nixos/vmalert: support multiple instances

vmalert only supports a single datasource for querying metrics and
managing alerts. Because of that, we need two instances to manage alerts
for both VictoriaLogs and VictoriaMetrics.

This is strongly inspired by the change made to Redis, i.e. a new
`instances` option was introduced with each option inside it.
With `mkRenamedOptionModule` it's ensured that existing configurations
still evaluate to the same result.

doc/release-notes/rl-2511.section.md

@@ -23,6 +23,9 @@
 
 - Added `rewriteURL` attribute to the nixpkgs `config`, to allow for rewriting the URLs downloaded by `fetchurl`.
 
+- `vmalert` now supports multiple instances with the option `services.vmalert.instances."".enable`
+  et al..
+
 ## Nixpkgs Library {#sec-nixpkgs-release-25.11-lib}
 
 <!-- To avoid merge conflicts, consider adding your item at an arbitrary place in the list instead. -->

nixos/doc/manual/release-notes/rl-2305.section.md

@@ -105,7 +105,7 @@ In addition to numerous new and updated packages, this release has the following
 
 - [ivpn](https://www.ivpn.net/), a secure, private VPN with fast WireGuard connections. Available as [services.ivpn](#opt-services.ivpn.enable).
 
-- [vmalert](https://victoriametrics.com/), an alerting engine for VictoriaMetrics. Available as [services.vmalert](#opt-services.vmalert.enable).
+- [vmalert](https://victoriametrics.com/), an alerting engine for VictoriaMetrics. Available as [services.vmalert.instances](#opt-services.vmalert.instances._name_.enable).
 
 - [jellyseerr](https://github.com/Fallenbagel/jellyseerr), a web-based requests manager for Jellyfin, forked from Overseerr. Available as [services.jellyseerr](#opt-services.jellyseerr.enable).
 

nixos/modules/services/monitoring/vmalert.nix

@@ -10,9 +10,9 @@ let
 
   format = pkgs.formats.yaml { };
 
-  confOpts = concatStringsSep " \\\n" (
+  mkConfOpts =
-    mapAttrsToList mkLine (filterAttrs (_: v: v != false) cfg.settings)
+    settings:
-  );
+    concatStringsSep " \\\n" (mapAttrsToList mkLine (filterAttrs (_: v: v != false) settings));
   confType =
     with types;
     let
@@ -33,124 +33,171 @@ let
       concatMapStringsSep " " (v: "-${key}=${escapeShellArg (toString v)}") value
     else
       "-${key}=${escapeShellArg (toString value)}";
+
+  vmalertName = name: "vmalert" + lib.optionalString (name != "") ("-" + name);
+  enabledInstances = lib.filterAttrs (name: conf: conf.enable) config.services.vmalert.instances;
 in
 {
+  imports = [
+    (lib.mkRenamedOptionModule
+      [ "services" "vmalert" "enable" ]
+      [ "services" "vmalert" "instances" "" "enable" ]
+    )
+    (lib.mkRenamedOptionModule
+      [ "services" "vmalert" "rules" ]
+      [ "services" "vmalert" "instances" "" "rules" ]
+    )
+    (lib.mkRenamedOptionModule
+      [ "services" "vmalert" "settings" ]
+      [ "services" "vmalert" "instances" "" "settings" ]
+    )
+  ];
+
   # interface
-  options.services.vmalert = {
+  options.services.vmalert.package = mkPackageOption pkgs "victoriametrics" { };
-    enable = lib.mkOption {
-      type = lib.types.bool;
-      default = false;
-      description = ''
-        Wether to enable VictoriaMetrics's `vmalert`.
 
-        `vmalert` evaluates alerting and recording rules against a data source, sends notifications via Alertmanager.
+  options.services.vmalert.instances = mkOption {
-      '';
+    default = { };
-    };
 
-    package = mkPackageOption pkgs "victoriametrics" { };
+    description = ''
+      Define multiple instances of vmalert.
+    '';
 
-    settings = mkOption {
+    type = types.attrsOf (
-      type = types.submodule {
+      types.submodule (
-        freeformType = confType;
+        { name, config, ... }:
-        options = {
+        {
+          options = {
+            enable = lib.mkOption {
+              type = lib.types.bool;
+              default = false;
+              description = ''
+                Wether to enable VictoriaMetrics's `vmalert`.
 
-          "datasource.url" = mkOption {
+                `vmalert` evaluates alerting and recording rules against a data source, sends notifications via Alertmanager.
-            type = types.nonEmptyStr;
+              '';
-            example = "http://localhost:8428";
+            };
-            description = ''
+
-              Datasource compatible with Prometheus HTTP API.
+            settings = mkOption {
-            '';
+              type = types.submodule {
+                freeformType = confType;
+                options = {
+
+                  "datasource.url" = mkOption {
+                    type = types.nonEmptyStr;
+                    example = "http://localhost:8428";
+                    description = ''
+                      Datasource compatible with Prometheus HTTP API.
+                    '';
+                  };
+
+                  "notifier.url" = mkOption {
+                    type = with types; listOf nonEmptyStr;
+                    default = [ ];
+                    example = [ "http://127.0.0.1:9093" ];
+                    description = ''
+                      Prometheus Alertmanager URL. List all Alertmanager URLs if it runs in the cluster mode to ensure high availability.
+                    '';
+                  };
+
+                  "rule" = mkOption {
+                    type = with types; listOf path;
+                    description = ''
+                      Path to the files with alerting and/or recording rules.
+
+                      ::: {.note}
+                      Consider using the {option}`services.vmalert.instances.<name>.rules` option as a convenient alternative for declaring rules
+                      directly in the `nix` language.
+                      :::
+                    '';
+                  };
+
+                };
+              };
+              default = { };
+              example = {
+                "datasource.url" = "http://localhost:8428";
+                "datasource.disableKeepAlive" = true;
+                "datasource.showURL" = false;
+                "rule" = [
+                  "http://<some-server-addr>/path/to/rules"
+                  "dir/*.yaml"
+                ];
+              };
+              description = ''
+                `vmalert` configuration, passed via command line flags. Refer to
+                <https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/app/vmalert/README.md#configuration>
+                for details on supported values.
+              '';
+            };
+
+            rules = mkOption {
+              type = format.type;
+              default = { };
+              example = {
+                group = [
+                  {
+                    name = "TestGroup";
+                    rules = [
+                      {
+                        alert = "ExampleAlertAlwaysFiring";
+                        expr = ''
+                          sum by(job)
+                          (up == 1)
+                        '';
+                      }
+                    ];
+                  }
+                ];
+              };
+              description = ''
+                A list of the given alerting or recording rules against configured `"datasource.url"` compatible with
+                Prometheus HTTP API for `vmalert` to execute. Refer to
+                <https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/app/vmalert/README.md#rules>
+                for details on supported values.
+              '';
+            };
           };
 
-          "notifier.url" = mkOption {
+          config = {
-            type = with types; listOf nonEmptyStr;
+            settings.rule = [
-            default = [ ];
+              "/etc/${vmalertName name}/rules.yml"
-            example = [ "http://127.0.0.1:9093" ];
-            description = ''
-              Prometheus Alertmanager URL. List all Alertmanager URLs if it runs in the cluster mode to ensure high availability.
-            '';
-          };
-
-          "rule" = mkOption {
-            type = with types; listOf path;
-            description = ''
-              Path to the files with alerting and/or recording rules.
-
-              ::: {.note}
-              Consider using the {option}`services.vmalert.rules` option as a convenient alternative for declaring rules
-              directly in the `nix` language.
-              :::
-            '';
-          };
-
-        };
-      };
-      default = { };
-      example = {
-        "datasource.url" = "http://localhost:8428";
-        "datasource.disableKeepAlive" = true;
-        "datasource.showURL" = false;
-        "rule" = [
-          "http://<some-server-addr>/path/to/rules"
-          "dir/*.yaml"
-        ];
-      };
-      description = ''
-        `vmalert` configuration, passed via command line flags. Refer to
-        <https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/app/vmalert/README.md#configuration>
-        for details on supported values.
-      '';
-    };
-
-    rules = mkOption {
-      type = format.type;
-      default = { };
-      example = {
-        group = [
-          {
-            name = "TestGroup";
-            rules = [
-              {
-                alert = "ExampleAlertAlwaysFiring";
-                expr = ''
-                  sum by(job)
-                  (up == 1)
-                '';
-              }
             ];
-          }
+          };
-        ];
+        }
-      };
+      )
-      description = ''
+    );
-        A list of the given alerting or recording rules against configured `"datasource.url"` compatible with
-        Prometheus HTTP API for `vmalert` to execute. Refer to
-        <https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/app/vmalert/README.md#rules>
-        for details on supported values.
-      '';
-    };
   };
 
   # implementation
-  config = mkIf cfg.enable {
+  config = mkIf (enabledInstances != { }) {
+    environment.etc = lib.mapAttrs' (
+      name:
+      { rules, ... }:
+      lib.nameValuePair "${vmalertName name}/rules.yml" {
+        source = format.generate "rules.yml" rules;
+      }
+    ) enabledInstances;
 
-    environment.etc."vmalert/rules.yml".source = format.generate "rules.yml" cfg.rules;
+    systemd.services = lib.mapAttrs' (
+      name:
+      { settings, ... }:
+      let
+        name' = vmalertName name;
+      in
+      lib.nameValuePair name' {
+        description = "vmalert service";
+        wantedBy = [ "multi-user.target" ];
+        after = [ "network.target" ];
+        reloadTriggers = [ config.environment.etc."${name'}/rules.yml".source ];
 
-    services.vmalert.settings.rule = [
+        serviceConfig = {
-      "/etc/vmalert/rules.yml"
+          DynamicUser = true;
-    ];
+          Restart = "on-failure";
-
+          ExecStart = "${cfg.package}/bin/vmalert ${mkConfOpts settings}";
-    systemd.services.vmalert = {
+          ExecReload = ''${pkgs.coreutils}/bin/kill -SIGHUP "$MAINPID"'';
-      description = "vmalert service";
+        };
-      wantedBy = [ "multi-user.target" ];
+      }
-      after = [ "network.target" ];
+    ) enabledInstances;
-      reloadTriggers = [ config.environment.etc."vmalert/rules.yml".source ];
-
-      serviceConfig = {
-        DynamicUser = true;
-        Restart = "on-failure";
-        ExecStart = "${cfg.package}/bin/vmalert ${confOpts}";
-        ExecReload = ''${pkgs.coreutils}/bin/kill -SIGHUP "$MAINPID"'';
-      };
-    };
   };
 }

nixos/tests/victoriametrics/vmalert.nix

@@ -55,7 +55,7 @@ import ../make-test-python.nix (
             };
           };
 
-          services.vmalert = {
+          services.vmalert.instances."" = {
             enable = true;
             settings = {
               "datasource.url" = "http://localhost:8428"; # victoriametrics' api