From 0d46fda9746f50f5e1d69950000b5ff8c6941a28 Mon Sep 17 00:00:00 2001 From: Maximilian Bosch Date: Sun, 25 May 2025 16:24:19 +0200 Subject: [PATCH] nixos/vmalert: support multiple instances vmalert only supports a single datasource for querying metrics and managing alerts. Because of that, we need two instances to manage alerts for both VictoriaLogs and VictoriaMetrics. This is strongly inspired by the change made to Redis, i.e. a new `instances` option was introduced with each option inside it. With `mkRenamedOptionModule` it's ensured that existing configurations still evaluate to the same result. --- doc/release-notes/rl-2511.section.md | 3 + .../manual/release-notes/rl-2305.section.md | 2 +- nixos/modules/services/monitoring/vmalert.nix | 261 +++++++++++------- nixos/tests/victoriametrics/vmalert.nix | 2 +- 4 files changed, 159 insertions(+), 109 deletions(-) diff --git a/doc/release-notes/rl-2511.section.md b/doc/release-notes/rl-2511.section.md index 02b08039812b..9214f1d49bfc 100644 --- a/doc/release-notes/rl-2511.section.md +++ b/doc/release-notes/rl-2511.section.md @@ -23,6 +23,9 @@ - Added `rewriteURL` attribute to the nixpkgs `config`, to allow for rewriting the URLs downloaded by `fetchurl`. +- `vmalert` now supports multiple instances with the option `services.vmalert.instances."".enable` + et al.. + ## Nixpkgs Library {#sec-nixpkgs-release-25.11-lib} diff --git a/nixos/doc/manual/release-notes/rl-2305.section.md b/nixos/doc/manual/release-notes/rl-2305.section.md index cae573c3452a..7ffe079977af 100644 --- a/nixos/doc/manual/release-notes/rl-2305.section.md +++ b/nixos/doc/manual/release-notes/rl-2305.section.md @@ -105,7 +105,7 @@ In addition to numerous new and updated packages, this release has the following - [ivpn](https://www.ivpn.net/), a secure, private VPN with fast WireGuard connections. Available as [services.ivpn](#opt-services.ivpn.enable). -- [vmalert](https://victoriametrics.com/), an alerting engine for VictoriaMetrics. Available as [services.vmalert](#opt-services.vmalert.enable). +- [vmalert](https://victoriametrics.com/), an alerting engine for VictoriaMetrics. Available as [services.vmalert.instances](#opt-services.vmalert.instances._name_.enable). - [jellyseerr](https://github.com/Fallenbagel/jellyseerr), a web-based requests manager for Jellyfin, forked from Overseerr. Available as [services.jellyseerr](#opt-services.jellyseerr.enable). diff --git a/nixos/modules/services/monitoring/vmalert.nix b/nixos/modules/services/monitoring/vmalert.nix index 438d067643fb..0baaa039e395 100644 --- a/nixos/modules/services/monitoring/vmalert.nix +++ b/nixos/modules/services/monitoring/vmalert.nix @@ -10,9 +10,9 @@ let format = pkgs.formats.yaml { }; - confOpts = concatStringsSep " \\\n" ( - mapAttrsToList mkLine (filterAttrs (_: v: v != false) cfg.settings) - ); + mkConfOpts = + settings: + concatStringsSep " \\\n" (mapAttrsToList mkLine (filterAttrs (_: v: v != false) settings)); confType = with types; let @@ -33,124 +33,171 @@ let concatMapStringsSep " " (v: "-${key}=${escapeShellArg (toString v)}") value else "-${key}=${escapeShellArg (toString value)}"; + + vmalertName = name: "vmalert" + lib.optionalString (name != "") ("-" + name); + enabledInstances = lib.filterAttrs (name: conf: conf.enable) config.services.vmalert.instances; in { + imports = [ + (lib.mkRenamedOptionModule + [ "services" "vmalert" "enable" ] + [ "services" "vmalert" "instances" "" "enable" ] + ) + (lib.mkRenamedOptionModule + [ "services" "vmalert" "rules" ] + [ "services" "vmalert" "instances" "" "rules" ] + ) + (lib.mkRenamedOptionModule + [ "services" "vmalert" "settings" ] + [ "services" "vmalert" "instances" "" "settings" ] + ) + ]; + # interface - options.services.vmalert = { - enable = lib.mkOption { - type = lib.types.bool; - default = false; - description = '' - Wether to enable VictoriaMetrics's `vmalert`. + options.services.vmalert.package = mkPackageOption pkgs "victoriametrics" { }; - `vmalert` evaluates alerting and recording rules against a data source, sends notifications via Alertmanager. - ''; - }; + options.services.vmalert.instances = mkOption { + default = { }; - package = mkPackageOption pkgs "victoriametrics" { }; + description = '' + Define multiple instances of vmalert. + ''; - settings = mkOption { - type = types.submodule { - freeformType = confType; - options = { + type = types.attrsOf ( + types.submodule ( + { name, config, ... }: + { + options = { + enable = lib.mkOption { + type = lib.types.bool; + default = false; + description = '' + Wether to enable VictoriaMetrics's `vmalert`. - "datasource.url" = mkOption { - type = types.nonEmptyStr; - example = "http://localhost:8428"; - description = '' - Datasource compatible with Prometheus HTTP API. - ''; + `vmalert` evaluates alerting and recording rules against a data source, sends notifications via Alertmanager. + ''; + }; + + settings = mkOption { + type = types.submodule { + freeformType = confType; + options = { + + "datasource.url" = mkOption { + type = types.nonEmptyStr; + example = "http://localhost:8428"; + description = '' + Datasource compatible with Prometheus HTTP API. + ''; + }; + + "notifier.url" = mkOption { + type = with types; listOf nonEmptyStr; + default = [ ]; + example = [ "http://127.0.0.1:9093" ]; + description = '' + Prometheus Alertmanager URL. List all Alertmanager URLs if it runs in the cluster mode to ensure high availability. + ''; + }; + + "rule" = mkOption { + type = with types; listOf path; + description = '' + Path to the files with alerting and/or recording rules. + + ::: {.note} + Consider using the {option}`services.vmalert.instances..rules` option as a convenient alternative for declaring rules + directly in the `nix` language. + ::: + ''; + }; + + }; + }; + default = { }; + example = { + "datasource.url" = "http://localhost:8428"; + "datasource.disableKeepAlive" = true; + "datasource.showURL" = false; + "rule" = [ + "http:///path/to/rules" + "dir/*.yaml" + ]; + }; + description = '' + `vmalert` configuration, passed via command line flags. Refer to + + for details on supported values. + ''; + }; + + rules = mkOption { + type = format.type; + default = { }; + example = { + group = [ + { + name = "TestGroup"; + rules = [ + { + alert = "ExampleAlertAlwaysFiring"; + expr = '' + sum by(job) + (up == 1) + ''; + } + ]; + } + ]; + }; + description = '' + A list of the given alerting or recording rules against configured `"datasource.url"` compatible with + Prometheus HTTP API for `vmalert` to execute. Refer to + + for details on supported values. + ''; + }; }; - "notifier.url" = mkOption { - type = with types; listOf nonEmptyStr; - default = [ ]; - example = [ "http://127.0.0.1:9093" ]; - description = '' - Prometheus Alertmanager URL. List all Alertmanager URLs if it runs in the cluster mode to ensure high availability. - ''; - }; - - "rule" = mkOption { - type = with types; listOf path; - description = '' - Path to the files with alerting and/or recording rules. - - ::: {.note} - Consider using the {option}`services.vmalert.rules` option as a convenient alternative for declaring rules - directly in the `nix` language. - ::: - ''; - }; - - }; - }; - default = { }; - example = { - "datasource.url" = "http://localhost:8428"; - "datasource.disableKeepAlive" = true; - "datasource.showURL" = false; - "rule" = [ - "http:///path/to/rules" - "dir/*.yaml" - ]; - }; - description = '' - `vmalert` configuration, passed via command line flags. Refer to - - for details on supported values. - ''; - }; - - rules = mkOption { - type = format.type; - default = { }; - example = { - group = [ - { - name = "TestGroup"; - rules = [ - { - alert = "ExampleAlertAlwaysFiring"; - expr = '' - sum by(job) - (up == 1) - ''; - } + config = { + settings.rule = [ + "/etc/${vmalertName name}/rules.yml" ]; - } - ]; - }; - description = '' - A list of the given alerting or recording rules against configured `"datasource.url"` compatible with - Prometheus HTTP API for `vmalert` to execute. Refer to - - for details on supported values. - ''; - }; + }; + } + ) + ); }; # implementation - config = mkIf cfg.enable { + config = mkIf (enabledInstances != { }) { + environment.etc = lib.mapAttrs' ( + name: + { rules, ... }: + lib.nameValuePair "${vmalertName name}/rules.yml" { + source = format.generate "rules.yml" rules; + } + ) enabledInstances; - environment.etc."vmalert/rules.yml".source = format.generate "rules.yml" cfg.rules; + systemd.services = lib.mapAttrs' ( + name: + { settings, ... }: + let + name' = vmalertName name; + in + lib.nameValuePair name' { + description = "vmalert service"; + wantedBy = [ "multi-user.target" ]; + after = [ "network.target" ]; + reloadTriggers = [ config.environment.etc."${name'}/rules.yml".source ]; - services.vmalert.settings.rule = [ - "/etc/vmalert/rules.yml" - ]; - - systemd.services.vmalert = { - description = "vmalert service"; - wantedBy = [ "multi-user.target" ]; - after = [ "network.target" ]; - reloadTriggers = [ config.environment.etc."vmalert/rules.yml".source ]; - - serviceConfig = { - DynamicUser = true; - Restart = "on-failure"; - ExecStart = "${cfg.package}/bin/vmalert ${confOpts}"; - ExecReload = ''${pkgs.coreutils}/bin/kill -SIGHUP "$MAINPID"''; - }; - }; + serviceConfig = { + DynamicUser = true; + Restart = "on-failure"; + ExecStart = "${cfg.package}/bin/vmalert ${mkConfOpts settings}"; + ExecReload = ''${pkgs.coreutils}/bin/kill -SIGHUP "$MAINPID"''; + }; + } + ) enabledInstances; }; } diff --git a/nixos/tests/victoriametrics/vmalert.nix b/nixos/tests/victoriametrics/vmalert.nix index f92c35a82010..aa5026cbd8f6 100644 --- a/nixos/tests/victoriametrics/vmalert.nix +++ b/nixos/tests/victoriametrics/vmalert.nix @@ -55,7 +55,7 @@ import ../make-test-python.nix ( }; }; - services.vmalert = { + services.vmalert.instances."" = { enable = true; settings = { "datasource.url" = "http://localhost:8428"; # victoriametrics' api