nixos/vmalert: support multiple instances

vmalert only supports a single datasource for querying metrics and
managing alerts. Because of that, we need two instances to manage alerts
for both VictoriaLogs and VictoriaMetrics.

This is strongly inspired by the change made to Redis, i.e. a new
`instances` option was introduced with each option inside it.
With `mkRenamedOptionModule` it's ensured that existing configurations
still evaluate to the same result.
This commit is contained in:
Maximilian Bosch 2025-05-25 16:24:19 +02:00
parent b932c31ba4
commit 0d46fda974
No known key found for this signature in database
4 changed files with 159 additions and 109 deletions

View file

@ -10,9 +10,9 @@ let
format = pkgs.formats.yaml { };
confOpts = concatStringsSep " \\\n" (
mapAttrsToList mkLine (filterAttrs (_: v: v != false) cfg.settings)
);
mkConfOpts =
settings:
concatStringsSep " \\\n" (mapAttrsToList mkLine (filterAttrs (_: v: v != false) settings));
confType =
with types;
let
@ -33,124 +33,171 @@ let
concatMapStringsSep " " (v: "-${key}=${escapeShellArg (toString v)}") value
else
"-${key}=${escapeShellArg (toString value)}";
vmalertName = name: "vmalert" + lib.optionalString (name != "") ("-" + name);
enabledInstances = lib.filterAttrs (name: conf: conf.enable) config.services.vmalert.instances;
in
{
imports = [
(lib.mkRenamedOptionModule
[ "services" "vmalert" "enable" ]
[ "services" "vmalert" "instances" "" "enable" ]
)
(lib.mkRenamedOptionModule
[ "services" "vmalert" "rules" ]
[ "services" "vmalert" "instances" "" "rules" ]
)
(lib.mkRenamedOptionModule
[ "services" "vmalert" "settings" ]
[ "services" "vmalert" "instances" "" "settings" ]
)
];
# interface
options.services.vmalert = {
enable = lib.mkOption {
type = lib.types.bool;
default = false;
description = ''
Wether to enable VictoriaMetrics's `vmalert`.
options.services.vmalert.package = mkPackageOption pkgs "victoriametrics" { };
`vmalert` evaluates alerting and recording rules against a data source, sends notifications via Alertmanager.
'';
};
options.services.vmalert.instances = mkOption {
default = { };
package = mkPackageOption pkgs "victoriametrics" { };
description = ''
Define multiple instances of vmalert.
'';
settings = mkOption {
type = types.submodule {
freeformType = confType;
options = {
type = types.attrsOf (
types.submodule (
{ name, config, ... }:
{
options = {
enable = lib.mkOption {
type = lib.types.bool;
default = false;
description = ''
Wether to enable VictoriaMetrics's `vmalert`.
"datasource.url" = mkOption {
type = types.nonEmptyStr;
example = "http://localhost:8428";
description = ''
Datasource compatible with Prometheus HTTP API.
'';
`vmalert` evaluates alerting and recording rules against a data source, sends notifications via Alertmanager.
'';
};
settings = mkOption {
type = types.submodule {
freeformType = confType;
options = {
"datasource.url" = mkOption {
type = types.nonEmptyStr;
example = "http://localhost:8428";
description = ''
Datasource compatible with Prometheus HTTP API.
'';
};
"notifier.url" = mkOption {
type = with types; listOf nonEmptyStr;
default = [ ];
example = [ "http://127.0.0.1:9093" ];
description = ''
Prometheus Alertmanager URL. List all Alertmanager URLs if it runs in the cluster mode to ensure high availability.
'';
};
"rule" = mkOption {
type = with types; listOf path;
description = ''
Path to the files with alerting and/or recording rules.
::: {.note}
Consider using the {option}`services.vmalert.instances.<name>.rules` option as a convenient alternative for declaring rules
directly in the `nix` language.
:::
'';
};
};
};
default = { };
example = {
"datasource.url" = "http://localhost:8428";
"datasource.disableKeepAlive" = true;
"datasource.showURL" = false;
"rule" = [
"http://<some-server-addr>/path/to/rules"
"dir/*.yaml"
];
};
description = ''
`vmalert` configuration, passed via command line flags. Refer to
<https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/app/vmalert/README.md#configuration>
for details on supported values.
'';
};
rules = mkOption {
type = format.type;
default = { };
example = {
group = [
{
name = "TestGroup";
rules = [
{
alert = "ExampleAlertAlwaysFiring";
expr = ''
sum by(job)
(up == 1)
'';
}
];
}
];
};
description = ''
A list of the given alerting or recording rules against configured `"datasource.url"` compatible with
Prometheus HTTP API for `vmalert` to execute. Refer to
<https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/app/vmalert/README.md#rules>
for details on supported values.
'';
};
};
"notifier.url" = mkOption {
type = with types; listOf nonEmptyStr;
default = [ ];
example = [ "http://127.0.0.1:9093" ];
description = ''
Prometheus Alertmanager URL. List all Alertmanager URLs if it runs in the cluster mode to ensure high availability.
'';
};
"rule" = mkOption {
type = with types; listOf path;
description = ''
Path to the files with alerting and/or recording rules.
::: {.note}
Consider using the {option}`services.vmalert.rules` option as a convenient alternative for declaring rules
directly in the `nix` language.
:::
'';
};
};
};
default = { };
example = {
"datasource.url" = "http://localhost:8428";
"datasource.disableKeepAlive" = true;
"datasource.showURL" = false;
"rule" = [
"http://<some-server-addr>/path/to/rules"
"dir/*.yaml"
];
};
description = ''
`vmalert` configuration, passed via command line flags. Refer to
<https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/app/vmalert/README.md#configuration>
for details on supported values.
'';
};
rules = mkOption {
type = format.type;
default = { };
example = {
group = [
{
name = "TestGroup";
rules = [
{
alert = "ExampleAlertAlwaysFiring";
expr = ''
sum by(job)
(up == 1)
'';
}
config = {
settings.rule = [
"/etc/${vmalertName name}/rules.yml"
];
}
];
};
description = ''
A list of the given alerting or recording rules against configured `"datasource.url"` compatible with
Prometheus HTTP API for `vmalert` to execute. Refer to
<https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/app/vmalert/README.md#rules>
for details on supported values.
'';
};
};
}
)
);
};
# implementation
config = mkIf cfg.enable {
config = mkIf (enabledInstances != { }) {
environment.etc = lib.mapAttrs' (
name:
{ rules, ... }:
lib.nameValuePair "${vmalertName name}/rules.yml" {
source = format.generate "rules.yml" rules;
}
) enabledInstances;
environment.etc."vmalert/rules.yml".source = format.generate "rules.yml" cfg.rules;
systemd.services = lib.mapAttrs' (
name:
{ settings, ... }:
let
name' = vmalertName name;
in
lib.nameValuePair name' {
description = "vmalert service";
wantedBy = [ "multi-user.target" ];
after = [ "network.target" ];
reloadTriggers = [ config.environment.etc."${name'}/rules.yml".source ];
services.vmalert.settings.rule = [
"/etc/vmalert/rules.yml"
];
systemd.services.vmalert = {
description = "vmalert service";
wantedBy = [ "multi-user.target" ];
after = [ "network.target" ];
reloadTriggers = [ config.environment.etc."vmalert/rules.yml".source ];
serviceConfig = {
DynamicUser = true;
Restart = "on-failure";
ExecStart = "${cfg.package}/bin/vmalert ${confOpts}";
ExecReload = ''${pkgs.coreutils}/bin/kill -SIGHUP "$MAINPID"'';
};
};
serviceConfig = {
DynamicUser = true;
Restart = "on-failure";
ExecStart = "${cfg.package}/bin/vmalert ${mkConfOpts settings}";
ExecReload = ''${pkgs.coreutils}/bin/kill -SIGHUP "$MAINPID"'';
};
}
) enabledInstances;
};
}