2020-01-19 18:55:56 +01:00
|
|
|
{
|
2024-10-29 10:45:10 +08:00
|
|
|
config,
|
|
|
|
pkgs,
|
|
|
|
lib,
|
|
|
|
...
|
|
|
|
}:
|
|
|
|
with lib;
|
|
|
|
let
|
|
|
|
cfg = config.services.victoriametrics;
|
|
|
|
settingsFormat = pkgs.formats.yaml { };
|
|
|
|
|
|
|
|
startCLIList =
|
|
|
|
[
|
|
|
|
"${cfg.package}/bin/victoria-metrics"
|
|
|
|
"-storageDataPath=/var/lib/${cfg.stateDir}"
|
|
|
|
"-httpListenAddr=${cfg.listenAddress}"
|
|
|
|
|
|
|
|
]
|
|
|
|
++ lib.optionals (cfg.retentionPeriod != null) [ "-retentionPeriod=${cfg.retentionPeriod}" ]
|
|
|
|
++ cfg.extraOptions;
|
|
|
|
prometheusConfigYml = checkedConfig (
|
|
|
|
settingsFormat.generate "prometheusConfig.yaml" cfg.prometheusConfig
|
|
|
|
);
|
|
|
|
|
|
|
|
checkedConfig =
|
|
|
|
file:
|
|
|
|
pkgs.runCommand "checked-config" { nativeBuildInputs = [ cfg.package ]; } ''
|
|
|
|
ln -s ${file} $out
|
|
|
|
${lib.escapeShellArgs startCLIList} -promscrape.config=${file} -dryRun
|
|
|
|
'';
|
|
|
|
in
|
|
|
|
{
|
|
|
|
options.services.victoriametrics = {
|
|
|
|
enable = mkEnableOption "VictoriaMetrics is a fast, cost-effective and scalable monitoring solution and time series database.";
|
2023-11-27 01:19:27 +01:00
|
|
|
package = mkPackageOption pkgs "victoriametrics" { };
|
2024-10-29 10:45:10 +08:00
|
|
|
|
2020-01-19 18:55:56 +01:00
|
|
|
listenAddress = mkOption {
|
|
|
|
default = ":8428";
|
|
|
|
type = types.str;
|
|
|
|
description = ''
|
2024-10-29 10:45:10 +08:00
|
|
|
TCP address to listen for incoming http requests.
|
2020-01-19 18:55:56 +01:00
|
|
|
'';
|
|
|
|
};
|
2024-10-29 10:45:10 +08:00
|
|
|
|
|
|
|
stateDir = mkOption {
|
|
|
|
type = types.str;
|
|
|
|
default = "victoriametrics";
|
|
|
|
description = ''
|
|
|
|
Directory below `/var/lib` to store VictoriaMetrics metrics data.
|
|
|
|
This directory will be created automatically using systemd's StateDirectory mechanism.
|
|
|
|
'';
|
|
|
|
};
|
|
|
|
|
2020-01-19 18:55:56 +01:00
|
|
|
retentionPeriod = mkOption {
|
2024-10-29 10:45:10 +08:00
|
|
|
type = types.nullOr types.str;
|
|
|
|
default = null;
|
|
|
|
example = "15d";
|
|
|
|
description = ''
|
|
|
|
How long to retain samples in storage.
|
|
|
|
The minimum retentionPeriod is 24h or 1d. See also -retentionFilter
|
|
|
|
The following optional suffixes are supported: s (second), h (hour), d (day), w (week), y (year).
|
|
|
|
If suffix isn't set, then the duration is counted in months (default 1)
|
|
|
|
'';
|
|
|
|
};
|
|
|
|
|
|
|
|
prometheusConfig = lib.mkOption {
|
|
|
|
type = lib.types.submodule { freeformType = settingsFormat.type; };
|
|
|
|
default = { };
|
|
|
|
example = literalExpression ''
|
|
|
|
{
|
|
|
|
scrape_configs = [
|
|
|
|
{
|
|
|
|
job_name = "postgres-exporter";
|
|
|
|
metrics_path = "/metrics";
|
|
|
|
static_configs = [
|
|
|
|
{
|
|
|
|
targets = ["1.2.3.4:9187"];
|
|
|
|
labels.type = "database";
|
|
|
|
}
|
|
|
|
];
|
|
|
|
}
|
|
|
|
{
|
|
|
|
job_name = "node-exporter";
|
|
|
|
metrics_path = "/metrics";
|
|
|
|
static_configs = [
|
|
|
|
{
|
|
|
|
targets = ["1.2.3.4:9100"];
|
|
|
|
labels.type = "node";
|
|
|
|
}
|
|
|
|
{
|
|
|
|
targets = ["5.6.7.8:9100"];
|
|
|
|
labels.type = "node";
|
|
|
|
}
|
|
|
|
];
|
|
|
|
}
|
|
|
|
];
|
|
|
|
}
|
|
|
|
'';
|
2020-01-19 18:55:56 +01:00
|
|
|
description = ''
|
2024-10-29 10:45:10 +08:00
|
|
|
Config for prometheus style metrics.
|
|
|
|
See the docs: <https://docs.victoriametrics.com/vmagent/#how-to-collect-metrics-in-prometheus-format>
|
|
|
|
for more information.
|
2020-01-19 18:55:56 +01:00
|
|
|
'';
|
|
|
|
};
|
2024-10-29 10:45:10 +08:00
|
|
|
|
2020-01-19 18:55:56 +01:00
|
|
|
extraOptions = mkOption {
|
|
|
|
type = types.listOf types.str;
|
2024-10-29 10:45:10 +08:00
|
|
|
default = [ ];
|
|
|
|
example = literalExpression ''
|
|
|
|
[
|
|
|
|
"-httpAuth.username=username"
|
|
|
|
"-httpAuth.password=file:///abs/path/to/file"
|
|
|
|
"-loggerLevel=WARN"
|
|
|
|
]
|
|
|
|
'';
|
2020-01-19 18:55:56 +01:00
|
|
|
description = ''
|
2024-10-29 10:45:10 +08:00
|
|
|
Extra options to pass to VictoriaMetrics. See the docs:
|
|
|
|
<https://docs.victoriametrics.com/single-server-victoriametrics/#list-of-command-line-flags>
|
|
|
|
or {command}`victoriametrics -help` for more information.
|
2020-01-19 18:55:56 +01:00
|
|
|
'';
|
|
|
|
};
|
|
|
|
};
|
2024-10-29 10:45:10 +08:00
|
|
|
|
2020-01-19 18:55:56 +01:00
|
|
|
config = lib.mkIf cfg.enable {
|
|
|
|
systemd.services.victoriametrics = {
|
|
|
|
description = "VictoriaMetrics time series database";
|
2024-10-29 10:45:10 +08:00
|
|
|
wantedBy = [ "multi-user.target" ];
|
2020-01-19 18:55:56 +01:00
|
|
|
after = [ "network.target" ];
|
2020-09-09 00:31:27 -07:00
|
|
|
startLimitBurst = 5;
|
2024-10-29 10:45:10 +08:00
|
|
|
|
2020-01-19 18:55:56 +01:00
|
|
|
serviceConfig = {
|
2024-10-29 10:45:10 +08:00
|
|
|
ExecStart = lib.escapeShellArgs (
|
|
|
|
startCLIList
|
2024-12-06 10:23:51 +08:00
|
|
|
++ lib.optionals (cfg.prometheusConfig != { }) [ "-promscrape.config=${prometheusConfigYml}" ]
|
2024-10-29 10:45:10 +08:00
|
|
|
);
|
|
|
|
|
2020-01-19 18:55:56 +01:00
|
|
|
DynamicUser = true;
|
2024-10-29 10:45:10 +08:00
|
|
|
RestartSec = 1;
|
|
|
|
Restart = "on-failure";
|
|
|
|
RuntimeDirectory = "victoriametrics";
|
|
|
|
RuntimeDirectoryMode = "0700";
|
|
|
|
StateDirectory = cfg.stateDir;
|
|
|
|
StateDirectoryMode = "0700";
|
|
|
|
|
|
|
|
# Increase the limit to avoid errors like 'too many open files' when merging small parts
|
nixos/victoriametrics: set LimitNOFILE=1048576 to fix panic and restart loop
This fixes:
```
systemd[1]: Started VictoriaMetrics time series database.
victoria-metrics[379550]: 2021-08-04T19:33:39.833Z panic VictoriaMetrics/lib/storage/partition.go:954 FATAL: unrecoverable error when merging small parts in the partition "/var/lib/victoriametrics/data/small/2021_08": cannot open source part for merging: cannot open metaindex file in stream mode: cannot open file "/var/lib/victoriametrics/data/small/2021_08/1228_1228_20210804184120.712_20210804184121.899_16982E83CD7A763A/metaindex.bin": open /var/lib/victoriametrics/data/small/2021_08/1228_1228_20210804184120.712_20210804184121.899_16982E83CD7A763A/metaindex.bin: too many open files
victoria-metrics[379550]: panic: FATAL: unrecoverable error when merging small parts in the partition "/var/lib/victoriametrics/data/small/2021_08": cannot open source part for merging: cannot open metaindex file in stream mode: cannot open file "/var/lib/victoriametrics/data/small/2021_08/1228_1228_20210804184120.712_20210804184121.899_16982E83CD7A763A/metaindex.bin": open /var/lib/victoriametrics/data/small/2021_08/1228_1228_20210804184120.712_20210804184121.899_16982E83CD7A763A/metaindex.bin: too many open files
victoria-metrics[379550]: goroutine 629 [running]:
victoria-metrics[379550]: github.com/VictoriaMetrics/VictoriaMetrics/lib/logger.logMessage(0xbb3ea1, 0x5, 0xc001113800, 0x1e7, 0x4)
victoria-metrics[379550]: github.com/VictoriaMetrics/VictoriaMetrics/lib/logger/logger.go:270 +0xc69
victoria-metrics[379550]: github.com/VictoriaMetrics/VictoriaMetrics/lib/logger.logLevelSkipframes(0x1, 0xbb3ea1, 0x5, 0xbe3f8b, 0x4b, 0xc000bb3f88, 0x2, 0x2)
victoria-metrics[379550]: github.com/VictoriaMetrics/VictoriaMetrics/lib/logger/logger.go:138 +0xd1
victoria-metrics[379550]: github.com/VictoriaMetrics/VictoriaMetrics/lib/logger.logLevel(...)
victoria-metrics[379550]: github.com/VictoriaMetrics/VictoriaMetrics/lib/logger/logger.go:130
victoria-metrics[379550]: github.com/VictoriaMetrics/VictoriaMetrics/lib/logger.Panicf(...)
victoria-metrics[379550]: github.com/VictoriaMetrics/VictoriaMetrics/lib/logger/logger.go:126
victoria-metrics[379550]: github.com/VictoriaMetrics/VictoriaMetrics/lib/storage.(*partition).smallPartsMerger(0xc0014d7980)
victoria-metrics[379550]: github.com/VictoriaMetrics/VictoriaMetrics/lib/storage/partition.go:954 +0x145
victoria-metrics[379550]: github.com/VictoriaMetrics/VictoriaMetrics/lib/storage.(*partition).startMergeWorkers.func1(0xc0014d7980)
victoria-metrics[379550]: github.com/VictoriaMetrics/VictoriaMetrics/lib/storage/partition.go:933 +0x2b
victoria-metrics[379550]: created by github.com/VictoriaMetrics/VictoriaMetrics/lib/storage.(*partition).startMergeWorkers
victoria-metrics[379550]: github.com/VictoriaMetrics/VictoriaMetrics/lib/storage/partition.go:932 +0x6c
systemd[1]: victoriametrics.service: Main process exited, code=exited, status=2/INVALIDARGUMENT
systemd[1]: victoriametrics.service: Failed with result 'exit-code'.
systemd[1]: victoriametrics.service: Consumed 587ms CPU time, received 6.5K IP traffic, sent 1.7K IP traffic.
systemd[1]: victoriametrics.service: Scheduled restart job, restart counter is at 2064.
systemd[1]: Stopped VictoriaMetrics time series database.
systemd[1]: victoriametrics.service: Consumed 587ms CPU time, received 6.5K IP traffic, sent 1.7K IP traffic.
systemd[1]: Starting VictoriaMetrics time series database...
```
2021-08-05 05:07:25 +00:00
|
|
|
LimitNOFILE = 1048576;
|
2024-10-29 10:45:10 +08:00
|
|
|
|
|
|
|
# Hardening
|
|
|
|
DeviceAllow = [ "/dev/null rw" ];
|
|
|
|
DevicePolicy = "strict";
|
|
|
|
LockPersonality = true;
|
|
|
|
MemoryDenyWriteExecute = true;
|
|
|
|
NoNewPrivileges = true;
|
|
|
|
PrivateDevices = true;
|
|
|
|
PrivateTmp = true;
|
|
|
|
PrivateUsers = true;
|
|
|
|
ProtectClock = true;
|
|
|
|
ProtectControlGroups = true;
|
|
|
|
ProtectHome = true;
|
|
|
|
ProtectHostname = true;
|
|
|
|
ProtectKernelLogs = true;
|
|
|
|
ProtectKernelModules = true;
|
|
|
|
ProtectKernelTunables = true;
|
|
|
|
ProtectProc = "invisible";
|
|
|
|
ProtectSystem = "full";
|
|
|
|
RemoveIPC = true;
|
|
|
|
RestrictAddressFamilies = [
|
|
|
|
"AF_INET"
|
|
|
|
"AF_INET6"
|
|
|
|
"AF_UNIX"
|
|
|
|
];
|
|
|
|
RestrictNamespaces = true;
|
|
|
|
RestrictRealtime = true;
|
|
|
|
RestrictSUIDSGID = true;
|
|
|
|
SystemCallArchitectures = "native";
|
|
|
|
SystemCallFilter = [
|
|
|
|
"@system-service"
|
|
|
|
"~@privileged"
|
|
|
|
];
|
2020-01-19 18:55:56 +01:00
|
|
|
};
|
|
|
|
|
|
|
|
postStart =
|
|
|
|
let
|
2024-10-29 10:45:10 +08:00
|
|
|
bindAddr =
|
|
|
|
(lib.optionalString (lib.hasPrefix ":" cfg.listenAddress) "127.0.0.1") + cfg.listenAddress;
|
2020-01-19 18:55:56 +01:00
|
|
|
in
|
|
|
|
lib.mkBefore ''
|
|
|
|
until ${lib.getBin pkgs.curl}/bin/curl -s -o /dev/null http://${bindAddr}/ping; do
|
|
|
|
sleep 1;
|
|
|
|
done
|
|
|
|
'';
|
|
|
|
};
|
|
|
|
};
|
|
|
|
}
|