nixos/pgbackrest: init module (#404384)

This commit is contained in:
Wolfgang Walther 2025-05-12 07:29:11 +00:00 committed by GitHub
commit 7da685054c
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
8 changed files with 703 additions and 23 deletions

View file

@ -208,6 +208,8 @@
- [GLPI-Agent](https://github.com/glpi-project/glpi-agent), GLPI Agent. Available as [services.glpiAgent](options.html#opt-services.glpiAgent.enable).
- [pgBackRest](https://pgbackrest.org), a reliable backup and restore solution for PostgreSQL. Available as [services.pgbackrest](options.html#opt-services.pgbackrest.enable).
- [Recyclarr](https://github.com/recyclarr/recyclarr) a TRaSH Guides synchronizer for Sonarr and Radarr. Available as [services.recyclarr](#opt-services.recyclarr.enable).
- [Rebuilderd](https://github.com/kpcyrd/rebuilderd) an independent verification of binary packages - Reproducible Builds. Available as [services.rebuilderd](#opt-services.rebuilderd.enable).

View file

@ -440,6 +440,7 @@
./services/backup/duplicati.nix
./services/backup/duplicity.nix
./services/backup/mysql-backup.nix
./services/backup/pgbackrest.nix
./services/backup/postgresql-backup.nix
./services/backup/postgresql-wal-receiver.nix
./services/backup/restic-rest-server.nix

View file

@ -0,0 +1,426 @@
{
config,
lib,
pkgs,
...
}:
let
cfg = config.services.pgbackrest;
settingsFormat = pkgs.formats.ini {
listsAsDuplicateKeys = true;
};
# pgBackRest "options"
settingsType =
with lib.types;
attrsOf (oneOf [
bool
ints.unsigned
str
(attrsOf str)
(listOf str)
]);
# Applied to both repoNNN-* and pgNNN-* options in global and stanza sections.
flattenWithIndex =
attrs: prefix:
lib.concatMapAttrs (
name:
let
index = lib.lists.findFirstIndex (n: n == name) null (lib.attrNames attrs);
index1 = index + 1;
in
lib.mapAttrs' (option: lib.nameValuePair "${prefix}${toString index1}-${option}")
) attrs;
# Remove nulls, turn attrsets into lists and bools into y/n
normalize =
x:
lib.pipe x [
(lib.filterAttrs (_: v: v != null))
(lib.mapAttrs (_: v: if lib.isAttrs v then lib.mapAttrsToList (n': v': "${n'}=${v'}") v else v))
(lib.mapAttrs (
_: v:
if v == true then
"y"
else if v == false then
"n"
else
v
))
];
fullConfig =
{
global = normalize (cfg.settings // flattenWithIndex cfg.repos "repo");
}
// lib.mapAttrs (
_: cfg': normalize (cfg'.settings // flattenWithIndex cfg'.instances "pg")
) cfg.stanzas;
namedJobs = lib.listToAttrs (
lib.flatten (
lib.mapAttrsToList (
stanza:
{ jobs, ... }:
lib.mapAttrsToList (
job: attrs: lib.nameValuePair "pgbackrest-${stanza}-${job}" (attrs // { inherit stanza job; })
) jobs
) cfg.stanzas
)
);
disabledOption = lib.mkOption {
default = null;
readOnly = true;
internal = true;
};
secretPathOption =
with lib.types;
lib.mkOption {
type = nullOr (pathWith {
inStore = false;
absolute = true;
});
default = null;
internal = true;
};
in
{
meta = {
maintainers = with lib.maintainers; [ wolfgangwalther ];
};
# TODO: Add enableServer option and corresponding pgBackRest TLS server service.
# TODO: Allow command-specific options
# TODO: Write wrapper around pgbackrest to turn --repo=<name> into --repo=<number>
# The following two are dependent on improvements upstream:
# https://github.com/pgbackrest/pgbackrest/issues/2621
# TODO: Add support for more repository types
# TODO: Support passing encryption key safely
options.services.pgbackrest = {
enable = lib.mkEnableOption "pgBackRest";
repos = lib.mkOption {
type =
with lib.types;
attrsOf (
submodule (
{ config, name, ... }:
let
setHostForType =
type:
if name == "localhost" then
null
# "posix" is the default repo type, which uses the -host option.
# Other types use prefixed options, for example -sftp-host.
else if config.type or "posix" != type then
null
else
name;
in
{
freeformType = settingsType;
options.host = lib.mkOption {
type = nullOr str;
default = setHostForType "posix";
defaultText = lib.literalExpression "name";
description = "Repository host when operating remotely";
};
options.sftp-host = lib.mkOption {
type = nullOr str;
default = setHostForType "sftp";
defaultText = lib.literalExpression "name";
description = "SFTP repository host";
};
options.sftp-private-key-file = lib.mkOption {
type = nullOr (pathWith {
inStore = false;
absolute = true;
});
default = null;
description = ''
SFTP private key file.
The file must be accessible by both the pgbackrest and the postgres users.
'';
};
# The following options should not be used; they would store secrets in the store.
options.azure-key = disabledOption;
options.cipher-pass = disabledOption;
options.s3-key = disabledOption;
options.s3-key-secret = disabledOption;
options.s3-kms-key-id = disabledOption; # unsure whether that's a secret or not
options.s3-sse-customer-key = disabledOption; # unsure whether that's a secret or not
options.s3-token = disabledOption;
options.sftp-private-key-passphrase = disabledOption;
# The following options are not fully supported / tested, yet, but point to files with secrets.
# Users can already set those options, but we'll force non-store paths.
options.gcs-key = secretPathOption;
options.host-cert-file = secretPathOption;
options.host-key-file = secretPathOption;
}
)
);
default = { };
description = ''
An attribute set of repositories as described in:
<https://pgbackrest.org/configuration.html#section-repository>
Each repository defaults to set `repo-host` to the attribute's name.
The special value "localhost" will unset `repo-host`.
::: {.note}
The prefix `repoNNN-` is added automatically.
Example: Use `path` instead of `repo1-path`.
:::
'';
example = lib.literalExpression ''
{
localhost.path = "/var/lib/backup";
"backup.example.com".host-type = "tls";
}
'';
};
stanzas = lib.mkOption {
type =
with lib.types;
attrsOf (submodule {
options = {
jobs = lib.mkOption {
type = lib.types.attrsOf (
lib.types.submodule {
options.schedule = lib.mkOption {
type = lib.types.str;
description = ''
When or how often the backup should run.
Must be in the format described in {manpage}`systemd.time(7)`.
'';
};
options.type = lib.mkOption {
type = lib.types.str;
description = ''
Backup type as described in:
<https://pgbackrest.org/command.html#command-backup/category-command/option-type>
'';
};
}
);
default = { };
description = ''
Backups jobs to schedule for this stanza as described in:
<https://pgbackrest.org/user-guide.html#quickstart/schedule-backup>
'';
example = lib.literalExpression ''
{
weekly = { schedule = "Sun, 6:30"; type = "full"; };
daily = { schedule = "Mon..Sat, 6:30"; type = "diff"; };
}
'';
};
instances = lib.mkOption {
type =
with lib.types;
attrsOf (
submodule (
{ name, ... }:
{
freeformType = settingsType;
options.host = lib.mkOption {
type = nullOr str;
default = if name == "localhost" then null else name;
defaultText = lib.literalExpression ''if name == "localhost" then null else name'';
description = "PostgreSQL host for operating remotely.";
};
# The following options are not fully supported / tested, yet, but point to files with secrets.
# Users can already set those options, but we'll force non-store paths.
options.host-cert-file = secretPathOption;
options.host-key-file = secretPathOption;
}
)
);
default = { };
description = ''
An attribute set of database instances as described in:
<https://pgbackrest.org/configuration.html#section-stanza>
Each instance defaults to set `pg-host` to the attribute's name.
The special value "localhost" will unset `pg-host`.
::: {.note}
The prefix `pgNNN-` is added automatically.
Example: Use `user` instead of `pg1-user`.
:::
'';
example = lib.literalExpression ''
{
localhost.database = "app";
"postgres.example.com".port = "5433";
}
'';
};
settings = lib.mkOption {
type = lib.types.submodule {
freeformType = settingsType;
# The following options are not fully supported / tested, yet, but point to files with secrets.
# Users can already set those options, but we'll force non-store paths.
options.tls-server-cert-file = secretPathOption;
options.tls-server-key-file = secretPathOption;
};
default = { };
description = ''
An attribute set of options as described in:
<https://pgbackrest.org/configuration.html>
All options can be used.
Repository options should be set via [`repos`](#opt-services.pgbackrest.repos) instead.
Stanza options should be set via [`instances`](#opt-services.pgbackrest.stanzas._name_.instances) instead.
'';
example = lib.literalExpression ''
{
process-max = 2;
}
'';
};
};
});
default = { };
description = ''
An attribute set of stanzas as described in:
<https://pgbackrest.org/user-guide.html#quickstart/configure-stanza>
'';
};
settings = lib.mkOption {
type = lib.types.submodule {
freeformType = settingsType;
# The following options are not fully supported / tested, yet, but point to files with secrets.
# Users can already set those options, but we'll force non-store paths.
options.tls-server-cert-file = secretPathOption;
options.tls-server-key-file = secretPathOption;
};
default = { };
description = ''
An attribute set of options as described in:
<https://pgbackrest.org/configuration.html>
All globally available options, i.e. all except stanza options, can be used.
Repository options should be set via [`repos`](#opt-services.pgbackrest.repos) instead.
'';
example = lib.literalExpression ''
{
process-max = 2;
}
'';
};
};
config = lib.mkIf cfg.enable (
lib.mkMerge [
{
services.pgbackrest.settings = {
log-level-console = lib.mkDefault "info";
log-level-file = lib.mkDefault "off";
cmd-ssh = lib.getExe pkgs.openssh;
};
environment.systemPackages = [ pkgs.pgbackrest ];
environment.etc."pgbackrest/pgbackrest.conf".source =
settingsFormat.generate "pgbackrest.conf" fullConfig;
users.users.pgbackrest = {
name = "pgbackrest";
group = "pgbackrest";
description = "pgBackRest service user";
isSystemUser = true;
useDefaultShell = true;
createHome = true;
home = cfg.repos.localhost.path or "/var/lib/pgbackrest";
};
users.groups.pgbackrest = { };
systemd.services = lib.mapAttrs (
_:
{
stanza,
job,
type,
...
}:
{
description = "pgBackRest job ${job} for stanza ${stanza}";
serviceConfig = {
User = "pgbackrest";
Group = "pgbackrest";
Type = "oneshot";
# stanza-create is idempotent, so safe to always run
ExecStartPre = "${lib.getExe pkgs.pgbackrest} --stanza='${stanza}' stanza-create";
ExecStart = "${lib.getExe pkgs.pgbackrest} --stanza='${stanza}' backup --type='${type}'";
};
}
) namedJobs;
systemd.timers = lib.mapAttrs (
name:
{
stanza,
job,
schedule,
...
}:
{
description = "pgBackRest job ${job} for stanza ${stanza}";
wantedBy = [ "timers.target" ];
after = [ "network-online.target" ];
wants = [ "network-online.target" ];
timerConfig = {
OnCalendar = schedule;
Persistent = true;
Unit = "${name}.service";
};
}
) namedJobs;
}
# The default stanza is set up for the local postgresql instance.
# It does not backup automatically, the systemd timer still needs to be set.
(lib.mkIf config.services.postgresql.enable {
services.pgbackrest.stanzas.default = {
settings.cmd = lib.getExe pkgs.pgbackrest;
instances.localhost = {
path = config.services.postgresql.dataDir;
user = "postgres";
};
};
services.postgresql.identMap = ''
postgres pgbackrest postgres
'';
services.postgresql.initdbArgs = [ "--allow-group-access" ];
users.users.pgbackrest.extraGroups = [ "postgres" ];
services.postgresql.settings = {
archive_command = ''${lib.getExe pkgs.pgbackrest} --stanza=default archive-push "%p"'';
archive_mode = lib.mkDefault "on";
};
users.groups.pgbackrest.members = [ "postgres" ];
})
]
);
}

View file

@ -1025,6 +1025,7 @@ in
peertube = handleTestOn [ "x86_64-linux" ] ./web-apps/peertube.nix { };
peroxide = handleTest ./peroxide.nix { };
pgadmin4 = runTest ./pgadmin4.nix;
pgbackrest = import ./pgbackrest { inherit runTest; };
pgbouncer = handleTest ./pgbouncer.nix { };
pghero = runTest ./pghero.nix;
pgweb = runTest ./pgweb.nix;

View file

@ -0,0 +1,5 @@
{ runTest }:
{
posix = runTest ./posix.nix;
sftp = runTest ./sftp.nix;
}

View file

@ -0,0 +1,147 @@
{ lib, pkgs, ... }:
let
inherit (import ../ssh-keys.nix pkgs) snakeOilPrivateKey snakeOilPublicKey;
backupPath = "/var/lib/pgbackrest";
in
{
name = "pgbackrest-posix";
meta = {
maintainers = with lib.maintainers; [ wolfgangwalther ];
};
nodes.primary =
{
pkgs,
...
}:
{
services.openssh.enable = true;
users.users.postgres.openssh.authorizedKeys.keys = [
snakeOilPublicKey
];
services.postgresql = {
enable = true;
initialScript = pkgs.writeText "init.sql" ''
CREATE TABLE t(c text);
INSERT INTO t VALUES ('hello world');
'';
};
services.pgbackrest = {
enable = true;
repos.backup = {
type = "posix";
path = backupPath;
host-user = "pgbackrest";
};
};
};
nodes.backup =
{
nodes,
...
}:
{
services.openssh.enable = true;
users.users.pgbackrest.openssh.authorizedKeys.keys = [
snakeOilPublicKey
];
services.pgbackrest = {
enable = true;
repos.localhost.path = backupPath;
stanzas.default = {
jobs.future = {
schedule = "3000-01-01";
type = "full";
};
instances.primary = {
path = nodes.primary.services.postgresql.dataDir;
user = "postgres";
};
};
# Examples from https://pgbackrest.org/configuration.html#introduction
# Not used for the test, except for dumping the config.
stanzas.config-format.settings = {
start-fast = true;
compress-level = 3;
buffer-size = "2MiB";
db-timeout = 600;
db-exclude = [
"db1"
"db2"
"db5"
];
tablespace-map = {
ts_01 = "/db/ts_01";
ts_02 = "/db/ts_02";
};
};
};
};
testScript =
{ nodes, ... }:
''
start_all()
primary.wait_for_unit("multi-user.target")
backup.wait_for_unit("multi-user.target")
with subtest("config file is written correctly"):
from textwrap import dedent
have = backup.succeed("cat /etc/pgbackrest/pgbackrest.conf")
want = dedent("""\
[config-format]
buffer-size=2MiB
compress-level=3
db-exclude=db1
db-exclude=db2
db-exclude=db5
db-timeout=600
start-fast=y
tablespace-map=ts_01=/db/ts_01
tablespace-map=ts_02=/db/ts_02
""")
assert want in have, repr((want, have))
primary.log(primary.succeed("""
HOME="${nodes.primary.services.postgresql.dataDir}"
mkdir -m 700 -p ~/.ssh
cat ${snakeOilPrivateKey} > ~/.ssh/id_ecdsa
chmod 400 ~/.ssh/id_ecdsa
ssh-keyscan backup >> ~/.ssh/known_hosts
chown -R postgres:postgres ~/.ssh
"""))
backup.log(backup.succeed("""
HOME="${backupPath}"
mkdir -m 700 -p ~/.ssh
cat ${snakeOilPrivateKey} > ~/.ssh/id_ecdsa
chmod 400 ~/.ssh/id_ecdsa
ssh-keyscan primary >> ~/.ssh/known_hosts
chown -R pgbackrest:pgbackrest ~
"""))
with subtest("backup/restore works with remote instance/local repo (SSH)"):
backup.succeed("sudo -u pgbackrest pgbackrest --stanza=default stanza-create")
backup.succeed("sudo -u pgbackrest pgbackrest --stanza=default check")
backup.systemctl("start pgbackrest-default-future")
# corrupt cluster
primary.systemctl("stop postgresql")
primary.execute("rm ${nodes.primary.services.postgresql.dataDir}/global/pg_control")
primary.succeed("sudo -u postgres pgbackrest --stanza=default restore --delta")
primary.systemctl("start postgresql")
primary.wait_for_unit("postgresql.service")
assert "hello world" in primary.succeed("sudo -u postgres psql -c 'TABLE t;'")
'';
}

View file

@ -0,0 +1,95 @@
{ lib, pkgs, ... }:
let
inherit (import ../ssh-keys.nix pkgs) snakeOilPrivateKey snakeOilPublicKey;
backupPath = "/home/backup";
in
{
name = "pgbackrest-sftp";
meta = {
maintainers = with lib.maintainers; [ wolfgangwalther ];
};
nodes.primary =
{
pkgs,
...
}:
{
services.postgresql = {
enable = true;
initialScript = pkgs.writeText "init.sql" ''
CREATE TABLE t(c text);
INSERT INTO t VALUES ('hello world');
'';
};
services.pgbackrest = {
enable = true;
repos.backup = {
type = "sftp";
path = "/home/backup";
sftp-host-key-check-type = "none";
sftp-host-key-hash-type = "sha256";
sftp-host-user = "backup";
sftp-private-key-file = "/var/lib/pgbackrest/sftp_key";
};
stanzas.default.jobs.future = {
schedule = "3000-01-01";
type = "diff";
};
};
};
nodes.backup =
{
nodes,
...
}:
{
services.openssh.enable = true;
users.users.backup = {
name = "backup";
group = "backup";
isNormalUser = true;
createHome = true;
openssh.authorizedKeys.keys = [
snakeOilPublicKey
];
};
users.groups.backup = { };
};
testScript =
{ nodes, ... }:
''
start_all()
primary.wait_for_unit("multi-user.target")
backup.wait_for_unit("multi-user.target")
primary.log(primary.succeed("""
HOME="/var/lib/pgbackrest"
cat ${snakeOilPrivateKey} > ~/sftp_key
chown -R pgbackrest:pgbackrest ~/sftp_key
chmod 770 ~
"""))
with subtest("backup/restore works with local instance/remote repo (SFTP)"):
primary.succeed("sudo -u pgbackrest pgbackrest --stanza=default stanza-create", timeout=10)
primary.succeed("sudo -u pgbackrest pgbackrest --stanza=default check")
primary.systemctl("start pgbackrest-default-future")
# corrupt cluster
primary.systemctl("stop postgresql")
primary.execute("rm ${nodes.primary.services.postgresql.dataDir}/global/pg_control")
primary.succeed("sudo -u postgres pgbackrest --stanza=default restore --delta")
primary.systemctl("start postgresql")
primary.wait_for_unit("postgresql.service")
assert "hello world" in primary.succeed("sudo -u postgres psql -c 'TABLE t;'")
'';
}

View file

@ -1,31 +1,32 @@
{
lib,
stdenv,
fetchFromGitHub,
meson,
ninja,
python3,
pkg-config,
libbacktrace,
bzip2,
lz4,
fetchFromGitHub,
lib,
libbacktrace,
libpq,
libssh2,
libxml2,
libyaml,
lz4,
meson,
ninja,
pkg-config,
python3,
stdenv,
zlib,
libssh2,
zstd,
nixosTests,
}:
stdenv.mkDerivation rec {
stdenv.mkDerivation (finalAttrs: {
pname = "pgbackrest";
version = "2.55.1";
src = fetchFromGitHub {
owner = "pgbackrest";
repo = "pgbackrest";
rev = "release/${version}";
sha256 = "sha256-A1dTywcCHBu7Ml0Q9k//VVPFN1C3kmmMkq4ok9T4g94=";
tag = "release/${finalAttrs.version}";
hash = "sha256-A1dTywcCHBu7Ml0Q9k//VVPFN1C3kmmMkq4ok9T4g94=";
};
strictDeps = true;
@ -33,28 +34,30 @@ stdenv.mkDerivation rec {
nativeBuildInputs = [
meson
ninja
python3
pkg-config
python3
];
buildInputs = [
libbacktrace
bzip2
lz4
libbacktrace
libpq
libssh2
libxml2
libyaml
lz4
zlib
libssh2
zstd
];
meta = with lib; {
passthru.tests = nixosTests.pgbackrest;
meta = {
description = "Reliable PostgreSQL backup & restore";
homepage = "https://pgbackrest.org/";
changelog = "https://github.com/pgbackrest/pgbackrest/releases";
license = licenses.mit;
homepage = "https://pgbackrest.org";
changelog = "https://github.com/pgbackrest/pgbackrest/releases/tag/release%2F${finalAttrs.version}";
license = lib.licenses.mit;
mainProgram = "pgbackrest";
maintainers = with maintainers; [ zaninime ];
maintainers = with lib.maintainers; [ zaninime ];
};
}
})