nixos/pgbackrest: init module (#404384)

nixos/doc/manual/release-notes/rl-2505.section.md

@@ -208,6 +208,8 @@
 
 - [GLPI-Agent](https://github.com/glpi-project/glpi-agent), GLPI Agent. Available as [services.glpiAgent](options.html#opt-services.glpiAgent.enable).
 
+- [pgBackRest](https://pgbackrest.org), a reliable backup and restore solution for PostgreSQL. Available as [services.pgbackrest](options.html#opt-services.pgbackrest.enable).
+
 - [Recyclarr](https://github.com/recyclarr/recyclarr) a TRaSH Guides synchronizer for Sonarr and Radarr. Available as [services.recyclarr](#opt-services.recyclarr.enable).
 
 - [Rebuilderd](https://github.com/kpcyrd/rebuilderd) an independent verification of binary packages - Reproducible Builds. Available as [services.rebuilderd](#opt-services.rebuilderd.enable).

nixos/modules/module-list.nix

@@ -440,6 +440,7 @@
   ./services/backup/duplicati.nix
   ./services/backup/duplicity.nix
   ./services/backup/mysql-backup.nix
+  ./services/backup/pgbackrest.nix
   ./services/backup/postgresql-backup.nix
   ./services/backup/postgresql-wal-receiver.nix
   ./services/backup/restic-rest-server.nix

nixos/modules/services/backup/pgbackrest.nix

@@ -0,0 +1,426 @@
+{
+  config,
+  lib,
+  pkgs,
+  ...
+}:
+
+let
+  cfg = config.services.pgbackrest;
+
+  settingsFormat = pkgs.formats.ini {
+    listsAsDuplicateKeys = true;
+  };
+
+  # pgBackRest "options"
+  settingsType =
+    with lib.types;
+    attrsOf (oneOf [
+      bool
+      ints.unsigned
+      str
+      (attrsOf str)
+      (listOf str)
+    ]);
+
+  # Applied to both repoNNN-* and pgNNN-* options in global and stanza sections.
+  flattenWithIndex =
+    attrs: prefix:
+    lib.concatMapAttrs (
+      name:
+      let
+        index = lib.lists.findFirstIndex (n: n == name) null (lib.attrNames attrs);
+        index1 = index + 1;
+      in
+      lib.mapAttrs' (option: lib.nameValuePair "${prefix}${toString index1}-${option}")
+    ) attrs;
+
+  # Remove nulls, turn attrsets into lists and bools into y/n
+  normalize =
+    x:
+    lib.pipe x [
+      (lib.filterAttrs (_: v: v != null))
+      (lib.mapAttrs (_: v: if lib.isAttrs v then lib.mapAttrsToList (n': v': "${n'}=${v'}") v else v))
+      (lib.mapAttrs (
+        _: v:
+        if v == true then
+          "y"
+        else if v == false then
+          "n"
+        else
+          v
+      ))
+    ];
+
+  fullConfig =
+    {
+      global = normalize (cfg.settings // flattenWithIndex cfg.repos "repo");
+    }
+    // lib.mapAttrs (
+      _: cfg': normalize (cfg'.settings // flattenWithIndex cfg'.instances "pg")
+    ) cfg.stanzas;
+
+  namedJobs = lib.listToAttrs (
+    lib.flatten (
+      lib.mapAttrsToList (
+        stanza:
+        { jobs, ... }:
+        lib.mapAttrsToList (
+          job: attrs: lib.nameValuePair "pgbackrest-${stanza}-${job}" (attrs // { inherit stanza job; })
+        ) jobs
+      ) cfg.stanzas
+    )
+  );
+
+  disabledOption = lib.mkOption {
+    default = null;
+    readOnly = true;
+    internal = true;
+  };
+
+  secretPathOption =
+    with lib.types;
+    lib.mkOption {
+      type = nullOr (pathWith {
+        inStore = false;
+        absolute = true;
+      });
+      default = null;
+      internal = true;
+    };
+in
+
+{
+  meta = {
+    maintainers = with lib.maintainers; [ wolfgangwalther ];
+  };
+
+  # TODO: Add enableServer option and corresponding pgBackRest TLS server service.
+  # TODO: Allow command-specific options
+  # TODO: Write wrapper around pgbackrest to turn --repo=<name> into --repo=<number>
+  # The following two are dependent on improvements upstream:
+  #   https://github.com/pgbackrest/pgbackrest/issues/2621
+  # TODO: Add support for more repository types
+  # TODO: Support passing encryption key safely
+  options.services.pgbackrest = {
+    enable = lib.mkEnableOption "pgBackRest";
+
+    repos = lib.mkOption {
+      type =
+        with lib.types;
+        attrsOf (
+          submodule (
+            { config, name, ... }:
+            let
+              setHostForType =
+                type:
+                if name == "localhost" then
+                  null
+                # "posix" is the default repo type, which uses the -host option.
+                # Other types use prefixed options, for example -sftp-host.
+                else if config.type or "posix" != type then
+                  null
+                else
+                  name;
+            in
+            {
+              freeformType = settingsType;
+
+              options.host = lib.mkOption {
+                type = nullOr str;
+                default = setHostForType "posix";
+                defaultText = lib.literalExpression "name";
+                description = "Repository host when operating remotely";
+              };
+
+              options.sftp-host = lib.mkOption {
+                type = nullOr str;
+                default = setHostForType "sftp";
+                defaultText = lib.literalExpression "name";
+                description = "SFTP repository host";
+              };
+
+              options.sftp-private-key-file = lib.mkOption {
+                type = nullOr (pathWith {
+                  inStore = false;
+                  absolute = true;
+                });
+                default = null;
+                description = ''
+                  SFTP private key file.
+
+                  The file must be accessible by both the pgbackrest and the postgres users.
+                '';
+              };
+
+              # The following options should not be used; they would store secrets in the store.
+              options.azure-key = disabledOption;
+              options.cipher-pass = disabledOption;
+              options.s3-key = disabledOption;
+              options.s3-key-secret = disabledOption;
+              options.s3-kms-key-id = disabledOption; # unsure whether that's a secret or not
+              options.s3-sse-customer-key = disabledOption; # unsure whether that's a secret or not
+              options.s3-token = disabledOption;
+              options.sftp-private-key-passphrase = disabledOption;
+
+              # The following options are not fully supported / tested, yet, but point to files with secrets.
+              # Users can already set those options, but we'll force non-store paths.
+              options.gcs-key = secretPathOption;
+              options.host-cert-file = secretPathOption;
+              options.host-key-file = secretPathOption;
+            }
+          )
+        );
+      default = { };
+      description = ''
+        An attribute set of repositories as described in:
+        <https://pgbackrest.org/configuration.html#section-repository>
+
+        Each repository defaults to set `repo-host` to the attribute's name.
+        The special value "localhost" will unset `repo-host`.
+
+        ::: {.note}
+        The prefix `repoNNN-` is added automatically.
+        Example: Use `path` instead of `repo1-path`.
+        :::
+      '';
+      example = lib.literalExpression ''
+        {
+          localhost.path = "/var/lib/backup";
+          "backup.example.com".host-type = "tls";
+        }
+      '';
+    };
+
+    stanzas = lib.mkOption {
+      type =
+        with lib.types;
+        attrsOf (submodule {
+          options = {
+            jobs = lib.mkOption {
+              type = lib.types.attrsOf (
+                lib.types.submodule {
+                  options.schedule = lib.mkOption {
+                    type = lib.types.str;
+                    description = ''
+                      When or how often the backup should run.
+                      Must be in the format described in {manpage}`systemd.time(7)`.
+                    '';
+                  };
+
+                  options.type = lib.mkOption {
+                    type = lib.types.str;
+                    description = ''
+                      Backup type as described in:
+                      <https://pgbackrest.org/command.html#command-backup/category-command/option-type>
+                    '';
+                  };
+                }
+              );
+              default = { };
+              description = ''
+                Backups jobs to schedule for this stanza as described in:
+                <https://pgbackrest.org/user-guide.html#quickstart/schedule-backup>
+              '';
+              example = lib.literalExpression ''
+                {
+                  weekly = { schedule = "Sun, 6:30"; type = "full"; };
+                  daily = { schedule = "Mon..Sat, 6:30"; type = "diff"; };
+                }
+              '';
+            };
+
+            instances = lib.mkOption {
+              type =
+                with lib.types;
+                attrsOf (
+                  submodule (
+                    { name, ... }:
+                    {
+                      freeformType = settingsType;
+                      options.host = lib.mkOption {
+                        type = nullOr str;
+                        default = if name == "localhost" then null else name;
+                        defaultText = lib.literalExpression ''if name == "localhost" then null else name'';
+                        description = "PostgreSQL host for operating remotely.";
+                      };
+
+                      # The following options are not fully supported / tested, yet, but point to files with secrets.
+                      # Users can already set those options, but we'll force non-store paths.
+                      options.host-cert-file = secretPathOption;
+                      options.host-key-file = secretPathOption;
+                    }
+                  )
+                );
+              default = { };
+              description = ''
+                An attribute set of database instances as described in:
+                <https://pgbackrest.org/configuration.html#section-stanza>
+
+                Each instance defaults to set `pg-host` to the attribute's name.
+                The special value "localhost" will unset `pg-host`.
+
+                ::: {.note}
+                The prefix `pgNNN-` is added automatically.
+                Example: Use `user` instead of `pg1-user`.
+                :::
+              '';
+              example = lib.literalExpression ''
+                {
+                  localhost.database = "app";
+                  "postgres.example.com".port = "5433";
+                }
+              '';
+            };
+
+            settings = lib.mkOption {
+              type = lib.types.submodule {
+                freeformType = settingsType;
+
+                # The following options are not fully supported / tested, yet, but point to files with secrets.
+                # Users can already set those options, but we'll force non-store paths.
+                options.tls-server-cert-file = secretPathOption;
+                options.tls-server-key-file = secretPathOption;
+              };
+              default = { };
+              description = ''
+                An attribute set of options as described in:
+                <https://pgbackrest.org/configuration.html>
+
+                All options can be used.
+                Repository options should be set via [`repos`](#opt-services.pgbackrest.repos) instead.
+                Stanza options should be set via [`instances`](#opt-services.pgbackrest.stanzas._name_.instances) instead.
+              '';
+              example = lib.literalExpression ''
+                {
+                  process-max = 2;
+                }
+              '';
+            };
+          };
+        });
+      default = { };
+      description = ''
+        An attribute set of stanzas as described in:
+        <https://pgbackrest.org/user-guide.html#quickstart/configure-stanza>
+      '';
+    };
+
+    settings = lib.mkOption {
+      type = lib.types.submodule {
+        freeformType = settingsType;
+
+        # The following options are not fully supported / tested, yet, but point to files with secrets.
+        # Users can already set those options, but we'll force non-store paths.
+        options.tls-server-cert-file = secretPathOption;
+        options.tls-server-key-file = secretPathOption;
+      };
+      default = { };
+      description = ''
+        An attribute set of options as described in:
+        <https://pgbackrest.org/configuration.html>
+
+        All globally available options, i.e. all except stanza options, can be used.
+        Repository options should be set via [`repos`](#opt-services.pgbackrest.repos) instead.
+      '';
+      example = lib.literalExpression ''
+        {
+          process-max = 2;
+        }
+      '';
+    };
+  };
+
+  config = lib.mkIf cfg.enable (
+    lib.mkMerge [
+      {
+        services.pgbackrest.settings = {
+          log-level-console = lib.mkDefault "info";
+          log-level-file = lib.mkDefault "off";
+          cmd-ssh = lib.getExe pkgs.openssh;
+        };
+
+        environment.systemPackages = [ pkgs.pgbackrest ];
+        environment.etc."pgbackrest/pgbackrest.conf".source =
+          settingsFormat.generate "pgbackrest.conf" fullConfig;
+
+        users.users.pgbackrest = {
+          name = "pgbackrest";
+          group = "pgbackrest";
+          description = "pgBackRest service user";
+          isSystemUser = true;
+          useDefaultShell = true;
+          createHome = true;
+          home = cfg.repos.localhost.path or "/var/lib/pgbackrest";
+        };
+        users.groups.pgbackrest = { };
+
+        systemd.services = lib.mapAttrs (
+          _:
+          {
+            stanza,
+            job,
+            type,
+            ...
+          }:
+          {
+            description = "pgBackRest job ${job} for stanza ${stanza}";
+
+            serviceConfig = {
+              User = "pgbackrest";
+              Group = "pgbackrest";
+              Type = "oneshot";
+              # stanza-create is idempotent, so safe to always run
+              ExecStartPre = "${lib.getExe pkgs.pgbackrest} --stanza='${stanza}' stanza-create";
+              ExecStart = "${lib.getExe pkgs.pgbackrest} --stanza='${stanza}' backup --type='${type}'";
+            };
+          }
+        ) namedJobs;
+
+        systemd.timers = lib.mapAttrs (
+          name:
+          {
+            stanza,
+            job,
+            schedule,
+            ...
+          }:
+          {
+            description = "pgBackRest job ${job} for stanza ${stanza}";
+            wantedBy = [ "timers.target" ];
+            after = [ "network-online.target" ];
+            wants = [ "network-online.target" ];
+            timerConfig = {
+              OnCalendar = schedule;
+              Persistent = true;
+              Unit = "${name}.service";
+            };
+          }
+        ) namedJobs;
+      }
+
+      # The default stanza is set up for the local postgresql instance.
+      # It does not backup automatically, the systemd timer still needs to be set.
+      (lib.mkIf config.services.postgresql.enable {
+        services.pgbackrest.stanzas.default = {
+          settings.cmd = lib.getExe pkgs.pgbackrest;
+          instances.localhost = {
+            path = config.services.postgresql.dataDir;
+            user = "postgres";
+          };
+        };
+        services.postgresql.identMap = ''
+          postgres pgbackrest postgres
+        '';
+        services.postgresql.initdbArgs = [ "--allow-group-access" ];
+        users.users.pgbackrest.extraGroups = [ "postgres" ];
+
+        services.postgresql.settings = {
+          archive_command = ''${lib.getExe pkgs.pgbackrest} --stanza=default archive-push "%p"'';
+          archive_mode = lib.mkDefault "on";
+        };
+        users.groups.pgbackrest.members = [ "postgres" ];
+      })
+    ]
+  );
+}

nixos/tests/all-tests.nix

@@ -1025,6 +1025,7 @@ in
   peertube = handleTestOn [ "x86_64-linux" ] ./web-apps/peertube.nix { };
   peroxide = handleTest ./peroxide.nix { };
   pgadmin4 = runTest ./pgadmin4.nix;
+  pgbackrest = import ./pgbackrest { inherit runTest; };
   pgbouncer = handleTest ./pgbouncer.nix { };
   pghero = runTest ./pghero.nix;
   pgweb = runTest ./pgweb.nix;

nixos/tests/pgbackrest/default.nix

@@ -0,0 +1,5 @@
+{ runTest }:
+{
+  posix = runTest ./posix.nix;
+  sftp = runTest ./sftp.nix;
+}

nixos/tests/pgbackrest/posix.nix

@@ -0,0 +1,147 @@
+{ lib, pkgs, ... }:
+let
+  inherit (import ../ssh-keys.nix pkgs) snakeOilPrivateKey snakeOilPublicKey;
+  backupPath = "/var/lib/pgbackrest";
+in
+{
+  name = "pgbackrest-posix";
+
+  meta = {
+    maintainers = with lib.maintainers; [ wolfgangwalther ];
+  };
+
+  nodes.primary =
+    {
+      pkgs,
+      ...
+    }:
+    {
+      services.openssh.enable = true;
+      users.users.postgres.openssh.authorizedKeys.keys = [
+        snakeOilPublicKey
+      ];
+
+      services.postgresql = {
+        enable = true;
+        initialScript = pkgs.writeText "init.sql" ''
+          CREATE TABLE t(c text);
+          INSERT INTO t VALUES ('hello world');
+        '';
+      };
+
+      services.pgbackrest = {
+        enable = true;
+        repos.backup = {
+          type = "posix";
+          path = backupPath;
+          host-user = "pgbackrest";
+        };
+      };
+    };
+
+  nodes.backup =
+    {
+      nodes,
+      ...
+    }:
+    {
+      services.openssh.enable = true;
+      users.users.pgbackrest.openssh.authorizedKeys.keys = [
+        snakeOilPublicKey
+      ];
+
+      services.pgbackrest = {
+        enable = true;
+        repos.localhost.path = backupPath;
+
+        stanzas.default = {
+          jobs.future = {
+            schedule = "3000-01-01";
+            type = "full";
+          };
+          instances.primary = {
+            path = nodes.primary.services.postgresql.dataDir;
+            user = "postgres";
+          };
+        };
+
+        # Examples from https://pgbackrest.org/configuration.html#introduction
+        # Not used for the test, except for dumping the config.
+        stanzas.config-format.settings = {
+          start-fast = true;
+          compress-level = 3;
+          buffer-size = "2MiB";
+          db-timeout = 600;
+          db-exclude = [
+            "db1"
+            "db2"
+            "db5"
+          ];
+          tablespace-map = {
+            ts_01 = "/db/ts_01";
+            ts_02 = "/db/ts_02";
+          };
+        };
+      };
+    };
+
+  testScript =
+    { nodes, ... }:
+    ''
+      start_all()
+
+      primary.wait_for_unit("multi-user.target")
+      backup.wait_for_unit("multi-user.target")
+
+      with subtest("config file is written correctly"):
+        from textwrap import dedent
+        have = backup.succeed("cat /etc/pgbackrest/pgbackrest.conf")
+        want = dedent("""\
+          [config-format]
+          buffer-size=2MiB
+          compress-level=3
+          db-exclude=db1
+          db-exclude=db2
+          db-exclude=db5
+          db-timeout=600
+          start-fast=y
+          tablespace-map=ts_01=/db/ts_01
+          tablespace-map=ts_02=/db/ts_02
+        """)
+        assert want in have, repr((want, have))
+
+      primary.log(primary.succeed("""
+        HOME="${nodes.primary.services.postgresql.dataDir}"
+        mkdir -m 700 -p ~/.ssh
+        cat ${snakeOilPrivateKey} > ~/.ssh/id_ecdsa
+        chmod 400 ~/.ssh/id_ecdsa
+        ssh-keyscan backup >> ~/.ssh/known_hosts
+        chown -R postgres:postgres ~/.ssh
+      """))
+
+      backup.log(backup.succeed("""
+        HOME="${backupPath}"
+        mkdir -m 700 -p ~/.ssh
+        cat ${snakeOilPrivateKey} > ~/.ssh/id_ecdsa
+        chmod 400 ~/.ssh/id_ecdsa
+        ssh-keyscan primary >> ~/.ssh/known_hosts
+        chown -R pgbackrest:pgbackrest ~
+      """))
+
+      with subtest("backup/restore works with remote instance/local repo (SSH)"):
+        backup.succeed("sudo -u pgbackrest pgbackrest --stanza=default stanza-create")
+        backup.succeed("sudo -u pgbackrest pgbackrest --stanza=default check")
+
+        backup.systemctl("start pgbackrest-default-future")
+
+        # corrupt cluster
+        primary.systemctl("stop postgresql")
+        primary.execute("rm ${nodes.primary.services.postgresql.dataDir}/global/pg_control")
+
+        primary.succeed("sudo -u postgres pgbackrest --stanza=default restore --delta")
+
+        primary.systemctl("start postgresql")
+        primary.wait_for_unit("postgresql.service")
+        assert "hello world" in primary.succeed("sudo -u postgres psql -c 'TABLE t;'")
+    '';
+}

nixos/tests/pgbackrest/sftp.nix

@@ -0,0 +1,95 @@
+{ lib, pkgs, ... }:
+let
+  inherit (import ../ssh-keys.nix pkgs) snakeOilPrivateKey snakeOilPublicKey;
+  backupPath = "/home/backup";
+in
+{
+  name = "pgbackrest-sftp";
+
+  meta = {
+    maintainers = with lib.maintainers; [ wolfgangwalther ];
+  };
+
+  nodes.primary =
+    {
+      pkgs,
+      ...
+    }:
+    {
+      services.postgresql = {
+        enable = true;
+        initialScript = pkgs.writeText "init.sql" ''
+          CREATE TABLE t(c text);
+          INSERT INTO t VALUES ('hello world');
+        '';
+      };
+
+      services.pgbackrest = {
+        enable = true;
+        repos.backup = {
+          type = "sftp";
+          path = "/home/backup";
+          sftp-host-key-check-type = "none";
+          sftp-host-key-hash-type = "sha256";
+          sftp-host-user = "backup";
+          sftp-private-key-file = "/var/lib/pgbackrest/sftp_key";
+        };
+
+        stanzas.default.jobs.future = {
+          schedule = "3000-01-01";
+          type = "diff";
+        };
+      };
+    };
+
+  nodes.backup =
+    {
+      nodes,
+      ...
+    }:
+    {
+      services.openssh.enable = true;
+      users.users.backup = {
+        name = "backup";
+        group = "backup";
+        isNormalUser = true;
+        createHome = true;
+        openssh.authorizedKeys.keys = [
+          snakeOilPublicKey
+        ];
+      };
+      users.groups.backup = { };
+    };
+
+  testScript =
+    { nodes, ... }:
+    ''
+      start_all()
+
+      primary.wait_for_unit("multi-user.target")
+      backup.wait_for_unit("multi-user.target")
+
+      primary.log(primary.succeed("""
+        HOME="/var/lib/pgbackrest"
+        cat ${snakeOilPrivateKey} > ~/sftp_key
+        chown -R pgbackrest:pgbackrest ~/sftp_key
+        chmod 770 ~
+      """))
+
+      with subtest("backup/restore works with local instance/remote repo (SFTP)"):
+        primary.succeed("sudo -u pgbackrest pgbackrest --stanza=default stanza-create", timeout=10)
+        primary.succeed("sudo -u pgbackrest pgbackrest --stanza=default check")
+
+        primary.systemctl("start pgbackrest-default-future")
+
+        # corrupt cluster
+        primary.systemctl("stop postgresql")
+        primary.execute("rm ${nodes.primary.services.postgresql.dataDir}/global/pg_control")
+
+        primary.succeed("sudo -u postgres pgbackrest --stanza=default restore --delta")
+
+        primary.systemctl("start postgresql")
+        primary.wait_for_unit("postgresql.service")
+        assert "hello world" in primary.succeed("sudo -u postgres psql -c 'TABLE t;'")
+    '';
+}

pkgs/by-name/pg/pgbackrest/package.nix

@@ -1,31 +1,32 @@
 {
-  lib,
-  stdenv,
-  fetchFromGitHub,
-  meson,
-  ninja,
-  python3,
-  pkg-config,
-  libbacktrace,
   bzip2,
-  lz4,
+  fetchFromGitHub,
+  lib,
+  libbacktrace,
   libpq,
+  libssh2,
   libxml2,
   libyaml,
+  lz4,
+  meson,
+  ninja,
+  pkg-config,
+  python3,
+  stdenv,
   zlib,
-  libssh2,
   zstd,
+  nixosTests,
 }:
 
-stdenv.mkDerivation rec {
+stdenv.mkDerivation (finalAttrs: {
   pname = "pgbackrest";
   version = "2.55.1";
 
   src = fetchFromGitHub {
     owner = "pgbackrest";
     repo = "pgbackrest";
-    rev = "release/${version}";
+    tag = "release/${finalAttrs.version}";
-    sha256 = "sha256-A1dTywcCHBu7Ml0Q9k//VVPFN1C3kmmMkq4ok9T4g94=";
+    hash = "sha256-A1dTywcCHBu7Ml0Q9k//VVPFN1C3kmmMkq4ok9T4g94=";
   };
 
   strictDeps = true;
@@ -33,28 +34,30 @@ stdenv.mkDerivation rec {
   nativeBuildInputs = [
     meson
     ninja
-    python3
     pkg-config
+    python3
   ];
 
   buildInputs = [
-    libbacktrace
     bzip2
-    lz4
+    libbacktrace
     libpq
+    libssh2
     libxml2
     libyaml
+    lz4
     zlib
-    libssh2
     zstd
   ];
 
-  meta = with lib; {
+  passthru.tests = nixosTests.pgbackrest;
+
+  meta = {
     description = "Reliable PostgreSQL backup & restore";
-    homepage = "https://pgbackrest.org/";
+    homepage = "https://pgbackrest.org";
-    changelog = "https://github.com/pgbackrest/pgbackrest/releases";
+    changelog = "https://github.com/pgbackrest/pgbackrest/releases/tag/release%2F${finalAttrs.version}";
-    license = licenses.mit;
+    license = lib.licenses.mit;
     mainProgram = "pgbackrest";
-    maintainers = with maintainers; [ zaninime ];
+    maintainers = with lib.maintainers; [ zaninime ];
   };
-}
+})