Merge pull request #293996 from wolfgangwalther/postgresql-cleanup

postgresql: more cleanup

.git-blame-ignore-revs

@@ -105,3 +105,6 @@ fb0e5be84331188a69b3edd31679ca6576edb75a
 
 # {pkgs/development/cuda-modules,pkgs/test/cuda,pkgs/top-level/cuda-packages.nix}: reformat all CUDA files with nixfmt-rfc-style 2023-03-01
 802a1b4d3338f24cbc4efd704616654456d75a94
+
+# postgresql: move packages.nix to ext/default.nix
+719034f6f6749d624faa28dff259309fc0e3e730

nixos/modules/services/databases/postgresql.nix

@@ -14,7 +14,7 @@ let
       #     package = pkgs.postgresql_<major>;
       #   };
       # works.
-      base = if cfg.enableJIT then cfg.package.withJIT else cfg.package;
+      base = if cfg.enableJIT then cfg.package.withJIT else cfg.package.withoutJIT;
     in
     if cfg.extraPlugins == []
       then base

nixos/tests/postgresql-jit.nix

@@ -1,6 +1,7 @@
 { system ? builtins.currentSystem
 , config ? {}
 , pkgs ? import ../.. { inherit system config; }
+, package ? null
 }:
 
 with import ../lib/testing-python.nix { inherit system pkgs; };
@@ -9,14 +10,17 @@ let
   inherit (pkgs) lib;
   packages = builtins.attrNames (import ../../pkgs/servers/sql/postgresql pkgs);
 
-  mkJitTest = packageName: makeTest {
+  mkJitTestFromName = name:
-    name = "${packageName}";
+    mkJitTest pkgs.${name};
+
+  mkJitTest = package: makeTest {
+    name = package.name;
     meta.maintainers = with lib.maintainers; [ ma27 ];
     nodes.machine = { pkgs, lib, ... }: {
       services.postgresql = {
+        inherit package;
         enable = true;
         enableJIT = true;
-        package = pkgs.${packageName};
         initialScript = pkgs.writeText "init.sql" ''
           create table demo (id int);
           insert into demo (id) select generate_series(1, 5);
@@ -45,4 +49,7 @@ let
     '';
   };
 in
-lib.genAttrs packages mkJitTest
+if package == null then
+  lib.genAttrs packages mkJitTestFromName
+else
+  mkJitTest package

nixos/tests/postgresql-wal-receiver.nix

@@ -1,6 +1,7 @@
 { system ? builtins.currentSystem,
   config ? {},
-  pkgs ? import ../.. { inherit system config; }
+  pkgs ? import ../.. { inherit system config; },
+  package ? null
 }:
 
 with import ../lib/testing-python.nix { inherit system pkgs; };
@@ -9,43 +10,38 @@ let
   lib = pkgs.lib;
 
   # Makes a test for a PostgreSQL package, given by name and looked up from `pkgs`.
-  makePostgresqlWalReceiverTest = postgresqlPackage:
+  makeTestAttribute = name:
   {
-    name = postgresqlPackage;
+    inherit name;
-    value =
+    value = makePostgresqlWalReceiverTest pkgs."${name}";
+  };
+
+  makePostgresqlWalReceiverTest = pkg:
     let
-        pkg = pkgs."${postgresqlPackage}";
       postgresqlDataDir = "/var/lib/postgresql/${pkg.psqlSchema}";
       replicationUser = "wal_receiver_user";
       replicationSlot = "wal_receiver_slot";
       replicationConn = "postgresql://${replicationUser}@localhost";
       baseBackupDir = "/tmp/pg_basebackup";
       walBackupDir = "/tmp/pg_wal";
-        atLeast12 = lib.versionAtLeast pkg.version "12.0";
 
-        recoveryFile = if atLeast12
+      recoveryFile = pkgs.writeTextDir "recovery.signal" "";
-            then pkgs.writeTextDir "recovery.signal" ""
-            else pkgs.writeTextDir "recovery.conf" "restore_command = 'cp ${walBackupDir}/%f %p'";
 
     in makeTest {
-        name = "postgresql-wal-receiver-${postgresqlPackage}";
+      name = "postgresql-wal-receiver-${pkg.name}";
       meta.maintainers = with lib.maintainers; [ pacien ];
 
       nodes.machine = { ... }: {
         services.postgresql = {
           package = pkg;
           enable = true;
-            settings = lib.mkMerge [
+          settings = {
-              {
-                wal_level = "archive"; # alias for replica on pg >= 9.6
-                max_wal_senders = 10;
             max_replication_slots = 10;
-              }
+            max_wal_senders = 10;
-              (lib.mkIf atLeast12 {
-                restore_command = "cp ${walBackupDir}/%f %p";
             recovery_end_command = "touch recovery.done";
-              })
+            restore_command = "cp ${walBackupDir}/%f %p";
-            ];
+            wal_level = "archive"; # alias for replica on pg >= 9.6
+          };
           authentication = ''
             host replication ${replicationUser} all trust
           '';
@@ -113,7 +109,11 @@ let
         )
       '';
     };
-    };
 
-# Maps the generic function over all attributes of PostgreSQL packages
+in
-in builtins.listToAttrs (map makePostgresqlWalReceiverTest (builtins.attrNames (import ../../pkgs/servers/sql/postgresql pkgs)))
+if package == null then
+  # all-tests.nix: Maps the generic function over all attributes of PostgreSQL packages
+  builtins.listToAttrs (map makeTestAttribute (builtins.attrNames (import ../../pkgs/servers/sql/postgresql pkgs)))
+else
+  # Called directly from <package>.tests
+  makePostgresqlWalReceiverTest package

pkgs/servers/sql/postgresql/12.nix

@@ -1,10 +1,4 @@
 import ./generic.nix {
   version = "12.18";
   hash = "sha256-T5kZcl2UHOmGjgf+HtHTqGdIWZtIM4ZUdYOSi3TDkYo=";
-  muslPatches = {
-    icu-collations-hack = {
-      url = "https://git.alpinelinux.org/aports/plain/testing/postgresql12/icu-collations-hack.patch?id=d5227c91adda59d4e7f55f13468f0314e8869174";
-      hash = "sha256-wuwjvGHArkRNwFo40g3p43W32OrJohretlt6iSRlJKg=";
-    };
-  };
 }

pkgs/servers/sql/postgresql/13.nix

@@ -2,10 +2,6 @@ import ./generic.nix {
   version = "13.14";
   hash = "sha256-uN8HhVGJiWC9UA3F04oXfpkFN234H+fytmChQH+mpe0=";
   muslPatches = {
-    icu-collations-hack = {
-      url = "https://git.alpinelinux.org/aports/plain/main/postgresql14/icu-collations-hack.patch?id=56999e6d0265ceff5c5239f85fdd33e146f06cb7";
-      hash = "sha256-wuwjvGHArkRNwFo40g3p43W32OrJohretlt6iSRlJKg=";
-    };
     disable-test-collate-icu-utf8 = {
       url = "https://git.alpinelinux.org/aports/plain/main/postgresql13/disable-test-collate.icu.utf8.patch?id=69faa146ec9fff3b981511068f17f9e629d4688b";
       hash = "sha256-jS/qxezaiaKhkWeMCXwpz1SDJwUWn9tzN0uKaZ3Ph2Y=";

pkgs/servers/sql/postgresql/14.nix

@@ -2,10 +2,6 @@ import ./generic.nix {
   version = "14.11";
   hash = "sha256-pnC9fc4i3K1Cl7JhE2s7HUoJpvVBcZViqhTKY78paKg=";
   muslPatches = {
-    icu-collations-hack = {
-      url = "https://git.alpinelinux.org/aports/plain/main/postgresql14/icu-collations-hack.patch?id=56999e6d0265ceff5c5239f85fdd33e146f06cb7";
-      hash = "sha256-wuwjvGHArkRNwFo40g3p43W32OrJohretlt6iSRlJKg=";
-    };
     disable-test-collate-icu-utf8 = {
       url = "https://git.alpinelinux.org/aports/plain/main/postgresql14/disable-test-collate.icu.utf8.patch?id=56999e6d0265ceff5c5239f85fdd33e146f06cb7";
       hash = "sha256-jXe23AxnFjEl+TZQm4R7rStk2Leo08ctxMNmu1xr5zM=";

pkgs/servers/sql/postgresql/15.nix

@@ -1,10 +1,4 @@
 import ./generic.nix {
   version = "15.6";
   hash = "sha256-hFUUbtnGnJOlfelUrq0DAsr60DXCskIXXWqh4X68svs=";
-  muslPatches = {
-    icu-collations-hack = {
-      url = "https://git.alpinelinux.org/aports/plain/main/postgresql15/icu-collations-hack.patch?id=f424e934e6d076c4ae065ce45e734aa283eecb9c";
-      hash = "sha256-HgtmhF4OJYU9macGJbTB9PjQi/yW7c3Akm3U0niWs8I=";
-    };
-  };
 }

pkgs/servers/sql/postgresql/16.nix

@@ -1,10 +1,4 @@
 import ./generic.nix {
   version = "16.2";
   hash = "sha256-RG6IKU28LJCFq0twYaZG+mBLS+wDUh1epnHC5a2bKVI=";
-  muslPatches = {
-    icu-collations-hack = {
-      url = "https://git.alpinelinux.org/aports/plain/main/postgresql16/icu-collations-hack.patch?id=08a24be262339fd093e641860680944c3590238e";
-      hash = "sha256-+urQdVIlADLdDPeT68XYv5rljhbK8M/7mPZn/cF+FT0=";
-    };
-  };
 }

pkgs/servers/sql/postgresql/default.nix

@@ -15,7 +15,6 @@ let
       in
       self.lib.nameValuePair attrName (import path {
         inherit jitSupport self;
-        thisAttr = attrName;
       })
     ) versions;
 

pkgs/servers/sql/postgresql/generic.nix

@@ -16,10 +16,10 @@ let
       , self, newScope, buildEnv
 
       # source specification
-      , version, hash, muslPatches
+      , version, hash, muslPatches ? {}
 
       # for tests
-      , testers, nixosTests, thisAttr
+      , testers, nixosTests
 
       # JIT
       , jitSupport
@@ -46,7 +46,8 @@ let
 
     stdenv' = if jitSupport then llvmPackages.stdenv else stdenv;
   in stdenv'.mkDerivation (finalAttrs: {
-    inherit pname version;
+    inherit version;
+    pname = pname + lib.optionalString jitSupport "-jit";
 
     src = fetchurl {
       url = "mirror://postgresql/source/v${version}/${pname}-${version}.tar.bz2";
@@ -81,16 +82,15 @@ let
     ]
       ++ lib.optionals jitSupport [ llvmPackages.llvm.dev nukeReferences patchelf ];
 
-    enableParallelBuilding = !stdenv'.isDarwin;
+    enableParallelBuilding = true;
 
     separateDebugInfo = true;
 
     buildFlags = [ "world" ];
 
-    env.NIX_CFLAGS_COMPILE = "-I${libxml2.dev}/include/libxml2";
+    # Makes cross-compiling work when xml2-config can't be executed on the host.
-
+    # Fixed upstream in https://github.com/postgres/postgres/commit/0bc8cebdb889368abdf224aeac8bc197fe4c9ae6
-    # Otherwise it retains a reference to compiler and fails; see #44767.  TODO: better.
+    env.NIX_CFLAGS_COMPILE = lib.optionalString (olderThan "13") "-I${libxml2.dev}/include/libxml2";
-    preConfigure = "CC=${stdenv'.cc.targetPrefix}cc";
 
     configureFlags = [
       "--with-openssl"
@@ -110,12 +110,11 @@ let
       ++ lib.optionals stdenv'.isLinux [ "--with-pam" ];
 
     patches = [
-      (if atLeast "16" then ./patches/disable-normalize_exec_path.patch
+      (if atLeast "16" then ./patches/relative-to-symlinks-16+.patch else ./patches/relative-to-symlinks.patch)
-       else ./patches/disable-resolve_symlinks.patch)
       ./patches/less-is-more.patch
-      ./patches/hardcode-pgxs-path.patch
+      ./patches/paths-for-split-outputs.patch
       ./patches/specify_pkglibdir_at_runtime.patch
-      ./patches/findstring.patch
+      ./patches/paths-with-postgresql-suffix.patch
 
       (substituteAll {
         src = ./patches/locale-binary-path.patch;
@@ -126,16 +125,14 @@ let
       # Using fetchurl instead of fetchpatch on purpose: https://github.com/NixOS/nixpkgs/issues/240141
       map fetchurl (lib.attrValues muslPatches)
     ) ++ lib.optionals stdenv'.isLinux  [
-      (if atLeast "13" then ./patches/socketdir-in-run-13.patch else ./patches/socketdir-in-run.patch)
+      (if atLeast "13" then ./patches/socketdir-in-run-13+.patch else ./patches/socketdir-in-run.patch)
     ];
 
     installTargets = [ "install-world" ];
 
-    LC_ALL = "C";
-
     postPatch = ''
       # Hardcode the path to pgxs so pg_config returns the path in $out
-      substituteInPlace "src/common/config_info.c" --replace HARDCODED_PGXS_PATH "$out/lib"
+      substituteInPlace "src/common/config_info.c" --subst-var out
     '' + lib.optionalString jitSupport ''
         # Force lookup of jit stuff in $out instead of $lib
         substituteInPlace src/backend/jit/jit.c --replace pkglib_path \"$out/lib\"
@@ -202,6 +199,7 @@ let
     # autodetection doesn't seem to able to find this, but it's there.
     checkTarget = "check";
 
+    # TODO: Remove after the next set of minor releases on May 9th 2024
     preCheck =
       # On musl, comment skip the following tests, because they break due to
       #     ! ERROR:  could not load library "/build/postgresql-11.5/tmp_install/nix/store/...-postgresql-11.5-lib/lib/libpqwalreceiver.so": Error loading shared library libpq.so.5: No such file or directory (needed by /build/postgresql-11.5/tmp_install/nix/store/...-postgresql-11.5-lib/lib/libpqwalreceiver.so)
@@ -254,10 +252,18 @@ let
                      this.pkgs;
 
       tests = {
-        postgresql = nixosTests.postgresql-wal-receiver.${thisAttr};
+        postgresql-wal-receiver = import ../../../../nixos/tests/postgresql-wal-receiver.nix {
+          inherit (stdenv) system;
+          pkgs = self;
+          package = this;
+        };
         pkg-config = testers.testMetaPkgConfig finalAttrs.finalPackage;
       } // lib.optionalAttrs jitSupport {
-        postgresql-jit = nixosTests.postgresql-jit.${thisAttr};
+        postgresql-jit = import ../../../../nixos/tests/postgresql-jit.nix {
+          inherit (stdenv) system;
+          pkgs = self;
+          package = this;
+        };
       };
     };
 
@@ -280,7 +286,9 @@ let
       # resulting LLVM IR isn't platform-independent this doesn't give you much.
       # In fact, I tried to test the result in a VM-test, but as soon as JIT was used to optimize
       # a query, postgres would coredump with `Illegal instruction`.
-      broken = jitSupport && (stdenv.hostPlatform != stdenv.buildPlatform);
+      broken = (jitSupport && stdenv.hostPlatform != stdenv.buildPlatform)
+        # Allmost all tests fail FATAL errors for v12 and v13
+        || (jitSupport && stdenv.hostPlatform.isMusl && olderThan "14");
     };
   });
 

pkgs/servers/sql/postgresql/patches/disable-resolve_symlinks.patch

@@ -1,12 +0,0 @@
---- a/src/common/exec.c	2014-09-04 20:19:12.236057588 +0200
-+++ b/src/common/exec.c	2014-09-04 20:19:50.550251633 +0200
-@@ -218,6 +218,9 @@
- static int
- resolve_symlinks(char *path)
- {
-+	// On NixOS we *want* stuff relative to symlinks.
-+	return 0;
-+
- #ifdef HAVE_READLINK
- 	struct stat buf;
- 	char		orig_wd[MAXPGPATH],

pkgs/servers/sql/postgresql/patches/hardcode-pgxs-path.patch

@@ -1,14 +0,0 @@
-diff -Naur postgresql-9.6.1-orig/src/common/config_info.c postgresql-9.6.1/src/common/config_info.c
---- postgresql-9.6.1-orig/src/common/config_info.c	2016-11-22 21:39:29.231929261 +0100
-+++ postgresql-9.6.1/src/common/config_info.c	2016-11-22 23:36:53.685163543 +0100
-@@ -118,7 +118,10 @@
- 	i++;
-
- 	configdata[i].name = pstrdup("PGXS");
-+	strlcpy(path, "HARDCODED_PGXS_PATH", sizeof(path));
-+/* commented out to be able to point to nix $out path
- 	get_pkglib_path(my_exec_path, path);
-+*/
- 	strlcat(path, "/pgxs/src/makefiles/pgxs.mk", sizeof(path));
- 	cleanup_path(path);
- 	configdata[i].setting = pstrdup(path);

pkgs/servers/sql/postgresql/patches/less-is-more.patch

@@ -1,6 +1,5 @@
-diff -Naur postgresql-9.6.1-orig/src/include/fe_utils/print.h postgresql-9.6.1/src/include/fe_utils/print.h
+--- a/src/include/fe_utils/print.h
---- postgresql-9.6.1-orig/src/include/fe_utils/print.h	2016-11-22 21:39:29.148932827 +0100
++++ b/src/include/fe_utils/print.h
-+++ postgresql-9.6.1/src/include/fe_utils/print.h	2016-11-22 21:39:36.283626258 +0100
 @@ -18,7 +18,7 @@
  
  /* This is not a particularly great place for this ... */

pkgs/servers/sql/postgresql/patches/locale-binary-path.patch

@@ -1,5 +1,3 @@
-diff --git a/src/backend/commands/collationcmds.c b/src/backend/commands/collationcmds.c
-index fcfc02d..d011394 100644
 --- a/src/backend/commands/collationcmds.c
 +++ b/src/backend/commands/collationcmds.c
 @@ -611,7 +611,7 @@ pg_import_system_collations(PG_FUNCTION_ARGS)

pkgs/servers/sql/postgresql/patches/paths-for-split-outputs.patch

@@ -0,0 +1,11 @@
+--- a/src/common/config_info.c
++++ b/src/common/config_info.c
+@@ -118,7 +118,7 @@
+ 	i++;
+
+ 	configdata[i].name = pstrdup("PGXS");
++	strlcpy(path, "@out@/lib", sizeof(path));
+-	get_pkglib_path(my_exec_path, path);
+ 	strlcat(path, "/pgxs/src/makefiles/pgxs.mk", sizeof(path));
+ 	cleanup_path(path);
+ 	configdata[i].setting = pstrdup(path);

pkgs/servers/sql/postgresql/patches/paths-with-postgresql-suffix.patch

@@ -1,7 +1,3 @@
-From: Matthew Bauer <mjbauer95@gmail.com>
-Date: Wed, 29 May 2019 22:51:52 -0400
-Subject: [PATCH] Add /postgresql suffix for Nix outputs
-
 Nix outputs put the `name' in each store path like
 /nix/store/...-<name>. This was confusing the Postgres make script
 because it thought its data directory already had postgresql in its
@@ -10,20 +6,6 @@ $out/share. To fix this, we just look for postgres or psql in the part
 after the / using make's notdir.
 
 ---
-From: Matthew Bauer <mjbauer95@gmail.com>
-Date: Wed, 29 May 2019 22:51:52 -0400
-Subject: [PATCH] Add /postgresql suffix for Nix outputs
-
-Nix outputs put the `name' in each store path like
-/nix/store/...-<name>. This was confusing the Postgres make script
-because it thought its data directory already had postgresql in its
-directory. This lead to Postgres installing all of its fils in
-$out/share. To fix this, we just look for postgres or psql in the part
-after the / using make's notdir.
-
----
-diff --git a/src/Makefile.global.in b/src/Makefile.global.in
-index b9d86acaa9..bce05464c3 100644
 --- a/src/Makefile.global.in
 +++ b/src/Makefile.global.in
 @@ -102,15 +102,15 @@ datarootdir := @datarootdir@

pkgs/servers/sql/postgresql/patches/relative-to-symlinks-16+.patch

@@ -1,10 +1,11 @@
+On NixOS we *want* stuff relative to symlinks.
+---
 --- a/src/common/exec.c
 +++ b/src/common/exec.c
-@@ -238,6 +238,9 @@
+@@ -238,6 +238,8 @@
  static int
  normalize_exec_path(char *path)
  {
-+	// On NixOS we *want* stuff relative to symlinks.
 +	return 0;
 +
  	/*

pkgs/servers/sql/postgresql/patches/relative-to-symlinks.patch

@@ -0,0 +1,13 @@
+On NixOS we *want* stuff relative to symlinks.
+---
+--- a/src/common/exec.c
++++ b/src/common/exec.c
+@@ -218,6 +218,8 @@
+ static int
+ resolve_symlinks(char *path)
+ {
++	return 0;
++
+ #ifdef HAVE_READLINK
+ 	struct stat buf;
+ 	char		orig_wd[MAXPGPATH],

pkgs/servers/sql/postgresql/patches/socketdir-in-run-13+.patch

@@ -1,7 +1,5 @@
-diff --git i/src/include/pg_config_manual.h w/src/include/pg_config_manual.h
+--- a/src/include/pg_config_manual.h
-index 8f3ec6bde1..4fc01e4a0a 100644
++++ b/src/include/pg_config_manual.h
---- i/src/include/pg_config_manual.h
-+++ w/src/include/pg_config_manual.h
 @@ -201,7 +201,7 @@
   * support them yet.
   */

pkgs/servers/sql/postgresql/patches/socketdir-in-run.patch

@@ -1,5 +1,3 @@
-diff --git a/src/include/pg_config_manual.h b/src/include/pg_config_manual.h
-index 743401cb96..be5c5f61d2 100644
 --- a/src/include/pg_config_manual.h
 +++ b/src/include/pg_config_manual.h
 @@ -179,7 +179,7 @@

pkgs/servers/sql/postgresql/patches/specify_pkglibdir_at_runtime.patch

@@ -1,6 +1,5 @@
-diff -ur postgresql-9.5.3-orig/src/port/path.c postgresql-9.5.3/src/port/path.c
+--- a/src/port/path.c
---- postgresql-9.5.3-orig/src/port/path.c	2016-05-09 22:50:23.000000000 +0200
++++ b/src/port/path.c
-+++ postgresql-9.5.3/src/port/path.c	2016-08-29 22:44:10.507377613 +0200
 @@ -714,7 +714,11 @@
  void
  get_lib_path(const char *my_exec_path, char *ret_path)