Merge pull request #170797 from Ma27/mailman

mailman: refactor package structure

nixos/modules/services/mail/mailman.nix

@@ -6,10 +6,10 @@ let
 
   cfg = config.services.mailman;
 
-  pythonEnv = pkgs.python3.withPackages (ps:
+  inherit (pkgs.mailmanPackages.buildEnvs { withHyperkitty = cfg.hyperkitty.enable; })
-    [ps.mailman ps.mailman-web]
+    mailmanEnv webEnv;
-    ++ lib.optional cfg.hyperkitty.enable ps.mailman-hyperkitty
+
-    ++ cfg.extraPythonPackages);
+  withPostgresql = config.services.postgresql.enable;
 
   # This deliberately doesn't use recursiveUpdate so users can
   # override the defaults.
@@ -72,6 +72,9 @@ in {
       stored in the world-readable Nix store.  To continue using
       Hyperkitty, you must set services.mailman.hyperkitty.enable = true.
     '')
+    (mkRemovedOptionModule [ "services" "mailman" "package" ] ''
+      Didn't have an effect for several years.
+    '')
   ];
 
   options = {
@@ -84,14 +87,6 @@ in {
         description = "Enable Mailman on this host. Requires an active MTA on the host (e.g. Postfix).";
       };
 
-      package = mkOption {
-        type = types.package;
-        default = pkgs.mailman;
-        defaultText = literalExpression "pkgs.mailman";
-        example = literalExpression "pkgs.mailman.override { archivers = []; }";
-        description = "Mailman package to use";
-      };
-
       enablePostfix = mkOption {
         type = types.bool;
         default = true;
@@ -185,7 +180,7 @@ in {
       mailman.layout = "fhs";
 
       "paths.fhs" = {
-        bin_dir = "${pkgs.python3Packages.mailman}/bin";
+        bin_dir = "${pkgs.mailmanPackages.mailman}/bin";
         var_dir = "/var/lib/mailman";
         queue_dir = "$var_dir/queue";
         template_dir = "$var_dir/templates";
@@ -295,9 +290,12 @@ in {
       name = "mailman-tools";
       # We don't want to pollute the system PATH with a python
       # interpreter etc. so let's pick only the stuff we actually
-      # want from pythonEnv
+      # want from {web,mailman}Env
       pathsToLink = ["/bin"];
-      paths = [pythonEnv];
+      paths = [ mailmanEnv webEnv ];
+      # Only mailman-related stuff is installed, the rest is removed
+      # in `postBuild`.
+      ignoreCollisions = true;
       postBuild = ''
         find $out/bin/ -mindepth 1 -not -name "mailman*" -delete
       '';
@@ -320,12 +318,14 @@ in {
         description = "GNU Mailman Master Process";
         before = lib.optional cfg.enablePostfix "postfix.service";
         after = [ "network.target" ]
-          ++ lib.optional cfg.enablePostfix "postfix-setup.service";
+          ++ lib.optional cfg.enablePostfix "postfix-setup.service"
+          ++ lib.optional withPostgresql "postgresql.service";
         restartTriggers = [ config.environment.etc."mailman.cfg".source ];
+        requires = optional withPostgresql "postgresql.service";
         wantedBy = [ "multi-user.target" ];
         serviceConfig = {
-          ExecStart = "${pythonEnv}/bin/mailman start";
+          ExecStart = "${mailmanEnv}/bin/mailman start";
-          ExecStop = "${pythonEnv}/bin/mailman stop";
+          ExecStop = "${mailmanEnv}/bin/mailman stop";
           User = "mailman";
           Group = "mailman";
           Type = "forking";
@@ -340,6 +340,8 @@ in {
         before = [ "mailman.service" "mailman-web-setup.service" "mailman-uwsgi.service" "hyperkitty.service" ];
         requiredBy = [ "mailman.service" "mailman-web-setup.service" "mailman-uwsgi.service" "hyperkitty.service" ];
         path = with pkgs; [ jq ];
+        after = optional withPostgresql "postgresql.service";
+        requires = optional withPostgresql "postgresql.service";
         serviceConfig.Type = "oneshot";
         script = ''
           mailmanDir=/var/lib/mailman
@@ -381,9 +383,9 @@ in {
         restartTriggers = [ config.environment.etc."mailman3/settings.py".source ];
         script = ''
           [[ -e "${webSettings.STATIC_ROOT}" ]] && find "${webSettings.STATIC_ROOT}/" -mindepth 1 -delete
-          ${pythonEnv}/bin/mailman-web migrate
+          ${webEnv}/bin/mailman-web migrate
-          ${pythonEnv}/bin/mailman-web collectstatic
+          ${webEnv}/bin/mailman-web collectstatic
-          ${pythonEnv}/bin/mailman-web compress
+          ${webEnv}/bin/mailman-web compress
         '';
         serviceConfig = {
           User = cfg.webUser;
@@ -397,14 +399,16 @@ in {
         uwsgiConfig.uwsgi = {
           type = "normal";
           plugins = ["python3"];
-          home = pythonEnv;
+          home = webEnv;
           module = "mailman_web.wsgi";
           http = "127.0.0.1:18507";
         };
         uwsgiConfigFile = pkgs.writeText "uwsgi-mailman.json" (builtins.toJSON uwsgiConfig);
       in {
         wantedBy = ["multi-user.target"];
-        requires = ["mailman-uwsgi.socket" "mailman-web-setup.service"];
+        after = optional withPostgresql "postgresql.service";
+        requires = ["mailman-uwsgi.socket" "mailman-web-setup.service"]
+          ++ optional withPostgresql "postgresql.service";
         restartTriggers = [ config.environment.etc."mailman3/settings.py".source ];
         serviceConfig = {
           # Since the mailman-web settings.py obstinately creates a logs
@@ -422,7 +426,7 @@ in {
         startAt = "daily";
         restartTriggers = [ config.environment.etc."mailman.cfg".source ];
         serviceConfig = {
-          ExecStart = "${pythonEnv}/bin/mailman digests --send";
+          ExecStart = "${mailmanEnv}/bin/mailman digests --send";
           User = "mailman";
           Group = "mailman";
         };
@@ -434,7 +438,7 @@ in {
         restartTriggers = [ config.environment.etc."mailman3/settings.py".source ];
         wantedBy = [ "mailman.service" "multi-user.target" ];
         serviceConfig = {
-          ExecStart = "${pythonEnv}/bin/mailman-web qcluster";
+          ExecStart = "${webEnv}/bin/mailman-web qcluster";
           User = cfg.webUser;
           Group = "mailman";
           WorkingDirectory = "/var/lib/mailman-web";
@@ -453,7 +457,7 @@ in {
         inherit startAt;
         restartTriggers = [ config.environment.etc."mailman3/settings.py".source ];
         serviceConfig = {
-          ExecStart = "${pythonEnv}/bin/mailman-web runjobs ${name}";
+          ExecStart = "${webEnv}/bin/mailman-web runjobs ${name}";
           User = cfg.webUser;
           Group = "mailman";
           WorkingDirectory = "/var/lib/mailman-web";
@@ -462,7 +466,7 @@ in {
   };
 
   meta = {
-    maintainers = with lib.maintainers; [ lheckemann qyliss ];
+    maintainers = with lib.maintainers; [ lheckemann qyliss ma27 ];
     doc = ./mailman.xml;
   };
 

pkgs/servers/mail/mailman/default.nix

@@ -1,95 +1,33 @@
-{ lib, buildPythonPackage, fetchPypi, fetchpatch, pythonOlder, python3, postfix, lynx
+{ newScope, lib, python3 }:
-}:
 
 let
-  # Mailman does not support sqlalchemy >= 1.4 https://gitlab.com/mailman/mailman/-/issues/845
+  callPackage = newScope self;
-  pythonOverride = python3.override {
-    packageOverrides = self: super: {
-      alembic = super.alembic.overridePythonAttrs (oldAttrs:  {
-        # does not find tests
-        doCheck = false;
-      });
-      sqlalchemy = super.sqlalchemy.overridePythonAttrs (oldAttrs: rec {
-        version = "1.3.24";
-        src = oldAttrs.src.override {
-          inherit version;
-          hash = "sha256-67t3fL+TEjWbiXv4G6ANrg9ctp+6KhgmXcwYpvXvdRk=";
-        };
-        # does not find tests
-        doCheck = false;
-      });
-    };
-  };
-in
 
-buildPythonPackage rec {
+  self = lib.makeExtensible (self: {
-  pname = "mailman";
+    python3 = callPackage ./python.nix { inherit python3; };
-  version = "3.3.5";
-  disabled = pythonOlder "3.6";
 
-  src = fetchPypi {
+    hyperkitty = callPackage ./hyperkitty.nix { };
-    inherit pname version;
-    sha256 = "12mgxs1ndhdjjkydx48b95na9k9h0disfqgrr6wxx7vda6dqvcwz";
-  };
 
-  propagatedBuildInputs = with pythonOverride.pkgs; [
+    mailman = callPackage ./package.nix { };
-    aiosmtpd
-    alembic
-    authheaders
-    click
-    dnspython
-    falcon
-    flufl_bounce
-    flufl_i18n
-    flufl_lock
-    gunicorn
-    importlib-resources
-    lazr_config
-    passlib
-    requests
-    sqlalchemy
-    zope_component
-    zope_configuration
-  ];
 
-  patches = [
+    mailman-hyperkitty = callPackage ./mailman-hyperkitty.nix { };
-    (fetchpatch {
-      url = "https://gitlab.com/mailman/mailman/-/commit/4b206e2a5267a0e17f345fd7b2d957122ba57566.patch";
-      sha256 = "06axmrn74p81wvcki36c7gfj5fp5q15zxz2yl3lrvijic7hbs4n2";
-    })
-    (fetchpatch {
-      url = "https://gitlab.com/mailman/mailman/-/commit/9613154f3c04fa2383fbf017031ef263c291418d.patch";
-      sha256 = "0vyw87s857vfxbf7kihwb6w094xyxmxbi1bpdqi3ybjamjycp55r";
-    })
-    ./log-stderr.patch
-  ];
 
-  postPatch = ''
+    postorius = callPackage ./postorius.nix { };
-    substituteInPlace setup.py \
-      --replace "alembic>=1.6.2,<1.7" "alembic>=1.6.2"
 
-    substituteInPlace src/mailman/config/postfix.cfg \
+    web = callPackage ./web.nix { };
-      --replace /usr/sbin/postmap ${postfix}/bin/postmap
-    substituteInPlace src/mailman/config/schema.cfg \
-      --replace /usr/bin/lynx ${lynx}/bin/lynx
-  '';
 
-  # Mailman assumes that those scripts in $out/bin are Python scripts. Wrapping
+    buildEnvs = { web ? self.web
-  # them in shell code breaks this assumption. Use the wrapped version (see
+                , mailman ? self.mailman
-  # wrapped.nix) if you need the CLI (rather than the Python library).
+                , mailman-hyperkitty ? self.mailman-hyperkitty
-  #
+                , withHyperkitty ? false
-  # This gives a properly wrapped 'mailman' command plus an interpreter that
+                }:
-  # has all the necessary search paths to execute unwrapped 'master' and
+      {
-  # 'runner' scripts.
+        mailmanEnv = self.python3.withPackages
-  dontWrapPythonPrograms = true;
+          (ps: [ mailman ps.psycopg2 ]
+            ++ lib.optional withHyperkitty mailman-hyperkitty);
+        webEnv = self.python3.withPackages
+          (ps: [ web ps.psycopg2 ]);
+      };
+  });
 
-  # requires flufl.testing, which the upstream has archived
+in self
-  doCheck = false;
-
-  meta = {
-    homepage = "https://www.gnu.org/software/mailman/";
-    description = "Free software for managing electronic mail discussion and newsletter lists";
-    license = lib.licenses.gpl3Plus;
-    maintainers = with lib.maintainers; [ qyliss ];
-  };
-}

pkgs/servers/mail/mailman/hyperkitty.nix

@@ -1,34 +1,10 @@
 { lib
-, buildPythonPackage
+, python3
 , fetchpatch
-, fetchPypi
-, pythonOlder
-
-# dependencies
-, defusedxml
-, django
-, django-gravatar2
-, django-haystack
-, django-mailman3
-, django-paintstore
-, django-q
-, django_compressor
-, django-extensions
-, djangorestframework
-, flufl_lock
-, mistune_2_0
-, networkx
-, psycopg2
-, python-dateutil
-, robot-detection
-
-# tests
-, beautifulsoup4
-, elasticsearch
-, mock
-, whoosh
 }:
 
+with python3.pkgs;
+
 buildPythonPackage rec {
   pname = "HyperKitty";
   # Note: Mailman core must be on the latest version before upgrading HyperKitty.

pkgs/servers/mail/mailman/mailman-hyperkitty.nix

@@ -1,15 +1,9 @@
 { lib
-, buildPythonPackage
+, python3
-, fetchPypi
 , mailman
-, mock
-, nose2
-, python
-, pythonOlder
-, requests
-, zope_interface
 }:
 
+with python3.pkgs;
 buildPythonPackage rec {
   pname = "mailman-hyperkitty";
   version = "1.2.0";

pkgs/servers/mail/mailman/package.nix

@@ -0,0 +1,76 @@
+{ lib, fetchpatch, python3, postfix, lynx
+}:
+
+with python3.pkgs;
+
+buildPythonPackage rec {
+  pname = "mailman";
+  version = "3.3.5";
+  disabled = pythonOlder "3.6";
+
+  src = fetchPypi {
+    inherit pname version;
+    sha256 = "12mgxs1ndhdjjkydx48b95na9k9h0disfqgrr6wxx7vda6dqvcwz";
+  };
+
+  propagatedBuildInputs = with python3.pkgs; [
+    aiosmtpd
+    alembic
+    authheaders
+    click
+    dnspython
+    falcon
+    flufl_bounce
+    flufl_i18n
+    flufl_lock
+    gunicorn
+    importlib-resources
+    lazr_config
+    passlib
+    requests
+    sqlalchemy
+    zope_component
+    zope_configuration
+  ];
+
+  patches = [
+    (fetchpatch {
+      url = "https://gitlab.com/mailman/mailman/-/commit/4b206e2a5267a0e17f345fd7b2d957122ba57566.patch";
+      sha256 = "06axmrn74p81wvcki36c7gfj5fp5q15zxz2yl3lrvijic7hbs4n2";
+    })
+    (fetchpatch {
+      url = "https://gitlab.com/mailman/mailman/-/commit/9613154f3c04fa2383fbf017031ef263c291418d.patch";
+      sha256 = "0vyw87s857vfxbf7kihwb6w094xyxmxbi1bpdqi3ybjamjycp55r";
+    })
+    ./log-stderr.patch
+  ];
+
+  postPatch = ''
+    substituteInPlace setup.py \
+      --replace "alembic>=1.6.2,<1.7" "alembic>=1.6.2"
+
+    substituteInPlace src/mailman/config/postfix.cfg \
+      --replace /usr/sbin/postmap ${postfix}/bin/postmap
+    substituteInPlace src/mailman/config/schema.cfg \
+      --replace /usr/bin/lynx ${lynx}/bin/lynx
+  '';
+
+  # Mailman assumes that those scripts in $out/bin are Python scripts. Wrapping
+  # them in shell code breaks this assumption. Use the wrapped version (see
+  # wrapped.nix) if you need the CLI (rather than the Python library).
+  #
+  # This gives a properly wrapped 'mailman' command plus an interpreter that
+  # has all the necessary search paths to execute unwrapped 'master' and
+  # 'runner' scripts.
+  dontWrapPythonPrograms = true;
+
+  # requires flufl.testing, which the upstream has archived
+  doCheck = false;
+
+  meta = {
+    homepage = "https://www.gnu.org/software/mailman/";
+    description = "Free software for managing electronic mail discussion and newsletter lists";
+    license = lib.licenses.gpl3Plus;
+    maintainers = with lib.maintainers; [ qyliss ma27 ];
+  };
+}

pkgs/servers/mail/mailman/postorius.nix

@@ -1,6 +1,6 @@
-{ lib, buildPythonPackage, fetchPypi, beautifulsoup4, vcrpy, mock
+{ lib, python3 }:
-, django-mailman3, mailmanclient, readme_renderer
+
-}:
+with python3.pkgs;
 
 buildPythonPackage rec {
   pname = "postorius";

pkgs/servers/mail/mailman/python.nix

@@ -0,0 +1,30 @@
+{ python3 }:
+
+python3.override {
+  packageOverrides = self: super: {
+    # does not find tests
+    alembic = super.alembic.overridePythonAttrs (oldAttrs:  {
+      doCheck = false;
+    });
+    # Needed by mailman, see https://gitlab.com/mailman/mailman/-/issues/964
+    sqlalchemy = super.sqlalchemy.overridePythonAttrs (oldAttrs: rec {
+      version = "1.3.24";
+      src = super.fetchPypi {
+        inherit version;
+        inherit (oldAttrs) pname;
+        sha256 = "06bmxzssc66cblk1hamskyv5q3xf1nh1py3vi6dka4lkpxy7gfzb";
+      };
+      # does not find tests
+      doCheck = false;
+    });
+    # Fixes `AssertionError: database connection isn't set to UTC`
+    psycopg2 = super.psycopg2.overridePythonAttrs (a: (rec {
+      version = "2.8.6";
+      src = super.fetchPypi {
+        inherit version;
+        inherit (a) pname;
+        sha256 = "fb23f6c71107c37fd667cb4ea363ddeb936b348bbd6449278eb92c189699f543";
+      };
+    }));
+  };
+}

pkgs/servers/mail/mailman/web.nix

@@ -1,7 +1,9 @@
-{ buildPythonPackage, lib, fetchPypi, pythonOlder
+{ lib, python3
-, sassc, hyperkitty, postorius, whoosh, setuptools-scm
+, sassc, hyperkitty, postorius
 }:
 
+with python3.pkgs;
+
 buildPythonPackage rec {
   pname = "mailman-web";
   version = "0.0.5";

pkgs/servers/mail/mailman/wrapped.nix

@@ -1,20 +0,0 @@
-{ runCommand, lib, makeWrapper, python3
-, archivers ? [ python3.pkgs.mailman-hyperkitty ]
-}:
-
-let
-  inherit (python3.pkgs) makePythonPath mailman;
-in
-
-runCommand "${mailman.name}-wrapped" {
-  inherit (mailman) meta;
-  nativeBuildInputs = [ makeWrapper ];
-  passthru = mailman.passthru // { unwrapped = mailman; };
-} ''
-  mkdir -p "$out/bin"
-  cd "${mailman}/bin"
-  for exe in *; do
-    makeWrapper "${mailman}/bin/$exe" "$out/bin/$exe" \
-        --set PYTHONPATH ${makePythonPath ([ mailman ] ++ archivers)}
-  done
-''

pkgs/top-level/all-packages.nix

@@ -21897,12 +21897,12 @@ with pkgs;
 
   mackerel-agent = callPackage ../servers/monitoring/mackerel-agent { };
 
-  mailman = callPackage ../servers/mail/mailman/wrapped.nix { };
+  mailmanPackages = callPackage ../servers/mail/mailman { };
+  inherit (mailmanPackages) mailman mailman-hyperkitty;
+  mailman-web = mailmanPackages.web;
 
   mailman-rss = callPackage ../tools/misc/mailman-rss { };
 
-  mailman-web = with python3.pkgs; toPythonApplication mailman-web;
-
   listadmin = callPackage ../applications/networking/listadmin {};
 
   maker-panel = callPackage ../tools/misc/maker-panel { };

pkgs/top-level/python-aliases.nix

@@ -77,6 +77,7 @@ mapAliases ({
   HAP-python = hap-python; # added 2021-06-01
   hbmqtt = throw "hbmqtt was removed because it is no longer maintained"; # added 2021-11-07
   hdlparse = throw "hdlparse has been removed, it was using setuptools 2to3 translation feature, which has been removed in setuptools 58"; # added 2022-01-18
+  hyperkitty = throw "Please use pkgs.mailmanPackages.hyperkitty"; # added 2022-04-29
   IMAPClient = imapclient; # added 2021-10-28
   jupyter_client = jupyter-client; # added 2021-10-15
   Keras = keras; # added 2021-11-25
@@ -85,12 +86,16 @@ mapAliases ({
   Markups = markups; # added 2022-02-14
   MechanicalSoup = mechanicalsoup; # added 2021-06-01
   memcached = python-memcached; # added 2022-05-06
+  mailman = throw "Please use pkgs.mailman"; # added 2022-04-29
+  mailman-hyperkitty = throw "Please use pkgs.mailmanPackages.mailman-hyperkitty"; # added 2022-04-29
+  mailman-web = throw "Please use pkgs.mailman-web"; # added 2022-04-29
   net2grid = gridnet; # add 2022-04-22
   nose-cover3 = throw "nose-cover3 has been removed, it was using setuptools 2to3 translation feature, which has been removed in setuptools 58"; # added 2022-02-16
   pam = python-pam; # added 2020-09-07.
   PasteDeploy = pastedeploy; # added 2021-10-07
   pathpy = path; # added 2022-04-12
   pep257 = pydocstyle; # added 2022-04-12
+  postorius = throw "Please use pkgs.mailmanPackages.postorius"; # added 2022-04-29
   powerlineMemSegment = powerline-mem-segment; # added 2021-10-08
   privacyidea = throw "privacyidea has been renamed to pkgs.privacyidea"; # added 2021-06-20
   prometheus_client = prometheus-client; # added 2021-06-10

pkgs/top-level/python-packages.nix

@@ -4023,8 +4023,6 @@ in {
 
   hyperion-py = callPackage ../development/python-modules/hyperion-py { };
 
-  hyperkitty = callPackage ../servers/mail/mailman/hyperkitty.nix { };
-
   hyperlink = callPackage ../development/python-modules/hyperlink { };
 
   hyperopt = callPackage ../development/python-modules/hyperopt { };
@@ -5076,14 +5074,8 @@ in {
 
   mailchimp = callPackage ../development/python-modules/mailchimp { };
 
-  mailman = callPackage ../servers/mail/mailman { };
-
   mailmanclient = callPackage ../development/python-modules/mailmanclient { };
 
-  mailman-hyperkitty = callPackage ../development/python-modules/mailman-hyperkitty { };
-
-  mailman-web = callPackage ../servers/mail/mailman/web.nix { };
-
   rtmixer = callPackage ../development/python-modules/rtmixer { };
 
   mail-parser = callPackage ../development/python-modules/mail-parser { };
@@ -6665,8 +6657,6 @@ in {
 
   poster3 = callPackage ../development/python-modules/poster3 { };
 
-  postorius = callPackage ../servers/mail/mailman/postorius.nix { };
-
   pot = callPackage ../development/python-modules/pot { };
 
   potentials = callPackage ../development/python-modules/potentials { };