Prefect init module (#386895)

nixos/modules/module-list.nix

@@ -1364,6 +1364,7 @@
   ./services/scheduling/atd.nix
   ./services/scheduling/cron.nix
   ./services/scheduling/fcron.nix
+  ./services/scheduling/prefect.nix
   ./services/scheduling/scx.nix
   ./services/search/elasticsearch-curator.nix
   ./services/search/elasticsearch.nix

nixos/modules/services/scheduling/prefect.nix

@@ -0,0 +1,232 @@
+{
+  lib,
+  pkgs,
+  config,
+  ...
+}:
+
+let
+  cfg = config.services.prefect;
+  inherit (lib.types)
+    bool
+    str
+    enum
+    path
+    attrsOf
+    nullOr
+    submodule
+    port
+    ;
+
+in
+{
+  options.services.prefect = {
+    enable = lib.mkOption {
+      type = bool;
+      default = false;
+      description = "enable prefect server and worker services";
+    };
+
+    package = lib.mkPackageOption pkgs "prefect" { };
+
+    host = lib.mkOption {
+      type = str;
+      default = "127.0.0.1";
+      example = "0.0.0.0";
+      description = "Prefect server host";
+    };
+
+    port = lib.mkOption {
+      type = port;
+      default = 4200;
+      description = "Prefect server port";
+    };
+
+    dataDir = lib.mkOption {
+      type = path;
+      default = "/var/lib/prefect-server";
+      description = ''
+        Specify the directory for Prefect.
+      '';
+    };
+
+    database = lib.mkOption {
+      type = enum [
+        "sqlite"
+        "postgres"
+      ];
+      default = "sqlite";
+      description = "which database to use for prefect server: sqlite or postgres";
+    };
+
+    databaseHost = lib.mkOption {
+      type = str;
+      default = "localhost";
+      description = "database host for postgres only";
+    };
+
+    databasePort = lib.mkOption {
+      type = str;
+      default = "5432";
+      description = "database port for postgres only";
+    };
+
+    databaseName = lib.mkOption {
+      type = str;
+      default = "prefect";
+      description = "database name for postgres only";
+    };
+
+    databaseUser = lib.mkOption {
+      type = str;
+      default = "postgres";
+      description = "database user for postgres only";
+    };
+
+    databasePasswordFile = lib.mkOption {
+      type = nullOr str;
+      default = null;
+      description = ''
+        path to a file containing e.g.:
+          DBPASSWORD=supersecret
+
+        stored outside the nix store, read by systemd as EnvironmentFile.
+      '';
+    };
+
+    # now define workerPools as an attribute set of submodules,
+    # each key is the pool name, and the submodule has an installPolicy
+    workerPools = lib.mkOption {
+      type = attrsOf (submodule {
+        options = {
+          installPolicy = lib.mkOption {
+            type = enum [
+              "always"
+              "if-not-present"
+              "never"
+              "prompt"
+            ];
+            default = "always";
+            description = "install policy for the worker (always, if-not-present, never, prompt)";
+          };
+        };
+      });
+      default = { };
+      description = ''
+        define a set of worker pools with submodule config. example:
+        workerPools.my-pool = {
+          installPolicy = "never";
+        };
+      '';
+    };
+
+    baseUrl = lib.mkOption {
+      type = nullOr str;
+      default = null;
+      description = "external url when served by a reverse proxy, e.g. https://example.com/prefect";
+    };
+  };
+
+  config = lib.mkIf cfg.enable {
+    # define systemd.services as the server plus any worker definitions
+    systemd.services =
+      {
+        "prefect-server" = {
+          description = "prefect server";
+          wantedBy = [ "multi-user.target" ];
+          after = [ "network.target" ];
+
+          serviceConfig = {
+            DynamicUser = true;
+            StateDirectory = "prefect-server";
+            # TODO all my efforts to setup the database url
+            # have failed with some unable to open file
+            Environment = [
+              "PREFECT_HOME=%S/prefect-server"
+              "PREFECT_UI_STATIC_DIRECTORY=%S/prefect-server"
+              "PREFECT_SERVER_ANALYTICS_ENABLED=off"
+              "PREFECT_UI_API_URL=${cfg.baseUrl}/api"
+              "PREFECT_UI_URL=${cfg.baseUrl}"
+            ];
+            EnvironmentFile =
+              if cfg.database == "postgres" && cfg.databasePasswordFile != null then
+                [ cfg.databasePasswordFile ]
+              else
+                [ ];
+
+            # ReadWritePaths = [ cfg.dataDir ];
+            ProtectSystem = "strict";
+            ProtectHome = true;
+            PrivateTmp = true;
+            NoNewPrivileges = true;
+            MemoryDenyWriteExecute = true;
+            LockPersonality = true;
+            CapabilityBoundingSet = [ ];
+            AmbientCapabilities = [ ];
+            RestrictSUIDSGID = true;
+            RestrictAddressFamilies = [
+              "AF_INET"
+              "AF_INET6"
+              "AF_UNIX"
+            ];
+            ProtectKernelTunables = true;
+            ProtectKernelModules = true;
+            ProtectKernelLogs = true;
+            ProtectControlGroups = true;
+            MemoryAccounting = true;
+            CPUAccounting = true;
+
+            ExecStart = "${pkgs.prefect}/bin/prefect server start --host ${cfg.host} --port ${toString cfg.port}";
+            Restart = "always";
+            WorkingDirectory = cfg.dataDir;
+          };
+        };
+      }
+      // lib.concatMapAttrs (poolName: poolCfg: {
+        # return a partial attr set with one key: "prefect-worker-..."
+        "prefect-worker-${poolName}" = {
+          description = "prefect worker for pool '${poolName}'";
+          wantedBy = [ "multi-user.target" ];
+          after = [ "network.target" ];
+
+          environment.systemPackages = cfg.package;
+
+          serviceConfig = {
+            DynamicUser = true;
+            StateDirectory = "prefect-worker-${poolName}";
+            Environment = [
+              "PREFECT_HOME=%S/prefect-worker-${poolName}"
+              "PREFECT_API_URL=${cfg.baseUrl}/api"
+            ];
+            ProtectSystem = "strict";
+            ProtectHome = true;
+            PrivateTmp = true;
+            NoNewPrivileges = true;
+            MemoryDenyWriteExecute = true;
+            LockPersonality = true;
+            CapabilityBoundingSet = [ ];
+            AmbientCapabilities = [ ];
+            RestrictSUIDSGID = true;
+            RestrictAddressFamilies = [
+              "AF_INET"
+              "AF_INET6"
+              "AF_UNIX"
+            ];
+            ProtectKernelTunables = true;
+            ProtectKernelModules = true;
+            ProtectKernelLogs = true;
+            ProtectControlGroups = true;
+            MemoryAccounting = true;
+            CPUAccounting = true;
+            ExecStart = ''
+              ${pkgs.prefect}/bin/prefect worker start \
+                --pool ${poolName} \
+                --type process \
+                --install-policy ${poolCfg.installPolicy}
+            '';
+            Restart = "always";
+          };
+        };
+      }) cfg.workerPools;
+  };
+}

nixos/tests/all-tests.nix

@@ -969,6 +969,7 @@ in {
   pppd = handleTest ./pppd.nix {};
   predictable-interface-names = handleTest ./predictable-interface-names.nix {};
   pretalx = runTest ./web-apps/pretalx.nix;
+  prefect = runTest ./prefect.nix;
   pretix = runTest ./web-apps/pretix.nix;
   printing-socket = handleTest ./printing.nix { socket = true; listenTcp = true; };
   printing-service = handleTest ./printing.nix { socket = false; listenTcp = true; };

nixos/tests/prefect.nix

@@ -0,0 +1,27 @@
+{ lib, ... }:
+let
+  mainPort = "4200";
+in
+{
+  name = "prefect";
+
+  nodes = {
+    machine =
+      { ... }:
+      {
+        services.prefect = {
+          enable = true;
+        };
+      };
+  };
+
+  testScript = ''
+    machine.start()
+    machine.wait_for_unit("prefect-server.service")
+    machine.wait_for_open_port("${mainPort}")
+  '';
+
+  meta = with lib.maintainers; {
+    maintainers = [ happysalada ];
+  };
+}

pkgs/by-name/pr/prefect/make_ui_files_writeable_on_startup.patch

@@ -0,0 +1,42 @@
+From a97d5f501ff3125d96e6c64dfa498ca1a598a4bd Mon Sep 17 00:00:00 2001
+From: happysalada <raphael@megzari.com>
+Date: Sun, 2 Mar 2025 08:30:36 -0500
+Subject: [PATCH] feat: ensure ui files are writeable On startup prefect copies
+ over files from the ui into the ui directory. If for any reason the ui files
+ were not writeable, the whole setup will fail. This PR ensures that the
+ copied files are writeable. To give a bit more context, I am currently
+ packaging Prefect for nixos. Nix having a little bit of a strict build
+ system, makes sure that the built package has only read-only files. this is
+ to ensure the build is deterministic. I understand that this might appear as
+ a detail related to nix build system only. I can patch the source when
+ building the nix package, but I thought I would try to contribute the patch.
+ No hard feelings if you are not interested in this patch. Thank you for
+ developping prefect!
+
+fix formatting
+---
+ src/prefect/server/api/server.py | 6 ++++++
+ 1 file changed, 6 insertions(+)
+
+diff --git a/src/prefect/server/api/server.py b/src/prefect/server/api/server.py
+index e5b64d527..ac64616ef 100644
+--- a/src/prefect/server/api/server.py
++++ b/src/prefect/server/api/server.py
+@@ -250,8 +250,14 @@ def copy_directory(directory: str, path: str) -> None:
+             if os.path.exists(destination):
+                 shutil.rmtree(destination)
+             shutil.copytree(source, destination, symlinks=True)
++            # ensure copied files are writeable
++            for root, dirs, files in os.walk(destination):
++                for f in files:
++                    os.chmod(os.path.join(root, f), 0o600)
+         else:
+             shutil.copy2(source, destination)
++            # Ensure copied file is writeable
++            os.chmod(destination, 0o600)
+ 
+ 
+ async def custom_internal_exception_handler(
+-- 
+2.48.1
+

pkgs/by-name/pr/prefect/package.nix

@@ -2,6 +2,7 @@
   lib,
   python3Packages,
   fetchPypi,
+  nixosTests,
 }:
 
 python3Packages.buildPythonApplication rec {
@@ -18,6 +19,10 @@ python3Packages.buildPythonApplication rec {
     hash = "sha256-4kwGrKvDihBi6Gcvcf6ophNI6GGd+M4qR0nnu/AUK1Q=";
   };
 
+  patches = [
+    ./make_ui_files_writeable_on_startup.patch
+  ];
+
   pythonRelaxDeps = [
     "websockets"
   ];
@@ -147,6 +152,16 @@ python3Packages.buildPythonApplication rec {
     ];
   };
 
+  makeWrapperArgs = [
+    # Add the installed directories to the python path so the worker can find them
+    "--prefix PYTHONPATH : ${python3Packages.makePythonPath dependencies}"
+    "--prefix PYTHONPATH : $out/${python3Packages.python.sitePackages}"
+  ];
+
+  passthru.tests = {
+    inherit (nixosTests) prefect;
+  };
+
   # Tests are not included in the pypi source
   doCheck = false;
   # nativeCheckInputs = (