nixpkgs/nixos/modules/services/cluster/k3s/default.nix

{
  config,
  lib,
  pkgs,
  ...
}:
let
  cfg = config.services.k3s;
  removeOption =
    config: instruction:
    lib.mkRemovedOptionModule (
      [
        "services"
        "k3s"
      ]
      ++ config
    ) instruction;

  manifestDir = "/var/lib/rancher/k3s/server/manifests";
  chartDir = "/var/lib/rancher/k3s/server/static/charts";
  imageDir = "/var/lib/rancher/k3s/agent/images";
  containerdConfigTemplateFile = "/var/lib/rancher/k3s/agent/etc/containerd/config.toml.tmpl";
  yamlFormat = pkgs.formats.yaml { };
  yamlDocSeparator = builtins.toFile "yaml-doc-separator" "\n---\n";
  # Manifests need a valid YAML suffix to be respected by k3s
  mkManifestTarget =
    name: if (lib.hasSuffix ".yaml" name || lib.hasSuffix ".yml" name) then name else name + ".yaml";
  # Produces a list containing all duplicate manifest names
  duplicateManifests =
    with builtins;
    lib.intersectLists (attrNames cfg.autoDeployCharts) (attrNames cfg.manifests);
  # Produces a list containing all duplicate chart names
  duplicateCharts =
    with builtins;
    lib.intersectLists (attrNames cfg.autoDeployCharts) (attrNames cfg.charts);

  # Converts YAML -> JSON -> Nix
  fromYaml =
    path:
    with builtins;
    fromJSON (
      readFile (
        pkgs.runCommand "${path}-converted.json" { nativeBuildInputs = [ yq-go ]; } ''
          yq --no-colors --output-format json ${path} > $out
        ''
      )
    );

  # Replace prefixes and characters that are problematic in file names
  cleanHelmChartName =
    name:
    let
      woPrefix = lib.removePrefix "https://" (lib.removePrefix "oci://" name);
    in
    lib.replaceStrings
      [
        "/"
        ":"
      ]
      [
        "-"
        "-"
      ]
      woPrefix;

  # Fetch a Helm chart from a public registry. This only supports a basic Helm pull.
  fetchHelm =
    {
      name,
      repo,
      version,
      hash ? lib.fakeHash,
    }:
    let
      isOci = lib.hasPrefix "oci://" repo;
      pullCmd = if isOci then repo else "--repo ${repo} ${name}";
      name' = if isOci then "${repo}-${version}" else "${repo}-${name}-${version}";
    in
    pkgs.runCommand (cleanHelmChartName "${name'}.tgz")
      {
        inherit (lib.fetchers.normalizeHash { } { inherit hash; }) outputHash outputHashAlgo;
        impureEnvVars = lib.fetchers.proxyImpureEnvVars;
        nativeBuildInputs = with pkgs; [
          kubernetes-helm
          cacert
        ];
      }
      ''
        helm pull ${pullCmd} --version ${version}
        mv ./*.tgz $out
      '';

  # Returns the path to a YAML manifest file
  mkExtraDeployManifest =
    x:
    # x is a derivation that provides a YAML file
    if lib.isDerivation x then
      x.outPath
    # x is an attribute set that needs to be converted to a YAML file
    else if builtins.isAttrs x then
      (yamlFormat.generate "extra-deploy-chart-manifest" x)
    # assume x is a path to a YAML file
    else
      x;

  # Generate a HelmChart custom resource.
  mkHelmChartCR =
    name: value:
    let
      chartValues = if (lib.isPath value.values) then fromYaml value.values else value.values;
      # use JSON for values as it's a subset of YAML and understood by the k3s Helm controller
      valuesContent = builtins.toJSON chartValues;
    in
    # merge with extraFieldDefinitions to allow setting advanced values and overwrite generated
    # values
    lib.recursiveUpdate {
      apiVersion = "helm.cattle.io/v1";
      kind = "HelmChart";
      metadata = {
        inherit name;
        namespace = "kube-system";
      };
      spec = {
        inherit valuesContent;
        inherit (value) targetNamespace createNamespace;
        chart = "https://%{KUBERNETES_API}%/static/charts/${name}.tgz";
      };
    } value.extraFieldDefinitions;

  # Generate a HelmChart custom resource together with extraDeploy manifests. This
  # generates possibly a multi document YAML file that the auto deploy mechanism of k3s
  # deploys.
  mkAutoDeployChartManifest = name: value: {
    # target is the final name of the link created for the manifest file
    target = mkManifestTarget name;
    inherit (value) enable package;
    # source is a store path containing the complete manifest file
    source = pkgs.concatText "auto-deploy-chart-${name}.yaml" (
      [
        (yamlFormat.generate "helm-chart-manifest-${name}.yaml" (mkHelmChartCR name value))
      ]
      # alternate the YAML doc separator (---) and extraDeploy manifests to create
      # multi document YAMLs
      ++ (lib.concatMap (x: [
        yamlDocSeparator
        (mkExtraDeployManifest x)
      ]) value.extraDeploy)
    );
  };

  autoDeployChartsModule = lib.types.submodule (
    { config, ... }:
    {
      options = {
        enable = lib.mkOption {
          type = lib.types.bool;
          default = true;
          example = false;
          description = ''
            Whether to enable the installation of this Helm chart. Note that setting
            this option to `false` will not uninstall the chart from the cluster, if
            it was previously installed. Please use the the `--disable` flag or `.skip`
            files to delete/disable Helm charts, as mentioned in the
            [docs](https://docs.k3s.io/installation/packaged-components#disabling-manifests).
          '';
        };

        repo = lib.mkOption {
          type = lib.types.nonEmptyStr;
          example = "https://kubernetes.github.io/ingress-nginx";
          description = ''
            The repo of the Helm chart. Only has an effect if `package` is not set.
            The Helm chart is fetched during build time and placed as a `.tgz` archive on the
            filesystem.
          '';
        };

        name = lib.mkOption {
          type = lib.types.nonEmptyStr;
          example = "ingress-nginx";
          description = ''
            The name of the Helm chart. Only has an effect if `package` is not set.
            The Helm chart is fetched during build time and placed as a `.tgz` archive on the
            filesystem.
          '';
        };

        version = lib.mkOption {
          type = lib.types.nonEmptyStr;
          example = "4.7.0";
          description = ''
            The version of the Helm chart. Only has an effect if `package` is not set.
            The Helm chart is fetched during build time and placed as a `.tgz` archive on the
            filesystem.
          '';
        };

        hash = lib.mkOption {
          type = lib.types.str;
          example = "sha256-ej+vpPNdiOoXsaj1jyRpWLisJgWo8EqX+Z5VbpSjsPA=";
          default = "";
          description = ''
            The hash of the packaged Helm chart. Only has an effect if `package` is not set.
            The Helm chart is fetched during build time and placed as a `.tgz` archive on the
            filesystem.
          '';
        };

        package = lib.mkOption {
          type = with lib.types; either path package;
          example = lib.literalExpression "../my-helm-chart.tgz";
          description = ''
            The packaged Helm chart. Overwrites the options `repo`, `name`, `version`
            and `hash` in case of conflicts.
          '';
        };

        targetNamespace = lib.mkOption {
          type = lib.types.nonEmptyStr;
          default = "default";
          example = "kube-system";
          description = "The namespace in which the Helm chart gets installed.";
        };

        createNamespace = lib.mkOption {
          type = lib.types.bool;
          default = false;
          example = true;
          description = "Whether to create the target namespace if not present.";
        };

        values = lib.mkOption {
          type = with lib.types; either path attrs;
          default = { };
          example = {
            replicaCount = 3;
            hostName = "my-host";
            server = {
              name = "nginx";
              port = 80;
            };
          };
          description = ''
            Override default chart values via Nix expressions. This is equivalent to setting
            values in a `values.yaml` file.

            WARNING: The values (including secrets!) specified here are exposed unencrypted
            in the world-readable nix store.
          '';
        };

        extraDeploy = lib.mkOption {
          type = with lib.types; listOf (either path attrs);
          default = [ ];
          example = lib.literalExpression ''
            [
              ../manifests/my-extra-deployment.yaml
              {
                apiVersion = "v1";
                kind = "Service";
                metadata = {
                  name = "app-service";
                };
                spec = {
                  selector = {
                    "app.kubernetes.io/name" = "MyApp";
                  };
                  ports = [
                    {
                      name = "name-of-service-port";
                      protocol = "TCP";
                      port = 80;
                      targetPort = "http-web-svc";
                    }
                  ];
                };
              }
            ];
          '';
          description = "List of extra Kubernetes manifests to deploy with this Helm chart.";
        };

        extraFieldDefinitions = lib.mkOption {
          inherit (yamlFormat) type;
          default = { };
          example = {
            spec = {
              bootstrap = true;
              helmVersion = "v2";
              backOffLimit = 3;
              jobImage = "custom-helm-controller:v0.0.1";
            };
          };
          description = ''
            Extra HelmChart field definitions that are merged with the rest of the HelmChart
            custom resource. This can be used to set advanced fields or to overwrite
            generated fields. See https://docs.k3s.io/helm#helmchart-field-definitions
            for possible fields.
          '';
        };
      };

      config.package = lib.mkDefault (fetchHelm {
        inherit (config)
          repo
          name
          version
          hash
          ;
      });
    }
  );

  manifestModule = lib.types.submodule (
    {
      name,
      config,
      options,
      ...
    }:
    {
      options = {
        enable = lib.mkOption {
          type = lib.types.bool;
          default = true;
          description = "Whether this manifest file should be generated.";
        };

        target = lib.mkOption {
          type = lib.types.nonEmptyStr;
          example = "manifest.yaml";
          description = ''
            Name of the symlink (relative to {file}`${manifestDir}`).
            Defaults to the attribute name.
          '';
        };

        content = lib.mkOption {
          type = with lib.types; nullOr (either attrs (listOf attrs));
          default = null;
          description = ''
            Content of the manifest file. A single attribute set will
            generate a single document YAML file. A list of attribute sets
            will generate multiple documents separated by `---` in a single
            YAML file.
          '';
        };

        source = lib.mkOption {
          type = lib.types.path;
          example = lib.literalExpression "./manifests/app.yaml";
          description = ''
            Path of the source `.yaml` file.
          '';
        };
      };

      config = {
        target = lib.mkDefault (mkManifestTarget name);
        source = lib.mkIf (config.content != null) (
          let
            name' = "k3s-manifest-" + builtins.baseNameOf name;
            docName = "k3s-manifest-doc-" + builtins.baseNameOf name;
            mkSource =
              value:
              if builtins.isList value then
                pkgs.concatText name' (
                  lib.concatMap (x: [
                    yamlDocSeparator
                    (yamlFormat.generate docName x)
                  ]) value
                )
              else
                yamlFormat.generate name' value;
          in
          lib.mkDerivedConfig options.content mkSource
        );
      };
    }
  );
in
{
  imports = [ (removeOption [ "docker" ] "k3s docker option is no longer supported.") ];

  # interface
  options.services.k3s = {
    enable = lib.mkEnableOption "k3s";

    package = lib.mkPackageOption pkgs "k3s" { };

    role = lib.mkOption {
      description = ''
        Whether k3s should run as a server or agent.

        If it's a server:

        - By default it also runs workloads as an agent.
        - Starts by default as a standalone server using an embedded sqlite datastore.
        - Configure `clusterInit = true` to switch over to embedded etcd datastore and enable HA mode.
        - Configure `serverAddr` to join an already-initialized HA cluster.

        If it's an agent:

        - `serverAddr` is required.
      '';
      default = "server";
      type = lib.types.enum [
        "server"
        "agent"
      ];
    };

    serverAddr = lib.mkOption {
      type = lib.types.str;
      description = ''
        The k3s server to connect to.

        Servers and agents need to communicate each other. Read
        [the networking docs](https://rancher.com/docs/k3s/latest/en/installation/installation-requirements/#networking)
        to know how to configure the firewall.
      '';
      example = "https://10.0.0.10:6443";
      default = "";
    };

    clusterInit = lib.mkOption {
      type = lib.types.bool;
      default = false;
      description = ''
        Initialize HA cluster using an embedded etcd datastore.

        If this option is `false` and `role` is `server`

        On a server that was using the default embedded sqlite backend,
        enabling this option will migrate to an embedded etcd DB.

        If an HA cluster using the embedded etcd datastore was already initialized,
        this option has no effect.

        This option only makes sense in a server that is not connecting to another server.

        If you are configuring an HA cluster with an embedded etcd,
        the 1st server must have `clusterInit = true`
        and other servers must connect to it using `serverAddr`.
      '';
    };

    token = lib.mkOption {
      type = lib.types.str;
      description = ''
        The k3s token to use when connecting to a server.

        WARNING: This option will expose store your token unencrypted world-readable in the nix store.
        If this is undesired use the tokenFile option instead.
      '';
      default = "";
    };

    tokenFile = lib.mkOption {
      type = lib.types.nullOr lib.types.path;
      description = "File path containing k3s token to use when connecting to the server.";
      default = null;
    };

    extraFlags = lib.mkOption {
      description = "Extra flags to pass to the k3s command.";
      type = with lib.types; either str (listOf str);
      default = [ ];
      example = [
        "--disable traefik"
        "--cluster-cidr 10.24.0.0/16"
      ];
    };

    disableAgent = lib.mkOption {
      type = lib.types.bool;
      default = false;
      description = "Only run the server. This option only makes sense for a server.";
    };

    environmentFile = lib.mkOption {
      type = lib.types.nullOr lib.types.path;
      description = ''
        File path containing environment variables for configuring the k3s service in the format of an EnvironmentFile. See {manpage}`systemd.exec(5)`.
      '';
      default = null;
    };

    configPath = lib.mkOption {
      type = lib.types.nullOr lib.types.path;
      default = null;
      description = "File path containing the k3s YAML config. This is useful when the config is generated (for example on boot).";
    };

    manifests = lib.mkOption {
      type = lib.types.attrsOf manifestModule;
      default = { };
      example = lib.literalExpression ''
        {
          deployment.source = ../manifests/deployment.yaml;
          my-service = {
            enable = false;
            target = "app-service.yaml";
            content = {
              apiVersion = "v1";
              kind = "Service";
              metadata = {
                name = "app-service";
              };
              spec = {
                selector = {
                  "app.kubernetes.io/name" = "MyApp";
                };
                ports = [
                  {
                    name = "name-of-service-port";
                    protocol = "TCP";
                    port = 80;
                    targetPort = "http-web-svc";
                  }
                ];
              };
            };
          };

          nginx.content = [
            {
              apiVersion = "v1";
              kind = "Pod";
              metadata = {
                name = "nginx";
                labels = {
                  "app.kubernetes.io/name" = "MyApp";
                };
              };
              spec = {
                containers = [
                  {
                    name = "nginx";
                    image = "nginx:1.14.2";
                    ports = [
                      {
                        containerPort = 80;
                        name = "http-web-svc";
                      }
                    ];
                  }
                ];
              };
            }
            {
              apiVersion = "v1";
              kind = "Service";
              metadata = {
                name = "nginx-service";
              };
              spec = {
                selector = {
                  "app.kubernetes.io/name" = "MyApp";
                };
                ports = [
                  {
                    name = "name-of-service-port";
                    protocol = "TCP";
                    port = 80;
                    targetPort = "http-web-svc";
                  }
                ];
              };
            }
          ];
        };
      '';
      description = ''
        Auto-deploying manifests that are linked to {file}`${manifestDir}` before k3s starts.
        Note that deleting manifest files will not remove or otherwise modify the resources
        it created. Please use the the `--disable` flag or `.skip` files to delete/disable AddOns,
        as mentioned in the [docs](https://docs.k3s.io/installation/packaged-components#disabling-manifests).
        This option only makes sense on server nodes (`role = server`).
        Read the [auto-deploying manifests docs](https://docs.k3s.io/installation/packaged-components#auto-deploying-manifests-addons)
        for further information.
      '';
    };

    charts = lib.mkOption {
      type = with lib.types; attrsOf (either path package);
      default = { };
      example = lib.literalExpression ''
        nginx = ../charts/my-nginx-chart.tgz;
        redis = ../charts/my-redis-chart.tgz;
      '';
      description = ''
        Packaged Helm charts that are linked to {file}`${chartDir}` before k3s starts.
        The attribute name will be used as the link target (relative to {file}`${chartDir}`).
        The specified charts will only be placed on the file system and made available to the
        Kubernetes APIServer from within the cluster. See the [](#opt-services.k3s.autoDeployCharts)
        option and the [k3s Helm controller docs](https://docs.k3s.io/helm#using-the-helm-controller)
        to deploy Helm charts. This option only makes sense on server nodes (`role = server`).
      '';
    };

    containerdConfigTemplate = lib.mkOption {
      type = lib.types.nullOr lib.types.str;
      default = null;
      example = lib.literalExpression ''
        # Base K3s config
        {{ template "base" . }}

        # Add a custom runtime
        [plugins."io.containerd.grpc.v1.cri".containerd.runtimes."custom"]
          runtime_type = "io.containerd.runc.v2"
        [plugins."io.containerd.grpc.v1.cri".containerd.runtimes."custom".options]
          BinaryName = "/path/to/custom-container-runtime"
      '';
      description = ''
        Config template for containerd, to be placed at
        `/var/lib/rancher/k3s/agent/etc/containerd/config.toml.tmpl`.
        See the K3s docs on [configuring containerd](https://docs.k3s.io/advanced#configuring-containerd).
      '';
    };

    images = lib.mkOption {
      type = with lib.types; listOf package;
      default = [ ];
      example = lib.literalExpression ''
        [
          (pkgs.dockerTools.pullImage {
            imageName = "docker.io/bitnami/keycloak";
            imageDigest = "sha256:714dfadc66a8e3adea6609bda350345bd3711657b7ef3cf2e8015b526bac2d6b";
            hash = "sha256-IM2BLZ0EdKIZcRWOtuFY9TogZJXCpKtPZnMnPsGlq0Y=";
            finalImageTag = "21.1.2-debian-11-r0";
          })

          config.services.k3s.package.airgapImages
        ]
      '';
      description = ''
        List of derivations that provide container images.
        All images are linked to {file}`${imageDir}` before k3s starts and consequently imported
        by the k3s agent. Consider importing the k3s airgap images archive of the k3s package in
        use, if you want to pre-provision this node with all k3s container images. This option
        only makes sense on nodes with an enabled agent.
      '';
    };

    gracefulNodeShutdown = {
      enable = lib.mkEnableOption ''
        graceful node shutdowns where the kubelet attempts to detect
        node system shutdown and terminates pods running on the node. See the
        [documentation](https://kubernetes.io/docs/concepts/cluster-administration/node-shutdown/#graceful-node-shutdown)
        for further information.
      '';

      shutdownGracePeriod = lib.mkOption {
        type = lib.types.nonEmptyStr;
        default = "30s";
        example = "1m30s";
        description = ''
          Specifies the total duration that the node should delay the shutdown by. This is the total
          grace period for pod termination for both regular and critical pods.
        '';
      };

      shutdownGracePeriodCriticalPods = lib.mkOption {
        type = lib.types.nonEmptyStr;
        default = "10s";
        example = "15s";
        description = ''
          Specifies the duration used to terminate critical pods during a node shutdown. This should be
          less than `shutdownGracePeriod`.
        '';
      };
    };

    extraKubeletConfig = lib.mkOption {
      type = with lib.types; attrsOf anything;
      default = { };
      example = {
        podsPerCore = 3;
        memoryThrottlingFactor = 0.69;
        containerLogMaxSize = "5Mi";
      };
      description = ''
        Extra configuration to add to the kubelet's configuration file. The subset of the kubelet's
        configuration that can be configured via a file is defined by the
        [KubeletConfiguration](https://kubernetes.io/docs/reference/config-api/kubelet-config.v1beta1/)
        struct. See the
        [documentation](https://kubernetes.io/docs/tasks/administer-cluster/kubelet-config-file/)
        for further information.
      '';
    };

    extraKubeProxyConfig = lib.mkOption {
      type = with lib.types; attrsOf anything;
      default = { };
      example = {
        mode = "nftables";
        clientConnection.kubeconfig = "/var/lib/rancher/k3s/agent/kubeproxy.kubeconfig";
      };
      description = ''
        Extra configuration to add to the kube-proxy's configuration file. The subset of the kube-proxy's
        configuration that can be configured via a file is defined by the
        [KubeProxyConfiguration](https://kubernetes.io/docs/reference/config-api/kube-proxy-config.v1alpha1/)
        struct. Note that the kubeconfig param will be override by `clientConnection.kubeconfig`, so you must
        set the `clientConnection.kubeconfig` if you want to use `extraKubeProxyConfig`.
      '';
    };

    autoDeployCharts = lib.mkOption {
      type = lib.types.attrsOf autoDeployChartsModule;
      apply = lib.mapAttrs mkAutoDeployChartManifest;
      default = { };
      example = lib.literalExpression ''
        {
          harbor = {
            name = "harbor";
            repo = "https://helm.goharbor.io";
            version = "1.14.0";
            hash = "sha256-fMP7q1MIbvzPGS9My91vbQ1d3OJMjwc+o8YE/BXZaYU=";
            values = {
              existingSecretAdminPassword = "harbor-admin";
              expose = {
                tls = {
                  enabled = true;
                  certSource = "secret";
                  secret.secretName = "my-tls-secret";
                };
                ingress = {
                  hosts.core = "example.com";
                  className = "nginx";
                };
              };
            };
          };
          nginx = {
            repo = "oci://registry-1.docker.io/bitnamicharts/nginx";
            version = "20.0.0";
            hash = "sha256-sy+tzB+i9jIl/tqOMzzuhVhTU4EZVsoSBtPznxF/36c=";
          };
          custom-chart = {
            package = ../charts/my-chart.tgz;
            values = ../values/my-values.yaml;
            extraFieldDefinitions = {
              spec.timeout = "60s";
            };
          };
        }
      '';
      description = ''
        Auto deploying Helm charts that are installed by the k3s Helm controller. Avoid to use
        attribute names that are also used in the [](#opt-services.k3s.manifests) and
        [](#opt-services.k3s.charts) options. Manifests with the same name will override
        auto deploying charts with the same name. Similiarly, charts with the same name will
        overwrite the Helm chart contained in auto deploying charts. This option only makes
        sense on server nodes (`role = server`). See the
        [k3s Helm documentation](https://docs.k3s.io/helm) for further information.
      '';
    };
  };

  # implementation

  config = lib.mkIf cfg.enable {
    warnings =
      (lib.optional (cfg.role != "server" && cfg.manifests != { })
        "k3s: Auto deploying manifests are only installed on server nodes (role == server), they will be ignored by this node."
      )
      ++ (lib.optional (cfg.role != "server" && cfg.charts != { })
        "k3s: Helm charts are only made available to the cluster on server nodes (role == server), they will be ignored by this node."
      )
      ++ (lib.optional (cfg.role != "server" && cfg.autoDeployCharts != { })
        "k3s: Auto deploying Helm charts are only installed on server nodes (role == server), they will be ignored by this node."
      )
      ++ (lib.optional (duplicateManifests != [ ])
        "k3s: The following auto deploying charts are overriden by manifests of the same name: ${toString duplicateManifests}."
      )
      ++ (lib.optional (duplicateCharts != [ ])
        "k3s: The following auto deploying charts are overriden by charts of the same name: ${toString duplicateCharts}."
      )
      ++ (lib.optional (
        cfg.disableAgent && cfg.images != [ ]
      ) "k3s: Images are only imported on nodes with an enabled agent, they will be ignored by this node")
      ++ (lib.optional (
        cfg.role == "agent" && cfg.configPath == null && cfg.serverAddr == ""
      ) "k3s: ServerAddr or configPath (with 'server' key) should be set if role is 'agent'")
      ++ (lib.optional
        (cfg.role == "agent" && cfg.configPath == null && cfg.tokenFile == null && cfg.token == "")
        "k3s: Token or tokenFile or configPath (with 'token' or 'token-file' keys) should be set if role is 'agent'"
      );

    assertions = [
      {
        assertion = cfg.role == "agent" -> !cfg.disableAgent;
        message = "k3s: disableAgent must be false if role is 'agent'";
      }
      {
        assertion = cfg.role == "agent" -> !cfg.clusterInit;
        message = "k3s: clusterInit must be false if role is 'agent'";
      }
    ];

    environment.systemPackages = [ config.services.k3s.package ];

    # Use systemd-tmpfiles to activate k3s content
    systemd.tmpfiles.settings."10-k3s" =
      let
        # Merge manifest with manifests generated from auto deploying charts, keep only enabled manifests
        enabledManifests = lib.filterAttrs (_: v: v.enable) (cfg.autoDeployCharts // cfg.manifests);
        # Merge charts with charts contained in enabled auto deploying charts
        helmCharts =
          (lib.concatMapAttrs (n: v: { ${n} = v.package; }) (
            lib.filterAttrs (_: v: v.enable) cfg.autoDeployCharts
          ))
          // cfg.charts;
        # Make a systemd-tmpfiles rule for a manifest
        mkManifestRule = manifest: {
          name = "${manifestDir}/${manifest.target}";
          value = {
            "L+".argument = "${manifest.source}";
          };
        };
        # Ensure that all chart targets have a .tgz suffix
        mkChartTarget = name: if (lib.hasSuffix ".tgz" name) then name else name + ".tgz";
        # Make a systemd-tmpfiles rule for a chart
        mkChartRule = target: source: {
          name = "${chartDir}/${mkChartTarget target}";
          value = {
            "L+".argument = "${source}";
          };
        };
        # Make a systemd-tmpfiles rule for a container image
        mkImageRule = image: {
          name = "${imageDir}/${image.name}";
          value = {
            "L+".argument = "${image}";
          };
        };
      in
      (lib.mapAttrs' (_: v: mkManifestRule v) enabledManifests)
      // (lib.mapAttrs' (n: v: mkChartRule n v) helmCharts)
      // (builtins.listToAttrs (map mkImageRule cfg.images))
      // (lib.optionalAttrs (cfg.containerdConfigTemplate != null) {
        ${containerdConfigTemplateFile} = {
          "L+".argument = "${pkgs.writeText "config.toml.tmpl" cfg.containerdConfigTemplate}";
        };
      });

    systemd.services.k3s =
      let
        kubeletParams =
          (lib.optionalAttrs (cfg.gracefulNodeShutdown.enable) {
            inherit (cfg.gracefulNodeShutdown) shutdownGracePeriod shutdownGracePeriodCriticalPods;
          })
          // cfg.extraKubeletConfig;
        kubeletConfig = (pkgs.formats.yaml { }).generate "k3s-kubelet-config" (
          {
            apiVersion = "kubelet.config.k8s.io/v1beta1";
            kind = "KubeletConfiguration";
          }
          // kubeletParams
        );

        kubeProxyConfig = (pkgs.formats.yaml { }).generate "k3s-kubeProxy-config" (
          {
            apiVersion = "kubeproxy.config.k8s.io/v1alpha1";
            kind = "KubeProxyConfiguration";
          }
          // cfg.extraKubeProxyConfig
        );
      in
      {
        description = "k3s service";
        after = [
          "firewall.service"
          "network-online.target"
        ];
        wants = [
          "firewall.service"
          "network-online.target"
        ];
        wantedBy = [ "multi-user.target" ];
        path = lib.optional config.boot.zfs.enabled config.boot.zfs.package;
        serviceConfig = {
          # See: https://github.com/rancher/k3s/blob/dddbd16305284ae4bd14c0aade892412310d7edc/install.sh#L197
          Type = if cfg.role == "agent" then "exec" else "notify";
          KillMode = "process";
          Delegate = "yes";
          Restart = "always";
          RestartSec = "5s";
          LimitNOFILE = 1048576;
          LimitNPROC = "infinity";
          LimitCORE = "infinity";
          TasksMax = "infinity";
          EnvironmentFile = cfg.environmentFile;
          ExecStart = lib.concatStringsSep " \\\n " (
            [ "${cfg.package}/bin/k3s ${cfg.role}" ]
            ++ (lib.optional cfg.clusterInit "--cluster-init")
            ++ (lib.optional cfg.disableAgent "--disable-agent")
            ++ (lib.optional (cfg.serverAddr != "") "--server ${cfg.serverAddr}")
            ++ (lib.optional (cfg.token != "") "--token ${cfg.token}")
            ++ (lib.optional (cfg.tokenFile != null) "--token-file ${cfg.tokenFile}")
            ++ (lib.optional (cfg.configPath != null) "--config ${cfg.configPath}")
            ++ (lib.optional (kubeletParams != { }) "--kubelet-arg=config=${kubeletConfig}")
            ++ (lib.optional (cfg.extraKubeProxyConfig != { }) "--kube-proxy-arg=config=${kubeProxyConfig}")
            ++ (lib.flatten cfg.extraFlags)
          );
        };
      };
  };

  meta.maintainers = lib.teams.k3s.members;
}