mirror of
https://github.com/NixOS/nixpkgs.git
synced 2025-07-14 06:00:33 +03:00
nixos/k3s: add autoDeployCharts
option
The `autoDeployCharts` option further improves the auto deploying capabilities of the k3s module by allowing to deploy and configure Helm charts that are then instaled via the k3s Helm controller. Although this was also previously possible by using auto deploying manifests, it required some knowledge of the k3s Helm controller and led to a lot of boilerplate code.
This commit is contained in:
parent
15884bd3a8
commit
95b894bad7
7 changed files with 671 additions and 148 deletions
|
@ -392,6 +392,8 @@
|
|||
- New options for the declarative configuration of the user space part of ALSA have been introduced under [hardware.alsa](options.html#opt-hardware.alsa.enable), including setting the default capture and playback device, defining sound card aliases and volume controls.
|
||||
Note: these are intended for users not running a sound server like PulseAudio or PipeWire, but having ALSA as their only sound system.
|
||||
|
||||
- `services.k3s` now provides the `autoDeployCharts` option that allows to automatically deploy Helm charts via the k3s Helm controller.
|
||||
|
||||
- Caddy can now be built with plugins by using `caddy.withPlugins`, a `passthru` function that accepts an attribute set as a parameter. The `plugins` argument represents a list of Caddy plugins, with each Caddy plugin being a versioned module. The `hash` argument represents the `vendorHash` of the resulting Caddy source code with the plugins added.
|
||||
|
||||
Example:
|
||||
|
|
|
@ -20,103 +20,386 @@ let
|
|||
chartDir = "/var/lib/rancher/k3s/server/static/charts";
|
||||
imageDir = "/var/lib/rancher/k3s/agent/images";
|
||||
containerdConfigTemplateFile = "/var/lib/rancher/k3s/agent/etc/containerd/config.toml.tmpl";
|
||||
yamlFormat = pkgs.formats.yaml { };
|
||||
yamlDocSeparator = builtins.toFile "yaml-doc-separator" "\n---\n";
|
||||
# Manifests need a valid YAML suffix to be respected by k3s
|
||||
mkManifestTarget =
|
||||
name: if (lib.hasSuffix ".yaml" name || lib.hasSuffix ".yml" name) then name else name + ".yaml";
|
||||
# Produces a list containing all duplicate manifest names
|
||||
duplicateManifests =
|
||||
with builtins;
|
||||
lib.intersectLists (attrNames cfg.autoDeployCharts) (attrNames cfg.manifests);
|
||||
# Produces a list containing all duplicate chart names
|
||||
duplicateCharts =
|
||||
with builtins;
|
||||
lib.intersectLists (attrNames cfg.autoDeployCharts) (attrNames cfg.charts);
|
||||
|
||||
manifestModule =
|
||||
let
|
||||
mkTarget =
|
||||
name: if (lib.hasSuffix ".yaml" name || lib.hasSuffix ".yml" name) then name else name + ".yaml";
|
||||
in
|
||||
lib.types.submodule (
|
||||
{
|
||||
name,
|
||||
config,
|
||||
options,
|
||||
...
|
||||
}:
|
||||
{
|
||||
options = {
|
||||
enable = lib.mkOption {
|
||||
type = lib.types.bool;
|
||||
default = true;
|
||||
description = "Whether this manifest file should be generated.";
|
||||
};
|
||||
|
||||
target = lib.mkOption {
|
||||
type = lib.types.nonEmptyStr;
|
||||
example = lib.literalExpression "manifest.yaml";
|
||||
description = ''
|
||||
Name of the symlink (relative to {file}`${manifestDir}`).
|
||||
Defaults to the attribute name.
|
||||
'';
|
||||
};
|
||||
|
||||
content = lib.mkOption {
|
||||
type = with lib.types; nullOr (either attrs (listOf attrs));
|
||||
default = null;
|
||||
description = ''
|
||||
Content of the manifest file. A single attribute set will
|
||||
generate a single document YAML file. A list of attribute sets
|
||||
will generate multiple documents separated by `---` in a single
|
||||
YAML file.
|
||||
'';
|
||||
};
|
||||
|
||||
source = lib.mkOption {
|
||||
type = lib.types.path;
|
||||
example = lib.literalExpression "./manifests/app.yaml";
|
||||
description = ''
|
||||
Path of the source `.yaml` file.
|
||||
'';
|
||||
};
|
||||
};
|
||||
|
||||
config = {
|
||||
target = lib.mkDefault (mkTarget name);
|
||||
source = lib.mkIf (config.content != null) (
|
||||
let
|
||||
name' = "k3s-manifest-" + builtins.baseNameOf name;
|
||||
docName = "k3s-manifest-doc-" + builtins.baseNameOf name;
|
||||
yamlDocSeparator = builtins.toFile "yaml-doc-separator" "\n---\n";
|
||||
mkYaml = name: x: (pkgs.formats.yaml { }).generate name x;
|
||||
mkSource =
|
||||
value:
|
||||
if builtins.isList value then
|
||||
pkgs.concatText name' (
|
||||
lib.concatMap (x: [
|
||||
yamlDocSeparator
|
||||
(mkYaml docName x)
|
||||
]) value
|
||||
)
|
||||
else
|
||||
mkYaml name' value;
|
||||
in
|
||||
lib.mkDerivedConfig options.content mkSource
|
||||
);
|
||||
};
|
||||
}
|
||||
# Converts YAML -> JSON -> Nix
|
||||
fromYaml =
|
||||
path:
|
||||
with builtins;
|
||||
fromJSON (
|
||||
readFile (
|
||||
pkgs.runCommand "${path}-converted.json" { nativeBuildInputs = [ yq-go ]; } ''
|
||||
yq --no-colors --output-format json ${path} > $out
|
||||
''
|
||||
)
|
||||
);
|
||||
|
||||
# Replace characters that are problematic in file names
|
||||
cleanHelmChartName =
|
||||
lib.replaceStrings
|
||||
[
|
||||
"/"
|
||||
":"
|
||||
]
|
||||
[
|
||||
"-"
|
||||
"-"
|
||||
];
|
||||
|
||||
# Fetch a Helm chart from a public registry. This only supports a basic Helm pull.
|
||||
fetchHelm =
|
||||
{
|
||||
name,
|
||||
repo,
|
||||
version,
|
||||
hash ? lib.fakeHash,
|
||||
}:
|
||||
pkgs.runCommand (cleanHelmChartName "${lib.removePrefix "https://" repo}-${name}-${version}.tgz")
|
||||
{
|
||||
inherit (lib.fetchers.normalizeHash { } { inherit hash; }) outputHash outputHashAlgo;
|
||||
impureEnvVars = lib.fetchers.proxyImpureEnvVars;
|
||||
nativeBuildInputs = with pkgs; [
|
||||
kubernetes-helm
|
||||
cacert
|
||||
];
|
||||
}
|
||||
''
|
||||
export HOME="$PWD"
|
||||
helm repo add repository ${repo}
|
||||
helm pull repository/${name} --version ${version}
|
||||
mv ./*.tgz $out
|
||||
'';
|
||||
|
||||
# Returns the path to a YAML manifest file
|
||||
mkExtraDeployManifest =
|
||||
x:
|
||||
# x is a derivation that provides a YAML file
|
||||
if lib.isDerivation x then
|
||||
x.outPath
|
||||
# x is an attribute set that needs to be converted to a YAML file
|
||||
else if builtins.isAttrs x then
|
||||
(yamlFormat.generate "extra-deploy-chart-manifest" x)
|
||||
# assume x is a path to a YAML file
|
||||
else
|
||||
x;
|
||||
|
||||
# Generate a HelmChart custom resource.
|
||||
mkHelmChartCR =
|
||||
name: value:
|
||||
let
|
||||
chartValues = if (lib.isPath value.values) then fromYaml value.values else value.values;
|
||||
# use JSON for values as it's a subset of YAML and understood by the k3s Helm controller
|
||||
valuesContent = builtins.toJSON chartValues;
|
||||
in
|
||||
# merge with extraFieldDefinitions to allow setting advanced values and overwrite generated
|
||||
# values
|
||||
lib.recursiveUpdate {
|
||||
apiVersion = "helm.cattle.io/v1";
|
||||
kind = "HelmChart";
|
||||
metadata = {
|
||||
inherit name;
|
||||
namespace = "kube-system";
|
||||
};
|
||||
spec = {
|
||||
inherit valuesContent;
|
||||
inherit (value) targetNamespace createNamespace;
|
||||
chart = "https://%{KUBERNETES_API}%/static/charts/${name}.tgz";
|
||||
};
|
||||
} value.extraFieldDefinitions;
|
||||
|
||||
# Generate a HelmChart custom resource together with extraDeploy manifests. This
|
||||
# generates possibly a multi document YAML file that the auto deploy mechanism of k3s
|
||||
# deploys.
|
||||
mkAutoDeployChartManifest = name: value: {
|
||||
# target is the final name of the link created for the manifest file
|
||||
target = mkManifestTarget name;
|
||||
inherit (value) enable package;
|
||||
# source is a store path containing the complete manifest file
|
||||
source = pkgs.concatText "auto-deploy-chart-${name}.yaml" (
|
||||
[
|
||||
(yamlFormat.generate "helm-chart-manifest-${name}.yaml" (mkHelmChartCR name value))
|
||||
]
|
||||
# alternate the YAML doc seperator (---) and extraDeploy manifests to create
|
||||
# multi document YAMLs
|
||||
++ (lib.concatMap (x: [
|
||||
yamlDocSeparator
|
||||
(mkExtraDeployManifest x)
|
||||
]) value.extraDeploy)
|
||||
);
|
||||
};
|
||||
|
||||
autoDeployChartsModule = lib.types.submodule (
|
||||
{ config, ... }:
|
||||
{
|
||||
options = {
|
||||
enable = lib.mkOption {
|
||||
type = lib.types.bool;
|
||||
default = true;
|
||||
example = false;
|
||||
description = ''
|
||||
Whether to enable the installation of this Helm chart. Note that setting
|
||||
this option to `false` will not uninstall the chart from the cluster, if
|
||||
it was previously installed. Please use the the `--disable` flag or `.skip`
|
||||
files to delete/disable Helm charts, as mentioned in the
|
||||
[docs](https://docs.k3s.io/installation/packaged-components#disabling-manifests).
|
||||
'';
|
||||
};
|
||||
|
||||
repo = lib.mkOption {
|
||||
type = lib.types.nonEmptyStr;
|
||||
example = "https://kubernetes.github.io/ingress-nginx";
|
||||
description = ''
|
||||
The repo of the Helm chart. Only has an effect if `package` is not set.
|
||||
The Helm chart is fetched during build time and placed as a `.tgz` archive on the
|
||||
filesystem.
|
||||
'';
|
||||
};
|
||||
|
||||
name = lib.mkOption {
|
||||
type = lib.types.nonEmptyStr;
|
||||
example = "ingress-nginx";
|
||||
description = ''
|
||||
The name of the Helm chart. Only has an effect if `package` is not set.
|
||||
The Helm chart is fetched during build time and placed as a `.tgz` archive on the
|
||||
filesystem.
|
||||
'';
|
||||
};
|
||||
|
||||
version = lib.mkOption {
|
||||
type = lib.types.nonEmptyStr;
|
||||
example = "4.7.0";
|
||||
description = ''
|
||||
The version of the Helm chart. Only has an effect if `package` is not set.
|
||||
The Helm chart is fetched during build time and placed as a `.tgz` archive on the
|
||||
filesystem.
|
||||
'';
|
||||
};
|
||||
|
||||
hash = lib.mkOption {
|
||||
type = lib.types.str;
|
||||
example = "sha256-ej+vpPNdiOoXsaj1jyRpWLisJgWo8EqX+Z5VbpSjsPA=";
|
||||
description = ''
|
||||
The hash of the packaged Helm chart. Only has an effect if `package` is not set.
|
||||
The Helm chart is fetched during build time and placed as a `.tgz` archive on the
|
||||
filesystem.
|
||||
'';
|
||||
};
|
||||
|
||||
package = lib.mkOption {
|
||||
type = with lib.types; either path package;
|
||||
example = lib.literalExpression "../my-helm-chart.tgz";
|
||||
description = ''
|
||||
The packaged Helm chart. Overwrites the options `repo`, `name`, `version`
|
||||
and `hash` in case of conflicts.
|
||||
'';
|
||||
};
|
||||
|
||||
targetNamespace = lib.mkOption {
|
||||
type = lib.types.nonEmptyStr;
|
||||
default = "default";
|
||||
example = "kube-system";
|
||||
description = "The namespace in which the Helm chart gets installed.";
|
||||
};
|
||||
|
||||
createNamespace = lib.mkOption {
|
||||
type = lib.types.bool;
|
||||
default = false;
|
||||
example = true;
|
||||
description = "Whether to create the target namespace if not present.";
|
||||
};
|
||||
|
||||
values = lib.mkOption {
|
||||
type = with lib.types; either path attrs;
|
||||
default = { };
|
||||
example = {
|
||||
replicaCount = 3;
|
||||
hostName = "my-host";
|
||||
server = {
|
||||
name = "nginx";
|
||||
port = 80;
|
||||
};
|
||||
};
|
||||
description = ''
|
||||
Override default chart values via Nix expressions. This is equivalent to setting
|
||||
values in a `values.yaml` file.
|
||||
|
||||
WARNING: The values (including secrets!) specified here are exposed unencrypted
|
||||
in the world-readable nix store.
|
||||
'';
|
||||
};
|
||||
|
||||
extraDeploy = lib.mkOption {
|
||||
type = with lib.types; listOf (either path attrs);
|
||||
default = [ ];
|
||||
example = lib.literalExpression ''
|
||||
[
|
||||
../manifests/my-extra-deployment.yaml
|
||||
{
|
||||
apiVersion = "v1";
|
||||
kind = "Service";
|
||||
metadata = {
|
||||
name = "app-service";
|
||||
};
|
||||
spec = {
|
||||
selector = {
|
||||
"app.kubernetes.io/name" = "MyApp";
|
||||
};
|
||||
ports = [
|
||||
{
|
||||
name = "name-of-service-port";
|
||||
protocol = "TCP";
|
||||
port = 80;
|
||||
targetPort = "http-web-svc";
|
||||
}
|
||||
];
|
||||
};
|
||||
}
|
||||
];
|
||||
'';
|
||||
description = "List of extra Kubernetes manifests to deploy with this Helm chart.";
|
||||
};
|
||||
|
||||
extraFieldDefinitions = lib.mkOption {
|
||||
inherit (yamlFormat) type;
|
||||
default = { };
|
||||
example = {
|
||||
spec = {
|
||||
bootstrap = true;
|
||||
helmVersion = "v2";
|
||||
backOffLimit = 3;
|
||||
jobImage = "custom-helm-controller:v0.0.1";
|
||||
};
|
||||
};
|
||||
description = ''
|
||||
Extra HelmChart field definitions that are merged with the rest of the HelmChart
|
||||
custom resource. This can be used to set advanced fields or to overwrite
|
||||
generated fields. See https://docs.k3s.io/helm#helmchart-field-definitions
|
||||
for possible fields.
|
||||
'';
|
||||
};
|
||||
};
|
||||
|
||||
config.package = lib.mkDefault (fetchHelm {
|
||||
inherit (config)
|
||||
repo
|
||||
name
|
||||
version
|
||||
hash
|
||||
;
|
||||
});
|
||||
}
|
||||
);
|
||||
|
||||
manifestModule = lib.types.submodule (
|
||||
{
|
||||
name,
|
||||
config,
|
||||
options,
|
||||
...
|
||||
}:
|
||||
{
|
||||
options = {
|
||||
enable = lib.mkOption {
|
||||
type = lib.types.bool;
|
||||
default = true;
|
||||
description = "Whether this manifest file should be generated.";
|
||||
};
|
||||
|
||||
target = lib.mkOption {
|
||||
type = lib.types.nonEmptyStr;
|
||||
example = "manifest.yaml";
|
||||
description = ''
|
||||
Name of the symlink (relative to {file}`${manifestDir}`).
|
||||
Defaults to the attribute name.
|
||||
'';
|
||||
};
|
||||
|
||||
content = lib.mkOption {
|
||||
type = with lib.types; nullOr (either attrs (listOf attrs));
|
||||
default = null;
|
||||
description = ''
|
||||
Content of the manifest file. A single attribute set will
|
||||
generate a single document YAML file. A list of attribute sets
|
||||
will generate multiple documents separated by `---` in a single
|
||||
YAML file.
|
||||
'';
|
||||
};
|
||||
|
||||
source = lib.mkOption {
|
||||
type = lib.types.path;
|
||||
example = lib.literalExpression "./manifests/app.yaml";
|
||||
description = ''
|
||||
Path of the source `.yaml` file.
|
||||
'';
|
||||
};
|
||||
};
|
||||
|
||||
config = {
|
||||
target = lib.mkDefault (mkManifestTarget name);
|
||||
source = lib.mkIf (config.content != null) (
|
||||
let
|
||||
name' = "k3s-manifest-" + builtins.baseNameOf name;
|
||||
docName = "k3s-manifest-doc-" + builtins.baseNameOf name;
|
||||
mkSource =
|
||||
value:
|
||||
if builtins.isList value then
|
||||
pkgs.concatText name' (
|
||||
lib.concatMap (x: [
|
||||
yamlDocSeparator
|
||||
(yamlFormat.generate docName x)
|
||||
]) value
|
||||
)
|
||||
else
|
||||
yamlFormat.generate name' value;
|
||||
in
|
||||
lib.mkDerivedConfig options.content mkSource
|
||||
);
|
||||
};
|
||||
}
|
||||
);
|
||||
|
||||
# TODO: use tmpfiles
|
||||
enabledManifests = lib.filter (m: m.enable) (lib.attrValues cfg.manifests);
|
||||
enabledHelmManifests = lib.filter (m: m.enable) (lib.attrValues cfg.autoDeployCharts);
|
||||
enabledAutoDeployCharts = lib.concatMapAttrs (n: v: { ${n} = v.package; }) (
|
||||
lib.filterAttrs (_: v: v.enable) cfg.autoDeployCharts
|
||||
);
|
||||
linkManifestEntry = m: "${pkgs.coreutils-full}/bin/ln -sfn ${m.source} ${manifestDir}/${m.target}";
|
||||
linkImageEntry = image: "${pkgs.coreutils-full}/bin/ln -sfn ${image} ${imageDir}/${image.name}";
|
||||
linkChartEntry =
|
||||
let
|
||||
mkTarget = name: if (lib.hasSuffix ".tgz" name) then name else name + ".tgz";
|
||||
mkChartTarget = name: if (lib.hasSuffix ".tgz" name) then name else name + ".tgz";
|
||||
in
|
||||
name: value:
|
||||
"${pkgs.coreutils-full}/bin/ln -sfn ${value} ${chartDir}/${mkTarget (builtins.baseNameOf name)}";
|
||||
"${pkgs.coreutils-full}/bin/ln -sfn ${value} ${chartDir}/${mkChartTarget (builtins.baseNameOf name)}";
|
||||
|
||||
activateK3sContent = pkgs.writeShellScript "activate-k3s-content" ''
|
||||
${lib.optionalString (
|
||||
builtins.length enabledManifests > 0
|
||||
builtins.length (enabledManifests ++ enabledHelmManifests) > 0
|
||||
) "${pkgs.coreutils-full}/bin/mkdir -p ${manifestDir}"}
|
||||
${lib.optionalString (cfg.charts != { }) "${pkgs.coreutils-full}/bin/mkdir -p ${chartDir}"}
|
||||
${lib.optionalString (
|
||||
cfg.charts != { } || enabledAutoDeployCharts != { }
|
||||
) "${pkgs.coreutils-full}/bin/mkdir -p ${chartDir}"}
|
||||
${lib.optionalString (
|
||||
builtins.length cfg.images > 0
|
||||
) "${pkgs.coreutils-full}/bin/mkdir -p ${imageDir}"}
|
||||
|
||||
${builtins.concatStringsSep "\n" (map linkManifestEntry enabledManifests)}
|
||||
${builtins.concatStringsSep "\n" (map linkManifestEntry enabledHelmManifests)}
|
||||
${builtins.concatStringsSep "\n" (lib.mapAttrsToList linkChartEntry cfg.charts)}
|
||||
${builtins.concatStringsSep "\n" (lib.mapAttrsToList linkChartEntry enabledAutoDeployCharts)}
|
||||
${builtins.concatStringsSep "\n" (map linkImageEntry cfg.images)}
|
||||
|
||||
${lib.optionalString (cfg.containerdConfigTemplate != null) ''
|
||||
|
@ -242,78 +525,80 @@ in
|
|||
type = lib.types.attrsOf manifestModule;
|
||||
default = { };
|
||||
example = lib.literalExpression ''
|
||||
deployment.source = ../manifests/deployment.yaml;
|
||||
my-service = {
|
||||
enable = false;
|
||||
target = "app-service.yaml";
|
||||
content = {
|
||||
apiVersion = "v1";
|
||||
kind = "Service";
|
||||
metadata = {
|
||||
name = "app-service";
|
||||
};
|
||||
spec = {
|
||||
selector = {
|
||||
"app.kubernetes.io/name" = "MyApp";
|
||||
{
|
||||
deployment.source = ../manifests/deployment.yaml;
|
||||
my-service = {
|
||||
enable = false;
|
||||
target = "app-service.yaml";
|
||||
content = {
|
||||
apiVersion = "v1";
|
||||
kind = "Service";
|
||||
metadata = {
|
||||
name = "app-service";
|
||||
};
|
||||
spec = {
|
||||
selector = {
|
||||
"app.kubernetes.io/name" = "MyApp";
|
||||
};
|
||||
ports = [
|
||||
{
|
||||
name = "name-of-service-port";
|
||||
protocol = "TCP";
|
||||
port = 80;
|
||||
targetPort = "http-web-svc";
|
||||
}
|
||||
];
|
||||
};
|
||||
ports = [
|
||||
{
|
||||
name = "name-of-service-port";
|
||||
protocol = "TCP";
|
||||
port = 80;
|
||||
targetPort = "http-web-svc";
|
||||
}
|
||||
];
|
||||
};
|
||||
}
|
||||
};
|
||||
};
|
||||
|
||||
nginx.content = [
|
||||
{
|
||||
apiVersion = "v1";
|
||||
kind = "Pod";
|
||||
metadata = {
|
||||
name = "nginx";
|
||||
labels = {
|
||||
"app.kubernetes.io/name" = "MyApp";
|
||||
nginx.content = [
|
||||
{
|
||||
apiVersion = "v1";
|
||||
kind = "Pod";
|
||||
metadata = {
|
||||
name = "nginx";
|
||||
labels = {
|
||||
"app.kubernetes.io/name" = "MyApp";
|
||||
};
|
||||
};
|
||||
};
|
||||
spec = {
|
||||
containers = [
|
||||
{
|
||||
name = "nginx";
|
||||
image = "nginx:1.14.2";
|
||||
ports = [
|
||||
{
|
||||
containerPort = 80;
|
||||
name = "http-web-svc";
|
||||
}
|
||||
];
|
||||
}
|
||||
];
|
||||
};
|
||||
}
|
||||
{
|
||||
apiVersion = "v1";
|
||||
kind = "Service";
|
||||
metadata = {
|
||||
name = "nginx-service";
|
||||
};
|
||||
spec = {
|
||||
selector = {
|
||||
"app.kubernetes.io/name" = "MyApp";
|
||||
spec = {
|
||||
containers = [
|
||||
{
|
||||
name = "nginx";
|
||||
image = "nginx:1.14.2";
|
||||
ports = [
|
||||
{
|
||||
containerPort = 80;
|
||||
name = "http-web-svc";
|
||||
}
|
||||
];
|
||||
}
|
||||
];
|
||||
};
|
||||
ports = [
|
||||
{
|
||||
name = "name-of-service-port";
|
||||
protocol = "TCP";
|
||||
port = 80;
|
||||
targetPort = "http-web-svc";
|
||||
}
|
||||
];
|
||||
};
|
||||
}
|
||||
];
|
||||
}
|
||||
{
|
||||
apiVersion = "v1";
|
||||
kind = "Service";
|
||||
metadata = {
|
||||
name = "nginx-service";
|
||||
};
|
||||
spec = {
|
||||
selector = {
|
||||
"app.kubernetes.io/name" = "MyApp";
|
||||
};
|
||||
ports = [
|
||||
{
|
||||
name = "name-of-service-port";
|
||||
protocol = "TCP";
|
||||
port = 80;
|
||||
targetPort = "http-web-svc";
|
||||
}
|
||||
];
|
||||
};
|
||||
}
|
||||
];
|
||||
};
|
||||
'';
|
||||
description = ''
|
||||
Auto-deploying manifests that are linked to {file}`${manifestDir}` before k3s starts.
|
||||
|
@ -337,10 +622,9 @@ in
|
|||
Packaged Helm charts that are linked to {file}`${chartDir}` before k3s starts.
|
||||
The attribute name will be used as the link target (relative to {file}`${chartDir}`).
|
||||
The specified charts will only be placed on the file system and made available to the
|
||||
Kubernetes APIServer from within the cluster, you may use the
|
||||
[k3s Helm controller](https://docs.k3s.io/helm#using-the-helm-controller)
|
||||
to deploy the charts. This option only makes sense on server nodes
|
||||
(`role = server`).
|
||||
Kubernetes APIServer from within the cluster. See the [](#opt-services.k3s.autoDeployCharts)
|
||||
option and the [k3s Helm controller docs](https://docs.k3s.io/helm#using-the-helm-controller)
|
||||
to deploy Helm charts. This option only makes sense on server nodes (`role = server`).
|
||||
'';
|
||||
};
|
||||
|
||||
|
@ -450,6 +734,53 @@ in
|
|||
set the `clientConnection.kubeconfig` if you want to use `extraKubeProxyConfig`.
|
||||
'';
|
||||
};
|
||||
|
||||
autoDeployCharts = lib.mkOption {
|
||||
type = lib.types.attrsOf autoDeployChartsModule;
|
||||
apply = lib.mapAttrs mkAutoDeployChartManifest;
|
||||
default = { };
|
||||
example = lib.literalExpression ''
|
||||
{
|
||||
harbor = {
|
||||
name = "harbor";
|
||||
repo = "https://helm.goharbor.io";
|
||||
version = "1.14.0";
|
||||
hash = "sha256-fMP7q1MIbvzPGS9My91vbQ1d3OJMjwc+o8YE/BXZaYU=";
|
||||
values = {
|
||||
existingSecretAdminPassword = "harbor-admin";
|
||||
expose = {
|
||||
tls = {
|
||||
enabled = true;
|
||||
certSource = "secret";
|
||||
secret.secretName = "my-tls-secret";
|
||||
};
|
||||
ingress = {
|
||||
hosts.core = "example.com";
|
||||
className = "nginx";
|
||||
};
|
||||
};
|
||||
};
|
||||
};
|
||||
|
||||
custom-chart = {
|
||||
package = ../charts/my-chart.tgz;
|
||||
values = ../values/my-values.yaml;
|
||||
extraFieldDefinitions = {
|
||||
spec.timeout = "60s";
|
||||
};
|
||||
};
|
||||
}
|
||||
'';
|
||||
description = ''
|
||||
Auto deploying Helm charts that are installed by the k3s Helm controller. Avoid to use
|
||||
attribute names that are also used in the [](#opt-services.k3s.manifests) and
|
||||
[](#opt-services.k3s.charts) options. Manifests with the same name will override
|
||||
auto deploying charts with the same name. Similiarly, charts with the same name will
|
||||
overwrite the Helm chart contained in auto deploying charts. This option only makes
|
||||
sense on server nodes (`role = server`). See the
|
||||
[k3s Helm documentation](https://docs.k3s.io/helm) for further information.
|
||||
'';
|
||||
};
|
||||
};
|
||||
|
||||
# implementation
|
||||
|
@ -462,6 +793,15 @@ in
|
|||
++ (lib.optional (cfg.role != "server" && cfg.charts != { })
|
||||
"k3s: Helm charts are only made available to the cluster on server nodes (role == server), they will be ignored by this node."
|
||||
)
|
||||
++ (lib.optional (cfg.role != "server" && cfg.autoDeployCharts != { })
|
||||
"k3s: Auto deploying Helm charts are only installed on server nodes (role == server), they will be ignored by this node."
|
||||
)
|
||||
++ (lib.optional (duplicateManifests != [ ])
|
||||
"k3s: The following auto deploying charts are overriden by manifests of the same name: ${toString duplicateManifests}."
|
||||
)
|
||||
++ (lib.optional (duplicateCharts != [ ])
|
||||
"k3s: The following auto deploying charts are overriden by charts of the same name: ${toString duplicateCharts}."
|
||||
)
|
||||
++ (lib.optional (
|
||||
cfg.disableAgent && cfg.images != [ ]
|
||||
) "k3s: Images are only imported on nodes with an enabled agent, they will be ignored by this node")
|
||||
|
|
135
nixos/tests/k3s/auto-deploy-charts.nix
Normal file
135
nixos/tests/k3s/auto-deploy-charts.nix
Normal file
|
@ -0,0 +1,135 @@
|
|||
# Tests whether container images are imported and auto deploying Helm charts work
|
||||
import ../make-test-python.nix (
|
||||
{
|
||||
k3s,
|
||||
lib,
|
||||
pkgs,
|
||||
...
|
||||
}:
|
||||
let
|
||||
testImageEnv = pkgs.buildEnv {
|
||||
name = "k3s-pause-image-env";
|
||||
paths = with pkgs; [
|
||||
busybox
|
||||
hello
|
||||
];
|
||||
};
|
||||
testImage = pkgs.dockerTools.buildImage {
|
||||
name = "test.local/test";
|
||||
tag = "local";
|
||||
# Slightly reduces the time needed to import image
|
||||
compressor = "zstd";
|
||||
copyToRoot = testImageEnv;
|
||||
};
|
||||
# pack the test helm chart as a .tgz archive
|
||||
package =
|
||||
pkgs.runCommand "k3s-test-chart.tgz"
|
||||
{
|
||||
nativeBuildInputs = [ pkgs.kubernetes-helm ];
|
||||
}
|
||||
''
|
||||
helm package ${./k3s-test-chart}
|
||||
mv ./*.tgz $out
|
||||
'';
|
||||
# The common Helm chart that is used in this test
|
||||
testChart = {
|
||||
inherit package;
|
||||
values = {
|
||||
runCommand = "hello";
|
||||
image = {
|
||||
repository = testImage.imageName;
|
||||
tag = testImage.imageTag;
|
||||
};
|
||||
};
|
||||
};
|
||||
in
|
||||
{
|
||||
name = "${k3s.name}-auto-deploy-helm";
|
||||
meta.maintainers = lib.teams.k3s.members;
|
||||
nodes.machine =
|
||||
{ pkgs, ... }:
|
||||
{
|
||||
# k3s uses enough resources the default vm fails.
|
||||
virtualisation = {
|
||||
memorySize = 1536;
|
||||
diskSize = 4096;
|
||||
};
|
||||
environment.systemPackages = [ pkgs.yq-go ];
|
||||
services.k3s = {
|
||||
enable = true;
|
||||
package = k3s;
|
||||
# Slightly reduce resource usage
|
||||
extraFlags = [
|
||||
"--disable coredns"
|
||||
"--disable local-storage"
|
||||
"--disable metrics-server"
|
||||
"--disable servicelb"
|
||||
"--disable traefik"
|
||||
];
|
||||
images = [
|
||||
# Provides the k3s Helm controller
|
||||
k3s.airgapImages
|
||||
testImage
|
||||
];
|
||||
autoDeployCharts = {
|
||||
# regular test chart that should get installed
|
||||
hello = testChart;
|
||||
# disabled chart that should not get installed
|
||||
disabled = testChart // {
|
||||
enable = false;
|
||||
};
|
||||
# advanced chart that should get installed in the "test" namespace with a custom
|
||||
# timeout and overridden values
|
||||
advanced = testChart // {
|
||||
# create the "test" namespace via extraDeploy for testing
|
||||
extraDeploy = [
|
||||
{
|
||||
apiVersion = "v1";
|
||||
kind = "Namespace";
|
||||
metadata.name = "test";
|
||||
}
|
||||
];
|
||||
extraFieldDefinitions = {
|
||||
spec = {
|
||||
# overwrite chart values
|
||||
valuesContent = ''
|
||||
runCommand: "echo 'advanced hello'"
|
||||
image:
|
||||
repository: ${testImage.imageName}
|
||||
tag: ${testImage.imageTag}
|
||||
'';
|
||||
# overwrite the chart namespace
|
||||
targetNamespace = "test";
|
||||
# set a custom timeout
|
||||
timeout = "69s";
|
||||
};
|
||||
};
|
||||
};
|
||||
};
|
||||
};
|
||||
};
|
||||
|
||||
testScript = # python
|
||||
''
|
||||
import json
|
||||
|
||||
machine.wait_for_unit("k3s")
|
||||
# check existence/absence of chart manifest files
|
||||
machine.succeed("test -e /var/lib/rancher/k3s/server/manifests/hello.yaml")
|
||||
machine.succeed("test ! -e /var/lib/rancher/k3s/server/manifests/disabled.yaml")
|
||||
machine.succeed("test -e /var/lib/rancher/k3s/server/manifests/advanced.yaml")
|
||||
# check that the timeout is set correctly, select only the first doc in advanced.yaml
|
||||
advancedManifest = json.loads(machine.succeed("yq -o json 'select(di == 0)' /var/lib/rancher/k3s/server/manifests/advanced.yaml"))
|
||||
assert advancedManifest["spec"]["timeout"] == "69s", f"unexpected value for spec.timeout: {advancedManifest["spec"]["timeout"]}"
|
||||
# wait for test jobs to complete
|
||||
machine.wait_until_succeeds("kubectl wait --for=condition=complete job/hello", timeout=180)
|
||||
machine.wait_until_succeeds("kubectl -n test wait --for=condition=complete job/advanced", timeout=180)
|
||||
# check output of test jobs
|
||||
hello_output = machine.succeed("kubectl logs -l batch.kubernetes.io/job-name=hello")
|
||||
advanced_output = machine.succeed("kubectl -n test logs -l batch.kubernetes.io/job-name=advanced")
|
||||
# strip the output to remove trailing whitespaces
|
||||
assert hello_output.rstrip() == "Hello, world!", f"unexpected output of hello job: {hello_output}"
|
||||
assert advanced_output.rstrip() == "advanced hello", f"unexpected output of advanced job: {advanced_output}"
|
||||
'';
|
||||
}
|
||||
)
|
|
@ -11,6 +11,9 @@ in
|
|||
_: k3s: import ./airgap-images.nix { inherit system pkgs k3s; }
|
||||
) allK3s;
|
||||
auto-deploy = lib.mapAttrs (_: k3s: import ./auto-deploy.nix { inherit system pkgs k3s; }) allK3s;
|
||||
auto-deploy-charts = lib.mapAttrs (
|
||||
_: k3s: import ./auto-deploy-charts.nix { inherit system pkgs k3s; }
|
||||
) allK3s;
|
||||
containerd-config = lib.mapAttrs (
|
||||
_: k3s: import ./containerd-config.nix { inherit system pkgs k3s; }
|
||||
) allK3s;
|
||||
|
|
24
nixos/tests/k3s/k3s-test-chart/Chart.yaml
Normal file
24
nixos/tests/k3s/k3s-test-chart/Chart.yaml
Normal file
|
@ -0,0 +1,24 @@
|
|||
apiVersion: v2
|
||||
name: k3s-test-chart
|
||||
description: A Helm chart that is used in k3s NixOS tests.
|
||||
|
||||
# A chart can be either an 'application' or a 'library' chart.
|
||||
#
|
||||
# Application charts are a collection of templates that can be packaged into versioned archives
|
||||
# to be deployed.
|
||||
#
|
||||
# Library charts provide useful utilities or functions for the chart developer. They're included as
|
||||
# a dependency of application charts to inject those utilities and functions into the rendering
|
||||
# pipeline. Library charts do not define any templates and therefore cannot be deployed.
|
||||
type: application
|
||||
|
||||
# This is the chart version. This version number should be incremented each time you make changes
|
||||
# to the chart and its templates, including the app version.
|
||||
# Versions are expected to follow Semantic Versioning (https://semver.org/)
|
||||
version: 0.1.0
|
||||
|
||||
# This is the version number of the application being deployed. This version number should be
|
||||
# incremented each time you make changes to the application. Versions are not expected to
|
||||
# follow Semantic Versioning. They should reflect the version the application is using.
|
||||
# It is recommended to use it with quotes.
|
||||
appVersion: "1.16.0"
|
14
nixos/tests/k3s/k3s-test-chart/templates/job.yaml
Normal file
14
nixos/tests/k3s/k3s-test-chart/templates/job.yaml
Normal file
|
@ -0,0 +1,14 @@
|
|||
apiVersion: batch/v1
|
||||
kind: Job
|
||||
metadata:
|
||||
name: {{ .Release.Name | quote }}
|
||||
namespace: {{ .Release.Namespace | quote }}
|
||||
spec:
|
||||
template:
|
||||
spec:
|
||||
containers:
|
||||
- name: test
|
||||
image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}"
|
||||
command: ["sh"]
|
||||
args: ["-c", "{{ .Values.runCommand }}"]
|
||||
restartPolicy: {{ .Values.restartPolicy | quote }}
|
5
nixos/tests/k3s/k3s-test-chart/values.yaml
Normal file
5
nixos/tests/k3s/k3s-test-chart/values.yaml
Normal file
|
@ -0,0 +1,5 @@
|
|||
restartPolicy: "Never"
|
||||
runCommand: ""
|
||||
image:
|
||||
repository: foo
|
||||
tag: 1.0.0
|
Loading…
Add table
Add a link
Reference in a new issue