diff --git a/nixos/modules/module-list.nix b/nixos/modules/module-list.nix index eed0b802b95e..dc152848fb48 100644 --- a/nixos/modules/module-list.nix +++ b/nixos/modules/module-list.nix @@ -772,6 +772,7 @@ ./services/networking/libreswan.nix ./services/networking/lldpd.nix ./services/networking/logmein-hamachi.nix + ./services/networking/lxd-image-server.nix ./services/networking/mailpile.nix ./services/networking/magic-wormhole-mailbox-server.nix ./services/networking/matterbridge.nix diff --git a/nixos/modules/services/networking/lxd-image-server.nix b/nixos/modules/services/networking/lxd-image-server.nix new file mode 100644 index 000000000000..5ec6cacffa49 --- /dev/null +++ b/nixos/modules/services/networking/lxd-image-server.nix @@ -0,0 +1,138 @@ +{ config, pkgs, lib, ... }: + +with lib; + +let + cfg = config.services.lxd-image-server; + format = pkgs.formats.toml {}; + + location = "/var/www/simplestreams"; +in +{ + options = { + services.lxd-image-server = { + enable = mkEnableOption "lxd-image-server"; + + group = mkOption { + type = types.str; + description = "Group assigned to the user and the webroot directory."; + default = "nginx"; + example = "www-data"; + }; + + settings = mkOption { + type = format.type; + description = '' + Configuration for lxd-image-server. + + Example see . + ''; + default = {}; + }; + + nginx = { + enable = mkEnableOption "nginx"; + domain = mkOption { + type = types.str; + description = "Domain to use for nginx virtual host."; + example = "images.example.org"; + }; + }; + }; + }; + + config = mkMerge [ + (mkIf (cfg.enable) { + users.users.lxd-image-server = { + isSystemUser = true; + group = cfg.group; + }; + users.groups.${cfg.group} = {}; + + environment.etc."lxd-image-server/config.toml".source = format.generate "config.toml" cfg.settings; + + services.logrotate.paths.lxd-image-server = { + path = "/var/log/lxd-image-server/lxd-image-server.log"; + frequency = "daily"; + keep = 21; + user = "lxd-image-server"; + group = cfg.group; + extraConfig = '' + missingok + compress + delaycompress + copytruncate + notifempty + ''; + }; + + systemd.tmpfiles.rules = [ + "d /var/www/simplestreams 0755 lxd-image-server ${cfg.group}" + ]; + + systemd.services.lxd-image-server = { + wantedBy = [ "multi-user.target" ]; + after = [ "network.target" ]; + + description = "LXD Image Server"; + + script = '' + ${pkgs.lxd-image-server}/bin/lxd-image-server init + ${pkgs.lxd-image-server}/bin/lxd-image-server watch + ''; + + serviceConfig = { + User = "lxd-image-server"; + Group = cfg.group; + DynamicUser = true; + LogsDirectory = "lxd-image-server"; + RuntimeDirectory = "lxd-image-server"; + ExecReload = "${pkgs.lxd-image-server}/bin/lxd-image-server reload"; + ReadWritePaths = [ location ]; + }; + }; + }) + # this is seperate so it can be enabled on mirrored hosts + (mkIf (cfg.nginx.enable) { + # https://github.com/Avature/lxd-image-server/blob/master/resources/nginx/includes/lxd-image-server.pkg.conf + services.nginx.virtualHosts = { + "${cfg.nginx.domain}" = { + forceSSL = true; + enableACME = mkDefault true; + + root = location; + + locations = { + "/streams/v1/" = { + index = "index.json"; + }; + + # Serve json files with content type header application/json + "~ \.json$" = { + extraConfig = '' + add_header Content-Type application/json; + ''; + }; + + "~ \.tar.xz$" = { + extraConfig = '' + add_header Content-Type application/octet-stream; + ''; + }; + + "~ \.tar.gz$" = { + extraConfig = '' + add_header Content-Type application/octet-stream; + ''; + }; + + # Deny access to document root and the images folder + "~ ^/(images/)?$" = { + return = "403"; + }; + }; + }; + }; + }) + ]; +} diff --git a/nixos/tests/all-tests.nix b/nixos/tests/all-tests.nix index 659e2f9e5699..a48dcda94d7e 100644 --- a/nixos/tests/all-tests.nix +++ b/nixos/tests/all-tests.nix @@ -238,6 +238,7 @@ in lxd = handleTest ./lxd.nix {}; lxd-image = handleTest ./lxd-image.nix {}; lxd-nftables = handleTest ./lxd-nftables.nix {}; + lxd-image-server = handleTest ./lxd-image-server.nix {}; #logstash = handleTest ./logstash.nix {}; lorri = handleTest ./lorri/default.nix {}; magic-wormhole-mailbox-server = handleTest ./magic-wormhole-mailbox-server.nix {}; diff --git a/nixos/tests/lxd-image-server.nix b/nixos/tests/lxd-image-server.nix new file mode 100644 index 000000000000..9f060fed38d8 --- /dev/null +++ b/nixos/tests/lxd-image-server.nix @@ -0,0 +1,127 @@ +import ./make-test-python.nix ({ pkgs, ...} : + +let + # Since we don't have access to the internet during the tests, we have to + # pre-fetch lxd containers beforehand. + # + # I've chosen to import Alpine Linux, because its image is turbo-tiny and, + # generally, sufficient for our tests. + alpine-meta = pkgs.fetchurl { + url = "https://tarballs.nixos.org/alpine/3.12/lxd.tar.xz"; + hash = "sha256-1tcKaO9lOkvqfmG/7FMbfAEToAuFy2YMewS8ysBKuLA="; + }; + + alpine-rootfs = pkgs.fetchurl { + url = "https://tarballs.nixos.org/alpine/3.12/rootfs.tar.xz"; + hash = "sha256-Tba9sSoaiMtQLY45u7p5DMqXTSDgs/763L/SQp0bkCA="; + }; + + lxd-config = pkgs.writeText "config.yaml" '' + storage_pools: + - name: default + driver: dir + config: + source: /var/lxd-pool + + networks: + - name: lxdbr0 + type: bridge + config: + ipv4.address: auto + ipv6.address: none + + profiles: + - name: default + devices: + eth0: + name: eth0 + network: lxdbr0 + type: nic + root: + path: / + pool: default + type: disk + ''; + + +in { + name = "lxd-image-server"; + + meta = with pkgs.lib.maintainers; { + maintainers = [ mkg20001 ]; + }; + + machine = { lib, ... }: { + virtualisation = { + cores = 2; + + memorySize = 2048; + diskSize = 4096; + + lxc.lxcfs.enable = true; + lxd.enable = true; + }; + + security.pki.certificates = [ + (builtins.readFile ./common/acme/server/ca.cert.pem) + ]; + + services.nginx = { + enable = true; + }; + + services.lxd-image-server = { + enable = true; + nginx = { + enable = true; + domain = "acme.test"; + }; + }; + + services.nginx.virtualHosts."acme.test" = { + enableACME = false; + sslCertificate = ./common/acme/server/acme.test.cert.pem; + sslCertificateKey = ./common/acme/server/acme.test.key.pem; + }; + + networking.hosts = { + "::1" = [ "acme.test" ]; + }; + }; + + testScript = '' + machine.wait_for_unit("sockets.target") + machine.wait_for_unit("lxd.service") + machine.wait_for_file("/var/lib/lxd/unix.socket") + + # It takes additional second for lxd to settle + machine.sleep(1) + + # lxd expects the pool's directory to already exist + machine.succeed("mkdir /var/lxd-pool") + + + machine.succeed( + "cat ${lxd-config} | lxd init --preseed" + ) + + machine.succeed( + "lxc image import ${alpine-meta} ${alpine-rootfs} --alias alpine" + ) + + loc = "/var/www/simplestreams/images/iats/alpine/amd64/default/v1" + + with subtest("push image to server"): + machine.succeed("lxc launch alpine test") + machine.succeed("lxc stop test") + machine.succeed("lxc publish --public test --alias=testimg") + machine.succeed("lxc image export testimg") + machine.succeed("ls >&2") + machine.succeed("mkdir -p " + loc) + machine.succeed("mv *.tar.gz " + loc) + + with subtest("pull image from server"): + machine.succeed("lxc remote add img https://acme.test --protocol=simplestreams") + machine.succeed("lxc image list img: >&2") + ''; +}) diff --git a/pkgs/development/python-modules/confight/default.nix b/pkgs/development/python-modules/confight/default.nix new file mode 100644 index 000000000000..ff07a0f8b646 --- /dev/null +++ b/pkgs/development/python-modules/confight/default.nix @@ -0,0 +1,30 @@ +{ lib +, buildPythonPackage +, fetchPypi +, toml +}: + +buildPythonPackage rec { + pname = "confight"; + version = "1.3.1"; + + src = fetchPypi { + inherit pname version; + sha256 = "sha256-fJr7f9Y/zEpCedWYd04AMuhkOFqZLJOw4sDiz8SDQ/Y="; + }; + + propagatedBuildInputs = [ + toml + ]; + + pythonImportsCheck = [ "confight" ]; + + doCheck = false; + + meta = with lib; { + description = "Python context manager for managing pid files"; + homepage = "https://github.com/avature/confight"; + license = with licenses; [ mit ]; + maintainers = with maintainers; [ mkg20001 ]; + }; +} diff --git a/pkgs/development/python-modules/inotify/default.nix b/pkgs/development/python-modules/inotify/default.nix new file mode 100644 index 000000000000..3590f53e1ecd --- /dev/null +++ b/pkgs/development/python-modules/inotify/default.nix @@ -0,0 +1,32 @@ +{ lib +, buildPythonPackage +, fetchFromGitHub +, nose +}: + +buildPythonPackage rec { + pname = "inotify"; + version = "unstable-2020-08-27"; + + src = fetchFromGitHub { + owner = "dsoprea"; + repo = "PyInotify"; + rev = "f77596ae965e47124f38d7bd6587365924dcd8f7"; + sha256 = "X0gu4s1R/Kg+tmf6s8SdZBab2HisJl4FxfdwKktubVc="; + fetchSubmodules = false; + }; + + checkInputs = [ + nose + ]; + + # dunno what's wrong but the module works regardless + doCheck = false; + + meta = with lib; { + homepage = "https://github.com/dsoprea/PyInotify"; + description = "Monitor filesystems events on Linux platforms with inotify"; + license = licenses.gpl2; + platforms = platforms.linux; + }; +} diff --git a/pkgs/tools/virtualization/lxd-image-server/default.nix b/pkgs/tools/virtualization/lxd-image-server/default.nix new file mode 100644 index 000000000000..3992f425a3cd --- /dev/null +++ b/pkgs/tools/virtualization/lxd-image-server/default.nix @@ -0,0 +1,47 @@ +{ lib +, openssl +, rsync +, python3 +, fetchFromGitHub +}: + +python3.pkgs.buildPythonApplication rec { + pname = "lxd-image-server"; + version = "0.0.4"; + + src = fetchFromGitHub { + owner = "Avature"; + repo = "lxd-image-server"; + rev = version; + sha256 = "yx8aUmMfSzyWaM6M7+WcL6ouuWwOpqLzODWSdNgwCwo="; + }; + + patches = [ + ./state.patch + ./run.patch + ]; + + propagatedBuildInputs = with python3.pkgs; [ + setuptools + attrs + click + inotify + cryptography + confight + python-pidfile + ]; + + makeWrapperArgs = [ + ''--prefix PATH ':' "${lib.makeBinPath [ openssl rsync ]}"'' + ]; + + doCheck = false; + + meta = with lib; { + description = "Creates and manages a simplestreams lxd image server on top of nginx"; + homepage = "https://github.com/Avature/lxd-image-server"; + license = licenses.apsl20; + platforms = platforms.unix; + maintainers = with maintainers; [ mkg20001 ]; + }; +} diff --git a/pkgs/tools/virtualization/lxd-image-server/run.patch b/pkgs/tools/virtualization/lxd-image-server/run.patch new file mode 100644 index 000000000000..bd1172c1f864 --- /dev/null +++ b/pkgs/tools/virtualization/lxd-image-server/run.patch @@ -0,0 +1,25 @@ +From df2ce9fb48a3790407646a388e0d220a75496c52 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Maciej=20Kr=C3=BCger?= +Date: Wed, 3 Nov 2021 14:23:38 +0100 +Subject: [PATCH] /var/run -> /run + +--- + lxd_image_server/tools/config.py | 2 +- + 1 file changed, 1 insertion(+), 1 deletion(-) + +diff --git a/lxd_image_server/tools/config.py b/lxd_image_server/tools/config.py +index 60e8973..23d392a 100644 +--- a/lxd_image_server/tools/config.py ++++ b/lxd_image_server/tools/config.py +@@ -9,7 +9,7 @@ import confight + class Config(): + + _lock = Lock() +- pidfile = Path('/var/run/lxd-image-server/pidfile') ++ pidfile = Path('/run/lxd-image-server/pidfile') + data = {} + + @classmethod +-- +2.33.0 + diff --git a/pkgs/tools/virtualization/lxd-image-server/state.patch b/pkgs/tools/virtualization/lxd-image-server/state.patch new file mode 100644 index 000000000000..c6677ea48e9c --- /dev/null +++ b/pkgs/tools/virtualization/lxd-image-server/state.patch @@ -0,0 +1,49 @@ +From 17a1e09eaf8957174425d05200be9ee3e77229f9 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Maciej=20Kr=C3=BCger?= +Date: Thu, 21 Oct 2021 00:39:08 +0200 +Subject: [PATCH] Remove system-state changing code + +This is already done by the module on nixOS +--- + lxd_image_server/cli.py | 15 +-------------- + 1 file changed, 1 insertion(+), 14 deletions(-) + +diff --git a/lxd_image_server/cli.py b/lxd_image_server/cli.py +index d276e6d..f759bf2 100644 +--- a/lxd_image_server/cli.py ++++ b/lxd_image_server/cli.py +@@ -140,30 +140,17 @@ def reload_config(): + @cli.command() + @click.option('--root_dir', default='/var/www/simplestreams', + show_default=True) +-@click.option('--ssl_dir', default='/etc/nginx/ssl', show_default=True, +- callback=lambda ctx, param, val: Path(val)) + @click.pass_context +-def init(ctx, root_dir, ssl_dir): ++def init(ctx, root_dir): + if not Path(root_dir).exists(): + logger.error('Root directory does not exists') + else: +- if not ssl_dir.exists(): +- os.makedirs(str(ssl_dir)) +- +- if not (ssl_dir / 'nginx.key').exists(): +- generate_cert(str(ssl_dir)) +- + img_dir = str(Path(root_dir, 'images')) + streams_dir = str(Path(root_dir, 'streams/v1')) + if not Path(img_dir).exists(): + os.makedirs(img_dir) + if not Path(streams_dir).exists(): + os.makedirs(streams_dir) +- conf_path = Path('/etc/nginx/sites-enabled/simplestreams.conf') +- if not conf_path.exists(): +- conf_path.symlink_to( +- '/etc/nginx/sites-available/simplestreams.conf') +- os.system('nginx -s reload') + + if not Path(root_dir, 'streams', 'v1', 'images.json').exists(): + ctx.invoke(update, img_dir=Path(root_dir, 'images'), +-- +2.33.0 + diff --git a/pkgs/top-level/all-packages.nix b/pkgs/top-level/all-packages.nix index b7caf86fe49f..392763488f16 100644 --- a/pkgs/top-level/all-packages.nix +++ b/pkgs/top-level/all-packages.nix @@ -7352,6 +7352,8 @@ with pkgs; lxcfs = callPackage ../os-specific/linux/lxcfs { }; lxd = callPackage ../tools/admin/lxd { }; + lxd-image-server = callPackage ../tools/virtualization/lxd-image-server { }; + lzfse = callPackage ../tools/compression/lzfse { }; lzham = callPackage ../tools/compression/lzham { }; diff --git a/pkgs/top-level/python-packages.nix b/pkgs/top-level/python-packages.nix index a94b532bfd71..8a6691cf2705 100644 --- a/pkgs/top-level/python-packages.nix +++ b/pkgs/top-level/python-packages.nix @@ -1705,6 +1705,8 @@ in { confuse = callPackage ../development/python-modules/confuse { }; + confight = callPackage ../development/python-modules/confight { }; + connexion = callPackage ../development/python-modules/connexion { }; consonance = callPackage ../development/python-modules/consonance { }; @@ -3770,6 +3772,8 @@ in { inkex = callPackage ../development/python-modules/inkex { }; + inotify = callPackage ../development/python-modules/inotify { }; + inotify-simple = callPackage ../development/python-modules/inotify-simple { }; inotifyrecursive = callPackage ../development/python-modules/inotifyrecursive { };