Merge pull request #144174 from mkg20001/lxdimage

nixos/modules/module-list.nix

@@ -772,6 +772,7 @@
   ./services/networking/libreswan.nix
   ./services/networking/lldpd.nix
   ./services/networking/logmein-hamachi.nix
+  ./services/networking/lxd-image-server.nix
   ./services/networking/mailpile.nix
   ./services/networking/magic-wormhole-mailbox-server.nix
   ./services/networking/matterbridge.nix

nixos/modules/services/networking/lxd-image-server.nix

@@ -0,0 +1,138 @@
+{ config, pkgs, lib, ... }:
+
+with lib;
+
+let
+  cfg = config.services.lxd-image-server;
+  format = pkgs.formats.toml {};
+
+  location = "/var/www/simplestreams";
+in
+{
+  options = {
+    services.lxd-image-server = {
+      enable = mkEnableOption "lxd-image-server";
+
+      group = mkOption {
+        type = types.str;
+        description = "Group assigned to the user and the webroot directory.";
+        default = "nginx";
+        example = "www-data";
+      };
+
+      settings = mkOption {
+        type = format.type;
+        description = ''
+          Configuration for lxd-image-server.
+
+          Example see <link xlink:href="https://github.com/Avature/lxd-image-server/blob/master/config.toml"/>.
+        '';
+        default = {};
+      };
+
+      nginx = {
+        enable = mkEnableOption "nginx";
+        domain = mkOption {
+          type = types.str;
+          description = "Domain to use for nginx virtual host.";
+          example = "images.example.org";
+        };
+      };
+    };
+  };
+
+  config = mkMerge [
+    (mkIf (cfg.enable) {
+      users.users.lxd-image-server = {
+        isSystemUser = true;
+        group = cfg.group;
+      };
+      users.groups.${cfg.group} = {};
+
+      environment.etc."lxd-image-server/config.toml".source = format.generate "config.toml" cfg.settings;
+
+      services.logrotate.paths.lxd-image-server = {
+        path = "/var/log/lxd-image-server/lxd-image-server.log";
+        frequency = "daily";
+        keep = 21;
+        user = "lxd-image-server";
+        group = cfg.group;
+        extraConfig = ''
+          missingok
+          compress
+          delaycompress
+          copytruncate
+          notifempty
+        '';
+      };
+
+      systemd.tmpfiles.rules = [
+        "d /var/www/simplestreams 0755 lxd-image-server ${cfg.group}"
+      ];
+
+      systemd.services.lxd-image-server = {
+        wantedBy = [ "multi-user.target" ];
+        after = [ "network.target" ];
+
+        description = "LXD Image Server";
+
+        script = ''
+          ${pkgs.lxd-image-server}/bin/lxd-image-server init
+          ${pkgs.lxd-image-server}/bin/lxd-image-server watch
+        '';
+
+        serviceConfig = {
+          User = "lxd-image-server";
+          Group = cfg.group;
+          DynamicUser = true;
+          LogsDirectory = "lxd-image-server";
+          RuntimeDirectory = "lxd-image-server";
+          ExecReload = "${pkgs.lxd-image-server}/bin/lxd-image-server reload";
+          ReadWritePaths = [ location ];
+        };
+      };
+    })
+    # this is seperate so it can be enabled on mirrored hosts
+    (mkIf (cfg.nginx.enable) {
+      # https://github.com/Avature/lxd-image-server/blob/master/resources/nginx/includes/lxd-image-server.pkg.conf
+      services.nginx.virtualHosts = {
+        "${cfg.nginx.domain}" = {
+          forceSSL = true;
+          enableACME = mkDefault true;
+
+          root = location;
+
+          locations = {
+            "/streams/v1/" = {
+              index = "index.json";
+            };
+
+            # Serve json files with content type header application/json
+            "~ \.json$" = {
+              extraConfig = ''
+                add_header Content-Type application/json;
+              '';
+            };
+
+            "~ \.tar.xz$" = {
+              extraConfig = ''
+                add_header Content-Type application/octet-stream;
+              '';
+            };
+
+            "~ \.tar.gz$" = {
+              extraConfig = ''
+                add_header Content-Type application/octet-stream;
+              '';
+            };
+
+            # Deny access to document root and the images folder
+            "~ ^/(images/)?$" = {
+              return = "403";
+            };
+          };
+        };
+      };
+    })
+  ];
+}

nixos/tests/all-tests.nix

@@ -238,6 +238,7 @@ in
   lxd = handleTest ./lxd.nix {};
   lxd-image = handleTest ./lxd-image.nix {};
   lxd-nftables = handleTest ./lxd-nftables.nix {};
+  lxd-image-server = handleTest ./lxd-image-server.nix {};
   #logstash = handleTest ./logstash.nix {};
   lorri = handleTest ./lorri/default.nix {};
   magic-wormhole-mailbox-server = handleTest ./magic-wormhole-mailbox-server.nix {};

nixos/tests/lxd-image-server.nix

@@ -0,0 +1,127 @@
+import ./make-test-python.nix ({ pkgs, ...} :
+
+let
+  # Since we don't have access to the internet during the tests, we have to
+  # pre-fetch lxd containers beforehand.
+  #
+  # I've chosen to import Alpine Linux, because its image is turbo-tiny and,
+  # generally, sufficient for our tests.
+  alpine-meta = pkgs.fetchurl {
+    url = "https://tarballs.nixos.org/alpine/3.12/lxd.tar.xz";
+    hash = "sha256-1tcKaO9lOkvqfmG/7FMbfAEToAuFy2YMewS8ysBKuLA=";
+  };
+
+  alpine-rootfs = pkgs.fetchurl {
+    url = "https://tarballs.nixos.org/alpine/3.12/rootfs.tar.xz";
+    hash = "sha256-Tba9sSoaiMtQLY45u7p5DMqXTSDgs/763L/SQp0bkCA=";
+  };
+
+  lxd-config = pkgs.writeText "config.yaml" ''
+    storage_pools:
+      - name: default
+        driver: dir
+        config:
+          source: /var/lxd-pool
+
+    networks:
+      - name: lxdbr0
+        type: bridge
+        config:
+          ipv4.address: auto
+          ipv6.address: none
+
+    profiles:
+      - name: default
+        devices:
+          eth0:
+            name: eth0
+            network: lxdbr0
+            type: nic
+          root:
+            path: /
+            pool: default
+            type: disk
+  '';
+
+
+in {
+  name = "lxd-image-server";
+
+  meta = with pkgs.lib.maintainers; {
+    maintainers = [ mkg20001 ];
+  };
+
+  machine = { lib, ... }: {
+    virtualisation = {
+      cores = 2;
+
+      memorySize = 2048;
+      diskSize = 4096;
+
+      lxc.lxcfs.enable = true;
+      lxd.enable = true;
+    };
+
+    security.pki.certificates = [
+      (builtins.readFile ./common/acme/server/ca.cert.pem)
+    ];
+
+    services.nginx = {
+      enable = true;
+    };
+
+    services.lxd-image-server = {
+      enable = true;
+      nginx = {
+        enable = true;
+        domain = "acme.test";
+      };
+    };
+
+    services.nginx.virtualHosts."acme.test" = {
+      enableACME = false;
+      sslCertificate = ./common/acme/server/acme.test.cert.pem;
+      sslCertificateKey = ./common/acme/server/acme.test.key.pem;
+    };
+
+    networking.hosts = {
+      "::1" = [ "acme.test" ];
+    };
+  };
+
+  testScript = ''
+    machine.wait_for_unit("sockets.target")
+    machine.wait_for_unit("lxd.service")
+    machine.wait_for_file("/var/lib/lxd/unix.socket")
+
+    # It takes additional second for lxd to settle
+    machine.sleep(1)
+
+    # lxd expects the pool's directory to already exist
+    machine.succeed("mkdir /var/lxd-pool")
+
+
+    machine.succeed(
+        "cat ${lxd-config} | lxd init --preseed"
+    )
+
+    machine.succeed(
+        "lxc image import ${alpine-meta} ${alpine-rootfs} --alias alpine"
+    )
+
+    loc = "/var/www/simplestreams/images/iats/alpine/amd64/default/v1"
+
+    with subtest("push image to server"):
+        machine.succeed("lxc launch alpine test")
+        machine.succeed("lxc stop test")
+        machine.succeed("lxc publish --public test --alias=testimg")
+        machine.succeed("lxc image export testimg")
+        machine.succeed("ls >&2")
+        machine.succeed("mkdir -p " + loc)
+        machine.succeed("mv *.tar.gz " + loc)
+
+    with subtest("pull image from server"):
+        machine.succeed("lxc remote add img https://acme.test --protocol=simplestreams")
+        machine.succeed("lxc image list img: >&2")
+  '';
+})

pkgs/development/python-modules/confight/default.nix

@@ -0,0 +1,30 @@
+{ lib
+, buildPythonPackage
+, fetchPypi
+, toml
+}:
+
+buildPythonPackage rec {
+  pname = "confight";
+  version = "1.3.1";
+
+  src = fetchPypi {
+    inherit pname version;
+    sha256 = "sha256-fJr7f9Y/zEpCedWYd04AMuhkOFqZLJOw4sDiz8SDQ/Y=";
+  };
+
+  propagatedBuildInputs = [
+    toml
+  ];
+
+  pythonImportsCheck = [ "confight" ];
+
+  doCheck = false;
+
+  meta = with lib; {
+    description = "Python context manager for managing pid files";
+    homepage = "https://github.com/avature/confight";
+    license = with licenses; [ mit ];
+    maintainers = with maintainers; [ mkg20001 ];
+  };
+}

pkgs/development/python-modules/inotify/default.nix

@@ -0,0 +1,32 @@
+{ lib
+, buildPythonPackage
+, fetchFromGitHub
+, nose
+}:
+
+buildPythonPackage rec {
+  pname = "inotify";
+  version = "unstable-2020-08-27";
+
+  src = fetchFromGitHub {
+    owner = "dsoprea";
+    repo = "PyInotify";
+    rev = "f77596ae965e47124f38d7bd6587365924dcd8f7";
+    sha256 = "X0gu4s1R/Kg+tmf6s8SdZBab2HisJl4FxfdwKktubVc=";
+    fetchSubmodules = false;
+  };
+
+  checkInputs = [
+    nose
+  ];
+
+  # dunno what's wrong but the module works regardless
+  doCheck = false;
+
+  meta = with lib; {
+    homepage = "https://github.com/dsoprea/PyInotify";
+    description = "Monitor filesystems events on Linux platforms with inotify";
+    license = licenses.gpl2;
+    platforms = platforms.linux;
+  };
+}

pkgs/tools/virtualization/lxd-image-server/default.nix

@@ -0,0 +1,47 @@
+{ lib
+, openssl
+, rsync
+, python3
+, fetchFromGitHub
+}:
+
+python3.pkgs.buildPythonApplication rec {
+  pname = "lxd-image-server";
+  version = "0.0.4";
+
+  src = fetchFromGitHub {
+    owner = "Avature";
+    repo = "lxd-image-server";
+    rev = version;
+    sha256 = "yx8aUmMfSzyWaM6M7+WcL6ouuWwOpqLzODWSdNgwCwo=";
+  };
+
+  patches = [
+    ./state.patch
+    ./run.patch
+  ];
+
+  propagatedBuildInputs = with python3.pkgs; [
+    setuptools
+    attrs
+    click
+    inotify
+    cryptography
+    confight
+    python-pidfile
+  ];
+
+  makeWrapperArgs = [
+    ''--prefix PATH ':' "${lib.makeBinPath [ openssl rsync ]}"''
+  ];
+
+  doCheck = false;
+
+  meta = with lib; {
+    description = "Creates and manages a simplestreams lxd image server on top of nginx";
+    homepage = "https://github.com/Avature/lxd-image-server";
+    license = licenses.apsl20;
+    platforms = platforms.unix;
+    maintainers = with maintainers; [ mkg20001 ];
+  };
+}

pkgs/tools/virtualization/lxd-image-server/run.patch

@@ -0,0 +1,25 @@
+From df2ce9fb48a3790407646a388e0d220a75496c52 Mon Sep 17 00:00:00 2001
+From: =?UTF-8?q?Maciej=20Kr=C3=BCger?= <mkg20001@gmail.com>
+Date: Wed, 3 Nov 2021 14:23:38 +0100
+Subject: [PATCH] /var/run -> /run
+
+---
+ lxd_image_server/tools/config.py | 2 +-
+ 1 file changed, 1 insertion(+), 1 deletion(-)
+
+diff --git a/lxd_image_server/tools/config.py b/lxd_image_server/tools/config.py
+index 60e8973..23d392a 100644
+--- a/lxd_image_server/tools/config.py
++++ b/lxd_image_server/tools/config.py
+@@ -9,7 +9,7 @@ import confight
+ class Config():
+ 
+     _lock = Lock()
+-    pidfile = Path('/var/run/lxd-image-server/pidfile')
++    pidfile = Path('/run/lxd-image-server/pidfile')
+     data = {}
+ 
+     @classmethod
+-- 
+2.33.0
+

pkgs/tools/virtualization/lxd-image-server/state.patch

@@ -0,0 +1,49 @@
+From 17a1e09eaf8957174425d05200be9ee3e77229f9 Mon Sep 17 00:00:00 2001
+From: =?UTF-8?q?Maciej=20Kr=C3=BCger?= <mkg20001@gmail.com>
+Date: Thu, 21 Oct 2021 00:39:08 +0200
+Subject: [PATCH] Remove system-state changing code
+
+This is already done by the module on nixOS
+---
+ lxd_image_server/cli.py | 15 +--------------
+ 1 file changed, 1 insertion(+), 14 deletions(-)
+
+diff --git a/lxd_image_server/cli.py b/lxd_image_server/cli.py
+index d276e6d..f759bf2 100644
+--- a/lxd_image_server/cli.py
++++ b/lxd_image_server/cli.py
+@@ -140,30 +140,17 @@ def reload_config():
+ @cli.command()
+ @click.option('--root_dir', default='/var/www/simplestreams',
+               show_default=True)
+-@click.option('--ssl_dir', default='/etc/nginx/ssl', show_default=True,
+-              callback=lambda ctx, param, val: Path(val))
+ @click.pass_context
+-def init(ctx, root_dir, ssl_dir):
++def init(ctx, root_dir):
+     if not Path(root_dir).exists():
+         logger.error('Root directory does not exists')
+     else:
+-        if not ssl_dir.exists():
+-            os.makedirs(str(ssl_dir))
+-
+-        if not (ssl_dir / 'nginx.key').exists():
+-            generate_cert(str(ssl_dir))
+-
+         img_dir = str(Path(root_dir, 'images'))
+         streams_dir = str(Path(root_dir, 'streams/v1'))
+         if not Path(img_dir).exists():
+             os.makedirs(img_dir)
+         if not Path(streams_dir).exists():
+             os.makedirs(streams_dir)
+-        conf_path = Path('/etc/nginx/sites-enabled/simplestreams.conf')
+-        if not conf_path.exists():
+-            conf_path.symlink_to(
+-                '/etc/nginx/sites-available/simplestreams.conf')
+-            os.system('nginx -s reload')
+ 
+         if not Path(root_dir, 'streams', 'v1', 'images.json').exists():
+             ctx.invoke(update, img_dir=Path(root_dir, 'images'),
+-- 
+2.33.0
+

pkgs/top-level/all-packages.nix

@@ -7352,6 +7352,8 @@ with pkgs;
   lxcfs = callPackage ../os-specific/linux/lxcfs { };
   lxd = callPackage ../tools/admin/lxd { };
 
+  lxd-image-server = callPackage ../tools/virtualization/lxd-image-server { };
+
   lzfse = callPackage ../tools/compression/lzfse { };
 
   lzham = callPackage ../tools/compression/lzham { };

pkgs/top-level/python-packages.nix

@@ -1705,6 +1705,8 @@ in {
 
   confuse = callPackage ../development/python-modules/confuse { };
 
+  confight = callPackage ../development/python-modules/confight { };
+
   connexion = callPackage ../development/python-modules/connexion { };
 
   consonance = callPackage ../development/python-modules/consonance { };
@@ -3770,6 +3772,8 @@ in {
 
   inkex = callPackage ../development/python-modules/inkex { };
 
+  inotify = callPackage ../development/python-modules/inotify { };
+
   inotify-simple = callPackage ../development/python-modules/inotify-simple { };
 
   inotifyrecursive = callPackage ../development/python-modules/inotifyrecursive { };