nixos/invidious: add option to run more invidious instances

nixos/modules/services/web-apps/invidious.nix

@@ -10,20 +10,58 @@ let
   generatedHmacKeyFile = "/var/lib/invidious/hmac_key";
   generateHmac = cfg.hmacKeyFile == null;
 
-  serviceConfig = {
+  commonInvidousServiceConfig = {
-    systemd.services.invidious = {
     description = "Invidious (An alternative YouTube front-end)";
     wants = [ "network-online.target" ];
-      after = [ "network-online.target" ];
+    after = [ "network-online.target" ] ++ lib.optional cfg.database.createLocally "postgresql.service";
+    requires = lib.optional cfg.database.createLocally "postgresql.service";
     wantedBy = [ "multi-user.target" ];
 
+    serviceConfig = {
+      RestartSec = "2s";
+      DynamicUser = true;
+      User = lib.mkIf (cfg.database.createLocally || cfg.serviceScale > 1) "invidious";
+      StateDirectory = "invidious";
+      StateDirectoryMode = "0750";
+
+      CapabilityBoundingSet = "";
+      PrivateDevices = true;
+      PrivateUsers = true;
+      ProtectHome = true;
+      ProtectKernelLogs = true;
+      ProtectProc = "invisible";
+      RestrictAddressFamilies = [ "AF_UNIX" "AF_INET" "AF_INET6" ];
+      RestrictNamespaces = true;
+      SystemCallArchitectures = "native";
+      SystemCallFilter = [ "@system-service" "~@privileged" "~@resources" ];
+
+      # Because of various issues Invidious must be restarted often, at least once a day, ideally
+      # every hour.
+      # This option enables the automatic restarting of the Invidious instance.
+      # To ensure multiple instances of Invidious are not restarted at the exact same time, a
+      # randomized extra offset of up to 5 minutes is added.
+      Restart = lib.mkDefault "always";
+      RuntimeMaxSec = lib.mkDefault "1h";
+      RuntimeRandomizedExtraSec = lib.mkDefault "5min";
+    };
+  };
+  mkInvidiousService = scaleIndex:
+    lib.foldl' lib.recursiveUpdate commonInvidousServiceConfig [
+      # only generate the hmac file in the first service
+      (lib.optionalAttrs (scaleIndex == 0) {
         preStart = lib.optionalString generateHmac ''
           if [[ ! -e "${generatedHmacKeyFile}" ]]; then
             ${pkgs.pwgen}/bin/pwgen 20 1 > "${generatedHmacKeyFile}"
             chmod 0600 "${generatedHmacKeyFile}"
           fi
         '';
-
+      })
+      # configure the secondary services to run after the first service
+      (lib.optionalAttrs (scaleIndex > 0) {
+        after = commonInvidousServiceConfig.after ++ [ "invidious.service" ];
+        wants = commonInvidousServiceConfig.wants ++ [ "invidious.service" ];
+      })
+      {
         script = ''
           configParts=()
         ''
@@ -47,40 +85,31 @@ let
         + lib.optionalString (cfg.hmacKeyFile != null) ''
           configParts+=("$(< ${lib.escapeShellArg cfg.hmacKeyFile})")
         ''
+        # configure threads for secondary instances
+        + lib.optionalString (scaleIndex > 0) ''
+          configParts+=('{"channel_threads":0, "feed_threads":0}')
+        ''
+        # configure different ports for the instances
+        + ''
+          configParts+=('{"port":${toString (cfg.port + scaleIndex)}}')
+        ''
         # merge all parts into a single configuration with later elements overriding previous elements
         + ''
           export INVIDIOUS_CONFIG="$(${pkgs.jq}/bin/jq -s 'reduce .[] as $item ({}; . * $item)' <<<"''${configParts[*]}")"
           exec ${cfg.package}/bin/invidious
         '';
+      }
+    ];
 
   serviceConfig = {
-        RestartSec = "2s";
+    systemd.services = builtins.listToAttrs (builtins.genList
-        DynamicUser = true;
+      (scaleIndex: {
-        StateDirectory = "invidious";
+        name = "invidious" + lib.optionalString (scaleIndex > 0) "-${builtins.toString scaleIndex}";
-        StateDirectoryMode = "0750";
+        value = mkInvidiousService scaleIndex;
-
+      })
-        CapabilityBoundingSet = "";
+      cfg.serviceScale);
-        PrivateDevices = true;
-        PrivateUsers = true;
-        ProtectHome = true;
-        ProtectKernelLogs = true;
-        ProtectProc = "invisible";
-        RestrictAddressFamilies = [ "AF_UNIX" "AF_INET" "AF_INET6" ];
-        RestrictNamespaces = true;
-        SystemCallArchitectures = "native";
-        SystemCallFilter = [ "@system-service" "~@privileged" "~@resources" ];
-
-        # Because of various issues Invidious must be restarted often, at least once a day, ideally
-        # every hour.
-        # This option enables the automatic restarting of the Invidious instance.
-        Restart = lib.mkDefault "always";
-        RuntimeMaxSec = lib.mkDefault "1h";
-      };
-    };
 
     services.invidious.settings = {
-      inherit (cfg) port;
-
       # Automatically initialises and migrates the database if necessary
       check_tables = true;
 
@@ -98,10 +127,16 @@ let
       inherit (cfg) domain;
     });
 
-    assertions = [{
+    assertions = [
+      {
         assertion = cfg.database.host != null -> cfg.database.passwordFile != null;
         message = "If database host isn't null, database password needs to be set";
-    }];
+      }
+      {
+        assertion = cfg.serviceScale >= 1;
+        message = "Service can't be scaled below one instance";
+      }
+    ];
   };
 
   # Settings necessary for running with an automatically managed local database
@@ -132,15 +167,6 @@ let
         local ${cfg.settings.db.dbname} ${cfg.settings.db.user} peer map=invidious
       '';
     };
-
-    systemd.services.invidious = {
-      requires = [ "postgresql.service" ];
-      after = [ "postgresql.service" ];
-
-      serviceConfig = {
-        User = "invidious";
-      };
-    };
   };
 
   nginxConfig = lib.mkIf cfg.nginx.enable {
@@ -152,11 +178,22 @@ let
     services.nginx = {
       enable = true;
       virtualHosts.${cfg.domain} = {
-        locations."/".proxyPass = "http://127.0.0.1:${toString cfg.port}";
+        locations."/".proxyPass =
+          if cfg.serviceScale == 1 then
+            "http://127.0.0.1:${toString cfg.port}"
+          else "http://upstream-invidious";
 
         enableACME = lib.mkDefault true;
         forceSSL = lib.mkDefault true;
       };
+      upstreams = lib.mkIf (cfg.serviceScale > 1) {
+        "upstream-invidious".servers = builtins.listToAttrs (builtins.genList
+          (scaleIndex: {
+            name = "127.0.0.1:${toString (cfg.port + scaleIndex)}";
+            value = { };
+          })
+          cfg.serviceScale);
+      };
     };
 
     assertions = [{
@@ -204,6 +241,20 @@ in
       '';
     };
 
+    serviceScale = lib.mkOption {
+      type = types.int;
+      default = 1;
+      description = lib.mdDoc ''
+        How many invidious instances to run.
+
+        See https://docs.invidious.io/improve-public-instance/#2-multiple-invidious-processes for more details
+        on how this is intended to work. All instances beyond the first one have the options `channel_threads`
+        and `feed_threads` set to 0 to avoid conflicts with multiple instances refreshing subscriptions. Instances
+        will be configured to bind to consecutive ports starting with {option}`services.invidious.port` for the
+        first instance.
+      '';
+    };
+
     # This needs to be outside of settings to avoid infinite recursion
     # (determining if nginx should be enabled and therefore the settings
     # modified).