Merge pull request #149532 from pennae/split-docs-build

nixos/*: split docs build

nixos/modules/misc/documentation.nix

@@ -1,19 +1,35 @@
-{ config, lib, pkgs, extendModules, noUserModules, ... }:
+{ config, options, lib, pkgs, utils, modules, baseModules, extraModules, modulesPath, ... }:
 
 with lib;
 
 let
 
   cfg = config.documentation;
+  allOpts = options;
 
   /* Modules for which to show options even when not imported. */
   extraDocModules = [ ../virtualisation/qemu-vm.nix ];
 
-  /* For the purpose of generating docs, evaluate options with each derivation
-    in `pkgs` (recursively) replaced by a fake with path "\${pkgs.attribute.path}".
-    It isn't perfect, but it seems to cover a vast majority of use cases.
-    Caveat: even if the package is reached by a different means,
-    the path above will be shown and not e.g. `${config.services.foo.package}`. */
+  canCacheDocs = m:
+    let
+      f = import m;
+      instance = f (mapAttrs (n: _: abort "evaluating ${n} for `meta` failed") (functionArgs f));
+    in
+      cfg.nixos.options.splitBuild
+        && builtins.isPath m
+        && isFunction f
+        && instance ? options
+        && instance.meta.buildDocsInSandbox or true;
+
+  docModules =
+    let
+      p = partition canCacheDocs (baseModules ++ extraDocModules);
+    in
+      {
+        lazy = p.right;
+        eager = p.wrong ++ optionals cfg.nixos.includeAllModules (extraModules ++ modules);
+      };
+
   manual = import ../../doc/manual rec {
     inherit pkgs config;
     version = config.system.nixos.release;
@@ -21,10 +37,17 @@ let
     extraSources = cfg.nixos.extraModuleSources;
     options =
       let
-        extendNixOS = if cfg.nixos.includeAllModules then extendModules else noUserModules.extendModules;
-        scrubbedEval = extendNixOS {
-          modules = extraDocModules;
-          specialArgs.pkgs = scrubDerivations "pkgs" pkgs;
+        scrubbedEval = evalModules {
+          modules = [ {
+            _module.check = false;
+          } ] ++ docModules.eager;
+          specialArgs = {
+            pkgs = scrubDerivations "pkgs" pkgs;
+            # allow access to arbitrary options for eager modules, eg for getting
+            # option types from lazy modules
+            options = allOpts;
+            inherit modulesPath utils;
+          };
         };
         scrubDerivations = namePrefix: pkgSet: mapAttrs
           (name: value:
@@ -36,6 +59,53 @@ let
           )
           pkgSet;
       in scrubbedEval.options;
+    baseOptionsJSON =
+      let
+        filter =
+          builtins.filterSource
+            (n: t:
+              (t == "directory" -> baseNameOf n != "tests")
+              && (t == "file" -> hasSuffix ".nix" n)
+            );
+        pull = dir:
+          if isStorePath pkgs.path
+          then "${builtins.storePath pkgs.path}/${dir}"
+          else filter "${toString pkgs.path}/${dir}";
+      in
+        pkgs.runCommand "lazy-options.json" {
+          libPath = pull "lib";
+          pkgsLibPath = pull "pkgs/pkgs-lib";
+          nixosPath = pull "nixos";
+          modules = map (p: ''"${removePrefix "${modulesPath}/" (toString p)}"'') docModules.lazy;
+        } ''
+          export NIX_STORE_DIR=$TMPDIR/store
+          export NIX_STATE_DIR=$TMPDIR/state
+          ${pkgs.nix}/bin/nix-instantiate \
+            --show-trace \
+            --eval --json --strict \
+            --argstr libPath "$libPath" \
+            --argstr pkgsLibPath "$pkgsLibPath" \
+            --argstr nixosPath "$nixosPath" \
+            --arg modules "[ $modules ]" \
+            --argstr stateVersion "${options.system.stateVersion.default}" \
+            --argstr release "${config.system.nixos.release}" \
+            $nixosPath/lib/eval-cacheable-options.nix > $out \
+            || {
+              echo -en "\e[1;31m"
+              echo 'Cacheable portion of option doc build failed.'
+              echo 'Usually this means that an option attribute that ends up in documentation (eg' \
+                '`default` or `description`) depends on the restricted module arguments' \
+                '`config` or `pkgs`.'
+              echo
+              echo 'Rebuild your configuration with `--show-trace` to find the offending' \
+                'location. Remove the references to restricted arguments (eg by escaping' \
+                'their antiquotations or adding a `defaultText`) or disable the sandboxed' \
+                'build for the failing module by setting `meta.buildDocsInSandbox = false`.'
+              echo -en "\e[0m"
+              exit 1
+            } >&2
+        '';
+    inherit (cfg.nixos.options) warningsAreErrors;
   };
 
 
@@ -178,6 +248,25 @@ in
         '';
       };
 
+      nixos.options.splitBuild = mkOption {
+        type = types.bool;
+        default = true;
+        description = ''
+          Whether to split the option docs build into a cacheable and an uncacheable part.
+          Splitting the build can substantially decrease the amount of time needed to build
+          the manual, but some user modules may be incompatible with this splitting.
+        '';
+      };
+
+      nixos.options.warningsAreErrors = mkOption {
+        type = types.bool;
+        default = true;
+        description = ''
+          Treat warning emitted during the option documentation build (eg for missing option
+          descriptions) as errors.
+        '';
+      };
+
       nixos.includeAllModules = mkOption {
         type = types.bool;
         default = false;