nixos/testing: Improve option docs

doc/stdenv/meta.chapter.md

@@ -213,6 +213,10 @@ runCommand "my-package-test" {
 
 A timeout (in seconds) for building the derivation. If the derivation takes longer than this time to build, it can fail due to breaking the timeout. However, all computers do not have the same computing power, hence some builders may decide to apply a multiplicative factor to this value. When filling this value in, try to keep it approximately consistent with other values already present in `nixpkgs`.
 
+`meta` attributes are not stored in the instantiated derivation.
+Therefore, this setting may be lost when the package is used as a dependency.
+To be effective, it must be presented directly to an evaluation process that handles the `meta.timeout` attribute.
+
 ### `hydraPlatforms` {#var-meta-hydraPlatforms}
 
 The list of Nix platform types for which the Hydra instance at `hydra.nixos.org` will build the package. (Hydra is the Nix-based continuous build system.) It defaults to the value of `meta.platforms`. Thus, the only reason to set `meta.hydraPlatforms` is if you want `hydra.nixos.org` to build the package on a subset of `meta.platforms`, or not at all, e.g.

nixos/doc/manual/default.nix

@@ -13,6 +13,8 @@
 with pkgs;
 
 let
+  inherit (lib) hasPrefix removePrefix;
+
   lib = pkgs.lib;
 
   docbook_xsl_ns = pkgs.docbook-xsl-ns.override {

nixos/doc/manual/development/writing-nixos-tests.section.md

@@ -22,12 +22,12 @@ A NixOS test is a module that has the following structure:
 ```
 
 We refer to the whole test above as a test module, whereas the values
-in `nodes.<name>` are NixOS modules. (A NixOS configuration is a module.)
+in [`nodes.<name>`](#opt-nodes) are NixOS modules themselves.
 
-The option `testScript` is a bit of Python code that executes the
+The option [`testScript`](#opt-testScript) is a piece of Python code that executes the
 test (described below). During the test, it will start one or more
 virtual machines, the configuration of which is described by
-the attribute `nodes`.
+the option [`nodes`](#opt-nodes).
 
 An example of a single-node test is
 [`login.nix`](https://github.com/NixOS/nixpkgs/blob/master/nixos/tests/login.nix).
@@ -58,7 +58,7 @@ For the purpose of constructing a test matrix, use the `matrix` options instead.
   hostname = runTest { imports = [ ./hostname.nix ]; defaults.networking.firewall.enable = false; };
 ```
 
-You can run a test with attribute name `mytest` in `all-tests.nix` by invoking:
+You can run a test with attribute name `mytest` in `nixos/tests/all-tests.nix` by invoking:
 
 ```shell
 nix-build -A nixosTests.mytest
@@ -181,7 +181,7 @@ The following methods are available on machine objects:
     least one will be returned.
 
     ::: {.note}
-    This requires passing `enableOCR` to the test attribute set.
+    This requires [`enableOCR`](#opt-enableOCR) to be set to `true`.
     :::
 
 `get_screen_text`
@@ -190,7 +190,7 @@ The following methods are available on machine objects:
     machine\'s screen using optical character recognition.
 
     ::: {.note}
-    This requires passing `enableOCR` to the test attribute set.
+    This requires [`enableOCR`](#opt-enableOCR) to be set to `true`.
     :::
 
 `send_monitor_command`
@@ -301,7 +301,7 @@ The following methods are available on machine objects:
     `get_screen_text` and `get_screen_text_variants`).
 
     ::: {.note}
-    This requires passing `enableOCR` to the test attribute set.
+    This requires [`enableOCR`](#opt-enableOCR) to be set to `true`.
     :::
 
 `wait_for_console_text`

nixos/doc/manual/from_md/development/writing-nixos-tests.section.xml

@@ -23,14 +23,17 @@
 </programlisting>
   <para>
     We refer to the whole test above as a test module, whereas the
-    values in <literal>nodes.&lt;name&gt;</literal> are NixOS modules.
+    values in
-    (A NixOS configuration is a module.)
+    <link linkend="opt-nodes"><literal>nodes.&lt;name&gt;</literal></link>
+    are NixOS modules themselves.
   </para>
   <para>
-    The option <literal>testScript</literal> is a bit of Python code
+    The option
-    that executes the test (described below). During the test, it will
+    <link linkend="opt-testScript"><literal>testScript</literal></link>
-    start one or more virtual machines, the configuration of which is
+    is a piece of Python code that executes the test (described below).
-    described by the attribute <literal>nodes</literal>.
+    During the test, it will start one or more virtual machines, the
+    configuration of which is described by the option
+    <link linkend="opt-nodes"><literal>nodes</literal></link>.
   </para>
   <para>
     An example of a single-node test is
@@ -73,7 +76,7 @@
 </programlisting>
       <para>
         You can run a test with attribute name <literal>mytest</literal>
-        in <literal>all-tests.nix</literal> by invoking:
+        in <literal>nixos/tests/all-tests.nix</literal> by invoking:
       </para>
       <programlisting>
 nix-build -A nixosTests.mytest
@@ -270,8 +273,9 @@ start_all()
           </para>
           <note>
             <para>
-              This requires passing <literal>enableOCR</literal> to the
+              This requires
-              test attribute set.
+              <link linkend="opt-enableOCR"><literal>enableOCR</literal></link>
+              to be set to <literal>true</literal>.
             </para>
           </note>
         </listitem>
@@ -287,8 +291,9 @@ start_all()
           </para>
           <note>
             <para>
-              This requires passing <literal>enableOCR</literal> to the
+              This requires
-              test attribute set.
+              <link linkend="opt-enableOCR"><literal>enableOCR</literal></link>
+              to be set to <literal>true</literal>.
             </para>
           </note>
         </listitem>
@@ -527,8 +532,9 @@ start_all()
           </para>
           <note>
             <para>
-              This requires passing <literal>enableOCR</literal> to the
+              This requires
-              test attribute set.
+              <link linkend="opt-enableOCR"><literal>enableOCR</literal></link>
+              to be set to <literal>true</literal>.
             </para>
           </note>
         </listitem>

nixos/lib/testing/driver.nix

@@ -1,6 +1,6 @@
 { config, lib, hostPkgs, ... }:
 let
-  inherit (lib) mkOption types;
+  inherit (lib) mkOption types literalMD mdDoc;
 
   # Reifies and correctly wraps the python test driver for
   # the respective qemu version and with or without ocr support
@@ -106,13 +106,13 @@ in
   options = {
 
     driver = mkOption {
-      description = "Script that runs the test.";
+      description = mdDoc "Package containing a script that runs the test.";
       type = types.package;
-      defaultText = lib.literalDocBook "set by the test framework";
+      defaultText = literalMD "set by the test framework";
     };
 
     hostPkgs = mkOption {
-      description = "Nixpkgs attrset used outside the nodes.";
+      description = mdDoc "Nixpkgs attrset used outside the nodes.";
       type = types.raw;
       example = lib.literalExpression ''
         import nixpkgs { inherit system config overlays; }
@@ -120,34 +120,39 @@ in
     };
 
     qemu.package = mkOption {
-      description = "Which qemu package to use.";
+      description = mdDoc "Which qemu package to use for the virtualisation of [{option}`nodes`](#opt-nodes).";
       type = types.package;
       default = hostPkgs.qemu_test;
       defaultText = "hostPkgs.qemu_test";
     };
 
     enableOCR = mkOption {
-      description = ''
+      description = mdDoc ''
         Whether to enable Optical Character Recognition functionality for
-        testing graphical programs.
+        testing graphical programs. See [Machine objects](`ssec-machine-objects`).
       '';
       type = types.bool;
       default = false;
     };
 
     extraPythonPackages = mkOption {
-      description = ''
+      description = mdDoc ''
         Python packages to add to the test driver.
 
         The argument is a Python package set, similar to `pkgs.pythonPackages`.
       '';
+      example = lib.literalExpression ''
+        p: [ p.numpy ]
+      '';
       type = types.functionTo (types.listOf types.package);
       default = ps: [ ];
     };
 
     extraDriverArgs = mkOption {
-      description = ''
+      description = mdDoc ''
         Extra arguments to pass to the test driver.
+
+        They become part of [{option}`driver`](#opt-driver) via `wrapProgram`.
       '';
       type = types.listOf types.str;
       default = [];
@@ -156,11 +161,19 @@ in
     skipLint = mkOption {
       type = types.bool;
       default = false;
+      description = mdDoc ''
+        Do not run the linters. This may speed up your iteration cycle, but it is not something you should commit.
+      '';
     };
 
     skipTypeCheck = mkOption {
       type = types.bool;
       default = false;
+      description = mdDoc ''
+        Disable type checking. This must not be enabled for new NixOS tests.
+
+        This may speed up your iteration cycle, unless you're working on the [{option}`testScript`](#opt-testScript).
+      '';
     };
   };
 

nixos/lib/testing/interactive.nix

@@ -1,12 +1,19 @@
 { config, lib, moduleType, hostPkgs, ... }:
 let
-  inherit (lib) mkOption types;
+  inherit (lib) mkOption types mdDoc;
 in
 {
   options = {
     interactive = mkOption {
-      description = "All the same options, but configured for interactive use.";
+      description = mdDoc ''
+        Tests [can be run interactively](#sec-running-nixos-tests-interactively).
+
+        When they are, the configuration will include anything set in this submodule.
+
+        You can set any top-level test option here.
+      '';
       type = moduleType;
+      visible = "shallow";
     };
   };
 

nixos/lib/testing/meta.nix

@@ -1,24 +1,38 @@
 { lib, ... }:
 let
-  inherit (lib) types mkOption;
+  inherit (lib) types mkOption mdDoc;
 in
 {
   options = {
     meta = lib.mkOption {
+      description = mdDoc ''
+        The [`meta`](https://nixos.org/manual/nixpkgs/stable/#chap-meta) attributes that will be set on the returned derivations.
+
+        Not all [`meta`](https://nixos.org/manual/nixpkgs/stable/#chap-meta) attributes are supported, but more can be added as desired.
+      '';
       apply = lib.filterAttrs (k: v: v != null);
       type = types.submodule {
         options = {
           maintainers = lib.mkOption {
             type = types.listOf types.raw;
             default = [];
+            description = mdDoc ''
+              The [list of maintainers](https://nixos.org/manual/nixpkgs/stable/#var-meta-maintainers) for this test.
+            '';
           };
           timeout = lib.mkOption {
             type = types.nullOr types.int;
-            default = null;
+            default = null; # NOTE: null values are filtered out by `meta`.
+            description = mdDoc ''
+              The [{option}`test`](#opt-test)'s [`meta.timeout`](https://nixos.org/manual/nixpkgs/stable/#var-meta-timeout) in seconds.
+            '';
           };
           broken = lib.mkOption {
             type = types.bool;
             default = false;
+            description = mdDoc ''
+              Sets the [`meta.broken`](https://nixos.org/manual/nixpkgs/stable/#var-meta-broken) attribute on the [{option}`test`](#opt-test) derivation.
+            '';
           };
         };
       };

nixos/lib/testing/name.nix

@@ -1,7 +1,14 @@
 { lib, ... }:
+let
+  inherit (lib) mkOption types mdDoc;
+in
 {
-  options.name = lib.mkOption {
+  options.name = mkOption {
-    description = "The name of the test.";
+    description = mdDoc ''
-    type = lib.types.str;
+      The name of the test.
+
+      This is used in the derivation names of the [{option}`driver`](#opt-driver) and [{option}`test`](#opt-test) runner.
+    '';
+    type = types.str;
   };
 }

nixos/lib/testing/network.nix

@@ -5,6 +5,7 @@ let
     attrNames concatMap concatMapStrings flip forEach head
     listToAttrs mkDefault mkOption nameValuePair optionalString
     range types zipListsWith zipLists
+    mdDoc
     ;
 
   nodeNumbers =
@@ -74,7 +75,7 @@ let
         default = name;
         # We need to force this in specilisations, otherwise it'd be
         # readOnly = true;
-        description = ''
+        description = mdDoc ''
           The `name` in `nodes.<name>`; stable across `specialisations`.
         '';
       };
@@ -83,7 +84,7 @@ let
         type = types.int;
         readOnly = true;
         default = nodeNumbers.${config.virtualisation.test.nodeName};
-        description = ''
+        description = mdDoc ''
           A unique number assigned for each node in `nodes`.
         '';
       };

nixos/lib/testing/nodes.nix

@@ -1,7 +1,7 @@
 testModuleArgs@{ config, lib, hostPkgs, nodes, ... }:
 
 let
-  inherit (lib) mkOption mkForce optional types mapAttrs mkDefault;
+  inherit (lib) mkOption mkForce optional types mapAttrs mkDefault mdDoc;
 
   system = hostPkgs.stdenv.hostPlatform.system;
 
@@ -39,11 +39,29 @@ in
 
     nodes = mkOption {
       type = types.lazyAttrsOf config.node.type;
+      visible = "shallow";
+      description = mdDoc ''
+        An attribute set of NixOS configuration modules.
+
+        The configurations are augmented by the [`defaults`](#opt-defaults) option.
+
+        They are assigned network addresses according to the `nixos/lib/testing/network.nix` module.
+
+        A few special options are available, that aren't in a plain NixOS configuration. See [Configuring the nodes](#sec-nixos-test-nodes)
+      '';
     };
 
     defaults = mkOption {
-      description = ''
+      description = mdDoc ''
-        NixOS configuration that is applied to all {option}`nodes`.
+        NixOS configuration that is applied to all [{option}`nodes`](#opt-nodes).
+      '';
+      type = types.deferredModule;
+      default = { };
+    };
+
+    extraBaseModules = mkOption {
+      description = mdDoc ''
+        NixOS configuration that, like [{option}`defaults`](#opt-defaults), is applied to all [{option}`nodes`](#opt-nodes) and can not be undone with [`specialisation.<name>.inheritParentConfig`](https://search.nixos.org/options?show=specialisation.%3Cname%3E.inheritParentConfig&from=0&size=50&sort=relevance&type=packages&query=specialisation).
       '';
       type = types.deferredModule;
       default = { };
@@ -52,15 +70,28 @@ in
     node.specialArgs = mkOption {
       type = types.lazyAttrsOf types.raw;
       default = { };
+      description = mdDoc ''
+        An attribute set of arbitrary values that will be made available as module arguments during the resolution of module `imports`.
+
+        Note that it is not possible to override these from within the NixOS configurations. If you argument is not relevant to `imports`, consider setting {option}`defaults._module.args.<name>` instead.
+      '';
     };
 
     minimal = mkOption {
       type = types.bool;
       default = false;
+      description = mdDoc ''
+        Enable to configure all [{option}`nodes`](#opt-nodes) to run with a minimal kernel.
+      '';
     };
 
     nodesCompat = mkOption {
       internal = true;
+      description = mdDoc ''
+        Basically `_module.args.nodes`, but with backcompat and warnings added.
+
+        This will go away.
+      '';
     };
   };
 

nixos/lib/testing/run.nix

@@ -1,12 +1,12 @@
 { config, hostPkgs, lib, ... }:
 let
-  inherit (lib) types mkOption;
+  inherit (lib) types mkOption mdDoc;
 in
 {
   options = {
     passthru = mkOption {
       type = types.lazyAttrsOf types.raw;
-      description = ''
+      description = mdDoc ''
         Attributes to add to the returned derivations,
         which are not necessarily part of the build.
 
@@ -18,8 +18,12 @@ in
 
     test = mkOption {
       type = types.package;
-      description = ''
+      # TODO: can the interactive driver be configured to access the network?
+      description = mdDoc ''
         Derivation that runs the test as its "build" process.
+
+        This implies that NixOS tests run isolated from the network, making them
+        more dependable.
       '';
     };
   };

nixos/lib/testing/testScript.nix

@@ -1,12 +1,16 @@
 testModuleArgs@{ config, lib, hostPkgs, nodes, moduleType, ... }:
 let
-  inherit (lib) mkOption types;
+  inherit (lib) mkOption types mdDoc;
   inherit (types) either str functionTo;
 in
 {
   options = {
     testScript = mkOption {
       type = either str (functionTo str);
+      description = ''
+        A series of python declarations and statements that you write to perform
+        the test.
+      '';
     };
     testScriptString = mkOption {
       type = str;
@@ -21,9 +25,11 @@ in
     };
     withoutTestScriptReferences = mkOption {
       type = moduleType;
-      description = ''
+      description = mdDoc ''
         A parallel universe where the testScript is invalid and has no references.
       '';
+      internal = true;
+      visible = false;
     };
   };
   config = {