jetbrains: add and improve tests (#403994)

pkgs/applications/editors/jetbrains/bin/darwin.nix

@@ -15,7 +15,7 @@
   plugins ? [ ],
   buildNumber,
   ...
-}:
+}@args:
 
 let
   loname = lib.toLower productShort;
@@ -29,6 +29,7 @@ stdenvNoCC.mkDerivation {
     ;
   passthru.buildNumber = buildNumber;
   passthru.product = product;
+  passthru.tests = args.passthru.tests;
   meta = meta // {
     mainProgram = loname;
   };

pkgs/applications/editors/jetbrains/bin/linux.nix

@@ -35,6 +35,7 @@
   extraLdPath ? [ ],
   extraWrapperArgs ? [ ],
   extraBuildInputs ? [ ],
+  ...
 }@args:
 
 let
@@ -48,6 +49,7 @@ lib.makeOverridable mkDerivation (
   rec {
     inherit pname version src;
     passthru.buildNumber = buildNumber;
+    passthru.tests = args.passthru.tests;
     meta = args.meta // {
       mainProgram = pname;
     };

pkgs/applications/editors/jetbrains/default.nix

@@ -77,6 +77,7 @@ let
       extraWrapperArgs ? [ ],
       extraLdPath ? [ ],
       extraBuildInputs ? [ ],
+      extraTests ? { },
     }:
     mkJetBrainsProductCore {
       inherit
@@ -100,6 +101,9 @@ let
       inherit (ideInfo."${pname}") wmClass product;
       productShort = ideInfo."${pname}".productShort or ideInfo."${pname}".product;
       meta = mkMeta ideInfo."${pname}".meta fromSource;
+      passthru.tests = extraTests // {
+        plugins = callPackage ./plugins/tests.nix { ideName = pname; };
+      };
       libdbm =
         if ideInfo."${pname}".meta.isOpenSource then
           communitySources."${pname}".libdbm

pkgs/applications/editors/jetbrains/plugins/tests.nix

@@ -1,36 +1,135 @@
-{ jetbrains, writeText }:
-
 {
-  # Check to see if the process for adding plugins is breaking anything, instead of the plugins themselves
-  default =
+  jetbrains,
+  symlinkJoin,
+  lib,
+  runCommand,
+  # If not set, all IDEs are tested.
+  ideName ? null,
+}:
+
 let
-      modify-ide = ide: jetbrains.plugins.addPlugins ide [ ];
+
+  # Known broken plugins, PLEASE remove entries here whenever possible.
+  broken-plugins = [
+    "github-copilot" # GitHub Copilot: https://github.com/NixOS/nixpkgs/issues/400317
+  ];
+
   ides =
+    if ideName == null then
       with jetbrains;
-        map modify-ide [
+      [
+        aqua
         clion
         datagrip
         dataspell
+        gateway
         goland
-          idea-community
+        idea-community-src
+        idea-community-bin
         idea-ultimate
         mps
         phpstorm
-          pycharm-community
+        pycharm-community-src
+        pycharm-community-bin
         pycharm-professional
         rider
         ruby-mine
         rust-rover
         webstorm
-        ];
-      paths = builtins.concatStringsSep " " ides;
+        writerside
+      ]
+    else
+      [ (jetbrains.${ideName}) ];
 in
-    writeText "jb-ides" paths;
+{
+  # Check to see if the process for adding plugins is breaking anything, instead of the plugins themselves
+  empty =
+    let
+      modify-ide = ide: jetbrains.plugins.addPlugins ide [ ];
+    in
+    symlinkJoin {
+      name = "jetbrains-test-plugins-empty";
+      paths = (map modify-ide ides);
+    };
 
-  idea-ce-with-plugins = jetbrains.plugins.addPlugins jetbrains.idea-community [
-    "ideavim"
-    "nixidea"
-    # test JAR plugins
-    "wakatime"
+  # Test all plugins. This will only build plugins compatible with the IDE and version. It will fail if the plugin is marked
+  # as compatible, but the build version is somehow not in the "builds" map (as that would indicate that something with update_plugins.py went wrong).
+  all =
+    let
+      plugins-json = builtins.fromJSON (builtins.readFile ./plugins.json);
+      plugins-for =
+        with lib.asserts;
+        ide:
+        builtins.map (plugin: plugin.name) (
+          builtins.filter (
+            plugin:
+            (
+              # Plugin has to not be broken
+              (!builtins.elem plugin.name broken-plugins)
+              # IDE has to be compatible
+              && (builtins.elem ide.pname plugin.compatible)
+              # Assert: The build number needs to be included (if marked compatible)
+              && (assertMsg (builtins.elem ide.buildNumber (builtins.attrNames plugin.builds)) "For plugin ${plugin.name} no entry for IDE build ${ide.buildNumber} is defined, even though ${ide.pname} is on that build.")
+              # The plugin has to exist for the build
+              && (plugin.builds.${ide.buildNumber} != null)
+            )
+          ) (builtins.attrValues plugins-json.plugins)
+        );
+      modify-ide = ide: jetbrains.plugins.addPlugins ide (plugins-for ide);
+    in
+    symlinkJoin {
+      name = "jetbrains-test-plugins-all";
+      paths = (map modify-ide ides);
+    };
+
+  # This test builds the IDEs with some plugins and checks that they can be discovered by the IDE.
+  # Test always succeeds on IDEs that the tested plugins don't support.
+  stored-correctly =
+    let
+      plugins-json = builtins.fromJSON (builtins.readFile ./plugins.json);
+      plugin-ids = [
+        # This is a "normal plugin", it's output must be linked into /${pname}/plugins.
+        "8607" # nixidea
+        # This is a plugin where the output contains a single JAR file. This JAR file needs to be linked directly in /${pname}/plugins.
+        "7425" # wakatime
       ];
+      check-if-supported =
+        ide:
+        builtins.all (
+          plugin:
+          (builtins.elem ide.pname plugins-json.plugins.${plugin}.compatible)
+          && (plugins-json.plugins.${plugin}.builds.${ide.buildNumber} != null)
+        ) plugin-ids;
+      modify-ide = ide: jetbrains.plugins.addPlugins ide plugin-ids;
+    in
+    runCommand "test-jetbrains-plugins-stored-correctly"
+      {
+        idePaths = (map modify-ide (builtins.filter check-if-supported ides));
+      }
+      # TODO: instead of globbing using $ide/*/plugins we could probably somehow get the package name here properly.
+      ''
+        set -e
+        exec &> >(tee -a "$out")
+
+        IFS=' ' read -ra ideArray <<< "$idePaths"
+        for ide in "''${ideArray[@]}"; do
+          echo "processing $ide"
+
+          echo "> ensure normal plugin is available"
+          (
+            set -x
+            find -L $ide/*/plugins -type f -iname 'NixIDEA-*.jar' | grep .
+          )
+
+          echo "> ensure single JAR file plugin is available"
+          (
+            set -x
+            PATH_TO_LINK=$(find $ide/*/plugins -maxdepth 1 -type l -iname '*wakatime.jar' | grep .)
+            test -f $(readlink $PATH_TO_LINK)
+          )
+          echo ""
+        done
+
+        echo "test done! ok!"
+      '';
 }

pkgs/applications/editors/jetbrains/readme.md

@@ -1,12 +1,19 @@
 This directory contains the build expressions needed to build any of the jetbrains IDEs.
 The jdk is in `pkgs/development/compilers/jetbrains-jdk`.
-To test the build process of every IDE (as well as the process for adding plugins), build `jetbrains.plugins.tests.default`.
+
+## Tests:
+- To test the build process of every IDE (as well as the process for adding plugins), build `jetbrains.plugins.tests.empty`.
+- To test the build process with all plugins\* supported by all IDEs, build `jetbrains.plugins.tests.all`.
+- To test only plugins for a specific IDE\*, build `jetbrains.ide-name.tests.plugins.all`.
+- To test that plugins are correctly stored in the plugins directory, build `jetbrains.plugins.tests.stored-correctly`.
+
+\*: Plugins marked as broken in nixpkgs are skipped: When updating/fixing plugins, please check the `broken-plugins` in `plugins/tests.nix` and update it if needed.
 
 ## How to use plugins:
  - Get the ide you want and call `jetbrains.plugins.addPlugins` with a list of plugins you want to add.
  - The list of plugins can be a list of ids or names (as in `plugins/plugins.json`)
  - Example: `jetbrains.plugins.addPlugins jetbrains.pycharm-professional [ "nixidea" ]`
- - The list can also contain a drv giving a `.jar` or `.zip` (this is how you use a plugin not added to nixpkgs)
+ - The list can also contain drvs giving the directory contents of the plugin (this is how you use a plugin not added to nixpkgs) or a single `.jar` (executable). For an example, look at the implementation of `fetchPluginSrc` in `plugins/default.nix`.
 
 ### How to add a new plugin to nixpkgs
  - Find the page for the plugin on https://plugins.jetbrains.com