diff --git a/nixos/tests/all-tests.nix b/nixos/tests/all-tests.nix
index 6ce214cc4ed6..3ff13868939b 100644
--- a/nixos/tests/all-tests.nix
+++ b/nixos/tests/all-tests.nix
@@ -135,7 +135,7 @@ in {
   archi = handleTest ./archi.nix {};
   aria2 = handleTest ./aria2.nix {};
   armagetronad = handleTest ./armagetronad.nix {};
-  artalk = handleTest ./artalk.nix {};
+  artalk = runTest ./artalk.nix;
   atd = handleTest ./atd.nix {};
   atop = handleTest ./atop.nix {};
   atticd = runTest ./atticd.nix;
diff --git a/nixos/tests/artalk.nix b/nixos/tests/artalk.nix
index 1338e5cd380c..336392ed67e3 100644
--- a/nixos/tests/artalk.nix
+++ b/nixos/tests/artalk.nix
@@ -1,28 +1,56 @@
-import ./make-test-python.nix (
-  { lib, pkgs, ... }:
-  {
+{ lib, pkgs, ... }:
+{
 
-    name = "artalk";
+  name = "artalk";
 
-    meta = {
-      maintainers = with lib.maintainers; [ moraxyc ];
-    };
+  meta = {
+    maintainers = with lib.maintainers; [ moraxyc ];
+  };
 
-    nodes.machine =
-      { pkgs, ... }:
-      {
-        environment.systemPackages = [ pkgs.curl ];
-        services.artalk = {
-          enable = true;
+  nodes.machine =
+    { pkgs, ... }:
+    {
+      environment.systemPackages = [
+        pkgs.curl
+        pkgs.artalk
+        pkgs.sudo
+      ];
+      services.artalk = {
+        enable = true;
+        settings = {
+          cache.enabled = true;
+          admin_users = [
+            {
+              name = "admin";
+              email = "admin@example.org";
+              # md5 for 'password'
+              password = "(md5)5F4DCC3B5AA765D61D8327DEB882CF99";
+            }
+          ];
         };
       };
+    };
 
-    testScript = ''
-      machine.wait_for_unit("artalk.service")
+  testScript = ''
+    import json
+    machine.wait_for_unit("artalk.service")
 
-      machine.wait_for_open_port(23366)
+    machine.wait_for_open_port(23366)
 
-      machine.succeed("curl --fail --max-time 10 http://127.0.0.1:23366/")
-    '';
-  }
-)
+    assert '${pkgs.artalk.version}' in machine.succeed("curl --fail --max-time 10 http://127.0.0.1:23366/api/v2/version")
+
+    # Get token
+    result = json.loads(machine.succeed("""
+      curl --fail -X POST --json '{
+              "email": "admin@example.org",
+              "password": "password"
+              }' 'http://127.0.0.1:23366/api/v2/auth/email/login'
+      """))
+    token = result['token']
+
+    # Test admin
+    machine.succeed(f"""
+      curl --fail -X POST --header 'Authorization: {token}' 'http://127.0.0.1:23366/api/v2/cache/flush'
+      """)
+  '';
+}