From c256d7b5ecd90d2a12ff6de378bdcb6c6607afbe Mon Sep 17 00:00:00 2001
From: Robert Rose <robert.rose@mailbox.org>
Date: Tue, 21 Jan 2025 17:37:05 +0100
Subject: [PATCH] nixos/k3s: use dynamic networking in multi node test

The dynamic setup requires less network configuration and allows to use
features of the nixos testing driver, e.g. port forwarding and SSH
access to nodes in interactive mode.
---
 nixos/tests/k3s/multi-node.nix | 49 ++++++++++------------------------
 1 file changed, 14 insertions(+), 35 deletions(-)

diff --git a/nixos/tests/k3s/multi-node.nix b/nixos/tests/k3s/multi-node.nix
index c1d89af2f3e2..335b8f8e6426 100644
--- a/nixos/tests/k3s/multi-node.nix
+++ b/nixos/tests/k3s/multi-node.nix
@@ -60,7 +60,7 @@ import ../make-test-python.nix (
 
     nodes = {
       server =
-        { pkgs, ... }:
+        { nodes, pkgs, ... }:
         {
           environment.systemPackages = with pkgs; [
             gzip
@@ -83,8 +83,12 @@ import ../make-test-python.nix (
               "--disable metrics-server"
               "--disable servicelb"
               "--disable traefik"
-              "--node-ip 192.168.1.1"
               "--pause-image test.local/pause:local"
+              "--node-ip ${nodes.server.networking.primaryIPAddress}"
+              # The interface selection logic of flannel would normally use eth0, as the nixos
+              # testing driver sets a default route via dev eth0. However, in test setups we
+              # have to use eth1 for inter-node communication.
+              "--flannel-iface eth1"
             ];
           };
           networking.firewall.allowedTCPPorts = [
@@ -93,19 +97,10 @@ import ../make-test-python.nix (
             6443
           ];
           networking.firewall.allowedUDPPorts = [ 8472 ];
-          networking.firewall.trustedInterfaces = [ "flannel.1" ];
-          networking.useDHCP = false;
-          networking.defaultGateway = "192.168.1.1";
-          networking.interfaces.eth1.ipv4.addresses = pkgs.lib.mkForce [
-            {
-              address = "192.168.1.1";
-              prefixLength = 24;
-            }
-          ];
         };
 
       server2 =
-        { pkgs, ... }:
+        { nodes, pkgs, ... }:
         {
           environment.systemPackages = with pkgs; [
             gzip
@@ -119,7 +114,7 @@ import ../make-test-python.nix (
             enable = true;
             package = k3s;
             images = [ pauseImage ];
-            serverAddr = "https://192.168.1.1:6443";
+            serverAddr = "https://${nodes.server.networking.primaryIPAddress}:6443";
             clusterInit = false;
             extraFlags = [
               "--disable coredns"
@@ -127,8 +122,9 @@ import ../make-test-python.nix (
               "--disable metrics-server"
               "--disable servicelb"
               "--disable traefik"
-              "--node-ip 192.168.1.3"
               "--pause-image test.local/pause:local"
+              "--node-ip ${nodes.server2.networking.primaryIPAddress}"
+              "--flannel-iface eth1"
             ];
           };
           networking.firewall.allowedTCPPorts = [
@@ -137,19 +133,10 @@ import ../make-test-python.nix (
             6443
           ];
           networking.firewall.allowedUDPPorts = [ 8472 ];
-          networking.firewall.trustedInterfaces = [ "flannel.1" ];
-          networking.useDHCP = false;
-          networking.defaultGateway = "192.168.1.3";
-          networking.interfaces.eth1.ipv4.addresses = pkgs.lib.mkForce [
-            {
-              address = "192.168.1.3";
-              prefixLength = 24;
-            }
-          ];
         };
 
       agent =
-        { pkgs, ... }:
+        { nodes, pkgs, ... }:
         {
           virtualisation.memorySize = 1024;
           virtualisation.diskSize = 2048;
@@ -159,23 +146,15 @@ import ../make-test-python.nix (
             role = "agent";
             package = k3s;
             images = [ pauseImage ];
-            serverAddr = "https://192.168.1.3:6443";
+            serverAddr = "https://${nodes.server2.networking.primaryIPAddress}:6443";
             extraFlags = [
               "--pause-image test.local/pause:local"
-              "--node-ip 192.168.1.2"
+              "--node-ip ${nodes.agent.networking.primaryIPAddress}"
+              "--flannel-iface eth1"
             ];
           };
           networking.firewall.allowedTCPPorts = [ 6443 ];
           networking.firewall.allowedUDPPorts = [ 8472 ];
-          networking.firewall.trustedInterfaces = [ "flannel.1" ];
-          networking.useDHCP = false;
-          networking.defaultGateway = "192.168.1.2";
-          networking.interfaces.eth1.ipv4.addresses = pkgs.lib.mkForce [
-            {
-              address = "192.168.1.2";
-              prefixLength = 24;
-            }
-          ];
         };
     };