diff --git a/.github/labeler-protected-branches.yml b/.github/labeler-protected-branches.yml
new file mode 100644
index 000000000000..9e2af0e06bde
--- /dev/null
+++ b/.github/labeler-protected-branches.yml
@@ -0,0 +1,12 @@
+# This file is used by .github/workflows/labels.yml
+# This version is only run for Pull Requests from protected branches like staging-next, haskell-updates or python-updates.
+
+"6.topic: haskell":
+  - any:
+    - head-branch:
+      - '^haskell-updates$'
+
+"6.topic: python":
+  - any:
+    - head-branch:
+      - '^python-updates$'
diff --git a/.github/workflows/labels.yml b/.github/workflows/labels.yml
index 0ae4ee09d5ca..4aefb306c048 100644
--- a/.github/workflows/labels.yml
+++ b/.github/workflows/labels.yml
@@ -20,12 +20,23 @@ jobs:
     if: "github.repository_owner == 'NixOS' && !contains(github.event.pull_request.title, '[skip treewide]')"
     steps:
       - uses: actions/labeler@8558fd74291d67161a8a78ce36a881fa63b766a9 # v5.0.0
+        if: "!(github.pull_request.head.repo == 'NixOS' && github.ref_protected)"
         with:
           repo-token: ${{ secrets.GITHUB_TOKEN }}
           configuration-path: .github/labeler.yml # default
           sync-labels: true
       - uses: actions/labeler@8558fd74291d67161a8a78ce36a881fa63b766a9 # v5.0.0
+        if: "!(github.pull_request.head.repo == 'NixOS' && github.ref_protected)"
         with:
           repo-token: ${{ secrets.GITHUB_TOKEN }}
           configuration-path: .github/labeler-no-sync.yml
           sync-labels: false
+      - uses: actions/labeler@8558fd74291d67161a8a78ce36a881fa63b766a9 # v5.0.0
+        # Protected branches like staging-next, haskell-updates and python-updates get special labels.
+        # This is to avoid the mass of labels there, which is mostly useless - and really annoying for
+        # the backport labels.
+        if: "github.pull_request.head.repo == 'NixOS' && github.ref_protected"
+        with:
+          repo-token: ${{ secrets.GITHUB_TOKEN }}
+          configuration-path: .github/labeler-protected-branches.yml
+          sync-labels: true