diff --git a/src/hooks/useOptimisticLabels.ts b/src/hooks/useOptimisticLabels.ts
index f5c787474..059abc672 100644
--- a/src/hooks/useOptimisticLabels.ts
+++ b/src/hooks/useOptimisticLabels.ts
@@ -95,7 +95,10 @@ export function useOptimisticLabels({
     }
 
     async function refresh() {
-        router.refresh();
+        // Only refresh if the labels have been modified
+        if (pendingActions.length > 0) {
+            router.refresh();
+        }
         setPendingActions([]);
     }