diff --git a/src/Avalonia.Base/Rendering/Composition/Server/ServerCompositionVisual/ServerCompositionVisual.ComputedProperties.cs b/src/Avalonia.Base/Rendering/Composition/Server/ServerCompositionVisual/ServerCompositionVisual.ComputedProperties.cs
index ed8860e04a..e2ce331318 100644
--- a/src/Avalonia.Base/Rendering/Composition/Server/ServerCompositionVisual/ServerCompositionVisual.ComputedProperties.cs
+++ b/src/Avalonia.Base/Rendering/Composition/Server/ServerCompositionVisual/ServerCompositionVisual.ComputedProperties.cs
@@ -50,7 +50,7 @@ partial class ServerCompositionVisual
     private LtrbRect? _ownClipRect;
 
     
-    private bool _hasExtraDirtyRect;
+    private bool _needsToAddExtraDirtyRectToDirtyRegion;
     private LtrbRect _extraDirtyRect;
 
     public virtual LtrbRect? ComputeOwnContentBounds() => null;
@@ -107,7 +107,7 @@ partial class ServerCompositionVisual
         _isDirtyForRender |= dirtyForRender;
         
         // If node itself is dirty for render, we don't need to keep track of extra dirty rects
-        _hasExtraDirtyRect = !dirtyForRender && (_hasExtraDirtyRect || additionalDirtyRegion);
+        _needsToAddExtraDirtyRectToDirtyRegion = !dirtyForRender && (_needsToAddExtraDirtyRectToDirtyRegion || additionalDirtyRegion);
     }
     
     public void RecomputeOwnProperties()
diff --git a/src/Avalonia.Base/Rendering/Composition/Server/ServerCompositionVisual/ServerCompositionVisual.DirtyInputs.cs b/src/Avalonia.Base/Rendering/Composition/Server/ServerCompositionVisual/ServerCompositionVisual.DirtyInputs.cs
index 8352fc70e2..35debea184 100644
--- a/src/Avalonia.Base/Rendering/Composition/Server/ServerCompositionVisual/ServerCompositionVisual.DirtyInputs.cs
+++ b/src/Avalonia.Base/Rendering/Composition/Server/ServerCompositionVisual/ServerCompositionVisual.DirtyInputs.cs
@@ -166,7 +166,7 @@ partial class ServerCompositionVisual
     
     protected void AddExtraDirtyRect(LtrbRect rect)
     {
-        _extraDirtyRect = _hasExtraDirtyRect ? _extraDirtyRect.Union(rect) : rect;
+        _extraDirtyRect = _delayPropagateHasExtraDirtyRects ? _extraDirtyRect.Union(rect) : rect;
         _delayPropagateHasExtraDirtyRects = true;
         EnqueueOwnPropertiesRecompute();
     }
diff --git a/src/Avalonia.Base/Rendering/Composition/Server/ServerCompositionVisual/ServerCompositionVisual.Update.cs b/src/Avalonia.Base/Rendering/Composition/Server/ServerCompositionVisual/ServerCompositionVisual.Update.cs
index b8322225bd..f9b65e01e0 100644
--- a/src/Avalonia.Base/Rendering/Composition/Server/ServerCompositionVisual/ServerCompositionVisual.Update.cs
+++ b/src/Avalonia.Base/Rendering/Composition/Server/ServerCompositionVisual/ServerCompositionVisual.Update.cs
@@ -56,7 +56,7 @@ internal partial class ServerCompositionVisual
         
         private bool NeedToPushBoundsAffectingProperties(ServerCompositionVisual node)
         {
-            return (node._isDirtyForRenderInSubgraph || node._hasExtraDirtyRect || node._contentChanged);
+            return (node._isDirtyForRenderInSubgraph || node._needsToAddExtraDirtyRectToDirtyRegion || node._contentChanged);
         }
         
         public void PreSubgraph(ServerCompositionVisual node, out bool visitChildren)
@@ -142,7 +142,7 @@ internal partial class ServerCompositionVisual
             // specified before the tranform, i.e. in inner space, hence we have to pick them
             // up before we pop the transform from the transform stack.
             //
-            if (node._hasExtraDirtyRect)
+            if (node._needsToAddExtraDirtyRectToDirtyRegion)
             {
                 AddToDirtyRegion(node._extraDirtyRect);
             }
@@ -169,7 +169,7 @@ internal partial class ServerCompositionVisual
             node._isDirtyForRender = false;
             node._isDirtyForRenderInSubgraph = false;
             node._needsBoundingBoxUpdate = false;
-            node._hasExtraDirtyRect = false;
+            node._needsToAddExtraDirtyRectToDirtyRegion = false;
             node._contentChanged = false;
         }
 
diff --git a/tests/Avalonia.Base.UnitTests/Rendering/CompositorInvalidationTests.cs b/tests/Avalonia.Base.UnitTests/Rendering/CompositorInvalidationTests.cs
index 699f450223..ef0e01a104 100644
--- a/tests/Avalonia.Base.UnitTests/Rendering/CompositorInvalidationTests.cs
+++ b/tests/Avalonia.Base.UnitTests/Rendering/CompositorInvalidationTests.cs
@@ -38,6 +38,31 @@ public class CompositorInvalidationTests : CompositorTestsBase
             s.AssertRects(new Rect(30, 50, 20, 10));
         }
     }
+    
+    [Fact]
+    public void Sibling_Controls_Should_Invalidate_Union_Rect_When_Removed()
+    {
+        using (var s = new CompositorCanvas())
+        {
+            var control = new Border()
+            {
+                Background = Brushes.Red, Width = 20, Height = 10,
+                [Canvas.LeftProperty] = 30, [Canvas.TopProperty] = 10
+            };
+            var control2 = new Border()
+            {
+                Background = Brushes.Blue, Width = 20, Height = 10,
+                [Canvas.LeftProperty] = 30, [Canvas.TopProperty] = 50
+            };
+            s.Canvas.Children.Add(control);
+            s.Canvas.Children.Add(control2);
+            s.RunJobs();
+            s.Events.Rects.Clear();
+            s.Canvas.Children.Remove(control);
+            s.Canvas.Children.Remove(control2);
+            s.AssertRects(new Rect(30, 10, 20, 50));
+        }
+    }
 
     [Fact]
     public void Control_Should_Invalidate_Both_Own_Rects_When_Moved()