diff --git a/src/Avalonia.Visuals/Rendering/DeferredRenderer.cs b/src/Avalonia.Visuals/Rendering/DeferredRenderer.cs
index efcc555159..0ff0285a04 100644
--- a/src/Avalonia.Visuals/Rendering/DeferredRenderer.cs
+++ b/src/Avalonia.Visuals/Rendering/DeferredRenderer.cs
@@ -540,7 +540,7 @@ namespace Avalonia.Rendering
                     foreach (var visual in _recalculateChildren)
                     {
                         var node = scene.FindNode(visual);
-                        ((VisualNode)node)?.SortChildren(scene);
+                        ((VisualNode)node)?.UpdateChildren(scene);
                     }
 
                     _recalculateChildren.Clear();
diff --git a/src/Avalonia.Visuals/Rendering/SceneGraph/VisualNode.cs b/src/Avalonia.Visuals/Rendering/SceneGraph/VisualNode.cs
index f579bf0a62..f079023c6d 100644
--- a/src/Avalonia.Visuals/Rendering/SceneGraph/VisualNode.cs
+++ b/src/Avalonia.Visuals/Rendering/SceneGraph/VisualNode.cs
@@ -174,12 +174,12 @@ namespace Avalonia.Rendering.SceneGraph
 
         /// <summary>
         /// Sorts the <see cref="Children"/> collection according to the order of the visual's
-        /// children and their z-index.
+        /// children and their z-index and removes controls that are no longer children.
         /// </summary>
         /// <param name="scene">The scene that the node is a part of.</param>
-        public void SortChildren(Scene scene)
+        public void UpdateChildren(Scene scene)
         {
-            if (_children == null || _children.Count <= 1)
+            if (_children == null || _children.Count == 0)
             {
                 return;
             }
@@ -193,9 +193,12 @@ namespace Avalonia.Rendering.SceneGraph
                 keys.Add(((long)zIndex << 32) + i);
             }
 
+            var toRemove = _children.ToList();
+
             keys.Sort();
             _children.Clear();
 
+
             foreach (var i in keys)
             {
                 var child = Visual.VisualChildren[(int)(i & 0xffffffff)];
@@ -204,8 +207,14 @@ namespace Avalonia.Rendering.SceneGraph
                 if (node != null)
                 {
                     _children.Add(node);
+                    toRemove.Remove(node);
                 }
             }
+
+            foreach (var node in toRemove)
+            {
+                scene.Remove(node);
+            }
         }
 
         /// <summary>
diff --git a/tests/Avalonia.Visuals.UnitTests/Rendering/SceneGraph/SceneBuilderTests.cs b/tests/Avalonia.Visuals.UnitTests/Rendering/SceneGraph/SceneBuilderTests.cs
index 13bcd27240..dcf23e94e2 100644
--- a/tests/Avalonia.Visuals.UnitTests/Rendering/SceneGraph/SceneBuilderTests.cs
+++ b/tests/Avalonia.Visuals.UnitTests/Rendering/SceneGraph/SceneBuilderTests.cs
@@ -521,8 +521,8 @@ namespace Avalonia.Visuals.UnitTests.Rendering.SceneGraph
                 moveFromNode = (VisualNode)scene.FindNode(moveFrom);
                 moveToNode = (VisualNode)scene.FindNode(moveTo);
 
-                moveFromNode.SortChildren(scene);
-                moveToNode.SortChildren(scene);
+                moveFromNode.UpdateChildren(scene);
+                moveToNode.UpdateChildren(scene);
                 sceneBuilder.Update(scene, moveFrom);
                 sceneBuilder.Update(scene, moveTo);
                 sceneBuilder.Update(scene, moveMe);
diff --git a/tests/Avalonia.Visuals.UnitTests/Rendering/SceneGraph/VisualNodeTests.cs b/tests/Avalonia.Visuals.UnitTests/Rendering/SceneGraph/VisualNodeTests.cs
index 24ba2d1c48..d4f7a6a142 100644
--- a/tests/Avalonia.Visuals.UnitTests/Rendering/SceneGraph/VisualNodeTests.cs
+++ b/tests/Avalonia.Visuals.UnitTests/Rendering/SceneGraph/VisualNodeTests.cs
@@ -99,7 +99,7 @@ namespace Avalonia.Visuals.UnitTests.Rendering.SceneGraph
             var node = new VisualNode(Mock.Of<IVisual>(), null);
             var scene = new Scene(Mock.Of<IVisual>());
 
-            node.SortChildren(scene);
+            node.UpdateChildren(scene);
         }
     }
 }