diff --git a/src/Avalonia.Visuals/Rendering/DeferredRenderer.cs b/src/Avalonia.Visuals/Rendering/DeferredRenderer.cs
index 0ff0285a04..efcc555159 100644
--- a/src/Avalonia.Visuals/Rendering/DeferredRenderer.cs
+++ b/src/Avalonia.Visuals/Rendering/DeferredRenderer.cs
@@ -540,7 +540,7 @@ namespace Avalonia.Rendering
                     foreach (var visual in _recalculateChildren)
                     {
                         var node = scene.FindNode(visual);
-                        ((VisualNode)node)?.UpdateChildren(scene);
+                        ((VisualNode)node)?.SortChildren(scene);
                     }
 
                     _recalculateChildren.Clear();
diff --git a/src/Avalonia.Visuals/Rendering/SceneGraph/SceneBuilder.cs b/src/Avalonia.Visuals/Rendering/SceneGraph/SceneBuilder.cs
index 2fcea8c205..161cbc099e 100644
--- a/src/Avalonia.Visuals/Rendering/SceneGraph/SceneBuilder.cs
+++ b/src/Avalonia.Visuals/Rendering/SceneGraph/SceneBuilder.cs
@@ -53,6 +53,16 @@ namespace Avalonia.Rendering.SceneGraph
 
             if (visual.VisualRoot != null)
             {
+                if (node?.Parent != null &&
+                    visual.VisualParent != null &&
+                    node.Parent.Visual != visual.VisualParent)
+                {
+                    // The control has changed parents. Remove the node and recurse into the new parent node.
+                    ((VisualNode)node.Parent).RemoveChild(node);
+                    Deindex(scene, node);
+                    node = (VisualNode)scene.FindNode(visual.VisualParent);
+                }
+
                 if (visual.IsVisible)
                 {
                     // If the node isn't yet part of the scene, find the nearest ancestor that is.
diff --git a/src/Avalonia.Visuals/Rendering/SceneGraph/VisualNode.cs b/src/Avalonia.Visuals/Rendering/SceneGraph/VisualNode.cs
index f079023c6d..f579bf0a62 100644
--- a/src/Avalonia.Visuals/Rendering/SceneGraph/VisualNode.cs
+++ b/src/Avalonia.Visuals/Rendering/SceneGraph/VisualNode.cs
@@ -174,12 +174,12 @@ namespace Avalonia.Rendering.SceneGraph
 
         /// <summary>
         /// Sorts the <see cref="Children"/> collection according to the order of the visual's
-        /// children and their z-index and removes controls that are no longer children.
+        /// children and their z-index.
         /// </summary>
         /// <param name="scene">The scene that the node is a part of.</param>
-        public void UpdateChildren(Scene scene)
+        public void SortChildren(Scene scene)
         {
-            if (_children == null || _children.Count == 0)
+            if (_children == null || _children.Count <= 1)
             {
                 return;
             }
@@ -193,12 +193,9 @@ namespace Avalonia.Rendering.SceneGraph
                 keys.Add(((long)zIndex << 32) + i);
             }
 
-            var toRemove = _children.ToList();
-
             keys.Sort();
             _children.Clear();
 
-
             foreach (var i in keys)
             {
                 var child = Visual.VisualChildren[(int)(i & 0xffffffff)];
@@ -207,14 +204,8 @@ namespace Avalonia.Rendering.SceneGraph
                 if (node != null)
                 {
                     _children.Add(node);
-                    toRemove.Remove(node);
                 }
             }
-
-            foreach (var node in toRemove)
-            {
-                scene.Remove(node);
-            }
         }
 
         /// <summary>
diff --git a/tests/Avalonia.Visuals.UnitTests/Rendering/DeferredRendererTests.cs b/tests/Avalonia.Visuals.UnitTests/Rendering/DeferredRendererTests.cs
index 568ccb81d8..b4743e900d 100644
--- a/tests/Avalonia.Visuals.UnitTests/Rendering/DeferredRendererTests.cs
+++ b/tests/Avalonia.Visuals.UnitTests/Rendering/DeferredRendererTests.cs
@@ -96,6 +96,55 @@ namespace Avalonia.Visuals.UnitTests.Rendering
             Assert.Equal(new List<IVisual> { root, decorator, border, canvas }, result);
         }
 
+        [Fact]
+        public void Should_Add_Dirty_Rect_On_Child_Remove()
+        {
+            var dispatcher = new ImmediateDispatcher();
+            var loop = new Mock<IRenderLoop>();
+
+            Decorator decorator;
+            Border border;
+            var root = new TestRoot
+            {
+                Width = 100,
+                Height= 100,
+                Child = decorator = new Decorator
+                {
+                    Child = border = new Border
+                    {
+                        Width = 50,
+                        Height = 50,
+                        Background = Brushes.Red,
+                    },
+                }
+            };
+
+            root.Measure(Size.Infinity);
+            root.Arrange(new Rect(root.DesiredSize));
+
+            var sceneBuilder = new SceneBuilder();
+            var target = new DeferredRenderer(
+                root,
+                loop.Object,
+                sceneBuilder: sceneBuilder,
+                dispatcher: dispatcher);
+
+            root.Renderer = target;
+            target.Start();
+            RunFrame(target);
+
+            decorator.Child = null;
+
+            RunFrame(target);
+
+            var scene = target.UnitTestScene();
+            var stackNode = scene.FindNode(decorator);
+            var dirty = scene.Layers[0].Dirty.ToList();
+
+            Assert.Equal(1, dirty.Count);
+            Assert.Equal(new Rect(25, 25, 50, 50), dirty[0]);
+        }
+
         [Fact]
         public void Should_Update_VisualNode_Order_On_Child_Remove_Insert()
         {
diff --git a/tests/Avalonia.Visuals.UnitTests/Rendering/SceneGraph/SceneBuilderTests.cs b/tests/Avalonia.Visuals.UnitTests/Rendering/SceneGraph/SceneBuilderTests.cs
index dcf23e94e2..13bcd27240 100644
--- a/tests/Avalonia.Visuals.UnitTests/Rendering/SceneGraph/SceneBuilderTests.cs
+++ b/tests/Avalonia.Visuals.UnitTests/Rendering/SceneGraph/SceneBuilderTests.cs
@@ -521,8 +521,8 @@ namespace Avalonia.Visuals.UnitTests.Rendering.SceneGraph
                 moveFromNode = (VisualNode)scene.FindNode(moveFrom);
                 moveToNode = (VisualNode)scene.FindNode(moveTo);
 
-                moveFromNode.UpdateChildren(scene);
-                moveToNode.UpdateChildren(scene);
+                moveFromNode.SortChildren(scene);
+                moveToNode.SortChildren(scene);
                 sceneBuilder.Update(scene, moveFrom);
                 sceneBuilder.Update(scene, moveTo);
                 sceneBuilder.Update(scene, moveMe);
diff --git a/tests/Avalonia.Visuals.UnitTests/Rendering/SceneGraph/VisualNodeTests.cs b/tests/Avalonia.Visuals.UnitTests/Rendering/SceneGraph/VisualNodeTests.cs
index d4f7a6a142..24ba2d1c48 100644
--- a/tests/Avalonia.Visuals.UnitTests/Rendering/SceneGraph/VisualNodeTests.cs
+++ b/tests/Avalonia.Visuals.UnitTests/Rendering/SceneGraph/VisualNodeTests.cs
@@ -99,7 +99,7 @@ namespace Avalonia.Visuals.UnitTests.Rendering.SceneGraph
             var node = new VisualNode(Mock.Of<IVisual>(), null);
             var scene = new Scene(Mock.Of<IVisual>());
 
-            node.UpdateChildren(scene);
+            node.SortChildren(scene);
         }
     }
 }