diff --git a/src/Avalonia.Controls/Presenters/TextPresenter.cs b/src/Avalonia.Controls/Presenters/TextPresenter.cs
index 9a89ccaf28..37fbbaa2e9 100644
--- a/src/Avalonia.Controls/Presenters/TextPresenter.cs
+++ b/src/Avalonia.Controls/Presenters/TextPresenter.cs
@@ -595,12 +595,6 @@ namespace Avalonia.Controls.Presenters
             InvalidateMeasure();
         }
 
-        protected override void OnLoaded(RoutedEventArgs e)
-        {
-            base.OnLoaded(e);
-            EnsureTextSelectionLayer();
-        }
-
         protected override Size MeasureOverride(Size availableSize)
         {
             _constraint = availableSize;
@@ -894,8 +888,7 @@ namespace Avalonia.Controls.Presenters
 
             ResetCaretTimer();
 
-            if (TextSelectionHandleCanvas is { } canvas && _layer != null && !_layer.Children.Contains(canvas))
-                _layer?.Add(TextSelectionHandleCanvas);
+            EnsureTextSelectionLayer();
         }
 
         private void EnsureTextSelectionLayer()
@@ -903,10 +896,17 @@ namespace Avalonia.Controls.Presenters
             if (TextSelectionHandleCanvas == null)
             {
                 TextSelectionHandleCanvas = new TextSelectionHandleCanvas();
-                TextSelectionHandleCanvas.SetPresenter(this);
             }
+            TextSelectionHandleCanvas.SetPresenter(this);
             _layer = TextSelectorLayer.GetTextSelectorLayer(this);
-            if (_layer != null && !_layer.Children.Contains(TextSelectionHandleCanvas))
+            if (TextSelectionHandleCanvas.VisualParent is { } parent && parent != _layer)
+            {
+                if (parent is TextSelectorLayer l)
+                {
+                    l.Remove(TextSelectionHandleCanvas);
+                }
+            }
+            if (_layer != null && TextSelectionHandleCanvas.VisualParent != _layer)
                 _layer?.Add(TextSelectionHandleCanvas);
         }
 
diff --git a/src/Avalonia.Controls/Primitives/TextSelectionCanvas.cs b/src/Avalonia.Controls/Primitives/TextSelectionCanvas.cs
index c9a58aed0b..07e48653d7 100644
--- a/src/Avalonia.Controls/Primitives/TextSelectionCanvas.cs
+++ b/src/Avalonia.Controls/Primitives/TextSelectionCanvas.cs
@@ -250,6 +250,8 @@ namespace Avalonia.Controls.Primitives
 
         internal void SetPresenter(TextPresenter? textPresenter)
         {
+            if (_presenter == textPresenter)
+                return;
             _presenter = textPresenter;
             if (_presenter != null)
             {