diff --git a/src/Avalonia.Controls/Presenters/TextPresenter.cs b/src/Avalonia.Controls/Presenters/TextPresenter.cs
index 246c12cf3c..e35d683a1f 100644
--- a/src/Avalonia.Controls/Presenters/TextPresenter.cs
+++ b/src/Avalonia.Controls/Presenters/TextPresenter.cs
@@ -462,6 +462,7 @@ namespace Avalonia.Controls.Presenters
         public void ShowCaret()
         {
             EnsureCaretTimer();
+            EnsureTextSelectionLayer();
             _caretBlink = true;
             _caretTimer?.Start();
             InvalidateVisual();
@@ -470,10 +471,7 @@ namespace Avalonia.Controls.Presenters
         public void HideCaret()
         {
             _caretBlink = false;
-            if (TextSelectionHandleCanvas != null)
-            {
-                TextSelectionHandleCanvas.ShowHandles = false;
-            }
+            RemoveTextSelectionCanvas();
             _caretTimer?.Stop();
             InvalidateVisual();
         }
@@ -908,8 +906,6 @@ namespace Avalonia.Controls.Presenters
             base.OnAttachedToVisualTree(e);
 
             ResetCaretTimer();
-
-            EnsureTextSelectionLayer();
         }
 
         private void EnsureTextSelectionLayer()
@@ -931,6 +927,17 @@ namespace Avalonia.Controls.Presenters
                 _layer?.Add(TextSelectionHandleCanvas);
         }
 
+        private void RemoveTextSelectionCanvas()
+        {
+            if(_layer != null && TextSelectionHandleCanvas is { } canvas)
+            {
+                canvas.SetPresenter(null);
+                _layer.Remove(canvas);
+            }
+
+            TextSelectionHandleCanvas = null;
+        }
+
         protected override void OnDetachedFromVisualTree(VisualTreeAttachmentEventArgs e)
         {
             base.OnDetachedFromVisualTree(e);
diff --git a/src/Avalonia.Controls/Primitives/TextSelectionCanvas.cs b/src/Avalonia.Controls/Primitives/TextSelectionCanvas.cs
index 3ec9221539..6e23495cb1 100644
--- a/src/Avalonia.Controls/Primitives/TextSelectionCanvas.cs
+++ b/src/Avalonia.Controls/Primitives/TextSelectionCanvas.cs
@@ -273,7 +273,6 @@ namespace Avalonia.Controls.Primitives
                 {
                     _textBox.AddHandler(TextBox.TextChangingEvent, TextChanged, handledEventsToo: true);
                     _textBox.AddHandler(KeyDownEvent, TextBoxKeyDown, handledEventsToo: true);
-                    _textBox.AddHandler(LostFocusEvent, TextBoxLostFocus, handledEventsToo: true);
                     _textBox.AddHandler(PointerReleasedEvent, TextBoxPointerReleased, handledEventsToo: true);
                     _textBox.AddHandler(Gestures.HoldingEvent, TextBoxHolding, handledEventsToo: true);
 
@@ -289,7 +288,6 @@ namespace Avalonia.Controls.Primitives
                     _textBox.RemoveHandler(TextBox.TextChangingEvent, TextChanged);
                     _textBox.RemoveHandler(KeyDownEvent, TextBoxKeyDown);
                     _textBox.RemoveHandler(PointerReleasedEvent, TextBoxPointerReleased);
-                    _textBox.RemoveHandler(LostFocusEvent, TextBoxLostFocus);
                     _textBox.RemoveHandler(Gestures.HoldingEvent, TextBoxHolding);
 
                     _textBox.PropertyChanged -= TextBoxPropertyChanged;
@@ -390,11 +388,6 @@ namespace Avalonia.Controls.Primitives
             }
         }
 
-        private void TextBoxLostFocus(object? sender, RoutedEventArgs e)
-        {
-            ShowHandles = false;
-        }
-
         private void TextBoxKeyDown(object? sender, KeyEventArgs e)
         {
             ShowHandles = false;