diff --git a/src/Avalonia.Controls/Primitives/TextSelectionCanvas.cs b/src/Avalonia.Controls/Primitives/TextSelectionCanvas.cs
index 07e48653d7..3ec9221539 100644
--- a/src/Avalonia.Controls/Primitives/TextSelectionCanvas.cs
+++ b/src/Avalonia.Controls/Primitives/TextSelectionCanvas.cs
@@ -4,6 +4,7 @@ using Avalonia.Controls.Presenters;
 using Avalonia.Input;
 using Avalonia.Interactivity;
 using Avalonia.Layout;
+using Avalonia.Media;
 using Avalonia.VisualTree;
 
 namespace Avalonia.Controls.Primitives
@@ -18,6 +19,7 @@ namespace Avalonia.Controls.Primitives
         private TextPresenter? _presenter;
         private TextBox? _textBox;
         private bool _showHandle;
+        private bool _canShowContextMenu = true;
 
         internal bool ShowHandles
         {
@@ -33,7 +35,7 @@ namespace Avalonia.Controls.Primitives
                     _caretHandle.IsVisible = false;
                 }
 
-                IsVisible = value;
+                IsVisible = !string.IsNullOrEmpty(_presenter?.Text) && value;
             }
         }
 
@@ -65,11 +67,20 @@ namespace Avalonia.Controls.Primitives
             _caretHandle.SetTopLeft(default);
             _endHandle.SetTopLeft(default);
 
+            _startHandle.PointerReleased += Handle_PointerReleased;
+            _caretHandle.PointerReleased += Handle_PointerReleased;
+            _endHandle.PointerReleased += Handle_PointerReleased;
+
             IsVisible = ShowHandles;
 
             ClipToBounds = false;
         }
 
+        private void Handle_PointerReleased(object? sender, PointerReleasedEventArgs e)
+        {
+            ShowContextMenu();
+        }
+
         private void Handle_DragStarted(object? sender, VectorEventArgs e)
         {
             if (_textBox?.ContextFlyout is { } flyout)
@@ -92,6 +103,7 @@ namespace Avalonia.Controls.Primitives
 
         private void CaretHandle_DragDelta(object? sender, VectorEventArgs e)
         {
+            _canShowContextMenu = false;
             if (_presenter != null && _textBox != null)
             {
                 var point = ToPresenter(_caretHandle.IndicatorPosition);
@@ -267,6 +279,7 @@ namespace Avalonia.Controls.Primitives
 
                     _textBox.PropertyChanged += TextBoxPropertyChanged;
                     _textBox.EffectiveViewportChanged += TextBoxEffectiveViewportChanged;
+                    _textBox.SizeChanged += TextBox_SizeChanged;
                 }
             }
             else
@@ -281,12 +294,18 @@ namespace Avalonia.Controls.Primitives
 
                     _textBox.PropertyChanged -= TextBoxPropertyChanged;
                     _textBox.EffectiveViewportChanged -= TextBoxEffectiveViewportChanged;
+                    _textBox.SizeChanged -= TextBox_SizeChanged;
                 }
 
                 _textBox = null;
             }
         }
 
+        private void TextBox_SizeChanged(object? sender, SizeChangedEventArgs e)
+        {
+            InvalidateMeasure();
+        }
+
         private void TextBoxEffectiveViewportChanged(object? sender, EffectiveViewportChangedEventArgs e)
         {
             if (ShowHandles)
@@ -304,7 +323,7 @@ namespace Avalonia.Controls.Primitives
 
         internal bool ShowContextMenu()
         {
-            if (_textBox != null)
+            if (_textBox != null && _canShowContextMenu)
             {
                 if (_textBox.ContextFlyout is PopupFlyoutBase flyout)
                 {
@@ -345,6 +364,8 @@ namespace Avalonia.Controls.Primitives
                 }
             }
 
+            _canShowContextMenu = true;
+
             return false;
         }
 
diff --git a/src/Avalonia.Controls/Primitives/TextSelectionHandle.cs b/src/Avalonia.Controls/Primitives/TextSelectionHandle.cs
index d9bf8b5529..550f82fe84 100644
--- a/src/Avalonia.Controls/Primitives/TextSelectionHandle.cs
+++ b/src/Avalonia.Controls/Primitives/TextSelectionHandle.cs
@@ -134,32 +134,42 @@ namespace Avalonia.Controls.Primitives
 
         protected override void OnPointerMoved(PointerEventArgs e)
         {
-            if (_lastPoint.HasValue)
+            VectorEventArgs ev;
+
+            if (!_lastPoint.HasValue)
             {
-                var ev = new VectorEventArgs
+                _lastPoint = e.GetPosition(VisualRoot as Visual);
+                e.Pointer.Capture(this);
+
+                ev = new VectorEventArgs
                 {
-                    RoutedEvent = DragDeltaEvent,
-                    Vector = e.GetPosition(VisualRoot as Visual) - _lastPoint.Value,
+                    RoutedEvent = DragStartedEvent,
+                    Vector = (Vector)_lastPoint,
                 };
+            }
+            else
+            {
+                var vector = e.GetPosition(VisualRoot as Visual) - _lastPoint.Value;
 
-                RaiseEvent(ev);
+                var tapSize = TopLevel.GetTopLevel(this)?.PlatformSettings?.GetTapSize(PointerType.Touch) ?? new Size(10, 10);
+
+                if (Math.Abs(vector.X) < tapSize.Width && Math.Abs(vector.Y) < tapSize.Height)
+                    return;
+
+                ev = new VectorEventArgs
+                {
+                    RoutedEvent = DragDeltaEvent,
+                    Vector = vector,
+                };
             }
+
+            RaiseEvent(ev);
         }
 
         protected override void OnPointerPressed(PointerPressedEventArgs e)
         {
             e.Handled = true;
-            _lastPoint = e.GetPosition(VisualRoot as Visual);
-
-            var ev = new VectorEventArgs
-            {
-                RoutedEvent = DragStartedEvent,
-                Vector = (Vector)_lastPoint,
-            };
-
             PseudoClasses.Add(":pressed");
-
-            RaiseEvent(ev);
         }
 
         protected override void OnPointerReleased(PointerReleasedEventArgs e)