diff --git a/src/Avalonia.Controls/Platform/InProcessDragSource.cs b/src/Avalonia.Controls/Platform/InProcessDragSource.cs
index ded2cd1ebb..aeedbe81ea 100644
--- a/src/Avalonia.Controls/Platform/InProcessDragSource.cs
+++ b/src/Avalonia.Controls/Platform/InProcessDragSource.cs
@@ -19,10 +19,9 @@ namespace Avalonia.Platform
 
         private DragDropEffects _allowedEffects;
         private IDataObject? _draggedData;
-        private IInputRoot? _lastRoot;
+        private TopLevel? _lastRoot;
         private Point _lastPosition;
-        private StandardCursorType _lastCursorType;
-        private object? _originalCursor;
+        private StandardCursorType? _lastCursorType;
         private RawInputModifiers? _initialInputModifiers;
 
         public InProcessDragSource()
@@ -77,7 +76,7 @@ namespace Avalonia.Platform
             tl?.PlatformImpl?.Input?.Invoke(rawEvent);
 
             var effect = GetPreferredEffect(rawEvent.Effects & _allowedEffects, modifiers);
-            UpdateCursor(root, effect);
+            UpdateCursor(tl, effect);
             return effect;
         }
 
@@ -103,41 +102,24 @@ namespace Avalonia.Platform
             return StandardCursorType.No;
         }
         
-        private void UpdateCursor(IInputRoot? root, DragDropEffects effect)
+        private void UpdateCursor(TopLevel? root, DragDropEffects effect)
         {
             if (_lastRoot != root)
             {
-                if (_lastRoot is InputElement ieLast)
-                {
-                    if (_originalCursor == AvaloniaProperty.UnsetValue)
-                        ieLast.ClearValue(InputElement.CursorProperty);
-                    else
-                        ieLast.Cursor = _originalCursor as Cursor;
-                }
-
-                if (root is InputElement ieNew)
-                {
-                    if (!ieNew.IsSet(InputElement.CursorProperty))
-                        _originalCursor = AvaloniaProperty.UnsetValue;
-                    else
-                        _originalCursor = root.Cursor;
-                }
-                else
-                    _originalCursor = null;
-
-                _lastCursorType = StandardCursorType.Arrow;
+                _lastRoot?.SetCursorOverride(null);
                 _lastRoot = root;
+                _lastCursorType = null;
             }
 
-            if (root is InputElement ie)
+            if (root != null)
             {
                 var ct = GetCursorForDropEffect(effect);
-                if (ct != _lastCursorType)
+                if (_lastCursorType != ct)
                 {
                     _lastCursorType = ct;
-                    ie.Cursor = new Cursor(ct);
+                    root.SetCursorOverride(new Cursor(ct));
                 }
-            }  
+            }
         }
 
         private void CancelDragging()
diff --git a/src/Avalonia.Controls/TopLevel.cs b/src/Avalonia.Controls/TopLevel.cs
index 62c860c7a9..3892925c15 100644
--- a/src/Avalonia.Controls/TopLevel.cs
+++ b/src/Avalonia.Controls/TopLevel.cs
@@ -139,6 +139,8 @@ namespace Avalonia.Controls
         private IStorageProvider? _storageProvider;
         private Screens? _screens;
         private LayoutDiagnosticBridge? _layoutDiagnosticBridge;
+        private Cursor? _cursor;
+        private Cursor? _cursorOverride;
         
         /// <summary>
         /// Initializes static members of the <see cref="TopLevel"/> class.
@@ -179,7 +181,7 @@ namespace Avalonia.Controls
 
                 if (e.NewValue is InputElement newInputElement)
                 {
-                    topLevel.PlatformImpl?.SetCursor(newInputElement.Cursor?.PlatformImpl);
+                    topLevel.SetCursor(newInputElement.Cursor);
                     newInputElement.PropertyChanged += topLevel.PointerOverElementOnPropertyChanged;
                 }
             });
@@ -867,11 +869,28 @@ namespace Avalonia.Controls
             return candidate;
         }
 
+        private void UpdateCursor() => PlatformImpl?.SetCursor(_cursorOverride?.PlatformImpl ?? _cursor?.PlatformImpl);
+
+        private void SetCursor(Cursor? cursor)
+        {
+            _cursor = cursor;
+            UpdateCursor();
+        }
+        
+        /// <summary>
+        /// This should only be used by InProcessDragSource
+        /// </summary>
+        internal void SetCursorOverride(Cursor? cursor)
+        {
+            _cursorOverride = cursor;
+            UpdateCursor();
+        }
+        
         private void PointerOverElementOnPropertyChanged(object? sender, AvaloniaPropertyChangedEventArgs e)
         {
             if (e.Property == CursorProperty && sender is InputElement inputElement)
             {
-                PlatformImpl?.SetCursor(inputElement.Cursor?.PlatformImpl);
+                SetCursor(inputElement.Cursor);
             }
         }