diff --git a/src/Avalonia.Layout/ElementManager.cs b/src/Avalonia.Layout/ElementManager.cs
index 0a8816480f..1748a3be03 100644
--- a/src/Avalonia.Layout/ElementManager.cs
+++ b/src/Avalonia.Layout/ElementManager.cs
@@ -276,8 +276,38 @@ namespace Avalonia.Layout
 
                     case NotifyCollectionChangedAction.Replace:
                     {
-                        OnItemsRemoved(args.OldStartingIndex, args.OldItems.Count);
-                        OnItemsAdded(args.NewStartingIndex, args.NewItems.Count);
+                        int oldSize = args.OldItems.Count;
+                        int newSize = args.NewItems.Count;
+                        int oldStartIndex = args.OldStartingIndex;
+                        int newStartIndex = args.NewStartingIndex;
+
+                        if (oldSize == newSize &&
+                            oldStartIndex == newStartIndex &&
+                            IsDataIndexRealized(oldStartIndex) &&
+                            IsDataIndexRealized(oldStartIndex + oldSize -1))
+                        {
+                            // Straight up replace of n items within the realization window.
+                            // Removing and adding might causes us to lose the anchor causing us
+                            // to throw away all containers and start from scratch.
+                            // Instead, we can just clear those items and set the element to
+                            // null (sentinel) and let the next measure get new containers for them.
+                            var startRealizedIndex = GetRealizedRangeIndexFromDataIndex(oldStartIndex);
+                            for (int realizedIndex = startRealizedIndex; realizedIndex < startRealizedIndex + oldSize; realizedIndex++)
+                            {
+                                var elementRef = _realizedElements[realizedIndex];
+
+                                if (elementRef != null)
+                                {
+                                    _context.RecycleElement(elementRef);
+                                    _realizedElements[realizedIndex] = null;
+                                }
+                            }
+                        }
+                        else
+                        {
+                            OnItemsRemoved(oldStartIndex, oldSize);
+                            OnItemsAdded(newStartIndex, newSize);
+                        }
                     }
                     break;
 
diff --git a/src/Avalonia.Layout/UniformGridLayout.cs b/src/Avalonia.Layout/UniformGridLayout.cs
index 6d157715a3..edc2042922 100644
--- a/src/Avalonia.Layout/UniformGridLayout.cs
+++ b/src/Avalonia.Layout/UniformGridLayout.cs
@@ -409,6 +409,10 @@ namespace Avalonia.Layout
                 _orientation.ScrollOrientation,
                 LayoutId);
 
+            // If after Measure the first item is in the realization rect, then we revoke grid state's ownership,
+            // and only use the layout when to clear it when it's done.
+            gridState.EnsureFirstElementOwnership(context);
+
             return new Size(desiredSize.Width, desiredSize.Height);
         }
 
diff --git a/src/Avalonia.Layout/UniformGridLayoutState.cs b/src/Avalonia.Layout/UniformGridLayoutState.cs
index 37660d0354..e30c1a3c4b 100644
--- a/src/Avalonia.Layout/UniformGridLayoutState.cs
+++ b/src/Avalonia.Layout/UniformGridLayoutState.cs
@@ -134,10 +134,13 @@ namespace Avalonia.Layout
             }
         }
 
-        internal void EnsureFirstElementOwnership()
+        internal void EnsureFirstElementOwnership(VirtualizingLayoutContext context)
         {
-            if (FlowAlgorithm.GetElementIfRealized(0) != null)
+            if (_cachedFirstElement != null && FlowAlgorithm.GetElementIfRealized(0) != null)
             {
+                // We created the element, but then flowlayout algorithm took ownership, so we can clear it and
+                // let flowlayout algorithm do its thing.
+                context.RecycleElement(_cachedFirstElement);
                 _cachedFirstElement = null;
             }
         }