diff --git a/src/Avalonia.Controls/ItemsControl.cs b/src/Avalonia.Controls/ItemsControl.cs
index 54ffba462f..4d71cc8d4f 100644
--- a/src/Avalonia.Controls/ItemsControl.cs
+++ b/src/Avalonia.Controls/ItemsControl.cs
@@ -94,7 +94,6 @@ namespace Avalonia.Controls
         private ItemContainerGenerator? _itemContainerGenerator;
         private EventHandler<ChildIndexChangedEventArgs>? _childIndexChanged;
         private IDataTemplate? _displayMemberItemTemplate;
-        private ScrollViewer? _scrollViewer;
         private ItemsPresenter? _itemsPresenter;
 
         /// <summary>
@@ -457,7 +456,6 @@ namespace Avalonia.Controls
         protected override void OnApplyTemplate(TemplateAppliedEventArgs e)
         {
             base.OnApplyTemplate(e);
-            _scrollViewer = e.NameScope.Find<ScrollViewer>("PART_ScrollViewer");
             _itemsPresenter = e.NameScope.Find<ItemsPresenter>("PART_ItemsPresenter");
         }
 
@@ -629,7 +627,6 @@ namespace Avalonia.Controls
         internal void ItemContainerPrepared(Control container, object? item, int index)
         {
             _childIndexChanged?.Invoke(this, new ChildIndexChangedEventArgs(container, index));
-            _scrollViewer?.RegisterAnchorCandidate(container);
             ContainerPrepared?.Invoke(this, new(container, index));
         }
 
@@ -642,7 +639,6 @@ namespace Avalonia.Controls
 
         internal void ClearItemContainer(Control container)
         {
-            _scrollViewer?.UnregisterAnchorCandidate(container);
             ClearContainerForItemOverride(container);
             ContainerClearing?.Invoke(this, new(container));
         }
diff --git a/src/Avalonia.Controls/Presenters/ScrollContentPresenter.cs b/src/Avalonia.Controls/Presenters/ScrollContentPresenter.cs
index bc86558ab3..261b7d3533 100644
--- a/src/Avalonia.Controls/Presenters/ScrollContentPresenter.cs
+++ b/src/Avalonia.Controls/Presenters/ScrollContentPresenter.cs
@@ -97,7 +97,7 @@ namespace Avalonia.Controls.Presenters
         private Size _viewport;
         private Dictionary<int, Vector>? _activeLogicalGestureScrolls;
         private Dictionary<int, Vector>? _scrollGestureSnapPoints;
-        private List<Control>? _anchorCandidates;
+        private HashSet<Control>? _anchorCandidates;
         private Control? _anchorElement;
         private Rect _anchorElementBounds;
         private bool _isAnchorElementDirty;
@@ -310,7 +310,7 @@ namespace Avalonia.Controls.Presenters
                     "An anchor control must be a visual descendent of the ScrollContentPresenter.");
             }
 
-            _anchorCandidates ??= new List<Control>();
+            _anchorCandidates ??= new();
             _anchorCandidates.Add(element);
             _isAnchorElementDirty = true;
         }
@@ -410,6 +410,7 @@ namespace Avalonia.Controls.Presenters
                     try
                     {
                         _arranging = true;
+
                         Offset = newOffset;
                     }
                     finally
diff --git a/src/Avalonia.Controls/VirtualizingStackPanel.cs b/src/Avalonia.Controls/VirtualizingStackPanel.cs
index 5a766b9cd3..ad2ae9278c 100644
--- a/src/Avalonia.Controls/VirtualizingStackPanel.cs
+++ b/src/Avalonia.Controls/VirtualizingStackPanel.cs
@@ -67,6 +67,7 @@ namespace Avalonia.Controls
         private double _lastEstimatedElementSizeU = 25;
         private RealizedStackElements? _measureElements;
         private RealizedStackElements? _realizedElements;
+        private ScrollViewer? _scrollViewer;
         private Rect _viewport = s_invalidViewport;
         private Stack<Control>? _recyclePool;
         private Control? _unrealizedFocusedElement;
@@ -203,6 +204,7 @@ namespace Avalonia.Controls
                             new Rect(u, 0, sizeU, finalSize.Height) :
                             new Rect(0, u, finalSize.Width, sizeU);
                         e.Arrange(rect);
+                        _scrollViewer?.RegisterAnchorCandidate(e);
                         u += orientation == Orientation.Horizontal ? rect.Width : rect.Height;
                     }
                 }
@@ -217,6 +219,18 @@ namespace Avalonia.Controls
             }
         }
 
+        protected override void OnAttachedToVisualTree(VisualTreeAttachmentEventArgs e)
+        {
+            base.OnAttachedToVisualTree(e);
+            _scrollViewer = this.FindAncestorOfType<ScrollViewer>();
+        }
+
+        protected override void OnDetachedFromVisualTree(VisualTreeAttachmentEventArgs e)
+        {
+            base.OnDetachedFromVisualTree(e);
+            _scrollViewer = null;
+        }
+
         protected override void OnItemsChanged(IReadOnlyList<object?> items, NotifyCollectionChangedEventArgs e)
         {
             InvalidateMeasure();
@@ -598,6 +612,8 @@ namespace Avalonia.Controls
         {
             Debug.Assert(ItemContainerGenerator is not null);
             
+            _scrollViewer?.UnregisterAnchorCandidate(element);
+
             if (element.IsSet(ItemIsOwnContainerProperty))
             {
                 element.IsVisible = false;