diff --git a/src/Avalonia.Controls/Presenters/ItemVirtualizerSimple.cs b/src/Avalonia.Controls/Presenters/ItemVirtualizerSimple.cs
index cd14211075..d27de7a80d 100644
--- a/src/Avalonia.Controls/Presenters/ItemVirtualizerSimple.cs
+++ b/src/Avalonia.Controls/Presenters/ItemVirtualizerSimple.cs
@@ -188,8 +188,8 @@ namespace Avalonia.Controls.Presenters
                         break;
 
                     case NotifyCollectionChangedAction.Remove:
-                        if (e.OldStartingIndex >= FirstIndex &&
-                            e.OldStartingIndex < NextIndex)
+                        if ((e.OldStartingIndex >= FirstIndex && e.OldStartingIndex < NextIndex) ||
+                            panel.Children.Count > ItemCount)
                         {
                             RecycleContainersOnRemove();
                         }
diff --git a/tests/Avalonia.Controls.UnitTests/ListBoxTests.cs b/tests/Avalonia.Controls.UnitTests/ListBoxTests.cs
index df2508a3ed..6e87a90ea4 100644
--- a/tests/Avalonia.Controls.UnitTests/ListBoxTests.cs
+++ b/tests/Avalonia.Controls.UnitTests/ListBoxTests.cs
@@ -4,6 +4,7 @@
 using System.Linq;
 using Avalonia.Collections;
 using Avalonia.Controls.Presenters;
+using Avalonia.Controls.Primitives;
 using Avalonia.Controls.Templates;
 using Avalonia.Input;
 using Avalonia.LogicalTree;
@@ -245,6 +246,23 @@ namespace Avalonia.Controls.UnitTests
             Assert.Equal(false, item.IsSelected);
         }
 
+        [Fact]
+        public void Can_Decrease_Number_Of_Materialized_Items_By_Removing_From_Source_Collection()
+        {
+            var items = new AvaloniaList<string>(Enumerable.Range(0, 20).Select(x => $"Item {x}"));
+            var target = new ListBox
+            {
+                Template = ListBoxTemplate(),
+                Items = items,
+                ItemTemplate = new FuncDataTemplate<string>((x, _) => new TextBlock { Height = 10 })
+            };
+
+            Prepare(target);
+            target.Scroll.Offset = new Vector(0, 1);
+
+            items.RemoveRange(0, 11);
+        }
+
         private void RaisePressedEvent(ListBox listBox, ListBoxItem item, MouseButton mouseButton)
         {
             _mouse.Click(listBox, item, mouseButton);
@@ -383,14 +401,26 @@ namespace Avalonia.Controls.UnitTests
         private FuncControlTemplate ScrollViewerTemplate()
         {
             return new FuncControlTemplate<ScrollViewer>((parent, scope) =>
-                new ScrollContentPresenter
+                new Panel
                 {
-                    Name = "PART_ContentPresenter",
-                    [~ScrollContentPresenter.ContentProperty] = parent.GetObservable(ScrollViewer.ContentProperty).ToBinding(),
-                    [~~ScrollContentPresenter.ExtentProperty] = parent[~~ScrollViewer.ExtentProperty],
-                    [~~ScrollContentPresenter.OffsetProperty] = parent[~~ScrollViewer.OffsetProperty],
-                    [~~ScrollContentPresenter.ViewportProperty] = parent[~~ScrollViewer.ViewportProperty],
-                }.RegisterInNameScope(scope));
+                    Children =
+                    {
+                        new ScrollContentPresenter
+                        {
+                            Name = "PART_ContentPresenter",
+                            [~ScrollContentPresenter.ContentProperty] = parent.GetObservable(ScrollViewer.ContentProperty).ToBinding(),
+                            [~~ScrollContentPresenter.ExtentProperty] = parent[~~ScrollViewer.ExtentProperty],
+                            [~~ScrollContentPresenter.OffsetProperty] = parent[~~ScrollViewer.OffsetProperty],
+                            [~~ScrollContentPresenter.ViewportProperty] = parent[~~ScrollViewer.ViewportProperty],
+                        }.RegisterInNameScope(scope),
+                        new ScrollBar
+                        {
+                            Name = "verticalScrollBar",
+                            [~ScrollBar.MaximumProperty] = parent[~ScrollViewer.VerticalScrollBarMaximumProperty],
+                            [~~ScrollBar.ValueProperty] = parent[~~ScrollViewer.VerticalScrollBarValueProperty],
+                        }
+                    }
+                });
         }
 
         private void Prepare(ListBox target)