diff --git a/src/Avalonia.Controls/Primitives/SelectingItemsControl.cs b/src/Avalonia.Controls/Primitives/SelectingItemsControl.cs
index b6ae567123..329b086a7c 100644
--- a/src/Avalonia.Controls/Primitives/SelectingItemsControl.cs
+++ b/src/Avalonia.Controls/Primitives/SelectingItemsControl.cs
@@ -367,17 +367,17 @@ namespace Avalonia.Controls.Primitives
             {
                 if ((container.ContainerControl as ISelectable)?.IsSelected == true)
                 {
-                    if (SelectedIndex == -1)
-                    {
-                        SelectedIndex = container.Index;
-                    }
-                    else
+                    if (SelectionMode.HasFlag(SelectionMode.Multiple))
                     {
                         if (_selection.Add(container.Index))
                         {
                             resetSelectedItems = true;
                         }
                     }
+                    else
+                    {
+                        SelectedIndex = container.Index;
+                    }
 
                     MarkContainerSelected(container.ContainerControl, true);
                 }