Restructure VisualLayerManager (#20905)

* refactor: Replace IsPopup with Enable*Layer properties on VisualLayerManager

- Remove IsPopup from VisualLayerManager, add granular Enable*Layer properties:
  EnableAdornerLayer (default true), EnableOverlayLayer (default false),
  EnablePopupOverlayLayer (internal, default false), EnableTextSelectorLayer (default false)
- Add PART_VisualLayerManager template part to TopLevel with protected property
- Window and EmbeddableControlRoot override OnApplyTemplate to enable
  overlay, popup overlay, and text selector layers
- OverlayLayer is now wrapped in a Panel with a dedicated AdornerLayer sibling
- AdornerLayer.GetAdornerLayer checks for OverlayLayer's dedicated AdornerLayer
- Update all 8 XAML templates (both themes) to name PART_VisualLayerManager
  and remove IsPopup="True" from PopupRoot/OverlayPopupHost

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>

* Add XML doc to VisualLayerManager

* Also search for AdornerLayer from TopLevel

---------

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
Co-authored-by: Julien Lebosquain <julien@lebosquain.net>

api/Avalonia.nupkg.xml

@@ -2269,6 +2269,12 @@
     <Left>baseline/Avalonia/lib/net10.0/Avalonia.Controls.dll</Left>
     <Right>current/Avalonia/lib/net10.0/Avalonia.Controls.dll</Right>
   </Suppression>
+  <Suppression>
+    <DiagnosticId>CP0002</DiagnosticId>
+    <Target>M:Avalonia.Controls.Primitives.VisualLayerManager.get_IsPopup</Target>
+    <Left>baseline/Avalonia/lib/net10.0/Avalonia.Controls.dll</Left>
+    <Right>current/Avalonia/lib/net10.0/Avalonia.Controls.dll</Right>
+  </Suppression>
   <Suppression>
     <DiagnosticId>CP0002</DiagnosticId>
     <Target>M:Avalonia.Controls.Primitives.VisualLayerManager.get_LightDismissOverlayLayer</Target>
@@ -2287,6 +2293,12 @@
     <Left>baseline/Avalonia/lib/net10.0/Avalonia.Controls.dll</Left>
     <Right>current/Avalonia/lib/net10.0/Avalonia.Controls.dll</Right>
   </Suppression>
+  <Suppression>
+    <DiagnosticId>CP0002</DiagnosticId>
+    <Target>M:Avalonia.Controls.Primitives.VisualLayerManager.set_IsPopup(System.Boolean)</Target>
+    <Left>baseline/Avalonia/lib/net10.0/Avalonia.Controls.dll</Left>
+    <Right>current/Avalonia/lib/net10.0/Avalonia.Controls.dll</Right>
+  </Suppression>
   <Suppression>
     <DiagnosticId>CP0002</DiagnosticId>
     <Target>M:Avalonia.Controls.Screens.ScreenFromWindow(Avalonia.Platform.IWindowBaseImpl)</Target>
@@ -3871,6 +3883,12 @@
     <Left>baseline/Avalonia/lib/net8.0/Avalonia.Controls.dll</Left>
     <Right>current/Avalonia/lib/net8.0/Avalonia.Controls.dll</Right>
   </Suppression>
+  <Suppression>
+    <DiagnosticId>CP0002</DiagnosticId>
+    <Target>M:Avalonia.Controls.Primitives.VisualLayerManager.get_IsPopup</Target>
+    <Left>baseline/Avalonia/lib/net8.0/Avalonia.Controls.dll</Left>
+    <Right>current/Avalonia/lib/net8.0/Avalonia.Controls.dll</Right>
+  </Suppression>
   <Suppression>
     <DiagnosticId>CP0002</DiagnosticId>
     <Target>M:Avalonia.Controls.Primitives.VisualLayerManager.get_LightDismissOverlayLayer</Target>
@@ -3889,6 +3907,12 @@
     <Left>baseline/Avalonia/lib/net8.0/Avalonia.Controls.dll</Left>
     <Right>current/Avalonia/lib/net8.0/Avalonia.Controls.dll</Right>
   </Suppression>
+  <Suppression>
+    <DiagnosticId>CP0002</DiagnosticId>
+    <Target>M:Avalonia.Controls.Primitives.VisualLayerManager.set_IsPopup(System.Boolean)</Target>
+    <Left>baseline/Avalonia/lib/net8.0/Avalonia.Controls.dll</Left>
+    <Right>current/Avalonia/lib/net8.0/Avalonia.Controls.dll</Right>
+  </Suppression>
   <Suppression>
     <DiagnosticId>CP0002</DiagnosticId>
     <Target>M:Avalonia.Controls.Screens.ScreenFromWindow(Avalonia.Platform.IWindowBaseImpl)</Target>

src/Avalonia.Controls/Embedding/EmbeddableControlRoot.cs

@@ -4,6 +4,7 @@ using Avalonia.Automation.Peers;
 using Avalonia.Controls.Automation;
 using Avalonia.Controls.Automation.Peers;
 using Avalonia.Controls.Platform;
+using Avalonia.Controls.Primitives;
 using Avalonia.Input;
 using Avalonia.Platform;
 
@@ -54,6 +55,12 @@ namespace Avalonia.Controls.Embedding
 
         protected override Type StyleKeyOverride => typeof(EmbeddableControlRoot);
 
+        protected override void OnApplyTemplate(TemplateAppliedEventArgs e)
+        {
+            base.OnApplyTemplate(e);
+            EnableVisualLayerManagerLayers();
+        }
+
         protected override AutomationPeer OnCreateAutomationPeer()
         {
             return new EmbeddableControlRootAutomationPeer(this);

src/Avalonia.Controls/Primitives/AdornerLayer.cs

@@ -1,5 +1,6 @@
 using System;
 using System.Collections.Specialized;
+using System.Linq;
 using Avalonia.Input.TextInput;
 using Avalonia.Media;
 using Avalonia.Reactive;
@@ -71,7 +72,32 @@ namespace Avalonia.Controls.Primitives
 
         public static AdornerLayer? GetAdornerLayer(Visual visual)
         {
-            return visual.FindAncestorOfType<VisualLayerManager>()?.AdornerLayer;
+            // Check if the visual is inside an OverlayLayer with a dedicated AdornerLayer
+            foreach (var ancestor in visual.GetVisualAncestors())
+            {
+                if (GetDirectAdornerLayer(ancestor) is { } adornerLayer)
+                    return adornerLayer;
+            }
+
+            if (TopLevel.GetTopLevel(visual) is { } topLevel)
+            {
+                foreach (var descendant in topLevel.GetVisualDescendants())
+                {
+                    if (GetDirectAdornerLayer(descendant) is { } adornerLayer)
+                        return adornerLayer;
+                }
+            }
+
+            return null;
+
+            static AdornerLayer? GetDirectAdornerLayer(Visual visual)
+            {
+                if (visual is OverlayLayer { AdornerLayer: { } adornerLayer })
+                    return adornerLayer;
+                if (visual is VisualLayerManager vlm)
+                    return vlm.AdornerLayer;
+                return null;
+            }
         }
 
         public static bool GetIsClipEnabled(Visual adorner)

src/Avalonia.Controls/Primitives/OverlayLayer.cs

@@ -13,6 +13,11 @@ namespace Avalonia.Controls.Primitives
 
         public Size AvailableSize { get; private set; }
 
+        /// <summary>
+        /// Gets the dedicated adorner layer for this overlay layer.
+        /// </summary>
+        internal AdornerLayer? AdornerLayer { get; set; }
+
         internal OverlayLayer()
         {
         }

src/Avalonia.Controls/Primitives/VisualLayerManager.cs

@@ -3,6 +3,9 @@ using Avalonia.LogicalTree;
 
 namespace Avalonia.Controls.Primitives
 {
+    /// <summary>
+    /// A control that manages multiple layers such as adorners, overlays, text selectors, and popups.
+    /// </summary>
     public sealed class VisualLayerManager : Decorator
     {
         private const int AdornerZIndex = int.MaxValue - 100;
@@ -13,13 +16,37 @@ namespace Avalonia.Controls.Primitives
 
         private ILogicalRoot? _logicalRoot;
         private readonly List<Control> _layers = new();
-
-        public bool IsPopup { get; set; }
-
-        internal AdornerLayer AdornerLayer
+        private OverlayLayer? _overlayLayer;
+
+        /// <summary>
+        /// Gets or sets a value indicating whether an <see cref="Avalonia.Controls.Primitives.AdornerLayer"/> is
+        /// created for this <see cref="VisualLayerManager"/>. When enabled, the adorner layer is added to the
+        /// visual tree, providing a dedicated layer for rendering adorners.
+        /// </summary>
+        public bool EnableAdornerLayer { get; set; } = true;
+
+        /// <summary>
+        /// Gets or sets a value indicating whether an <see cref="Avalonia.Controls.Primitives.OverlayLayer"/> is
+        /// created for this <see cref="VisualLayerManager"/>. When enabled, the overlay layer is added to the
+        /// visual tree, providing a dedicated layer for rendering overlay visuals.
+        /// </summary>
+        public bool EnableOverlayLayer { get; set; }
+
+        internal bool EnablePopupOverlayLayer { get; set; }
+
+        /// <summary>
+        /// Gets or sets a value indicating whether a <see cref="Avalonia.Controls.Primitives.TextSelectorLayer"/> is
+        /// created for this <see cref="VisualLayerManager"/>. When enabled, the overlay layer is added to the
+        /// visual tree, providing a dedicated layer for rendering text selection handles.
+        /// </summary>
+        public bool EnableTextSelectorLayer { get; set; }
+
+        internal AdornerLayer? AdornerLayer
         {
             get
             {
+                if (!EnableAdornerLayer)
+                    return null;
                 var rv = FindLayer<AdornerLayer>();
                 if (rv == null)
                     AddLayer(rv = new AdornerLayer(), AdornerZIndex);
@@ -31,7 +58,7 @@ namespace Avalonia.Controls.Primitives
         {
             get
             {
-                if (IsPopup)
+                if (!EnablePopupOverlayLayer)
                     return null;
                 var rv = FindLayer<PopupOverlayLayer>();
                 if (rv == null)
@@ -44,12 +71,21 @@ namespace Avalonia.Controls.Primitives
         {
             get
             {
-                if (IsPopup)
+                if (!EnableOverlayLayer)
                     return null;
-                var rv = FindLayer<OverlayLayer>();
-                if (rv == null)
-                    AddLayer(rv = new OverlayLayer(), OverlayZIndex);
-                return rv;
+                if (_overlayLayer == null)
+                {
+                    _overlayLayer = new OverlayLayer();
+                    var adorner = new AdornerLayer();
+                    _overlayLayer.AdornerLayer = adorner;
+                    
+                    var panel = new Panel();
+                    panel.Children.Add(_overlayLayer);
+                    panel.Children.Add(adorner);
+                    
+                    AddLayer(panel, OverlayZIndex);
+                }
+                return _overlayLayer;
             }
         }
 
@@ -57,7 +93,7 @@ namespace Avalonia.Controls.Primitives
         {
             get
             {
-                if (IsPopup)
+                if (!EnableTextSelectorLayer)
                     return null;
                 var rv = FindLayer<TextSelectorLayer>();
                 if (rv == null)

src/Avalonia.Controls/TopLevel.cs

@@ -38,6 +38,7 @@ namespace Avalonia.Controls
     /// tracking the widget's <see cref="ClientSize"/>.
     /// </remarks>
     [TemplatePart("PART_TransparencyFallback", typeof(Border))]
+    [TemplatePart("PART_VisualLayerManager", typeof(VisualLayerManager))]
     public abstract class TopLevel : ContentControl,
         ICloseable,
         IStyleHost,
@@ -125,6 +126,7 @@ namespace Avalonia.Controls
         private Size? _frameSize;
         private WindowTransparencyLevel _actualTransparencyLevel;
         private Border? _transparencyFallbackBorder;
+        private VisualLayerManager? _visualLayerManager;
         private TargetWeakEventSubscriber<TopLevel, ResourcesChangedEventArgs>? _resourcesChangesSubscriber;
         private IStorageProvider? _storageProvider;
         private Screens? _screens;
@@ -133,6 +135,18 @@ namespace Avalonia.Controls
         internal TopLevelHost TopLevelHost => _topLevelHost;
         internal new PresentationSource PresentationSource => _source;
         internal IInputRoot InputRoot => _source;
+        
+        private protected VisualLayerManager? VisualLayerManager => _visualLayerManager;
+
+        private protected void EnableVisualLayerManagerLayers()
+        {
+            if (_visualLayerManager is { } vlm)
+            {
+                vlm.EnableOverlayLayer = true;
+                vlm.EnablePopupOverlayLayer = true;
+                vlm.EnableTextSelectorLayer = true;
+            }
+        }
 
         /// <summary>
         /// Initializes static members of the <see cref="TopLevel"/> class.
@@ -723,6 +737,8 @@ namespace Avalonia.Controls
         {
             base.OnApplyTemplate(e);
 
+            _visualLayerManager = e.NameScope.Find<VisualLayerManager>("PART_VisualLayerManager");
+
             if (PlatformImpl is null)
                 return;
 

src/Avalonia.Controls/Window.cs

@@ -6,6 +6,7 @@ using System.Threading.Tasks;
 using Avalonia.Automation.Peers;
 using Avalonia.Controls.Chrome;
 using Avalonia.Controls.Platform;
+using Avalonia.Controls.Primitives;
 using Avalonia.Input;
 using Avalonia.Interactivity;
 using Avalonia.Layout;
@@ -795,6 +796,12 @@ namespace Avalonia.Controls
             ShowCore<object>(null, false);
         }
 
+        protected override void OnApplyTemplate(TemplateAppliedEventArgs e)
+        {
+            base.OnApplyTemplate(e);
+            EnableVisualLayerManagerLayers();
+        }
+
         protected override void IsVisibleChanged(AvaloniaPropertyChangedEventArgs e)
         {
             if (!IgnoreVisibilityChanges)

src/Avalonia.Themes.Fluent/Controls/EmbeddableControlRoot.xaml

@@ -12,7 +12,7 @@
         <Panel>
           <Border Name="PART_TransparencyFallback" IsHitTestVisible="False" />
           <Border Background="{TemplateBinding Background}">
-            <VisualLayerManager>
+            <VisualLayerManager Name="PART_VisualLayerManager">
               <ContentPresenter Name="PART_ContentPresenter"
                                 ContentTemplate="{TemplateBinding ContentTemplate}"
                                 Content="{TemplateBinding Content}"

src/Avalonia.Themes.Fluent/Controls/OverlayPopupHost.xaml

@@ -11,7 +11,7 @@
     <Setter Property="Template">
       <ControlTemplate>
         <LayoutTransformControl LayoutTransform="{TemplateBinding Transform}">
-          <VisualLayerManager IsPopup="True">
+          <VisualLayerManager Name="PART_VisualLayerManager">
             <ContentPresenter Name="PART_ContentPresenter"
                               Background="{TemplateBinding Background}"
                               ContentTemplate="{TemplateBinding ContentTemplate}"

src/Avalonia.Themes.Fluent/Controls/PopupRoot.xaml

@@ -15,7 +15,7 @@
         <LayoutTransformControl LayoutTransform="{TemplateBinding Transform}">
           <Panel>
             <Border Name="PART_TransparencyFallback" IsHitTestVisible="False" />
-            <VisualLayerManager IsPopup="True">
+            <VisualLayerManager Name="PART_VisualLayerManager">
               <ContentPresenter Name="PART_ContentPresenter"
                                 Background="{TemplateBinding Background}"
                                 ContentTemplate="{TemplateBinding ContentTemplate}"

src/Avalonia.Themes.Fluent/Controls/Window.xaml

@@ -14,7 +14,7 @@
           <Border Name="PART_TransparencyFallback" IsHitTestVisible="False" />
           <Border Background="{TemplateBinding Background}" IsHitTestVisible="False" />
           <Panel Background="Transparent" Margin="{TemplateBinding WindowDecorationMargin}" />
-          <VisualLayerManager>
+          <VisualLayerManager Name="PART_VisualLayerManager">
             <ContentPresenter Name="PART_ContentPresenter"
                               ContentTemplate="{TemplateBinding ContentTemplate}"
                               Content="{TemplateBinding Content}"

src/Avalonia.Themes.Simple/Controls/EmbeddableControlRoot.xaml

@@ -13,7 +13,7 @@
           <Border Name="PART_TransparencyFallback"
                   IsHitTestVisible="False" />
           <Border Background="{TemplateBinding Background}">
-            <VisualLayerManager>
+            <VisualLayerManager Name="PART_VisualLayerManager">
               <ContentPresenter Name="PART_ContentPresenter"
                                 Margin="{TemplateBinding Padding}"
                                 Content="{TemplateBinding Content}"

src/Avalonia.Themes.Simple/Controls/OverlayPopupHost.xaml

@@ -13,7 +13,7 @@
       <ControlTemplate>
         <!--  Do not forget to update Templated_Control_With_Popup_In_Template_Should_Set_TemplatedParent test  -->
         <LayoutTransformControl LayoutTransform="{TemplateBinding Transform}">
-          <VisualLayerManager IsPopup="True">
+          <VisualLayerManager Name="PART_VisualLayerManager">
             <ContentPresenter Name="PART_ContentPresenter"
                               Padding="{TemplateBinding Padding}"
                               Background="{TemplateBinding Background}"

src/Avalonia.Themes.Simple/Controls/PopupRoot.xaml

@@ -17,7 +17,7 @@
           <Panel>
             <Border Name="PART_TransparencyFallback"
                     IsHitTestVisible="False" />
-            <VisualLayerManager IsPopup="True">
+            <VisualLayerManager Name="PART_VisualLayerManager">
               <ContentPresenter Name="PART_ContentPresenter"
                                 Padding="{TemplateBinding Padding}"
                                 Background="{TemplateBinding Background}"

src/Avalonia.Themes.Simple/Controls/Window.xaml

@@ -17,7 +17,7 @@
                   IsHitTestVisible="False" />
           <Panel Margin="{TemplateBinding WindowDecorationMargin}"
                  Background="Transparent" />
-          <VisualLayerManager>
+          <VisualLayerManager Name="PART_VisualLayerManager">
             <ContentPresenter Name="PART_ContentPresenter"
                               Margin="{TemplateBinding Padding}"
                               HorizontalContentAlignment="{TemplateBinding HorizontalContentAlignment}"

tests/Avalonia.Controls.UnitTests/Primitives/VisualLayerManagerTests.cs

@@ -0,0 +1,37 @@
+using Avalonia.Controls.Primitives;
+using Avalonia.UnitTests;
+using Xunit;
+
+namespace Avalonia.Controls.UnitTests.Primitives
+{
+    public class VisualLayerManagerTests : ScopedTestBase
+    {
+        [Fact]
+        public void GetAdornerLayer_Returns_Dedicated_AdornerLayer_For_Controls_Inside_OverlayLayer()
+        {
+            var button = new Button();
+            var vlm = new VisualLayerManager { EnableOverlayLayer = true, Child = button };
+            var root = new TestRoot { Child = vlm };
+
+            root.Measure(new Size(100, 100));
+            root.Arrange(new Rect(0, 0, 100, 100));
+
+            var overlayLayer = vlm.OverlayLayer;
+            Assert.NotNull(overlayLayer);
+
+            var overlayChild = new Border();
+            overlayLayer.Children.Add(overlayChild);
+
+            // The adorner layer for a control inside the OverlayLayer
+            // should be the dedicated one, not the main VLM adorner layer.
+            var overlayAdornerLayer = AdornerLayer.GetAdornerLayer(overlayChild);
+            Assert.NotNull(overlayAdornerLayer);
+            Assert.Same(overlayLayer.AdornerLayer, overlayAdornerLayer);
+
+            // The main VLM adorner layer should be different.
+            var mainAdornerLayer = AdornerLayer.GetAdornerLayer(button);
+            Assert.NotNull(mainAdornerLayer);
+            Assert.NotSame(overlayAdornerLayer, mainAdornerLayer);
+        }
+    }
+}