Improve updates of Win32 window's WindowStyles (#12752)

* compute win32 window styles from properties, instead of toggling relevant styles

* save client rect early, before swithing to full screen

* add callbacks for window style and wndproc

* move new options class to Avalonia.Controls

* use toplevel instance to set the callbacks instead

* remove redundant casting

* call wndproc callback before the default handler

* remove unused using

* ensure window state when decorations changed

* move win32 specific delegates

* make Win32SpecificOptions static

* fix build

* replace Set callbacks with Add/Remove callbacks

* fix docs

* only recreate styles when the change isn't caused by a fullscreen update

src/Avalonia.Controls/Platform/IWin32OptionsTopLevelImpl.cs

@@ -0,0 +1,25 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+using Avalonia.Metadata;
+using Avalonia.Platform;
+using static Avalonia.Controls.Platform.Win32SpecificOptions;
+
+namespace Avalonia.Controls.Platform
+{
+    [PrivateApi]
+    public interface IWin32OptionsTopLevelImpl : ITopLevelImpl
+    {
+        /// <summary>
+        /// Gets or sets a callback to set the window styles. 
+        /// </summary>
+        public CustomWindowStylesCallback? WindowStylesCallback { get; set; }
+
+        /// <summary>
+        /// Gets or sets a custom callback for the window's WndProc
+        /// </summary>
+        public CustomWndProcHookCallback? WndProcHookCallback { get; set; }
+    }
+}

src/Avalonia.Controls/Platform/Win32SpecificOptions.cs

@@ -0,0 +1,70 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+using Avalonia.Controls;
+using Avalonia.Metadata;
+using Avalonia.Platform;
+using static Avalonia.Controls.Platform.IWin32OptionsTopLevelImpl;
+
+namespace Avalonia.Controls.Platform
+{
+    public static class Win32SpecificOptions
+    {
+        public delegate (uint style, uint exStyle) CustomWindowStylesCallback(uint style, uint exStyle);
+        public delegate IntPtr CustomWndProcHookCallback(IntPtr hWnd, uint msg, IntPtr wParam, IntPtr lParam, ref bool handled);
+
+        /// <summary>
+        /// Adds a callback to set the window's style.
+        /// </summary>
+        /// <param name="topLevel">The window implementation</param>
+        /// <param name="callback">The callback</param>
+        public static void AddWindowStylesCallback(TopLevel topLevel, CustomWindowStylesCallback? callback)
+        {
+            if (topLevel.PlatformImpl is IWin32OptionsTopLevelImpl toplevelImpl)
+            {
+                toplevelImpl.WindowStylesCallback += callback;
+            }
+        }
+
+        /// <summary>
+        /// Removes a callback to set the window's style.
+        /// </summary>
+        /// <param name="topLevel">The window implementation</param>
+        /// <param name="callback">The callback</param>
+        public static void RemoveWindowStylesCallback(TopLevel topLevel, CustomWindowStylesCallback? callback)
+        {
+            if (topLevel.PlatformImpl is IWin32OptionsTopLevelImpl toplevelImpl)
+            {
+                toplevelImpl.WindowStylesCallback -= callback;
+            }
+        }
+
+        /// <summary>
+        /// Adds a custom callback for the window's WndProc
+        /// </summary>
+        /// <param name="topLevel">The window</param>
+        /// <param name="callback">The callback</param>
+        public static void AddWndProcHookCallback(TopLevel topLevel, CustomWndProcHookCallback? callback)
+        {
+            if (topLevel.PlatformImpl is IWin32OptionsTopLevelImpl toplevelImpl)
+            {
+                toplevelImpl.WndProcHookCallback += callback;
+            }
+        }
+
+        /// <summary>
+        /// Removes a custom callback for the window's WndProc
+        /// </summary>
+        /// <param name="topLevel">The window</param>
+        /// <param name="callback">The callback</param>
+        public static void RemoveWndProcHookCallback(TopLevel topLevel, CustomWndProcHookCallback? callback)
+        {
+            if (topLevel.PlatformImpl is IWin32OptionsTopLevelImpl toplevelImpl)
+            {
+                toplevelImpl.WndProcHookCallback -= callback;
+            }
+        }
+    }
+}

src/Windows/Avalonia.Win32/WindowImpl.AppWndProc.cs

@@ -59,7 +59,7 @@ namespace Avalonia.Win32
 
                 case WindowsMessage.WM_NCCALCSIZE:
                     {
-                        if (ToInt32(wParam) == 1 && _windowProperties.Decorations == SystemDecorations.None ||  _isClientAreaExtended)
+                        if (ToInt32(wParam) == 1 && (_windowProperties.Decorations == SystemDecorations.None || _isClientAreaExtended))
                         {
                             return IntPtr.Zero;
                         }

src/Windows/Avalonia.Win32/WindowImpl.WndProc.cs

@@ -2,6 +2,7 @@
 // Licensed under the MIT license. See licence.md file in the project root for full license information.
 
 using System;
+using System.Runtime.InteropServices;
 using Avalonia.Controls.Platform;
 
 namespace Avalonia.Win32

src/Windows/Avalonia.Win32/WindowImpl.cs

@@ -26,13 +26,15 @@ using Avalonia.Win32.WinRT;
 using static Avalonia.Win32.Interop.UnmanagedMethods;
 using Avalonia.Input.Platform;
 using System.Diagnostics;
+using static Avalonia.Controls.Platform.IWin32OptionsTopLevelImpl;
+using static Avalonia.Controls.Platform.Win32SpecificOptions;
 
 namespace Avalonia.Win32
 {
     /// <summary>
     /// Window implementation for Win32 platform.
     /// </summary>
-    internal partial class WindowImpl : IWindowImpl, EglGlPlatformSurface.IEglWindowGlPlatformSurfaceInfo
+    internal partial class WindowImpl : IWindowImpl, EglGlPlatformSurface.IEglWindowGlPlatformSurfaceInfo, IWin32OptionsTopLevelImpl
     {
         private static readonly List<WindowImpl> s_instances = new();
 
@@ -823,7 +825,7 @@ namespace Avalonia.Win32
         private void CreateWindow()
         {
             // Ensure that the delegate doesn't get garbage collected by storing it as a field.
-            _wndProcDelegate = WndProc;
+            _wndProcDelegate = WndProcMessageHandler;
 
             _className = $"Avalonia-{Guid.NewGuid().ToString()}";
 
@@ -876,6 +878,20 @@ namespace Avalonia.Win32
             }
         }
 
+        private IntPtr WndProcMessageHandler(IntPtr hWnd, uint msg, IntPtr wParam, IntPtr lParam)
+        {
+            bool handled = false;
+            IntPtr ret = IntPtr.Zero;
+
+            if (WndProcHookCallback is { } callback)
+                ret = callback(hWnd, msg, wParam, lParam, ref handled);
+
+            if (handled)
+                return ret;
+
+            return WndProc(hWnd, msg, wParam, lParam);
+        }
+
         private void CreateDropTarget(IInputRoot inputRoot)
         {
             if (AvaloniaLocator.Current.GetService<IDragDropDevice>() is { } dragDropDevice)
@@ -905,7 +921,7 @@ namespace Avalonia.Win32
                 clientRect.right += windowRect.left;
                 clientRect.top += windowRect.top;
                 clientRect.bottom += windowRect.top;
-                
+
                 _savedWindowInfo.WindowRect = clientRect;
 
                 var current = GetStyle();
@@ -1256,51 +1272,60 @@ namespace Avalonia.Win32
             // according to the new values already.
             _windowProperties = newProperties;
 
-            if ((oldProperties.ShowInTaskbar != newProperties.ShowInTaskbar) || forceChanges)
+            if (oldProperties.IsFullScreen == newProperties.IsFullScreen)
             {
-                var exStyle = GetExtendedStyle();
+                var exStyle = WindowStyles.WS_EX_WINDOWEDGE | (_isUsingComposition ? WindowStyles.WS_EX_NOREDIRECTIONBITMAP : 0);
 
-                if (newProperties.ShowInTaskbar)
+                if ((oldProperties.ShowInTaskbar != newProperties.ShowInTaskbar) || forceChanges)
                 {
-                    exStyle |= WindowStyles.WS_EX_APPWINDOW;
-
-                    if (_hiddenWindowIsParent)
+                    if (newProperties.ShowInTaskbar)
                     {
-                        // Can't enable the taskbar icon by clearing the parent window unless the window
-                        // is hidden. Hide the window and show it again with the same activation state
-                        // when we've finished. Interestingly it seems to work fine the other way.
-                        var shown = IsWindowVisible(_hwnd);
-                        var activated = GetActiveWindow() == _hwnd;
+                        exStyle |= WindowStyles.WS_EX_APPWINDOW;
+
+                        if (_hiddenWindowIsParent)
+                        {
+                            // Can't enable the taskbar icon by clearing the parent window unless the window
+                            // is hidden. Hide the window and show it again with the same activation state
+                            // when we've finished. Interestingly it seems to work fine the other way.
+                            var shown = IsWindowVisible(_hwnd);
+                            var activated = GetActiveWindow() == _hwnd;
 
-                        if (shown)
-                            Hide();
+                            if (shown)
+                                Hide();
 
-                        _hiddenWindowIsParent = false;
-                        SetParent(null);
+                            _hiddenWindowIsParent = false;
+                            SetParent(null);
 
-                        if (shown)
-                            Show(activated, false);
+                            if (shown)
+                                Show(activated, false);
+                        }
                     }
-                }
-                else
-                {
-                    // To hide a non-owned window's taskbar icon we need to parent it to a hidden window.
-                    if (_parent is null)
+                    else
                     {
-                        SetWindowLongPtr(_hwnd, (int)WindowLongParam.GWL_HWNDPARENT, OffscreenParentWindow.Handle);
-                        _hiddenWindowIsParent = true;
+                        // To hide a non-owned window's taskbar icon we need to parent it to a hidden window.
+                        if (_parent is null)
+                        {
+                            SetWindowLongPtr(_hwnd, (int)WindowLongParam.GWL_HWNDPARENT, OffscreenParentWindow.Handle);
+                            _hiddenWindowIsParent = true;
+                        }
+
+                        exStyle &= ~WindowStyles.WS_EX_APPWINDOW;
                     }
+                }
 
+                if (newProperties.ShowInTaskbar)
+                {
+                    exStyle |= WindowStyles.WS_EX_APPWINDOW;
+                }
+                else
+                {
                     exStyle &= ~WindowStyles.WS_EX_APPWINDOW;
                 }
 
-                SetExtendedStyle(exStyle);
-            }
+                WindowStyles style = WindowStyles.WS_CLIPCHILDREN | WindowStyles.WS_OVERLAPPEDWINDOW | WindowStyles.WS_CLIPSIBLINGS;
 
-            WindowStyles style;
-            if ((oldProperties.IsResizable != newProperties.IsResizable) || forceChanges)
-            {
-                style = GetStyle();
+                if (IsWindowVisible(_hwnd))
+                    style |= WindowStyles.WS_VISIBLE;
 
                 if (newProperties.IsResizable)
                 {
@@ -1313,18 +1338,6 @@ namespace Avalonia.Win32
                     style &= ~WindowStyles.WS_MAXIMIZEBOX;
                 }
 
-                SetStyle(style);
-            }
-
-            if (oldProperties.IsFullScreen != newProperties.IsFullScreen)
-            {
-                SetFullScreen(newProperties.IsFullScreen);
-            }
-
-            if ((oldProperties.Decorations != newProperties.Decorations) || forceChanges)
-            {
-                style = GetStyle();
-
                 const WindowStyles fullDecorationFlags = WindowStyles.WS_CAPTION | WindowStyles.WS_SYSMENU;
 
                 if (newProperties.Decorations == SystemDecorations.Full)
@@ -1334,39 +1347,70 @@ namespace Avalonia.Win32
                 else
                 {
                     style &= ~fullDecorationFlags;
+
+                    if (newProperties.Decorations == SystemDecorations.BorderOnly)
+                    {
+                        style |= WindowStyles.WS_THICKFRAME | WindowStyles.WS_BORDER;
+                    }
                 }
 
-                SetStyle(style);
+                var windowStates = GetWindowStateStyles();
+                style &= ~WindowStateMask;
+                style |= windowStates;
 
-                if (!_isFullScreenActive)
+                _savedWindowInfo.Style = style;
+                _savedWindowInfo.ExStyle = exStyle;
+
+                if (WindowStylesCallback is { } callback)
                 {
-                    var margins = new MARGINS
-                    {
-                        cyBottomHeight = 0,
-                        cxRightWidth = 0,
-                        cxLeftWidth = 0,
-                        cyTopHeight = 0
-                    };
+                    var (s, e) = callback((uint)style, (uint)exStyle);
 
-                    DwmExtendFrameIntoClientArea(_hwnd, ref margins);
+                    style = (WindowStyles)s;
+                    exStyle = (WindowStyles)e;
+                }
 
-                    GetClientRect(_hwnd, out var oldClientRect);
-                    var oldClientRectOrigin = new POINT();
-                    ClientToScreen(_hwnd, ref oldClientRectOrigin);
-                    oldClientRect.Offset(oldClientRectOrigin);
+                SetStyle(style);
+                SetExtendedStyle(exStyle);
+            }
+            else
+                SetFullScreen(newProperties.IsFullScreen);
 
-                    var newRect = oldClientRect;
+            if (!_isFullScreenActive)
+            {
+                var style = GetStyle();
 
-                    if (newProperties.Decorations == SystemDecorations.Full)
-                    {
-                        AdjustWindowRectEx(ref newRect, (uint)style, false, (uint)GetExtendedStyle());
-                    }
+                var margin = newProperties.Decorations == SystemDecorations.BorderOnly ? 1 : 0;
+
+                var margins = new MARGINS
+                {
+                    cyBottomHeight = margin,
+                    cxRightWidth = margin,
+                    cxLeftWidth = margin,
+                    cyTopHeight = margin
+                };
+
+                DwmExtendFrameIntoClientArea(_hwnd, ref margins);
+
+                GetClientRect(_hwnd, out var oldClientRect);
+                var oldClientRectOrigin = new POINT();
+                ClientToScreen(_hwnd, ref oldClientRectOrigin);
+                oldClientRect.Offset(oldClientRectOrigin);
+
+                var newRect = oldClientRect;
 
-                    SetWindowPos(_hwnd, IntPtr.Zero, newRect.left, newRect.top, newRect.Width, newRect.Height,
-                        SetWindowPosFlags.SWP_NOZORDER | SetWindowPosFlags.SWP_NOACTIVATE |
-                        SetWindowPosFlags.SWP_FRAMECHANGED);
+                if (newProperties.Decorations == SystemDecorations.Full)
+                {
+                    AdjustWindowRectEx(ref newRect, (uint)style, false, (uint)GetExtendedStyle());
                 }
+
+                SetWindowPos(_hwnd, IntPtr.Zero, newRect.left, newRect.top, newRect.Width, newRect.Height,
+                    SetWindowPosFlags.SWP_NOZORDER | SetWindowPosFlags.SWP_NOACTIVATE |
+                    SetWindowPosFlags.SWP_FRAMECHANGED);
             }
+
+            // Ensure window state if decorations change
+            if (oldProperties.Decorations != newProperties.Decorations)
+                ShowWindow(WindowState, false);
         }
 
         private const int MF_BYCOMMAND = 0x0;
@@ -1452,6 +1496,12 @@ namespace Avalonia.Win32
         /// <inheritdoc/>
         public AcrylicPlatformCompensationLevels AcrylicCompensationLevels { get; } = new AcrylicPlatformCompensationLevels(1, 0.8, 0);
 
+        /// <inheritdoc/>
+        public CustomWindowStylesCallback? WindowStylesCallback { get; set; }
+
+        /// <inheritdoc/>
+        public CustomWndProcHookCallback? WndProcHookCallback { get; set; }
+
         private ResizeReasonScope SetResizeReason(WindowResizeReason reason)
         {
             var old = _resizeReason;