Fix missing Win32 system menu on title bar right-click when ExtendClientAreaToDecorationsHint is enabled (#21630)

* Fix Win32 system menu for extended client area title bars

* Fix Win32 system menu behavior in extended client area and fullscreen

* Improve  system menu display and default item behavior

* Clean up

* Fix SetMenuDefaultItem P/Invoke signature

* Fix system menu handling for Space key

* Allow controls to handle non-client right-click events

src/Windows/Avalonia.Win32/Interop/UnmanagedMethods.cs

@@ -967,6 +967,13 @@ namespace Avalonia.Win32.Interop
             SCF_ISSECURE = 0x00000001,
         }
 
+        [Flags]
+        public enum TrackPopupMenuFlags : uint
+        {
+            TPM_RIGHTBUTTON = 0x0002,
+            TPM_RETURNCMD = 0x0100,
+        }
+
         [Flags]
         public enum PointerFlags
         {
@@ -1398,9 +1405,16 @@ namespace Avalonia.Win32.Interop
         [DllImport("user32.dll")]
         public static extern IntPtr GetSystemMenu(IntPtr hWnd, bool bRevert);
 
+        [DllImport("user32.dll", SetLastError = true)]
+        public static extern uint TrackPopupMenu(IntPtr hMenu, TrackPopupMenuFlags uFlags, int x, int y, int nReserved,
+            IntPtr hWnd, IntPtr prcRect);
+
         [DllImport("user32.dll")]
         public static extern bool EnableMenuItem(IntPtr hMenu, uint uIDEnableItem, uint uEnable);
 
+        [DllImport("user32.dll")]
+        public static extern bool SetMenuDefaultItem(IntPtr hMenu, uint uItem, uint fByPos);
+
         [DllImport("user32.dll", SetLastError = true)]
         public static extern bool GetWindowPlacement(IntPtr hWnd, out WINDOWPLACEMENT lpwndpl);
 

src/Windows/Avalonia.Win32/WindowImpl.AppWndProc.cs

@@ -238,7 +238,7 @@ namespace Avalonia.Win32
 
                 case WindowsMessage.WM_SYSCOMMAND:
                     // Disable system handling of Alt/F10 menu keys.
-                    if ((SysCommands)wParam == SysCommands.SC_KEYMENU && HighWord(ToInt32(lParam)) <= 0)
+                    if (GetSysCommand(wParam) == SysCommands.SC_KEYMENU && HighWord(ToInt32(lParam)) <= 0)
                         return IntPtr.Zero;
                     break;
 
@@ -1349,7 +1349,7 @@ namespace Avalonia.Win32
 
             Imm32InputMethod.Current.SetLanguageAndWindow(this, Hwnd, hkl);
         }
-        
+
         // GetPointerDeviceRects is part of the WM_POINTER API (Windows 8+) but is not implemented
         // by Wine/Proton. Probe once and fall back to the integer pixel location when missing,
         // otherwise the P/Invoke throws EntryPointNotFoundException for every pointer message.
@@ -1394,6 +1394,8 @@ namespace Avalonia.Win32
 
         private static int HighWord(int param) => param >> 16;
 
+        private static SysCommands GetSysCommand(IntPtr wParam) => (SysCommands)(ToInt32(wParam) & 0xfff0);
+
         private Point DipFromLParam(IntPtr lParam)
         {
             return new Point((short)(ToInt32(lParam) & 0xffff), (short)(ToInt32(lParam) >> 16)) / RenderScaling;

src/Windows/Avalonia.Win32/WindowImpl.CustomCaptionProc.cs

@@ -122,6 +122,14 @@ namespace Avalonia.Win32
                     }
                     break;
 
+                case WindowsMessage.WM_NCRBUTTONUP when (HitTestValues)ToInt32(wParam) == HitTestValues.HTCAPTION:
+                    ShowSystemMenu(PointFromLParam(lParam));
+                    break;
+
+                case WindowsMessage.WM_INITMENU:
+                    UpdateSystemMenu(GetSystemMenu(hWnd, false));
+                    break;
+
                 // Normally, Avalonia doesn't handles non-client input as a special NonClientLeftButtonDown, ignoring move and up events.
                 // What makes it a problem, Avalonia has to mark templated caption buttons as a non-client area.
                 // Meaning, these buttons no longer can accept normal client input.
@@ -224,7 +232,7 @@ namespace Avalonia.Win32
         private HitTestValues HitTestVisual(IntPtr lParam)
         {
             var position = PointToClient(PointFromLParam(lParam));
-            
+
             // First, check new cross-platform ElementRole via chrome hit-test
             if (_owner is IInputRoot inputRoot)
             {
@@ -254,9 +262,9 @@ namespace Avalonia.Win32
                     };
                 }
             }
-            
+
             // Fall back to Win32-specific NonClientHitTestResult attached property
-            if (_owner?.RootElement is {} window)
+            if (_owner?.RootElement is { } window)
             {
                 var visual = window.GetVisualAt(position, x =>
                 {
@@ -292,5 +300,81 @@ namespace Avalonia.Win32
                 or HitTestValues.HTMENU
                 or HitTestValues.HTSYSMENU;
         }
+
+        private void ShowSystemMenu(PixelPoint screenPoint)
+        {
+            var menu = GetSystemMenu(_hwnd, false);
+            if (menu == IntPtr.Zero)
+            {
+                return;
+            }
+
+            SetForegroundWindow(_hwnd);
+
+            var command = TrackPopupMenu(
+                menu,
+                TrackPopupMenuFlags.TPM_RIGHTBUTTON | TrackPopupMenuFlags.TPM_RETURNCMD,
+                screenPoint.X,
+                screenPoint.Y,
+                0,
+                _hwnd,
+                IntPtr.Zero);
+
+            PostMessage(_hwnd, (uint)WindowsMessage.WM_NULL, IntPtr.Zero, IntPtr.Zero);
+
+            if (command != 0)
+            {
+                if ((SysCommands)command == SysCommands.SC_RESTORE && WindowState == WindowState.FullScreen)
+                {
+                    WindowState = WindowState.Normal;
+                }
+                else
+                {
+                    SendMessage(_hwnd, (int)WindowsMessage.WM_SYSCOMMAND, (IntPtr)command, MakeLParam(screenPoint));
+                }
+            }
+        }
+
+        private void UpdateSystemMenu(IntPtr menu)
+        {
+            if (menu == IntPtr.Zero)
+            {
+                return;
+            }
+
+            var state = WindowState;
+            var isMinimized = state == WindowState.Minimized;
+            var isMaximized = state == WindowState.Maximized;
+            var isFullScreen = state == WindowState.FullScreen;
+            var isNormal = state == WindowState.Normal;
+
+            SetSystemMenuItemEnabled(menu, SysCommands.SC_RESTORE, isMinimized || isMaximized || isFullScreen);
+            SetSystemMenuItemEnabled(menu, SysCommands.SC_MOVE, isNormal || isFullScreen);
+            SetSystemMenuItemEnabled(menu, SysCommands.SC_SIZE, isNormal && _windowProperties.IsResizable);
+            SetSystemMenuItemEnabled(menu, SysCommands.SC_MINIMIZE, !isMinimized && !isFullScreen && _windowProperties.IsMinimizable);
+            SetSystemMenuItemEnabled(menu, SysCommands.SC_MAXIMIZE, !isMaximized && !isFullScreen && _windowProperties.IsMaximizable);
+
+            if (isMinimized || isMaximized || isFullScreen)
+            {
+                SetMenuDefaultItem(menu, (uint)SysCommands.SC_RESTORE, 0u);
+            }
+            else if (isNormal && _windowProperties.IsMaximizable)
+            {
+                SetMenuDefaultItem(menu, (uint)SysCommands.SC_MAXIMIZE, 0u);
+            }
+            else
+            {
+                SetMenuDefaultItem(menu, (uint)SysCommands.SC_CLOSE, 0u);
+            }
+        }
+
+        private static void SetSystemMenuItemEnabled(IntPtr menu, SysCommands command, bool enabled)
+        {
+            EnableMenuItem(menu, (uint)command,
+                (uint)(MF_BYCOMMAND | (enabled ? MF_ENABLED : MF_DISABLED | MF_GRAYED)));
+        }
+
+        private static IntPtr MakeLParam(PixelPoint point)
+            => unchecked(((point.Y & 0xffff) << 16) | (point.X & 0xffff));
     }
 }

src/Windows/Avalonia.Win32/WindowImpl.cs

@@ -1110,6 +1110,12 @@ namespace Avalonia.Win32
                 _savedWindowInfo.Style = current;
                 _savedWindowInfo.ExStyle = currentEx;
 
+                if (current.HasAllFlags(WindowStyles.WS_SYSMENU))
+                {
+                    // Create the system menu copy before fullscreen removes WS_SYSMENU.
+                    GetSystemMenu(_hwnd, false);
+                }
+
                 // Set new window style and size.
                 SetStyle(current & ~WindowStyles.WS_OVERLAPPEDWINDOW, false);
                 SetExtendedStyle(currentEx & ~(WindowStyles.WS_EX_DLGMODALFRAME | WindowStyles.WS_EX_WINDOWEDGE | WindowStyles.WS_EX_CLIENTEDGE | WindowStyles.WS_EX_STATICEDGE), false);
@@ -1243,15 +1249,6 @@ namespace Avalonia.Win32
                 SetNCRenderingPolicy(DwmNCRenderingPolicy.DWMNCRP_USEWINDOWSTYLE);
             }
 
-            if (!_isClientAreaExtended)
-            {
-                EnableCloseButton(_hwnd);
-            }
-            else
-            {
-                DisableCloseButton(_hwnd);
-            }
-
             // Inform the application of the frame change.
             SetWindowPos(_hwnd,
                 IntPtr.Zero,
@@ -1547,19 +1544,6 @@ namespace Avalonia.Win32
         private const int MF_ENABLED = 0x0;
         private const int MF_GRAYED = 0x1;
         private const int MF_DISABLED = 0x2;
-        private const int SC_CLOSE = 0xF060;
-
-        private static void DisableCloseButton(IntPtr hwnd)
-        {
-            EnableMenuItem(GetSystemMenu(hwnd, false), SC_CLOSE,
-                           MF_BYCOMMAND | MF_DISABLED | MF_GRAYED);
-        }
-
-        private static void EnableCloseButton(IntPtr hwnd)
-        {
-            EnableMenuItem(GetSystemMenu(hwnd, false), SC_CLOSE,
-                           MF_BYCOMMAND | MF_ENABLED);
-        }
 
         private RECT ClientRectToWindowRect(RECT clientRect, WindowStyles? styleOverride = null, WindowStyles? extendedStyleOverride = null)
         {