Improve system menu display and default item behavior

src/Windows/Avalonia.Win32/Interop/UnmanagedMethods.cs

@@ -1407,6 +1407,9 @@ namespace Avalonia.Win32.Interop
         [DllImport("user32.dll")]
         public static extern bool EnableMenuItem(IntPtr hMenu, uint uIDEnableItem, uint uEnable);
 
+        [DllImport("user32.dll")]
+        public static extern bool SetMenuDefaultItem(IntPtr hMenu, uint uItem, bool fByPos);
+
         [DllImport("user32.dll", SetLastError = true)]
         public static extern bool GetWindowPlacement(IntPtr hWnd, out WINDOWPLACEMENT lpwndpl);
 

src/Windows/Avalonia.Win32/WindowImpl.AppWndProc.cs

@@ -246,11 +246,6 @@ namespace Avalonia.Win32
                         return IntPtr.Zero;
                     break;
 
-                case WindowsMessage.WM_CONTEXTMENU
-                    when ToInt32(lParam) != -1 && IsCaptionHitForSystemMenu(hWnd, lParam):
-                    ShowSystemMenu(PointFromLParam(lParam));
-                    return IntPtr.Zero;
-
                 case WindowsMessage.WM_MENUCHAR:
                     {
                         // mute the system beep
@@ -488,13 +483,6 @@ namespace Avalonia.Win32
                         {
                             break;
                         }
-                        // Keep caption right-clicks out of Avalonia input; the menu is shown on button up.
-                        if (_isClientAreaExtended
-                            && message == WindowsMessage.WM_NCRBUTTONDOWN
-                            && (HitTestValues)ToInt32(wParam) == HitTestValues.HTCAPTION)
-                        {
-                            break;
-                        }
                         e = new RawPointerEventArgs(
                             _mouseDevice,
                             timestamp,

src/Windows/Avalonia.Win32/WindowImpl.CustomCaptionProc.cs

@@ -122,12 +122,26 @@ namespace Avalonia.Win32
                     }
                     break;
 
-                case WindowsMessage.WM_NCRBUTTONUP
+                // Prevent right-clicks from triggering controls underneath while the cursor is over the window edge or HTCAPTION.
-                    when (HitTestValues)ToInt32(wParam) == HitTestValues.HTCAPTION && IsCaptionHitForSystemMenu(hWnd, lParam):
+                case WindowsMessage.WM_NCRBUTTONDOWN
+                    when (HitTestValues)ToInt32(wParam) is HitTestValues.HTCAPTION
+                    or HitTestValues.HTTOP or HitTestValues.HTBOTTOM
+                    or HitTestValues.HTLEFT or HitTestValues.HTRIGHT
+                    or HitTestValues.HTTOPLEFT or HitTestValues.HTTOPRIGHT
+                    or HitTestValues.HTBOTTOMLEFT or HitTestValues.HTBOTTOMRIGHT:
+                    callDwp = false;
+                    return IntPtr.Zero;
+
+                case WindowsMessage.WM_NCRBUTTONUP when (HitTestValues)ToInt32(wParam) == HitTestValues.HTCAPTION:
                     ShowSystemMenu(PointFromLParam(lParam));
                     callDwp = false;
                     return IntPtr.Zero;
 
+                case WindowsMessage.WM_INITMENU:
+                    UpdateSystemMenu(GetSystemMenu(hWnd, false));
+                    callDwp = false;
+                    return IntPtr.Zero;
+
                 // Normally, Avalonia doesn't handles non-client input as a special NonClientLeftButtonDown, ignoring move and up events.
                 // What makes it a problem, Avalonia has to mark templated caption buttons as a non-client area.
                 // Meaning, these buttons no longer can accept normal client input.
@@ -299,21 +313,6 @@ namespace Avalonia.Win32
                 or HitTestValues.HTSYSMENU;
         }
 
-        private bool IsCaptionHitForSystemMenu(IntPtr hWnd, IntPtr lParam)
-        {
-            if (!_isClientAreaExtended)
-            {
-                return false;
-            }
-
-            if (HitTestNCA(hWnd, IntPtr.Zero, lParam) is not HitTestValues.HTCAPTION and not HitTestValues.HTNOWHERE)
-            {
-                return false;
-            }
-
-            return HitTestVisual(lParam) is HitTestValues.HTCAPTION or HitTestValues.HTNOWHERE;
-        }
-
         private void ShowSystemMenu(PixelPoint screenPoint)
         {
             var menu = GetSystemMenu(_hwnd, false);
@@ -323,8 +322,6 @@ namespace Avalonia.Win32
             }
 
             SetForegroundWindow(_hwnd);
-            SendMessage(_hwnd, (int)WindowsMessage.WM_INITMENU, menu, IntPtr.Zero);
-            UpdateSystemMenu(menu);
 
             var command = TrackPopupMenu(
                 menu,
@@ -352,6 +349,11 @@ namespace Avalonia.Win32
 
         private void UpdateSystemMenu(IntPtr menu)
         {
+            if (menu == IntPtr.Zero)
+            {
+                return;
+            }
+
             var state = WindowState;
             var isMinimized = state == WindowState.Minimized;
             var isMaximized = state == WindowState.Maximized;
@@ -364,6 +366,19 @@ namespace Avalonia.Win32
             SetSystemMenuItemEnabled(menu, SysCommands.SC_MINIMIZE, !isMinimized && !isFullScreen && _windowProperties.IsMinimizable);
             SetSystemMenuItemEnabled(menu, SysCommands.SC_MAXIMIZE, !isMaximized && !isFullScreen && _windowProperties.IsMaximizable);
             SetSystemMenuItemEnabled(menu, SysCommands.SC_CLOSE, true);
+
+            if (isMinimized || isMaximized || isFullScreen)
+            {
+                SetMenuDefaultItem(menu, (uint)SysCommands.SC_RESTORE, false);
+            }
+            else if (isNormal && _windowProperties.IsMaximizable)
+            {
+                SetMenuDefaultItem(menu, (uint)SysCommands.SC_MAXIMIZE, false);
+            }
+            else
+            {
+                SetMenuDefaultItem(menu, (uint)SysCommands.SC_CLOSE, false);
+            }
         }
 
         private static void SetSystemMenuItemEnabled(IntPtr menu, SysCommands command, bool enabled)