Merge pull request #4857 from MarchingCube/fix-custom-titlebar-hittest

[win32] Fix hittest returning top-left and top-right for non-resize corner areas

src/Windows/Avalonia.Win32/WindowImpl.CustomCaptionProc.cs

@@ -1,5 +1,4 @@
 using System;
-using System.Diagnostics;
 using Avalonia.Controls;
 using Avalonia.Input;
 using static Avalonia.Win32.Interop.UnmanagedMethods;
@@ -11,70 +10,78 @@ namespace Avalonia.Win32
     public partial class WindowImpl
     {
         // Hit test the frame for resizing and moving.
-        HitTestValues HitTestNCA(IntPtr hWnd, IntPtr wParam, IntPtr lParam)
+        private HitTestValues HitTestNCA(IntPtr hWnd, IntPtr wParam, IntPtr lParam)
         {
-            // Get the point coordinates for the hit test.
+            // Get the point coordinates for the hit test (screen space).
             var ptMouse = PointFromLParam(lParam);
 
-            // Get the window rectangle.            
+            // Get the window rectangle.
             GetWindowRect(hWnd, out var rcWindow);
 
             // Get the frame rectangle, adjusted for the style without a caption.
-            RECT rcFrame = new RECT();
+            var rcFrame = new RECT();
             AdjustWindowRectEx(ref rcFrame, (uint)(WindowStyles.WS_OVERLAPPEDWINDOW & ~WindowStyles.WS_CAPTION), false, 0);
 
-            RECT border_thickness = new RECT();
+            var borderThickness = new RECT();
             if (GetStyle().HasFlag(WindowStyles.WS_THICKFRAME))
             {
-                AdjustWindowRectEx(ref border_thickness, (uint)(GetStyle()), false, 0);
+                AdjustWindowRectEx(ref borderThickness, (uint)(GetStyle()), false, 0);
-                border_thickness.left *= -1;
+                borderThickness.left *= -1;
-                border_thickness.top *= -1;
+                borderThickness.top *= -1;
             }
             else if (GetStyle().HasFlag(WindowStyles.WS_BORDER))
             {
-                border_thickness = new RECT { bottom = 1, left = 1, right = 1, top = 1 };
+                borderThickness = new RECT { bottom = 1, left = 1, right = 1, top = 1 };
             }
 
             if (_extendTitleBarHint >= 0)
             {
-                border_thickness.top = (int)(_extendedMargins.Top * RenderScaling);
+                borderThickness.top = (int)(_extendedMargins.Top * RenderScaling);
             }
 
             // Determine if the hit test is for resizing. Default middle (1,1).
             ushort uRow = 1;
             ushort uCol = 1;
-            bool fOnResizeBorder = false;
+            bool onResizeBorder = false;
 
-            // Determine if the point is at the top or bottom of the window.
+            // Determine if the point is at the left or right of the window.
-            if (ptMouse.Y >= rcWindow.top && ptMouse.Y < rcWindow.top + border_thickness.top)
+            if (ptMouse.X >= rcWindow.left && ptMouse.X < rcWindow.left + borderThickness.left)
             {
-                fOnResizeBorder = (ptMouse.Y < (rcWindow.top - rcFrame.top));
+                uCol = 0; // left side
-                uRow = 0;
             }
-            else if (ptMouse.Y < rcWindow.bottom && ptMouse.Y >= rcWindow.bottom - border_thickness.bottom)
+            else if (ptMouse.X < rcWindow.right && ptMouse.X >= rcWindow.right - borderThickness.right)
             {
-                uRow = 2;
+                uCol = 2; // right side
             }
 
-            // Determine if the point is at the left or right of the window.
+            // Determine if the point is at the top or bottom of the window.
-            if (ptMouse.X >= rcWindow.left && ptMouse.X < rcWindow.left + border_thickness.left)
+            if (ptMouse.Y >= rcWindow.top && ptMouse.Y < rcWindow.top + borderThickness.top)
             {
-                uCol = 0; // left side
+                onResizeBorder = (ptMouse.Y < (rcWindow.top - rcFrame.top));
+
+                // Two cases where we have a valid row 0 hit test:
+                // - window resize border (top resize border hit)
+                // - area below resize border that is actual titlebar (caption hit).
+                if (onResizeBorder || uCol == 1)
+                {
+                    uRow = 0;
+                }
             }
-            else if (ptMouse.X < rcWindow.right && ptMouse.X >= rcWindow.right - border_thickness.right)
+            else if (ptMouse.Y < rcWindow.bottom && ptMouse.Y >= rcWindow.bottom - borderThickness.bottom)
             {
-                uCol = 2; // right side
+                uRow = 2;
             }
 
-            // Hit test (HTTOPLEFT, ... HTBOTTOMRIGHT)
+            ReadOnlySpan<HitTestValues> hitZones = stackalloc HitTestValues[]
-            HitTestValues[][] hitTests = new[]
             {
-                new []{ HitTestValues.HTTOPLEFT,    fOnResizeBorder ? HitTestValues.HTTOP : HitTestValues.HTCAPTION,    HitTestValues.HTTOPRIGHT },
+                HitTestValues.HTTOPLEFT, onResizeBorder ? HitTestValues.HTTOP : HitTestValues.HTCAPTION,
-                new []{ HitTestValues.HTLEFT,      HitTestValues.HTNOWHERE,     HitTestValues.HTRIGHT },
+                HitTestValues.HTTOPRIGHT, HitTestValues.HTLEFT, HitTestValues.HTNOWHERE, HitTestValues.HTRIGHT,
-                new []{ HitTestValues.HTBOTTOMLEFT, HitTestValues.HTBOTTOM, HitTestValues.HTBOTTOMRIGHT },
+                HitTestValues.HTBOTTOMLEFT, HitTestValues.HTBOTTOM, HitTestValues.HTBOTTOMRIGHT
             };
 
-            return hitTests[uRow][uCol];
+            var zoneIndex = uRow * 3 + uCol;
+
+            return hitZones[zoneIndex];
         }
 
         protected virtual IntPtr CustomCaptionProc(IntPtr hWnd, uint msg, IntPtr wParam, IntPtr lParam, ref bool callDwp)