[macOS][X11] Release mouse capture when dialog shown (#16205)

* Added integration test for #14525.

* Release mouse capture when dialog shown.

Fixes #14525.

* Release X11 pointer capture when dialog shown.

samples/IntegrationTestApp/DelegateCommand.cs

@@ -0,0 +1,19 @@
+using System;
+using System.Windows.Input;
+
+namespace IntegrationTestApp;
+
+internal class DelegateCommand : ICommand
+{
+    private readonly Action _action;
+    private readonly Func<object?, bool> _canExecute;
+    public DelegateCommand(Action action, Func<object?, bool>? canExecute = default)
+    {
+        _action = action;
+        _canExecute = canExecute ?? new(_ => true);
+    }
+
+    public event EventHandler? CanExecuteChanged { add { } remove { } }
+    public bool CanExecute(object? parameter) => _canExecute(parameter);
+    public void Execute(object? parameter) => _action();
+}

samples/IntegrationTestApp/MainWindow.axaml

@@ -136,6 +136,21 @@
           </StackPanel>
         </DockPanel>
       </TabItem>
+
+      <TabItem Header="Pointer">
+        <StackPanel>
+          <!-- Trigger with PointerPressed rather using a Button so we have access to the pointer. -->
+          <Border Name="PointerPageShowDialog"
+                  Background="{DynamicResource ButtonBackground}"
+                  HorizontalAlignment="Left"
+                  Padding="{DynamicResource ButtonPadding}"
+                  AutomationProperties.AccessibilityView="Control"
+                  PointerPressed="PointerPageShowDialogPressed">
+            <TextBlock>Show Dialog</TextBlock>
+          </Border>
+          <TextBlock Name="PointerCaptureStatus"/>
+        </StackPanel>
+      </TabItem>
       
       <TabItem Header="Window">
         <Grid ColumnDefinitions="*,8,*">

samples/IntegrationTestApp/MainWindow.axaml.cs

@@ -1,3 +1,4 @@
+using System;
 using System.Collections.Generic;
 using System.Linq;
 using Avalonia;
@@ -335,5 +336,37 @@ namespace IntegrationTestApp
             OnApplyWindowDecorations(window);
             window.Show();
         }
+
+        private void PointerPageShowDialogPressed(object? sender, PointerPressedEventArgs e)
+        {
+            void CaptureLost(object? sender, PointerCaptureLostEventArgs e)
+            {
+                PointerCaptureStatus.Text = "None";
+                ((Control)sender!).PointerCaptureLost -= CaptureLost;
+            }
+
+            var captured = e.Pointer.Captured as Control;
+
+            if (captured is not null)
+            {
+                captured.PointerCaptureLost += CaptureLost;
+            }
+
+            PointerCaptureStatus.Text = captured?.ToString() ?? "None";
+
+            var dialog = new Window
+            {
+                Width = 200,
+                Height = 200,
+            };
+            
+            dialog.Content = new Button
+            {
+                Content = "Close",
+                Command = new DelegateCommand(() => dialog.Close()),
+            };
+
+            dialog.ShowDialog(this);
+        }
     }
 }

src/Avalonia.Base/Input/MouseDevice.cs

@@ -305,5 +305,10 @@ namespace Avalonia.Input
         {
             return _pointer;
         }
+
+        internal void PlatformCaptureLost()
+        {
+            _pointer.Capture(null);
+        }
     }
 }

src/Avalonia.Base/Input/TouchDevice.cs

@@ -160,5 +160,11 @@ namespace Avalonia.Input
                 ? pointer
                 : null;
         }
+
+        internal void PlatformCaptureLost()
+        {
+            foreach (var pointer in _pointers.Values)
+                pointer.Capture(null);
+        }
     }
 }

src/Avalonia.Native/WindowImpl.cs

@@ -225,6 +225,13 @@ namespace Avalonia.Native
         public void SetEnabled(bool enable)
         {
             _native.SetEnabled(enable.AsComBool());
+
+            // Showing a dialog should result in mouse capture being lost. macOS doesn't have the concept of mouse
+            // capture, so no we have no OS-level event to hook into. Instead, release the mouse capture when the
+            // owner window is disabled. This behavior matches win32, which sends a WM_CANCELMODE message when
+            // EnableWindow(hWnd, false) is called from SetEnabled.
+            if (!enable && MouseDevice is MouseDevice mouse)
+                mouse.PlatformCaptureLost();
         }
 
         public override object TryGetFeature(Type featureType)

src/Avalonia.X11/X11Window.cs

@@ -1318,6 +1318,15 @@ namespace Avalonia.X11
                 // so setting it again forces the update
                 UpdateMotifHints();
             }
+            else
+            {
+                // Showing a dialog should result in pointer capture being lost. We don't currently use XGrabPointer on
+                // X11 to implement pointer capture, so no we have no OS-level event to hook into. Instead, release the 
+                // pointer capture when the owner window is disabled. This behavior matches win32, which sends a
+                // WM_CANCELMODE message when EnableWindow(hWnd, false) is called from SetEnabled.
+                _mouse.PlatformCaptureLost();
+                _touch.PlatformCaptureLost();
+            }
         }
 
         private void UpdateWMHints()

tests/Avalonia.IntegrationTests.Appium/PointerTests.cs

@@ -0,0 +1,31 @@
+using OpenQA.Selenium.Interactions;
+using Xunit;
+
+namespace Avalonia.IntegrationTests.Appium
+{
+    [Collection("Default")]
+    public class PointerTests
+    {
+        private readonly AppiumDriver _session;
+
+        public PointerTests(DefaultAppFixture fixture)
+        {
+            _session = fixture.Session;
+
+            var tabs = _session.FindElementByAccessibilityId("MainTabs");
+            var tab = tabs.FindElementByName("Pointer");
+            tab.Click();
+        }
+
+        [Fact]
+        public void Pointer_Capture_Is_Released_When_Showing_Dialog()
+        {
+            var button = _session.FindElementByAccessibilityId("PointerPageShowDialog");
+
+            button.OpenWindowWithClick().Dispose();
+
+            var status = _session.FindElementByAccessibilityId("PointerCaptureStatus");
+            Assert.Equal("None", status.Text);
+        }
+    }
+}