Fix viewport calculation with non-invertible render transform (#13188)

* Added failing test for viewport with 0 scale transform

* Fix viewport calculation with non-invertible render transform

src/Avalonia.Base/Layout/LayoutManager.cs

@@ -423,12 +423,16 @@ namespace Avalonia.Layout
             // Translate the viewport into this control's coordinate space.
             viewport = viewport.Translate(-control.Bounds.Position);
 
-            if (control != target && control.RenderTransform is object)
+            if (control != target && control.RenderTransform is { } transform)
             {
-                var origin = control.RenderTransformOrigin.ToPixels(control.Bounds.Size);
+                if (transform.Value.TryInvert(out var invertedMatrix))
-                var offset = Matrix.CreateTranslation(origin);
+                {
-                var renderTransform = (-offset) * control.RenderTransform.Value.Invert() * (offset);
+                    var origin = control.RenderTransformOrigin.ToPixels(control.Bounds.Size);
-                viewport = viewport.TransformToAABB(renderTransform);
+                    var offset = Matrix.CreateTranslation(origin);
+                    viewport = viewport.TransformToAABB(-offset * invertedMatrix * offset);
+                }
+                else
+                    viewport = default;
             }
         }
 

tests/Avalonia.Base.UnitTests/Layout/LayoutableTests_EffectiveViewportChanged.cs

@@ -341,6 +341,32 @@ namespace Avalonia.Base.UnitTests.Layout
             });
         }
 
+        // https://github.com/AvaloniaUI/Avalonia/issues/12452
+        [Fact]
+        public async Task Zero_ScaleTransform_Sets_Empty_EffectiveViewport()
+        {
+            await RunOnUIThread.Execute(async () =>
+            {
+                var effectiveViewport = new Rect(Size.Infinity);
+
+                var root = CreateRoot();
+                var target = new Canvas { Width = 100, Height = 100 };
+                var parent = new Border { Width = 100, Height = 100, Child = target };
+
+                target.EffectiveViewportChanged += (_, e) => effectiveViewport = e.EffectiveViewport;
+
+                root.Child = parent;
+
+                await ExecuteInitialLayoutPass(root);
+
+                parent.RenderTransform = new ScaleTransform(0, 0);
+
+                await ExecuteLayoutPass(root);
+
+                Assert.Equal(new Rect(0, 0, 0, 0), effectiveViewport);
+            });
+        }
+
         private static TestRoot CreateRoot() => new TestRoot { Width = 1200, Height = 900 };
 
         private static Task ExecuteInitialLayoutPass(TestRoot root)