Make OpacityMask work in deferred renderer.

src/Avalonia.Visuals/Media/IDrawingContextImpl.cs

@@ -31,6 +31,15 @@ namespace Avalonia.Media
         /// <param name="destRect">The rect in the output to draw to.</param>
         void DrawImage(IBitmapImpl source, double opacity, Rect sourceRect, Rect destRect);
 
+        /// <summary>
+        /// Draws a bitmap image.
+        /// </summary>
+        /// <param name="source">The bitmap image.</param>
+        /// <param name="opacityMask">The opacity mask to draw with.</param>
+        /// <param name="opacityMaskRect">The destination rect for the opacity mask.</param>
+        /// <param name="destRect">The rect in the output to draw to.</param>
+        void DrawImage(IBitmapImpl source, IBrush opacityMask, Rect opacityMaskRect, Rect destRect);
+
         /// <summary>
         /// Draws a line.
         /// </summary>

src/Avalonia.Visuals/Rendering/DeferredRenderer.cs

@@ -262,7 +262,15 @@ namespace Avalonia.Rendering
                     {
                         var bitmap = _layers[layer.LayerRoot].Bitmap;
                         var sourceRect = new Rect(0, 0, bitmap.PixelWidth, bitmap.PixelHeight);
-                        context.DrawImage(bitmap, layer.Opacity, sourceRect, clientRect);
+
+                        if (layer.OpacityMask == null)
+                        {
+                            context.DrawImage(bitmap, layer.Opacity, sourceRect, clientRect);
+                        }
+                        else
+                        {
+                            context.DrawImage(bitmap, layer.OpacityMask, layer.OpacityMaskRect, sourceRect);
+                        }
                     }
 
                     if (_overlay != null)

src/Avalonia.Visuals/Rendering/SceneGraph/DeferredDrawingContextImpl.cs

@@ -93,6 +93,11 @@ namespace Avalonia.Rendering.SceneGraph
             }
         }
 
+        public void DrawImage(IBitmapImpl source, IBrush opacityMask, Rect opacityMaskRect, Rect sourceRect)
+        {
+            throw new NotImplementedException();
+        }
+
         public void DrawLine(Pen pen, Point p1, Point p2)
         {
             var next = NextDrawAs<LineNode>();

src/Avalonia.Visuals/Rendering/SceneGraph/SceneBuilder.cs

@@ -152,6 +152,8 @@ namespace Avalonia.Rendering.SceneGraph
                 using (context.PushPostTransform(m))
                 using (context.PushTransformContainer())
                 {
+                    var startLayer = opacity < 1 || visual.OpacityMask != null;
+
                     forceRecurse = forceRecurse || node.Transform != contextImpl.Transform;
 
                     node.Transform = contextImpl.Transform;
@@ -161,25 +163,22 @@ namespace Avalonia.Rendering.SceneGraph
                     node.Opacity = opacity;
                     node.OpacityMask = visual.OpacityMask;
 
-                    if (opacity < 1 && node.LayerRoot != visual)
+                    if (startLayer)
                     {
-                        SetLayer(scene, node, node.Visual);
+                        if (node.LayerRoot != visual)
+                        {
+                            MakeLayer(scene, node);
+                        }
+                        else
+                        {
+                            UpdateLayer(node, scene.Layers[node.LayerRoot]);
+                        }
                     }
-                    else if (opacity >= 1 && node.LayerRoot == node.Visual && node.Parent != null)
+                    else if (!startLayer && node.LayerRoot == node.Visual && node.Parent != null)
                     {
                         ClearLayer(scene, node);
                     }
 
-                    if (node.LayerRoot == visual)
-                    {
-                        var layer = scene.Layers.Find(visual);
-
-                        if (layer != null)
-                        {
-                            layer.Opacity = visual.Opacity;
-                        }
-                    }
-
                     if (node.ClipToBounds)
                     {
                         clip = clip.Intersect(node.ClipBounds);
@@ -290,33 +289,57 @@ namespace Avalonia.Rendering.SceneGraph
                 newDirtyRects.Add(r);
             }
 
-            SetLayer(scene, node, newLayerRoot);
-            scene.Layers.Remove(oldLayerRoot);
+            var oldLayer = scene.Layers[oldLayerRoot];
+            SetDescendentsLayer(node, scene.Layers[newLayerRoot], oldLayer);
+            scene.Layers.Remove(oldLayer);
+        }
+
+        private static void MakeLayer(Scene scene, VisualNode node)
+        {
+            var oldLayerRoot = node.LayerRoot;
+            var layer = scene.Layers.Add(node.Visual);
+            var oldLayer = scene.Layers[oldLayerRoot];
+
+            UpdateLayer(node, layer);
+            SetDescendentsLayer(node, layer, scene.Layers[oldLayerRoot]);
         }
 
-        private static void SetLayer(Scene scene, VisualNode node, IVisual layerRoot)
+        private static void UpdateLayer(VisualNode node, SceneLayer layer)
         {
-            if (node.LayerRoot == layerRoot)
+            layer.Opacity = node.Visual.Opacity;
+
+            if (node.Visual.OpacityMask != null)
             {
-                throw new AvaloniaInternalException("Called SetLayer with unchanged LayerRoot.");
+                layer.OpacityMask = ToImmutable(node.Visual.OpacityMask);
+                layer.OpacityMaskRect = node.ClipBounds;
             }
+            else
+            {
+                layer.OpacityMask = null;
+                layer.OpacityMaskRect = Rect.Empty;
+            }
+        }
 
-            var oldLayerRoot = node.LayerRoot;
-
-            node.LayerRoot = layerRoot;
+        private static void SetDescendentsLayer(VisualNode node, SceneLayer layer, SceneLayer oldLayer)
+        {
+            node.LayerRoot = layer.LayerRoot;
 
-            var layer = scene.Layers.GetOrAdd(layerRoot);
             layer.Dirty.Add(node.Bounds);
-            scene.Layers[oldLayerRoot].Dirty.Add(node.Bounds);
+            oldLayer.Dirty.Add(node.Bounds);
 
             foreach (VisualNode child in node.Children)
             {
                 // If the child is not the start of a new layer, recurse.
                 if (child.LayerRoot != child.Visual)
                 {
-                    SetLayer(scene, child, layerRoot);
+                    SetDescendentsLayer(child, layer, oldLayer);
                 }
             }
         }
+
+        private static IBrush ToImmutable(IBrush brush)
+        {
+            return (brush as IMutableBrush)?.ToImmutable() ?? brush;
+        }
     }
 }

src/Avalonia.Visuals/Rendering/SceneGraph/SceneLayer.cs

@@ -1,4 +1,5 @@
 using System;
+using Avalonia.Media;
 using Avalonia.VisualTree;
 
 namespace Avalonia.Rendering.SceneGraph
@@ -16,7 +17,9 @@ namespace Avalonia.Rendering.SceneGraph
         {
             return new SceneLayer(LayerRoot, DistanceFromRoot)
             {
-                Opacity = Opacity
+                Opacity = Opacity,
+                OpacityMask = OpacityMask,
+                OpacityMaskRect = OpacityMaskRect,
             };
         }
 
@@ -24,5 +27,7 @@ namespace Avalonia.Rendering.SceneGraph
         public DirtyRects Dirty { get; }
         public int DistanceFromRoot { get; }
         public double Opacity { get; set; } = 1;
+        public IBrush OpacityMask { get; set; }
+        public Rect OpacityMaskRect { get; set; }
     }
 }

src/Avalonia.Visuals/Rendering/SceneGraph/SceneLayers.cs

@@ -99,13 +99,20 @@ namespace Avalonia.Rendering.SceneGraph
 
             if (_index.TryGetValue(layerRoot, out layer))
             {
-                _index.Remove(layerRoot);
-                _inner.Remove(layer);
+                Remove(layer);
             }
 
             return layer != null;
         }
 
+        public bool Remove(SceneLayer layer)
+        {
+            Contract.Requires<ArgumentNullException>(layer != null);
+
+            _index.Remove(layer.LayerRoot);
+            return _inner.Remove(layer);
+        }
+
         public IEnumerator<SceneLayer> GetEnumerator() => _inner.GetEnumerator();
         IEnumerator IEnumerable.GetEnumerator() => GetEnumerator();
 

src/Gtk/Avalonia.Cairo/Media/DrawingContext.cs

@@ -118,6 +118,11 @@ namespace Avalonia.Cairo.Media
             _context.Restore();
         }
 
+        public void DrawImage(IBitmapImpl source, IBrush opacityMask, Rect opacityMaskRect, Rect destRect)
+        {
+            throw new NotImplementedException();
+        }
+
         /// <summary>
         /// Draws a line.
         /// </summary>

src/Skia/Avalonia.Skia/DrawingContextImpl.cs

@@ -37,6 +37,11 @@ namespace Avalonia.Skia
             }
         }
 
+        public void DrawImage(IBitmapImpl source, IBrush opacityMask, Rect opacityMaskRect, Rect destRect)
+        {
+            throw new NotImplementedException();
+        }
+
         public void DrawLine(Pen pen, Point p1, Point p2)
         {
             using (var paint = CreatePaint(pen, new Size(Math.Abs(p2.X - p1.X), Math.Abs(p2.Y - p1.Y))))

src/Windows/Avalonia.Direct2D1/Media/DrawingContextImpl.cs

@@ -100,6 +100,29 @@ namespace Avalonia.Direct2D1.Media
             }
         }
 
+        /// <summary>
+        /// Draws a bitmap image.
+        /// </summary>
+        /// <param name="source">The bitmap image.</param>
+        /// <param name="opacityMask">The opacity mask to draw with.</param>
+        /// <param name="opacityMaskRect">The destination rect for the opacity mask.</param>
+        /// <param name="destRect">The rect in the output to draw to.</param>
+        public void DrawImage(IBitmapImpl source, IBrush opacityMask, Rect opacityMaskRect, Rect destRect)
+        {
+            using (var d2dSource = ((BitmapImpl)source).GetDirect2DBitmap(_renderTarget))
+            using (var sourceBrush = new BitmapBrush(_renderTarget, d2dSource))
+            using (var d2dOpacityMask = CreateBrush(opacityMask, opacityMaskRect.Size))
+            using (var geometry = new SharpDX.Direct2D1.RectangleGeometry(_renderTarget.Factory, destRect.ToDirect2D()))
+            {
+                d2dOpacityMask.PlatformBrush.Transform = Matrix.CreateTranslation(opacityMaskRect.Position).ToDirect2D();
+
+                _renderTarget.FillGeometry(
+                    geometry,
+                    sourceBrush,
+                    d2dOpacityMask.PlatformBrush);
+            }
+        }
+
         /// <summary>
         /// Draws a line.
         /// </summary>

tests/Avalonia.Visuals.UnitTests/Rendering/SceneGraph/SceneBuilderTests_Layers.cs

@@ -79,6 +79,72 @@ namespace Avalonia.Visuals.UnitTests.Rendering.SceneGraph
             }
         }
 
+        [Fact]
+        public void Control_With_OpacityMask_Should_Start_New_Layer()
+        {
+            using (TestApplication())
+            {
+                Decorator decorator;
+                Border border;
+                Canvas canvas;
+                var tree = new TestRoot
+                {
+                    Padding = new Thickness(10),
+                    Width = 100,
+                    Height = 120,
+                    Child = decorator = new Decorator
+                    {
+                        Padding = new Thickness(11),
+                        Child = border = new Border
+                        {
+                            OpacityMask = Brushes.Red,
+                            Background = Brushes.Red,
+                            Padding = new Thickness(12),
+                            Child = canvas = new Canvas(),
+                        }
+                    }
+                };
+
+                var layout = AvaloniaLocator.Current.GetService<ILayoutManager>();
+                layout.ExecuteInitialLayoutPass(tree);
+
+                var scene = new Scene(tree);
+                var sceneBuilder = new SceneBuilder();
+                sceneBuilder.UpdateAll(scene);
+
+                var rootNode = (VisualNode)scene.Root;
+                var borderNode = (VisualNode)scene.FindNode(border);
+                var canvasNode = (VisualNode)scene.FindNode(canvas);
+
+                Assert.Same(tree, rootNode.LayerRoot);
+                Assert.Same(border, borderNode.LayerRoot);
+                Assert.Same(border, canvasNode.LayerRoot);
+                Assert.Equal(Brushes.Red, scene.Layers[border].OpacityMask);
+
+                Assert.Equal(2, scene.Layers.Count());
+                Assert.Empty(scene.Layers.Select(x => x.LayerRoot).Except(new IVisual[] { tree, border }));
+
+                border.OpacityMask = null;
+                scene = scene.Clone();
+
+                sceneBuilder.Update(scene, border);
+
+                rootNode = (VisualNode)scene.Root;
+                borderNode = (VisualNode)scene.FindNode(border);
+                canvasNode = (VisualNode)scene.FindNode(canvas);
+
+                Assert.Same(tree, rootNode.LayerRoot);
+                Assert.Same(tree, borderNode.LayerRoot);
+                Assert.Same(tree, canvasNode.LayerRoot);
+                Assert.Equal(1, scene.Layers.Count());
+
+                var rootDirty = scene.Layers[tree].Dirty;
+
+                Assert.Equal(1, rootDirty.Count());
+                Assert.Equal(new Rect(21, 21, 58, 78), rootDirty.Single());
+            }
+        }
+
         [Fact]
         public void Removing_Transparent_Control_Should_Remove_Layers()
         {