Implemented a proper framebuffer for RemoteServerTopLevelImpl

src/Avalonia.Controls/Embedding/Offscreen/OffscreenTopLevelImpl.cs

@@ -1,14 +1,10 @@
 using System;
 using System.Collections.Generic;
-using System.Diagnostics;
 using Avalonia.Input;
 using Avalonia.Input.Raw;
-using Avalonia.Media;
 using Avalonia.Metadata;
 using Avalonia.Platform;
-using Avalonia.Rendering;
 using Avalonia.Rendering.Composition;
-using Avalonia.Threading;
 
 namespace Avalonia.Controls.Embedding.Offscreen
 {
@@ -17,7 +13,6 @@ namespace Avalonia.Controls.Embedding.Offscreen
     {
         private double _scaling = 1;
         private Size _clientSize;
-        private ManualRenderTimer _manualRenderTimer = new();
 
         public IInputRoot? InputRoot { get; private set; }
         public bool IsDisposed { get; private set; }
@@ -27,21 +22,10 @@ namespace Avalonia.Controls.Embedding.Offscreen
             IsDisposed = true;
         }
 
-        class ManualRenderTimer : IRenderTimer
-        {
-            static Stopwatch St = Stopwatch.StartNew(); 
-            public event Action<TimeSpan>? Tick;
-            public bool RunsInBackground => false;
-            public void TriggerTick() => Tick?.Invoke(St.Elapsed);
-        }
-
         public Compositor Compositor { get; }
 
         public OffscreenTopLevelImplBase()
-        {
+            => Compositor = new Compositor(null);
-            Compositor = new Compositor(new RenderLoop(_manualRenderTimer), null, false,
-                MediaContext.Instance, false);
-        }
 
         public abstract IEnumerable<object> Surfaces { get; }
 
@@ -76,7 +60,6 @@ namespace Avalonia.Controls.Embedding.Offscreen
 
         public void SetFrameThemeVariant(PlatformThemeVariant themeVariant) { }
 
-        /// <inheritdoc/>
         public AcrylicPlatformCompensationLevels AcrylicCompensationLevels { get; } = new AcrylicPlatformCompensationLevels(1, 1, 1);
 
         public void SetInputRoot(IInputRoot inputRoot) => InputRoot = inputRoot;

src/Avalonia.Controls/Remote/Server/RemoteServerTopLevelImpl.Framebuffer.cs

@@ -0,0 +1,116 @@
+using System;
+using System.Runtime.InteropServices;
+using System.Threading;
+using Avalonia.Platform;
+using Avalonia.Remote.Protocol.Viewport;
+using PlatformPixelFormat = Avalonia.Platform.PixelFormat;
+using ProtocolPixelFormat = Avalonia.Remote.Protocol.Viewport.PixelFormat;
+
+namespace Avalonia.Controls.Remote.Server
+{
+    internal partial class RemoteServerTopLevelImpl
+    {
+        private enum FrameStatus
+        {
+            NotRendered,
+            Rendered,
+            CopiedToMessage
+        }
+
+        private sealed class Framebuffer
+        {
+            public static Framebuffer Empty { get; } =
+                new(ProtocolPixelFormat.Rgba8888, default, new Vector(96.0, 96.0));
+
+            private readonly object _dataLock = new();
+            private readonly byte[] _data; // for rendering only
+            private readonly byte[] _dataCopy; // for messages only
+            private FrameStatus _status = FrameStatus.NotRendered;
+
+            public Framebuffer(ProtocolPixelFormat format, Size clientSize, Vector dpi)
+            {
+                var frameSize = PixelSize.FromSizeWithDpi(clientSize, dpi);
+                if (frameSize.Width <= 0 || frameSize.Height <= 0)
+                    frameSize = PixelSize.Empty;
+
+                var bpp = format == ProtocolPixelFormat.Rgb565 ? 2 : 4;
+                var stride = frameSize.Width * bpp;
+                var dataLength = Math.Max(0, stride * frameSize.Height);
+
+                Format = format;
+                FrameSize = frameSize;
+                ClientSize = clientSize;
+                Dpi = dpi;
+
+                (Stride, _data, _dataCopy) = dataLength > 0 ?
+                    (stride, new byte[dataLength], new byte[dataLength]) :
+                    (0, Array.Empty<byte>(), Array.Empty<byte>());
+            }
+
+            public ProtocolPixelFormat Format { get; }
+
+            public Size ClientSize { get; }
+
+            public Vector Dpi { get; }
+
+            public PixelSize FrameSize { get; }
+
+            public int Stride { get; }
+
+            public FrameStatus GetStatus()
+            {
+                lock (_dataLock)
+                    return _status;
+            }
+
+            public ILockedFramebuffer Lock(Action onUnlocked)
+            {
+                var handle = GCHandle.Alloc(_data, GCHandleType.Pinned);
+                Monitor.Enter(_dataLock);
+
+                try
+                {
+                    return new LockedFramebuffer(
+                        handle.AddrOfPinnedObject(),
+                        FrameSize,
+                        Stride,
+                        Dpi,
+                        new PlatformPixelFormat((PixelFormatEnum)Format),
+                        () =>
+                        {
+                            handle.Free();
+                            Array.Copy(_data, _dataCopy, _data.Length);
+                            _status = FrameStatus.Rendered;
+                            Monitor.Exit(_dataLock);
+                            onUnlocked();
+                        });
+                }
+                catch
+                {
+                    handle.Free();
+                    Monitor.Exit(_dataLock);
+                    throw;
+                }
+            }
+
+            /// <remarks>The returned message must be kept around, as it contains a shared buffer.</remarks>
+            public FrameMessage ToMessage(long sequenceId)
+            {
+                lock (_dataLock)
+                    _status = FrameStatus.CopiedToMessage;
+
+                return new FrameMessage
+                {
+                    SequenceId = sequenceId,
+                    Data = _dataCopy,
+                    Format = Format,
+                    Width = FrameSize.Width,
+                    Height = FrameSize.Height,
+                    Stride = Stride,
+                    DpiX = Dpi.X,
+                    DpiY = Dpi.Y
+                };
+            }
+        }
+    }
+}

src/Avalonia.Controls/Remote/Server/RemoteServerTopLevelImpl.cs

@@ -1,6 +1,5 @@
 using System;
 using System.Collections.Generic;
-using System.Runtime.InteropServices;
 using Avalonia.Controls.Embedding.Offscreen;
 using Avalonia.Controls.Platform.Surfaces;
 using Avalonia.Input;
@@ -11,7 +10,6 @@ using Avalonia.Platform;
 using Avalonia.Remote.Protocol;
 using Avalonia.Remote.Protocol.Input;
 using Avalonia.Remote.Protocol.Viewport;
-using Avalonia.Rendering;
 using Avalonia.Threading;
 using Key = Avalonia.Input.Key;
 using ProtocolPixelFormat = Avalonia.Remote.Protocol.Viewport.PixelFormat;
@@ -20,28 +18,29 @@ using ProtocolMouseButton = Avalonia.Remote.Protocol.Input.MouseButton;
 namespace Avalonia.Controls.Remote.Server
 {
     [Unstable]
-    internal class RemoteServerTopLevelImpl : OffscreenTopLevelImplBase, IFramebufferPlatformSurface, ITopLevelImpl
+    internal partial class RemoteServerTopLevelImpl : OffscreenTopLevelImplBase, IFramebufferPlatformSurface, ITopLevelImpl
     {
         private readonly IAvaloniaRemoteTransportConnection _transport;
-        private LockedFramebuffer? _framebuffer;
         private readonly object _lock = new();
+        private readonly Action _sendLastFrameIfNeeded;
+        private readonly Action _renderAndSendFrameIfNeeded;
+        private Framebuffer _framebuffer = Framebuffer.Empty;
         private long _lastSentFrame = -1;
         private long _lastReceivedFrame = -1;
         private long _nextFrameNumber = 1;
         private ClientViewportAllocatedMessage? _pendingAllocation;
-        private bool _queuedNextRender;
+        private Vector _dpi = new(96, 96);
-        private bool _inRender;
+        private ProtocolPixelFormat? _format;
-        private Vector _dpi = new Vector(96, 96);
-        private ProtocolPixelFormat[]? _supportedFormats;
 
         public RemoteServerTopLevelImpl(IAvaloniaRemoteTransportConnection transport)
         {
+            _sendLastFrameIfNeeded = SendLastFrameIfNeeded;
+            _renderAndSendFrameIfNeeded = RenderAndSendFrameIfNeeded;
+
             _transport = transport;
             _transport.OnMessage += OnMessage;
 
             KeyboardDevice = AvaloniaLocator.Current.GetRequiredService<IKeyboardDevice>();
-            QueueNextRender();
-            Compositor.AfterCommit += QueueNextRender;
         }
 
         private static RawPointerEventType GetAvaloniaEventType(ProtocolMouseButton button, bool pressed)
@@ -118,23 +117,23 @@ namespace Avalonia.Controls.Remote.Server
                     {
                         _lastReceivedFrame = Math.Max(lastFrame.SequenceId, _lastReceivedFrame);
                     }
-                    Dispatcher.UIThread.Post(RenderIfNeeded);
+                    Dispatcher.UIThread.Post(_sendLastFrameIfNeeded);
                 }
-                if(obj is ClientRenderInfoMessage renderInfo)
+                if (obj is ClientRenderInfoMessage renderInfo)
                 {
-                    lock(_lock)
+                    lock (_lock)
                     {
                         _dpi = new Vector(renderInfo.DpiX, renderInfo.DpiY);
-                        _queuedNextRender = true;
                     }
-                    
+                    Dispatcher.UIThread.Post(_renderAndSendFrameIfNeeded);
-                    Dispatcher.UIThread.Post(RenderIfNeeded);
                 }
                 if (obj is ClientSupportedPixelFormatsMessage supportedFormats)
                 {
                     lock (_lock)
-                        _supportedFormats = supportedFormats.Formats;
+                    {
-                    Dispatcher.UIThread.Post(RenderIfNeeded);
+                        _format = TryGetValidPixelFormat(supportedFormats.Formats);
+                    }
+                    Dispatcher.UIThread.Post(_renderAndSendFrameIfNeeded);
                 }
                 if (obj is MeasureViewportMessage measure)
                     Dispatcher.UIThread.Post(() =>
@@ -161,7 +160,7 @@ namespace Avalonia.Controls.Remote.Server
                                 }
                                 _dpi = new Vector(allocation.DpiX, allocation.DpiY);
                                 ClientSize = new Size(allocation.Width, allocation.Height);
-                                RenderIfNeeded();
+                                RenderAndSendFrameIfNeeded();
                             });
 
                         _pendingAllocation = allocated;
@@ -250,10 +249,24 @@ namespace Avalonia.Controls.Remote.Server
             }
         }
 
+        private static ProtocolPixelFormat? TryGetValidPixelFormat(ProtocolPixelFormat[]? formats)
+        {
+            if (formats is not null)
+            {
+                foreach (var format in formats)
+                {
+                    if (format is >= 0 and <= ProtocolPixelFormat.MaxValue)
+                        return format;
+                }
+            }
+
+            return null;
+        }
+
         protected void SetDpi(Vector dpi)
         {
             _dpi = dpi;
-            RenderIfNeeded();
+            RenderAndSendFrameIfNeeded();
         }
 
         protected virtual Size Measure(Size constraint)
@@ -265,88 +278,63 @@ namespace Avalonia.Controls.Remote.Server
 
         public override IEnumerable<object> Surfaces => new[] { this };
 
-        private FrameMessage RenderFrame(int width, int height, ProtocolPixelFormat? format)
+        private Framebuffer GetOrCreateFramebuffer()
         {
-            var scalingX = _dpi.X / 96.0;
+            lock (_lock)
-            var scalingY = _dpi.Y / 96.0;
-
-            width = (int)(width * scalingX);
-            height = (int)(height * scalingY);
-
-            var fmt = format ?? ProtocolPixelFormat.Rgba8888;
-            var bpp = fmt == ProtocolPixelFormat.Rgb565 ? 2 : 4;
-            var data = new byte[width * height * bpp];
-            var handle = GCHandle.Alloc(data, GCHandleType.Pinned);
-
-            try
-            {
-                if (width > 0 && height > 0)
-                {
-                    _framebuffer = new LockedFramebuffer(handle.AddrOfPinnedObject(), new PixelSize(width, height), width * bpp, _dpi, new((PixelFormatEnum)fmt),
-                        null);
-                    Paint?.Invoke(new Rect(0, 0, width, height));
-                }
-            }
-            finally
             {
-                _framebuffer = null;
+                if (_format is not { } format)
-                handle.Free();
+                    _framebuffer = Framebuffer.Empty;
+                else if (_framebuffer.Format != format || _framebuffer.ClientSize != ClientSize || _framebuffer.Dpi != _dpi)
+                    _framebuffer = new Framebuffer(format, ClientSize, _dpi);
+
+                return _framebuffer;
             }
-            return new FrameMessage
-            {
-                Data = data,
-                Format = fmt,
-                Width = width,
-                Height = height,
-                Stride = width * bpp,
-                DpiX = _dpi.X,
-                DpiY = _dpi.Y
-            };
         }
 
         public ILockedFramebuffer Lock()
-        {
+            => GetOrCreateFramebuffer().Lock(_sendLastFrameIfNeeded);
-            if (_framebuffer == null)
-                throw new InvalidOperationException("Paint was not requested, wait for Paint event");
-            return _framebuffer;
-        }
 
-        protected void RenderIfNeeded()
+        private void SendLastFrameIfNeeded()
         {
-            lock (_lock)
+            if (IsDisposed)
-            {
+                return;
-                if (_lastReceivedFrame != _lastSentFrame || !_queuedNextRender || _supportedFormats == null)
-                    return;
 
-            }
+            Framebuffer framebuffer;
-
+            long sequenceId;
-            var format = ProtocolPixelFormat.Rgba8888;
-            foreach(var fmt in _supportedFormats)
-                if (fmt <= ProtocolPixelFormat.MaxValue)
-                {
-                    format = fmt;
-                    break;
-                }
 
-            _inRender = true;
-            var frame = RenderFrame((int) ClientSize.Width, (int) ClientSize.Height, format);
             lock (_lock)
             {
+                // Ideally we should only send a frame if its status is Rendered: since the renderer might not be
+                // initialized at the start, we're sending black frames in this case. However, this was the historical
+                // behavior and some external programs are depending on receiving a frame asap.
+                if (_lastReceivedFrame != _lastSentFrame || _framebuffer.GetStatus() == FrameStatus.CopiedToMessage)
+                    return;
+
+                framebuffer = _framebuffer;
                 _lastSentFrame = _nextFrameNumber++;
-                frame.SequenceId = _lastSentFrame;
+                sequenceId = _lastSentFrame;
-                _queuedNextRender = false;
             }
-            _inRender = false;
+
-            _transport.Send(frame);
+            _transport.Send(framebuffer.ToMessage(sequenceId));
         }
 
-        private void QueueNextRender()
+        protected void RenderAndSendFrameIfNeeded()
         {
-            if (!_inRender && !IsDisposed)
+            if (IsDisposed)
+                return;
+
+            lock (_lock)
             {
-                _queuedNextRender = true;
+                if (_lastReceivedFrame != _lastSentFrame || _format is null)
-                DispatcherTimer.RunOnce(RenderIfNeeded, TimeSpan.FromMilliseconds(2), DispatcherPriority.Background);
+                    return;
             }
+
+            var framebuffer = GetOrCreateFramebuffer();
+
+            if (framebuffer.Stride > 0)
+                Paint?.Invoke(new Rect(framebuffer.ClientSize));
+
+            SendLastFrameIfNeeded();
         }
 
         public override IMouseDevice MouseDevice { get; } = new MouseDevice();

src/Avalonia.DesignerSupport/Remote/PreviewerWindowImpl.cs

@@ -67,7 +67,7 @@ namespace Avalonia.DesignerSupport.Remote
                 Height = clientSize.Height
             });
             ClientSize = clientSize;
-            RenderIfNeeded();
+            RenderAndSendFrameIfNeeded();
         }
 
         public void Move(PixelPoint point)

src/Avalonia.DesignerSupport/Remote/PreviewerWindowingPlatform.cs

@@ -52,7 +52,7 @@ namespace Avalonia.DesignerSupport.Remote
                 .Bind<IKeyboardDevice>().ToConstant(Keyboard)
                 .Bind<IPlatformSettings>().ToSingleton<DefaultPlatformSettings>()
                 .Bind<IDispatcherImpl>().ToConstant(new ManagedDispatcherImpl(null))
-                .Bind<IRenderTimer>().ToConstant(new DefaultRenderTimer(60))
+                .Bind<IRenderTimer>().ToConstant(new UiThreadRenderTimer(60))
                 .Bind<IWindowingPlatform>().ToConstant(instance)
                 .Bind<IPlatformIconLoader>().ToSingleton<IconLoaderStub>()
                 .Bind<PlatformHotkeyConfiguration>().ToSingleton<PlatformHotkeyConfiguration>();