From 23886742246edf610e2917a0b7a068f1ffaae36b Mon Sep 17 00:00:00 2001 From: Julian Preece Date: Tue, 30 May 2023 14:42:26 +0100 Subject: [PATCH 1/4] Fix vnc colour format --- .../HeadlessVncPlatformExtensions.cs | 4 +++- .../Avalonia.Headless/AvaloniaHeadlessPlatform.cs | 12 +++++++++--- src/Headless/Avalonia.Headless/HeadlessWindowImpl.cs | 6 ++++-- 3 files changed, 16 insertions(+), 6 deletions(-) diff --git a/src/Headless/Avalonia.Headless.Vnc/HeadlessVncPlatformExtensions.cs b/src/Headless/Avalonia.Headless.Vnc/HeadlessVncPlatformExtensions.cs index 8e5cd1a316..29ca911a82 100644 --- a/src/Headless/Avalonia.Headless.Vnc/HeadlessVncPlatformExtensions.cs +++ b/src/Headless/Avalonia.Headless.Vnc/HeadlessVncPlatformExtensions.cs @@ -5,6 +5,7 @@ using Avalonia.Controls; using Avalonia.Controls.ApplicationLifetimes; using Avalonia.Headless; using Avalonia.Headless.Vnc; +using Avalonia.Platform; using RemoteViewing.Vnc; using RemoteViewing.Vnc.Server; @@ -22,7 +23,8 @@ namespace Avalonia return builder .UseHeadless(new AvaloniaHeadlessPlatformOptions { - UseHeadlessDrawing = false + UseHeadlessDrawing = false, + FrameBufferFormat = PixelFormat.Bgra8888 }) .AfterSetup(_ => { diff --git a/src/Headless/Avalonia.Headless/AvaloniaHeadlessPlatform.cs b/src/Headless/Avalonia.Headless/AvaloniaHeadlessPlatform.cs index 411cdd5a95..2d8506f92b 100644 --- a/src/Headless/Avalonia.Headless/AvaloniaHeadlessPlatform.cs +++ b/src/Headless/Avalonia.Headless/AvaloniaHeadlessPlatform.cs @@ -51,11 +51,16 @@ namespace Avalonia.Headless private class HeadlessWindowingPlatform : IWindowingPlatform { - public IWindowImpl CreateWindow() => new HeadlessWindowImpl(false); + readonly PixelFormat _frameBufferFormat; + public HeadlessWindowingPlatform(PixelFormat frameBufferFormat) + { + _frameBufferFormat = frameBufferFormat; + } + public IWindowImpl CreateWindow() => new HeadlessWindowImpl(false, _frameBufferFormat); public IWindowImpl CreateEmbeddableWindow() => throw new PlatformNotSupportedException(); - public IPopupImpl CreatePopup() => new HeadlessWindowImpl(true); + public IPopupImpl CreatePopup() => new HeadlessWindowImpl(true, _frameBufferFormat); public ITrayIconImpl? CreateTrayIcon() => null; } @@ -70,7 +75,7 @@ namespace Avalonia.Headless .Bind().ToSingleton() .Bind().ToConstant(new KeyboardDevice()) .Bind().ToConstant(new RenderTimer(60)) - .Bind().ToConstant(new HeadlessWindowingPlatform()) + .Bind().ToConstant(new HeadlessWindowingPlatform(opts.FrameBufferFormat)) .Bind().ToSingleton(); Compositor = new Compositor( null); } @@ -92,6 +97,7 @@ namespace Avalonia.Headless public class AvaloniaHeadlessPlatformOptions { public bool UseHeadlessDrawing { get; set; } = true; + public PixelFormat FrameBufferFormat { get; set; } = PixelFormat.Rgba8888; } public static class AvaloniaHeadlessPlatformExtensions diff --git a/src/Headless/Avalonia.Headless/HeadlessWindowImpl.cs b/src/Headless/Avalonia.Headless/HeadlessWindowImpl.cs index 3f486d98f6..7807b24cf8 100644 --- a/src/Headless/Avalonia.Headless/HeadlessWindowImpl.cs +++ b/src/Headless/Avalonia.Headless/HeadlessWindowImpl.cs @@ -24,9 +24,10 @@ namespace Avalonia.Headless private readonly Pointer _mousePointer; private WriteableBitmap? _lastRenderedFrame; private readonly object _sync = new object(); + private readonly PixelFormat _frameBufferFormat; public bool IsPopup { get; } - public HeadlessWindowImpl(bool isPopup) + public HeadlessWindowImpl(bool isPopup, PixelFormat frameBufferFormat) { IsPopup = isPopup; Surfaces = new object[] { this }; @@ -34,6 +35,7 @@ namespace Avalonia.Headless _mousePointer = new Pointer(Pointer.GetNextFreeId(), PointerType.Mouse, true); MouseDevice = new MouseDevice(_mousePointer); ClientSize = new Size(1024, 768); + _frameBufferFormat = frameBufferFormat; } public void Dispose() @@ -201,7 +203,7 @@ namespace Avalonia.Headless public ILockedFramebuffer Lock() { - var bmp = new WriteableBitmap(PixelSize.FromSize(ClientSize, RenderScaling), new Vector(96, 96) * RenderScaling, PixelFormat.Rgba8888, AlphaFormat.Premul); + var bmp = new WriteableBitmap(PixelSize.FromSize(ClientSize, RenderScaling), new Vector(96, 96) * RenderScaling, _frameBufferFormat, AlphaFormat.Premul); var fb = bmp.Lock(); return new FramebufferProxy(fb, () => { From 5df3a4664501e35d81462de2d48f4837a4d4c104 Mon Sep 17 00:00:00 2001 From: Julian Preece Date: Tue, 30 May 2023 14:56:05 +0100 Subject: [PATCH 2/4] Handle keyboard input for vnc --- .../HeadlessVncFramebufferSource.cs | 259 +++++++++++++++++- .../HeadlessWindowExtensions.cs | 6 + .../Avalonia.Headless/HeadlessWindowImpl.cs | 8 + .../Avalonia.Headless/IHeadlessWindow.cs | 1 + 4 files changed, 272 insertions(+), 2 deletions(-) diff --git a/src/Headless/Avalonia.Headless.Vnc/HeadlessVncFramebufferSource.cs b/src/Headless/Avalonia.Headless.Vnc/HeadlessVncFramebufferSource.cs index 1a7bb2ec2f..086c4d06d3 100644 --- a/src/Headless/Avalonia.Headless.Vnc/HeadlessVncFramebufferSource.cs +++ b/src/Headless/Avalonia.Headless.Vnc/HeadlessVncFramebufferSource.cs @@ -1,8 +1,7 @@ using System; -using System.Runtime.InteropServices; +using System.Globalization; using Avalonia.Controls; using Avalonia.Input; -using Avalonia.Platform; using Avalonia.Threading; using RemoteViewing.Vnc; using RemoteViewing.Vnc.Server; @@ -16,6 +15,8 @@ namespace Avalonia.Headless.Vnc public VncFramebuffer _framebuffer = new VncFramebuffer("Avalonia", 1, 1, VncPixelFormat.RGB32); private VncButton _previousButtons; + private RawInputModifiers _keyState; + public HeadlessVncFramebufferSource(VncServerSession session, Window window) { Window = window; @@ -49,9 +50,263 @@ namespace Avalonia.Headless.Vnc _previousButtons = buttons; }, DispatcherPriority.Input); }; + + session.KeyChanged += (_, args) => + { + Key? key = TranslateKey(args.Keysym); + if (key == null) + return; + + //we only care about text input on key up + string? inputText = args.Pressed ? null : KeyToText(args.Keysym); + Dispatcher.UIThread.Post(() => + { + if (args.Pressed) + Window?.KeyPress(key.Value, _keyState); + else + Window?.KeyRelease(key.Value, _keyState); + + if (inputText != null) + Window?.KeyTextInput(inputText); + }, DispatcherPriority.Input); + }; + } + + private bool CheckKeyIsInputModifier(KeyChangedEventArgs args) + { + RawInputModifiers? toggleModifier = args.Keysym switch + { + KeySym.ShiftLeft or KeySym.ShiftRight => RawInputModifiers.Shift, + KeySym.ControlLeft or KeySym.ControlRight => RawInputModifiers.Control, + KeySym.AltLeft or KeySym.AltRight => RawInputModifiers.Alt, + _ => null + }; + if(!toggleModifier.HasValue) + return false; + + if(args.Pressed) + _keyState |= toggleModifier.Value; + else + _keyState &= ~toggleModifier.Value; + + return true; + } + + private static string? KeyToText(KeySym key) + { + int keyCode = (int)key; + if (key >= KeySym.Space && key <= KeySym.AsciiTilde) + return new string((char)key, 1); + + //handle as normal text chars 0-9 + if (key >= KeySym.NumPad0 && key <= KeySym.NumPad9) + return new string((char)(key - 65408), 1); + + switch (key) + { + case KeySym.NumPadAdd: return "+"; + case KeySym.NumPadSubtract: return "-"; + case KeySym.NumPadMultiply: return "*"; + case KeySym.NumPadDivide: return "/"; + case KeySym.NumPadSeparator: return NumberFormatInfo.CurrentInfo.NumberDecimalSeparator; + } + + return null; } + private static Key? TranslateKey(KeySym key) => + key switch + { + KeySym.Backspace => Key.Back, + KeySym.Tab => Key.Tab, + KeySym.LineFeed => Key.LineFeed, + KeySym.Clear => Key.Clear, + KeySym.Return => Key.Return, + KeySym.Pause => Key.Pause, + KeySym.Escape => Key.Escape, + KeySym.Delete => Key.Delete, + KeySym.Home => Key.Home, + KeySym.Left => Key.Left, + KeySym.Up => Key.Up, + KeySym.Right => Key.Right, + KeySym.Down => Key.Down, + KeySym.PageUp => Key.PageUp, + KeySym.PageDown => Key.PageDown, + KeySym.End => Key.End, + KeySym.Begin => Key.Home, + KeySym.Select => Key.Select, + KeySym.Print => Key.Print, + KeySym.Execute => Key.Execute, + KeySym.Insert => Key.Insert, + KeySym.Cancel => Key.Cancel, + KeySym.Help => Key.Help, + KeySym.Break => Key.Pause, + KeySym.Num_Lock => Key.NumLock, + KeySym.NumPadSpace => Key.Space, + KeySym.NumPadTab => Key.Tab, + KeySym.NumPadEnter => Key.Enter, + KeySym.NumPadF1 => Key.F1, + KeySym.NumPadF2 => Key.F2, + KeySym.NumPadF3 => Key.F3, + KeySym.NumPadF4 => Key.F4, + KeySym.NumPadHome => Key.Home, + KeySym.NumPadLeft => Key.Left, + KeySym.NumPadUp => Key.Up, + KeySym.NumPadRight => Key.Right, + KeySym.NumPadDown => Key.Down, + KeySym.NumPadPageUp => Key.PageUp, + KeySym.NumPadPageDown => Key.PageDown, + KeySym.NumPadEnd => Key.End, + KeySym.NumPadBegin => Key.Home, + KeySym.NumPadInsert => Key.Insert, + KeySym.NumPadDelete => Key.Delete, + KeySym.NumPadEqual => Key.Return, + KeySym.NumPadMultiply => Key.Multiply, + KeySym.NumPadAdd => Key.Add, + KeySym.NumPadSeparator => Key.Separator, + KeySym.NumPadSubtract => Key.Subtract, + KeySym.NumPadDecimal => Key.Decimal, + KeySym.NumPadDivide => Key.Divide, + KeySym.NumPad0 => Key.NumPad0, + KeySym.NumPad1 => Key.NumPad1, + KeySym.NumPad2 => Key.NumPad2, + KeySym.NumPad3 => Key.NumPad3, + KeySym.NumPad4 => Key.NumPad4, + KeySym.NumPad5 => Key.NumPad5, + KeySym.NumPad6 => Key.NumPad6, + KeySym.NumPad7 => Key.NumPad7, + KeySym.NumPad8 => Key.NumPad8, + KeySym.NumPad9 => Key.NumPad9, + KeySym.F1 => Key.F1, + KeySym.F2 => Key.F2, + KeySym.F3 => Key.F3, + KeySym.F4 => Key.F4, + KeySym.F5 => Key.F5, + KeySym.F6 => Key.F6, + KeySym.F7 => Key.F7, + KeySym.F8 => Key.F8, + KeySym.F9 => Key.F9, + KeySym.F10 => Key.F10, + KeySym.F11 => Key.F11, + KeySym.F12 => Key.F12, + KeySym.F13 => Key.F13, + KeySym.F14 => Key.F14, + KeySym.F15 => Key.F15, + KeySym.F16 => Key.F16, + KeySym.F17 => Key.F17, + KeySym.F18 => Key.F18, + KeySym.F19 => Key.F19, + KeySym.F20 => Key.F20, + KeySym.F21 => Key.F21, + KeySym.F22 => Key.F22, + KeySym.F23 => Key.F23, + KeySym.F24 => Key.F24, + KeySym.ShiftLeft => Key.LeftShift, + KeySym.ShiftRight => Key.RightShift, + KeySym.ControlLeft => Key.LeftCtrl, + KeySym.ControlRight => Key.RightCtrl, + KeySym.CapsLock => Key.CapsLock, + KeySym.AltLeft => Key.LeftAlt, + KeySym.AltRight => Key.RightAlt, + KeySym.Space => Key.Space, + KeySym.Exclamation => Key.D1, + KeySym.Quote => Key.D2, + KeySym.NumberSign => Key.D3, + KeySym.Dollar => Key.D4, + KeySym.Percent => Key.D5, + KeySym.Ampersand => Key.D7, + KeySym.Apostrophe => Key.Oem3, + KeySym.ParenthesisLeft => Key.D9, + KeySym.ParenthesisRight => Key.D0, + KeySym.Asterisk => Key.D8, + KeySym.Plus => Key.OemPlus, + KeySym.Comma => Key.OemComma, + KeySym.Minus => Key.OemMinus, + KeySym.Period => Key.OemPeriod, + KeySym.Slash => Key.OemQuestion, + KeySym.D0 => Key.D0, + KeySym.D1 => Key.D1, + KeySym.D2 => Key.D2, + KeySym.D3 => Key.D3, + KeySym.D4 => Key.D4, + KeySym.D5 => Key.D5, + KeySym.D6 => Key.D6, + KeySym.D7 => Key.D7, + KeySym.D8 => Key.D8, + KeySym.D9 => Key.D9, + KeySym.Colon => Key.OemSemicolon, + KeySym.Semicolon => Key.OemSemicolon, + KeySym.Less => Key.OemComma, + KeySym.Equal => Key.OemPlus, + KeySym.Greater => Key.OemPeriod, + KeySym.Question => Key.OemQuestion, + KeySym.At => Key.Oem3, + KeySym.A => Key.A, + KeySym.B => Key.B, + KeySym.C => Key.C, + KeySym.D => Key.D, + KeySym.E => Key.E, + KeySym.F => Key.F, + KeySym.G => Key.G, + KeySym.H => Key.H, + KeySym.I => Key.I, + KeySym.J => Key.J, + KeySym.K => Key.K, + KeySym.L => Key.L, + KeySym.M => Key.M, + KeySym.N => Key.N, + KeySym.O => Key.O, + KeySym.P => Key.P, + KeySym.Q => Key.Q, + KeySym.R => Key.R, + KeySym.S => Key.S, + KeySym.T => Key.T, + KeySym.U => Key.U, + KeySym.V => Key.V, + KeySym.W => Key.W, + KeySym.X => Key.X, + KeySym.Y => Key.Y, + KeySym.Z => Key.Z, + KeySym.BracketLeft => Key.OemOpenBrackets, + KeySym.Backslash => Key.OemPipe, + KeySym.Bracketright => Key.OemCloseBrackets, + KeySym.Underscore => Key.OemMinus, + KeySym.Grave => Key.Oem8, + KeySym.a => Key.A, + KeySym.b => Key.B, + KeySym.c => Key.C, + KeySym.d => Key.D, + KeySym.e => Key.E, + KeySym.f => Key.F, + KeySym.g => Key.G, + KeySym.h => Key.H, + KeySym.i => Key.I, + KeySym.j => Key.J, + KeySym.k => Key.K, + KeySym.l => Key.L, + KeySym.m => Key.M, + KeySym.n => Key.M, + KeySym.o => Key.O, + KeySym.p => Key.P, + KeySym.q => Key.Q, + KeySym.r => Key.R, + KeySym.s => Key.S, + KeySym.t => Key.T, + KeySym.u => Key.U, + KeySym.v => Key.V, + KeySym.w => Key.W, + KeySym.x => Key.X, + KeySym.y => Key.Y, + KeySym.z => Key.Z, + KeySym.BraceLeft => Key.OemOpenBrackets, + KeySym.Bar => Key.OemPipe, + KeySym.BraceRight => Key.OemCloseBrackets, + KeySym.AsciiTilde => Key.OemTilde, + _ => null + }; + + [Flags] enum VncButton { diff --git a/src/Headless/Avalonia.Headless/HeadlessWindowExtensions.cs b/src/Headless/Avalonia.Headless/HeadlessWindowExtensions.cs index 61659dee2b..3d07cac64e 100644 --- a/src/Headless/Avalonia.Headless/HeadlessWindowExtensions.cs +++ b/src/Headless/Avalonia.Headless/HeadlessWindowExtensions.cs @@ -52,6 +52,12 @@ public static class HeadlessWindowExtensions public static void KeyRelease(this TopLevel topLevel, Key key, RawInputModifiers modifiers) => RunJobsOnImpl(topLevel, w => w.KeyRelease(key, modifiers)); + /// + /// Simulates a text input event on the headless window/toplevel + /// + public static void KeyTextInput(this TopLevel topLevel, string text) => + RunJobsOnImpl(topLevel, w => w.TextInput(text)); + /// /// Simulates mouse down on the headless window/toplevel. /// diff --git a/src/Headless/Avalonia.Headless/HeadlessWindowImpl.cs b/src/Headless/Avalonia.Headless/HeadlessWindowImpl.cs index 3f486d98f6..117d315cba 100644 --- a/src/Headless/Avalonia.Headless/HeadlessWindowImpl.cs +++ b/src/Headless/Avalonia.Headless/HeadlessWindowImpl.cs @@ -274,6 +274,14 @@ namespace Avalonia.Headless Input?.Invoke(new RawKeyEventArgs(_keyboard, Timestamp, InputRoot!, RawKeyEventType.KeyUp, key, modifiers)); } + void IHeadlessWindow.TextInput(string text) + { + if (InputRoot == null) + return; + + Input?.Invoke(new RawTextInputEventArgs(_keyboard, 0, InputRoot, text)); + } + void IHeadlessWindow.MouseDown(Point point, MouseButton button, RawInputModifiers modifiers) { Input?.Invoke(new RawPointerEventArgs(MouseDevice, Timestamp, InputRoot!, diff --git a/src/Headless/Avalonia.Headless/IHeadlessWindow.cs b/src/Headless/Avalonia.Headless/IHeadlessWindow.cs index ef501b3800..6d34dbbd4b 100644 --- a/src/Headless/Avalonia.Headless/IHeadlessWindow.cs +++ b/src/Headless/Avalonia.Headless/IHeadlessWindow.cs @@ -11,6 +11,7 @@ namespace Avalonia.Headless WriteableBitmap? GetLastRenderedFrame(); void KeyPress(Key key, RawInputModifiers modifiers); void KeyRelease(Key key, RawInputModifiers modifiers); + void TextInput(string text); void MouseDown(Point point, MouseButton button, RawInputModifiers modifiers = RawInputModifiers.None); void MouseMove(Point point, RawInputModifiers modifiers = RawInputModifiers.None); void MouseUp(Point point, MouseButton button, RawInputModifiers modifiers = RawInputModifiers.None); From 0027ace53b65244617dbe072d6166b9817aef95f Mon Sep 17 00:00:00 2001 From: Julian Preece Date: Tue, 30 May 2023 14:56:23 +0100 Subject: [PATCH 3/4] handle mouse wheel for vnc --- .../Avalonia.Headless.Vnc/HeadlessVncFramebufferSource.cs | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/Headless/Avalonia.Headless.Vnc/HeadlessVncFramebufferSource.cs b/src/Headless/Avalonia.Headless.Vnc/HeadlessVncFramebufferSource.cs index 086c4d06d3..b1cedb684e 100644 --- a/src/Headless/Avalonia.Headless.Vnc/HeadlessVncFramebufferSource.cs +++ b/src/Headless/Avalonia.Headless.Vnc/HeadlessVncFramebufferSource.cs @@ -47,6 +47,14 @@ namespace Avalonia.Headless.Vnc foreach (var btn in CheckedButtons) if (!_previousButtons.HasFlag(btn) && buttons.HasFlag(btn)) Window?.MouseDown(pt, TranslateButton(btn), modifiers); + + + if (buttons == VncButton.ScrollUp) + Window?.MouseWheel(pt, Vector.One, modifiers); + + else if (buttons == VncButton.ScrollDown) + Window?.MouseWheel(pt, Vector.One.Negate(), modifiers); + _previousButtons = buttons; }, DispatcherPriority.Input); }; From 4511be1765f65a8c55f464a1abcd2cc40bbfeb40 Mon Sep 17 00:00:00 2001 From: Julian Preece Date: Tue, 30 May 2023 15:31:33 +0100 Subject: [PATCH 4/4] Fix using key modifier for input --- .../HeadlessVncFramebufferSource.cs | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/src/Headless/Avalonia.Headless.Vnc/HeadlessVncFramebufferSource.cs b/src/Headless/Avalonia.Headless.Vnc/HeadlessVncFramebufferSource.cs index b1cedb684e..91ef942723 100644 --- a/src/Headless/Avalonia.Headless.Vnc/HeadlessVncFramebufferSource.cs +++ b/src/Headless/Avalonia.Headless.Vnc/HeadlessVncFramebufferSource.cs @@ -61,12 +61,19 @@ namespace Avalonia.Headless.Vnc session.KeyChanged += (_, args) => { + bool isModifierKey = CheckKeyIsInputModifier(args); + if (isModifierKey) + return; + Key? key = TranslateKey(args.Keysym); if (key == null) return; - //we only care about text input on key up - string? inputText = args.Pressed ? null : KeyToText(args.Keysym); + //we only care about text input on key up if not using Ctrl or Alt + string? inputText = args.Pressed || _keyState.HasFlag(RawInputModifiers.Control) || _keyState.HasFlag(RawInputModifiers.Alt) + ? null + : KeyToText(args.Keysym); + Dispatcher.UIThread.Post(() => { if (args.Pressed)