experiments

samples/GpuInterop/VulkanDemo/InteropTransferDirection.cs

@@ -0,0 +1,8 @@
+namespace GpuInterop.VulkanDemo;
+
+public enum InteropTransferDirection
+{
+    None,
+    AppToAvalonia,
+    AvaloniaToApp
+}

samples/GpuInterop/VulkanDemo/VulkanContent.cs

@@ -156,7 +156,7 @@ unsafe class VulkanContent : IDisposable
 
         _colorAttachment!.TransitionLayout(commandBuffer.InternalHandle,
             ImageLayout.Undefined, AccessFlags.None,
-            ImageLayout.ColorAttachmentOptimal, AccessFlags.ColorAttachmentWriteBit);
+            ImageLayout.ColorAttachmentOptimal, AccessFlags.ColorAttachmentWriteBit, false);
 
         var commandBufferHandle = new CommandBuffer(commandBuffer.Handle);
 
@@ -215,8 +215,8 @@ unsafe class VulkanContent : IDisposable
         
         api.CmdEndRenderPass(commandBufferHandle);
         
-        _colorAttachment.TransitionLayout(commandBuffer.InternalHandle, ImageLayout.TransferSrcOptimal, AccessFlags.TransferReadBit);
-        image.TransitionLayout(commandBuffer.InternalHandle, ImageLayout.TransferDstOptimal, AccessFlags.TransferWriteBit);
+        _colorAttachment.TransitionLayout(commandBuffer.InternalHandle, ImageLayout.TransferSrcOptimal, AccessFlags.TransferReadBit, false);
+        image.TransitionLayout(commandBuffer.InternalHandle, ImageLayout.TransferDstOptimal, AccessFlags.TransferWriteBit, false);
         
         
         var srcBlitRegion = new ImageBlit

samples/GpuInterop/VulkanDemo/VulkanImage.cs

@@ -25,6 +25,7 @@ public unsafe class VulkanImage : IDisposable
         private readonly PhysicalDevice _physicalDevice;
         private readonly VulkanCommandBufferPool _commandBufferPool;
         private ImageLayout _currentLayout;
+        private bool _ownedByInterop;
         private AccessFlags _currentAccessFlags;
         private ImageUsageFlags _imageUsageFlags { get; }
         private ImageView _imageView { get; set; }
@@ -120,16 +121,18 @@ public unsafe class VulkanImage : IDisposable
                 Api.GetImageMemoryRequirements(_device, InternalHandle,
                     out var memoryRequirements);
 
-                var dedicatedAllocation = new MemoryDedicatedAllocateInfoKHR
+                var dedicatedAllocationStruct = new MemoryDedicatedAllocateInfoKHR
                 {
                     SType = StructureType.MemoryDedicatedAllocateInfoKhr, Image = image
                 };
+                var dedicatedAllocationPtr = &dedicatedAllocationStruct;
+                dedicatedAllocationPtr = null;
 
                 var fdExport = new ExportMemoryAllocateInfo
                 {
                     HandleTypes = handleType,
                     SType = StructureType.ExportMemoryAllocateInfo,
-                    PNext = &dedicatedAllocation
+                    PNext = dedicatedAllocationPtr
                 };
 
                 ImportMemoryWin32HandleInfoKHR handleImport = default;
@@ -141,7 +144,7 @@ public unsafe class VulkanImage : IDisposable
 
                     handleImport = new ImportMemoryWin32HandleInfoKHR
                     {
-                        PNext = &dedicatedAllocation,
+                        PNext = dedicatedAllocationPtr,
                         SType = StructureType.ImportMemoryWin32HandleInfoKhr,
                         HandleType = ExternalMemoryHandleTypeFlags.D3D11TextureBit,
                         Handle = dxgi.CreateSharedHandle(null, SharedResourceFlags.Read | SharedResourceFlags.Write),
@@ -199,7 +202,7 @@ public unsafe class VulkanImage : IDisposable
 
             _currentLayout = ImageLayout.Undefined;
 
-            TransitionLayout(ImageLayout.ColorAttachmentOptimal, AccessFlags.NoneKhr);
+            TransitionLayout(ImageLayout.ColorAttachmentOptimal, AccessFlags.NoneKhr, false);
         }
 
         public int ExportFd()
@@ -279,36 +282,44 @@ public unsafe class VulkanImage : IDisposable
         
         internal void TransitionLayout(CommandBuffer commandBuffer,
             ImageLayout fromLayout, AccessFlags fromAccessFlags,
-            ImageLayout destinationLayout, AccessFlags destinationAccessFlags)
+            ImageLayout destinationLayout, AccessFlags destinationAccessFlags, bool transferToInterop)
         {
+            var direction =
+                transferToInterop && !_ownedByInterop
+                    ? InteropTransferDirection.AppToAvalonia
+                    : !transferToInterop && _ownedByInterop
+                        ? InteropTransferDirection.AvaloniaToApp
+                        : InteropTransferDirection.None;
+            
             VulkanMemoryHelper.TransitionLayout(Api, commandBuffer, InternalHandle,
                 fromLayout,
                 fromAccessFlags,
                 destinationLayout, destinationAccessFlags,
-                MipLevels);
-            
+                MipLevels, _vk.QueueFamilyIndex, direction);
+
+            _ownedByInterop = transferToInterop;
             _currentLayout = destinationLayout;
             _currentAccessFlags = destinationAccessFlags;
         }
 
         internal void TransitionLayout(CommandBuffer commandBuffer,
-            ImageLayout destinationLayout, AccessFlags destinationAccessFlags)
+            ImageLayout destinationLayout, AccessFlags destinationAccessFlags, bool transferToInterop)
             => TransitionLayout(commandBuffer, _currentLayout, _currentAccessFlags, destinationLayout,
-                destinationAccessFlags);
+                destinationAccessFlags, transferToInterop);
         
         
-        internal void TransitionLayout(ImageLayout destinationLayout, AccessFlags destinationAccessFlags)
+        internal void TransitionLayout(ImageLayout destinationLayout, AccessFlags destinationAccessFlags, bool transferToInterop)
         {
             var commandBuffer = _commandBufferPool.CreateCommandBuffer();
             commandBuffer.BeginRecording();
-            TransitionLayout(commandBuffer.InternalHandle, destinationLayout, destinationAccessFlags);
+            TransitionLayout(commandBuffer.InternalHandle, destinationLayout, destinationAccessFlags, transferToInterop);
             commandBuffer.EndRecording();
             commandBuffer.Submit();
         }
 
-        public void TransitionLayout(uint destinationLayout, uint destinationAccessFlags)
+        public void TransitionLayout(uint destinationLayout, uint destinationAccessFlags, bool transferToInterop)
         {
-            TransitionLayout((ImageLayout)destinationLayout, (AccessFlags)destinationAccessFlags);
+            TransitionLayout((ImageLayout)destinationLayout, (AccessFlags)destinationAccessFlags, transferToInterop);
         }
 
         public unsafe void Dispose()

samples/GpuInterop/VulkanDemo/VulkanMemoryHelper.cs

@@ -27,7 +27,7 @@ internal static class VulkanMemoryHelper
         AccessFlags sourceAccessMask,
         ImageLayout destinationLayout,
         AccessFlags destinationAccessMask,
-        uint mipLevels)
+        uint mipLevels, uint appQueueFamily, InteropTransferDirection direction)
     {
         var subresourceRange = new ImageSubresourceRange(ImageAspectFlags.ColorBit, 0, mipLevels, 0, 1);
 
@@ -38,8 +38,10 @@ internal static class VulkanMemoryHelper
             DstAccessMask = destinationAccessMask,
             OldLayout = sourceLayout,
             NewLayout = destinationLayout,
-            SrcQueueFamilyIndex = Vk.QueueFamilyIgnored,
-            DstQueueFamilyIndex = Vk.QueueFamilyIgnored,
+            SrcQueueFamilyIndex = direction == InteropTransferDirection.None ? Vk.QueueFamilyIgnored
+                : direction == InteropTransferDirection.AppToAvalonia ? appQueueFamily : Vk.QueueFamilyExternal,
+            DstQueueFamilyIndex = direction == InteropTransferDirection.None ? Vk.QueueFamilyIgnored 
+                : direction == InteropTransferDirection.AppToAvalonia ? Vk.QueueFamilyExternal : appQueueFamily,
             Image = image,
             SubresourceRange = subresourceRange
         };

samples/GpuInterop/VulkanDemo/VulkanSwapchain.cs

@@ -89,7 +89,7 @@ class VulkanSwapchainImage : ISwapchainImage
 
         _image.TransitionLayout(buffer.InternalHandle, 
             ImageLayout.Undefined, AccessFlags.None,
-            ImageLayout.ColorAttachmentOptimal, AccessFlags.ColorAttachmentReadBit);
+            ImageLayout.ColorAttachmentOptimal, AccessFlags.ColorAttachmentReadBit, false);
 
         if(_image.IsDirectXBacked)
             buffer.Submit(null,null,null, null, new VulkanCommandBufferPool.VulkanCommandBuffer.KeyedMutexSubmitInfo
@@ -132,7 +132,7 @@ class VulkanSwapchainImage : ISwapchainImage
     {
         var buffer = _vk.Pool.CreateCommandBuffer();
         buffer.BeginRecording();
-        _image.TransitionLayout(buffer.InternalHandle, ImageLayout.TransferSrcOptimal, AccessFlags.TransferWriteBit);
+        _image.TransitionLayout(buffer.InternalHandle, ImageLayout.TransferSrcOptimal, AccessFlags.TransferWriteBit, true);
 
 
         if (_image.IsDirectXBacked)

src/Avalonia.Base/Rendering/Composition/Server/ServerCompositionContainerVisual.cs

@@ -64,6 +64,8 @@ namespace Avalonia.Rendering.Composition.Server
             IsDirtyComposition = false;
             return new(_transformedContentBounds, oldInvalidated, newInvalidated);
         }
+        
+        
 
         void AddEffectPaddedDirtyRect(IImmutableEffect effect, LtrbRect transformedBounds)
         {

src/Avalonia.OpenGL/ArbDebugHandler.cs

@@ -0,0 +1,45 @@
+using System;
+using System.Runtime.CompilerServices;
+using System.Runtime.InteropServices;
+
+namespace Avalonia.OpenGL;
+
+public unsafe class ArbDebugHandler
+{
+    /*        typedef void (APIENTRY *DEBUGPROCARB)(enum source,
+                                             enum type,
+                                             uint id,
+                                             enum severity,
+                                             sizei length,
+                                             const char* message,
+                                             const void* userParam);*/
+    
+    #if NET6_0_OR_GREATER
+    private static IntPtr s_Callback =
+        (IntPtr)(delegate* unmanaged[Stdcall]<int, int, uint, int, IntPtr, IntPtr, IntPtr, void>)&DebugCallback;
+    
+    [UnmanagedCallersOnly(CallConvs = [typeof(CallConvStdcall)])]
+    #else
+    [UnmanagedFunctionPointer(CallingConvention.StdCall)]
+    delegate void DebugCallbackDelegate(int source, int type, uint id, int severity, IntPtr length, IntPtr message,
+        IntPtr userParam);
+    private static DebugCallbackDelegate s_ManagedCallback = DebugCallback;
+    private static IntPtr s_Callback = Marshal.GetFunctionPointerForDelegate(s_ManagedCallback);
+    
+    #endif
+    static void DebugCallback(int source, int type, uint id, int severity, IntPtr length, IntPtr message, IntPtr userParam)
+    {
+        string msg = Marshal.PtrToStringAnsi(message) ?? string.Empty;
+        Console.WriteLine($"GL DEBUG MESSAGE: Source={source}, Type={type}, ID={id}, Severity={severity}, Message={msg}");
+    }
+    
+    public static void Install(GlInterface gl)
+    {
+        if (gl.IsDebugMessageCallbackAvailable)
+        {
+            gl.DebugMessageCallback(s_Callback, IntPtr.Zero);
+            gl.DebugMessageControl(GlConsts.GL_DONT_CARE, GlConsts.GL_DONT_CARE, GlConsts.GL_DONT_CARE, IntPtr.Zero,
+                null, 1);
+        }
+    }
+}

src/Avalonia.OpenGL/Features/ExternalObjectsOpenGlExtensionFeature.cs

@@ -159,7 +159,7 @@ public class ExternalObjectsOpenGlExtensionFeature : IGlContextExternalObjectsFe
             _ext.CreateMemoryObjectsEXT(1, out var memoryObject);
             _ext.ImportMemoryFdEXT(memoryObject, properties.MemorySize, GL_HANDLE_TYPE_OPAQUE_FD_EXT,
                 handle.Handle.ToInt32());
-            
+            Console.WriteLine($"{properties.Width}\t{properties.Height}\t{properties.MemorySize}\t{properties.MemorySize:X}");
             var err = _context.GlInterface.GetError();
             if (err != 0)
                 throw OpenGlException.GetFormattedException("glImportMemoryFdEXT", err);
@@ -170,6 +170,7 @@ public class ExternalObjectsOpenGlExtensionFeature : IGlContextExternalObjectsFe
             _context.GlInterface.BindTexture(GL_TEXTURE_2D, texture);
             _ext.TexStorageMem2DEXT(GL_TEXTURE_2D, 1, GL_RGBA8, properties.Width, properties.Height,
                 memoryObject, properties.MemoryOffset);
+            _context.GlInterface.Finish();
             err = _context.GlInterface.GetError();
 
             _context.GlInterface.BindTexture(GL_TEXTURE_2D, oldTexture);

src/Avalonia.OpenGL/GlConsts.cs

@@ -355,7 +355,7 @@ namespace Avalonia.OpenGL
 //        public const int GL_LINE_SMOOTH_HINT = 0x0C52;
 //        public const int GL_POLYGON_SMOOTH_HINT = 0x0C53;
 //        public const int GL_FOG_HINT = 0x0C54;
-//        public const int GL_DONT_CARE = 0x1100;
+        public const int GL_DONT_CARE = 0x1100;
 //        public const int GL_FASTEST = 0x1101;
 //        public const int GL_NICEST = 0x1102;
 //        public const int GL_SCISSOR_BOX = 0x0C10;

src/Avalonia.OpenGL/GlInterface.cs

@@ -385,6 +385,16 @@ namespace Avalonia.OpenGL
         [GlMinVersionEntryPoint("glGenVertexArrays", 3, 0)]
         [GlExtensionEntryPoint("glGenVertexArraysOES", "GL_OES_vertex_array_object")]
         public partial void GenVertexArrays(int n, int* rv);
+        
+        [GetProcAddress(true)]
+        [GlMinVersionEntryPoint("glDebugMessageCallback", 4, 3)]
+        [GlExtensionEntryPoint("glDebugMessageCallbackARB", "GL_ARB_debug_output")]
+        public partial void DebugMessageCallback(IntPtr proc, IntPtr userParam);
+
+        [GetProcAddress(true)]
+        [GlMinVersionEntryPoint("glDebugMessageControl", 4, 3)]
+        [GlExtensionEntryPoint("glDebugMessageControlARB", "GL_ARB_debug_output")]
+        public partial void DebugMessageControl(int source, int type, int severity, IntPtr count, uint* ids, int enabled);
 
         public int GenVertexArray()
         {

src/Avalonia.X11/Glx/GlxContext.cs

@@ -38,6 +38,7 @@ namespace Avalonia.X11.Glx
             {
                 GlInterface = new GlInterface(version, GlxInterface.SafeGetProcAddress);
                 _externalObjects = ExternalObjectsOpenGlExtensionFeature.TryCreate(this);
+                ArbDebugHandler.Install(GlInterface);
             }
         }