Don't use Linq and test for common path first.

src/ImageSharp/Image{TPixel}.cs

@@ -201,14 +201,14 @@ namespace SixLabors.ImageSharp
         public bool TryGetSinglePixelSpan(out Span<TPixel> span)
         {
             IMemoryGroup<TPixel> mg = this.GetPixelMemoryGroup();
-            if (mg.Count > 1)
+            if (mg.Count == 1)
             {
-                span = default;
+                span = mg[0].Span;
-                return false;
+                return true;
             }
 
-            span = mg.Single().Span;
+            span = default;
-            return true;
+            return false;
         }
 
         /// <summary>