diff --git a/src/Avalonia.Controls/Image.cs b/src/Avalonia.Controls/Image.cs
index fcbbd7e82b..c2cdcb4e69 100644
--- a/src/Avalonia.Controls/Image.cs
+++ b/src/Avalonia.Controls/Image.cs
@@ -27,7 +27,9 @@ namespace Avalonia.Controls
         /// Defines the <see cref="StretchDirection"/> property.
         /// </summary>
         public static readonly StyledProperty<StretchDirection> StretchDirectionProperty =
-            AvaloniaProperty.Register<Image, StretchDirection>(nameof(StretchDirection));
+            AvaloniaProperty.Register<Image, StretchDirection>(
+                nameof(StretchDirection),
+                StretchDirection.Both);
 
         static Image()
         {
@@ -74,7 +76,7 @@ namespace Avalonia.Controls
             {
                 Rect viewPort = new Rect(Bounds.Size);
                 Size sourceSize = new Size(source.PixelSize.Width, source.PixelSize.Height);
-                Vector scale = Stretch.CalculateScaling(Bounds.Size, sourceSize);
+                Vector scale = Stretch.CalculateScaling(Bounds.Size, sourceSize, StretchDirection);
                 Size scaledSize = sourceSize * scale;
                 Rect destRect = viewPort
                     .CenterRect(new Rect(scaledSize))
@@ -100,15 +102,8 @@ namespace Avalonia.Controls
 
             if (source != null)
             {
-                Size sourceSize = new Size(source.PixelSize.Width, source.PixelSize.Height);
-                if (double.IsInfinity(availableSize.Width) || double.IsInfinity(availableSize.Height))
-                {
-                    result = sourceSize;
-                }
-                else
-                {
-                    result = Stretch.CalculateSize(availableSize, sourceSize);
-                }
+                var sourceSize = new Size(source.PixelSize.Width, source.PixelSize.Height);
+                result = Stretch.CalculateSize(availableSize, sourceSize, StretchDirection);
             }
 
             return result;
diff --git a/src/Avalonia.Visuals/Media/MediaExtensions.cs b/src/Avalonia.Visuals/Media/MediaExtensions.cs
index 95d17b454e..36bda5f483 100644
--- a/src/Avalonia.Visuals/Media/MediaExtensions.cs
+++ b/src/Avalonia.Visuals/Media/MediaExtensions.cs
@@ -2,6 +2,7 @@
 // Licensed under the MIT license. See licence.md file in the project root for full license information.
 
 using System;
+using Avalonia.Utilities;
 
 namespace Avalonia.Media
 {
@@ -16,24 +17,82 @@ namespace Avalonia.Media
         /// <param name="stretch">The stretch mode.</param>
         /// <param name="destinationSize">The size of the destination viewport.</param>
         /// <param name="sourceSize">The size of the source.</param>
+        /// <param name="stretchDirection">The stretch direction.</param>
         /// <returns>A vector with the X and Y scaling factors.</returns>
-        public static Vector CalculateScaling(this Stretch stretch, Size destinationSize, Size sourceSize)
+        public static Vector CalculateScaling(
+            this Stretch stretch,
+            Size destinationSize,
+            Size sourceSize,
+            StretchDirection stretchDirection = StretchDirection.Both)
         {
-            double scaleX = 1;
-            double scaleY = 1;
+            var scaleX = 1.0;
+            var scaleY = 1.0;
 
-            if (stretch != Stretch.None)
+            bool isConstrainedWidth = !double.IsPositiveInfinity(destinationSize.Width);
+            bool isConstrainedHeight = !double.IsPositiveInfinity(destinationSize.Height);
+
+            if ((stretch == Stretch.Uniform || stretch == Stretch.UniformToFill || stretch == Stretch.Fill)
+                 && (isConstrainedWidth || isConstrainedHeight))
             {
-                scaleX = destinationSize.Width / sourceSize.Width;
-                scaleY = destinationSize.Height / sourceSize.Height;
+                // Compute scaling factors for both axes
+                scaleX = MathUtilities.IsZero(sourceSize.Width) ? 0.0 : destinationSize.Width / sourceSize.Width;
+                scaleY = MathUtilities.IsZero(sourceSize.Height) ? 0.0 : destinationSize.Height / sourceSize.Height;
 
-                switch (stretch)
+                if (!isConstrainedWidth)
+                {
+                    scaleX = scaleY;
+                }
+                else if (!isConstrainedHeight)
+                {
+                    scaleY = scaleX;
+                }
+                else
                 {
-                    case Stretch.Uniform:
-                        scaleX = scaleY = Math.Min(scaleX, scaleY);
+                    // If not preserving aspect ratio, then just apply transform to fit
+                    switch (stretch)
+                    {
+                        case Stretch.Uniform:
+                            // Find minimum scale that we use for both axes
+                            double minscale = scaleX < scaleY ? scaleX : scaleY;
+                            scaleX = scaleY = minscale;
+                            break;
+
+                        case Stretch.UniformToFill:
+                            // Find maximum scale that we use for both axes
+                            double maxscale = scaleX > scaleY ? scaleX : scaleY;
+                            scaleX = scaleY = maxscale;
+                            break;
+
+                        case Stretch.Fill:
+                            // We already computed the fill scale factors above, so just use them
+                            break;
+                    }
+                }
+
+                // Apply stretch direction by bounding scales.
+                // In the uniform case, scaleX=scaleY, so this sort of clamping will maintain aspect ratio
+                // In the uniform fill case, we have the same result too.
+                // In the fill case, note that we change aspect ratio, but that is okay
+                switch (stretchDirection)
+                {
+                    case StretchDirection.UpOnly:
+                        if (scaleX < 1.0)
+                            scaleX = 1.0;
+                        if (scaleY < 1.0)
+                            scaleY = 1.0;
+                        break;
+
+                    case StretchDirection.DownOnly:
+                        if (scaleX > 1.0)
+                            scaleX = 1.0;
+                        if (scaleY > 1.0)
+                            scaleY = 1.0;
                         break;
-                    case Stretch.UniformToFill:
-                        scaleX = scaleY = Math.Max(scaleX, scaleY);
+
+                    case StretchDirection.Both:
+                        break;
+
+                    default:
                         break;
                 }
             }
@@ -47,10 +106,15 @@ namespace Avalonia.Media
         /// <param name="stretch">The stretch mode.</param>
         /// <param name="destinationSize">The size of the destination viewport.</param>
         /// <param name="sourceSize">The size of the source.</param>
+        /// <param name="stretchDirection">The stretch direction.</param>
         /// <returns>The size of the stretched source.</returns>
-        public static Size CalculateSize(this Stretch stretch, Size destinationSize, Size sourceSize)
+        public static Size CalculateSize(
+            this Stretch stretch,
+            Size destinationSize,
+            Size sourceSize,
+            StretchDirection stretchDirection = StretchDirection.Both)
         {
-            return sourceSize * stretch.CalculateScaling(destinationSize, sourceSize);
+            return sourceSize * stretch.CalculateScaling(destinationSize, sourceSize, stretchDirection);
         }
     }
 }