diff --git a/src/ImageSharp/Formats/ImageFormatManager.cs b/src/ImageSharp/Formats/ImageFormatManager.cs
index 63fd02d8d6..fdbc4ee442 100644
--- a/src/ImageSharp/Formats/ImageFormatManager.cs
+++ b/src/ImageSharp/Formats/ImageFormatManager.cs
@@ -13,6 +13,12 @@ namespace SixLabors.ImageSharp.Formats
     /// </summary>
     public class ImageFormatManager
     {
+        /// <summary>
+        /// Used for locking against as there is no ConcurrentSet type.
+        /// <see href="https://github.com/dotnet/corefx/issues/6318"/>
+        /// </summary>
+        private static readonly object HashLock = new object();
+
         /// <summary>
         /// The list of supported <see cref="IImageEncoder"/> keyed to mime types.
         /// </summary>
@@ -26,7 +32,7 @@ namespace SixLabors.ImageSharp.Formats
         /// <summary>
         /// The list of supported <see cref="IImageFormat"/>s.
         /// </summary>
-        private readonly ConcurrentBag<IImageFormat> imageFormats = new ConcurrentBag<IImageFormat>();
+        private readonly HashSet<IImageFormat> imageFormats = new HashSet<IImageFormat>();
 
         /// <summary>
         /// The list of supported <see cref="IImageFormatDetector"/>s.
@@ -74,7 +80,14 @@ namespace SixLabors.ImageSharp.Formats
             Guard.NotNull(format, nameof(format));
             Guard.NotNull(format.MimeTypes, nameof(format.MimeTypes));
             Guard.NotNull(format.FileExtensions, nameof(format.FileExtensions));
-            this.imageFormats.Add(format);
+
+            lock (HashLock)
+            {
+                if (!this.imageFormats.Contains(format))
+                {
+                    this.imageFormats.Add(format);
+                }
+            }
         }
 
         /// <summary>
diff --git a/tests/ImageSharp.Tests/ConfigurationTests.cs b/tests/ImageSharp.Tests/ConfigurationTests.cs
index 1a7183df81..5c5eb9e9d9 100644
--- a/tests/ImageSharp.Tests/ConfigurationTests.cs
+++ b/tests/ImageSharp.Tests/ConfigurationTests.cs
@@ -2,13 +2,9 @@
 // Licensed under the Apache License, Version 2.0.
 
 using System;
-using System.Collections.Generic;
-using System.IO;
 using System.Linq;
-using SixLabors.ImageSharp.Formats;
-using SixLabors.ImageSharp.IO;
-using SixLabors.ImageSharp.PixelFormats;
 using Moq;
+using SixLabors.ImageSharp.IO;
 using Xunit;
 // ReSharper disable InconsistentNaming
 
@@ -96,5 +92,13 @@ namespace SixLabors.ImageSharp.Tests
 
             provider.Verify(x => x.Configure(config));
         }
+
+        [Fact]
+        public void DefaultConfigurationHasCorrectFormatCount()
+        {
+            Configuration config = Configuration.Default;
+
+            Assert.Equal(4, config.ImageFormats.Count());
+        }
     }
 }
\ No newline at end of file