Cache functions enhanced to allow 360000 items

Former-commit-id: 160a64285e1a768638494526c07c3ce92b9dd5fb

src/ImageProcessor.Web/Caching/DiskCache.cs

@@ -10,9 +10,11 @@ namespace ImageProcessor.Web.Caching
     #region Using
     using System;
     using System.Collections.Generic;
+    using System.Globalization;
     using System.IO;
     using System.Linq;
     using System.Text.RegularExpressions;
+    using System.Threading.Tasks;
     using System.Web;
     using System.Web.Hosting;
     using ImageProcessor.Helpers.Extensions;
@@ -26,67 +28,109 @@ namespace ImageProcessor.Web.Caching
     {
         #region Fields
         /// <summary>
-        ///     The maximum number of days to cache files on the system for.
+        /// The maximum number of days to cache files on the system for.
         /// </summary>
         internal static readonly int MaxFileCachedDuration = ImageProcessorConfig.Instance.MaxCacheDays;
 
+        /// <summary>
+        /// The valid sub directory chars. This used in combination with the file limit per folder
+        /// allows the storage of 360,000 image files in the cache.
+        /// </summary>
+        private const string ValidSubDirectoryChars = "abcdefghijklmnopqrstuvwxyz0123456789";
+
         /// <summary>
         /// The maximum number of files allowed in the directory.
         /// </summary>
         /// <remarks>
-        /// NTFS directories can handle up to 8000 files in the directory before slowing down. 
+        /// NTFS directories can handle up to 10,000 files in the directory before slowing down. 
-        /// This buffer will help us to ensure that we rarely hit anywhere near that limit.
+        /// This will help us to ensure that don't go over that limit.
+        /// <see cref="http://stackoverflow.com/questions/197162/ntfs-performance-and-large-volumes-of-files-and-directories"/>
+        /// <see cref="http://stackoverflow.com/questions/115882/how-do-you-deal-with-lots-of-small-files"/>
+        /// <see cref="http://stackoverflow.com/questions/1638219/millions-of-small-graphics-files-and-how-to-overcome-slow-file-system-access-on"/>
         /// </remarks>
-        private const int MaxFilesCount = 7500;
+        private const int MaxFilesCount = 10000;
+
+        /// <summary>
+        /// The regular expression to search strings for file extensions.
+        /// </summary>
+        private static readonly Regex FormatRegex = new Regex(
+            @"(jpeg|png|bmp|gif)", RegexOptions.RightToLeft | RegexOptions.Compiled);
 
         /// <summary>
-        /// The regular expression to search strings for extension changes.
+        /// The regular expression to search strings for valid subfolder names.
+        /// We're specifically not using a shorter regex as we need to be able to iterate through
+        /// each match group.
         /// </summary>
-        private static readonly Regex FormatRegex = new Regex(@"(jpeg|png|bmp|gif)", RegexOptions.RightToLeft | RegexOptions.Compiled);
+        private static readonly Regex SubFolderRegex = new Regex(@"(\/(a|b|c|d|e|f|g|h|i|j|k|l|m|n|o|p|q|r|s|t|u|v|w|x|y|z|0|1|2|3|4|5|6|7|8|9)\/)", RegexOptions.Compiled);
 
         /// <summary>
-        /// The default paths for Cached folders on the server.
+        /// The absolute path to virtual cache path on the server.
         /// </summary>
-        private static readonly string CachePath = ImageProcessorConfig.Instance.VirtualCachePath;
+        private static readonly string AbsoluteCachePath =
+            HostingEnvironment.MapPath(ImageProcessorConfig.Instance.VirtualCachePath);
+
         #endregion
 
         #region Methods
+        /// <summary>
+        /// The create cache paths.
+        /// </summary>
+        /// <returns>
+        /// The true if the cache directories are created successfully; otherwise, false.
+        /// </returns>
+        internal static bool CreateCacheDirectories()
+        {
+            try
+            {
+                Parallel.ForEach(
+                    ValidSubDirectoryChars.ToCharArray(),
+                    (extension, loop) =>
+                    {
+                        string path = Path.Combine(AbsoluteCachePath, extension.ToString(CultureInfo.InvariantCulture));
+                        DirectoryInfo directoryInfo = new DirectoryInfo(path);
+
+                        if (!directoryInfo.Exists)
+                        {
+                            directoryInfo.Create();
+                        }
+                    });
+            }
+            catch
+            {
+                return false;
+            }
+
+            return true;
+        }
+
         /// <summary>
         /// Gets the full transformed cached path for the image.
+        /// The file names are stored as MD5 encrypted versions of the full request path.
+        /// This should make them unique enough to 
         /// </summary>
         /// <param name="imagePath">The original image path.</param>
         /// <param name="imageName">The original image name.</param>
         /// <returns>The full cached path for the image.</returns>
         internal static string GetCachePath(string imagePath, string imageName)
         {
-            string virtualCachePath = CachePath;
-            string absoluteCachePath = HostingEnvironment.MapPath(virtualCachePath);
             string cachedPath = string.Empty;
 
-            if (absoluteCachePath != null)
+            if (AbsoluteCachePath != null)
             {
+                // Use an md5 hash of the full path including the querystring to create the image name. 
+                // That name can also be used as a key for the cached image and we should be able to use 
+                // The first character of that hash as a subfolder.
                 string parsedExtension = ParseExtension(imagePath);
                 string fallbackExtension = imageName.Substring(imageName.LastIndexOf(".", StringComparison.Ordinal) + 1);
-                string subpath = !string.IsNullOrWhiteSpace(parsedExtension) ? parsedExtension : fallbackExtension;
+                string encryptedName = imagePath.ToMD5Fingerprint();
+                string subpath = encryptedName.Substring(0, 1);
 
                 string cachedFileName = string.Format(
                     "{0}.{1}",
-                    imagePath.ToMD5Fingerprint(),
+                    encryptedName,
                     !string.IsNullOrWhiteSpace(parsedExtension) ? parsedExtension : fallbackExtension);
-                cachedPath = Path.Combine(absoluteCachePath, subpath, cachedFileName);
-
-                string cachedDirectory = Path.GetDirectoryName(cachedPath);
 
-                if (cachedDirectory != null)
+                cachedPath = Path.Combine(AbsoluteCachePath, subpath, cachedFileName);
-                {
-                    DirectoryInfo directoryInfo = new DirectoryInfo(cachedDirectory);
-
-                    if (!directoryInfo.Exists)
-                    {
-                        // Create the directory.
-                        directoryInfo.Create();
-                    }
-                }
             }
 
             return cachedPath;
@@ -166,10 +210,12 @@ namespace ImageProcessor.Web.Caching
                 return true;
             }
 
-            FileInfo imageFileInfo = new FileInfo(imagePath);
+            // Test now for locally requested files.
-            if (imageFileInfo.Exists)
+            if (PersistantDictionary.Instance.TryGetValue(key, out cachedImage))
             {
-                if (PersistantDictionary.Instance.TryGetValue(key, out cachedImage))
+                FileInfo imageFileInfo = new FileInfo(imagePath);
+
+                if (imageFileInfo.Exists)
                 {
                     // Check to see if the last write time is different of whether the
                     // cached image is set to expire or if the max age is different.
@@ -183,11 +229,11 @@ namespace ImageProcessor.Web.Caching
                         }
                     }
                 }
-                else
+            }
-                {
+            else
-                    // Nothing in the cache so we should return true.
+            {
-                    return true;
+                // Nothing in the cache so we should return true.
-                }
+                return true;
             }
 
             return false;
@@ -239,7 +285,7 @@ namespace ImageProcessor.Web.Caching
             // Group each cache folder and clear any expired items or any that exeed
             // the maximum allowable count.
             var groups = PersistantDictionary.Instance.ToList()
-                .GroupBy(x => FormatRegex.Match(x.Value.Path).Value)
+                .GroupBy(x => SubFolderRegex.Match(x.Value.Path).Value)
                 .Where(g => g.Count() > MaxFilesCount);
 
             foreach (var group in groups)
@@ -269,11 +315,10 @@ namespace ImageProcessor.Web.Caching
                             groupCount -= 1;
                         }
                     }
-                    catch (Exception ex)
+                    catch (Exception)
                     {
                         // Do Nothing, skip to the next.
                         // TODO: Should we handle this?   
-                        throw ex;
                         continue;
                     }
                 }

src/ImageProcessor.Web/Caching/PersistantDictionary.cs

@@ -141,9 +141,8 @@ namespace ImageProcessor.Web.Caching
 
                 return SQLContext.AddImage(key, cachedImage);
             }
-            catch (Exception ex)
+            catch
             {
-                throw ex;
                 return false;
             }
         }

src/ImageProcessor.Web/Caching/SQLContext.cs

@@ -129,9 +129,8 @@ namespace ImageProcessor.Web.Caching
 
                 return true;
             }
-            catch (Exception ex)
+            catch
             {
-                throw ex;
                 return false;
             }
         }
@@ -168,9 +167,8 @@ namespace ImageProcessor.Web.Caching
 
                 return true;
             }
-            catch (Exception ex)
+            catch
             {
-                throw ex;
                 return false;
             }
         }
@@ -213,9 +211,8 @@ namespace ImageProcessor.Web.Caching
 
                 return dictionary;
             }
-            catch (Exception ex)
+            catch
             {
-                throw ex;
                 return new Dictionary<string, CachedImage>();
             }
         }

src/ImageProcessor.Web/Config/ImageCacheSection.cs

@@ -13,7 +13,7 @@ namespace ImageProcessor.Web.Config
     #endregion
 
     /// <summary>
-    /// Represents an imagecache section within a configuration file.
+    /// Represents an image cache section within a configuration file.
     /// </summary>
     public class ImageCacheSection : ConfigurationSection
     {

src/ImageProcessor.Web/Config/ImageProcessingSection.cs

@@ -13,8 +13,8 @@ namespace ImageProcessor.Web.Config
     #endregion
 
     /// <summary>
-    /// Represents an imageprocessing section within a configuration file.
+    /// Represents an image processing section within a configuration file.
-    /// Nested syntax adapted from http://tneustaedter.blogspot.co.uk/2011/09/how-to-create-one-or-more-nested.html
+    /// Nested syntax adapted from <see cref="http://tneustaedter.blogspot.co.uk/2011/09/how-to-create-one-or-more-nested.html"/>
     /// </summary>
     public class ImageProcessingSection : ConfigurationSection
     {
@@ -132,12 +132,12 @@ namespace ImageProcessor.Web.Config
 
                 set
                 {
-                    if (BaseGet(index) != null)
+                    if (this.BaseGet(index) != null)
                     {
-                        BaseRemoveAt(index);
+                        this.BaseRemoveAt(index);
                     }
 
-                    BaseAdd(index, value);
+                   this.BaseAdd(index, value);
                 }
             }
 
@@ -253,20 +253,20 @@ namespace ImageProcessor.Web.Config
 
                 set
                 {
-                    if (BaseGet(index) != null)
+                    if (this.BaseGet(index) != null)
                     {
-                        BaseRemoveAt(index);
+                       this.BaseRemoveAt(index);
                     }
 
-                    BaseAdd(index, value);
+                   this.BaseAdd(index, value);
                 }
             }
 
             /// <summary>
             /// Returns the setting element with the specified key.
             /// </summary>
-            /// <param name="key">knkn knk</param>
+            /// <param name="key">the key representing the element</param>
-            /// <returns>jn jnj </returns>
+            /// <returns>the setting element</returns>
             public new SettingElement this[string key]
             {
                 get { return (SettingElement)BaseGet(key); }

src/ImageProcessor.Web/Config/ImageProcessorConfig.cs

@@ -10,22 +10,20 @@ namespace ImageProcessor.Web.Config
     #region Using
     using System;
     using System.Collections.Generic;
-    using System.Diagnostics.CodeAnalysis;
     using System.Linq;
     using ImageProcessor.Processors;
     #endregion
 
     /// <summary>
     /// Encapsulates methods to allow the retrieval of ImageProcessor settings.
-    /// http://csharpindepth.com/Articles/General/Singleton.aspx
+    /// <see cref="http://csharpindepth.com/Articles/General/Singleton.aspx"/>
     /// </summary>
-    [SuppressMessage("StyleCop.CSharp.DocumentationRules", "SA1650:ElementDocumentationMustBeSpelledCorrectly", Justification = "Reviewed. Suppression is OK here.")]
     public class ImageProcessorConfig
     {
         #region Fields
         /// <summary>
         /// A new instance Initializes a new instance of the <see cref="T:ImageProcessor.Web.Config.ImageProcessorConfig"/> class.
-        /// intitialized lazily.
+        /// with lazy initialization.
         /// </summary>
         private static readonly Lazy<ImageProcessorConfig> Lazy =
                         new Lazy<ImageProcessorConfig>(() => new ImageProcessorConfig());
@@ -107,9 +105,8 @@ namespace ImageProcessor.Web.Config
 
         #region Security
         /// <summary>
-        /// Gets a list of whitelisted urls that images can be downloaded from.
+        /// Gets a list of whitelisted url[s] that images can be downloaded from.
         /// </summary>
-        [SuppressMessage("StyleCop.CSharp.DocumentationRules", "SA1650:ElementDocumentationMustBeSpelledCorrectly", Justification = "Reviewed. Suppression is OK here.")]
         public Uri[] RemoteFileWhiteList
         {
             get

src/ImageProcessor.Web/Helpers/RemoteFile.cs

@@ -24,7 +24,7 @@ namespace ImageProcessor.Web.Helpers
     /// </summary>
     /// <remarks>
     /// <para>
-    /// The purpose of this class is so there's one core way of downloading remote files with urls that are from
+    /// The purpose of this class is so there's one core way of downloading remote files with url[s] that are from
     /// outside users. There's various areas in application where an attacker could supply an external url to the server
     /// and tie up resources.
     /// </para>
@@ -43,7 +43,7 @@ namespace ImageProcessor.Web.Helpers
     {
         #region Fields
         /// <summary>
-        /// The white-list of urls from which to download remote files.
+        /// The white-list of url[s] from which to download remote files.
         /// </summary>
         private static readonly Uri[] RemoteFileWhiteList = ImageProcessorConfig.Instance.RemoteFileWhiteList;
 

src/ImageProcessor.Web/HttpModules/ImageProcessingModule.cs

@@ -49,6 +49,7 @@ namespace ImageProcessor.Web.HttpModules
         /// <param name="context">An <see cref="T:System.Web.HttpApplication"/> that provides access to the methods, properties, and events common to all application objects within an ASP.NET application </param>
         public void Init(HttpApplication context)
         {
+            DiskCache.CreateCacheDirectories();
             context.BeginRequest += this.ContextBeginRequest;
             context.PreSendRequestHeaders += this.ContextPreSendRequestHeaders;
         }

src/ImageProcessor/Helpers/Extensions/EnumExtensions.cs

@@ -1,6 +1,7 @@
 // -----------------------------------------------------------------------
 // <copyright file="EnumExtensions.cs" company="James South">
-// TODO: Update copyright text.
+//     Copyright (c) James South.
+//     Dual licensed under the MIT or GPL Version 2 licenses.
 // </copyright>
 // -----------------------------------------------------------------------
 

src/ImageProcessor/Helpers/Extensions/StringExtensions.cs

@@ -1,6 +1,7 @@
 // -----------------------------------------------------------------------
 // <copyright file="StringExtensions.cs" company="James South">
-// TODO: Update copyright text.
+//     Copyright (c) James South.
+//     Dual licensed under the MIT or GPL Version 2 licenses.
 // </copyright>
 // -----------------------------------------------------------------------
 
@@ -29,8 +30,6 @@ namespace ImageProcessor.Helpers.Extensions
         /// <returns>An MD5 fingerprint of the String.</returns>
         public static string ToMD5Fingerprint(this string expression)
         {
-            Contract.Requires(!string.IsNullOrWhiteSpace(expression));
-
             byte[] bytes = Encoding.Unicode.GetBytes(expression.ToCharArray());
 
             using (MD5CryptoServiceProvider md5 = new MD5CryptoServiceProvider())
@@ -41,7 +40,49 @@ namespace ImageProcessor.Helpers.Extensions
                 return hash.Aggregate(
                     new StringBuilder(32),
                     (sb, b) => sb.Append(b.ToString("X2", CultureInfo.InvariantCulture)))
-                    .ToString();
+                    .ToString().ToLowerInvariant();
+            }
+        }
+
+        /// <summary>
+        /// Creates an SHA1 fingerprint of the String.
+        /// </summary>
+        /// <param name="expression">The <see cref="T:System.String">String</see> instance that this method extends.</param>
+        /// <returns>An SHA1 fingerprint of the String.</returns>
+        public static string ToSHA1Fingerprint(this string expression)
+        {
+            byte[] bytes = Encoding.ASCII.GetBytes(expression.ToCharArray());
+
+            using (SHA1CryptoServiceProvider sha1 = new SHA1CryptoServiceProvider())
+            {
+                byte[] hash = sha1.ComputeHash(bytes);
+
+                // Concatenate the hash bytes into one long String.
+                return hash.Aggregate(
+                    new StringBuilder(40),
+                    (sb, b) => sb.Append(b.ToString("X2", CultureInfo.InvariantCulture)))
+                    .ToString().ToLowerInvariant();
+            }
+        }
+
+        /// <summary>
+        /// Creates an SHA256 fingerprint of the String.
+        /// </summary>
+        /// <param name="expression">The <see cref="T:System.String">String</see> instance that this method extends.</param>
+        /// <returns>An SHA256 fingerprint of the String.</returns>
+        public static string ToSHA256Fingerprint(this string expression)
+        {
+            byte[] bytes = Encoding.ASCII.GetBytes(expression.ToCharArray());
+
+            using (SHA256CryptoServiceProvider sha256 = new SHA256CryptoServiceProvider())
+            {
+                byte[] hash = sha256.ComputeHash(bytes);
+
+                // Concatenate the hash bytes into one long String.
+                return hash.Aggregate(
+                    new StringBuilder(64),
+                    (sb, b) => sb.Append(b.ToString("X2", CultureInfo.InvariantCulture)))
+                    .ToString().ToLowerInvariant();
             }
         }
         #endregion
@@ -82,8 +123,6 @@ namespace ImageProcessor.Helpers.Extensions
         /// <returns>True if the given string is a valid virtual path name</returns>
         public static bool IsValidVirtualPathName(this string expression)
         {
-            Contract.Requires(!string.IsNullOrWhiteSpace(expression));
-
             // Check the start of the string.
             if (expression.StartsWith("~/"))
             {
@@ -97,14 +136,15 @@ namespace ImageProcessor.Helpers.Extensions
 
         /// <summary>
         /// Checks the string to see whether the value is a valid path name.
-        /// http://stackoverflow.com/questions/62771/how-check-if-given-string-is-legal-allowed-file-name-under-windows/
         /// </summary>
+        /// <remarks>
+        /// For an explanation 
+        /// <see cref="http://stackoverflow.com/questions/62771/how-check-if-given-string-is-legal-allowed-file-name-under-windows"/>
+        /// </remarks>
         /// <param name="expression">The <see cref="T:System.String">String</see> instance that this method extends.</param>
         /// <returns>True if the given string is a valid path name</returns>
         public static bool IsValidPathName(this string expression)
         {
-            Contract.Requires(!string.IsNullOrWhiteSpace(expression));
-
             // Create a regex of invalid characters and test it.
             string invalidPathNameChars = new string(Path.GetInvalidFileNameChars());
             Regex regFixPathName = new Regex("[" + Regex.Escape(invalidPathNameChars) + "]");

src/ImageProcessor/Imaging/OctreeQuantizer.cs

@@ -126,12 +126,12 @@ namespace ImageProcessor.Imaging
             /// <summary>
             /// Mask used when getting the appropriate pixels for a given node
             /// </summary>
-            private static int[] mask = new int[8] { 0x80, 0x40, 0x20, 0x10, 0x08, 0x04, 0x02, 0x01 };
+            private static readonly int[] mask = new int[8] { 0x80, 0x40, 0x20, 0x10, 0x08, 0x04, 0x02, 0x01 };
 
             /// <summary>
             /// The root of the octree
             /// </summary>
-            private OctreeNode root;
+            private readonly OctreeNode root;
 
             /// <summary>
             /// Number of leaves in the tree

src/ImageProcessor/Imaging/Quantizer.cs

@@ -15,7 +15,7 @@ namespace ImageProcessor.Imaging
     #endregion
 
     /// <summary>
-    /// Encapsulates methods to calulate the colour pallete of an image.
+    /// Encapsulates methods to calculate the color palette of an image.
     /// </summary>
     internal abstract class Quantizer
     {
@@ -250,11 +250,11 @@ namespace ImageProcessor.Imaging
         /// Retrieve the palette for the quantized image
         /// </summary>
         /// <param name="original">Any old palette, this is overwritten</param>
-        /// <returns>The new colour palette</returns>
+        /// <returns>The new color palette</returns>
         protected abstract ColorPalette GetPalette(ColorPalette original);
 
         /// <summary>
-        /// Structure that defines a 32 bpp colour
+        /// Structure that defines a 32 bit color
         /// </summary>
         /// <remarks>
         /// This structure is used to read data from a 32 bits per pixel image
@@ -265,31 +265,31 @@ namespace ImageProcessor.Imaging
         public struct Color32
         {
             /// <summary>
-            /// Holds the blue component of the colour
+            /// Holds the blue component of the color
             /// </summary>
             [FieldOffset(0)]
             public byte Blue;
 
             /// <summary>
-            /// Holds the green component of the colour
+            /// Holds the green component of the color
             /// </summary>
             [FieldOffset(1)]
             public byte Green;
 
             /// <summary>
-            /// Holds the red component of the colour
+            /// Holds the red component of the color
             /// </summary>
             [FieldOffset(2)]
             public byte Red;
 
             /// <summary>
-            /// Holds the alpha component of the colour
+            /// Holds the alpha component of the color
             /// </summary>
             [FieldOffset(3)]
             public byte Alpha;
 
             /// <summary>
-            /// Permits the color32 to be treated as an int32
+            /// Permits the color32 to be treated as a 32 bit integer.
             /// </summary>
             [FieldOffset(0)]
             public int ARGB;
@@ -304,7 +304,7 @@ namespace ImageProcessor.Imaging
             }
 
             /// <summary>
-            /// Gets the colour for this Color32 object
+            /// Gets the color for this Color32 object
             /// </summary>
             public Color Color
             {

src/Test/Test/Controllers/HomeController.cs

@@ -10,6 +10,7 @@ namespace Test.Controllers
     using System.Threading.Tasks;
     using System.Web.Hosting;
 
+    using ImageProcessor.Helpers.Extensions;
     using ImageProcessor.Web.Caching;
 
     public class HomeController : Controller
@@ -58,6 +59,29 @@ namespace Test.Controllers
         {
             DateTime start = DateTime.Now;
 
+            List<double> collisions = new List<double>();
+const int Iterations = 1;
+const int Maxitems = 360000;
+
+for (int i = 0; i < Iterations; i++)
+{
+    List<string> paths = new List<string>();
+
+    for (int j = 0; j < Maxitems; j++)
+    {
+        string path = Path.GetRandomFileName().ToSHA256Fingerprint().Substring(0, 8);
+
+        paths.Add(path);
+    }
+
+    int count = paths.Distinct().Count();
+
+    double collisionRate = ((Maxitems - count) * 100D) / Maxitems;
+    collisions.Add(collisionRate);
+}
+
+double averageCollisionRate = collisions.Average();
+
             //PersistantDictionary persistantDictionary = PersistantDictionary.Instance;
 
             //for (int i = 0; i < 1000; i++)
@@ -77,7 +101,8 @@ namespace Test.Controllers
 
             TimeSpan timeSpan = DateTime.Now - start;
 
-            //ViewBag.Count = persistantDictionary.Count();
+            //ViewBag.Count = count;
+            ViewBag.Collision = averageCollisionRate;
 
             return this.View(timeSpan);
         }

src/Test/Test/Views/Home/Index.cshtml

@@ -90,7 +90,7 @@
             </div>
             <div class="clmn2">
                 <h2>Remote</h2>
-               @* <img src="/remote.axd?http://images.mymovies.net/images/film/cin/500x377/fid11707.jpg?width=300" />*@
+                <img src="/remote.axd?http://images.mymovies.net/images/film/cin/500x377/fid11707.jpg?width=300" />
             </div>
         </div>
     </section>

src/Test/Test/Views/Home/Speed.cshtml

@@ -14,7 +14,8 @@
 <body>
     <div>
         Speed In Milliseconds: @s<br/>
-        Dictionary Count: @ViewBag.Count
+    @*    Distinct Count: @ViewBag.Count*@
+        Collision Rate: @ViewBag.Collision%
     </div>
 </body>
 </html>