diff --git a/src/ImageProcessor.Web/Caching/DiskCache.cs b/src/ImageProcessor.Web/Caching/DiskCache.cs index f7c0e6dac..84fc204e3 100644 --- a/src/ImageProcessor.Web/Caching/DiskCache.cs +++ b/src/ImageProcessor.Web/Caching/DiskCache.cs @@ -10,9 +10,11 @@ namespace ImageProcessor.Web.Caching #region Using using System; using System.Collections.Generic; + using System.Globalization; using System.IO; using System.Linq; using System.Text.RegularExpressions; + using System.Threading.Tasks; using System.Web; using System.Web.Hosting; using ImageProcessor.Helpers.Extensions; @@ -26,67 +28,109 @@ namespace ImageProcessor.Web.Caching { #region Fields /// - /// The maximum number of days to cache files on the system for. + /// The maximum number of days to cache files on the system for. /// internal static readonly int MaxFileCachedDuration = ImageProcessorConfig.Instance.MaxCacheDays; + /// + /// The valid sub directory chars. This used in combination with the file limit per folder + /// allows the storage of 360,000 image files in the cache. + /// + private const string ValidSubDirectoryChars = "abcdefghijklmnopqrstuvwxyz0123456789"; + /// /// The maximum number of files allowed in the directory. /// /// - /// NTFS directories can handle up to 8000 files in the directory before slowing down. - /// This buffer will help us to ensure that we rarely hit anywhere near that limit. + /// NTFS directories can handle up to 10,000 files in the directory before slowing down. + /// This will help us to ensure that don't go over that limit. + /// + /// + /// /// - private const int MaxFilesCount = 7500; + private const int MaxFilesCount = 10000; + + /// + /// The regular expression to search strings for file extensions. + /// + private static readonly Regex FormatRegex = new Regex( + @"(jpeg|png|bmp|gif)", RegexOptions.RightToLeft | RegexOptions.Compiled); /// - /// The regular expression to search strings for extension changes. + /// The regular expression to search strings for valid subfolder names. + /// We're specifically not using a shorter regex as we need to be able to iterate through + /// each match group. /// - private static readonly Regex FormatRegex = new Regex(@"(jpeg|png|bmp|gif)", RegexOptions.RightToLeft | RegexOptions.Compiled); + private static readonly Regex SubFolderRegex = new Regex(@"(\/(a|b|c|d|e|f|g|h|i|j|k|l|m|n|o|p|q|r|s|t|u|v|w|x|y|z|0|1|2|3|4|5|6|7|8|9)\/)", RegexOptions.Compiled); /// - /// The default paths for Cached folders on the server. + /// The absolute path to virtual cache path on the server. /// - private static readonly string CachePath = ImageProcessorConfig.Instance.VirtualCachePath; + private static readonly string AbsoluteCachePath = + HostingEnvironment.MapPath(ImageProcessorConfig.Instance.VirtualCachePath); + #endregion #region Methods + /// + /// The create cache paths. + /// + /// + /// The true if the cache directories are created successfully; otherwise, false. + /// + internal static bool CreateCacheDirectories() + { + try + { + Parallel.ForEach( + ValidSubDirectoryChars.ToCharArray(), + (extension, loop) => + { + string path = Path.Combine(AbsoluteCachePath, extension.ToString(CultureInfo.InvariantCulture)); + DirectoryInfo directoryInfo = new DirectoryInfo(path); + + if (!directoryInfo.Exists) + { + directoryInfo.Create(); + } + }); + } + catch + { + return false; + } + + return true; + } + /// /// Gets the full transformed cached path for the image. + /// The file names are stored as MD5 encrypted versions of the full request path. + /// This should make them unique enough to /// /// The original image path. /// The original image name. /// The full cached path for the image. internal static string GetCachePath(string imagePath, string imageName) { - string virtualCachePath = CachePath; - string absoluteCachePath = HostingEnvironment.MapPath(virtualCachePath); string cachedPath = string.Empty; - if (absoluteCachePath != null) + if (AbsoluteCachePath != null) { + // Use an md5 hash of the full path including the querystring to create the image name. + // That name can also be used as a key for the cached image and we should be able to use + // The first character of that hash as a subfolder. string parsedExtension = ParseExtension(imagePath); string fallbackExtension = imageName.Substring(imageName.LastIndexOf(".", StringComparison.Ordinal) + 1); - string subpath = !string.IsNullOrWhiteSpace(parsedExtension) ? parsedExtension : fallbackExtension; + string encryptedName = imagePath.ToMD5Fingerprint(); + string subpath = encryptedName.Substring(0, 1); string cachedFileName = string.Format( "{0}.{1}", - imagePath.ToMD5Fingerprint(), + encryptedName, !string.IsNullOrWhiteSpace(parsedExtension) ? parsedExtension : fallbackExtension); - cachedPath = Path.Combine(absoluteCachePath, subpath, cachedFileName); - - string cachedDirectory = Path.GetDirectoryName(cachedPath); - if (cachedDirectory != null) - { - DirectoryInfo directoryInfo = new DirectoryInfo(cachedDirectory); - - if (!directoryInfo.Exists) - { - // Create the directory. - directoryInfo.Create(); - } - } + cachedPath = Path.Combine(AbsoluteCachePath, subpath, cachedFileName); } return cachedPath; @@ -166,10 +210,12 @@ namespace ImageProcessor.Web.Caching return true; } - FileInfo imageFileInfo = new FileInfo(imagePath); - if (imageFileInfo.Exists) + // Test now for locally requested files. + if (PersistantDictionary.Instance.TryGetValue(key, out cachedImage)) { - if (PersistantDictionary.Instance.TryGetValue(key, out cachedImage)) + FileInfo imageFileInfo = new FileInfo(imagePath); + + if (imageFileInfo.Exists) { // Check to see if the last write time is different of whether the // cached image is set to expire or if the max age is different. @@ -183,11 +229,11 @@ namespace ImageProcessor.Web.Caching } } } - else - { - // Nothing in the cache so we should return true. - return true; - } + } + else + { + // Nothing in the cache so we should return true. + return true; } return false; @@ -239,7 +285,7 @@ namespace ImageProcessor.Web.Caching // Group each cache folder and clear any expired items or any that exeed // the maximum allowable count. var groups = PersistantDictionary.Instance.ToList() - .GroupBy(x => FormatRegex.Match(x.Value.Path).Value) + .GroupBy(x => SubFolderRegex.Match(x.Value.Path).Value) .Where(g => g.Count() > MaxFilesCount); foreach (var group in groups) @@ -269,11 +315,10 @@ namespace ImageProcessor.Web.Caching groupCount -= 1; } } - catch (Exception ex) + catch (Exception) { // Do Nothing, skip to the next. // TODO: Should we handle this? - throw ex; continue; } } diff --git a/src/ImageProcessor.Web/Caching/PersistantDictionary.cs b/src/ImageProcessor.Web/Caching/PersistantDictionary.cs index 9c76f8ca1..81f79c669 100644 --- a/src/ImageProcessor.Web/Caching/PersistantDictionary.cs +++ b/src/ImageProcessor.Web/Caching/PersistantDictionary.cs @@ -141,9 +141,8 @@ namespace ImageProcessor.Web.Caching return SQLContext.AddImage(key, cachedImage); } - catch (Exception ex) + catch { - throw ex; return false; } } diff --git a/src/ImageProcessor.Web/Caching/SQLContext.cs b/src/ImageProcessor.Web/Caching/SQLContext.cs index d6c9a3b16..aef2bb4b7 100644 --- a/src/ImageProcessor.Web/Caching/SQLContext.cs +++ b/src/ImageProcessor.Web/Caching/SQLContext.cs @@ -129,9 +129,8 @@ namespace ImageProcessor.Web.Caching return true; } - catch (Exception ex) + catch { - throw ex; return false; } } @@ -168,9 +167,8 @@ namespace ImageProcessor.Web.Caching return true; } - catch (Exception ex) + catch { - throw ex; return false; } } @@ -213,9 +211,8 @@ namespace ImageProcessor.Web.Caching return dictionary; } - catch (Exception ex) + catch { - throw ex; return new Dictionary(); } } diff --git a/src/ImageProcessor.Web/Config/ImageCacheSection.cs b/src/ImageProcessor.Web/Config/ImageCacheSection.cs index ecb42ed5d..012178f06 100644 --- a/src/ImageProcessor.Web/Config/ImageCacheSection.cs +++ b/src/ImageProcessor.Web/Config/ImageCacheSection.cs @@ -13,7 +13,7 @@ namespace ImageProcessor.Web.Config #endregion /// - /// Represents an imagecache section within a configuration file. + /// Represents an image cache section within a configuration file. /// public class ImageCacheSection : ConfigurationSection { diff --git a/src/ImageProcessor.Web/Config/ImageProcessingSection.cs b/src/ImageProcessor.Web/Config/ImageProcessingSection.cs index 3d55cc9ba..524cb7b49 100644 --- a/src/ImageProcessor.Web/Config/ImageProcessingSection.cs +++ b/src/ImageProcessor.Web/Config/ImageProcessingSection.cs @@ -13,8 +13,8 @@ namespace ImageProcessor.Web.Config #endregion /// - /// Represents an imageprocessing section within a configuration file. - /// Nested syntax adapted from http://tneustaedter.blogspot.co.uk/2011/09/how-to-create-one-or-more-nested.html + /// Represents an image processing section within a configuration file. + /// Nested syntax adapted from /// public class ImageProcessingSection : ConfigurationSection { @@ -132,12 +132,12 @@ namespace ImageProcessor.Web.Config set { - if (BaseGet(index) != null) + if (this.BaseGet(index) != null) { - BaseRemoveAt(index); + this.BaseRemoveAt(index); } - BaseAdd(index, value); + this.BaseAdd(index, value); } } @@ -253,20 +253,20 @@ namespace ImageProcessor.Web.Config set { - if (BaseGet(index) != null) + if (this.BaseGet(index) != null) { - BaseRemoveAt(index); + this.BaseRemoveAt(index); } - BaseAdd(index, value); + this.BaseAdd(index, value); } } /// /// Returns the setting element with the specified key. /// - /// knkn knk - /// jn jnj + /// the key representing the element + /// the setting element public new SettingElement this[string key] { get { return (SettingElement)BaseGet(key); } diff --git a/src/ImageProcessor.Web/Config/ImageProcessorConfig.cs b/src/ImageProcessor.Web/Config/ImageProcessorConfig.cs index 7332b2d1b..94320e7a3 100644 --- a/src/ImageProcessor.Web/Config/ImageProcessorConfig.cs +++ b/src/ImageProcessor.Web/Config/ImageProcessorConfig.cs @@ -10,22 +10,20 @@ namespace ImageProcessor.Web.Config #region Using using System; using System.Collections.Generic; - using System.Diagnostics.CodeAnalysis; using System.Linq; using ImageProcessor.Processors; #endregion /// /// Encapsulates methods to allow the retrieval of ImageProcessor settings. - /// http://csharpindepth.com/Articles/General/Singleton.aspx + /// /// - [SuppressMessage("StyleCop.CSharp.DocumentationRules", "SA1650:ElementDocumentationMustBeSpelledCorrectly", Justification = "Reviewed. Suppression is OK here.")] public class ImageProcessorConfig { #region Fields /// /// A new instance Initializes a new instance of the class. - /// intitialized lazily. + /// with lazy initialization. /// private static readonly Lazy Lazy = new Lazy(() => new ImageProcessorConfig()); @@ -107,9 +105,8 @@ namespace ImageProcessor.Web.Config #region Security /// - /// Gets a list of whitelisted urls that images can be downloaded from. + /// Gets a list of whitelisted url[s] that images can be downloaded from. /// - [SuppressMessage("StyleCop.CSharp.DocumentationRules", "SA1650:ElementDocumentationMustBeSpelledCorrectly", Justification = "Reviewed. Suppression is OK here.")] public Uri[] RemoteFileWhiteList { get diff --git a/src/ImageProcessor.Web/Helpers/RemoteFile.cs b/src/ImageProcessor.Web/Helpers/RemoteFile.cs index 4e96faa58..aa17b0f4e 100644 --- a/src/ImageProcessor.Web/Helpers/RemoteFile.cs +++ b/src/ImageProcessor.Web/Helpers/RemoteFile.cs @@ -24,7 +24,7 @@ namespace ImageProcessor.Web.Helpers /// /// /// - /// The purpose of this class is so there's one core way of downloading remote files with urls that are from + /// The purpose of this class is so there's one core way of downloading remote files with url[s] that are from /// outside users. There's various areas in application where an attacker could supply an external url to the server /// and tie up resources. /// @@ -43,7 +43,7 @@ namespace ImageProcessor.Web.Helpers { #region Fields /// - /// The white-list of urls from which to download remote files. + /// The white-list of url[s] from which to download remote files. /// private static readonly Uri[] RemoteFileWhiteList = ImageProcessorConfig.Instance.RemoteFileWhiteList; diff --git a/src/ImageProcessor.Web/HttpModules/ImageProcessingModule.cs b/src/ImageProcessor.Web/HttpModules/ImageProcessingModule.cs index 1ef7908c0..68895043f 100644 --- a/src/ImageProcessor.Web/HttpModules/ImageProcessingModule.cs +++ b/src/ImageProcessor.Web/HttpModules/ImageProcessingModule.cs @@ -49,6 +49,7 @@ namespace ImageProcessor.Web.HttpModules /// An that provides access to the methods, properties, and events common to all application objects within an ASP.NET application public void Init(HttpApplication context) { + DiskCache.CreateCacheDirectories(); context.BeginRequest += this.ContextBeginRequest; context.PreSendRequestHeaders += this.ContextPreSendRequestHeaders; } diff --git a/src/ImageProcessor/Helpers/Extensions/EnumExtensions.cs b/src/ImageProcessor/Helpers/Extensions/EnumExtensions.cs index 9d40047f2..7aa226bf4 100644 --- a/src/ImageProcessor/Helpers/Extensions/EnumExtensions.cs +++ b/src/ImageProcessor/Helpers/Extensions/EnumExtensions.cs @@ -1,6 +1,7 @@ // ----------------------------------------------------------------------- // -// TODO: Update copyright text. +// Copyright (c) James South. +// Dual licensed under the MIT or GPL Version 2 licenses. // // ----------------------------------------------------------------------- diff --git a/src/ImageProcessor/Helpers/Extensions/StringExtensions.cs b/src/ImageProcessor/Helpers/Extensions/StringExtensions.cs index 32976d937..6a15c1677 100644 --- a/src/ImageProcessor/Helpers/Extensions/StringExtensions.cs +++ b/src/ImageProcessor/Helpers/Extensions/StringExtensions.cs @@ -1,6 +1,7 @@ // ----------------------------------------------------------------------- // -// TODO: Update copyright text. +// Copyright (c) James South. +// Dual licensed under the MIT or GPL Version 2 licenses. // // ----------------------------------------------------------------------- @@ -29,8 +30,6 @@ namespace ImageProcessor.Helpers.Extensions /// An MD5 fingerprint of the String. public static string ToMD5Fingerprint(this string expression) { - Contract.Requires(!string.IsNullOrWhiteSpace(expression)); - byte[] bytes = Encoding.Unicode.GetBytes(expression.ToCharArray()); using (MD5CryptoServiceProvider md5 = new MD5CryptoServiceProvider()) @@ -41,7 +40,49 @@ namespace ImageProcessor.Helpers.Extensions return hash.Aggregate( new StringBuilder(32), (sb, b) => sb.Append(b.ToString("X2", CultureInfo.InvariantCulture))) - .ToString(); + .ToString().ToLowerInvariant(); + } + } + + /// + /// Creates an SHA1 fingerprint of the String. + /// + /// The String instance that this method extends. + /// An SHA1 fingerprint of the String. + public static string ToSHA1Fingerprint(this string expression) + { + byte[] bytes = Encoding.ASCII.GetBytes(expression.ToCharArray()); + + using (SHA1CryptoServiceProvider sha1 = new SHA1CryptoServiceProvider()) + { + byte[] hash = sha1.ComputeHash(bytes); + + // Concatenate the hash bytes into one long String. + return hash.Aggregate( + new StringBuilder(40), + (sb, b) => sb.Append(b.ToString("X2", CultureInfo.InvariantCulture))) + .ToString().ToLowerInvariant(); + } + } + + /// + /// Creates an SHA256 fingerprint of the String. + /// + /// The String instance that this method extends. + /// An SHA256 fingerprint of the String. + public static string ToSHA256Fingerprint(this string expression) + { + byte[] bytes = Encoding.ASCII.GetBytes(expression.ToCharArray()); + + using (SHA256CryptoServiceProvider sha256 = new SHA256CryptoServiceProvider()) + { + byte[] hash = sha256.ComputeHash(bytes); + + // Concatenate the hash bytes into one long String. + return hash.Aggregate( + new StringBuilder(64), + (sb, b) => sb.Append(b.ToString("X2", CultureInfo.InvariantCulture))) + .ToString().ToLowerInvariant(); } } #endregion @@ -82,8 +123,6 @@ namespace ImageProcessor.Helpers.Extensions /// True if the given string is a valid virtual path name public static bool IsValidVirtualPathName(this string expression) { - Contract.Requires(!string.IsNullOrWhiteSpace(expression)); - // Check the start of the string. if (expression.StartsWith("~/")) { @@ -97,14 +136,15 @@ namespace ImageProcessor.Helpers.Extensions /// /// Checks the string to see whether the value is a valid path name. - /// http://stackoverflow.com/questions/62771/how-check-if-given-string-is-legal-allowed-file-name-under-windows/ /// + /// + /// For an explanation + /// + /// /// The String instance that this method extends. /// True if the given string is a valid path name public static bool IsValidPathName(this string expression) { - Contract.Requires(!string.IsNullOrWhiteSpace(expression)); - // Create a regex of invalid characters and test it. string invalidPathNameChars = new string(Path.GetInvalidFileNameChars()); Regex regFixPathName = new Regex("[" + Regex.Escape(invalidPathNameChars) + "]"); diff --git a/src/ImageProcessor/Imaging/OctreeQuantizer.cs b/src/ImageProcessor/Imaging/OctreeQuantizer.cs index ac0929c05..997be4fad 100644 --- a/src/ImageProcessor/Imaging/OctreeQuantizer.cs +++ b/src/ImageProcessor/Imaging/OctreeQuantizer.cs @@ -126,12 +126,12 @@ namespace ImageProcessor.Imaging /// /// Mask used when getting the appropriate pixels for a given node /// - private static int[] mask = new int[8] { 0x80, 0x40, 0x20, 0x10, 0x08, 0x04, 0x02, 0x01 }; + private static readonly int[] mask = new int[8] { 0x80, 0x40, 0x20, 0x10, 0x08, 0x04, 0x02, 0x01 }; /// /// The root of the octree /// - private OctreeNode root; + private readonly OctreeNode root; /// /// Number of leaves in the tree diff --git a/src/ImageProcessor/Imaging/Quantizer.cs b/src/ImageProcessor/Imaging/Quantizer.cs index 29933f321..c12bde5e3 100644 --- a/src/ImageProcessor/Imaging/Quantizer.cs +++ b/src/ImageProcessor/Imaging/Quantizer.cs @@ -15,7 +15,7 @@ namespace ImageProcessor.Imaging #endregion /// - /// Encapsulates methods to calulate the colour pallete of an image. + /// Encapsulates methods to calculate the color palette of an image. /// internal abstract class Quantizer { @@ -250,11 +250,11 @@ namespace ImageProcessor.Imaging /// Retrieve the palette for the quantized image /// /// Any old palette, this is overwritten - /// The new colour palette + /// The new color palette protected abstract ColorPalette GetPalette(ColorPalette original); /// - /// Structure that defines a 32 bpp colour + /// Structure that defines a 32 bit color /// /// /// This structure is used to read data from a 32 bits per pixel image @@ -265,31 +265,31 @@ namespace ImageProcessor.Imaging public struct Color32 { /// - /// Holds the blue component of the colour + /// Holds the blue component of the color /// [FieldOffset(0)] public byte Blue; /// - /// Holds the green component of the colour + /// Holds the green component of the color /// [FieldOffset(1)] public byte Green; /// - /// Holds the red component of the colour + /// Holds the red component of the color /// [FieldOffset(2)] public byte Red; /// - /// Holds the alpha component of the colour + /// Holds the alpha component of the color /// [FieldOffset(3)] public byte Alpha; /// - /// Permits the color32 to be treated as an int32 + /// Permits the color32 to be treated as a 32 bit integer. /// [FieldOffset(0)] public int ARGB; @@ -304,7 +304,7 @@ namespace ImageProcessor.Imaging } /// - /// Gets the colour for this Color32 object + /// Gets the color for this Color32 object /// public Color Color { diff --git a/src/Test/Test/Controllers/HomeController.cs b/src/Test/Test/Controllers/HomeController.cs index 532188eb8..9e859b0a6 100644 --- a/src/Test/Test/Controllers/HomeController.cs +++ b/src/Test/Test/Controllers/HomeController.cs @@ -10,6 +10,7 @@ namespace Test.Controllers using System.Threading.Tasks; using System.Web.Hosting; + using ImageProcessor.Helpers.Extensions; using ImageProcessor.Web.Caching; public class HomeController : Controller @@ -58,6 +59,29 @@ namespace Test.Controllers { DateTime start = DateTime.Now; + List collisions = new List(); +const int Iterations = 1; +const int Maxitems = 360000; + +for (int i = 0; i < Iterations; i++) +{ + List paths = new List(); + + for (int j = 0; j < Maxitems; j++) + { + string path = Path.GetRandomFileName().ToSHA256Fingerprint().Substring(0, 8); + + paths.Add(path); + } + + int count = paths.Distinct().Count(); + + double collisionRate = ((Maxitems - count) * 100D) / Maxitems; + collisions.Add(collisionRate); +} + +double averageCollisionRate = collisions.Average(); + //PersistantDictionary persistantDictionary = PersistantDictionary.Instance; //for (int i = 0; i < 1000; i++) @@ -77,7 +101,8 @@ namespace Test.Controllers TimeSpan timeSpan = DateTime.Now - start; - //ViewBag.Count = persistantDictionary.Count(); + //ViewBag.Count = count; + ViewBag.Collision = averageCollisionRate; return this.View(timeSpan); } diff --git a/src/Test/Test/Views/Home/Index.cshtml b/src/Test/Test/Views/Home/Index.cshtml index bf32df04f..05b7a8885 100644 --- a/src/Test/Test/Views/Home/Index.cshtml +++ b/src/Test/Test/Views/Home/Index.cshtml @@ -90,7 +90,7 @@

Remote

- @* *@ +
diff --git a/src/Test/Test/Views/Home/Speed.cshtml b/src/Test/Test/Views/Home/Speed.cshtml index 862dd76a8..55cd03e4a 100644 --- a/src/Test/Test/Views/Home/Speed.cshtml +++ b/src/Test/Test/Views/Home/Speed.cshtml @@ -14,7 +14,8 @@
Speed In Milliseconds: @s
- Dictionary Count: @ViewBag.Count + @* Distinct Count: @ViewBag.Count*@ + Collision Rate: @ViewBag.Collision%