From efe26a3a8cd0f0b3f1ce82180bb1bfae3d2ae8b7 Mon Sep 17 00:00:00 2001 From: James South Date: Mon, 9 Dec 2013 23:54:31 +0000 Subject: [PATCH] Can now cache billions of items TODO: An efficient cleanup mechanism Former-commit-id: aec71d092116d0de1d2f92d84327e5dc8e550582 --- .../NET45/Caching/DiskCache.cs | 97 +++---------------- .../HttpModules/ImageProcessingModule.cs | 2 +- src/ImageProcessor/ImageFactory.cs | 13 +++ 3 files changed, 29 insertions(+), 83 deletions(-) diff --git a/src/ImageProcessor.Web/NET45/Caching/DiskCache.cs b/src/ImageProcessor.Web/NET45/Caching/DiskCache.cs index 3bbc0a42b..d8572050d 100644 --- a/src/ImageProcessor.Web/NET45/Caching/DiskCache.cs +++ b/src/ImageProcessor.Web/NET45/Caching/DiskCache.cs @@ -37,12 +37,6 @@ namespace ImageProcessor.Web.Caching /// internal static readonly int MaxFileCachedDuration = ImageProcessorConfig.Instance.MaxCacheDays; - /// - /// The valid sub directory chars. This used in combination with the file limit per folder - /// allows the storage of 360,000 image files in the cache. - /// - private const string ValidSubDirectoryChars = "abcdefghijklmnopqrstuvwxyz0123456789"; - /// /// The maximum number of files allowed in the directory. /// @@ -53,13 +47,7 @@ namespace ImageProcessor.Web.Caching /// /// /// - private const int MaxFilesCount = 10000; - - /// - /// The regular expression to search strings for file extensions. - /// - private static readonly Regex FormatRegex = new Regex( - @"(jpeg|png|bmp|gif)", RegexOptions.RightToLeft | RegexOptions.Compiled); + private const int MaxFilesCount = 100; /// /// The regular expression to search strings for valid subfolder names. @@ -142,62 +130,6 @@ namespace ImageProcessor.Web.Caching #region Methods #region Internal - /// - /// Creates the series of directories required to house our cached images. - /// The images are stored in paths that are based upon the MD5 of their full request path - /// taking the first and last characters of the hash to determine their location. - /// ~/cache/a/1/ab04g67p91.jpg - /// This allows us to store 36 folders within 36 folders giving us a total of 12,960,000 images. - /// - /// - /// True if the directories are successfully created; otherwise, false. - /// - [SuppressMessage("StyleCop.CSharp.DocumentationRules", "SA1650:ElementDocumentationMustBeSpelledCorrectly", Justification = "Reviewed. Suppression is OK here.")] - internal static bool CreateDirectories() - { - bool success = true; - - try - { - // Split up our characters into an array to loop though. - char[] characters = ValidSubDirectoryChars.ToCharArray(); - - // Loop through and create the first level. - Parallel.ForEach( - characters, - (character, loop) => - { - string firstSubPath = Path.Combine(AbsoluteCachePath, character.ToString(CultureInfo.InvariantCulture)); - DirectoryInfo directoryInfo = new DirectoryInfo(firstSubPath); - - if (!directoryInfo.Exists) - { - directoryInfo.Create(); - - // Loop through and create the second level. - Parallel.ForEach( - characters, - (subCharacter, subLoop) => - { - string secondSubPath = Path.Combine(firstSubPath, subCharacter.ToString(CultureInfo.InvariantCulture)); - DirectoryInfo subDirectoryInfo = new DirectoryInfo(secondSubPath); - - if (!subDirectoryInfo.Exists) - { - subDirectoryInfo.Create(); - } - }); - } - }); - } - catch - { - success = false; - } - - return success; - } - /// /// Gets the virtual path to the cached processed image. /// @@ -407,12 +339,13 @@ namespace ImageProcessor.Web.Caching { int groupCount = group.Count(); - foreach (CleanupImage pair in group.OrderBy(x => x.ExpiresUtc)) + foreach (CleanupImage image in group.OrderBy(x => x.ExpiresUtc)) { // If the group count is equal to the max count minus 1 then we know we // are counting down from a full directory not simply clearing out // expired items. - if (groupCount == MaxFilesCount - 1) + if (groupCount <= MaxFilesCount - 1 + && image.ExpiresUtc >= DateTime.UtcNow.AddDays(-MaxFileCachedDuration)) { break; } @@ -420,7 +353,7 @@ namespace ImageProcessor.Web.Caching try { // Remove from the cache and delete each CachedImage. - FileInfo fileInfo = new FileInfo(pair.Path); + FileInfo fileInfo = new FileInfo(image.Path); string key = Path.GetFileNameWithoutExtension(fileInfo.Name); if (await MemoryCache.Instance.RemoveAsync(key)) @@ -441,10 +374,10 @@ namespace ImageProcessor.Web.Caching /// /// Gets the full transformed cached path for the image. - /// The images are stored in paths that are based upon the MD5 of their full request path - /// taking the first and last characters of the hash to determine their location. - /// ~/cache/a/1/ab04g67p91.jpg - /// This allows us to store 36 folders within 36 folders giving us a total of 12,960,000 images. + /// The images are stored in paths that are based upon the sha1 of their full request path + /// taking the individual characters of the hash to determine their location. + /// This allows us to store 40 folders within 40 folders giving us a total of 3.0223145e+64 potential images. + /// Answers on a post card if you can figure out a way to store their details in a db for fast recovery. /// /// The full cached path for the image. [SuppressMessage("StyleCop.CSharp.DocumentationRules", "SA1650:ElementDocumentationMustBeSpelledCorrectly", Justification = "Reviewed. Suppression is OK here.")] @@ -454,21 +387,21 @@ namespace ImageProcessor.Web.Caching if (AbsoluteCachePath != null) { - // Use an md5 hash of the full path including the querystring to create the image name. + // Use an sha1 hash of the full path including the querystring to create the image name. // That name can also be used as a key for the cached image and we should be able to use - // The first character of that hash as a subfolder. + // The characters of that hash as subfolders. string parsedExtension = ImageUtils.GetExtension(this.fullPath); string fallbackExtension = this.imageName.Substring(this.imageName.LastIndexOf(".", StringComparison.Ordinal) + 1); - string encryptedName = this.fullPath.ToMD5Fingerprint(); - string firstSubpath = encryptedName.Substring(0, 1); - string secondSubpath = encryptedName.Substring(31, 1); + string encryptedName = this.fullPath.ToSHA1Fingerprint(); + + string pathFromKey = string.Join("\\", encryptedName.ToCharArray()); string cachedFileName = string.Format( "{0}.{1}", encryptedName, !string.IsNullOrWhiteSpace(parsedExtension) ? parsedExtension.Replace(".", string.Empty) : fallbackExtension); - cachedPath = Path.Combine(AbsoluteCachePath, firstSubpath, secondSubpath, cachedFileName); + cachedPath = Path.Combine(AbsoluteCachePath, pathFromKey, cachedFileName); } return cachedPath; diff --git a/src/ImageProcessor.Web/NET45/HttpModules/ImageProcessingModule.cs b/src/ImageProcessor.Web/NET45/HttpModules/ImageProcessingModule.cs index e90b47221..73f7b813d 100644 --- a/src/ImageProcessor.Web/NET45/HttpModules/ImageProcessingModule.cs +++ b/src/ImageProcessor.Web/NET45/HttpModules/ImageProcessingModule.cs @@ -79,7 +79,7 @@ namespace ImageProcessor.Web.HttpModules if (!this.hasModuleInitialized) { Interlocked.CompareExchange(ref initCheck, 1, 0); - DiskCache.CreateDirectories(); + // DiskCache.CreateDirectories(); } #if NET45 diff --git a/src/ImageProcessor/ImageFactory.cs b/src/ImageProcessor/ImageFactory.cs index a25f31878..049d289cb 100644 --- a/src/ImageProcessor/ImageFactory.cs +++ b/src/ImageProcessor/ImageFactory.cs @@ -677,6 +677,9 @@ namespace ImageProcessor // Fix the colour palette of indexed images. this.FixIndexedPallete(); + // ReSharper disable once AssignNullToNotNullAttribute + DirectoryInfo directoryInfo = new DirectoryInfo(Path.GetDirectoryName(filePath)); + if (this.ImageFormat.Equals(ImageFormat.Jpeg)) { // Jpegs can be saved with different settings to include a quality setting for the JPEG compression. @@ -693,6 +696,11 @@ namespace ImageProcessor { try { + if (!directoryInfo.Exists) + { + directoryInfo.Create(); + } + this.Image.Save(filePath, imageCodecInfo, encoderParameters); break; } @@ -710,6 +718,11 @@ namespace ImageProcessor { try { + if (!directoryInfo.Exists) + { + directoryInfo.Create(); + } + this.Image.Save(filePath, this.ImageFormat); break; }