From bc6e29807464ca054c8a0cc6c2fb715255676a4c Mon Sep 17 00:00:00 2001 From: JimBobSquarePants Date: Tue, 5 Mar 2013 02:36:10 +0000 Subject: [PATCH] Enhancing DiskCache to handle 24,000 files Former-commit-id: bb3d1ded41e10fff13d46fa9009a838c2bd958f0 --- src/ImageProcessor.Web/Caching/DiskCache.cs | 81 +++++++++++-------- .../Config/ImageProcessorConfig.cs | 2 +- .../Helpers/FileCompareLastwritetime.cs | 6 +- .../HttpModules/ImageProcessingModule.cs | 28 +++++-- .../ImageFactoryExtensions.cs | 45 +++++------ src/Test/Test/Web.config | 2 +- 6 files changed, 94 insertions(+), 70 deletions(-) diff --git a/src/ImageProcessor.Web/Caching/DiskCache.cs b/src/ImageProcessor.Web/Caching/DiskCache.cs index 19e3e5466..af3334939 100644 --- a/src/ImageProcessor.Web/Caching/DiskCache.cs +++ b/src/ImageProcessor.Web/Caching/DiskCache.cs @@ -84,22 +84,28 @@ namespace ImageProcessor.Web.Caching if (absoluteCachePath != null) { - DirectoryInfo di = new DirectoryInfo(absoluteCachePath); - - if (!di.Exists) - { - // Create the directory. - Directory.CreateDirectory(absoluteCachePath); - } - string parsedExtension = ParseExtension(imagePath); - string fallbackExtension = imageName.Substring(imageName.LastIndexOf(".", StringComparison.Ordinal)); + string fallbackExtension = imageName.Substring(imageName.LastIndexOf(".", StringComparison.Ordinal) + 1); + string subpath = !string.IsNullOrWhiteSpace(parsedExtension) ? parsedExtension : fallbackExtension; string cachedFileName = string.Format( - "{0}{1}", + "{0}.{1}", imagePath.ToMD5Fingerprint(), !string.IsNullOrWhiteSpace(parsedExtension) ? parsedExtension : fallbackExtension); - cachedPath = Path.Combine(absoluteCachePath, cachedFileName); + cachedPath = Path.Combine(absoluteCachePath, subpath, cachedFileName); + + string cachedDirectory = Path.GetDirectoryName(cachedPath); + + if (cachedDirectory != null) + { + DirectoryInfo directoryInfo = new DirectoryInfo(cachedDirectory); + + if (!directoryInfo.Exists) + { + // Create the directory. + Directory.CreateDirectory(cachedDirectory); + } + } } return cachedPath; @@ -190,33 +196,42 @@ namespace ImageProcessor.Web.Caching if (directoryInfo.Exists) { - // Get all the files in the cache ordered by LastAccessTime - oldest first. - List fileInfos = directoryInfo.EnumerateFiles("*", SearchOption.AllDirectories) - .OrderBy(x => x.LastAccessTime).ToList(); - - int counter = fileInfos.Count; + List directoryInfos = directoryInfo + .EnumerateDirectories("*", SearchOption.AllDirectories) + .ToList(); Parallel.ForEach( - fileInfos, - fileInfo => + directoryInfos, + dir => { - lock (SyncRoot) - { - try + // Get all the files in the cache ordered by LastAccessTime - oldest first. + List fileInfos = dir.EnumerateFiles("*", SearchOption.AllDirectories) + .OrderBy(x => x.LastAccessTime).ToList(); + + int counter = fileInfos.Count; + + Parallel.ForEach( + fileInfos, + fileInfo => { - // Delete the file if we are nearing our limit buffer. - if (counter >= MaxFilesCount || fileInfo.LastAccessTime < DateTime.Now.AddDays(-MaxFileCachedDuration)) + lock (SyncRoot) { - fileInfo.Delete(); - counter -= 1; + try + { + // Delete the file if we are nearing our limit buffer. + if (counter >= MaxFilesCount && fileInfo.LastAccessTime.ToUniversalTime() < DateTime.UtcNow.AddHours(1) + || fileInfo.LastAccessTime.ToUniversalTime() < DateTime.UtcNow.AddDays(-MaxFileCachedDuration)) + { + fileInfo.Delete(); + counter -= 1; + } + } + catch (IOException) + { + // Do Nothing, skip to the next. + } } - } - catch - { - // TODO: Sort out the try/catch. - throw; - } - } + }); }); } } @@ -237,7 +252,7 @@ namespace ImageProcessor.Web.Caching { if (match.Success) { - return "." + match.Value.Split('=')[1]; + return match.Value.Split('=')[1]; } } diff --git a/src/ImageProcessor.Web/Config/ImageProcessorConfig.cs b/src/ImageProcessor.Web/Config/ImageProcessorConfig.cs index 0e08e3e41..1c35ff9bc 100644 --- a/src/ImageProcessor.Web/Config/ImageProcessorConfig.cs +++ b/src/ImageProcessor.Web/Config/ImageProcessorConfig.cs @@ -76,7 +76,7 @@ namespace ImageProcessor.Web.Config /// /// Gets the list of available GraphicsProcessors. /// - public List GraphicsProcessors { get; private set; } + public IList GraphicsProcessors { get; private set; } #region Caching /// diff --git a/src/ImageProcessor.Web/Helpers/FileCompareLastwritetime.cs b/src/ImageProcessor.Web/Helpers/FileCompareLastwritetime.cs index c91a031a0..8b943e2e2 100644 --- a/src/ImageProcessor.Web/Helpers/FileCompareLastwritetime.cs +++ b/src/ImageProcessor.Web/Helpers/FileCompareLastwritetime.cs @@ -28,7 +28,7 @@ namespace ImageProcessor.Web.Helpers /// public static DateTime ToMinute(DateTime value) { - return new DateTime(value.Year, value.Month, value.Day, value.Hour, value.Minute, 0, value.Kind); + return new DateTime(value.Year, value.Month, value.Day, value.Hour, value.Minute, 0, value.Kind).ToUniversalTime(); } /// @@ -43,7 +43,7 @@ namespace ImageProcessor.Web.Helpers /// true if the specified objects are equal; otherwise, false. public bool Equals(System.IO.FileInfo f1, System.IO.FileInfo f2) { - return ToMinute(f1.LastWriteTime) == ToMinute(f2.LastWriteTime); + return ToMinute(f1.LastWriteTime.ToUniversalTime()) == ToMinute(f2.LastWriteTime.ToUniversalTime()); } /// @@ -53,7 +53,7 @@ namespace ImageProcessor.Web.Helpers /// A hash code for the specified . public int GetHashCode(System.IO.FileInfo fi) { - return ToMinute(fi.LastWriteTime).GetHashCode(); + return ToMinute(fi.LastWriteTime.ToUniversalTime()).GetHashCode(); } } } diff --git a/src/ImageProcessor.Web/HttpModules/ImageProcessingModule.cs b/src/ImageProcessor.Web/HttpModules/ImageProcessingModule.cs index f28724d3a..f0687d575 100644 --- a/src/ImageProcessor.Web/HttpModules/ImageProcessingModule.cs +++ b/src/ImageProcessor.Web/HttpModules/ImageProcessingModule.cs @@ -78,19 +78,24 @@ namespace ImageProcessor.Web.HttpModules // Is this a remote file. bool isRemote = context.Request.Path.Equals(RemotePrefix, StringComparison.OrdinalIgnoreCase); - string path; + string path = string.Empty; string queryString = string.Empty; if (isRemote) { // We need to split the querystring to get the actual values we want. - string[] paths = HttpUtility.UrlDecode(context.Request.QueryString.ToString()).Split('?'); + string urlDecode = HttpUtility.UrlDecode(context.Request.QueryString.ToString()); - path = paths[0]; - - if (paths.Length > 1) + if (urlDecode != null) { - queryString = paths[1]; + string[] paths = urlDecode.Split('?'); + + path = paths[0]; + + if (paths.Length > 1) + { + queryString = paths[1]; + } } } else @@ -148,7 +153,16 @@ namespace ImageProcessor.Web.HttpModules } else { - imageFactory.Load(fullPath).AutoProcess().Save(cachedPath); + try + { + imageFactory.Load(fullPath).AutoProcess().Save(cachedPath); + } + catch (Exception ex) + { + + throw ex; + } + } } diff --git a/src/ImageProcessor.Web/ImageFactoryExtensions.cs b/src/ImageProcessor.Web/ImageFactoryExtensions.cs index 23eaa1eb7..d135cbb67 100644 --- a/src/ImageProcessor.Web/ImageFactoryExtensions.cs +++ b/src/ImageProcessor.Web/ImageFactoryExtensions.cs @@ -8,15 +8,10 @@ namespace ImageProcessor.Web { #region Using - - using System; using System.Collections.Generic; using System.Linq; - using System.Reflection; - using ImageProcessor.Processors; using ImageProcessor.Web.Config; - using ImageProcessor.Web.Helpers; #endregion /// @@ -36,31 +31,31 @@ namespace ImageProcessor.Web /// /// The current instance of the class. /// - public static ImageFactory AutoProcess(this ImageFactory factory) +public static ImageFactory AutoProcess(this ImageFactory factory) +{ + if (factory.ShouldProcess) + { + // TODO: This is going to be a bottleneck for speed. Find a faster way. + lock (SyncLock) { - if (factory.ShouldProcess) + // Get a list of all graphics processors that have parsed and matched the querystring. + List list = + ImageProcessorConfig.Instance.GraphicsProcessors + .Where(x => x.MatchRegexIndex(factory.QueryString) != int.MaxValue) + .OrderBy(y => y.SortOrder) + .ToList(); + + // Loop through and process the image. + foreach (IGraphicsProcessor graphicsProcessor in list) { - // TODO: This is going to be a bottleneck for speed. Find a faster way. - lock (SyncLock) - { - // Get a list of all graphics processors that have parsed and matched the querystring. - List list = - ImageProcessorConfig.Instance.GraphicsProcessors - .Where(x => x.MatchRegexIndex(factory.QueryString) != int.MaxValue) - .OrderBy(y => y.SortOrder) - .ToList(); - - // Loop through and process the image. - foreach (IGraphicsProcessor graphicsProcessor in list) - { - factory.Image = graphicsProcessor.ProcessImage(factory); - } - } + factory.Image = graphicsProcessor.ProcessImage(factory); } + } + } - return factory; + return factory; - } +} } diff --git a/src/Test/Test/Web.config b/src/Test/Test/Web.config index d13d11155..a10f918e9 100644 --- a/src/Test/Test/Web.config +++ b/src/Test/Test/Web.config @@ -72,7 +72,7 @@ - +