Enhancing DiskCache to handle 24,000 files

Former-commit-id: 44c08e3be09bf9afa8f801f751e1c7c3766563e8

src/ImageProcessor.Web/Caching/DiskCache.cs

@@ -84,22 +84,28 @@ namespace ImageProcessor.Web.Caching
 
             if (absoluteCachePath != null)
             {
-                DirectoryInfo di = new DirectoryInfo(absoluteCachePath);
-
-                if (!di.Exists)
-                {
-                    // Create the directory.
-                    Directory.CreateDirectory(absoluteCachePath);
-                }
-
                 string parsedExtension = ParseExtension(imagePath);
-                string fallbackExtension = imageName.Substring(imageName.LastIndexOf(".", StringComparison.Ordinal));
+                string fallbackExtension = imageName.Substring(imageName.LastIndexOf(".", StringComparison.Ordinal) + 1);
+                string subpath = !string.IsNullOrWhiteSpace(parsedExtension) ? parsedExtension : fallbackExtension;
 
                 string cachedFileName = string.Format(
-                    "{0}{1}",
+                    "{0}.{1}",
                     imagePath.ToMD5Fingerprint(),
                     !string.IsNullOrWhiteSpace(parsedExtension) ? parsedExtension : fallbackExtension);
-                cachedPath = Path.Combine(absoluteCachePath, cachedFileName);
+                cachedPath = Path.Combine(absoluteCachePath, subpath, cachedFileName);
+
+                string cachedDirectory = Path.GetDirectoryName(cachedPath);
+
+                if (cachedDirectory != null)
+                {
+                    DirectoryInfo directoryInfo = new DirectoryInfo(cachedDirectory);
+
+                    if (!directoryInfo.Exists)
+                    {
+                        // Create the directory.
+                        Directory.CreateDirectory(cachedDirectory);
+                    }
+                }
             }
 
             return cachedPath;
@@ -190,33 +196,42 @@ namespace ImageProcessor.Web.Caching
 
                 if (directoryInfo.Exists)
                 {
-                    // Get all the files in the cache ordered by LastAccessTime - oldest first.
+                    List<DirectoryInfo> directoryInfos = directoryInfo
-                    List<FileInfo> fileInfos = directoryInfo.EnumerateFiles("*", SearchOption.AllDirectories)
+                        .EnumerateDirectories("*", SearchOption.AllDirectories)
-                        .OrderBy(x => x.LastAccessTime).ToList();
+                        .ToList();
-
-                    int counter = fileInfos.Count;
 
                     Parallel.ForEach(
-                        fileInfos,
+                        directoryInfos,
-                        fileInfo =>
+                        dir =>
                         {
-                            lock (SyncRoot)
+                            // Get all the files in the cache ordered by LastAccessTime - oldest first.
-                            {
+                            List<FileInfo> fileInfos = dir.EnumerateFiles("*", SearchOption.AllDirectories)
-                                try
+                                .OrderBy(x => x.LastAccessTime).ToList();
+
+                            int counter = fileInfos.Count;
+
+                            Parallel.ForEach(
+                                fileInfos,
+                                fileInfo =>
                                 {
-                                    // Delete the file if we are nearing our limit buffer.
+                                    lock (SyncRoot)
-                                    if (counter >= MaxFilesCount || fileInfo.LastAccessTime < DateTime.Now.AddDays(-MaxFileCachedDuration))
                                     {
-                                        fileInfo.Delete();
+                                        try
-                                        counter -= 1;
+                                        {
+                                            // Delete the file if we are nearing our limit buffer.
+                                            if (counter >= MaxFilesCount && fileInfo.LastAccessTime.ToUniversalTime() < DateTime.UtcNow.AddHours(1)
+                                                || fileInfo.LastAccessTime.ToUniversalTime() < DateTime.UtcNow.AddDays(-MaxFileCachedDuration))
+                                            {
+                                                fileInfo.Delete();
+                                                counter -= 1;
+                                            }
+                                        }
+                                        catch (IOException)
+                                        {
+                                            // Do Nothing, skip to the next.
+                                        }
                                     }
-                                }
+                                });
-                                catch
-                                {
-                                    // TODO: Sort out the try/catch.
-                                    throw;
-                                }
-                            }
                         });
                 }
             }
@@ -237,7 +252,7 @@ namespace ImageProcessor.Web.Caching
             {
                 if (match.Success)
                 {
-                    return "." + match.Value.Split('=')[1];
+                    return match.Value.Split('=')[1];
                 }
             }
 

src/ImageProcessor.Web/Config/ImageProcessorConfig.cs

@@ -76,7 +76,7 @@ namespace ImageProcessor.Web.Config
         /// <summary>
         /// Gets the list of available GraphicsProcessors.
         /// </summary>
-        public List<IGraphicsProcessor> GraphicsProcessors { get; private set; }
+        public IList<IGraphicsProcessor> GraphicsProcessors { get; private set; }
 
         #region Caching
         /// <summary>

src/ImageProcessor.Web/Helpers/FileCompareLastwritetime.cs

@@ -28,7 +28,7 @@ namespace ImageProcessor.Web.Helpers
         /// </returns>
         public static DateTime ToMinute(DateTime value)
         {
-            return new DateTime(value.Year, value.Month, value.Day, value.Hour, value.Minute, 0, value.Kind);
+            return new DateTime(value.Year, value.Month, value.Day, value.Hour, value.Minute, 0, value.Kind).ToUniversalTime();
         }
 
         /// <summary>
@@ -43,7 +43,7 @@ namespace ImageProcessor.Web.Helpers
         /// <returns>true if the specified objects are equal; otherwise, false.</returns>
         public bool Equals(System.IO.FileInfo f1, System.IO.FileInfo f2)
         {
-            return ToMinute(f1.LastWriteTime) == ToMinute(f2.LastWriteTime);
+            return ToMinute(f1.LastWriteTime.ToUniversalTime()) == ToMinute(f2.LastWriteTime.ToUniversalTime());
         }
 
         /// <summary>
@@ -53,7 +53,7 @@ namespace ImageProcessor.Web.Helpers
         /// <returns>A hash code for the specified <see cref="T:System.IO.FileInfo"/>.</returns>
         public int GetHashCode(System.IO.FileInfo fi)
         {
-            return ToMinute(fi.LastWriteTime).GetHashCode();
+            return ToMinute(fi.LastWriteTime.ToUniversalTime()).GetHashCode();
         }
     }
 }

src/ImageProcessor.Web/HttpModules/ImageProcessingModule.cs

@@ -78,19 +78,24 @@ namespace ImageProcessor.Web.HttpModules
 
             // Is this a remote file.
             bool isRemote = context.Request.Path.Equals(RemotePrefix, StringComparison.OrdinalIgnoreCase);
-            string path;
+            string path = string.Empty;
             string queryString = string.Empty;
 
             if (isRemote)
             {
                 // We need to split the querystring to get the actual values we want.
-                string[] paths = HttpUtility.UrlDecode(context.Request.QueryString.ToString()).Split('?');
+                string urlDecode = HttpUtility.UrlDecode(context.Request.QueryString.ToString());
 
-                path = paths[0];
+                if (urlDecode != null)
-
-                if (paths.Length > 1)
                 {
-                    queryString = paths[1];
+                    string[] paths = urlDecode.Split('?');
+
+                    path = paths[0];
+
+                    if (paths.Length > 1)
+                    {
+                        queryString = paths[1];
+                    }
                 }
             }
             else
@@ -148,7 +153,16 @@ namespace ImageProcessor.Web.HttpModules
                             }
                             else
                             {
-                                imageFactory.Load(fullPath).AutoProcess().Save(cachedPath);
+                                try
+                                {
+                                    imageFactory.Load(fullPath).AutoProcess().Save(cachedPath);
+                                }
+                                catch (Exception ex)
+                                {
+
+                                    throw ex;
+                                }
+
                             }
                         }
 

src/ImageProcessor.Web/ImageFactoryExtensions.cs

@@ -8,15 +8,10 @@
 namespace ImageProcessor.Web
 {
     #region Using
-
-    using System;
     using System.Collections.Generic;
     using System.Linq;
-    using System.Reflection;
-
     using ImageProcessor.Processors;
     using ImageProcessor.Web.Config;
-    using ImageProcessor.Web.Helpers;
     #endregion
 
     /// <summary>
@@ -36,31 +31,31 @@ namespace ImageProcessor.Web
         /// <returns>
         /// The current instance of the <see cref="T:ImageProcessor.ImageFactory"/> class.
         /// </returns>
-        public static ImageFactory AutoProcess(this ImageFactory factory)
+public static ImageFactory AutoProcess(this ImageFactory factory)
+{
+    if (factory.ShouldProcess)
+    {
+        // TODO: This is going to be a bottleneck for speed. Find a faster way.
+        lock (SyncLock)
         {
-            if (factory.ShouldProcess)
+            // Get a list of all graphics processors that have parsed and matched the querystring.
+            List<IGraphicsProcessor> list =
+                ImageProcessorConfig.Instance.GraphicsProcessors
+                .Where(x => x.MatchRegexIndex(factory.QueryString) != int.MaxValue)
+                .OrderBy(y => y.SortOrder)
+                .ToList();
+
+            // Loop through and process the image.
+            foreach (IGraphicsProcessor graphicsProcessor in list)
             {
-                // TODO: This is going to be a bottleneck for speed. Find a faster way.
+                factory.Image = graphicsProcessor.ProcessImage(factory);
-                lock (SyncLock)
-                {
-                    // Get a list of all graphics processors that have parsed and matched the querystring.
-                    List<IGraphicsProcessor> list =
-                        ImageProcessorConfig.Instance.GraphicsProcessors
-                        .Where(x => x.MatchRegexIndex(factory.QueryString) != int.MaxValue)
-                        .OrderBy(y => y.SortOrder)
-                        .ToList();
-
-                    // Loop through and process the image.
-                    foreach (IGraphicsProcessor graphicsProcessor in list)
-                    {
-                        factory.Image = graphicsProcessor.ProcessImage(factory);
-                    }
-                }
             }
+        }
+    }
 
-            return factory;
+    return factory;
 
-        }
+}
 
 
     }

src/Test/Test/Web.config

@@ -72,7 +72,7 @@
         <add url="http://images.mymovies.net"/>
       </whiteList>
     </security>
-    <cache virtualPath="~/cache" maxDays="365" />
+    <cache virtualPath="~/cache" maxDays="28" />
     <processing>
       <plugins>
         <plugin name ="Resize">