diff --git a/src/ImageProcessor.Web/Caching/CachedImage.cs b/src/ImageProcessor.Web/Caching/CachedImage.cs index 2757e830e..34316d58c 100644 --- a/src/ImageProcessor.Web/Caching/CachedImage.cs +++ b/src/ImageProcessor.Web/Caching/CachedImage.cs @@ -14,7 +14,7 @@ namespace ImageProcessor.Web.Caching /// /// Describes a cached image /// - internal sealed class CachedImage + internal sealed class CachedImage : IComparable { /// /// Initializes a new instance of the class. @@ -49,5 +49,15 @@ namespace ImageProcessor.Web.Caching /// Gets or sets when the cached image should expire from the cache. /// public DateTime ExpiresUtc { get; set; } + + /// + /// + /// + /// + /// + public int CompareTo(CachedImage other) + { + return this.ExpiresUtc.CompareTo(other.ExpiresUtc); + } } } diff --git a/src/ImageProcessor.Web/Caching/DiskCache.cs b/src/ImageProcessor.Web/Caching/DiskCache.cs index 0d2ca67ea..95100c640 100644 --- a/src/ImageProcessor.Web/Caching/DiskCache.cs +++ b/src/ImageProcessor.Web/Caching/DiskCache.cs @@ -27,17 +27,6 @@ namespace ImageProcessor.Web.Caching internal sealed class DiskCache { #region Fields - /// - /// The maximum number or time a new file should be cached before checking the - /// cache controller and running any clearing mechanisms. - /// - /// - /// NTFS file systems can handle up to 8000 files in one directory. The Cache controller will clear out any - /// time we hit 6000 so if we tell the handler to run at every 1000 times an image is added to the cache we - /// should have a 1000 file buffer. - /// - internal const int MaxRunsBeforeCacheClear = 1000; - /// /// The maximum number of days to cache files on the system for. /// @@ -47,10 +36,10 @@ namespace ImageProcessor.Web.Caching /// The maximum number of files allowed in the directory. /// /// - /// NTFS Folder can handle up to 8000 files in a directory. + /// NTFS directories can handle up to 8000 files in the directory before slowing down. /// This buffer will help us to ensure that we rarely hit anywhere near that limit. /// - private const int MaxFilesCount = 6500; + private const int MaxFilesCount = 7500; /// /// The regular expression to search strings for extension changes. @@ -58,12 +47,7 @@ namespace ImageProcessor.Web.Caching private static readonly Regex FormatRegex = new Regex(@"format=(jpeg|png|bmp|gif)", RegexOptions.Compiled); /// - /// The object to lock against. - /// - private static readonly object SyncRoot = new object(); - - /// - /// The default paths for Cached folders on the server. + /// The default paths for Cached folders on the server. /// private static readonly string CachePath = ImageProcessorConfig.Instance.VirtualCachePath; #endregion @@ -122,8 +106,8 @@ namespace ImageProcessor.Web.Caching internal static void AddImageToCache(string cachedPath, DateTime lastWriteTimeUtc) { string key = Path.GetFileNameWithoutExtension(cachedPath); - DateTime expires = lastWriteTimeUtc.AddDays(MaxFileCachedDuration).ToUniversalTime(); - CachedImage cachedImage = new CachedImage(key, lastWriteTimeUtc, expires); + DateTime expires = DateTime.UtcNow.AddDays(MaxFileCachedDuration).ToUniversalTime(); + CachedImage cachedImage = new CachedImage(cachedPath, lastWriteTimeUtc, expires); PersistantDictionary.Instance.Add(key, cachedImage); } @@ -171,17 +155,27 @@ namespace ImageProcessor.Web.Caching /// internal static bool IsUpdatedFile(string imagePath, string cachedImagePath) { + string key = Path.GetFileNameWithoutExtension(cachedImagePath); + CachedImage cachedImage; + bool isUpdated = false; + if (File.Exists(imagePath)) { - CachedImage image; - string key = Path.GetFileNameWithoutExtension(cachedImagePath); - PersistantDictionary.Instance.TryGetValue(key, out image); FileInfo imageFileInfo = new FileInfo(imagePath); - return image != null && imageFileInfo.LastWriteTimeUtc.Equals(image.LastWriteTimeUtc); + if (PersistantDictionary.Instance.TryGetValue(key, out cachedImage)) + { + if (!imageFileInfo.LastWriteTimeUtc.Equals(cachedImage.LastWriteTimeUtc)) + { + if (PersistantDictionary.Instance.TryRemove(key, out cachedImage)) + { + isUpdated = true; + } + } + } } - return true; + return isUpdated; } /// @@ -198,99 +192,124 @@ namespace ImageProcessor.Web.Caching /// internal static DateTime SetCachedLastWriteTime(string imagePath, string cachedImagePath) { - lock (SyncRoot) + if (File.Exists(imagePath) && File.Exists(cachedImagePath)) { - if (File.Exists(imagePath) && File.Exists(cachedImagePath)) - { - DateTime dateTime = File.GetLastWriteTimeUtc(imagePath); - File.SetLastWriteTimeUtc(cachedImagePath, dateTime); - return dateTime; - } + DateTime dateTime = File.GetLastWriteTimeUtc(imagePath); + File.SetLastWriteTimeUtc(cachedImagePath, dateTime); + return dateTime; } - return DateTime.MinValue; + return DateTime.MinValue.ToUniversalTime(); } /// /// Purges any files from the file-system cache in the given folders. /// - private static void PurgeFolders() + internal static void PurgeFolders() { + // Group each cache folder and clear any expired items or any that exeed + // the maximum allowable count. Regex searchTerm = new Regex(@"(jpeg|png|bmp|gif)"); - var list = PersistantDictionary.Instance.ToList() - .GroupBy(x => searchTerm.Match(x.Value.Path)) - .Select(y => new - { - Path = y.Key, - Expires = y.Select(z => z.Value.ExpiresUtc), - Count = y.Sum(z => z.Key.Count()) - }) - .AsEnumerable(); - - foreach (var path in list) - { - - } + var groups = PersistantDictionary.Instance.ToList() + .GroupBy(x => searchTerm.Match(x.Value.Path).Value) + .Where(g => g.Count() > MaxFilesCount); + //.Where(g => g.Count() > MaxFilesCount + // || g.Select(a => a.Value.ExpiresUtc < DateTime.UtcNow.AddDays(-MaxFileCachedDuration)).Count() > 0); - - - string folder = HostingEnvironment.MapPath(CachePath); - - if (folder != null) + foreach (var group in groups) { - DirectoryInfo directoryInfo = new DirectoryInfo(folder); + int groupCount = group.Count(); - if (directoryInfo.Exists) + foreach (KeyValuePair pair in group.OrderBy(x => x.Value.ExpiresUtc)) { - List directoryInfos = directoryInfo - .EnumerateDirectories("*", SearchOption.TopDirectoryOnly) - .ToList(); + // If the group count is equal to the max count minus 1 then we know we + // are counting down from a full directory not simply clearing out + // expired items. + if (groupCount == MaxFilesCount - 1) + { + break; + } + + // Delete each CachedImage. + try + { + FileInfo fileInfo = new FileInfo(pair.Value.Path); + // Remove from the cache. + string key = Path.GetFileNameWithoutExtension(fileInfo.Name); + CachedImage cachedImage; - Parallel.ForEach( - directoryInfos, - subDirectoryInfo => + if (PersistantDictionary.Instance.TryRemove(key, out cachedImage)) { - // Get all the files in the cache ordered by LastAccessTime - oldest first. - List fileInfos = subDirectoryInfo.EnumerateFiles("*", SearchOption.TopDirectoryOnly) - .OrderBy(x => x.LastAccessTimeUtc).ToList(); - - int counter = fileInfos.Count; - - Parallel.ForEach( - fileInfos, - fileInfo => - { - // Delete the file if we are nearing our limit buffer. - if (counter >= MaxFilesCount || fileInfo.LastAccessTimeUtc < DateTime.UtcNow.AddDays(-MaxFileCachedDuration)) - { - lock (SyncRoot) - { - try - { - // Remove from the cache. - string key = Path.GetFileNameWithoutExtension(fileInfo.Name); - CachedImage cachedImage; - - if (PersistantDictionary.Instance.TryGetValue(key, out cachedImage)) - { - if (PersistantDictionary.Instance.TryRemove(key, out cachedImage)) - { - fileInfo.Delete(); - counter -= 1; - } - } - } - catch (IOException) - { - // Do Nothing, skip to the next. - // TODO: Should we handle this? - } - } - } - }); - }); + fileInfo.Delete(); + groupCount -= 1; + } + } + catch (Exception) + { + // Do Nothing, skip to the next. + // TODO: Should we handle this? + continue; + } } } + + //string folder = HostingEnvironment.MapPath(CachePath); + + //if (folder != null) + //{ + // DirectoryInfo directoryInfo = new DirectoryInfo(folder); + + // if (directoryInfo.Exists) + // { + // List directoryInfos = directoryInfo + // .EnumerateDirectories("*", SearchOption.TopDirectoryOnly) + // .ToList(); + + // Parallel.ForEach( + // directoryInfos, + // subDirectoryInfo => + // { + // // Get all the files in the cache ordered by LastAccessTime - oldest first. + // List fileInfos = subDirectoryInfo.EnumerateFiles("*", SearchOption.TopDirectoryOnly) + // .OrderBy(x => x.LastAccessTimeUtc).ToList(); + + // int counter = fileInfos.Count; + + // Parallel.ForEach( + // fileInfos, + // fileInfo => + // { + // // Delete the file if we are nearing our limit buffer. + // if (counter >= MaxFilesCount || fileInfo.LastAccessTimeUtc < DateTime.UtcNow.AddDays(-MaxFileCachedDuration)) + // { + // lock (SyncRoot) + // { + // try + // { + // // Remove from the cache. + // string key = Path.GetFileNameWithoutExtension(fileInfo.Name); + // CachedImage cachedImage; + + // if (PersistantDictionary.Instance.TryGetValue(key, out cachedImage)) + // { + // if (PersistantDictionary.Instance.TryRemove(key, out cachedImage)) + // { + // fileInfo.Delete(); + // counter -= 1; + // } + // } + // } + // catch (IOException) + // { + // // Do Nothing, skip to the next. + // // TODO: Should we handle this? + // } + // } + // } + // }); + // }); + // } + //} } /// diff --git a/src/ImageProcessor.Web/Caching/SQLContext.cs b/src/ImageProcessor.Web/Caching/SQLContext.cs index af8e4c5aa..fcea2fe33 100644 --- a/src/ImageProcessor.Web/Caching/SQLContext.cs +++ b/src/ImageProcessor.Web/Caching/SQLContext.cs @@ -72,7 +72,7 @@ namespace ImageProcessor.Web.Caching LastWriteTimeUtc TEXT, ExpiresUtc TEXT, PRIMARY KEY (Key), - UNIQUE (Value));"; + UNIQUE (Path));"; command.ExecuteNonQuery(); } @@ -198,7 +198,7 @@ namespace ImageProcessor.Web.Caching CachedImage image = new CachedImage( reader["Path"].ToString(), DateTime.Parse(reader["LastWriteTimeUtc"].ToString()).ToUniversalTime(), - DateTime.Parse(reader["LastWriteTimeUtc"].ToString()).ToUniversalTime()); + DateTime.Parse(reader["ExpiresUtc"].ToString()).ToUniversalTime()); dictionary.Add(key, image); } diff --git a/src/ImageProcessor.Web/HttpModules/ImageProcessingModule.cs b/src/ImageProcessor.Web/HttpModules/ImageProcessingModule.cs index 1bc4a98d6..e1187e2f3 100644 --- a/src/ImageProcessor.Web/HttpModules/ImageProcessingModule.cs +++ b/src/ImageProcessor.Web/HttpModules/ImageProcessingModule.cs @@ -37,14 +37,9 @@ namespace ImageProcessor.Web.HttpModules private static readonly string RemotePrefix = ImageProcessorConfig.Instance.RemotePrefix; /// - /// Whether this is the first run of the handler. + /// The object to lock against. /// - private static bool isFirstRun = true; - - /// - /// A counter for keeping track of how many images have been added to the cache. - /// - private static int cachedImageCounter; + private static readonly object SyncRoot = new object(); #endregion #region IHttpModule Members @@ -106,16 +101,6 @@ namespace ImageProcessor.Web.HttpModules if (ImageUtils.IsValidImageExtension(path) && !string.IsNullOrWhiteSpace(queryString)) { - // Check to see if this is the first run and if so run the cache controller. - if (isFirstRun) - { - // Trim the cache. - DiskCache.PurgeCachedFolders(); - - // Disable the controller. - isFirstRun = false; - } - string fullPath = string.Format("{0}?{1}", path, queryString); string imageName = Path.GetFileName(path); string cachedPath = DiskCache.GetCachePath(fullPath, imageName); @@ -141,46 +126,51 @@ namespace ImageProcessor.Web.HttpModules { if (responseStream != null) { - responseStream.CopyTo(memoryStream); + lock (SyncRoot) + { + // Trim the cache. + DiskCache.PurgeFolders(); + + responseStream.CopyTo(memoryStream); + + imageFactory.Load(memoryStream) + .AddQueryString(queryString) + .Format(ImageUtils.GetImageFormat(imageName)) + .AutoProcess().Save(cachedPath); + + // Ensure that the LastWriteTime property of the source and cached file match. + DateTime dateTime = DiskCache.SetCachedLastWriteTime(path, cachedPath); - imageFactory.Load(memoryStream) - .AddQueryString(queryString) - .Format(ImageUtils.GetImageFormat(imageName)) - .AutoProcess().Save(cachedPath); + // Add to the cache. + DiskCache.AddImageToCache(cachedPath, dateTime); + } } } } } else { - imageFactory.Load(fullPath).AutoProcess().Save(cachedPath); - } - } + lock (SyncRoot) + { + // Trim the cache. + DiskCache.PurgeFolders(); - // Add 1 to the counter - cachedImageCounter += 1; + imageFactory.Load(fullPath).AutoProcess().Save(cachedPath); - // Ensure that the LastWriteTime property of the source and cached file match. - DateTime dateTime = DiskCache.SetCachedLastWriteTime(path, cachedPath); + // Ensure that the LastWriteTime property of the source and cached file match. + DateTime dateTime = DiskCache.SetCachedLastWriteTime(path, cachedPath); - // Add to the cache. - DiskCache.AddImageToCache(cachedPath, dateTime); + // Add to the cache. + DiskCache.AddImageToCache(cachedPath, dateTime); + } + } + } } context.Items[CachedResponseTypeKey] = ImageUtils.GetResponseType(imageName).ToDescription(); // The cached file is valid so just rewrite the path. context.RewritePath(DiskCache.GetVirtualPath(cachedPath, context.Request), false); - - // If the number of cached imaged hits the maximum allowed for this session then we clear - // the cache again and reset the counter. - // TODO: There is a potential concurrency issue here but collision probability is very low - // it would be nice to nail it though. - if (cachedImageCounter >= DiskCache.MaxRunsBeforeCacheClear) - { - DiskCache.PurgeCachedFolders(); - cachedImageCounter = 0; - } } } } diff --git a/src/ImageProcessor.Web/ImageFactoryExtensions.cs b/src/ImageProcessor.Web/ImageFactoryExtensions.cs index 1b05bb669..22c2583ab 100644 --- a/src/ImageProcessor.Web/ImageFactoryExtensions.cs +++ b/src/ImageProcessor.Web/ImageFactoryExtensions.cs @@ -39,8 +39,8 @@ namespace ImageProcessor.Web if (factory.ShouldProcess) { // TODO: This is going to be a bottleneck for speed. Find a faster way. - lock (SyncRoot) - { + //lock (SyncRoot) + //{ // Get a list of all graphics processors that have parsed and matched the querystring. List list = ImageProcessorConfig.Instance.GraphicsProcessors @@ -53,7 +53,7 @@ namespace ImageProcessor.Web { factory.Image = graphicsProcessor.ProcessImage(factory); } - } + //} } return factory; diff --git a/src/Test/Test/Web.config b/src/Test/Test/Web.config index 323aca4c7..cf4f6d258 100644 --- a/src/Test/Test/Web.config +++ b/src/Test/Test/Web.config @@ -3,45 +3,40 @@ For more information on how to configure your ASP.NET application, please visit http://go.microsoft.com/fwlink/?LinkId=152368 --> -
-
-
+
+
- - - + - - - - - + + + + + - - + - - - - - - + + + + + @@ -51,31 +46,28 @@ - - - - + + - - + - + @@ -84,4 +76,4 @@ - + \ No newline at end of file