tsai
/
ImageSharp
mirror of https://github.com/SixLabors/ImageSharp
1
0
Fork 0
Code Issues Projects Releases Wiki Activity
Browse Source

Bug fixes in cache 

Former-commit-id: 07050d4d50fcaadffd6dda4300ddc27b69b15127
pull/17/head
James South 13 years ago
parent
3af4298f05
commit
0cd274e5a5
6 changed files with 186 additions and 175 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 12
      src/ImageProcessor.Web/Caching/CachedImage.cs
  2. 221
      src/ImageProcessor.Web/Caching/DiskCache.cs
  3. 4
      src/ImageProcessor.Web/Caching/SQLContext.cs
  4. 72
      src/ImageProcessor.Web/HttpModules/ImageProcessingModule.cs
  5. 6
      src/ImageProcessor.Web/ImageFactoryExtensions.cs
  6. 46
      src/Test/Test/Web.config

12
src/ImageProcessor.Web/Caching/CachedImage.cs
View File

@ -14,7 +14,7 @@ namespace ImageProcessor.Web.Caching
/// <summary>
/// Describes a cached image
/// </summary>
internal sealed class CachedImage
internal sealed class CachedImage : IComparable<CachedImage>
{
/// <summary>
/// Initializes a new instance of the <see cref="CachedImage"/> class.
@ -49,5 +49,15 @@ namespace ImageProcessor.Web.Caching
/// Gets or sets when the cached image should expire from the cache.
/// </summary>
public DateTime ExpiresUtc { get; set; }
/// <summary>
///
/// </summary>
/// <param name="other"></param>
/// <returns></returns>
public int CompareTo(CachedImage other)
{
return this.ExpiresUtc.CompareTo(other.ExpiresUtc);
}
}
}

221
src/ImageProcessor.Web/Caching/DiskCache.cs
View File

@ -27,17 +27,6 @@ namespace ImageProcessor.Web.Caching
internal sealed class DiskCache
{
#region Fields
/// <summary>
/// The maximum number or time a new file should be cached before checking the
/// cache controller and running any clearing mechanisms.
/// </summary>
/// <remarks>
/// NTFS file systems can handle up to 8000 files in one directory. The Cache controller will clear out any
/// time we hit 6000 so if we tell the handler to run at every 1000 times an image is added to the cache we
/// should have a 1000 file buffer.
/// </remarks>
internal const int MaxRunsBeforeCacheClear = 1000;
/// <summary>
/// The maximum number of days to cache files on the system for.
/// </summary>
@ -47,10 +36,10 @@ namespace ImageProcessor.Web.Caching
/// The maximum number of files allowed in the directory.
/// </summary>
/// <remarks>
/// NTFS Folder can handle up to 8000 files in a directory.
/// NTFS directories can handle up to 8000 files in the directory before slowing down.
/// This buffer will help us to ensure that we rarely hit anywhere near that limit.
/// </remarks>
private const int MaxFilesCount = 6500;
private const int MaxFilesCount = 7500;
/// <summary>
/// The regular expression to search strings for extension changes.
@ -58,12 +47,7 @@ namespace ImageProcessor.Web.Caching
private static readonly Regex FormatRegex = new Regex(@"format=(jpeg|png|bmp|gif)", RegexOptions.Compiled);
/// <summary>
/// The object to lock against.
/// </summary>
private static readonly object SyncRoot = new object();
/// <summary>
/// The default paths for Cached folders on the server.
/// The default paths for Cached folders on the server.
/// </summary>
private static readonly string CachePath = ImageProcessorConfig.Instance.VirtualCachePath;
#endregion
@ -122,8 +106,8 @@ namespace ImageProcessor.Web.Caching
internal static void AddImageToCache(string cachedPath, DateTime lastWriteTimeUtc)
{
string key = Path.GetFileNameWithoutExtension(cachedPath);
DateTime expires = lastWriteTimeUtc.AddDays(MaxFileCachedDuration).ToUniversalTime();
CachedImage cachedImage = new CachedImage(key, lastWriteTimeUtc, expires);
DateTime expires = DateTime.UtcNow.AddDays(MaxFileCachedDuration).ToUniversalTime();
CachedImage cachedImage = new CachedImage(cachedPath, lastWriteTimeUtc, expires);
PersistantDictionary.Instance.Add(key, cachedImage);
}
@ -171,17 +155,27 @@ namespace ImageProcessor.Web.Caching
/// </returns>
internal static bool IsUpdatedFile(string imagePath, string cachedImagePath)
{
string key = Path.GetFileNameWithoutExtension(cachedImagePath);
CachedImage cachedImage;
bool isUpdated = false;
if (File.Exists(imagePath))
{
CachedImage image;
string key = Path.GetFileNameWithoutExtension(cachedImagePath);
PersistantDictionary.Instance.TryGetValue(key, out image);
FileInfo imageFileInfo = new FileInfo(imagePath);
return image != null && imageFileInfo.LastWriteTimeUtc.Equals(image.LastWriteTimeUtc);
if (PersistantDictionary.Instance.TryGetValue(key, out cachedImage))
{
if (!imageFileInfo.LastWriteTimeUtc.Equals(cachedImage.LastWriteTimeUtc))
{
if (PersistantDictionary.Instance.TryRemove(key, out cachedImage))
{
isUpdated = true;
}
}
}
}
return true;
return isUpdated;
}
/// <summary>
@ -198,99 +192,124 @@ namespace ImageProcessor.Web.Caching
/// </returns>
internal static DateTime SetCachedLastWriteTime(string imagePath, string cachedImagePath)
{
lock (SyncRoot)
if (File.Exists(imagePath) && File.Exists(cachedImagePath))
{
if (File.Exists(imagePath) && File.Exists(cachedImagePath))
{
DateTime dateTime = File.GetLastWriteTimeUtc(imagePath);
File.SetLastWriteTimeUtc(cachedImagePath, dateTime);
return dateTime;
}
DateTime dateTime = File.GetLastWriteTimeUtc(imagePath);
File.SetLastWriteTimeUtc(cachedImagePath, dateTime);
return dateTime;
}
return DateTime.MinValue;
return DateTime.MinValue.ToUniversalTime();
}
/// <summary>
/// Purges any files from the file-system cache in the given folders.
/// </summary>
private static void PurgeFolders()
internal static void PurgeFolders()
{
// Group each cache folder and clear any expired items or any that exeed
// the maximum allowable count.
Regex searchTerm = new Regex(@"(jpeg|png|bmp|gif)");
var list = PersistantDictionary.Instance.ToList()
.GroupBy(x => searchTerm.Match(x.Value.Path))
.Select(y => new
{
Path = y.Key,
Expires = y.Select(z => z.Value.ExpiresUtc),
Count = y.Sum(z => z.Key.Count())
})
.AsEnumerable();
foreach (var path in list)
{
}
var groups = PersistantDictionary.Instance.ToList()
.GroupBy(x => searchTerm.Match(x.Value.Path).Value)
.Where(g => g.Count() > MaxFilesCount);
//.Where(g => g.Count() > MaxFilesCount
// || g.Select(a => a.Value.ExpiresUtc < DateTime.UtcNow.AddDays(-MaxFileCachedDuration)).Count() > 0);
string folder = HostingEnvironment.MapPath(CachePath);
if (folder != null)
foreach (var group in groups)
{
DirectoryInfo directoryInfo = new DirectoryInfo(folder);
int groupCount = group.Count();
if (directoryInfo.Exists)
foreach (KeyValuePair<string, CachedImage> pair in group.OrderBy(x => x.Value.ExpiresUtc))
{
List<DirectoryInfo> directoryInfos = directoryInfo
.EnumerateDirectories("*", SearchOption.TopDirectoryOnly)
.ToList();
// If the group count is equal to the max count minus 1 then we know we
// are counting down from a full directory not simply clearing out
// expired items.
if (groupCount == MaxFilesCount - 1)
{
break;
}
// Delete each CachedImage.
try
{
FileInfo fileInfo = new FileInfo(pair.Value.Path);
// Remove from the cache.
string key = Path.GetFileNameWithoutExtension(fileInfo.Name);
CachedImage cachedImage;
Parallel.ForEach(
directoryInfos,
subDirectoryInfo =>
if (PersistantDictionary.Instance.TryRemove(key, out cachedImage))
{
// Get all the files in the cache ordered by LastAccessTime - oldest first.
List<FileInfo> fileInfos = subDirectoryInfo.EnumerateFiles("*", SearchOption.TopDirectoryOnly)
.OrderBy(x => x.LastAccessTimeUtc).ToList();
int counter = fileInfos.Count;
Parallel.ForEach(
fileInfos,
fileInfo =>
{
// Delete the file if we are nearing our limit buffer.
if (counter >= MaxFilesCount || fileInfo.LastAccessTimeUtc < DateTime.UtcNow.AddDays(-MaxFileCachedDuration))
{
lock (SyncRoot)
{
try
{
// Remove from the cache.
string key = Path.GetFileNameWithoutExtension(fileInfo.Name);
CachedImage cachedImage;
if (PersistantDictionary.Instance.TryGetValue(key, out cachedImage))
{
if (PersistantDictionary.Instance.TryRemove(key, out cachedImage))
{
fileInfo.Delete();
counter -= 1;
}
}
}
catch (IOException)
{
// Do Nothing, skip to the next.
// TODO: Should we handle this?
}
}
}
});
});
fileInfo.Delete();
groupCount -= 1;
}
}
catch (Exception)
{
// Do Nothing, skip to the next.
// TODO: Should we handle this?
continue;
}
}
}
//string folder = HostingEnvironment.MapPath(CachePath);
//if (folder != null)
//{
// DirectoryInfo directoryInfo = new DirectoryInfo(folder);
// if (directoryInfo.Exists)
// {
// List<DirectoryInfo> directoryInfos = directoryInfo
// .EnumerateDirectories("*", SearchOption.TopDirectoryOnly)
// .ToList();
// Parallel.ForEach(
// directoryInfos,
// subDirectoryInfo =>
// {
// // Get all the files in the cache ordered by LastAccessTime - oldest first.
// List<FileInfo> fileInfos = subDirectoryInfo.EnumerateFiles("*", SearchOption.TopDirectoryOnly)
// .OrderBy(x => x.LastAccessTimeUtc).ToList();
// int counter = fileInfos.Count;
// Parallel.ForEach(
// fileInfos,
// fileInfo =>
// {
// // Delete the file if we are nearing our limit buffer.
// if (counter >= MaxFilesCount || fileInfo.LastAccessTimeUtc < DateTime.UtcNow.AddDays(-MaxFileCachedDuration))
// {
// lock (SyncRoot)
// {
// try
// {
// // Remove from the cache.
// string key = Path.GetFileNameWithoutExtension(fileInfo.Name);
// CachedImage cachedImage;
// if (PersistantDictionary.Instance.TryGetValue(key, out cachedImage))
// {
// if (PersistantDictionary.Instance.TryRemove(key, out cachedImage))
// {
// fileInfo.Delete();
// counter -= 1;
// }
// }
// }
// catch (IOException)
// {
// // Do Nothing, skip to the next.
// // TODO: Should we handle this?
// }
// }
// }
// });
// });
// }
//}
}
/// <summary>

4
src/ImageProcessor.Web/Caching/SQLContext.cs
View File

@ -72,7 +72,7 @@ namespace ImageProcessor.Web.Caching
LastWriteTimeUtc TEXT,
ExpiresUtc TEXT,
PRIMARY KEY (Key),
UNIQUE (Value));";
UNIQUE (Path));";
command.ExecuteNonQuery();
}
@ -198,7 +198,7 @@ namespace ImageProcessor.Web.Caching
CachedImage image = new CachedImage(
reader["Path"].ToString(),
DateTime.Parse(reader["LastWriteTimeUtc"].ToString()).ToUniversalTime(),
DateTime.Parse(reader["LastWriteTimeUtc"].ToString()).ToUniversalTime());
DateTime.Parse(reader["ExpiresUtc"].ToString()).ToUniversalTime());
dictionary.Add(key, image);
}

72
src/ImageProcessor.Web/HttpModules/ImageProcessingModule.cs
View File

@ -37,14 +37,9 @@ namespace ImageProcessor.Web.HttpModules
private static readonly string RemotePrefix = ImageProcessorConfig.Instance.RemotePrefix;
/// <summary>
/// Whether this is the first run of the handler.
/// The object to lock against.
/// </summary>
private static bool isFirstRun = true;
/// <summary>
/// A counter for keeping track of how many images have been added to the cache.
/// </summary>
private static int cachedImageCounter;
private static readonly object SyncRoot = new object();
#endregion
#region IHttpModule Members
@ -106,16 +101,6 @@ namespace ImageProcessor.Web.HttpModules
if (ImageUtils.IsValidImageExtension(path) && !string.IsNullOrWhiteSpace(queryString))
{
// Check to see if this is the first run and if so run the cache controller.
if (isFirstRun)
{
// Trim the cache.
DiskCache.PurgeCachedFolders();
// Disable the controller.
isFirstRun = false;
}
string fullPath = string.Format("{0}?{1}", path, queryString);
string imageName = Path.GetFileName(path);
string cachedPath = DiskCache.GetCachePath(fullPath, imageName);
@ -141,46 +126,51 @@ namespace ImageProcessor.Web.HttpModules
{
if (responseStream != null)
{
responseStream.CopyTo(memoryStream);
lock (SyncRoot)
{
// Trim the cache.
DiskCache.PurgeFolders();
responseStream.CopyTo(memoryStream);
imageFactory.Load(memoryStream)
.AddQueryString(queryString)
.Format(ImageUtils.GetImageFormat(imageName))
.AutoProcess().Save(cachedPath);
// Ensure that the LastWriteTime property of the source and cached file match.
DateTime dateTime = DiskCache.SetCachedLastWriteTime(path, cachedPath);
imageFactory.Load(memoryStream)
.AddQueryString(queryString)
.Format(ImageUtils.GetImageFormat(imageName))
.AutoProcess().Save(cachedPath);
// Add to the cache.
DiskCache.AddImageToCache(cachedPath, dateTime);
}
}
}
}
}
else
{
imageFactory.Load(fullPath).AutoProcess().Save(cachedPath);
}
}
lock (SyncRoot)
{
// Trim the cache.
DiskCache.PurgeFolders();
// Add 1 to the counter
cachedImageCounter += 1;
imageFactory.Load(fullPath).AutoProcess().Save(cachedPath);
// Ensure that the LastWriteTime property of the source and cached file match.
DateTime dateTime = DiskCache.SetCachedLastWriteTime(path, cachedPath);
// Ensure that the LastWriteTime property of the source and cached file match.
DateTime dateTime = DiskCache.SetCachedLastWriteTime(path, cachedPath);
// Add to the cache.
DiskCache.AddImageToCache(cachedPath, dateTime);
// Add to the cache.
DiskCache.AddImageToCache(cachedPath, dateTime);
}
}
}
}
context.Items[CachedResponseTypeKey] = ImageUtils.GetResponseType(imageName).ToDescription();
// The cached file is valid so just rewrite the path.
context.RewritePath(DiskCache.GetVirtualPath(cachedPath, context.Request), false);
// If the number of cached imaged hits the maximum allowed for this session then we clear
// the cache again and reset the counter.
// TODO: There is a potential concurrency issue here but collision probability is very low
// it would be nice to nail it though.
if (cachedImageCounter >= DiskCache.MaxRunsBeforeCacheClear)
{
DiskCache.PurgeCachedFolders();
cachedImageCounter = 0;
}
}
}
}

6
src/ImageProcessor.Web/ImageFactoryExtensions.cs
View File

@ -39,8 +39,8 @@ namespace ImageProcessor.Web
if (factory.ShouldProcess)
{
// TODO: This is going to be a bottleneck for speed. Find a faster way.
lock (SyncRoot)
{
//lock (SyncRoot)
//{
// Get a list of all graphics processors that have parsed and matched the querystring.
List<IGraphicsProcessor> list =
ImageProcessorConfig.Instance.GraphicsProcessors
@ -53,7 +53,7 @@ namespace ImageProcessor.Web
{
factory.Image = graphicsProcessor.ProcessImage(factory);
}
}
//}
}
return factory;

46
src/Test/Test/Web.config
View File

@ -3,45 +3,40 @@
For more information on how to configure your ASP.NET application, please visit
http://go.microsoft.com/fwlink/?LinkId=152368
-->
<configuration>
<!-- Configuration section-handler declaration area. -->
<configSections>
<sectionGroup name="imageProcessor">
<section name="security" requirePermission="false" type="ImageProcessor.Web.Config.ImageSecuritySection, ImageProcessor.Web"/>
<section name="processing" requirePermission="false" type="ImageProcessor.Web.Config.ImageProcessingSection, ImageProcessor.Web" />
<section name="cache" requirePermission="false" type="ImageProcessor.Web.Config.ImageCacheSection, ImageProcessor.Web" />
<section name="processing" requirePermission="false" type="ImageProcessor.Web.Config.ImageProcessingSection, ImageProcessor.Web"/>
<section name="cache" requirePermission="false" type="ImageProcessor.Web.Config.ImageCacheSection, ImageProcessor.Web"/>
</sectionGroup>
</configSections>
<appSettings>
<add key="webpages:Version" value="1.0.0.0"/>
<add key="ClientValidationEnabled" value="true"/>
<add key="UnobtrusiveJavaScriptEnabled" value="true"/>
</appSettings>
<system.web>
<compilation debug="false" targetFramework="4.0">
<compilation debug="true" targetFramework="4.0">
<assemblies>
<add assembly="System.Web.Abstractions, Version=4.0.0.0, Culture=neutral, PublicKeyToken=31BF3856AD364E35" />
<add assembly="System.Web.Helpers, Version=1.0.0.0, Culture=neutral, PublicKeyToken=31BF3856AD364E35" />
<add assembly="System.Web.Routing, Version=4.0.0.0, Culture=neutral, PublicKeyToken=31BF3856AD364E35" />
<add assembly="System.Web.Mvc, Version=3.0.0.0, Culture=neutral, PublicKeyToken=31BF3856AD364E35" />
<add assembly="System.Web.WebPages, Version=1.0.0.0, Culture=neutral, PublicKeyToken=31BF3856AD364E35" />
<add assembly="System.Web.Abstractions, Version=4.0.0.0, Culture=neutral, PublicKeyToken=31BF3856AD364E35"/>
<add assembly="System.Web.Helpers, Version=1.0.0.0, Culture=neutral, PublicKeyToken=31BF3856AD364E35"/>
<add assembly="System.Web.Routing, Version=4.0.0.0, Culture=neutral, PublicKeyToken=31BF3856AD364E35"/>
<add assembly="System.Web.Mvc, Version=3.0.0.0, Culture=neutral, PublicKeyToken=31BF3856AD364E35"/>
<add assembly="System.Web.WebPages, Version=1.0.0.0, Culture=neutral, PublicKeyToken=31BF3856AD364E35"/>
</assemblies>
</compilation>
<authentication mode="Forms">
<forms loginUrl="~/Account/LogOn" timeout="2880" />
<forms loginUrl="~/Account/LogOn" timeout="2880"/>
</authentication>
<pages>
<namespaces>
<add namespace="System.Web.Helpers" />
<add namespace="System.Web.Mvc" />
<add namespace="System.Web.Mvc.Ajax" />
<add namespace="System.Web.Mvc.Html" />
<add namespace="System.Web.Routing" />
<add namespace="System.Web.Helpers"/>
<add namespace="System.Web.Mvc"/>
<add namespace="System.Web.Mvc.Ajax"/>
<add namespace="System.Web.Mvc.Html"/>
<add namespace="System.Web.Routing"/>
<add namespace="System.Web.WebPages"/>
</namespaces>
</pages>
@ -51,31 +46,28 @@
<!--Set the trust level.-->
<!--<trust level="Medium"/>-->
</system.web>
<system.webServer>
<validation validateIntegratedModeConfiguration="false"/>
<modules runAllManagedModulesForAllRequests="true"/>
</system.webServer>
<runtime>
<assemblyBinding xmlns="urn:schemas-microsoft-com:asm.v1">
<dependentAssembly>
<assemblyIdentity name="System.Web.Mvc" publicKeyToken="31bf3856ad364e35" />
<bindingRedirect oldVersion="1.0.0.0-2.0.0.0" newVersion="3.0.0.0" />
<assemblyIdentity name="System.Web.Mvc" publicKeyToken="31bf3856ad364e35"/>
<bindingRedirect oldVersion="1.0.0.0-2.0.0.0" newVersion="3.0.0.0"/>
</dependentAssembly>
</assemblyBinding>
</runtime>
<imageProcessor>
<security allowRemoteDownloads="true" timeout="300000" maxBytes="524288" remotePrefix="/remote.axd">
<whiteList>
<add url="http://images.mymovies.net"/>
</whiteList>
</security>
<cache virtualPath="~/cache" maxDays="28" />
<cache virtualPath="~/cache" maxDays="28"/>
<processing>
<plugins>
<plugin name ="Resize">
<plugin name="Resize">
<settings>
<setting key="MaxWidth" value="1024"/>
<setting key="MaxHeight" value="768"/>
@ -84,4 +76,4 @@
</plugins>
</processing>
</imageProcessor>
</configuration>
</configuration>
Write Preview
Loading…
Cancel
Save