@ -12,11 +12,11 @@ namespace ImageProcessor.Web.Caching
{ {
#region Using
#region Using
using System ; using System ;
using System.Collections.Generic ;
using System.Diagnostics.CodeAnalysis ; using System.Diagnostics.CodeAnalysis ;
using System.Globalization ; using System.Globalization ;
using System.IO ; using System.IO ;
using System.Linq ; using System.Linq ;
using System.Text.RegularExpressions ;
using System.Threading.Tasks ; using System.Threading.Tasks ;
using System.Web ; using System.Web ;
using System.Web.Hosting ; using System.Web.Hosting ;
@ -47,17 +47,7 @@ namespace ImageProcessor.Web.Caching
/// <see cref="http://stackoverflow.com/questions/115882/how-do-you-deal-with-lots-of-small-files"/>
/// <see cref="http://stackoverflow.com/questions/115882/how-do-you-deal-with-lots-of-small-files"/>
/// <see cref="http://stackoverflow.com/questions/1638219/millions-of-small-graphics-files-and-how-to-overcome-slow-file-system-access-on"/>
/// <see cref="http://stackoverflow.com/questions/1638219/millions-of-small-graphics-files-and-how-to-overcome-slow-file-system-access-on"/>
/// </remarks>
/// </remarks>
private const int MaxFilesCount = 1 0 0 ; private const int MaxFilesCount = 5 0 ;
/// <summary>
/// The regular expression to search strings for valid subfolder names.
/// We're specifically not using a shorter regex as we need to be able to iterate through
/// each match group.
/// </summary>
private static readonly Regex SubFolderRegex =
new Regex (
@"(\/([a-z]|[0-9])\/(a|b|c|d|e|f|g|h|i|j|k|l|m|n|o|p|q|r|s|t|u|v|w|x|y|z|0|1|2|3|4|5|6|7|8|9)\/)" ,
RegexOptions . Compiled ) ;
/// <summary>
/// <summary>
/// The absolute path to virtual cache path on the server.
/// The absolute path to virtual cache path on the server.
@ -171,7 +161,7 @@ namespace ImageProcessor.Web.Caching
ExpiresUtc = expires ExpiresUtc = expires
} ; } ;
await Memory Cache. Instance . AddAsync ( key , cachedImage ) ; await CacheIndexer . Instance . AddAsync ( key , cachedImage ) ;
} }
/// <summary>
/// <summary>
@ -188,7 +178,7 @@ namespace ImageProcessor.Web.Caching
if ( this . isRemote ) if ( this . isRemote )
{ {
cachedImage = await Memory Cache. Instance . GetValueAsync ( key ) ; cachedImage = await CacheIndexer . Instance . GetValueAsync ( key ) ;
if ( cachedImage ! = null ) if ( cachedImage ! = null )
{ {
@ -197,7 +187,7 @@ namespace ImageProcessor.Web.Caching
if ( cachedImage . ExpiresUtc < DateTime . UtcNow . AddDays ( - MaxFileCachedDuration ) if ( cachedImage . ExpiresUtc < DateTime . UtcNow . AddDays ( - MaxFileCachedDuration )
| | cachedImage . MaxAge ! = MaxFileCachedDuration ) | | cachedImage . MaxAge ! = MaxFileCachedDuration )
{ {
if ( await Memory Cache. Instance . RemoveAsync ( key ) ) if ( await CacheIndexer . Instance . RemoveAsync ( key ) )
{ {
isUpdated = true ; isUpdated = true ;
} }
@ -212,7 +202,7 @@ namespace ImageProcessor.Web.Caching
else else
{ {
// Test now for locally requested files.
// Test now for locally requested files.
cachedImage = await Memory Cache. Instance . GetValueAsync ( key ) ; cachedImage = await CacheIndexer . Instance . GetValueAsync ( key ) ;
if ( cachedImage ! = null ) if ( cachedImage ! = null )
{ {
@ -226,7 +216,7 @@ namespace ImageProcessor.Web.Caching
| | cachedImage . ExpiresUtc < DateTime . UtcNow . AddDays ( - MaxFileCachedDuration ) | | cachedImage . ExpiresUtc < DateTime . UtcNow . AddDays ( - MaxFileCachedDuration )
| | cachedImage . MaxAge ! = MaxFileCachedDuration ) | | cachedImage . MaxAge ! = MaxFileCachedDuration )
{ {
if ( await Memory Cache. Instance . RemoveAsync ( key ) ) if ( await CacheIndexer . Instance . RemoveAsync ( key ) )
{ {
isUpdated = true ; isUpdated = true ;
} }
@ -254,7 +244,7 @@ namespace ImageProcessor.Web.Caching
string key = Path . GetFileNameWithoutExtension ( this . CachedPath ) ; string key = Path . GetFileNameWithoutExtension ( this . CachedPath ) ;
DateTime dateTime = DateTime . UtcNow ; DateTime dateTime = DateTime . UtcNow ;
CachedImage cachedImage = await Memory Cache. Instance . GetValueAsync ( key ) ; CachedImage cachedImage = await CacheIndexer . Instance . GetValueAsync ( key ) ;
if ( cachedImage ! = null ) if ( cachedImage ! = null )
{ {
@ -277,15 +267,17 @@ namespace ImageProcessor.Web.Caching
} }
/// <summary>
/// <summary>
/// Purges any files from the file-system cache in the given folders .
/// Trims a cached folder ensuring that it does not exceed the maximum file count .
/// </summary>
/// </summary>
/// <param name="path">
/// The path to the folder.
/// </param>
/// <returns>
/// <returns>
/// The <see cref="T:System.Threading.Tasks.Task"/>.
/// The <see cref="T:System.Threading.Tasks.Task"/>.
/// </returns>
/// </returns>
internal async Task TrimCachedFolders Async ( ) internal async Task TrimCachedFolderAsync ( string path )
{ {
// Create Action delegate for TrimCachedFolders.
await TaskHelpers . Run ( ( ) = > this . TrimCachedFolder ( path ) ) ;
await TaskHelpers . Run ( this . TrimCachedFolders ) ;
} }
#endregion
#endregion
@ -325,50 +317,42 @@ namespace ImageProcessor.Web.Caching
} }
/// <summary>
/// <summary>
/// Purges any files from the file-system cache in the given folders .
/// Trims a cached folder ensuring that it does not exceed the maximum file count .
/// </summary>
/// </summary>
private async void TrimCachedFolders ( ) /// <param name="path">
/// The path to the folder.
/// </param>
private async void TrimCachedFolder ( string path )
{ {
// Group each cache folder and clear any expired items or any that exceed
// ReSharper disable once AssignNullToNotNullAttribute
// the maximum allowable count.
DirectoryInfo directoryInfo = new DirectoryInfo ( Path . GetDirectoryName ( path ) ) ;
var groups = SQLContext . GetImagesForCleanup ( ) IEnumerable < FileInfo > files = directoryInfo . EnumerateFiles ( ) . OrderBy ( f = > f . LastWriteTimeUtc ) ;
. GroupBy ( x = > SubFolderRegex . Match ( x . Path ) . Value ) int count = files . Count ( ) ;
. Where ( g = > g . Count ( ) > MaxFilesCount ) ;
foreach ( var group in group s) foreach ( FileInfo fileInfo in file s)
{ {
int groupCount = group . Count ( ) ; try
foreach ( CleanupImage image in group . OrderBy ( x = > x . ExpiresUtc ) )
{ {
// If the group count is equal to the max count minus 1 then we know we
// If the group count is equal to the max count minus 1 then we know we
// are counting down from a full directory not simply clearing out
// have reduced the number of items below the maximum allowed.
// expired items.
if ( count < = MaxFilesCount - 1 )
if ( groupCount < = MaxFilesCount - 1
& & image . ExpiresUtc > = DateTime . UtcNow . AddDays ( - MaxFileCachedDuration ) )
{ {
break ; break ;
} }
try // Remove from the cache and delete each CachedImage.
string key = Path . GetFileNameWithoutExtension ( fileInfo . Name ) ;
if ( await CacheIndexer . Instance . RemoveAsync ( key ) )
{ {
// Remove from the cache and delete each CachedImage.
fileInfo . Delete ( ) ;
FileInfo fileInfo = new FileInfo ( image . Path ) ; count - = 1 ;
string key = Path . GetFileNameWithoutExtension ( fileInfo . Name ) ;
if ( await MemoryCache . Instance . RemoveAsync ( key ) )
{
fileInfo . Delete ( ) ;
groupCount - = 1 ;
}
}
// ReSharper disable EmptyGeneralCatchClause
catch
// ReSharper restore EmptyGeneralCatchClause
{
// Do nothing; skip to the next file.
} }
} }
// ReSharper disable once EmptyGeneralCatchClause
catch
{
// Do nothing; skip to the next file.
}
} }
} }
@ -376,8 +360,8 @@ namespace ImageProcessor.Web.Caching
/// Gets the full transformed cached path for the image.
/// Gets the full transformed cached path for the image.
/// The images are stored in paths that are based upon the sha1 of their full request path
/// The images are stored in paths that are based upon the sha1 of their full request path
/// taking the individual characters of the hash to determine their location.
/// taking the individual characters of the hash to determine their location.
/// This allows us to store 40 folders within 40 folders giving us a total of 3.0223145e+64 potential images.
/// This allows us to store millions of images.
/// Answers on a post card if you can figure out a way to store their details in a db for fast recovery.
/// Answers on a post card if you can figure out a way to store that many details in a db for fast recovery.
/// </summary>
/// </summary>
/// <returns>The full cached path for the image.</returns>
/// <returns>The full cached path for the image.</returns>
[SuppressMessage("StyleCop.CSharp.DocumentationRules", "SA1650:ElementDocumentationMustBeSpelledCorrectly", Justification = "Reviewed. Suppression is OK here.")] [SuppressMessage("StyleCop.CSharp.DocumentationRules", "SA1650:ElementDocumentationMustBeSpelledCorrectly", Justification = "Reviewed. Suppression is OK here.")]
@ -394,7 +378,8 @@ namespace ImageProcessor.Web.Caching
string fallbackExtension = this . imageName . Substring ( this . imageName . LastIndexOf ( "." , StringComparison . Ordinal ) + 1 ) ; string fallbackExtension = this . imageName . Substring ( this . imageName . LastIndexOf ( "." , StringComparison . Ordinal ) + 1 ) ;
string encryptedName = this . fullPath . ToSHA1Fingerprint ( ) ; string encryptedName = this . fullPath . ToSHA1Fingerprint ( ) ;
string pathFromKey = string . Join ( "\\" , encryptedName . ToCharArray ( ) ) ; // Collision rate of about 1 in 1000 for the folder structure.
string pathFromKey = string . Join ( "\\" , encryptedName . ToCharArray ( ) . Take ( 5 ) ) ;
string cachedFileName = string . Format ( string cachedFileName = string . Format (
"{0}.{1}" , "{0}.{1}" ,
James South
12 years ago