diff --git a/src/ImageProcessor.Web.AzureBlobCache/AzureBlobCache.cs b/src/ImageProcessor.Web.AzureBlobCache/AzureBlobCache.cs index 4706fd857..d6beca980 100644 --- a/src/ImageProcessor.Web.AzureBlobCache/AzureBlobCache.cs +++ b/src/ImageProcessor.Web.AzureBlobCache/AzureBlobCache.cs @@ -1,11 +1,17 @@ namespace ImageProcessor.Web.AzureBlobCache { using System; + using System.Collections.Generic; using System.Configuration; + using System.Globalization; + using System.IO; + using System.Linq; using System.Threading.Tasks; using System.Web; using ImageProcessor.Web.Caching; + using ImageProcessor.Web.Extensions; + using ImageProcessor.Web.Helpers; using Microsoft.WindowsAzure; using Microsoft.WindowsAzure.Storage; @@ -13,39 +19,72 @@ public class AzureBlobCache : ImageCacheBase { - private CloudStorageAccount cloudStorageAccount; + /// + /// The max age. + /// + private readonly int maxAge; + + private CloudStorageAccount cloudCachedStorageAccount; + + private CloudStorageAccount cloudSourceStorageAccount; + + private CloudBlobClient cloudCachedBlobClient; - private CloudBlobClient cloudBlobClient; + private CloudBlobClient cloudSourceBlobClient; - private CloudBlobContainer cloudBlobContainer; + private CloudBlobContainer cloudCachedBlobContainer; + + private CloudBlobContainer cloudSourceBlobContainer; + + private string cachedContainerRoot; + + /// + /// The physical cached path. + /// + private string physicalCachedPath; public AzureBlobCache(string requestPath, string fullPath, string querystring) : base(requestPath, fullPath, querystring) { - // TODO: These should all be in the configuration. + // TODO: Get from configuration. + this.Settings = new Dictionary(); - // Retrieve storage account from connection string. - this.cloudStorageAccount = CloudStorageAccount.Parse( - CloudConfigurationManager.GetSetting("StorageConnectionString")); + this.maxAge = Convert.ToInt32(this.Settings["MaxAge"]); - // Create the blob client. - this.cloudBlobClient = this.cloudStorageAccount.CreateCloudBlobClient(); + // Retrieve storage accounts from connection string. + this.cloudCachedStorageAccount = CloudStorageAccount.Parse(this.Settings["CachedStorageAccount"]); + this.cloudSourceStorageAccount = CloudStorageAccount.Parse(this.Settings["SourceStorageAccount"]); - // Retrieve reference to a previously created container. - this.cloudBlobContainer = this.cloudBlobClient.GetContainerReference("mycontainer"); + // Create the blob clients. + this.cloudCachedBlobClient = this.cloudCachedStorageAccount.CreateCloudBlobClient(); + this.cloudSourceBlobClient = this.cloudSourceStorageAccount.CreateCloudBlobClient(); + + // Retrieve references to a previously created containers. + this.cloudCachedBlobContainer = this.cloudCachedBlobClient.GetContainerReference(this.Settings["CachedBlobContainer"]); + this.cloudSourceBlobContainer = this.cloudSourceBlobClient.GetContainerReference(this.Settings["SourceBlobContainer"]); + + this.cachedContainerRoot = this.Settings["CachedContainerRoot"]; } public override int MaxAge { - get { throw new System.NotImplementedException(); } + get + { + return this.maxAge; + } } public override async Task IsNewOrUpdatedAsync() { string cachedFileName = await this.CreateCachedFileName(); - // TODO: Generate cache path. - CloudBlockBlob blockBlob = new CloudBlockBlob(new Uri("")); + // Collision rate of about 1 in 10000 for the folder structure. + // That gives us massive scope to store millions of files. + string pathFromKey = string.Join("\\", cachedFileName.ToCharArray().Take(6)); + this.CachedPath = Path.Combine(this.cachedContainerRoot, pathFromKey, cachedFileName).Replace(@"\", "/"); + + ICloudBlob blockBlob = await this.cloudCachedBlobContainer + .GetBlobReferenceFromServerAsync(this.RequestPath); bool isUpdated = false; if (!await blockBlob.ExistsAsync()) @@ -53,46 +92,126 @@ // Nothing in the cache so we should return true. isUpdated = true; } - else if (blockBlob.Properties.LastModified.HasValue) + else { - // Check to see if the cached image is set to expire. - if (this.IsExpired(blockBlob.Properties.LastModified.Value.UtcDateTime)) + // Pull the latest info. + await blockBlob.FetchAttributesAsync(); + if (blockBlob.Properties.LastModified.HasValue) { - isUpdated = true; + // Check to see if the cached image is set to expire. + if (this.IsExpired(blockBlob.Properties.LastModified.Value.UtcDateTime)) + { + isUpdated = true; + } } } return isUpdated; } - public override async Task AddImageToCacheAsync(System.IO.Stream stream) + public override async Task AddImageToCacheAsync(Stream stream) { - throw new System.NotImplementedException(); + CloudBlockBlob blockBlob = this.cloudCachedBlobContainer.GetBlockBlobReference(this.CachedPath); + await blockBlob.UploadFromStreamAsync(stream); } public override async Task TrimCacheAsync() { - throw new System.NotImplementedException(); + Uri uri = new Uri(this.CachedPath); + string path = uri.GetLeftPart(UriPartial.Path); + string directory = path.Substring(0, path.LastIndexOf('/')); + string parent = directory.Substring(0, path.LastIndexOf('/')); + + BlobContinuationToken continuationToken = null; + CloudBlobDirectory directoryBlob = this.cloudCachedBlobContainer.GetDirectoryReference(parent); + List results = new List(); + + // Loop through the all the files in a non blocking fashion. + do + { + BlobResultSegment response = await directoryBlob.ListBlobsSegmentedAsync(continuationToken); + continuationToken = response.ContinuationToken; + results.AddRange(response.Results); + } + while (continuationToken != null); + + // Now leap through and delete. + foreach (CloudBlockBlob blob in results + .Where((blobItem, type) => blobItem is CloudBlockBlob) + .Cast() + .OrderBy(b => b.Properties.LastModified != null ? b.Properties.LastModified.Value.UtcDateTime : new DateTime())) + { + if (blob.Properties.LastModified.HasValue && !this.IsExpired(blob.Properties.LastModified.Value.UtcDateTime)) + { + await blob.DeleteAsync(); + } + } } - public override void RewritePath(HttpContext context) + public override async Task CreateCachedFileName() { - throw new System.NotImplementedException(); + string streamHash = string.Empty; + + try + { + if (new Uri(this.RequestPath).IsFile) + { + ICloudBlob blockBlob = await this.cloudSourceBlobContainer + .GetBlobReferenceFromServerAsync(this.RequestPath); + + if (await blockBlob.ExistsAsync()) + { + // Pull the latest info. + await blockBlob.FetchAttributesAsync(); + + if (blockBlob.Properties.LastModified.HasValue) + { + string creation = blockBlob.Properties.LastModified.Value.UtcDateTime.ToString(CultureInfo.InvariantCulture); + string length = blockBlob.Properties.Length.ToString(CultureInfo.InvariantCulture); + streamHash = string.Format("{0}{1}", creation, length); + } + } + else + { + // Get the hash for the filestream. That way we can ensure that if the image is + // updated but has the same name we will know. + FileInfo imageFileInfo = new FileInfo(this.RequestPath); + if (imageFileInfo.Exists) + { + // Pull the latest info. + imageFileInfo.Refresh(); + + // Checking the stream itself is far too processor intensive so we make a best guess. + string creation = imageFileInfo.CreationTimeUtc.ToString(CultureInfo.InvariantCulture); + string length = imageFileInfo.Length.ToString(CultureInfo.InvariantCulture); + streamHash = string.Format("{0}{1}", creation, length); + } + } + } + } + catch + { + streamHash = string.Empty; + } + + // Use an sha1 hash of the full path including the querystring to create the image name. + // That name can also be used as a key for the cached image and we should be able to use + // The characters of that hash as sub-folders. + string parsedExtension = ImageHelpers.GetExtension(this.FullPath, this.Querystring); + string encryptedName = (streamHash + this.FullPath).ToSHA1Fingerprint(); + + string cachedFileName = string.Format( + "{0}.{1}", + encryptedName, + !string.IsNullOrWhiteSpace(parsedExtension) ? parsedExtension.Replace(".", string.Empty) : "jpg"); + + return cachedFileName; } - /// - /// Gets a value indicating whether the given images creation date is out with - /// the prescribed limit. - /// - /// - /// The creation date. - /// - /// - /// The true if the date is out with the limit, otherwise; false. - /// - private bool IsExpired(DateTime creationDate) + public override void RewritePath(HttpContext context) { - return creationDate.AddDays(this.MaxAge) < DateTime.UtcNow.AddDays(-this.MaxAge); + // The cached file is valid so just rewrite the path. + context.RewritePath(this.CachedPath, false); } } } diff --git a/src/ImageProcessor.Web.AzureBlobCache/ImageProcessor.Web.AzureBlobCache.csproj b/src/ImageProcessor.Web.AzureBlobCache/ImageProcessor.Web.AzureBlobCache.csproj index 4d30e737d..f2b15b3ab 100644 --- a/src/ImageProcessor.Web.AzureBlobCache/ImageProcessor.Web.AzureBlobCache.csproj +++ b/src/ImageProcessor.Web.AzureBlobCache/ImageProcessor.Web.AzureBlobCache.csproj @@ -47,11 +47,15 @@ ..\packages\Microsoft.WindowsAzure.ConfigurationManager.1.8.0.0\lib\net35-full\Microsoft.WindowsAzure.Configuration.dll + + ..\packages\WindowsAzure.Storage.4.3.0\lib\net40\Microsoft.WindowsAzure.Storage.dll + ..\packages\Newtonsoft.Json.5.0.8\lib\net45\Newtonsoft.Json.dll True + False diff --git a/src/ImageProcessor.Web/Caching/DiskCache2.cs b/src/ImageProcessor.Web/Caching/DiskCache2.cs index 2ea9e34a6..f4e04e7c3 100644 --- a/src/ImageProcessor.Web/Caching/DiskCache2.cs +++ b/src/ImageProcessor.Web/Caching/DiskCache2.cs @@ -26,29 +26,33 @@ private const int MaxFilesCount = 100; /// - /// The virtual cache path. + /// The max age. /// - private static readonly string VirtualCachePath = ImageProcessorConfiguration.Instance.VirtualCachePath; + private readonly int maxAge; /// - /// The absolute path to virtual cache path on the server. - /// TODO: Change this so configuration is determined per IImageCache instance. + /// The virtual cache path. /// - private static readonly string AbsoluteCachePath = HostingEnvironment.MapPath(VirtualCachePath); + private readonly string virtualCachePath; /// - /// The physical cached path. + /// The absolute path to virtual cache path on the server. /// - private string physicalCachedPath; + private readonly string absoluteCachePath; /// - /// The virtual cached path. + /// The virtual cached path to the cached file. /// - private string virtualCachedPath; + private string virtualCachedFilePath; public DiskCache2(string requestPath, string fullPath, string querystring) : base(requestPath, fullPath, querystring) { + // TODO: Get from configuration. + this.Settings = new Dictionary(); + this.maxAge = Convert.ToInt32(this.Settings["MaxAge"]); + this.virtualCachePath = this.Settings["VirtualCachePath"]; + this.absoluteCachePath = HostingEnvironment.MapPath(this.virtualCachePath); } /// @@ -59,7 +63,7 @@ { get { - return ImageProcessorConfiguration.Instance.MaxCacheDays; + return this.maxAge; } } @@ -68,15 +72,14 @@ string cachedFileName = await this.CreateCachedFileName(); // Collision rate of about 1 in 10000 for the folder structure. - // That gives us massive scope for files. + // That gives us massive scope to store millions of files. string pathFromKey = string.Join("\\", cachedFileName.ToCharArray().Take(6)); string virtualPathFromKey = pathFromKey.Replace(@"\", "/"); - this.physicalCachedPath = Path.Combine(AbsoluteCachePath, pathFromKey, cachedFileName); - this.virtualCachedPath = Path.Combine(VirtualCachePath, virtualPathFromKey, cachedFileName).Replace(@"\", "/"); - this.CachedPath = this.physicalCachedPath; + this.CachedPath = Path.Combine(this.absoluteCachePath, pathFromKey, cachedFileName); + this.virtualCachedFilePath = Path.Combine(this.virtualCachePath, virtualPathFromKey, cachedFileName).Replace(@"\", "/"); bool isUpdated = false; - CachedImage cachedImage = CacheIndexer.GetValue(this.physicalCachedPath); + CachedImage cachedImage = CacheIndexer.GetValue(this.CachedPath); if (cachedImage == null) { @@ -88,7 +91,7 @@ // Check to see if the cached image is set to expire. if (this.IsExpired(cachedImage.CreationTimeUtc)) { - CacheIndexer.Remove(this.physicalCachedPath); + CacheIndexer.Remove(this.CachedPath); isUpdated = true; } } @@ -99,13 +102,13 @@ public override async Task AddImageToCacheAsync(Stream stream) { // ReSharper disable once AssignNullToNotNullAttribute - DirectoryInfo directoryInfo = new DirectoryInfo(Path.GetDirectoryName(this.physicalCachedPath)); + DirectoryInfo directoryInfo = new DirectoryInfo(Path.GetDirectoryName(this.CachedPath)); if (!directoryInfo.Exists) { directoryInfo.Create(); } - using (FileStream fileStream = File.Create(this.physicalCachedPath)) + using (FileStream fileStream = File.Create(this.CachedPath)) { await stream.CopyToAsync(fileStream); } @@ -113,7 +116,7 @@ public override async Task TrimCacheAsync() { - string directory = Path.GetDirectoryName(this.physicalCachedPath); + string directory = Path.GetDirectoryName(this.CachedPath); if (directory != null) { @@ -159,22 +162,7 @@ public override void RewritePath(HttpContext context) { // The cached file is valid so just rewrite the path. - context.RewritePath(this.virtualCachedPath, false); - } - - /// - /// Gets a value indicating whether the given images creation date is out with - /// the prescribed limit. - /// - /// - /// The creation date. - /// - /// - /// The true if the date is out with the limit, otherwise; false. - /// - private bool IsExpired(DateTime creationDate) - { - return creationDate.AddDays(this.MaxAge) < DateTime.UtcNow.AddDays(-this.MaxAge); + context.RewritePath(this.virtualCachedFilePath, false); } } } diff --git a/src/ImageProcessor.Web/Caching/IImageCache.cs b/src/ImageProcessor.Web/Caching/IImageCache.cs index 6153c1f8a..d6404ff51 100644 --- a/src/ImageProcessor.Web/Caching/IImageCache.cs +++ b/src/ImageProcessor.Web/Caching/IImageCache.cs @@ -1,12 +1,18 @@  namespace ImageProcessor.Web.Caching { + using System.Collections.Generic; using System.IO; using System.Threading.Tasks; using System.Web; public interface IImageCache { + /// + /// Gets or sets any additional settings required by the cache. + /// + Dictionary Settings { get; } + string CachedPath { get; } int MaxAge { get; } diff --git a/src/ImageProcessor.Web/Caching/ImageCacheBase.cs b/src/ImageProcessor.Web/Caching/ImageCacheBase.cs index c33a03e60..cfb7ea5ce 100644 --- a/src/ImageProcessor.Web/Caching/ImageCacheBase.cs +++ b/src/ImageProcessor.Web/Caching/ImageCacheBase.cs @@ -1,6 +1,7 @@ namespace ImageProcessor.Web.Caching { using System; + using System.Collections.Generic; using System.Globalization; using System.IO; using System.Reflection; @@ -12,25 +13,25 @@ public abstract class ImageCacheBase : IImageCache { - /// - /// The assembly version. - /// - private static readonly string AssemblyVersion = Assembly.GetExecutingAssembly().GetName().Version.ToString(); - /// /// The request path for the image. /// - private readonly string requestPath; + protected readonly string RequestPath; /// /// The full path for the image. /// - private readonly string fullPath; + protected readonly string FullPath; /// /// The querystring containing processing instructions. /// - private readonly string querystring; + protected readonly string Querystring; + + /// + /// The assembly version. + /// + private static readonly string AssemblyVersion = Assembly.GetExecutingAssembly().GetName().Version.ToString(); /// /// Initializes a new instance of the class. @@ -46,11 +47,16 @@ /// protected ImageCacheBase(string requestPath, string fullPath, string querystring) { - this.requestPath = requestPath; - this.fullPath = fullPath; - this.querystring = querystring; + this.RequestPath = requestPath; + this.FullPath = fullPath; + this.Querystring = querystring; } + /// + /// Gets any additional settings required by the cache. + /// + public Dictionary Settings { get; set; } + public string CachedPath { get; protected set; } public abstract int MaxAge { get; } @@ -61,17 +67,17 @@ public abstract Task TrimCacheAsync(); - public Task CreateCachedFileName() + public virtual Task CreateCachedFileName() { string streamHash = string.Empty; try { - if (new Uri(this.requestPath).IsFile) + if (new Uri(this.RequestPath).IsFile) { // Get the hash for the filestream. That way we can ensure that if the image is // updated but has the same name we will know. - FileInfo imageFileInfo = new FileInfo(this.requestPath); + FileInfo imageFileInfo = new FileInfo(this.RequestPath); if (imageFileInfo.Exists) { // Pull the latest info. @@ -92,34 +98,32 @@ // Use an sha1 hash of the full path including the querystring to create the image name. // That name can also be used as a key for the cached image and we should be able to use // The characters of that hash as sub-folders. - string parsedExtension = ImageHelpers.GetExtension(this.fullPath, this.querystring); - string encryptedName = (streamHash + this.fullPath).ToSHA1Fingerprint(); + string parsedExtension = ImageHelpers.GetExtension(this.FullPath, this.Querystring); + string encryptedName = (streamHash + this.FullPath).ToSHA1Fingerprint(); string cachedFileName = string.Format( "{0}.{1}", encryptedName, !string.IsNullOrWhiteSpace(parsedExtension) ? parsedExtension.Replace(".", string.Empty) : "jpg"); - this.CachedPath = cachedFileName; return Task.FromResult(cachedFileName); } public abstract void RewritePath(HttpContext context); - public virtual void SetHeaders(HttpContext context, string responseType) + /// + /// Gets a value indicating whether the given images creation date is out with + /// the prescribed limit. + /// + /// + /// The creation date. + /// + /// + /// The true if the date is out with the limit, otherwise; false. + /// + protected virtual bool IsExpired(DateTime creationDate) { - HttpResponse response = context.Response; - - response.ContentType = responseType; - - if (response.Headers["Image-Served-By"] == null) - { - response.AddHeader("Image-Served-By", "ImageProcessor.Web/" + AssemblyVersion); - } - - HttpCachePolicy cache = response.Cache; - cache.SetCacheability(HttpCacheability.Public); - cache.VaryByHeaders["Accept-Encoding"] = true; + return creationDate.AddDays(this.MaxAge) < DateTime.UtcNow.AddDays(-this.MaxAge); } } } diff --git a/src/ImageProcessor.sln b/src/ImageProcessor.sln index 8caaeabc7..a5b0609f3 100644 --- a/src/ImageProcessor.sln +++ b/src/ImageProcessor.sln @@ -36,6 +36,8 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ImageProcessor.Playground", EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ImageProcessor.Web.PostProcessor", "ImageProcessor.Web.PostProcessor\ImageProcessor.Web.PostProcessor.csproj", "{55D08737-7D7E-4995-8892-BD9F944329E6}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ImageProcessor.Web.AzureBlobCache", "ImageProcessor.Web.AzureBlobCache\ImageProcessor.Web.AzureBlobCache.csproj", "{3C805E4C-D679-43F8-8C43-8909CDB4D4D7}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution All|Any CPU = All|Any CPU @@ -198,6 +200,21 @@ Global {55D08737-7D7E-4995-8892-BD9F944329E6}.Release|Mixed Platforms.ActiveCfg = Release|Any CPU {55D08737-7D7E-4995-8892-BD9F944329E6}.Release|Mixed Platforms.Build.0 = Release|Any CPU {55D08737-7D7E-4995-8892-BD9F944329E6}.Release|x86.ActiveCfg = Release|Any CPU + {3C805E4C-D679-43F8-8C43-8909CDB4D4D7}.All|Any CPU.ActiveCfg = Release|Any CPU + {3C805E4C-D679-43F8-8C43-8909CDB4D4D7}.All|Any CPU.Build.0 = Release|Any CPU + {3C805E4C-D679-43F8-8C43-8909CDB4D4D7}.All|Mixed Platforms.ActiveCfg = Release|Any CPU + {3C805E4C-D679-43F8-8C43-8909CDB4D4D7}.All|Mixed Platforms.Build.0 = Release|Any CPU + {3C805E4C-D679-43F8-8C43-8909CDB4D4D7}.All|x86.ActiveCfg = Release|Any CPU + {3C805E4C-D679-43F8-8C43-8909CDB4D4D7}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {3C805E4C-D679-43F8-8C43-8909CDB4D4D7}.Debug|Any CPU.Build.0 = Debug|Any CPU + {3C805E4C-D679-43F8-8C43-8909CDB4D4D7}.Debug|Mixed Platforms.ActiveCfg = Debug|Any CPU + {3C805E4C-D679-43F8-8C43-8909CDB4D4D7}.Debug|Mixed Platforms.Build.0 = Debug|Any CPU + {3C805E4C-D679-43F8-8C43-8909CDB4D4D7}.Debug|x86.ActiveCfg = Debug|Any CPU + {3C805E4C-D679-43F8-8C43-8909CDB4D4D7}.Release|Any CPU.ActiveCfg = Release|Any CPU + {3C805E4C-D679-43F8-8C43-8909CDB4D4D7}.Release|Any CPU.Build.0 = Release|Any CPU + {3C805E4C-D679-43F8-8C43-8909CDB4D4D7}.Release|Mixed Platforms.ActiveCfg = Release|Any CPU + {3C805E4C-D679-43F8-8C43-8909CDB4D4D7}.Release|Mixed Platforms.Build.0 = Release|Any CPU + {3C805E4C-D679-43F8-8C43-8909CDB4D4D7}.Release|x86.ActiveCfg = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE