Browse Source

More tooling in Azure cache.

TODO: Config

Former-commit-id: d4bb114a2a45ee9e1cb5286d700668d9ad243cbd
Former-commit-id: 12c00c04646fefc73f241ee93a7fb358ad32d2e7
af/merge-core
James South 11 years ago
parent
commit
8d32ac8e78
  1. 189
      src/ImageProcessor.Web.AzureBlobCache/AzureBlobCache.cs
  2. 4
      src/ImageProcessor.Web.AzureBlobCache/ImageProcessor.Web.AzureBlobCache.csproj
  3. 58
      src/ImageProcessor.Web/Caching/DiskCache2.cs
  4. 6
      src/ImageProcessor.Web/Caching/IImageCache.cs
  5. 64
      src/ImageProcessor.Web/Caching/ImageCacheBase.cs
  6. 17
      src/ImageProcessor.sln

189
src/ImageProcessor.Web.AzureBlobCache/AzureBlobCache.cs

@ -1,11 +1,17 @@
namespace ImageProcessor.Web.AzureBlobCache
{
using System;
using System.Collections.Generic;
using System.Configuration;
using System.Globalization;
using System.IO;
using System.Linq;
using System.Threading.Tasks;
using System.Web;
using ImageProcessor.Web.Caching;
using ImageProcessor.Web.Extensions;
using ImageProcessor.Web.Helpers;
using Microsoft.WindowsAzure;
using Microsoft.WindowsAzure.Storage;
@ -13,39 +19,72 @@
public class AzureBlobCache : ImageCacheBase
{
private CloudStorageAccount cloudStorageAccount;
/// <summary>
/// The max age.
/// </summary>
private readonly int maxAge;
private CloudStorageAccount cloudCachedStorageAccount;
private CloudStorageAccount cloudSourceStorageAccount;
private CloudBlobClient cloudCachedBlobClient;
private CloudBlobClient cloudBlobClient;
private CloudBlobClient cloudSourceBlobClient;
private CloudBlobContainer cloudBlobContainer;
private CloudBlobContainer cloudCachedBlobContainer;
private CloudBlobContainer cloudSourceBlobContainer;
private string cachedContainerRoot;
/// <summary>
/// The physical cached path.
/// </summary>
private string physicalCachedPath;
public AzureBlobCache(string requestPath, string fullPath, string querystring)
: base(requestPath, fullPath, querystring)
{
// TODO: These should all be in the configuration.
// TODO: Get from configuration.
this.Settings = new Dictionary<string, string>();
// Retrieve storage account from connection string.
this.cloudStorageAccount = CloudStorageAccount.Parse(
CloudConfigurationManager.GetSetting("StorageConnectionString"));
this.maxAge = Convert.ToInt32(this.Settings["MaxAge"]);
// Create the blob client.
this.cloudBlobClient = this.cloudStorageAccount.CreateCloudBlobClient();
// Retrieve storage accounts from connection string.
this.cloudCachedStorageAccount = CloudStorageAccount.Parse(this.Settings["CachedStorageAccount"]);
this.cloudSourceStorageAccount = CloudStorageAccount.Parse(this.Settings["SourceStorageAccount"]);
// Retrieve reference to a previously created container.
this.cloudBlobContainer = this.cloudBlobClient.GetContainerReference("mycontainer");
// Create the blob clients.
this.cloudCachedBlobClient = this.cloudCachedStorageAccount.CreateCloudBlobClient();
this.cloudSourceBlobClient = this.cloudSourceStorageAccount.CreateCloudBlobClient();
// Retrieve references to a previously created containers.
this.cloudCachedBlobContainer = this.cloudCachedBlobClient.GetContainerReference(this.Settings["CachedBlobContainer"]);
this.cloudSourceBlobContainer = this.cloudSourceBlobClient.GetContainerReference(this.Settings["SourceBlobContainer"]);
this.cachedContainerRoot = this.Settings["CachedContainerRoot"];
}
public override int MaxAge
{
get { throw new System.NotImplementedException(); }
get
{
return this.maxAge;
}
}
public override async Task<bool> IsNewOrUpdatedAsync()
{
string cachedFileName = await this.CreateCachedFileName();
// TODO: Generate cache path.
CloudBlockBlob blockBlob = new CloudBlockBlob(new Uri(""));
// Collision rate of about 1 in 10000 for the folder structure.
// That gives us massive scope to store millions of files.
string pathFromKey = string.Join("\\", cachedFileName.ToCharArray().Take(6));
this.CachedPath = Path.Combine(this.cachedContainerRoot, pathFromKey, cachedFileName).Replace(@"\", "/");
ICloudBlob blockBlob = await this.cloudCachedBlobContainer
.GetBlobReferenceFromServerAsync(this.RequestPath);
bool isUpdated = false;
if (!await blockBlob.ExistsAsync())
@ -53,46 +92,126 @@
// Nothing in the cache so we should return true.
isUpdated = true;
}
else if (blockBlob.Properties.LastModified.HasValue)
else
{
// Check to see if the cached image is set to expire.
if (this.IsExpired(blockBlob.Properties.LastModified.Value.UtcDateTime))
// Pull the latest info.
await blockBlob.FetchAttributesAsync();
if (blockBlob.Properties.LastModified.HasValue)
{
isUpdated = true;
// Check to see if the cached image is set to expire.
if (this.IsExpired(blockBlob.Properties.LastModified.Value.UtcDateTime))
{
isUpdated = true;
}
}
}
return isUpdated;
}
public override async Task AddImageToCacheAsync(System.IO.Stream stream)
public override async Task AddImageToCacheAsync(Stream stream)
{
throw new System.NotImplementedException();
CloudBlockBlob blockBlob = this.cloudCachedBlobContainer.GetBlockBlobReference(this.CachedPath);
await blockBlob.UploadFromStreamAsync(stream);
}
public override async Task TrimCacheAsync()
{
throw new System.NotImplementedException();
Uri uri = new Uri(this.CachedPath);
string path = uri.GetLeftPart(UriPartial.Path);
string directory = path.Substring(0, path.LastIndexOf('/'));
string parent = directory.Substring(0, path.LastIndexOf('/'));
BlobContinuationToken continuationToken = null;
CloudBlobDirectory directoryBlob = this.cloudCachedBlobContainer.GetDirectoryReference(parent);
List<IListBlobItem> results = new List<IListBlobItem>();
// Loop through the all the files in a non blocking fashion.
do
{
BlobResultSegment response = await directoryBlob.ListBlobsSegmentedAsync(continuationToken);
continuationToken = response.ContinuationToken;
results.AddRange(response.Results);
}
while (continuationToken != null);
// Now leap through and delete.
foreach (CloudBlockBlob blob in results
.Where((blobItem, type) => blobItem is CloudBlockBlob)
.Cast<CloudBlockBlob>()
.OrderBy(b => b.Properties.LastModified != null ? b.Properties.LastModified.Value.UtcDateTime : new DateTime()))
{
if (blob.Properties.LastModified.HasValue && !this.IsExpired(blob.Properties.LastModified.Value.UtcDateTime))
{
await blob.DeleteAsync();
}
}
}
public override void RewritePath(HttpContext context)
public override async Task<string> CreateCachedFileName()
{
throw new System.NotImplementedException();
string streamHash = string.Empty;
try
{
if (new Uri(this.RequestPath).IsFile)
{
ICloudBlob blockBlob = await this.cloudSourceBlobContainer
.GetBlobReferenceFromServerAsync(this.RequestPath);
if (await blockBlob.ExistsAsync())
{
// Pull the latest info.
await blockBlob.FetchAttributesAsync();
if (blockBlob.Properties.LastModified.HasValue)
{
string creation = blockBlob.Properties.LastModified.Value.UtcDateTime.ToString(CultureInfo.InvariantCulture);
string length = blockBlob.Properties.Length.ToString(CultureInfo.InvariantCulture);
streamHash = string.Format("{0}{1}", creation, length);
}
}
else
{
// Get the hash for the filestream. That way we can ensure that if the image is
// updated but has the same name we will know.
FileInfo imageFileInfo = new FileInfo(this.RequestPath);
if (imageFileInfo.Exists)
{
// Pull the latest info.
imageFileInfo.Refresh();
// Checking the stream itself is far too processor intensive so we make a best guess.
string creation = imageFileInfo.CreationTimeUtc.ToString(CultureInfo.InvariantCulture);
string length = imageFileInfo.Length.ToString(CultureInfo.InvariantCulture);
streamHash = string.Format("{0}{1}", creation, length);
}
}
}
}
catch
{
streamHash = string.Empty;
}
// Use an sha1 hash of the full path including the querystring to create the image name.
// That name can also be used as a key for the cached image and we should be able to use
// The characters of that hash as sub-folders.
string parsedExtension = ImageHelpers.GetExtension(this.FullPath, this.Querystring);
string encryptedName = (streamHash + this.FullPath).ToSHA1Fingerprint();
string cachedFileName = string.Format(
"{0}.{1}",
encryptedName,
!string.IsNullOrWhiteSpace(parsedExtension) ? parsedExtension.Replace(".", string.Empty) : "jpg");
return cachedFileName;
}
/// <summary>
/// Gets a value indicating whether the given images creation date is out with
/// the prescribed limit.
/// </summary>
/// <param name="creationDate">
/// The creation date.
/// </param>
/// <returns>
/// The true if the date is out with the limit, otherwise; false.
/// </returns>
private bool IsExpired(DateTime creationDate)
public override void RewritePath(HttpContext context)
{
return creationDate.AddDays(this.MaxAge) < DateTime.UtcNow.AddDays(-this.MaxAge);
// The cached file is valid so just rewrite the path.
context.RewritePath(this.CachedPath, false);
}
}
}

4
src/ImageProcessor.Web.AzureBlobCache/ImageProcessor.Web.AzureBlobCache.csproj

@ -47,11 +47,15 @@
<Reference Include="Microsoft.WindowsAzure.Configuration">
<HintPath>..\packages\Microsoft.WindowsAzure.ConfigurationManager.1.8.0.0\lib\net35-full\Microsoft.WindowsAzure.Configuration.dll</HintPath>
</Reference>
<Reference Include="Microsoft.WindowsAzure.Storage">
<HintPath>..\packages\WindowsAzure.Storage.4.3.0\lib\net40\Microsoft.WindowsAzure.Storage.dll</HintPath>
</Reference>
<Reference Include="Newtonsoft.Json">
<HintPath>..\packages\Newtonsoft.Json.5.0.8\lib\net45\Newtonsoft.Json.dll</HintPath>
<Private>True</Private>
</Reference>
<Reference Include="System" />
<Reference Include="System.Configuration" />
<Reference Include="System.Core" />
<Reference Include="System.Spatial, Version=5.6.2.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>

58
src/ImageProcessor.Web/Caching/DiskCache2.cs

@ -26,29 +26,33 @@
private const int MaxFilesCount = 100;
/// <summary>
/// The virtual cache path.
/// The max age.
/// </summary>
private static readonly string VirtualCachePath = ImageProcessorConfiguration.Instance.VirtualCachePath;
private readonly int maxAge;
/// <summary>
/// The absolute path to virtual cache path on the server.
/// TODO: Change this so configuration is determined per IImageCache instance.
/// The virtual cache path.
/// </summary>
private static readonly string AbsoluteCachePath = HostingEnvironment.MapPath(VirtualCachePath);
private readonly string virtualCachePath;
/// <summary>
/// The physical cached path.
/// The absolute path to virtual cache path on the server.
/// </summary>
private string physicalCachedPath;
private readonly string absoluteCachePath;
/// <summary>
/// The virtual cached path.
/// The virtual cached path to the cached file.
/// </summary>
private string virtualCachedPath;
private string virtualCachedFilePath;
public DiskCache2(string requestPath, string fullPath, string querystring)
: base(requestPath, fullPath, querystring)
{
// TODO: Get from configuration.
this.Settings = new Dictionary<string, string>();
this.maxAge = Convert.ToInt32(this.Settings["MaxAge"]);
this.virtualCachePath = this.Settings["VirtualCachePath"];
this.absoluteCachePath = HostingEnvironment.MapPath(this.virtualCachePath);
}
/// <summary>
@ -59,7 +63,7 @@
{
get
{
return ImageProcessorConfiguration.Instance.MaxCacheDays;
return this.maxAge;
}
}
@ -68,15 +72,14 @@
string cachedFileName = await this.CreateCachedFileName();
// Collision rate of about 1 in 10000 for the folder structure.
// That gives us massive scope for files.
// That gives us massive scope to store millions of files.
string pathFromKey = string.Join("\\", cachedFileName.ToCharArray().Take(6));
string virtualPathFromKey = pathFromKey.Replace(@"\", "/");
this.physicalCachedPath = Path.Combine(AbsoluteCachePath, pathFromKey, cachedFileName);
this.virtualCachedPath = Path.Combine(VirtualCachePath, virtualPathFromKey, cachedFileName).Replace(@"\", "/");
this.CachedPath = this.physicalCachedPath;
this.CachedPath = Path.Combine(this.absoluteCachePath, pathFromKey, cachedFileName);
this.virtualCachedFilePath = Path.Combine(this.virtualCachePath, virtualPathFromKey, cachedFileName).Replace(@"\", "/");
bool isUpdated = false;
CachedImage cachedImage = CacheIndexer.GetValue(this.physicalCachedPath);
CachedImage cachedImage = CacheIndexer.GetValue(this.CachedPath);
if (cachedImage == null)
{
@ -88,7 +91,7 @@
// Check to see if the cached image is set to expire.
if (this.IsExpired(cachedImage.CreationTimeUtc))
{
CacheIndexer.Remove(this.physicalCachedPath);
CacheIndexer.Remove(this.CachedPath);
isUpdated = true;
}
}
@ -99,13 +102,13 @@
public override async Task AddImageToCacheAsync(Stream stream)
{
// ReSharper disable once AssignNullToNotNullAttribute
DirectoryInfo directoryInfo = new DirectoryInfo(Path.GetDirectoryName(this.physicalCachedPath));
DirectoryInfo directoryInfo = new DirectoryInfo(Path.GetDirectoryName(this.CachedPath));
if (!directoryInfo.Exists)
{
directoryInfo.Create();
}
using (FileStream fileStream = File.Create(this.physicalCachedPath))
using (FileStream fileStream = File.Create(this.CachedPath))
{
await stream.CopyToAsync(fileStream);
}
@ -113,7 +116,7 @@
public override async Task TrimCacheAsync()
{
string directory = Path.GetDirectoryName(this.physicalCachedPath);
string directory = Path.GetDirectoryName(this.CachedPath);
if (directory != null)
{
@ -159,22 +162,7 @@
public override void RewritePath(HttpContext context)
{
// The cached file is valid so just rewrite the path.
context.RewritePath(this.virtualCachedPath, false);
}
/// <summary>
/// Gets a value indicating whether the given images creation date is out with
/// the prescribed limit.
/// </summary>
/// <param name="creationDate">
/// The creation date.
/// </param>
/// <returns>
/// The true if the date is out with the limit, otherwise; false.
/// </returns>
private bool IsExpired(DateTime creationDate)
{
return creationDate.AddDays(this.MaxAge) < DateTime.UtcNow.AddDays(-this.MaxAge);
context.RewritePath(this.virtualCachedFilePath, false);
}
}
}

6
src/ImageProcessor.Web/Caching/IImageCache.cs

@ -1,12 +1,18 @@

namespace ImageProcessor.Web.Caching
{
using System.Collections.Generic;
using System.IO;
using System.Threading.Tasks;
using System.Web;
public interface IImageCache
{
/// <summary>
/// Gets or sets any additional settings required by the cache.
/// </summary>
Dictionary<string, string> Settings { get; }
string CachedPath { get; }
int MaxAge { get; }

64
src/ImageProcessor.Web/Caching/ImageCacheBase.cs

@ -1,6 +1,7 @@
namespace ImageProcessor.Web.Caching
{
using System;
using System.Collections.Generic;
using System.Globalization;
using System.IO;
using System.Reflection;
@ -12,25 +13,25 @@
public abstract class ImageCacheBase : IImageCache
{
/// <summary>
/// The assembly version.
/// </summary>
private static readonly string AssemblyVersion = Assembly.GetExecutingAssembly().GetName().Version.ToString();
/// <summary>
/// The request path for the image.
/// </summary>
private readonly string requestPath;
protected readonly string RequestPath;
/// <summary>
/// The full path for the image.
/// </summary>
private readonly string fullPath;
protected readonly string FullPath;
/// <summary>
/// The querystring containing processing instructions.
/// </summary>
private readonly string querystring;
protected readonly string Querystring;
/// <summary>
/// The assembly version.
/// </summary>
private static readonly string AssemblyVersion = Assembly.GetExecutingAssembly().GetName().Version.ToString();
/// <summary>
/// Initializes a new instance of the <see cref="ImageCacheBase"/> class.
@ -46,11 +47,16 @@
/// </param>
protected ImageCacheBase(string requestPath, string fullPath, string querystring)
{
this.requestPath = requestPath;
this.fullPath = fullPath;
this.querystring = querystring;
this.RequestPath = requestPath;
this.FullPath = fullPath;
this.Querystring = querystring;
}
/// <summary>
/// Gets any additional settings required by the cache.
/// </summary>
public Dictionary<string, string> Settings { get; set; }
public string CachedPath { get; protected set; }
public abstract int MaxAge { get; }
@ -61,17 +67,17 @@
public abstract Task TrimCacheAsync();
public Task<string> CreateCachedFileName()
public virtual Task<string> CreateCachedFileName()
{
string streamHash = string.Empty;
try
{
if (new Uri(this.requestPath).IsFile)
if (new Uri(this.RequestPath).IsFile)
{
// Get the hash for the filestream. That way we can ensure that if the image is
// updated but has the same name we will know.
FileInfo imageFileInfo = new FileInfo(this.requestPath);
FileInfo imageFileInfo = new FileInfo(this.RequestPath);
if (imageFileInfo.Exists)
{
// Pull the latest info.
@ -92,34 +98,32 @@
// Use an sha1 hash of the full path including the querystring to create the image name.
// That name can also be used as a key for the cached image and we should be able to use
// The characters of that hash as sub-folders.
string parsedExtension = ImageHelpers.GetExtension(this.fullPath, this.querystring);
string encryptedName = (streamHash + this.fullPath).ToSHA1Fingerprint();
string parsedExtension = ImageHelpers.GetExtension(this.FullPath, this.Querystring);
string encryptedName = (streamHash + this.FullPath).ToSHA1Fingerprint();
string cachedFileName = string.Format(
"{0}.{1}",
encryptedName,
!string.IsNullOrWhiteSpace(parsedExtension) ? parsedExtension.Replace(".", string.Empty) : "jpg");
this.CachedPath = cachedFileName;
return Task.FromResult(cachedFileName);
}
public abstract void RewritePath(HttpContext context);
public virtual void SetHeaders(HttpContext context, string responseType)
/// <summary>
/// Gets a value indicating whether the given images creation date is out with
/// the prescribed limit.
/// </summary>
/// <param name="creationDate">
/// The creation date.
/// </param>
/// <returns>
/// The true if the date is out with the limit, otherwise; false.
/// </returns>
protected virtual bool IsExpired(DateTime creationDate)
{
HttpResponse response = context.Response;
response.ContentType = responseType;
if (response.Headers["Image-Served-By"] == null)
{
response.AddHeader("Image-Served-By", "ImageProcessor.Web/" + AssemblyVersion);
}
HttpCachePolicy cache = response.Cache;
cache.SetCacheability(HttpCacheability.Public);
cache.VaryByHeaders["Accept-Encoding"] = true;
return creationDate.AddDays(this.MaxAge) < DateTime.UtcNow.AddDays(-this.MaxAge);
}
}
}

17
src/ImageProcessor.sln

@ -36,6 +36,8 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ImageProcessor.Playground",
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ImageProcessor.Web.PostProcessor", "ImageProcessor.Web.PostProcessor\ImageProcessor.Web.PostProcessor.csproj", "{55D08737-7D7E-4995-8892-BD9F944329E6}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ImageProcessor.Web.AzureBlobCache", "ImageProcessor.Web.AzureBlobCache\ImageProcessor.Web.AzureBlobCache.csproj", "{3C805E4C-D679-43F8-8C43-8909CDB4D4D7}"
EndProject
Global
GlobalSection(SolutionConfigurationPlatforms) = preSolution
All|Any CPU = All|Any CPU
@ -198,6 +200,21 @@ Global
{55D08737-7D7E-4995-8892-BD9F944329E6}.Release|Mixed Platforms.ActiveCfg = Release|Any CPU
{55D08737-7D7E-4995-8892-BD9F944329E6}.Release|Mixed Platforms.Build.0 = Release|Any CPU
{55D08737-7D7E-4995-8892-BD9F944329E6}.Release|x86.ActiveCfg = Release|Any CPU
{3C805E4C-D679-43F8-8C43-8909CDB4D4D7}.All|Any CPU.ActiveCfg = Release|Any CPU
{3C805E4C-D679-43F8-8C43-8909CDB4D4D7}.All|Any CPU.Build.0 = Release|Any CPU
{3C805E4C-D679-43F8-8C43-8909CDB4D4D7}.All|Mixed Platforms.ActiveCfg = Release|Any CPU
{3C805E4C-D679-43F8-8C43-8909CDB4D4D7}.All|Mixed Platforms.Build.0 = Release|Any CPU
{3C805E4C-D679-43F8-8C43-8909CDB4D4D7}.All|x86.ActiveCfg = Release|Any CPU
{3C805E4C-D679-43F8-8C43-8909CDB4D4D7}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{3C805E4C-D679-43F8-8C43-8909CDB4D4D7}.Debug|Any CPU.Build.0 = Debug|Any CPU
{3C805E4C-D679-43F8-8C43-8909CDB4D4D7}.Debug|Mixed Platforms.ActiveCfg = Debug|Any CPU
{3C805E4C-D679-43F8-8C43-8909CDB4D4D7}.Debug|Mixed Platforms.Build.0 = Debug|Any CPU
{3C805E4C-D679-43F8-8C43-8909CDB4D4D7}.Debug|x86.ActiveCfg = Debug|Any CPU
{3C805E4C-D679-43F8-8C43-8909CDB4D4D7}.Release|Any CPU.ActiveCfg = Release|Any CPU
{3C805E4C-D679-43F8-8C43-8909CDB4D4D7}.Release|Any CPU.Build.0 = Release|Any CPU
{3C805E4C-D679-43F8-8C43-8909CDB4D4D7}.Release|Mixed Platforms.ActiveCfg = Release|Any CPU
{3C805E4C-D679-43F8-8C43-8909CDB4D4D7}.Release|Mixed Platforms.Build.0 = Release|Any CPU
{3C805E4C-D679-43F8-8C43-8909CDB4D4D7}.Release|x86.ActiveCfg = Release|Any CPU
EndGlobalSection
GlobalSection(SolutionProperties) = preSolution
HideSolutionNode = FALSE

Loading…
Cancel
Save