From a0eeec81a7986b31e6a92c5a42a11ffdffbb48bb Mon Sep 17 00:00:00 2001 From: James South Date: Fri, 22 Mar 2013 16:57:27 +0000 Subject: [PATCH] More refactoring. Former-commit-id: ee57be6d460b0e9bbf4af2b6a205b398ce7ce92d --- .../Caching/{Cacher.cs => Cache.cs} | 236 +++++++----- .../Helpers/AsyncIoExtensions.cs | 64 ++++ .../Helpers/Copy of RemoteFile.cs | 347 ++++++++++++++++++ src/ImageProcessor.Web/Helpers/RemoteFile.cs | 86 +++-- .../Copy of ImageProcessingModule.cs | 342 +++++++++++++++++ .../HttpModules/ImageProcessingModule.cs | 175 +++++---- .../ImageProcessor.Web.csproj | 3 +- 7 files changed, 1060 insertions(+), 193 deletions(-) rename src/ImageProcessor.Web/Caching/{Cacher.cs => Cache.cs} (69%) create mode 100644 src/ImageProcessor.Web/Helpers/AsyncIoExtensions.cs create mode 100644 src/ImageProcessor.Web/Helpers/Copy of RemoteFile.cs create mode 100644 src/ImageProcessor.Web/HttpModules/Copy of ImageProcessingModule.cs diff --git a/src/ImageProcessor.Web/Caching/Cacher.cs b/src/ImageProcessor.Web/Caching/Cache.cs similarity index 69% rename from src/ImageProcessor.Web/Caching/Cacher.cs rename to src/ImageProcessor.Web/Caching/Cache.cs index a4a6f67af..541173412 100644 --- a/src/ImageProcessor.Web/Caching/Cacher.cs +++ b/src/ImageProcessor.Web/Caching/Cache.cs @@ -1,4 +1,9 @@ - +// ----------------------------------------------------------------------- +// +// Copyright (c) James South. +// Dual licensed under the MIT or GPL Version 2 licenses. +// +// ----------------------------------------------------------------------- namespace ImageProcessor.Web.Caching { @@ -8,6 +13,7 @@ namespace ImageProcessor.Web.Caching using System.Collections.Generic; using System.Globalization; using System.IO; + using System.Linq; using System.Text.RegularExpressions; using System.Threading.Tasks; using System.Web; @@ -22,7 +28,6 @@ namespace ImageProcessor.Web.Caching internal sealed class Cache { #region Fields - /// /// The maximum number of days to cache files on the system for. /// @@ -71,23 +76,43 @@ namespace ImageProcessor.Web.Caching /// /// The concurrent dictionary. /// - private ConcurrentDictionary concurrentDictionary = + private static ConcurrentDictionary concurrentDictionary = new ConcurrentDictionary(); - #endregion - - #region Methods + /// + /// The request path for the image. + /// + private string requestPath; - public void Test() - { - Task task = this.CreateDirectoriesAsync(); - if (task.Result) - { + /// + /// The image name + /// + private string imageName; - } + /// + /// Whether the request is for a remote image. + /// + private bool isRemote; + #endregion + #region Constructors + public Cache(string requestPath, string fullPath, string imageName, bool isRemote) + { + this.requestPath = requestPath; + this.imageName = imageName; + this.isRemote = isRemote; + this.CachedPath = this.GetCachePath(fullPath, imageName); } + #endregion + #region Properties + /// + /// Gets the cached path. + /// + internal string CachedPath { get; private set; } + #endregion + + #region Methods #region Internal /// /// Converts an absolute file path @@ -110,48 +135,15 @@ namespace ImageProcessor.Web.Caching "We can only map an absolute back to a relative path if the application path is available."); } - /// - /// Gets the full transformed cached path for the image. - /// The file names are stored as MD5 encrypted versions of the full request path. - /// This should make them unique enough to - /// - /// The original image path. - /// The original image name. - /// The full cached path for the image. - internal string GetCachePath(string imagePath, string imageName) - { - string cachedPath = string.Empty; - - if (AbsoluteCachePath != null) - { - // Use an md5 hash of the full path including the querystring to create the image name. - // That name can also be used as a key for the cached image and we should be able to use - // The first character of that hash as a subfolder. - string parsedExtension = this.ParseExtension(imagePath); - string fallbackExtension = imageName.Substring(imageName.LastIndexOf(".", StringComparison.Ordinal) + 1); - string encryptedName = imagePath.ToMD5Fingerprint(); - string subpath = encryptedName.Substring(0, 1); - - string cachedFileName = string.Format( - "{0}.{1}", - encryptedName, - !string.IsNullOrWhiteSpace(parsedExtension) ? parsedExtension : fallbackExtension); - - cachedPath = Path.Combine(AbsoluteCachePath, subpath, cachedFileName); - } - - return cachedPath; - } - /// /// Creates the cache directories for storing images. /// /// /// The true if the cache directories are created successfully; otherwise, false. /// - internal /*async*/ Task CreateDirectoriesAsync() + internal static /*async*/ Task CreateDirectoriesAsync() { - return this.CreateDirectoriesAsyncTasks().ToTask(); + return CreateDirectoriesAsyncTasks().ToTask(); } /// @@ -166,43 +158,39 @@ namespace ImageProcessor.Web.Caching /// /// The task. /// - internal Task /*async*/ AddImageToCacheAsync(string cachedPath, DateTime lastWriteTimeUtc) + internal Task /*async*/ AddImageToCacheAsync(DateTime lastWriteTimeUtc) { - return this.AddImageToCacheAsyncTask(cachedPath, lastWriteTimeUtc).ToTask(); + return this.AddImageToCacheAsyncTask(lastWriteTimeUtc).ToTask(); } /// - /// Returns a value indicating whether the original file has been updated. + /// Returns a value indicating whether the original file is new or has been updated. /// - /// The original image path. - /// The cached image path. - /// Whether the file is a remote request. /// - /// True if the the original file has been updated; otherwise, false. + /// True if the the original file is new or has been updated; otherwise, false. /// - internal /*async*/ Task IsUpdatedFileAsync(string imagePath, string cachedImagePath, bool isRemote) + internal /*async*/ Task isNewOrUpdatedFileAsync() { - return this.IsUpdatedFileAsyncTask(imagePath, cachedImagePath, isRemote).ToTask(); + return this.isNewOrUpdatedFileAsyncTask().ToTask(); } /// /// Sets the LastWriteTime of the cached file to match the original file. /// - /// - /// The original image path. - /// - /// - /// The cached image path. - /// - /// Whether the file is remote. - /// /// The set to the last write time of the file. /// - internal /*async*/ Task SetCachedLastWriteTimeAsync(string imagePath, string cachedImagePath, bool isRemote) + internal /*async*/ Task SetCachedLastWriteTimeAsync() { - return this.SetCachedLastWriteTimeAsyncTask(imagePath, cachedImagePath, isRemote).ToTask(); + return this.SetCachedLastWriteTimeAsyncTask().ToTask(); } + /// + /// Purges any files from the file-system cache in the given folders. + /// + internal /*async*/ Task TrimCachedFoldersAsync() + { + return this.TrimCachedFoldersAsyncTask().ToTask(); + } #endregion #region Private @@ -212,7 +200,7 @@ namespace ImageProcessor.Web.Caching /// /// The . /// - private IEnumerable CreateDirectoriesAsyncTasks() + private static IEnumerable CreateDirectoriesAsyncTasks() { bool success = true; @@ -242,41 +230,35 @@ namespace ImageProcessor.Web.Caching /// /// Adds an image to the cache. /// - /// - /// The cached path. - /// /// /// The last write time. /// /// /// The . /// - private IEnumerable AddImageToCacheAsyncTask(string cachedPath, DateTime lastWriteTimeUtc) + private IEnumerable AddImageToCacheAsyncTask(DateTime lastWriteTimeUtc) { - string key = Path.GetFileNameWithoutExtension(cachedPath); + string key = Path.GetFileNameWithoutExtension(this.CachedPath); DateTime expires = DateTime.UtcNow.AddDays(MaxFileCachedDuration).ToUniversalTime(); - CachedImage cachedImage = new CachedImage(cachedPath, MaxFileCachedDuration, lastWriteTimeUtc, expires); + CachedImage cachedImage = new CachedImage(this.CachedPath, MaxFileCachedDuration, lastWriteTimeUtc, expires); PersistantDictionary.Instance.Add(key, cachedImage); yield break; } /// - /// Returns a value indicating whether the original file has been updated. + /// Returns a value indicating whether the original file is new or has been updated. /// - /// The original image path. - /// The cached image path. - /// Whether the file is a remote request. /// /// The . /// - private IEnumerable IsUpdatedFileAsyncTask(string imagePath, string cachedImagePath, bool isRemote) + private IEnumerable isNewOrUpdatedFileAsyncTask() { - string key = Path.GetFileNameWithoutExtension(cachedImagePath); + string key = Path.GetFileNameWithoutExtension(this.CachedPath); CachedImage cachedImage; bool isUpdated = false; - if (isRemote) + if (this.isRemote) { if (PersistantDictionary.Instance.TryGetValue(key, out cachedImage)) { @@ -301,7 +283,7 @@ namespace ImageProcessor.Web.Caching // Test now for locally requested files. if (PersistantDictionary.Instance.TryGetValue(key, out cachedImage)) { - FileInfo imageFileInfo = new FileInfo(imagePath); + FileInfo imageFileInfo = new FileInfo(this.requestPath); if (imageFileInfo.Exists) { @@ -340,9 +322,9 @@ namespace ImageProcessor.Web.Caching /// /// The . /// - private IEnumerable SetCachedLastWriteTimeAsyncTask(string imagePath, string cachedImagePath, bool isRemote) + private IEnumerable SetCachedLastWriteTimeAsyncTask() { - FileInfo cachedFileInfo = new FileInfo(cachedImagePath); + FileInfo cachedFileInfo = new FileInfo(this.CachedPath); DateTime lastWriteTime = DateTime.MinValue.ToUniversalTime(); if (isRemote) @@ -354,7 +336,7 @@ namespace ImageProcessor.Web.Caching } else { - FileInfo imageFileInfo = new FileInfo(imagePath); + FileInfo imageFileInfo = new FileInfo(this.requestPath); if (imageFileInfo.Exists && cachedFileInfo.Exists) { @@ -368,6 +350,92 @@ namespace ImageProcessor.Web.Caching yield return TaskEx.FromResult(lastWriteTime); } + /// + /// Purges any files from the file-system cache in the given folders. + /// + /// + /// The . + /// + private IEnumerable TrimCachedFoldersAsyncTask() + { + // Group each cache folder and clear any expired items or any that exeed + // the maximum allowable count. + var groups = PersistantDictionary.Instance.ToList() + .GroupBy(x => SubFolderRegex.Match(x.Value.Path).Value) + .Where(g => g.Count() > MaxFilesCount); + + foreach (var group in groups) + { + int groupCount = group.Count(); + + foreach (KeyValuePair pair in group.OrderBy(x => x.Value.ExpiresUtc)) + { + // If the group count is equal to the max count minus 1 then we know we + // are counting down from a full directory not simply clearing out + // expired items. + if (groupCount == MaxFilesCount - 1) + { + break; + } + + try + { + // Remove from the cache and delete each CachedImage. + FileInfo fileInfo = new FileInfo(pair.Value.Path); + string key = Path.GetFileNameWithoutExtension(fileInfo.Name); + CachedImage cachedImage; + + if (PersistantDictionary.Instance.TryRemove(key, out cachedImage)) + { + fileInfo.Delete(); + groupCount -= 1; + } + } + catch (Exception) + { + // Do Nothing, skip to the next. + // TODO: Should we handle this? + continue; + } + } + } + + yield break; + } + + /// + /// Gets the full transformed cached path for the image. + /// The file names are stored as MD5 encrypted versions of the full request path. + /// This should make them unique enough to + /// + /// The original image path. + /// The original image name. + /// The full cached path for the image. + private string GetCachePath(string fullPath, string imageName) + { + string cachedPath = string.Empty; + + if (AbsoluteCachePath != null) + { + // Use an md5 hash of the full path including the querystring to create the image name. + // That name can also be used as a key for the cached image and we should be able to use + // The first character of that hash as a subfolder. + string parsedExtension = this.ParseExtension(fullPath); + string fallbackExtension = imageName.Substring(imageName.LastIndexOf(".", StringComparison.Ordinal) + 1); + string encryptedName = fullPath.ToMD5Fingerprint(); + string subpath = encryptedName.Substring(0, 1); + + string cachedFileName = string.Format( + "{0}.{1}", + encryptedName, + !string.IsNullOrWhiteSpace(parsedExtension) ? parsedExtension : fallbackExtension); + + cachedPath = Path.Combine(AbsoluteCachePath, subpath, cachedFileName); + } + + return cachedPath; + } + /// /// Returns the correct file extension for the given string input /// diff --git a/src/ImageProcessor.Web/Helpers/AsyncIoExtensions.cs b/src/ImageProcessor.Web/Helpers/AsyncIoExtensions.cs new file mode 100644 index 000000000..977416786 --- /dev/null +++ b/src/ImageProcessor.Web/Helpers/AsyncIoExtensions.cs @@ -0,0 +1,64 @@ +// License: CPOL at http://www.codeproject.com/info/cpol10.aspx +using System.Collections.Generic; +using System.Net; +using System.Threading; +using System.Threading.Tasks; + +namespace System.IO +{ + public static class AsyncIoExtensions + { + public static Task GetRequestStreamAsync(this WebRequest webRequest) + { + return Task.Factory.FromAsync( + webRequest.BeginGetRequestStream, + ar => webRequest.EndGetRequestStream(ar), + null); + } + + public static Task GetResponseAsync(this WebRequest webRequest) + { + return Task.Factory.FromAsync( + webRequest.BeginGetResponse, + ar => webRequest.EndGetResponse(ar), + null); + } + + public static Task ReadAsync(this Stream input, Byte[] buffer, int offset, int count) + { + return Task.Factory.FromAsync( + input.BeginRead, + (Func)input.EndRead, + buffer, offset, count, + null); + } + + public static Task WriteAsync(this Stream input, Byte[] buffer, int offset, int count) + { + return Task.Factory.FromAsync( + input.BeginWrite, + input.EndWrite, + buffer, offset, count, + null); + } + + public static /*async*/ Task CopyToAsync(this Stream input, Stream output, CancellationToken cancellationToken = default(CancellationToken)) + { + return CopyToAsyncTasks(input, output, cancellationToken).ToTask(); + } + private static IEnumerable CopyToAsyncTasks(Stream input, Stream output, CancellationToken cancellationToken) + { + byte[] buffer = new byte[0x1000]; // 4 KiB + while (true) + { + cancellationToken.ThrowIfCancellationRequested(); + var readTask = input.ReadAsync(buffer, 0, buffer.Length); + yield return readTask; + if (readTask.Result == 0) break; + + cancellationToken.ThrowIfCancellationRequested(); + yield return output.WriteAsync(buffer, 0, readTask.Result); + } + } + } +} diff --git a/src/ImageProcessor.Web/Helpers/Copy of RemoteFile.cs b/src/ImageProcessor.Web/Helpers/Copy of RemoteFile.cs new file mode 100644 index 000000000..aa17b0f4e --- /dev/null +++ b/src/ImageProcessor.Web/Helpers/Copy of RemoteFile.cs @@ -0,0 +1,347 @@ +// ----------------------------------------------------------------------- +// +// Copyright (c) James South. +// Dual licensed under the MIT or GPL Version 2 licenses. +// +// ----------------------------------------------------------------------- + +namespace ImageProcessor.Web.Helpers +{ + #region Using + using System; + using System.Diagnostics.Contracts; + using System.Globalization; + using System.IO; + using System.Linq; + using System.Net; + using System.Security; + using System.Text; + using ImageProcessor.Web.Config; + #endregion + + /// + /// Encapsulates methods used to download files from a website address. + /// + /// + /// + /// The purpose of this class is so there's one core way of downloading remote files with url[s] that are from + /// outside users. There's various areas in application where an attacker could supply an external url to the server + /// and tie up resources. + /// + /// For example, the ImageProcessingModule accepts off-server addresses as a path. An attacker could, for instance, pass the url + /// to a file that's a few gigs in size, causing the server to get out-of-memory exceptions or some other errors. An attacker + /// could also use this same method to use one application instance to hammer another site by, again, passing an off-server + /// address of the victims site to the ImageProcessingModule. + /// This class will not throw an exception if the Uri supplied points to a resource local to the running application instance. + /// + /// There shouldn't be any security issues there, as the internal WebRequest instance is still calling it remotely. + /// Any local files that shouldn't be accessed by this won't be allowed by the remote call. + /// + /// Adapted from BlogEngine.Net + /// + internal sealed class RemoteFile + { + #region Fields + /// + /// The white-list of url[s] from which to download remote files. + /// + private static readonly Uri[] RemoteFileWhiteList = ImageProcessorConfig.Instance.RemoteFileWhiteList; + + /// + /// The length of time, in milliseconds, that a remote file download attempt can last before timing out. + /// + private static readonly int TimeoutMilliseconds = ImageProcessorConfig.Instance.Timeout; + + /// + /// The maximum size, in bytes, that a remote file download attempt can download. + /// + private static readonly int MaxBytes = ImageProcessorConfig.Instance.MaxBytes; + + /// + /// Whether to allow remote downloads. + /// + private static readonly bool AllowRemoteDownloads = ImageProcessorConfig.Instance.AllowRemoteDownloads; + + /// + /// Whether this RemoteFile instance is ignoring remote download rules set in the current application + /// instance. + /// + private readonly bool ignoreRemoteDownloadSettings; + + /// + /// The Uri of the remote file being downloaded. + /// + private readonly Uri url; + + /// + /// The maximum allowable download size in bytes. + /// + private readonly int maxDownloadSize; + + /// + /// The length of time, in milliseconds, that a remote file download attempt can last before timing out. + /// + private int timeoutLength; + + /// + /// The WebResponse object used internally for this RemoteFile instance. + /// + private WebRequest webRequest; + #endregion + + #region Constructors + /// + /// Initializes a new instance of the RemoteFile class. + /// + /// The url of the file to be downloaded. + /// + /// If set to , then RemoteFile should ignore the current the applications instance's remote download settings; otherwise,. + /// + internal RemoteFile(Uri filePath, bool ignoreRemoteDownloadSettings) + { + Contract.Requires(filePath != null); + + this.url = filePath; + this.ignoreRemoteDownloadSettings = ignoreRemoteDownloadSettings; + this.timeoutLength = TimeoutMilliseconds; + this.maxDownloadSize = MaxBytes; + } + #endregion + + #region Properties + /// + /// Gets a value indicating whether this RemoteFile instance is ignoring remote download rules set in the + /// current application instance. + /// + /// This should only be set to true if the supplied url is a verified resource. Use at your own risk. + /// + /// + /// + /// if this RemoteFile instance is ignoring remote download rules set in the current + /// application instance; otherwise, . + /// + public bool IgnoreRemoteDownloadSettings + { + get + { + return this.ignoreRemoteDownloadSettings; + } + } + + /// + /// Gets the Uri of the remote file being downloaded. + /// + public Uri Uri + { + get + { + return this.url; + } + } + + /// + /// Gets or sets the length of time, in milliseconds, that a remote file download attempt can + /// last before timing out. + /// + /// + /// This value can only be set if the instance is supposed to ignore the remote download settings set + /// in the current application instance. + /// + /// + /// Set this value to 0 if there should be no timeout. + /// + /// + /// + public int TimeoutLength + { + get + { + return this.IgnoreRemoteDownloadSettings ? this.timeoutLength : TimeoutMilliseconds; + } + + set + { + if (!this.IgnoreRemoteDownloadSettings) + { + throw new SecurityException("Timeout length can not be adjusted on remote files that are abiding by remote download rules"); + } + + if (value < 0) + { + throw new ArgumentOutOfRangeException("TimeoutLength"); + } + + this.timeoutLength = value; + } + } + + /// + /// Gets or sets the maximum download size, in bytes, that a remote file download attempt can be. + /// + /// + /// This value can only be set if the instance is supposed to ignore the remote download settings set + /// in the current application instance. + /// + /// + /// Set this value to 0 if there should be no timeout. + /// + /// + /// + public int MaxDownloadSize + { + get + { + return this.IgnoreRemoteDownloadSettings ? this.maxDownloadSize : MaxBytes; + } + + set + { + if (!this.IgnoreRemoteDownloadSettings) + { + throw new SecurityException("Max Download Size can not be adjusted on remote files that are abiding by remote download rules"); + } + + if (value < 0) + { + throw new ArgumentOutOfRangeException("MaxDownloadSize"); + } + + this.timeoutLength = value; + } + } + #endregion + + #region Methods + #region Public + /// + /// Returns the WebResponse used to download this file. + /// + /// + /// This method is meant for outside users who need specific access to the WebResponse this class + /// generates. They're responsible for disposing of it. + /// + /// + /// + /// The WebResponse used to download this file. + public WebResponse GetWebResponse() + { + WebResponse response = this.GetWebRequest().GetResponse(); + + long contentLength = response.ContentLength; + + // WebResponse.ContentLength doesn't always know the value, it returns -1 in this case. + if (contentLength == -1) + { + // Response headers may still have the Content-Length inside of it. + string headerContentLength = response.Headers["Content-Length"]; + + if (!string.IsNullOrWhiteSpace(headerContentLength)) + { + contentLength = long.Parse(headerContentLength, CultureInfo.InvariantCulture); + } + } + + // We don't need to check the url here since any external urls are available only from the web.config. + if ((this.MaxDownloadSize > 0) && (contentLength > this.MaxDownloadSize)) + { + response.Close(); + throw new SecurityException("An attempt to download a remote file has been halted because the file is larger than allowed."); + } + + return response; + } + + /// + /// Returns the remote file as a String. + /// + /// This returns the resulting stream as a string as passed through a StreamReader. + /// + /// + /// The remote file as a String. + public string GetFileAsString() + { + using (WebResponse response = this.GetWebResponse()) + { + Stream responseStream = response.GetResponseStream(); + + if (responseStream != null) + { + // Pipe the stream to a stream reader with the required encoding format. + using (StreamReader reader = new StreamReader(responseStream, Encoding.UTF8)) + { + return reader.ReadToEnd(); + } + } + + return string.Empty; + } + } + #endregion + + #region Private + /// + /// Performs a check to see whether the application is able to download remote files. + /// + private void CheckCanDownload() + { + if (!this.IgnoreRemoteDownloadSettings && !AllowRemoteDownloads) + { + throw new SecurityException("application is not configured to allow remote file downloads."); + } + } + + /// + /// Creates the WebRequest object used internally for this RemoteFile instance. + /// + /// + /// + /// The WebRequest should not be passed outside of this instance, as it will allow tampering. Anyone + /// that needs more fine control over the downloading process should probably be using the WebRequest + /// class on its own. + /// + /// + private WebRequest GetWebRequest() + { + // Check downloads are allowed. + this.CheckCanDownload(); + + // Check the url is from a whitelisted location. + this.CheckSafeUrlLocation(); + + if (this.webRequest == null) + { + HttpWebRequest request = (HttpWebRequest)WebRequest.Create(this.Uri); + request.Headers["Accept-Encoding"] = "gzip"; + request.Headers["Accept-Language"] = "en-us"; + request.Credentials = CredentialCache.DefaultNetworkCredentials; + request.AutomaticDecompression = DecompressionMethods.GZip; + + if (this.TimeoutLength > 0) + { + request.Timeout = this.TimeoutLength; + } + + this.webRequest = request; + } + + return this.webRequest; + } + + /// + /// Returns a value indicating whether the current url is in a list of safe download locations. + /// + private void CheckSafeUrlLocation() + { + bool validUrl = RemoteFileWhiteList.Any(item => item.Host.ToUpperInvariant().Equals(this.url.Host.ToUpperInvariant())); + + if (!validUrl) + { + throw new SecurityException("application is not configured to allow remote file downloads from this domain."); + } + } + #endregion + #endregion + } +} + + diff --git a/src/ImageProcessor.Web/Helpers/RemoteFile.cs b/src/ImageProcessor.Web/Helpers/RemoteFile.cs index aa17b0f4e..794e6e98a 100644 --- a/src/ImageProcessor.Web/Helpers/RemoteFile.cs +++ b/src/ImageProcessor.Web/Helpers/RemoteFile.cs @@ -17,6 +17,8 @@ namespace ImageProcessor.Web.Helpers using System.Security; using System.Text; using ImageProcessor.Web.Config; + using System.Threading.Tasks; + using System.Collections.Generic; #endregion /// @@ -99,7 +101,10 @@ namespace ImageProcessor.Web.Helpers /// internal RemoteFile(Uri filePath, bool ignoreRemoteDownloadSettings) { - Contract.Requires(filePath != null); + if (filePath == null) + { + throw new ArgumentNullException("filePath"); + } this.url = filePath; this.ignoreRemoteDownloadSettings = ignoreRemoteDownloadSettings; @@ -212,7 +217,7 @@ namespace ImageProcessor.Web.Helpers #endregion #region Methods - #region Public + #region Internal /// /// Returns the WebResponse used to download this file. /// @@ -223,32 +228,12 @@ namespace ImageProcessor.Web.Helpers /// /// /// The WebResponse used to download this file. - public WebResponse GetWebResponse() + /// + /// The . + /// + internal /*async*/ Task GetWebResponseAsync() { - WebResponse response = this.GetWebRequest().GetResponse(); - - long contentLength = response.ContentLength; - - // WebResponse.ContentLength doesn't always know the value, it returns -1 in this case. - if (contentLength == -1) - { - // Response headers may still have the Content-Length inside of it. - string headerContentLength = response.Headers["Content-Length"]; - - if (!string.IsNullOrWhiteSpace(headerContentLength)) - { - contentLength = long.Parse(headerContentLength, CultureInfo.InvariantCulture); - } - } - - // We don't need to check the url here since any external urls are available only from the web.config. - if ((this.MaxDownloadSize > 0) && (contentLength > this.MaxDownloadSize)) - { - response.Close(); - throw new SecurityException("An attempt to download a remote file has been halted because the file is larger than allowed."); - } - - return response; + return this.GetWebResponseAsyncTask().ToTask(); } /// @@ -260,7 +245,9 @@ namespace ImageProcessor.Web.Helpers /// The remote file as a String. public string GetFileAsString() { - using (WebResponse response = this.GetWebResponse()) + Task responseTask = this.GetWebResponseAsync(); + + using (WebResponse response = responseTask.Result) { Stream responseStream = response.GetResponseStream(); @@ -279,6 +266,49 @@ namespace ImageProcessor.Web.Helpers #endregion #region Private + /// + /// Returns the WebResponse used to download this file. + /// + /// + /// This method is meant for outside users who need specific access to the WebResponse this class + /// generates. They're responsible for disposing of it. + /// + /// + /// + /// + /// The . + /// + private IEnumerable GetWebResponseAsyncTask() + { + Task responseTask = this.GetWebRequest().GetResponseAsync(); + yield return responseTask; + + WebResponse response = responseTask.Result; + + long contentLength = response.ContentLength; + + // WebResponse.ContentLength doesn't always know the value, it returns -1 in this case. + if (contentLength == -1) + { + // Response headers may still have the Content-Length inside of it. + string headerContentLength = response.Headers["Content-Length"]; + + if (!string.IsNullOrWhiteSpace(headerContentLength)) + { + contentLength = long.Parse(headerContentLength, CultureInfo.InvariantCulture); + } + } + + // We don't need to check the url here since any external urls are available only from the web.config. + if ((this.MaxDownloadSize > 0) && (contentLength > this.MaxDownloadSize)) + { + response.Close(); + throw new SecurityException("An attempt to download a remote file has been halted because the file is larger than allowed."); + } + + yield return TaskEx.FromResult(response); + } + /// /// Performs a check to see whether the application is able to download remote files. /// diff --git a/src/ImageProcessor.Web/HttpModules/Copy of ImageProcessingModule.cs b/src/ImageProcessor.Web/HttpModules/Copy of ImageProcessingModule.cs new file mode 100644 index 000000000..409181289 --- /dev/null +++ b/src/ImageProcessor.Web/HttpModules/Copy of ImageProcessingModule.cs @@ -0,0 +1,342 @@ +// ----------------------------------------------------------------------- +// +// Copyright (c) James South. +// Dual licensed under the MIT or GPL Version 2 licenses. +// +// ----------------------------------------------------------------------- + +namespace ImageProcessor.Web.HttpModules +{ + #region Using + using System; + using System.IO; + using System.Net; + using System.Reflection; + using System.Web; + using System.Web.Hosting; + using ImageProcessor.Helpers.Extensions; + using ImageProcessor.Imaging; + using ImageProcessor.Web.Caching; + using ImageProcessor.Web.Config; + using ImageProcessor.Web.Helpers; + #endregion + + /// + /// Processes any image requests within the web application. + /// + public class ImageProcessingModule : IHttpModule + { + #region Fields + /// + /// The key for storing the response type of the current image. + /// + private const string CachedResponseTypeKey = "CACHED_IMAGE_RESPONSE_TYPE"; + + /// + /// The value to prefix any remote image requests with to ensure they get captured. + /// + private static readonly string RemotePrefix = ImageProcessorConfig.Instance.RemotePrefix; + + /// + /// The object to lock against. + /// + private static readonly object SyncRoot = new object(); + + /// + /// The assembly version. + /// + private static readonly string AssemblyVersion = Assembly.GetExecutingAssembly().GetName().Version.ToString(); + + /// + /// A value indicating whether the application has started. + /// + private static bool hasModuleInitialized; + #endregion + + /// + /// The delegate void representing the ProcessImage method. + /// + /// + /// the HttpContext object that provides + /// references to the intrinsic server objects + /// + private delegate void ProcessImageDelegate(HttpContext context); + + #region IHttpModule Members + /// + /// Initializes a module and prepares it to handle requests. + /// + /// + /// An that provides + /// access to the methods, properties, and events common to all + /// application objects within an ASP.NET application + /// + public void Init(HttpApplication context) + { + if (!hasModuleInitialized) + { + lock (SyncRoot) + { + if (!hasModuleInitialized) + { + DiskCache.CreateCacheDirectories(); + hasModuleInitialized = true; + } + } + } + + context.AddOnBeginRequestAsync(this.OnBeginAsync, this.OnEndAsync); + context.PreSendRequestHeaders += this.ContextPreSendRequestHeaders; + } + + /// + /// Disposes of the resources (other than memory) used by the module that implements . + /// + public void Dispose() + { + // Nothing to dispose. + } + #endregion + + /// + /// The that starts asynchronous processing + /// of the . + /// + /// The source of the event. + /// + /// An EventArgs that contains + /// the event data. + /// + /// + /// The delegate to call when the asynchronous method call is complete. + /// If the callback is null, the delegate is not called. + /// + /// + /// Any additional data needed to process the request. + /// + /// + /// The status of the asynchronous operation. + /// + private IAsyncResult OnBeginAsync(object sender, EventArgs e, AsyncCallback callBack, object state) + { + HttpContext context = ((HttpApplication)sender).Context; + + ProcessImageDelegate processImage = this.ProcessImage; + + return processImage.BeginInvoke(context, callBack, state); + } + + /// + /// The method that handles asynchronous events such as application events. + /// + /// + /// The that is the result of the + /// operation. + /// + private void OnEndAsync(IAsyncResult result) + { + // Ensure our ProcessImage has completed in the background. + while (!result.IsCompleted) + { + System.Threading.Thread.Sleep(1); + } + } + + /// + /// Occurs just before ASP.NET send HttpHeaders to the client. + /// + /// The source of the event. + /// An EventArgs that contains the event data. + private void ContextPreSendRequestHeaders(object sender, EventArgs e) + { + HttpContext context = ((HttpApplication)sender).Context; + + object responseTypeObject = context.Items[CachedResponseTypeKey]; + + if (responseTypeObject != null) + { + string responseType = (string)responseTypeObject; + + // Set the headers + this.SetHeaders(context, responseType); + + context.Items[CachedResponseTypeKey] = null; + } + } + + #region Private + /// + /// Processes the image. + /// + /// + /// the HttpContext object that provides + /// references to the intrinsic server objects + /// + private void ProcessImage(HttpContext context) + { + // Is this a remote file. + bool isRemote = context.Request.Path.Equals(RemotePrefix, StringComparison.OrdinalIgnoreCase); + string path = string.Empty; + string queryString = string.Empty; + + if (isRemote) + { + // We need to split the querystring to get the actual values we want. + string urlDecode = HttpUtility.UrlDecode(context.Request.QueryString.ToString()); + + if (urlDecode != null) + { + string[] paths = urlDecode.Split('?'); + + path = paths[0]; + + if (paths.Length > 1) + { + queryString = paths[1]; + } + } + } + else + { + path = HostingEnvironment.MapPath(context.Request.Path); + queryString = HttpUtility.UrlDecode(context.Request.QueryString.ToString()); + } + + // Only process requests that pass our sanitizing filter. + if (ImageUtils.IsValidImageExtension(path) && !string.IsNullOrWhiteSpace(queryString)) + { + if (this.FileExists(path, isRemote)) + { + string fullPath = string.Format("{0}?{1}", path, queryString); + string imageName = Path.GetFileName(path); + string cachedPath = DiskCache.GetCachePath(fullPath, imageName); + bool isUpdated = DiskCache.IsUpdatedFile(path, cachedPath, isRemote); + + // Only process if the file has been updated. + if (isUpdated) + { + // Process the image. + using (ImageFactory imageFactory = new ImageFactory()) + { + if (isRemote) + { + Uri uri = new Uri(path); + RemoteFile remoteFile = new RemoteFile(uri, false); + + using (MemoryStream memoryStream = new MemoryStream()) + { + using (Stream responseStream = remoteFile.GetWebResponse().GetResponseStream()) + { + if (responseStream != null) + { + //lock (SyncRoot) + //{ + // Trim the cache. + DiskCache.TrimCachedFolders(); + + responseStream.CopyTo(memoryStream); + + imageFactory.Load(memoryStream) + .AddQueryString(queryString) + .Format(ImageUtils.GetImageFormat(imageName)) + .AutoProcess().Save(cachedPath); + + // Ensure that the LastWriteTime property of the source and cached file match. + DateTime dateTime = DiskCache.SetCachedLastWriteTime(path, cachedPath, true); + + // Add to the cache. + DiskCache.AddImageToCache(cachedPath, dateTime); + //} + } + } + } + } + else + { + //lock (SyncRoot) + //{ + // Trim the cache. + DiskCache.TrimCachedFolders(); + + imageFactory.Load(fullPath).AutoProcess().Save(cachedPath); + + // Ensure that the LastWriteTime property of the source and cached file match. + DateTime dateTime = DiskCache.SetCachedLastWriteTime(path, cachedPath, false); + + // Add to the cache. + DiskCache.AddImageToCache(cachedPath, dateTime); + //} + } + } + } + + context.Items[CachedResponseTypeKey] = ImageUtils.GetResponseType(imageName).ToDescription(); + + // The cached file is valid so just rewrite the path. + context.RewritePath(DiskCache.GetVirtualPath(cachedPath, context.Request), false); + } + } + } + + /// + /// returns a value indicating whether a file exists. + /// + /// The path to the file to check. + /// Whether the file is remote. + /// True if the file exists, otherwise false. + /// If the file is remote the method will always return true. + private bool FileExists(string path, bool isRemote) + { + if (isRemote) + { + return true; + } + + FileInfo fileInfo = new FileInfo(path); + return fileInfo.Exists; + } + + /// + /// This will make the browser and server keep the output + /// in its cache and thereby improve performance. + /// See http://en.wikipedia.org/wiki/HTTP_ETag + /// + /// + /// the HttpContext object that provides + /// references to the intrinsic server objects + /// + /// The HTTP MIME type to to send. + private void SetHeaders(HttpContext context, string responseType) + { + HttpResponse response = context.Response; + + response.ContentType = responseType; + + response.AddHeader("Image-Served-By", "ImageProcessor/" + AssemblyVersion); + + HttpCachePolicy cache = response.Cache; + + cache.VaryByHeaders["Accept-Encoding"] = true; + + int maxDays = DiskCache.MaxFileCachedDuration; + + cache.SetExpires(DateTime.Now.ToUniversalTime().AddDays(maxDays)); + cache.SetMaxAge(new TimeSpan(maxDays, 0, 0, 0)); + cache.SetRevalidation(HttpCacheRevalidation.AllCaches); + + string incomingEtag = context.Request.Headers["If-None-Match"]; + + cache.SetCacheability(HttpCacheability.Public); + + if (incomingEtag == null) + { + return; + } + + response.Clear(); + response.StatusCode = (int)HttpStatusCode.NotModified; + response.SuppressContent = true; + } + #endregion + } +} \ No newline at end of file diff --git a/src/ImageProcessor.Web/HttpModules/ImageProcessingModule.cs b/src/ImageProcessor.Web/HttpModules/ImageProcessingModule.cs index 409181289..431582a76 100644 --- a/src/ImageProcessor.Web/HttpModules/ImageProcessingModule.cs +++ b/src/ImageProcessor.Web/HttpModules/ImageProcessingModule.cs @@ -19,6 +19,8 @@ namespace ImageProcessor.Web.HttpModules using ImageProcessor.Web.Caching; using ImageProcessor.Web.Config; using ImageProcessor.Web.Helpers; + using System.Threading.Tasks; + using System.Collections.Generic; #endregion /// @@ -53,15 +55,6 @@ namespace ImageProcessor.Web.HttpModules private static bool hasModuleInitialized; #endregion - /// - /// The delegate void representing the ProcessImage method. - /// - /// - /// the HttpContext object that provides - /// references to the intrinsic server objects - /// - private delegate void ProcessImageDelegate(HttpContext context); - #region IHttpModule Members /// /// Initializes a module and prepares it to handle requests. @@ -79,13 +72,14 @@ namespace ImageProcessor.Web.HttpModules { if (!hasModuleInitialized) { - DiskCache.CreateCacheDirectories(); + Cache.CreateDirectoriesAsync(); + //DiskCache.CreateCacheDirectories(); hasModuleInitialized = true; } } } - context.AddOnBeginRequestAsync(this.OnBeginAsync, this.OnEndAsync); + context.BeginRequest += this.ContextBeginRequest; context.PreSendRequestHeaders += this.ContextPreSendRequestHeaders; } @@ -99,47 +93,14 @@ namespace ImageProcessor.Web.HttpModules #endregion /// - /// The that starts asynchronous processing - /// of the . + /// Occurs as the first event in the HTTP pipeline chain of execution when ASP.NET responds to a request. /// /// The source of the event. - /// - /// An EventArgs that contains - /// the event data. - /// - /// - /// The delegate to call when the asynchronous method call is complete. - /// If the callback is null, the delegate is not called. - /// - /// - /// Any additional data needed to process the request. - /// - /// - /// The status of the asynchronous operation. - /// - private IAsyncResult OnBeginAsync(object sender, EventArgs e, AsyncCallback callBack, object state) + /// An EventArgs that contains the event data. + private void ContextBeginRequest(object sender, EventArgs e) { HttpContext context = ((HttpApplication)sender).Context; - - ProcessImageDelegate processImage = this.ProcessImage; - - return processImage.BeginInvoke(context, callBack, state); - } - - /// - /// The method that handles asynchronous events such as application events. - /// - /// - /// The that is the result of the - /// operation. - /// - private void OnEndAsync(IAsyncResult result) - { - // Ensure our ProcessImage has completed in the background. - while (!result.IsCompleted) - { - System.Threading.Thread.Sleep(1); - } + this.ProcessImage(context); } /// @@ -173,10 +134,37 @@ namespace ImageProcessor.Web.HttpModules /// references to the intrinsic server objects /// private void ProcessImage(HttpContext context) + { + this.ProcessImageAsync(context); + } + + /// + /// Processes the image. + /// + /// + /// the HttpContext object that provides + /// references to the intrinsic server objects + /// + private /*async*/ Task ProcessImageAsync(HttpContext context) + { + return this.ProcessImageAsyncTask(context).ToTask(); + } + + /// + /// Processes the image. + /// + /// + /// the HttpContext object that provides + /// references to the intrinsic server objects + /// + /// + /// The . + /// + private IEnumerable ProcessImageAsyncTask(HttpContext context) { // Is this a remote file. bool isRemote = context.Request.Path.Equals(RemotePrefix, StringComparison.OrdinalIgnoreCase); - string path = string.Empty; + string requestPath = string.Empty; string queryString = string.Empty; if (isRemote) @@ -188,7 +176,7 @@ namespace ImageProcessor.Web.HttpModules { string[] paths = urlDecode.Split('?'); - path = paths[0]; + requestPath = paths[0]; if (paths.Length > 1) { @@ -198,74 +186,96 @@ namespace ImageProcessor.Web.HttpModules } else { - path = HostingEnvironment.MapPath(context.Request.Path); + requestPath = HostingEnvironment.MapPath(context.Request.Path); queryString = HttpUtility.UrlDecode(context.Request.QueryString.ToString()); } // Only process requests that pass our sanitizing filter. - if (ImageUtils.IsValidImageExtension(path) && !string.IsNullOrWhiteSpace(queryString)) + if (ImageUtils.IsValidImageExtension(requestPath) && !string.IsNullOrWhiteSpace(queryString)) { - if (this.FileExists(path, isRemote)) + if (this.FileExists(requestPath, isRemote)) { - string fullPath = string.Format("{0}?{1}", path, queryString); - string imageName = Path.GetFileName(path); - string cachedPath = DiskCache.GetCachePath(fullPath, imageName); - bool isUpdated = DiskCache.IsUpdatedFile(path, cachedPath, isRemote); + + string fullPath = string.Format("{0}?{1}", requestPath, queryString); + string imageName = Path.GetFileName(requestPath); + + // Create a new cache to help process and cache the request. + Cache cache = new Cache(requestPath, fullPath, imageName, isRemote); + + // Is the file new or updated? + Task isUpdatedTask = cache.isNewOrUpdatedFileAsync(); + yield return isUpdatedTask; + bool isNewOrUpdated = isUpdatedTask.Result; // Only process if the file has been updated. - if (isUpdated) + if (isNewOrUpdated) { // Process the image. using (ImageFactory imageFactory = new ImageFactory()) { if (isRemote) { - Uri uri = new Uri(path); - RemoteFile remoteFile = new RemoteFile(uri, false); + Uri uri = new Uri(requestPath); + + HttpWebRequest webRequest = (HttpWebRequest)WebRequest.Create(uri); + + Task responseTask = webRequest.GetResponseAsync(); + yield return responseTask; + //RemoteFile remoteFile = new RemoteFile(uri, false); + + //Task getWebResponseTask = remoteFile.GetWebResponseAsync(); + //yield return getWebResponseTask; using (MemoryStream memoryStream = new MemoryStream()) { - using (Stream responseStream = remoteFile.GetWebResponse().GetResponseStream()) + using (WebResponse response = responseTask.Result) { - if (responseStream != null) + using (Stream responseStream = response.GetResponseStream()) { - //lock (SyncRoot) - //{ + if (responseStream != null) + { // Trim the cache. - DiskCache.TrimCachedFolders(); + Task trimCachedFoldersTask = cache.TrimCachedFoldersAsync(); + yield return trimCachedFoldersTask; responseStream.CopyTo(memoryStream); imageFactory.Load(memoryStream) .AddQueryString(queryString) .Format(ImageUtils.GetImageFormat(imageName)) - .AutoProcess().Save(cachedPath); + .AutoProcess().Save(cache.CachedPath); // Ensure that the LastWriteTime property of the source and cached file match. - DateTime dateTime = DiskCache.SetCachedLastWriteTime(path, cachedPath, true); + Task setCachedLastWriteTimeTask = cache.SetCachedLastWriteTimeAsync(); + yield return setCachedLastWriteTimeTask; + DateTime dateTime = setCachedLastWriteTimeTask.Result; // Add to the cache. - DiskCache.AddImageToCache(cachedPath, dateTime); - //} + Task addImageToCacheTask = cache.AddImageToCacheAsync(dateTime); + yield return addImageToCacheTask; + } } + } } } else { - //lock (SyncRoot) - //{ - // Trim the cache. - DiskCache.TrimCachedFolders(); + // Trim the cache. + Task trimCachedFoldersTask = cache.TrimCachedFoldersAsync(); + yield return trimCachedFoldersTask; + + imageFactory.Load(fullPath).AutoProcess().Save(cache.CachedPath); - imageFactory.Load(fullPath).AutoProcess().Save(cachedPath); + // Ensure that the LastWriteTime property of the source and cached file match. + Task setCachedLastWriteTimeTask = cache.SetCachedLastWriteTimeAsync(); + yield return setCachedLastWriteTimeTask; + DateTime dateTime = setCachedLastWriteTimeTask.Result; - // Ensure that the LastWriteTime property of the source and cached file match. - DateTime dateTime = DiskCache.SetCachedLastWriteTime(path, cachedPath, false); + // Add to the cache. + Task addImageToCacheTask = cache.AddImageToCacheAsync(dateTime); + yield return addImageToCacheTask; - // Add to the cache. - DiskCache.AddImageToCache(cachedPath, dateTime); - //} } } } @@ -273,11 +283,16 @@ namespace ImageProcessor.Web.HttpModules context.Items[CachedResponseTypeKey] = ImageUtils.GetResponseType(imageName).ToDescription(); // The cached file is valid so just rewrite the path. - context.RewritePath(DiskCache.GetVirtualPath(cachedPath, context.Request), false); + context.RewritePath(cache.GetVirtualPath(cache.CachedPath, context.Request), false); + yield break; + } } + + yield break; } + /// /// returns a value indicating whether a file exists. /// diff --git a/src/ImageProcessor.Web/ImageProcessor.Web.csproj b/src/ImageProcessor.Web/ImageProcessor.Web.csproj index 42983cf84..8ea6cfe26 100644 --- a/src/ImageProcessor.Web/ImageProcessor.Web.csproj +++ b/src/ImageProcessor.Web/ImageProcessor.Web.csproj @@ -87,8 +87,9 @@ - + +