diff --git a/MediaBrowser.Common.Implementations/BaseApplicationHost.cs b/MediaBrowser.Common.Implementations/BaseApplicationHost.cs index 4578da8279..14d7f87c3e 100644 --- a/MediaBrowser.Common.Implementations/BaseApplicationHost.cs +++ b/MediaBrowser.Common.Implementations/BaseApplicationHost.cs @@ -273,7 +273,7 @@ namespace MediaBrowser.Common.Implementations RegisterSingleInstance(TaskManager); - HttpClient = new HttpClientManager.HttpClientManager(ApplicationPaths, Logger); + HttpClient = new HttpClientManager.HttpClientManager(ApplicationPaths, Logger, JsonSerializer); RegisterSingleInstance(HttpClient); NetworkManager = new NetworkManager(); diff --git a/MediaBrowser.Common.Implementations/HttpClientManager/HttpClientManager.cs b/MediaBrowser.Common.Implementations/HttpClientManager/HttpClientManager.cs index 1b1d37ff26..a8e5b3e797 100644 --- a/MediaBrowser.Common.Implementations/HttpClientManager/HttpClientManager.cs +++ b/MediaBrowser.Common.Implementations/HttpClientManager/HttpClientManager.cs @@ -1,14 +1,17 @@ using MediaBrowser.Common.Configuration; +using MediaBrowser.Common.Extensions; using MediaBrowser.Common.IO; using MediaBrowser.Common.Net; using MediaBrowser.Model.Logging; using MediaBrowser.Model.Net; +using MediaBrowser.Model.Serialization; using System; using System.Collections.Concurrent; using System.Collections.Generic; using System.Globalization; using System.IO; using System.Linq; +using System.Net; using System.Net.Cache; using System.Net.Http; using System.Text; @@ -31,13 +34,22 @@ namespace MediaBrowser.Common.Implementations.HttpClientManager /// The _app paths /// private readonly IApplicationPaths _appPaths; - + + private readonly IJsonSerializer _jsonSerializer; + //private readonly FileSystemRepository _cacheRepository; + /// /// Initializes a new instance of the class. /// /// The kernel. /// The logger. - public HttpClientManager(IApplicationPaths appPaths, ILogger logger) + /// The json serializer. + /// + /// appPaths + /// or + /// logger + /// + public HttpClientManager(IApplicationPaths appPaths, ILogger logger, IJsonSerializer jsonSerializer) { if (appPaths == null) { @@ -47,9 +59,12 @@ namespace MediaBrowser.Common.Implementations.HttpClientManager { throw new ArgumentNullException("logger"); } - + _logger = logger; + _jsonSerializer = jsonSerializer; _appPaths = appPaths; + + //_cacheRepository = new FileSystemRepository(Path.Combine(_appPaths.CachePath, "http")); } /// @@ -77,8 +92,7 @@ namespace MediaBrowser.Common.Implementations.HttpClientManager { var handler = new WebRequestHandler { - //AutomaticDecompression = DecompressionMethods.Deflate, - CachePolicy = new RequestCachePolicy(RequestCacheLevel.Revalidate) + CachePolicy = new RequestCachePolicy(RequestCacheLevel.BypassCache) }; client = new HttpClient(handler); @@ -102,10 +116,42 @@ namespace MediaBrowser.Common.Implementations.HttpClientManager { ValidateParams(url, cancellationToken); + //var urlHash = url.GetMD5().ToString(); + //var infoPath = _cacheRepository.GetResourcePath(urlHash + ".js"); + //var responsePath = _cacheRepository.GetResourcePath(urlHash + ".dat"); + + //HttpResponseInfo cachedInfo = null; + + //try + //{ + // cachedInfo = _jsonSerializer.DeserializeFromFile(infoPath); + //} + //catch (FileNotFoundException) + //{ + + //} + + //if (cachedInfo != null && !cachedInfo.MustRevalidate && cachedInfo.Expires.HasValue && cachedInfo.Expires.Value > DateTime.UtcNow) + //{ + // return GetCachedResponse(responsePath); + //} + cancellationToken.ThrowIfCancellationRequested(); var message = new HttpRequestMessage(HttpMethod.Get, url); + //if (cachedInfo != null) + //{ + // if (!string.IsNullOrEmpty(cachedInfo.Etag)) + // { + // message.Headers.Add("If-None-Match", cachedInfo.Etag); + // } + // else if (cachedInfo.LastModified.HasValue) + // { + // message.Headers.IfModifiedSince = new DateTimeOffset(cachedInfo.LastModified.Value); + // } + //} + if (resourcePool != null) { await resourcePool.WaitAsync(cancellationToken).ConfigureAwait(false); @@ -123,6 +169,20 @@ namespace MediaBrowser.Common.Implementations.HttpClientManager cancellationToken.ThrowIfCancellationRequested(); + //cachedInfo = UpdateInfoCache(cachedInfo, url, infoPath, response); + + //if (response.StatusCode == HttpStatusCode.NotModified) + //{ + // return GetCachedResponse(responsePath); + //} + + //if (!string.IsNullOrEmpty(cachedInfo.Etag) || cachedInfo.LastModified.HasValue || (cachedInfo.Expires.HasValue && cachedInfo.Expires.Value > DateTime.UtcNow)) + //{ + // await UpdateResponseCache(response, responsePath).ConfigureAwait(false); + + // return GetCachedResponse(responsePath); + //} + return await response.Content.ReadAsStreamAsync().ConfigureAwait(false); } } @@ -150,7 +210,108 @@ namespace MediaBrowser.Common.Implementations.HttpClientManager } } } - + + /// + /// Gets the cached response. + /// + /// The response path. + /// Stream. + private Stream GetCachedResponse(string responsePath) + { + return File.OpenRead(responsePath); + } + + /// + /// Updates the cache. + /// + /// The cached info. + /// The URL. + /// The path. + /// The response. + private HttpResponseInfo UpdateInfoCache(HttpResponseInfo cachedInfo, string url, string path, HttpResponseMessage response) + { + var fileExists = true; + + if (cachedInfo == null) + { + cachedInfo = new HttpResponseInfo(); + fileExists = false; + } + + cachedInfo.Url = url; + + var etag = response.Headers.ETag; + if (etag != null) + { + cachedInfo.Etag = etag.Tag; + } + + var modified = response.Content.Headers.LastModified; + + if (modified.HasValue) + { + cachedInfo.LastModified = modified.Value.UtcDateTime; + } + else if (response.Headers.Age.HasValue) + { + cachedInfo.LastModified = DateTime.UtcNow.Subtract(response.Headers.Age.Value); + } + + var expires = response.Content.Headers.Expires; + + if (expires.HasValue) + { + cachedInfo.Expires = expires.Value.UtcDateTime; + } + else + { + var cacheControl = response.Headers.CacheControl; + + if (cacheControl != null) + { + if (cacheControl.MaxAge.HasValue) + { + var baseline = cachedInfo.LastModified ?? DateTime.UtcNow; + cachedInfo.Expires = baseline.Add(cacheControl.MaxAge.Value); + } + + cachedInfo.MustRevalidate = cacheControl.MustRevalidate; + } + } + + if (string.IsNullOrEmpty(cachedInfo.Etag) && !cachedInfo.Expires.HasValue && !cachedInfo.LastModified.HasValue) + { + // Nothing to cache + if (fileExists) + { + File.Delete(path); + } + } + else + { + _jsonSerializer.SerializeToFile(cachedInfo, path); + } + + return cachedInfo; + } + + /// + /// Updates the response cache. + /// + /// The response. + /// The path. + /// Task. + private async Task UpdateResponseCache(HttpResponseMessage response, string path) + { + using (var stream = await response.Content.ReadAsStreamAsync().ConfigureAwait(false)) + { + using (var fs = new FileStream(path, FileMode.Create, FileAccess.Write, FileShare.Read, StreamDefaults.DefaultFileStreamBufferSize, FileOptions.Asynchronous)) + { + await stream.CopyToAsync(fs).ConfigureAwait(false); + } + } + } + /// /// Performs a POST request /// @@ -259,10 +420,9 @@ namespace MediaBrowser.Common.Implementations.HttpClientManager options.CancellationToken.ThrowIfCancellationRequested(); - IEnumerable lengthValues; + var contentLength = GetContentLength(response); - if (!response.Headers.TryGetValues("content-length", out lengthValues) && - !response.Content.Headers.TryGetValues("content-length", out lengthValues)) + if (!contentLength.HasValue) { // We're not able to track progress using (var stream = await response.Content.ReadAsStreamAsync().ConfigureAwait(false)) @@ -275,9 +435,7 @@ namespace MediaBrowser.Common.Implementations.HttpClientManager } else { - var length = long.Parse(string.Join(string.Empty, lengthValues.ToArray()), UsCulture); - - using (var stream = ProgressStream.CreateReadProgressStream(await response.Content.ReadAsStreamAsync().ConfigureAwait(false), options.Progress.Report, length)) + using (var stream = ProgressStream.CreateReadProgressStream(await response.Content.ReadAsStreamAsync().ConfigureAwait(false), options.Progress.Report, contentLength.Value)) { using (var fs = new FileStream(tempFile, FileMode.Create, FileAccess.Write, FileShare.Read, StreamDefaults.DefaultFileStreamBufferSize, FileOptions.Asynchronous)) { @@ -306,6 +464,23 @@ namespace MediaBrowser.Common.Implementations.HttpClientManager return tempFile; } + /// + /// Gets the length of the content. + /// + /// The response. + /// System.Nullable{System.Int64}. + private long? GetContentLength(HttpResponseMessage response) + { + IEnumerable lengthValues; + + if (!response.Headers.TryGetValues("content-length", out lengthValues) && !response.Content.Headers.TryGetValues("content-length", out lengthValues)) + { + return null; + } + + return long.Parse(string.Join(string.Empty, lengthValues.ToArray()), UsCulture); + } + protected static readonly CultureInfo UsCulture = new CultureInfo("en-US"); /// @@ -348,41 +523,6 @@ namespace MediaBrowser.Common.Implementations.HttpClientManager throw ex; } - - /// - /// Downloads the contents of a given url into a MemoryStream - /// - /// The URL. - /// The resource pool. - /// The cancellation token. - /// Task{MemoryStream}. - /// - public async Task GetMemoryStream(string url, SemaphoreSlim resourcePool, CancellationToken cancellationToken) - { - ValidateParams(url, cancellationToken); - - cancellationToken.ThrowIfCancellationRequested(); - - _logger.Info("HttpClientManager.GetMemoryStream url: {0}", url); - - var ms = new MemoryStream(); - - try - { - using (var stream = await Get(url, resourcePool, cancellationToken).ConfigureAwait(false)) - { - await stream.CopyToAsync(ms, StreamDefaults.DefaultCopyToBufferSize, cancellationToken).ConfigureAwait(false); - } - - return ms; - } - catch - { - ms.Dispose(); - - throw; - } - } /// /// Validates the params. @@ -499,16 +639,5 @@ namespace MediaBrowser.Common.Implementations.HttpClientManager { return Post(url, postData, null, cancellationToken); } - - /// - /// Gets the memory stream. - /// - /// The URL. - /// The cancellation token. - /// Task{MemoryStream}. - public Task GetMemoryStream(string url, CancellationToken cancellationToken) - { - return GetMemoryStream(url, null, cancellationToken); - } } } diff --git a/MediaBrowser.Common.Implementations/HttpClientManager/HttpResponseInfo.cs b/MediaBrowser.Common.Implementations/HttpClientManager/HttpResponseInfo.cs new file mode 100644 index 0000000000..240e99d796 --- /dev/null +++ b/MediaBrowser.Common.Implementations/HttpClientManager/HttpResponseInfo.cs @@ -0,0 +1,40 @@ +using System; + +namespace MediaBrowser.Common.Implementations.HttpClientManager +{ + /// + /// Class HttpResponseOutput + /// + public class HttpResponseInfo + { + /// + /// Gets or sets the URL. + /// + /// The URL. + public string Url { get; set; } + + /// + /// Gets or sets the etag. + /// + /// The etag. + public string Etag { get; set; } + + /// + /// Gets or sets the last modified. + /// + /// The last modified. + public DateTime? LastModified { get; set; } + + /// + /// Gets or sets the expires. + /// + /// The expires. + public DateTime? Expires { get; set; } + + /// + /// Gets or sets a value indicating whether [must revalidate]. + /// + /// true if [must revalidate]; otherwise, false. + public bool MustRevalidate { get; set; } + } +} diff --git a/MediaBrowser.Common.Implementations/MediaBrowser.Common.Implementations.csproj b/MediaBrowser.Common.Implementations/MediaBrowser.Common.Implementations.csproj index 2fe309ceeb..1f5c93988a 100644 --- a/MediaBrowser.Common.Implementations/MediaBrowser.Common.Implementations.csproj +++ b/MediaBrowser.Common.Implementations/MediaBrowser.Common.Implementations.csproj @@ -65,6 +65,7 @@ + diff --git a/MediaBrowser.Common/Net/IHttpClient.cs b/MediaBrowser.Common/Net/IHttpClient.cs index f443341a0e..87f9b5d712 100644 --- a/MediaBrowser.Common/Net/IHttpClient.cs +++ b/MediaBrowser.Common/Net/IHttpClient.cs @@ -58,23 +58,5 @@ namespace MediaBrowser.Common.Net /// progress /// Task GetTempFile(HttpRequestOptions options); - - /// - /// Downloads the contents of a given url into a MemoryStream - /// - /// The URL. - /// The resource pool. - /// The cancellation token. - /// Task{MemoryStream}. - /// - Task GetMemoryStream(string url, SemaphoreSlim resourcePool, CancellationToken cancellationToken); - - /// - /// Gets the memory stream. - /// - /// The URL. - /// The cancellation token. - /// Task{MemoryStream}. - Task GetMemoryStream(string url, CancellationToken cancellationToken); } } \ No newline at end of file diff --git a/MediaBrowser.Controller/Providers/Movies/TmdbPersonProvider.cs b/MediaBrowser.Controller/Providers/Movies/TmdbPersonProvider.cs index 699c5473b4..583e0bb97b 100644 --- a/MediaBrowser.Controller/Providers/Movies/TmdbPersonProvider.cs +++ b/MediaBrowser.Controller/Providers/Movies/TmdbPersonProvider.cs @@ -335,7 +335,7 @@ namespace MediaBrowser.Controller.Providers.Movies var localPath = Path.Combine(item.MetaLocation, targetName); if (!item.ResolveArgs.ContainsMetaFileByName(targetName)) { - using (var sourceStream = await HttpClient.GetMemoryStream(source, MovieDbProvider.Current.MovieDbResourcePool, cancellationToken).ConfigureAwait(false)) + using (var sourceStream = await HttpClient.Get(source, MovieDbProvider.Current.MovieDbResourcePool, cancellationToken).ConfigureAwait(false)) { await ProviderManager.SaveToLibraryFilesystem(item, localPath, sourceStream, cancellationToken).ConfigureAwait(false); diff --git a/MediaBrowser.Server.Implementations/Providers/ProviderManager.cs b/MediaBrowser.Server.Implementations/Providers/ProviderManager.cs index 97548140c6..e4f57e8d94 100644 --- a/MediaBrowser.Server.Implementations/Providers/ProviderManager.cs +++ b/MediaBrowser.Server.Implementations/Providers/ProviderManager.cs @@ -202,7 +202,7 @@ namespace MediaBrowser.Server.Implementations.Providers { item.ProviderData[_supportedProvidersKey] = supportedProvidersInfo; } - + return result || providersChanged; } @@ -282,7 +282,7 @@ namespace MediaBrowser.Server.Implementations.Providers } catch (ObjectDisposedException) { - + } } @@ -358,7 +358,7 @@ namespace MediaBrowser.Server.Implementations.Providers Path.Combine(item.MetaLocation, targetName) : _remoteImageCache.GetResourcePath(item.GetType().FullName + item.Path.ToLower(), targetName); - var img = await _httpClient.GetMemoryStream(source, resourcePool, cancellationToken).ConfigureAwait(false); + var img = await _httpClient.Get(source, resourcePool, cancellationToken).ConfigureAwait(false); if (ConfigurationManager.Configuration.SaveLocalMeta) // queue to media directories { @@ -422,12 +422,32 @@ namespace MediaBrowser.Server.Implementations.Providers throw new ArgumentNullException(); } - cancellationToken.ThrowIfCancellationRequested(); + if (cancellationToken.IsCancellationRequested) + { + dataToSave.Dispose(); + cancellationToken.ThrowIfCancellationRequested(); + } //Tell the watchers to ignore _directoryWatchers.TemporarilyIgnore(path); - dataToSave.Position = 0; + if (dataToSave.CanSeek) + { + dataToSave.Position = 0; + } + + if (!(dataToSave is MemoryStream || dataToSave is FileStream)) + { + var ms = new MemoryStream(); + + using (var input = dataToSave) + { + await input.CopyToAsync(ms).ConfigureAwait(false); + } + + ms.Position = 0; + dataToSave = ms; + } try {