diff --git a/MediaBrowser.Controller/Providers/IProviderManager.cs b/MediaBrowser.Controller/Providers/IProviderManager.cs index da46c0bce6..99860bac0d 100644 --- a/MediaBrowser.Controller/Providers/IProviderManager.cs +++ b/MediaBrowser.Controller/Providers/IProviderManager.cs @@ -21,6 +21,8 @@ namespace MediaBrowser.Controller.Providers /// item Task DownloadAndSaveImage(BaseItem item, string source, string targetName, bool saveLocally, SemaphoreSlim resourcePool, CancellationToken cancellationToken); + Task SaveImage(BaseItem item, Stream source, string targetName, bool saveLocally, CancellationToken cancellationToken); + /// /// Saves to library filesystem. /// diff --git a/MediaBrowser.Controller/Providers/Movies/MovieDbImagesProvider.cs b/MediaBrowser.Controller/Providers/Movies/MovieDbImagesProvider.cs index 0b76afbc33..8260a0568f 100644 --- a/MediaBrowser.Controller/Providers/Movies/MovieDbImagesProvider.cs +++ b/MediaBrowser.Controller/Providers/Movies/MovieDbImagesProvider.cs @@ -207,11 +207,10 @@ namespace MediaBrowser.Controller.Providers.Movies /// Task{MovieImages}. private async Task FetchImages(BaseItem item, string id, CancellationToken cancellationToken) { - using (var json = await _httpClient.Get(new HttpRequestOptions + using (var json = await MovieDbProvider.Current.GetMovieDbResponse(new HttpRequestOptions { Url = string.Format(GetImages, id, MovieDbProvider.ApiKey, item is BoxSet ? "collection" : "movie"), CancellationToken = cancellationToken, - ResourcePool = MovieDbProvider.Current.MovieDbResourcePool, AcceptHeader = MovieDbProvider.AcceptHeader, EnableResponseCache = true @@ -264,14 +263,14 @@ namespace MediaBrowser.Controller.Providers.Movies } if (poster != null) { - try + var img = await MovieDbProvider.Current.GetMovieDbResponse(new HttpRequestOptions { - item.PrimaryImagePath = await _providerManager.DownloadAndSaveImage(item, tmdbImageUrl + poster.file_path, "folder" + Path.GetExtension(poster.file_path), ConfigurationManager.Configuration.SaveLocalMeta && item.LocationType == LocationType.FileSystem, MovieDbProvider.Current.MovieDbResourcePool, cancellationToken).ConfigureAwait(false); - } - catch (HttpException) - { - status = ProviderRefreshStatus.CompletedWithErrors; - } + Url = tmdbImageUrl + poster.file_path, + CancellationToken = cancellationToken + + }).ConfigureAwait(false); + + item.PrimaryImagePath = await _providerManager.SaveImage(item, img, "folder" + Path.GetExtension(poster.file_path), ConfigurationManager.Configuration.SaveLocalMeta && item.LocationType == LocationType.FileSystem, cancellationToken).ConfigureAwait(false); } } @@ -295,14 +294,14 @@ namespace MediaBrowser.Controller.Providers.Movies if (ConfigurationManager.Configuration.RefreshItemImages || !hasLocalBackdrop) { - try + var img = await MovieDbProvider.Current.GetMovieDbResponse(new HttpRequestOptions { - item.BackdropImagePaths.Add(await _providerManager.DownloadAndSaveImage(item, tmdbImageUrl + images.backdrops[i].file_path, bdName + Path.GetExtension(images.backdrops[i].file_path), ConfigurationManager.Configuration.SaveLocalMeta && item.LocationType == LocationType.FileSystem, MovieDbProvider.Current.MovieDbResourcePool, cancellationToken).ConfigureAwait(false)); - } - catch (HttpException) - { - status = ProviderRefreshStatus.CompletedWithErrors; - } + Url = tmdbImageUrl + images.backdrops[i].file_path, + CancellationToken = cancellationToken + + }).ConfigureAwait(false); + + item.BackdropImagePaths.Add(await _providerManager.SaveImage(item, img, bdName + Path.GetExtension(images.backdrops[i].file_path), ConfigurationManager.Configuration.SaveLocalMeta && item.LocationType == LocationType.FileSystem, cancellationToken).ConfigureAwait(false)); } } } diff --git a/MediaBrowser.Controller/Providers/Movies/MovieDbProvider.cs b/MediaBrowser.Controller/Providers/Movies/MovieDbProvider.cs index 38d80883a8..a222c87855 100644 --- a/MediaBrowser.Controller/Providers/Movies/MovieDbProvider.cs +++ b/MediaBrowser.Controller/Providers/Movies/MovieDbProvider.cs @@ -40,7 +40,7 @@ namespace MediaBrowser.Controller.Providers.Movies /// /// The movie db /// - internal readonly SemaphoreSlim MovieDbResourcePool = new SemaphoreSlim(1,1); + private readonly SemaphoreSlim _movieDbResourcePool = new SemaphoreSlim(1,1); internal static MovieDbProvider Current { get; private set; } @@ -81,7 +81,7 @@ namespace MediaBrowser.Controller.Providers.Movies { if (dispose) { - MovieDbResourcePool.Dispose(); + _movieDbResourcePool.Dispose(); } } @@ -172,7 +172,7 @@ namespace MediaBrowser.Controller.Providers.Movies { get { - LazyInitializer.EnsureInitialized(ref _tmdbSettingsTask, ref _tmdbSettingsTaskInitialized, ref _tmdbSettingsTaskSyncLock, () => GetTmdbSettings(JsonSerializer, HttpClient)); + LazyInitializer.EnsureInitialized(ref _tmdbSettingsTask, ref _tmdbSettingsTaskInitialized, ref _tmdbSettingsTaskSyncLock, () => GetTmdbSettings(JsonSerializer)); return _tmdbSettingsTask; } } @@ -181,15 +181,14 @@ namespace MediaBrowser.Controller.Providers.Movies /// Gets the TMDB settings. /// /// Task{TmdbSettingsResult}. - private static async Task GetTmdbSettings(IJsonSerializer jsonSerializer, IHttpClient httpClient) + private async Task GetTmdbSettings(IJsonSerializer jsonSerializer) { try { - using (var json = await httpClient.Get(new HttpRequestOptions + using (var json = await GetMovieDbResponse(new HttpRequestOptions { Url = string.Format(TmdbConfigUrl, ApiKey), CancellationToken = CancellationToken.None, - ResourcePool = Current.MovieDbResourcePool, AcceptHeader = AcceptHeader, EnableResponseCache = true @@ -560,11 +559,10 @@ namespace MediaBrowser.Controller.Providers.Movies try { - using (Stream json = await HttpClient.Get(new HttpRequestOptions + using (Stream json = await GetMovieDbResponse(new HttpRequestOptions { Url = url3, CancellationToken = cancellationToken, - ResourcePool = Current.MovieDbResourcePool, AcceptHeader = AcceptHeader, EnableResponseCache = true @@ -600,11 +598,10 @@ namespace MediaBrowser.Controller.Providers.Movies try { - using (var json = await HttpClient.Get(new HttpRequestOptions + using (var json = await GetMovieDbResponse(new HttpRequestOptions { Url = url3, CancellationToken = cancellationToken, - ResourcePool = Current.MovieDbResourcePool, AcceptHeader = AcceptHeader, EnableResponseCache = true @@ -647,11 +644,10 @@ namespace MediaBrowser.Controller.Providers.Movies try { - using (var json = await HttpClient.Get(new HttpRequestOptions + using (var json = await GetMovieDbResponse(new HttpRequestOptions { Url = url3, CancellationToken = cancellationToken, - ResourcePool = Current.MovieDbResourcePool, AcceptHeader = AcceptHeader, EnableResponseCache = true @@ -737,11 +733,10 @@ namespace MediaBrowser.Controller.Providers.Movies try { - using (Stream json = await HttpClient.Get(new HttpRequestOptions + using (Stream json = await GetMovieDbResponse(new HttpRequestOptions { Url = url, CancellationToken = cancellationToken, - ResourcePool = Current.MovieDbResourcePool, AcceptHeader = AcceptHeader, EnableResponseCache = true @@ -822,11 +817,10 @@ namespace MediaBrowser.Controller.Providers.Movies try { - using (var json = await HttpClient.Get(new HttpRequestOptions + using (var json = await GetMovieDbResponse(new HttpRequestOptions { Url = url, CancellationToken = cancellationToken, - ResourcePool = Current.MovieDbResourcePool, AcceptHeader = AcceptHeader, EnableResponseCache = true @@ -863,11 +857,10 @@ namespace MediaBrowser.Controller.Providers.Movies try { - using (Stream json = await HttpClient.Get(new HttpRequestOptions + using (Stream json = await GetMovieDbResponse(new HttpRequestOptions { Url = url, CancellationToken = cancellationToken, - ResourcePool = Current.MovieDbResourcePool, AcceptHeader = AcceptHeader, EnableResponseCache = true @@ -1022,6 +1015,38 @@ namespace MediaBrowser.Controller.Providers.Movies } + private DateTime _lastRequestDate = DateTime.MinValue; + + /// + /// Gets the movie db response. + /// + internal async Task GetMovieDbResponse(HttpRequestOptions options) + { + var cancellationToken = options.CancellationToken; + + await _movieDbResourcePool.WaitAsync(cancellationToken).ConfigureAwait(false); + + try + { + // Limit to three requests per second + var diff = 330 - (DateTime.Now - _lastRequestDate).TotalMilliseconds; + + if (diff > 0) + { + await Task.Delay(Convert.ToInt32(diff), cancellationToken).ConfigureAwait(false); + } + + _lastRequestDate = DateTime.Now; + + return await HttpClient.Get(options).ConfigureAwait(false); + } + finally + { + _lastRequestDate = DateTime.Now; + + _movieDbResourcePool.Release(); + } + } /// /// The remove diff --git a/MediaBrowser.Controller/Providers/Movies/PersonProviderFromJson.cs b/MediaBrowser.Controller/Providers/Movies/PersonProviderFromJson.cs index f1c5d21c5f..da53e316e5 100644 --- a/MediaBrowser.Controller/Providers/Movies/PersonProviderFromJson.cs +++ b/MediaBrowser.Controller/Providers/Movies/PersonProviderFromJson.cs @@ -1,5 +1,4 @@ -using MediaBrowser.Common.Net; -using MediaBrowser.Controller.Configuration; +using MediaBrowser.Controller.Configuration; using MediaBrowser.Controller.Entities; using MediaBrowser.Model.Logging; using MediaBrowser.Model.Serialization; @@ -15,8 +14,8 @@ namespace MediaBrowser.Controller.Providers.Movies /// class PersonProviderFromJson : TmdbPersonProvider { - public PersonProviderFromJson(IHttpClient httpClient, IJsonSerializer jsonSerializer, ILogManager logManager, IServerConfigurationManager configurationManager, IProviderManager providerManager) - : base(httpClient, jsonSerializer, logManager, configurationManager, providerManager) + public PersonProviderFromJson(IJsonSerializer jsonSerializer, ILogManager logManager, IServerConfigurationManager configurationManager, IProviderManager providerManager) + : base(jsonSerializer, logManager, configurationManager, providerManager) { } diff --git a/MediaBrowser.Controller/Providers/Movies/TmdbPersonProvider.cs b/MediaBrowser.Controller/Providers/Movies/TmdbPersonProvider.cs index 4a5fa8ca25..36ca5dc1a6 100644 --- a/MediaBrowser.Controller/Providers/Movies/TmdbPersonProvider.cs +++ b/MediaBrowser.Controller/Providers/Movies/TmdbPersonProvider.cs @@ -28,18 +28,13 @@ namespace MediaBrowser.Controller.Providers.Movies protected readonly IProviderManager ProviderManager; - public TmdbPersonProvider(IHttpClient httpClient, IJsonSerializer jsonSerializer, ILogManager logManager, IServerConfigurationManager configurationManager, IProviderManager providerManager) + public TmdbPersonProvider(IJsonSerializer jsonSerializer, ILogManager logManager, IServerConfigurationManager configurationManager, IProviderManager providerManager) : base(logManager, configurationManager) { if (jsonSerializer == null) { throw new ArgumentNullException("jsonSerializer"); } - if (httpClient == null) - { - throw new ArgumentNullException("httpClient"); - } - HttpClient = httpClient; JsonSerializer = jsonSerializer; ProviderManager = providerManager; } @@ -50,12 +45,6 @@ namespace MediaBrowser.Controller.Providers.Movies /// The json serializer. protected IJsonSerializer JsonSerializer { get; private set; } - /// - /// Gets the HTTP client. - /// - /// The HTTP client. - protected IHttpClient HttpClient { get; private set; } - /// /// Supportses the specified item. /// @@ -172,11 +161,10 @@ namespace MediaBrowser.Controller.Providers.Movies try { - using (Stream json = await HttpClient.Get(new HttpRequestOptions + using (Stream json = await MovieDbProvider.Current.GetMovieDbResponse(new HttpRequestOptions { Url = url, CancellationToken = cancellationToken, - ResourcePool = MovieDbProvider.Current.MovieDbResourcePool, AcceptHeader = MovieDbProvider.AcceptHeader, EnableResponseCache = true @@ -204,11 +192,10 @@ namespace MediaBrowser.Controller.Providers.Movies string url = string.Format(@"http://api.themoviedb.org/3/person/{1}?api_key={0}&append_to_response=credits,images", MovieDbProvider.ApiKey, id); PersonResult searchResult = null; - using (var json = await HttpClient.Get(new HttpRequestOptions + using (var json = await MovieDbProvider.Current.GetMovieDbResponse(new HttpRequestOptions { Url = url, CancellationToken = cancellationToken, - ResourcePool = MovieDbProvider.Current.MovieDbResourcePool, AcceptHeader = MovieDbProvider.AcceptHeader, EnableResponseCache = true @@ -340,7 +327,12 @@ namespace MediaBrowser.Controller.Providers.Movies var localPath = Path.Combine(item.MetaLocation, targetName); if (!item.ResolveArgs.ContainsMetaFileByName(targetName)) { - using (var sourceStream = await HttpClient.Get(source, MovieDbProvider.Current.MovieDbResourcePool, cancellationToken).ConfigureAwait(false)) + using (var sourceStream = await MovieDbProvider.Current.GetMovieDbResponse(new HttpRequestOptions + { + Url = source, + CancellationToken = cancellationToken + + }).ConfigureAwait(false)) { await ProviderManager.SaveToLibraryFilesystem(item, localPath, sourceStream, cancellationToken).ConfigureAwait(false); diff --git a/MediaBrowser.Controller/Providers/Music/FanArtAlbumProvider.cs b/MediaBrowser.Controller/Providers/Music/FanArtAlbumProvider.cs index 7683cbafb8..537015c3fa 100644 --- a/MediaBrowser.Controller/Providers/Music/FanArtAlbumProvider.cs +++ b/MediaBrowser.Controller/Providers/Music/FanArtAlbumProvider.cs @@ -206,9 +206,7 @@ namespace MediaBrowser.Controller.Providers.Music /// /// The _last music brainz request /// - private DateTime _lastMusicBrainzRequest = DateTime.MinValue; - - private readonly SemaphoreSlim _musicBrainzSemaphore = new SemaphoreSlim(1, 1); + private DateTime _lastRequestDate = DateTime.MinValue; /// /// Gets the music brainz response. @@ -222,7 +220,7 @@ namespace MediaBrowser.Controller.Providers.Music try { - var diff = 1500 - (DateTime.Now - _lastMusicBrainzRequest).TotalMilliseconds; + var diff = 1500 - (DateTime.Now - _lastRequestDate).TotalMilliseconds; // MusicBrainz is extremely adamant about limiting to one request per second @@ -231,7 +229,7 @@ namespace MediaBrowser.Controller.Providers.Music await Task.Delay(Convert.ToInt32(diff), cancellationToken).ConfigureAwait(false); } - _lastMusicBrainzRequest = DateTime.Now; + _lastRequestDate = DateTime.Now; var doc = new XmlDocument(); @@ -239,7 +237,6 @@ namespace MediaBrowser.Controller.Providers.Music { Url = url, CancellationToken = cancellationToken, - ResourcePool = _musicBrainzSemaphore, UserAgent = "MediaBrowserServer/www.mediabrowser3.com", EnableResponseCache = true @@ -255,7 +252,7 @@ namespace MediaBrowser.Controller.Providers.Music } finally { - _lastMusicBrainzRequest = DateTime.Now; + _lastRequestDate = DateTime.Now; _musicBrainzResourcePool.Release(); } diff --git a/MediaBrowser.Server.Implementations/Providers/ProviderManager.cs b/MediaBrowser.Server.Implementations/Providers/ProviderManager.cs index cc586ccbf9..9e6a14442f 100644 --- a/MediaBrowser.Server.Implementations/Providers/ProviderManager.cs +++ b/MediaBrowser.Server.Implementations/Providers/ProviderManager.cs @@ -357,16 +357,22 @@ namespace MediaBrowser.Server.Implementations.Providers throw new ArgumentNullException("resourcePool"); } + var img = await _httpClient.Get(source, resourcePool, cancellationToken).ConfigureAwait(false); + + //download and save locally + return await SaveImage(item, img, targetName, saveLocally, cancellationToken).ConfigureAwait(false); + } + + public async Task SaveImage(BaseItem item, Stream source, string targetName, bool saveLocally, CancellationToken cancellationToken) + { //download and save locally var localPath = (saveLocally && item.MetaLocation != null) ? Path.Combine(item.MetaLocation, targetName) : _remoteImageCache.GetResourcePath(item.GetType().FullName + item.Path.ToLower(), targetName); - var img = await _httpClient.Get(source, resourcePool, cancellationToken).ConfigureAwait(false); - if (saveLocally) // queue to media directories { - await SaveToLibraryFilesystem(item, localPath, img, cancellationToken).ConfigureAwait(false); + await SaveToLibraryFilesystem(item, localPath, source, cancellationToken).ConfigureAwait(false); } else { @@ -376,7 +382,7 @@ namespace MediaBrowser.Server.Implementations.Providers { using (var fs = new FileStream(localPath, FileMode.Create, FileAccess.Write, FileShare.Read, StreamDefaults.DefaultFileStreamBufferSize, FileOptions.Asynchronous)) { - await img.CopyToAsync(fs, StreamDefaults.DefaultCopyToBufferSize, cancellationToken).ConfigureAwait(false); + await source.CopyToAsync(fs, StreamDefaults.DefaultCopyToBufferSize, cancellationToken).ConfigureAwait(false); } } catch (OperationCanceledException) @@ -390,7 +396,7 @@ namespace MediaBrowser.Server.Implementations.Providers } finally { - img.Dispose(); + source.Dispose(); } }