mirror of
https://github.com/jellyfin/jellyfin.git
synced 2024-09-07 06:17:14 -04:00
limit number of tmdb requests per second
This commit is contained in:
parent
13d997a97a
commit
25314e1fc9
@ -21,6 +21,8 @@ namespace MediaBrowser.Controller.Providers
|
||||
/// <exception cref="System.ArgumentNullException">item</exception>
|
||||
Task<string> DownloadAndSaveImage(BaseItem item, string source, string targetName, bool saveLocally, SemaphoreSlim resourcePool, CancellationToken cancellationToken);
|
||||
|
||||
Task<string> SaveImage(BaseItem item, Stream source, string targetName, bool saveLocally, CancellationToken cancellationToken);
|
||||
|
||||
/// <summary>
|
||||
/// Saves to library filesystem.
|
||||
/// </summary>
|
||||
|
@ -207,11 +207,10 @@ namespace MediaBrowser.Controller.Providers.Movies
|
||||
/// <returns>Task{MovieImages}.</returns>
|
||||
private async Task<MovieImages> FetchImages(BaseItem item, string id, CancellationToken cancellationToken)
|
||||
{
|
||||
using (var json = await _httpClient.Get(new HttpRequestOptions
|
||||
using (var json = await MovieDbProvider.Current.GetMovieDbResponse(new HttpRequestOptions
|
||||
{
|
||||
Url = string.Format(GetImages, id, MovieDbProvider.ApiKey, item is BoxSet ? "collection" : "movie"),
|
||||
CancellationToken = cancellationToken,
|
||||
ResourcePool = MovieDbProvider.Current.MovieDbResourcePool,
|
||||
AcceptHeader = MovieDbProvider.AcceptHeader,
|
||||
EnableResponseCache = true
|
||||
|
||||
@ -264,14 +263,14 @@ namespace MediaBrowser.Controller.Providers.Movies
|
||||
}
|
||||
if (poster != null)
|
||||
{
|
||||
try
|
||||
var img = await MovieDbProvider.Current.GetMovieDbResponse(new HttpRequestOptions
|
||||
{
|
||||
item.PrimaryImagePath = await _providerManager.DownloadAndSaveImage(item, tmdbImageUrl + poster.file_path, "folder" + Path.GetExtension(poster.file_path), ConfigurationManager.Configuration.SaveLocalMeta && item.LocationType == LocationType.FileSystem, MovieDbProvider.Current.MovieDbResourcePool, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
catch (HttpException)
|
||||
{
|
||||
status = ProviderRefreshStatus.CompletedWithErrors;
|
||||
}
|
||||
Url = tmdbImageUrl + poster.file_path,
|
||||
CancellationToken = cancellationToken
|
||||
|
||||
}).ConfigureAwait(false);
|
||||
|
||||
item.PrimaryImagePath = await _providerManager.SaveImage(item, img, "folder" + Path.GetExtension(poster.file_path), ConfigurationManager.Configuration.SaveLocalMeta && item.LocationType == LocationType.FileSystem, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
}
|
||||
|
||||
@ -295,14 +294,14 @@ namespace MediaBrowser.Controller.Providers.Movies
|
||||
|
||||
if (ConfigurationManager.Configuration.RefreshItemImages || !hasLocalBackdrop)
|
||||
{
|
||||
try
|
||||
var img = await MovieDbProvider.Current.GetMovieDbResponse(new HttpRequestOptions
|
||||
{
|
||||
item.BackdropImagePaths.Add(await _providerManager.DownloadAndSaveImage(item, tmdbImageUrl + images.backdrops[i].file_path, bdName + Path.GetExtension(images.backdrops[i].file_path), ConfigurationManager.Configuration.SaveLocalMeta && item.LocationType == LocationType.FileSystem, MovieDbProvider.Current.MovieDbResourcePool, cancellationToken).ConfigureAwait(false));
|
||||
}
|
||||
catch (HttpException)
|
||||
{
|
||||
status = ProviderRefreshStatus.CompletedWithErrors;
|
||||
}
|
||||
Url = tmdbImageUrl + images.backdrops[i].file_path,
|
||||
CancellationToken = cancellationToken
|
||||
|
||||
}).ConfigureAwait(false);
|
||||
|
||||
item.BackdropImagePaths.Add(await _providerManager.SaveImage(item, img, bdName + Path.GetExtension(images.backdrops[i].file_path), ConfigurationManager.Configuration.SaveLocalMeta && item.LocationType == LocationType.FileSystem, cancellationToken).ConfigureAwait(false));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
@ -40,7 +40,7 @@ namespace MediaBrowser.Controller.Providers.Movies
|
||||
/// <summary>
|
||||
/// The movie db
|
||||
/// </summary>
|
||||
internal readonly SemaphoreSlim MovieDbResourcePool = new SemaphoreSlim(1,1);
|
||||
private readonly SemaphoreSlim _movieDbResourcePool = new SemaphoreSlim(1,1);
|
||||
|
||||
internal static MovieDbProvider Current { get; private set; }
|
||||
|
||||
@ -81,7 +81,7 @@ namespace MediaBrowser.Controller.Providers.Movies
|
||||
{
|
||||
if (dispose)
|
||||
{
|
||||
MovieDbResourcePool.Dispose();
|
||||
_movieDbResourcePool.Dispose();
|
||||
}
|
||||
}
|
||||
|
||||
@ -172,7 +172,7 @@ namespace MediaBrowser.Controller.Providers.Movies
|
||||
{
|
||||
get
|
||||
{
|
||||
LazyInitializer.EnsureInitialized(ref _tmdbSettingsTask, ref _tmdbSettingsTaskInitialized, ref _tmdbSettingsTaskSyncLock, () => GetTmdbSettings(JsonSerializer, HttpClient));
|
||||
LazyInitializer.EnsureInitialized(ref _tmdbSettingsTask, ref _tmdbSettingsTaskInitialized, ref _tmdbSettingsTaskSyncLock, () => GetTmdbSettings(JsonSerializer));
|
||||
return _tmdbSettingsTask;
|
||||
}
|
||||
}
|
||||
@ -181,15 +181,14 @@ namespace MediaBrowser.Controller.Providers.Movies
|
||||
/// Gets the TMDB settings.
|
||||
/// </summary>
|
||||
/// <returns>Task{TmdbSettingsResult}.</returns>
|
||||
private static async Task<TmdbSettingsResult> GetTmdbSettings(IJsonSerializer jsonSerializer, IHttpClient httpClient)
|
||||
private async Task<TmdbSettingsResult> GetTmdbSettings(IJsonSerializer jsonSerializer)
|
||||
{
|
||||
try
|
||||
{
|
||||
using (var json = await httpClient.Get(new HttpRequestOptions
|
||||
using (var json = await GetMovieDbResponse(new HttpRequestOptions
|
||||
{
|
||||
Url = string.Format(TmdbConfigUrl, ApiKey),
|
||||
CancellationToken = CancellationToken.None,
|
||||
ResourcePool = Current.MovieDbResourcePool,
|
||||
AcceptHeader = AcceptHeader,
|
||||
EnableResponseCache = true
|
||||
|
||||
@ -560,11 +559,10 @@ namespace MediaBrowser.Controller.Providers.Movies
|
||||
|
||||
try
|
||||
{
|
||||
using (Stream json = await HttpClient.Get(new HttpRequestOptions
|
||||
using (Stream json = await GetMovieDbResponse(new HttpRequestOptions
|
||||
{
|
||||
Url = url3,
|
||||
CancellationToken = cancellationToken,
|
||||
ResourcePool = Current.MovieDbResourcePool,
|
||||
AcceptHeader = AcceptHeader,
|
||||
EnableResponseCache = true
|
||||
|
||||
@ -600,11 +598,10 @@ namespace MediaBrowser.Controller.Providers.Movies
|
||||
|
||||
try
|
||||
{
|
||||
using (var json = await HttpClient.Get(new HttpRequestOptions
|
||||
using (var json = await GetMovieDbResponse(new HttpRequestOptions
|
||||
{
|
||||
Url = url3,
|
||||
CancellationToken = cancellationToken,
|
||||
ResourcePool = Current.MovieDbResourcePool,
|
||||
AcceptHeader = AcceptHeader,
|
||||
EnableResponseCache = true
|
||||
|
||||
@ -647,11 +644,10 @@ namespace MediaBrowser.Controller.Providers.Movies
|
||||
|
||||
try
|
||||
{
|
||||
using (var json = await HttpClient.Get(new HttpRequestOptions
|
||||
using (var json = await GetMovieDbResponse(new HttpRequestOptions
|
||||
{
|
||||
Url = url3,
|
||||
CancellationToken = cancellationToken,
|
||||
ResourcePool = Current.MovieDbResourcePool,
|
||||
AcceptHeader = AcceptHeader,
|
||||
EnableResponseCache = true
|
||||
|
||||
@ -737,11 +733,10 @@ namespace MediaBrowser.Controller.Providers.Movies
|
||||
|
||||
try
|
||||
{
|
||||
using (Stream json = await HttpClient.Get(new HttpRequestOptions
|
||||
using (Stream json = await GetMovieDbResponse(new HttpRequestOptions
|
||||
{
|
||||
Url = url,
|
||||
CancellationToken = cancellationToken,
|
||||
ResourcePool = Current.MovieDbResourcePool,
|
||||
AcceptHeader = AcceptHeader,
|
||||
EnableResponseCache = true
|
||||
|
||||
@ -822,11 +817,10 @@ namespace MediaBrowser.Controller.Providers.Movies
|
||||
|
||||
try
|
||||
{
|
||||
using (var json = await HttpClient.Get(new HttpRequestOptions
|
||||
using (var json = await GetMovieDbResponse(new HttpRequestOptions
|
||||
{
|
||||
Url = url,
|
||||
CancellationToken = cancellationToken,
|
||||
ResourcePool = Current.MovieDbResourcePool,
|
||||
AcceptHeader = AcceptHeader,
|
||||
EnableResponseCache = true
|
||||
|
||||
@ -863,11 +857,10 @@ namespace MediaBrowser.Controller.Providers.Movies
|
||||
|
||||
try
|
||||
{
|
||||
using (Stream json = await HttpClient.Get(new HttpRequestOptions
|
||||
using (Stream json = await GetMovieDbResponse(new HttpRequestOptions
|
||||
{
|
||||
Url = url,
|
||||
CancellationToken = cancellationToken,
|
||||
ResourcePool = Current.MovieDbResourcePool,
|
||||
AcceptHeader = AcceptHeader,
|
||||
EnableResponseCache = true
|
||||
|
||||
@ -1022,6 +1015,38 @@ namespace MediaBrowser.Controller.Providers.Movies
|
||||
|
||||
}
|
||||
|
||||
private DateTime _lastRequestDate = DateTime.MinValue;
|
||||
|
||||
/// <summary>
|
||||
/// Gets the movie db response.
|
||||
/// </summary>
|
||||
internal async Task<Stream> GetMovieDbResponse(HttpRequestOptions options)
|
||||
{
|
||||
var cancellationToken = options.CancellationToken;
|
||||
|
||||
await _movieDbResourcePool.WaitAsync(cancellationToken).ConfigureAwait(false);
|
||||
|
||||
try
|
||||
{
|
||||
// Limit to three requests per second
|
||||
var diff = 330 - (DateTime.Now - _lastRequestDate).TotalMilliseconds;
|
||||
|
||||
if (diff > 0)
|
||||
{
|
||||
await Task.Delay(Convert.ToInt32(diff), cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
_lastRequestDate = DateTime.Now;
|
||||
|
||||
return await HttpClient.Get(options).ConfigureAwait(false);
|
||||
}
|
||||
finally
|
||||
{
|
||||
_lastRequestDate = DateTime.Now;
|
||||
|
||||
_movieDbResourcePool.Release();
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// The remove
|
||||
|
@ -1,5 +1,4 @@
|
||||
using MediaBrowser.Common.Net;
|
||||
using MediaBrowser.Controller.Configuration;
|
||||
using MediaBrowser.Controller.Configuration;
|
||||
using MediaBrowser.Controller.Entities;
|
||||
using MediaBrowser.Model.Logging;
|
||||
using MediaBrowser.Model.Serialization;
|
||||
@ -15,8 +14,8 @@ namespace MediaBrowser.Controller.Providers.Movies
|
||||
/// </summary>
|
||||
class PersonProviderFromJson : TmdbPersonProvider
|
||||
{
|
||||
public PersonProviderFromJson(IHttpClient httpClient, IJsonSerializer jsonSerializer, ILogManager logManager, IServerConfigurationManager configurationManager, IProviderManager providerManager)
|
||||
: base(httpClient, jsonSerializer, logManager, configurationManager, providerManager)
|
||||
public PersonProviderFromJson(IJsonSerializer jsonSerializer, ILogManager logManager, IServerConfigurationManager configurationManager, IProviderManager providerManager)
|
||||
: base(jsonSerializer, logManager, configurationManager, providerManager)
|
||||
{
|
||||
}
|
||||
|
||||
|
@ -28,18 +28,13 @@ namespace MediaBrowser.Controller.Providers.Movies
|
||||
|
||||
protected readonly IProviderManager ProviderManager;
|
||||
|
||||
public TmdbPersonProvider(IHttpClient httpClient, IJsonSerializer jsonSerializer, ILogManager logManager, IServerConfigurationManager configurationManager, IProviderManager providerManager)
|
||||
public TmdbPersonProvider(IJsonSerializer jsonSerializer, ILogManager logManager, IServerConfigurationManager configurationManager, IProviderManager providerManager)
|
||||
: base(logManager, configurationManager)
|
||||
{
|
||||
if (jsonSerializer == null)
|
||||
{
|
||||
throw new ArgumentNullException("jsonSerializer");
|
||||
}
|
||||
if (httpClient == null)
|
||||
{
|
||||
throw new ArgumentNullException("httpClient");
|
||||
}
|
||||
HttpClient = httpClient;
|
||||
JsonSerializer = jsonSerializer;
|
||||
ProviderManager = providerManager;
|
||||
}
|
||||
@ -50,12 +45,6 @@ namespace MediaBrowser.Controller.Providers.Movies
|
||||
/// <value>The json serializer.</value>
|
||||
protected IJsonSerializer JsonSerializer { get; private set; }
|
||||
|
||||
/// <summary>
|
||||
/// Gets the HTTP client.
|
||||
/// </summary>
|
||||
/// <value>The HTTP client.</value>
|
||||
protected IHttpClient HttpClient { get; private set; }
|
||||
|
||||
/// <summary>
|
||||
/// Supportses the specified item.
|
||||
/// </summary>
|
||||
@ -172,11 +161,10 @@ namespace MediaBrowser.Controller.Providers.Movies
|
||||
|
||||
try
|
||||
{
|
||||
using (Stream json = await HttpClient.Get(new HttpRequestOptions
|
||||
using (Stream json = await MovieDbProvider.Current.GetMovieDbResponse(new HttpRequestOptions
|
||||
{
|
||||
Url = url,
|
||||
CancellationToken = cancellationToken,
|
||||
ResourcePool = MovieDbProvider.Current.MovieDbResourcePool,
|
||||
AcceptHeader = MovieDbProvider.AcceptHeader,
|
||||
EnableResponseCache = true
|
||||
|
||||
@ -204,11 +192,10 @@ namespace MediaBrowser.Controller.Providers.Movies
|
||||
string url = string.Format(@"http://api.themoviedb.org/3/person/{1}?api_key={0}&append_to_response=credits,images", MovieDbProvider.ApiKey, id);
|
||||
PersonResult searchResult = null;
|
||||
|
||||
using (var json = await HttpClient.Get(new HttpRequestOptions
|
||||
using (var json = await MovieDbProvider.Current.GetMovieDbResponse(new HttpRequestOptions
|
||||
{
|
||||
Url = url,
|
||||
CancellationToken = cancellationToken,
|
||||
ResourcePool = MovieDbProvider.Current.MovieDbResourcePool,
|
||||
AcceptHeader = MovieDbProvider.AcceptHeader,
|
||||
EnableResponseCache = true
|
||||
|
||||
@ -340,7 +327,12 @@ namespace MediaBrowser.Controller.Providers.Movies
|
||||
var localPath = Path.Combine(item.MetaLocation, targetName);
|
||||
if (!item.ResolveArgs.ContainsMetaFileByName(targetName))
|
||||
{
|
||||
using (var sourceStream = await HttpClient.Get(source, MovieDbProvider.Current.MovieDbResourcePool, cancellationToken).ConfigureAwait(false))
|
||||
using (var sourceStream = await MovieDbProvider.Current.GetMovieDbResponse(new HttpRequestOptions
|
||||
{
|
||||
Url = source,
|
||||
CancellationToken = cancellationToken
|
||||
|
||||
}).ConfigureAwait(false))
|
||||
{
|
||||
await ProviderManager.SaveToLibraryFilesystem(item, localPath, sourceStream, cancellationToken).ConfigureAwait(false);
|
||||
|
||||
|
@ -206,9 +206,7 @@ namespace MediaBrowser.Controller.Providers.Music
|
||||
/// <summary>
|
||||
/// The _last music brainz request
|
||||
/// </summary>
|
||||
private DateTime _lastMusicBrainzRequest = DateTime.MinValue;
|
||||
|
||||
private readonly SemaphoreSlim _musicBrainzSemaphore = new SemaphoreSlim(1, 1);
|
||||
private DateTime _lastRequestDate = DateTime.MinValue;
|
||||
|
||||
/// <summary>
|
||||
/// Gets the music brainz response.
|
||||
@ -222,7 +220,7 @@ namespace MediaBrowser.Controller.Providers.Music
|
||||
|
||||
try
|
||||
{
|
||||
var diff = 1500 - (DateTime.Now - _lastMusicBrainzRequest).TotalMilliseconds;
|
||||
var diff = 1500 - (DateTime.Now - _lastRequestDate).TotalMilliseconds;
|
||||
|
||||
// MusicBrainz is extremely adamant about limiting to one request per second
|
||||
|
||||
@ -231,7 +229,7 @@ namespace MediaBrowser.Controller.Providers.Music
|
||||
await Task.Delay(Convert.ToInt32(diff), cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
_lastMusicBrainzRequest = DateTime.Now;
|
||||
_lastRequestDate = DateTime.Now;
|
||||
|
||||
var doc = new XmlDocument();
|
||||
|
||||
@ -239,7 +237,6 @@ namespace MediaBrowser.Controller.Providers.Music
|
||||
{
|
||||
Url = url,
|
||||
CancellationToken = cancellationToken,
|
||||
ResourcePool = _musicBrainzSemaphore,
|
||||
UserAgent = "MediaBrowserServer/www.mediabrowser3.com",
|
||||
EnableResponseCache = true
|
||||
|
||||
@ -255,7 +252,7 @@ namespace MediaBrowser.Controller.Providers.Music
|
||||
}
|
||||
finally
|
||||
{
|
||||
_lastMusicBrainzRequest = DateTime.Now;
|
||||
_lastRequestDate = DateTime.Now;
|
||||
|
||||
_musicBrainzResourcePool.Release();
|
||||
}
|
||||
|
@ -357,16 +357,22 @@ namespace MediaBrowser.Server.Implementations.Providers
|
||||
throw new ArgumentNullException("resourcePool");
|
||||
}
|
||||
|
||||
var img = await _httpClient.Get(source, resourcePool, cancellationToken).ConfigureAwait(false);
|
||||
|
||||
//download and save locally
|
||||
return await SaveImage(item, img, targetName, saveLocally, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
public async Task<string> SaveImage(BaseItem item, Stream source, string targetName, bool saveLocally, CancellationToken cancellationToken)
|
||||
{
|
||||
//download and save locally
|
||||
var localPath = (saveLocally && item.MetaLocation != null) ?
|
||||
Path.Combine(item.MetaLocation, targetName) :
|
||||
_remoteImageCache.GetResourcePath(item.GetType().FullName + item.Path.ToLower(), targetName);
|
||||
|
||||
var img = await _httpClient.Get(source, resourcePool, cancellationToken).ConfigureAwait(false);
|
||||
|
||||
if (saveLocally) // queue to media directories
|
||||
{
|
||||
await SaveToLibraryFilesystem(item, localPath, img, cancellationToken).ConfigureAwait(false);
|
||||
await SaveToLibraryFilesystem(item, localPath, source, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
else
|
||||
{
|
||||
@ -376,7 +382,7 @@ namespace MediaBrowser.Server.Implementations.Providers
|
||||
{
|
||||
using (var fs = new FileStream(localPath, FileMode.Create, FileAccess.Write, FileShare.Read, StreamDefaults.DefaultFileStreamBufferSize, FileOptions.Asynchronous))
|
||||
{
|
||||
await img.CopyToAsync(fs, StreamDefaults.DefaultCopyToBufferSize, cancellationToken).ConfigureAwait(false);
|
||||
await source.CopyToAsync(fs, StreamDefaults.DefaultCopyToBufferSize, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
}
|
||||
catch (OperationCanceledException)
|
||||
@ -390,7 +396,7 @@ namespace MediaBrowser.Server.Implementations.Providers
|
||||
}
|
||||
finally
|
||||
{
|
||||
img.Dispose();
|
||||
source.Dispose();
|
||||
}
|
||||
|
||||
}
|
||||
|
Loading…
Reference in New Issue
Block a user