mirror of
https://github.com/jellyfin/jellyfin.git
synced 2024-09-05 21:37:10 -04:00
reduce fanart movie requests using their updates api
This commit is contained in:
parent
e5936e008e
commit
70cecb346b
@ -75,6 +75,7 @@
|
||||
<Compile Include="Entities\Audio\MusicAlbumDisc.cs" />
|
||||
<Compile Include="Library\ILibraryPostScanTask.cs" />
|
||||
<Compile Include="Library\ILibraryPrescanTask.cs" />
|
||||
<Compile Include="Providers\Movies\FanArtMovieUpdatesPrescanTask.cs" />
|
||||
<Compile Include="Providers\Movies\MovieDbImagesProvider.cs" />
|
||||
<Compile Include="Providers\Music\ArtistsPostScanTask.cs" />
|
||||
<Compile Include="Providers\Music\FanArtUpdatesPrescanTask.cs" />
|
||||
|
@ -1,4 +1,6 @@
|
||||
using MediaBrowser.Common.Extensions;
|
||||
using MediaBrowser.Common.Configuration;
|
||||
using MediaBrowser.Common.Extensions;
|
||||
using MediaBrowser.Common.IO;
|
||||
using MediaBrowser.Common.Net;
|
||||
using MediaBrowser.Controller.Configuration;
|
||||
using MediaBrowser.Controller.Entities;
|
||||
@ -7,6 +9,8 @@ using MediaBrowser.Model.Entities;
|
||||
using MediaBrowser.Model.Logging;
|
||||
using System;
|
||||
using System.Globalization;
|
||||
using System.IO;
|
||||
using System.Linq;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using System.Xml;
|
||||
@ -16,7 +20,7 @@ namespace MediaBrowser.Controller.Providers.Movies
|
||||
/// <summary>
|
||||
/// Class FanArtMovieProvider
|
||||
/// </summary>
|
||||
class FanArtMovieProvider : FanartBaseProvider, IDisposable
|
||||
class FanArtMovieProvider : FanartBaseProvider
|
||||
{
|
||||
/// <summary>
|
||||
/// Gets the HTTP client.
|
||||
@ -24,10 +28,18 @@ namespace MediaBrowser.Controller.Providers.Movies
|
||||
/// <value>The HTTP client.</value>
|
||||
protected IHttpClient HttpClient { get; private set; }
|
||||
|
||||
/// <summary>
|
||||
/// The _provider manager
|
||||
/// </summary>
|
||||
private readonly IProviderManager _providerManager;
|
||||
|
||||
/// <summary>
|
||||
/// The us culture
|
||||
/// </summary>
|
||||
private static readonly CultureInfo UsCulture = new CultureInfo("en-US");
|
||||
|
||||
internal static FanArtMovieProvider Current { get; private set; }
|
||||
|
||||
/// <summary>
|
||||
/// Initializes a new instance of the <see cref="FanArtMovieProvider" /> class.
|
||||
/// </summary>
|
||||
@ -45,18 +57,7 @@ namespace MediaBrowser.Controller.Providers.Movies
|
||||
}
|
||||
HttpClient = httpClient;
|
||||
_providerManager = providerManager;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Releases unmanaged and - optionally - managed resources.
|
||||
/// </summary>
|
||||
/// <param name="dispose"><c>true</c> to release both managed and unmanaged resources; <c>false</c> to release only unmanaged resources.</param>
|
||||
protected virtual void Dispose(bool dispose)
|
||||
{
|
||||
if (dispose)
|
||||
{
|
||||
FanArtResourcePool.Dispose();
|
||||
}
|
||||
Current = this;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
@ -79,7 +80,7 @@ namespace MediaBrowser.Controller.Providers.Movies
|
||||
{
|
||||
get
|
||||
{
|
||||
return "12";
|
||||
return "13";
|
||||
}
|
||||
}
|
||||
|
||||
@ -140,10 +141,62 @@ namespace MediaBrowser.Controller.Providers.Movies
|
||||
/// <summary>
|
||||
/// Gets the comparison data.
|
||||
/// </summary>
|
||||
/// <param name="id">The id.</param>
|
||||
/// <returns>Guid.</returns>
|
||||
private Guid GetComparisonData(string id)
|
||||
{
|
||||
return string.IsNullOrEmpty(id) ? Guid.Empty : id.GetMD5();
|
||||
if (!string.IsNullOrEmpty(id))
|
||||
{
|
||||
// Process images
|
||||
var path = GetMovieDataPath(ConfigurationManager.ApplicationPaths, id);
|
||||
|
||||
var files = new DirectoryInfo(path)
|
||||
.EnumerateFiles("*.xml", SearchOption.TopDirectoryOnly)
|
||||
.Select(i => i.FullName + i.LastWriteTimeUtc.Ticks)
|
||||
.ToArray();
|
||||
|
||||
if (files.Length > 0)
|
||||
{
|
||||
return string.Join(string.Empty, files).GetMD5();
|
||||
}
|
||||
}
|
||||
|
||||
return Guid.Empty;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Gets the movie data path.
|
||||
/// </summary>
|
||||
/// <param name="appPaths">The app paths.</param>
|
||||
/// <param name="tmdbId">The TMDB id.</param>
|
||||
/// <returns>System.String.</returns>
|
||||
internal static string GetMovieDataPath(IApplicationPaths appPaths, string tmdbId)
|
||||
{
|
||||
var dataPath = Path.Combine(GetMoviesDataPath(appPaths), tmdbId);
|
||||
|
||||
if (!Directory.Exists(dataPath))
|
||||
{
|
||||
Directory.CreateDirectory(dataPath);
|
||||
}
|
||||
|
||||
return dataPath;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Gets the movie data path.
|
||||
/// </summary>
|
||||
/// <param name="appPaths">The app paths.</param>
|
||||
/// <returns>System.String.</returns>
|
||||
internal static string GetMoviesDataPath(IApplicationPaths appPaths)
|
||||
{
|
||||
var dataPath = Path.Combine(appPaths.DataPath, "fanart-movies");
|
||||
|
||||
if (!Directory.Exists(dataPath))
|
||||
{
|
||||
Directory.CreateDirectory(dataPath);
|
||||
}
|
||||
|
||||
return dataPath;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
@ -165,15 +218,43 @@ namespace MediaBrowser.Controller.Providers.Movies
|
||||
item.ProviderData[Id] = data;
|
||||
}
|
||||
|
||||
var status = ProviderRefreshStatus.Success;
|
||||
var movieId = item.GetProviderId(MetadataProviders.Tmdb);
|
||||
|
||||
var movie = item;
|
||||
var movieDataPath = GetMovieDataPath(ConfigurationManager.ApplicationPaths, movieId);
|
||||
var xmlPath = Path.Combine(movieDataPath, "fanart.xml");
|
||||
|
||||
var language = ConfigurationManager.Configuration.PreferredMetadataLanguage.ToLower();
|
||||
var url = string.Format(FanArtBaseUrl, ApiKey, movie.GetProviderId(MetadataProviders.Tmdb));
|
||||
var doc = new XmlDocument();
|
||||
// Only download the xml if it doesn't already exist. The prescan task will take care of getting updates
|
||||
if (!File.Exists(xmlPath))
|
||||
{
|
||||
await DownloadMovieXml(movieDataPath, movieId, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
using (var xml = await HttpClient.Get(new HttpRequestOptions
|
||||
if (File.Exists(xmlPath))
|
||||
{
|
||||
await FetchFromXml(item, xmlPath, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
data.Data = GetComparisonData(item.GetProviderId(MetadataProviders.Tmdb));
|
||||
SetLastRefreshed(item, DateTime.UtcNow);
|
||||
return true;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Downloads the movie XML.
|
||||
/// </summary>
|
||||
/// <param name="movieDataPath">The movie data path.</param>
|
||||
/// <param name="tmdbId">The TMDB id.</param>
|
||||
/// <param name="cancellationToken">The cancellation token.</param>
|
||||
/// <returns>Task.</returns>
|
||||
internal async Task DownloadMovieXml(string movieDataPath, string tmdbId, CancellationToken cancellationToken)
|
||||
{
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
|
||||
string url = string.Format(FanArtBaseUrl, ApiKey, tmdbId);
|
||||
|
||||
var xmlPath = Path.Combine(movieDataPath, "fanart.xml");
|
||||
|
||||
using (var response = await HttpClient.Get(new HttpRequestOptions
|
||||
{
|
||||
Url = url,
|
||||
ResourcePool = FanArtResourcePool,
|
||||
@ -181,8 +262,26 @@ namespace MediaBrowser.Controller.Providers.Movies
|
||||
|
||||
}).ConfigureAwait(false))
|
||||
{
|
||||
doc.Load(xml);
|
||||
using (var xmlFileStream = new FileStream(xmlPath, FileMode.Create, FileAccess.Write, FileShare.Read, StreamDefaults.DefaultFileStreamBufferSize, FileOptions.Asynchronous))
|
||||
{
|
||||
await response.CopyToAsync(xmlFileStream).ConfigureAwait(false);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Fetches from XML.
|
||||
/// </summary>
|
||||
/// <param name="item">The item.</param>
|
||||
/// <param name="xmlFilePath">The XML file path.</param>
|
||||
/// <param name="cancellationToken">The cancellation token.</param>
|
||||
/// <returns>Task.</returns>
|
||||
private async Task FetchFromXml(BaseItem item, string xmlFilePath, CancellationToken cancellationToken)
|
||||
{
|
||||
var doc = new XmlDocument();
|
||||
doc.Load(xmlFilePath);
|
||||
|
||||
var language = ConfigurationManager.Configuration.PreferredMetadataLanguage.ToLower();
|
||||
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
|
||||
@ -205,7 +304,7 @@ namespace MediaBrowser.Controller.Providers.Movies
|
||||
path = node != null ? node.Value : null;
|
||||
if (!string.IsNullOrEmpty(path))
|
||||
{
|
||||
movie.SetImage(ImageType.Logo, await _providerManager.DownloadAndSaveImage(movie, path, LogoFile, saveLocal, FanArtResourcePool, cancellationToken).ConfigureAwait(false));
|
||||
item.SetImage(ImageType.Logo, await _providerManager.DownloadAndSaveImage(item, path, LogoFile, saveLocal, FanArtResourcePool, cancellationToken).ConfigureAwait(false));
|
||||
}
|
||||
}
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
@ -220,7 +319,7 @@ namespace MediaBrowser.Controller.Providers.Movies
|
||||
path = node != null ? node.Value : null;
|
||||
if (!string.IsNullOrEmpty(path))
|
||||
{
|
||||
movie.SetImage(ImageType.Art, await _providerManager.DownloadAndSaveImage(movie, path, ArtFile, saveLocal, FanArtResourcePool, cancellationToken).ConfigureAwait(false));
|
||||
item.SetImage(ImageType.Art, await _providerManager.DownloadAndSaveImage(item, path, ArtFile, saveLocal, FanArtResourcePool, cancellationToken).ConfigureAwait(false));
|
||||
}
|
||||
}
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
@ -232,7 +331,7 @@ namespace MediaBrowser.Controller.Providers.Movies
|
||||
path = node != null ? node.Value : null;
|
||||
if (!string.IsNullOrEmpty(path))
|
||||
{
|
||||
movie.SetImage(ImageType.Disc, await _providerManager.DownloadAndSaveImage(movie, path, DiscFile, saveLocal, FanArtResourcePool, cancellationToken).ConfigureAwait(false));
|
||||
item.SetImage(ImageType.Disc, await _providerManager.DownloadAndSaveImage(item, path, DiscFile, saveLocal, FanArtResourcePool, cancellationToken).ConfigureAwait(false));
|
||||
}
|
||||
}
|
||||
|
||||
@ -245,7 +344,7 @@ namespace MediaBrowser.Controller.Providers.Movies
|
||||
path = node != null ? node.Value : null;
|
||||
if (!string.IsNullOrEmpty(path))
|
||||
{
|
||||
movie.SetImage(ImageType.Banner, await _providerManager.DownloadAndSaveImage(movie, path, BannerFile, saveLocal, FanArtResourcePool, cancellationToken).ConfigureAwait(false));
|
||||
item.SetImage(ImageType.Banner, await _providerManager.DownloadAndSaveImage(item, path, BannerFile, saveLocal, FanArtResourcePool, cancellationToken).ConfigureAwait(false));
|
||||
}
|
||||
}
|
||||
|
||||
@ -258,7 +357,7 @@ namespace MediaBrowser.Controller.Providers.Movies
|
||||
path = node != null ? node.Value : null;
|
||||
if (!string.IsNullOrEmpty(path))
|
||||
{
|
||||
movie.SetImage(ImageType.Thumb, await _providerManager.DownloadAndSaveImage(movie, path, ThumbFile, saveLocal, FanArtResourcePool, cancellationToken).ConfigureAwait(false));
|
||||
item.SetImage(ImageType.Thumb, await _providerManager.DownloadAndSaveImage(item, path, ThumbFile, saveLocal, FanArtResourcePool, cancellationToken).ConfigureAwait(false));
|
||||
}
|
||||
}
|
||||
|
||||
@ -286,15 +385,6 @@ namespace MediaBrowser.Controller.Providers.Movies
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
data.Data = GetComparisonData(item.GetProviderId(MetadataProviders.Tmdb));
|
||||
SetLastRefreshed(movie, DateTime.UtcNow, status);
|
||||
return true;
|
||||
}
|
||||
|
||||
public void Dispose()
|
||||
{
|
||||
Dispose(true);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
@ -0,0 +1,184 @@
|
||||
using MediaBrowser.Common.Net;
|
||||
using MediaBrowser.Controller.Configuration;
|
||||
using MediaBrowser.Controller.Library;
|
||||
using MediaBrowser.Controller.Providers.Music;
|
||||
using MediaBrowser.Model.Logging;
|
||||
using MediaBrowser.Model.Net;
|
||||
using MediaBrowser.Model.Serialization;
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Globalization;
|
||||
using System.IO;
|
||||
using System.Linq;
|
||||
using System.Text;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
|
||||
namespace MediaBrowser.Controller.Providers.Movies
|
||||
{
|
||||
class FanArtMovieUpdatesPrescanTask : ILibraryPrescanTask
|
||||
{
|
||||
private const string UpdatesUrl = "http://api.fanart.tv/webservice/newmovies/{0}/{1}/";
|
||||
|
||||
/// <summary>
|
||||
/// The _HTTP client
|
||||
/// </summary>
|
||||
private readonly IHttpClient _httpClient;
|
||||
/// <summary>
|
||||
/// The _logger
|
||||
/// </summary>
|
||||
private readonly ILogger _logger;
|
||||
/// <summary>
|
||||
/// The _config
|
||||
/// </summary>
|
||||
private readonly IServerConfigurationManager _config;
|
||||
private readonly IJsonSerializer _jsonSerializer;
|
||||
|
||||
private static readonly CultureInfo UsCulture = new CultureInfo("en-US");
|
||||
|
||||
public FanArtMovieUpdatesPrescanTask(IJsonSerializer jsonSerializer, IServerConfigurationManager config, ILogger logger, IHttpClient httpClient)
|
||||
{
|
||||
_jsonSerializer = jsonSerializer;
|
||||
_config = config;
|
||||
_logger = logger;
|
||||
_httpClient = httpClient;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Runs the specified progress.
|
||||
/// </summary>
|
||||
/// <param name="progress">The progress.</param>
|
||||
/// <param name="cancellationToken">The cancellation token.</param>
|
||||
/// <returns>Task.</returns>
|
||||
public async Task Run(IProgress<double> progress, CancellationToken cancellationToken)
|
||||
{
|
||||
if (!_config.Configuration.EnableInternetProviders)
|
||||
{
|
||||
progress.Report(100);
|
||||
return;
|
||||
}
|
||||
|
||||
var path = FanArtMovieProvider.GetMoviesDataPath(_config.CommonApplicationPaths);
|
||||
|
||||
var timestampFile = Path.Combine(path, "time.txt");
|
||||
|
||||
var timestampFileInfo = new FileInfo(timestampFile);
|
||||
|
||||
if (_config.Configuration.MetadataRefreshDays > 0 && timestampFileInfo.Exists && (DateTime.UtcNow - timestampFileInfo.LastWriteTimeUtc).TotalDays < _config.Configuration.MetadataRefreshDays)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
// Find out the last time we queried for updates
|
||||
var lastUpdateTime = timestampFileInfo.Exists ? File.ReadAllText(timestampFile, Encoding.UTF8) : string.Empty;
|
||||
|
||||
var existingDirectories = Directory.EnumerateDirectories(path).Select(Path.GetFileName).ToList();
|
||||
|
||||
// If this is our first time, don't do any updates and just record the timestamp
|
||||
if (!string.IsNullOrEmpty(lastUpdateTime))
|
||||
{
|
||||
var moviesToUpdate = await GetMovieIdsToUpdate(existingDirectories, lastUpdateTime, cancellationToken).ConfigureAwait(false);
|
||||
|
||||
progress.Report(5);
|
||||
|
||||
await UpdateMovies(moviesToUpdate, path, progress, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
var newUpdateTime = Convert.ToInt64(DateTimeToUnixTimestamp(DateTime.UtcNow)).ToString(UsCulture);
|
||||
|
||||
File.WriteAllText(timestampFile, newUpdateTime, Encoding.UTF8);
|
||||
|
||||
progress.Report(100);
|
||||
}
|
||||
|
||||
private async Task<IEnumerable<string>> GetMovieIdsToUpdate(IEnumerable<string> existingIds, string lastUpdateTime, CancellationToken cancellationToken)
|
||||
{
|
||||
// First get last time
|
||||
using (var stream = await _httpClient.Get(new HttpRequestOptions
|
||||
{
|
||||
Url = string.Format(UpdatesUrl, FanartBaseProvider.ApiKey, lastUpdateTime),
|
||||
CancellationToken = cancellationToken,
|
||||
EnableHttpCompression = true,
|
||||
ResourcePool = FanartBaseProvider.FanArtResourcePool
|
||||
|
||||
}).ConfigureAwait(false))
|
||||
{
|
||||
// If empty fanart will return a string of "null", rather than an empty list
|
||||
using (var reader = new StreamReader(stream))
|
||||
{
|
||||
var json = await reader.ReadToEndAsync().ConfigureAwait(false);
|
||||
|
||||
if (string.Equals(json, "null", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
return new List<string>();
|
||||
}
|
||||
|
||||
var updates = _jsonSerializer.DeserializeFromString<List<FanArtUpdatesPrescanTask.FanArtUpdate>>(json);
|
||||
|
||||
return updates.Select(i => i.id).Where(i => existingIds.Contains(i, StringComparer.OrdinalIgnoreCase));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private async Task UpdateMovies(IEnumerable<string> idList, string moviesDataPath, IProgress<double> progress, CancellationToken cancellationToken)
|
||||
{
|
||||
var list = idList.ToList();
|
||||
var numComplete = 0;
|
||||
|
||||
foreach (var id in list)
|
||||
{
|
||||
try
|
||||
{
|
||||
await UpdateMovie(id, moviesDataPath, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
catch (HttpException ex)
|
||||
{
|
||||
// Already logged at lower levels, but don't fail the whole operation, unless something other than a timeout
|
||||
if (!ex.IsTimedOut)
|
||||
{
|
||||
throw;
|
||||
}
|
||||
}
|
||||
|
||||
numComplete++;
|
||||
double percent = numComplete;
|
||||
percent /= list.Count;
|
||||
percent *= 95;
|
||||
|
||||
progress.Report(percent + 5);
|
||||
}
|
||||
}
|
||||
|
||||
private Task UpdateMovie(string tmdbId, string movieDataPath, CancellationToken cancellationToken)
|
||||
{
|
||||
_logger.Info("Updating movie " + tmdbId);
|
||||
|
||||
movieDataPath = Path.Combine(movieDataPath, tmdbId);
|
||||
|
||||
if (!Directory.Exists(movieDataPath))
|
||||
{
|
||||
Directory.CreateDirectory(movieDataPath);
|
||||
}
|
||||
|
||||
return FanArtMovieProvider.Current.DownloadMovieXml(movieDataPath, tmdbId, cancellationToken);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Dates the time to unix timestamp.
|
||||
/// </summary>
|
||||
/// <param name="dateTime">The date time.</param>
|
||||
/// <returns>System.Double.</returns>
|
||||
private static double DateTimeToUnixTimestamp(DateTime dateTime)
|
||||
{
|
||||
return (dateTime - new DateTime(1970, 1, 1).ToUniversalTime()).TotalSeconds;
|
||||
}
|
||||
|
||||
public class FanArtUpdate
|
||||
{
|
||||
public string id { get; set; }
|
||||
public string name { get; set; }
|
||||
public string new_images { get; set; }
|
||||
public string total_images { get; set; }
|
||||
}
|
||||
}
|
||||
}
|
@ -3,7 +3,6 @@ using MediaBrowser.Controller.Configuration;
|
||||
using MediaBrowser.Controller.Entities;
|
||||
using MediaBrowser.Model.Entities;
|
||||
using MediaBrowser.Model.Logging;
|
||||
using MediaBrowser.Model.Net;
|
||||
using MediaBrowser.Model.Serialization;
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
|
@ -358,7 +358,7 @@ namespace MediaBrowser.Server.Implementations.HttpServer
|
||||
{
|
||||
var address = endpoint.ToString();
|
||||
|
||||
_localEndPoints.AddOrUpdate(address, address, (key, existing) => address);
|
||||
_localEndPoints.GetOrAdd(address, address);
|
||||
}
|
||||
|
||||
if (EnableHttpRequestLogging)
|
||||
|
Loading…
Reference in New Issue
Block a user