diff --git a/MediaBrowser.Controller/Library/ILibraryManager.cs b/MediaBrowser.Controller/Library/ILibraryManager.cs index f930385d34..059c427bb7 100644 --- a/MediaBrowser.Controller/Library/ILibraryManager.cs +++ b/MediaBrowser.Controller/Library/ILibraryManager.cs @@ -166,6 +166,7 @@ namespace MediaBrowser.Controller.Library /// The item comparers. /// The prescan tasks. /// The postscan tasks. + /// The people prescan tasks. /// The savers. void AddParts(IEnumerable rules, IEnumerable pluginFolders, @@ -174,6 +175,7 @@ namespace MediaBrowser.Controller.Library IEnumerable itemComparers, IEnumerable prescanTasks, IEnumerable postscanTasks, + IEnumerable peoplePrescanTasks, IEnumerable savers); /// diff --git a/MediaBrowser.Controller/Library/IPeoplePrescanTask.cs b/MediaBrowser.Controller/Library/IPeoplePrescanTask.cs new file mode 100644 index 0000000000..04d179baef --- /dev/null +++ b/MediaBrowser.Controller/Library/IPeoplePrescanTask.cs @@ -0,0 +1,20 @@ +using System; +using System.Threading; +using System.Threading.Tasks; + +namespace MediaBrowser.Controller.Library +{ + /// + /// Interface IPeoplePrescanTask + /// + public interface IPeoplePrescanTask + { + /// + /// Runs the specified progress. + /// + /// The progress. + /// The cancellation token. + /// Task. + Task Run(IProgress progress, CancellationToken cancellationToken); + } +} diff --git a/MediaBrowser.Controller/MediaBrowser.Controller.csproj b/MediaBrowser.Controller/MediaBrowser.Controller.csproj index b032da8264..5cde659ddf 100644 --- a/MediaBrowser.Controller/MediaBrowser.Controller.csproj +++ b/MediaBrowser.Controller/MediaBrowser.Controller.csproj @@ -96,6 +96,7 @@ + diff --git a/MediaBrowser.Providers/MediaBrowser.Providers.csproj b/MediaBrowser.Providers/MediaBrowser.Providers.csproj index 9d664b3708..972578b7d6 100644 --- a/MediaBrowser.Providers/MediaBrowser.Providers.csproj +++ b/MediaBrowser.Providers/MediaBrowser.Providers.csproj @@ -67,6 +67,7 @@ + diff --git a/MediaBrowser.Providers/Movies/MovieDbProvider.cs b/MediaBrowser.Providers/Movies/MovieDbProvider.cs index 424070f704..c61e37bde1 100644 --- a/MediaBrowser.Providers/Movies/MovieDbProvider.cs +++ b/MediaBrowser.Providers/Movies/MovieDbProvider.cs @@ -31,7 +31,7 @@ namespace MediaBrowser.Providers.Movies /// /// The movie db /// - private readonly SemaphoreSlim _movieDbResourcePool = new SemaphoreSlim(1, 1); + internal readonly SemaphoreSlim MovieDbResourcePool = new SemaphoreSlim(1, 1); internal static MovieDbProvider Current { get; private set; } @@ -72,7 +72,7 @@ namespace MediaBrowser.Providers.Movies { if (dispose) { - _movieDbResourcePool.Dispose(); + MovieDbResourcePool.Dispose(); } } @@ -726,7 +726,7 @@ namespace MediaBrowser.Providers.Movies { var cancellationToken = options.CancellationToken; - await _movieDbResourcePool.WaitAsync(cancellationToken).ConfigureAwait(false); + await MovieDbResourcePool.WaitAsync(cancellationToken).ConfigureAwait(false); try { @@ -746,7 +746,7 @@ namespace MediaBrowser.Providers.Movies { _lastRequestDate = DateTime.Now; - _movieDbResourcePool.Release(); + MovieDbResourcePool.Release(); } } diff --git a/MediaBrowser.Providers/Movies/OpenMovieDatabaseProvider.cs b/MediaBrowser.Providers/Movies/OpenMovieDatabaseProvider.cs index 4356c9e2ea..5ee3f11415 100644 --- a/MediaBrowser.Providers/Movies/OpenMovieDatabaseProvider.cs +++ b/MediaBrowser.Providers/Movies/OpenMovieDatabaseProvider.cs @@ -46,7 +46,7 @@ namespace MediaBrowser.Providers.Movies { get { - return "11"; + return "12"; } } diff --git a/MediaBrowser.Providers/Movies/PersonUpdatesPreScanTask.cs b/MediaBrowser.Providers/Movies/PersonUpdatesPreScanTask.cs new file mode 100644 index 0000000000..0bc85f9345 --- /dev/null +++ b/MediaBrowser.Providers/Movies/PersonUpdatesPreScanTask.cs @@ -0,0 +1,227 @@ +using MediaBrowser.Common.Net; +using MediaBrowser.Controller.Configuration; +using MediaBrowser.Controller.Library; +using MediaBrowser.Model.Logging; +using MediaBrowser.Model.Net; +using MediaBrowser.Model.Serialization; +using System; +using System.Collections.Generic; +using System.Globalization; +using System.IO; +using System.Linq; +using System.Text; +using System.Threading; +using System.Threading.Tasks; + +namespace MediaBrowser.Providers.Movies +{ + public class PersonUpdatesPreScanTask : IPeoplePrescanTask + { + /// + /// The updates URL + /// + private const string UpdatesUrl = "http://api.themoviedb.org/3/person/changes?start_date={0}&api_key={1}&page={2}"; + + /// + /// The _HTTP client + /// + private readonly IHttpClient _httpClient; + /// + /// The _logger + /// + private readonly ILogger _logger; + /// + /// The _config + /// + private readonly IServerConfigurationManager _config; + private readonly IJsonSerializer _json; + + /// + /// Initializes a new instance of the class. + /// + /// The logger. + /// The HTTP client. + /// The config. + public PersonUpdatesPreScanTask(ILogger logger, IHttpClient httpClient, IServerConfigurationManager config, IJsonSerializer json) + { + _logger = logger; + _httpClient = httpClient; + _config = config; + _json = json; + } + + protected readonly CultureInfo UsCulture = new CultureInfo("en-US"); + + /// + /// Runs the specified progress. + /// + /// The progress. + /// The cancellation token. + /// Task. + public async Task Run(IProgress progress, CancellationToken cancellationToken) + { + if (!_config.Configuration.EnableInternetProviders) + { + progress.Report(100); + return; + } + + var path = TmdbPersonProvider.GetPersonsDataPath(_config.CommonApplicationPaths); + + Directory.CreateDirectory(path); + + var timestampFile = Path.Combine(path, "time.txt"); + + var timestampFileInfo = new FileInfo(timestampFile); + + // Don't check for tvdb updates anymore frequently than 24 hours + if (timestampFileInfo.Exists && (DateTime.UtcNow - timestampFileInfo.LastWriteTimeUtc).TotalDays < 1) + { + return; + } + + // Find out the last time we queried tvdb for updates + var lastUpdateTime = timestampFileInfo.Exists ? File.ReadAllText(timestampFile, Encoding.UTF8) : string.Empty; + + var existingDirectories = Directory.EnumerateDirectories(path).Select(Path.GetFileName).ToList(); + + if (!string.IsNullOrEmpty(lastUpdateTime)) + { + long lastUpdateTicks; + + if (long.TryParse(lastUpdateTime, NumberStyles.Any, UsCulture, out lastUpdateTicks)) + { + var lastUpdateDate = new DateTime(lastUpdateTicks, DateTimeKind.Utc); + + var updatedIds = await GetIdsToUpdate(lastUpdateDate, 1, cancellationToken).ConfigureAwait(false); + + var existingDictionary = existingDirectories.ToDictionary(i => i, StringComparer.OrdinalIgnoreCase); + + var idsToUpdate = updatedIds.Where(i => !string.IsNullOrWhiteSpace(i) && existingDictionary.ContainsKey(i)); + + await UpdatePeople(idsToUpdate, path, progress, cancellationToken).ConfigureAwait(false); + } + } + + File.WriteAllText(timestampFile, DateTime.UtcNow.Ticks.ToString(UsCulture), Encoding.UTF8); + progress.Report(100); + } + + /// + /// Gets the ids to update. + /// + /// The last update time. + /// The page. + /// The cancellation token. + /// Task{IEnumerable{System.String}}. + private async Task> GetIdsToUpdate(DateTime lastUpdateTime, int page, CancellationToken cancellationToken) + { + var hasMorePages = false; + var list = new List(); + + // First get last time + using (var stream = await _httpClient.Get(new HttpRequestOptions + { + Url = string.Format(UpdatesUrl, lastUpdateTime.ToString("yyyy-MM-dd"), MovieDbProvider.ApiKey, page), + CancellationToken = cancellationToken, + EnableHttpCompression = true, + ResourcePool = MovieDbProvider.Current.MovieDbResourcePool, + AcceptHeader = MovieDbProvider.AcceptHeader + + }).ConfigureAwait(false)) + { + var obj = _json.DeserializeFromStream(stream); + + var data = obj.results.Select(i => i.id.ToString(UsCulture)); + + list.AddRange(data); + + hasMorePages = page < obj.total_pages; + } + + if (hasMorePages) + { + var more = await GetIdsToUpdate(lastUpdateTime, page + 1, cancellationToken).ConfigureAwait(false); + + list.AddRange(more); + } + + return list; + } + + /// + /// Updates the people. + /// + /// The ids. + /// The people data path. + /// The progress. + /// The cancellation token. + /// Task. + private async Task UpdatePeople(IEnumerable ids, string peopleDataPath, IProgress progress, CancellationToken cancellationToken) + { + var list = ids.ToList(); + var numComplete = 0; + + foreach (var id in list) + { + try + { + await UpdatePerson(id, peopleDataPath, cancellationToken).ConfigureAwait(false); + } + catch (HttpException ex) + { + // Already logged at lower levels, but don't fail the whole operation, unless timed out + // We have to fail this to make it run again otherwise new episode data could potentially be missing + if (ex.IsTimedOut) + { + throw; + } + } + + numComplete++; + double percent = numComplete; + percent /= list.Count; + percent *= 100; + + progress.Report(percent); + } + } + + /// + /// Updates the person. + /// + /// The id. + /// The people data path. + /// The cancellation token. + /// Task. + private Task UpdatePerson(string id, string peopleDataPath, CancellationToken cancellationToken) + { + _logger.Info("Updating person from tmdb " + id); + + var personDataPath = Path.Combine(peopleDataPath, id); + + Directory.CreateDirectory(peopleDataPath); + + return TmdbPersonProvider.Current.DownloadPersonInfo(id, personDataPath, cancellationToken); + } + + class Result + { + public int id { get; set; } + public bool? adult { get; set; } + } + + class RootObject + { + public List results { get; set; } + public int page { get; set; } + public int total_pages { get; set; } + public int total_results { get; set; } + + public RootObject() + { + results = new List(); + } + } + } +} diff --git a/MediaBrowser.Providers/Movies/TmdbPersonProvider.cs b/MediaBrowser.Providers/Movies/TmdbPersonProvider.cs index fdc4b86c5a..f0c8e3dcad 100644 --- a/MediaBrowser.Providers/Movies/TmdbPersonProvider.cs +++ b/MediaBrowser.Providers/Movies/TmdbPersonProvider.cs @@ -1,4 +1,6 @@ -using MediaBrowser.Common.Net; +using MediaBrowser.Common.Configuration; +using MediaBrowser.Common.IO; +using MediaBrowser.Common.Net; using MediaBrowser.Controller.Configuration; using MediaBrowser.Controller.Entities; using MediaBrowser.Controller.Library; @@ -24,6 +26,8 @@ namespace MediaBrowser.Providers.Movies { protected readonly IProviderManager ProviderManager; + internal static TmdbPersonProvider Current { get; private set; } + public TmdbPersonProvider(IJsonSerializer jsonSerializer, ILogManager logManager, IServerConfigurationManager configurationManager, IProviderManager providerManager) : base(logManager, configurationManager) { @@ -33,6 +37,7 @@ namespace MediaBrowser.Providers.Movies } JsonSerializer = jsonSerializer; ProviderManager = providerManager; + Current = this; } /// @@ -75,12 +80,49 @@ namespace MediaBrowser.Providers.Movies } } - protected override bool NeedsRefreshInternal(BaseItem item, BaseProviderInfo providerInfo) + protected override bool NeedsRefreshBasedOnCompareDate(BaseItem item, BaseProviderInfo providerInfo) { - if (HasAltMeta(item)) - return false; + var provderId = item.GetProviderId(MetadataProviders.Tmdb); - return base.NeedsRefreshInternal(item, providerInfo); + if (!string.IsNullOrEmpty(provderId)) + { + // Process images + var path = GetPersonDataPath(ConfigurationManager.ApplicationPaths, provderId); + + try + { + var files = new DirectoryInfo(path) + .EnumerateFiles("*.json", SearchOption.TopDirectoryOnly) + .Select(i => i.LastWriteTimeUtc) + .ToList(); + + if (files.Count > 0) + { + return files.Max() > providerInfo.LastRefreshed; + } + } + catch (DirectoryNotFoundException) + { + // Don't blow up + return true; + } + } + + return base.NeedsRefreshBasedOnCompareDate(item, providerInfo); + } + + internal static string GetPersonDataPath(IApplicationPaths appPaths, string tmdbId) + { + var seriesDataPath = Path.Combine(GetPersonsDataPath(appPaths), tmdbId); + + return seriesDataPath; + } + + internal static string GetPersonsDataPath(IApplicationPaths appPaths) + { + var dataPath = Path.Combine(appPaths.DataPath, "tmdb-people"); + + return dataPath; } private bool HasAltMeta(BaseItem item) @@ -181,8 +223,40 @@ namespace MediaBrowser.Providers.Movies /// Task. private async Task FetchInfo(Person person, string id, CancellationToken cancellationToken) { - string url = string.Format(@"http://api.themoviedb.org/3/person/{1}?api_key={0}&append_to_response=credits,images", MovieDbProvider.ApiKey, id); - PersonResult searchResult = null; + var personDataPath = GetPersonDataPath(ConfigurationManager.ApplicationPaths, id); + + Directory.CreateDirectory(personDataPath); + + var files = Directory.EnumerateFiles(personDataPath, "*.json", SearchOption.TopDirectoryOnly) + .Select(Path.GetFileName) + .ToList(); + + const string dataFileName = "info.json"; + + // Only download if not already there + // The prescan task will take care of updates so we don't need to re-download here + if (!files.Contains(dataFileName, StringComparer.OrdinalIgnoreCase)) + { + await DownloadPersonInfo(id, personDataPath, cancellationToken).ConfigureAwait(false); + } + + //if (!HasAltMeta(person)) + { + var info = JsonSerializer.DeserializeFromFile(Path.Combine(personDataPath, dataFileName)); + + cancellationToken.ThrowIfCancellationRequested(); + + ProcessInfo(person, info); + + Logger.Debug("TmdbPersonProvider downloaded and saved information for {0}", person.Name); + + await FetchImages(person, info.images, cancellationToken).ConfigureAwait(false); + } + } + + internal async Task DownloadPersonInfo(string id, string personDataPath, CancellationToken cancellationToken) + { + var url = string.Format(@"http://api.themoviedb.org/3/person/{1}?api_key={0}&append_to_response=credits,images", MovieDbProvider.ApiKey, id); using (var json = await MovieDbProvider.Current.GetMovieDbResponse(new HttpRequestOptions { @@ -192,18 +266,10 @@ namespace MediaBrowser.Providers.Movies }).ConfigureAwait(false)) { - searchResult = JsonSerializer.DeserializeFromStream(json); - } - - cancellationToken.ThrowIfCancellationRequested(); - - if (searchResult != null) - { - ProcessInfo(person, searchResult); - - Logger.Debug("TmdbPersonProvider downloaded and saved information for {0}", person.Name); - - await FetchImages(person, searchResult.images, cancellationToken).ConfigureAwait(false); + using (var fs = new FileStream(Path.Combine(personDataPath, "info.json"), FileMode.Create, FileAccess.Write, FileShare.Read, StreamDefaults.DefaultFileStreamBufferSize, true)) + { + await json.CopyToAsync(fs).ConfigureAwait(false); + } } } diff --git a/MediaBrowser.Server.Implementations/Library/LibraryManager.cs b/MediaBrowser.Server.Implementations/Library/LibraryManager.cs index e9c824198f..b632792f64 100644 --- a/MediaBrowser.Server.Implementations/Library/LibraryManager.cs +++ b/MediaBrowser.Server.Implementations/Library/LibraryManager.cs @@ -15,7 +15,6 @@ using MediaBrowser.Model.Entities; using MediaBrowser.Model.Logging; using MediaBrowser.Server.Implementations.Library.Validators; using MediaBrowser.Server.Implementations.ScheduledTasks; -using MoreLinq; using System; using System.Collections.Concurrent; using System.Collections.Generic; @@ -44,6 +43,12 @@ namespace MediaBrowser.Server.Implementations.Library /// The prescan tasks. private IEnumerable PrescanTasks { get; set; } + /// + /// Gets or sets the people prescan tasks. + /// + /// The people prescan tasks. + private IEnumerable PeoplePrescanTasks { get; set; } + /// /// Gets the intro providers. /// @@ -197,6 +202,7 @@ namespace MediaBrowser.Server.Implementations.Library /// The item comparers. /// The prescan tasks. /// The postscan tasks. + /// The people prescan tasks. /// The savers. public void AddParts(IEnumerable rules, IEnumerable pluginFolders, @@ -205,6 +211,7 @@ namespace MediaBrowser.Server.Implementations.Library IEnumerable itemComparers, IEnumerable prescanTasks, IEnumerable postscanTasks, + IEnumerable peoplePrescanTasks, IEnumerable savers) { EntityResolutionIgnoreRules = rules; @@ -214,6 +221,7 @@ namespace MediaBrowser.Server.Implementations.Library Comparers = itemComparers; PrescanTasks = prescanTasks; PostscanTasks = postscanTasks; + PeoplePrescanTasks = peoplePrescanTasks; _savers = savers; } @@ -771,45 +779,9 @@ namespace MediaBrowser.Server.Implementations.Library /// The cancellation token. /// The progress. /// Task. - public async Task ValidatePeople(CancellationToken cancellationToken, IProgress progress) + public Task ValidatePeople(CancellationToken cancellationToken, IProgress progress) { - var people = RootFolder.GetRecursiveChildren() - .SelectMany(c => c.People) - .DistinctBy(p => p.Name, StringComparer.OrdinalIgnoreCase) - .ToList(); - - var numComplete = 0; - - foreach (var person in people) - { - cancellationToken.ThrowIfCancellationRequested(); - - try - { - var item = GetPerson(person.Name); - - await item.RefreshMetadata(cancellationToken).ConfigureAwait(false); - } - catch (Exception ex) - { - _logger.ErrorException("Error validating IBN entry {0}", ex, person.Name); - } - - // Update progress - numComplete++; - double percent = numComplete; - percent /= people.Count; - - progress.Report(100 * percent); - } - - progress.Report(100); - - _logger.Info("People validation complete"); - - // Bad practice, i know. But we keep a lot in memory, unfortunately. - GC.Collect(2, GCCollectionMode.Forced, true); - GC.Collect(2, GCCollectionMode.Forced, true); + return new PeopleValidator(this, PeoplePrescanTasks, _logger).ValidatePeople(cancellationToken, progress); } /// @@ -1153,7 +1125,7 @@ namespace MediaBrowser.Server.Implementations.Library private Video ResolveIntro(IntroInfo info) { Video video = null; - + if (info.ItemId.HasValue) { // Get an existing item by Id diff --git a/MediaBrowser.Server.Implementations/Library/Validators/PeopleValidator.cs b/MediaBrowser.Server.Implementations/Library/Validators/PeopleValidator.cs new file mode 100644 index 0000000000..9554290ed8 --- /dev/null +++ b/MediaBrowser.Server.Implementations/Library/Validators/PeopleValidator.cs @@ -0,0 +1,147 @@ +using MediaBrowser.Common.Progress; +using MediaBrowser.Controller.Library; +using MediaBrowser.Model.Logging; +using MoreLinq; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; + +namespace MediaBrowser.Server.Implementations.Library.Validators +{ + /// + /// Class PeopleValidator + /// + public class PeopleValidator + { + /// + /// The _library manager + /// + private readonly ILibraryManager _libraryManager; + /// + /// The _logger + /// + private readonly ILogger _logger; + + private readonly IEnumerable _prescanTasks; + + /// + /// Initializes a new instance of the class. + /// + /// The library manager. + /// The prescan tasks. + /// The logger. + public PeopleValidator(ILibraryManager libraryManager, IEnumerable prescanTasks, ILogger logger) + { + _libraryManager = libraryManager; + _logger = logger; + _prescanTasks = prescanTasks; + } + + /// + /// Validates the people. + /// + /// The cancellation token. + /// The progress. + /// Task. + public async Task ValidatePeople(CancellationToken cancellationToken, IProgress progress) + { + var innerProgress = new ActionableProgress(); + + innerProgress.RegisterAction(pct => progress.Report(pct * .15)); + + // Run prescan tasks + await RunPrescanTasks(innerProgress, cancellationToken).ConfigureAwait(false); + + progress.Report(15); + + var people = _libraryManager.RootFolder.GetRecursiveChildren() + .SelectMany(c => c.People) + .DistinctBy(p => p.Name, StringComparer.OrdinalIgnoreCase) + .ToList(); + + var numComplete = 0; + + foreach (var person in people) + { + cancellationToken.ThrowIfCancellationRequested(); + + try + { + var item = _libraryManager.GetPerson(person.Name); + + await item.RefreshMetadata(cancellationToken).ConfigureAwait(false); + } + catch (Exception ex) + { + _logger.ErrorException("Error validating IBN entry {0}", ex, person.Name); + } + + // Update progress + numComplete++; + double percent = numComplete; + percent /= people.Count; + + progress.Report(15 + 85 * percent); + } + + progress.Report(100); + + _logger.Info("People validation complete"); + + // Bad practice, i know. But we keep a lot in memory, unfortunately. + GC.Collect(2, GCCollectionMode.Forced, true); + GC.Collect(2, GCCollectionMode.Forced, true); + } + + /// + /// Runs the prescan tasks. + /// + /// The progress. + /// The cancellation token. + /// Task. + private async Task RunPrescanTasks(IProgress progress, CancellationToken cancellationToken) + { + var tasks = _prescanTasks.ToList(); + + var numComplete = 0; + var numTasks = tasks.Count; + + foreach (var task in tasks) + { + var innerProgress = new ActionableProgress(); + + // Prevent access to modified closure + var currentNumComplete = numComplete; + + innerProgress.RegisterAction(pct => + { + double innerPercent = (currentNumComplete * 100) + pct; + innerPercent /= numTasks; + progress.Report(innerPercent); + }); + + try + { + await task.Run(innerProgress, cancellationToken); + } + catch (OperationCanceledException) + { + _logger.Info("Pre-scan task cancelled: {0}", task.GetType().Name); + } + catch (Exception ex) + { + _logger.ErrorException("Error running pre-scan task", ex); + } + + numComplete++; + double percent = numComplete; + percent /= numTasks; + progress.Report(percent * 100); + } + + progress.Report(100); + } + } +} diff --git a/MediaBrowser.Server.Implementations/MediaBrowser.Server.Implementations.csproj b/MediaBrowser.Server.Implementations/MediaBrowser.Server.Implementations.csproj index 39a9b77895..f10eff1665 100644 --- a/MediaBrowser.Server.Implementations/MediaBrowser.Server.Implementations.csproj +++ b/MediaBrowser.Server.Implementations/MediaBrowser.Server.Implementations.csproj @@ -160,6 +160,7 @@ + diff --git a/MediaBrowser.ServerApplication/ApplicationHost.cs b/MediaBrowser.ServerApplication/ApplicationHost.cs index ed9179b076..ae4f09d3ec 100644 --- a/MediaBrowser.ServerApplication/ApplicationHost.cs +++ b/MediaBrowser.ServerApplication/ApplicationHost.cs @@ -439,6 +439,7 @@ namespace MediaBrowser.ServerApplication GetExports(), GetExports(), GetExports(), + GetExports(), GetExports()); ProviderManager.AddParts(GetExports());