From b9fc0d26287e46017515e4ac3e569ca2c60f622f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jes=C3=BAs=20Higueras?= Date: Mon, 23 Mar 2020 20:05:49 +0100 Subject: [PATCH] Add BlurHash support to backend --- Emby.Drawing/ImageProcessor.cs | 4 ++ Emby.Drawing/NullImageEncoder.cs | 6 +++ .../Data/SqliteItemRepository.cs | 17 ++++++- Emby.Server.Implementations/Dto/DtoService.cs | 7 +++ .../Library/LibraryManager.cs | 30 +++++++++++- .../Jellyfin.Drawing.Skia.csproj | 2 + Jellyfin.Drawing.Skia/SkiaEncoder.cs | 48 ++++++++++++++++++- MediaBrowser.Api/Images/ImageService.cs | 7 ++- .../Drawing/IImageEncoder.cs | 7 +++ .../Drawing/IImageProcessor.cs | 7 +++ MediaBrowser.Controller/Entities/BaseItem.cs | 1 + .../Entities/ItemImageInfo.cs | 6 +++ MediaBrowser.Model/Dto/BaseItemDto.cs | 6 +++ MediaBrowser.Model/Dto/ImageInfo.cs | 6 +++ 14 files changed, 149 insertions(+), 5 deletions(-) diff --git a/Emby.Drawing/ImageProcessor.cs b/Emby.Drawing/ImageProcessor.cs index 0b3bbe29ef..1237b603b6 100644 --- a/Emby.Drawing/ImageProcessor.cs +++ b/Emby.Drawing/ImageProcessor.cs @@ -313,6 +313,10 @@ namespace Emby.Drawing public ImageDimensions GetImageDimensions(string path) => _imageEncoder.GetImageSize(path); + /// + public string GetImageHash(string path) + => _imageEncoder.GetImageHash(path); + /// public string GetImageCacheTag(BaseItem item, ItemImageInfo image) => (item.Path + image.DateModified.Ticks).GetMD5().ToString("N", CultureInfo.InvariantCulture); diff --git a/Emby.Drawing/NullImageEncoder.cs b/Emby.Drawing/NullImageEncoder.cs index 5af7f16225..fa89b4c638 100644 --- a/Emby.Drawing/NullImageEncoder.cs +++ b/Emby.Drawing/NullImageEncoder.cs @@ -42,5 +42,11 @@ namespace Emby.Drawing { throw new NotImplementedException(); } + + /// + public string GetImageHash(string inputPath) + { + throw new NotImplementedException(); + } } } diff --git a/Emby.Server.Implementations/Data/SqliteItemRepository.cs b/Emby.Server.Implementations/Data/SqliteItemRepository.cs index ca5cd6fdd5..5a43a138bd 100644 --- a/Emby.Server.Implementations/Data/SqliteItemRepository.cs +++ b/Emby.Server.Implementations/Data/SqliteItemRepository.cs @@ -1144,12 +1144,18 @@ namespace Emby.Server.Implementations.Data var delimeter = "*"; var path = image.Path; + var hash = image.Hash; if (path == null) { path = string.Empty; } + if (hash == null) + { + hash = string.Empty; + } + return GetPathToSave(path) + delimeter + image.DateModified.Ticks.ToString(CultureInfo.InvariantCulture) + @@ -1158,7 +1164,11 @@ namespace Emby.Server.Implementations.Data delimeter + image.Width.ToString(CultureInfo.InvariantCulture) + delimeter + - image.Height.ToString(CultureInfo.InvariantCulture); + image.Height.ToString(CultureInfo.InvariantCulture) + + delimeter + + // Replace delimiters with other characters. + // This can be removed when we migrate to a proper DB. + hash.Replace('*', '/').Replace('|', '\\'); } public ItemImageInfo ItemImageInfoFromValueString(string value) @@ -1192,6 +1202,11 @@ namespace Emby.Server.Implementations.Data image.Width = width; image.Height = height; } + + if (parts.Length >= 6) + { + image.Hash = parts[5].Replace('/', '*').Replace('\\', '|'); + } } return image; diff --git a/Emby.Server.Implementations/Dto/DtoService.cs b/Emby.Server.Implementations/Dto/DtoService.cs index c4b65d2654..a34a3a192f 100644 --- a/Emby.Server.Implementations/Dto/DtoService.cs +++ b/Emby.Server.Implementations/Dto/DtoService.cs @@ -718,6 +718,7 @@ namespace Emby.Server.Implementations.Dto if (options.EnableImages) { dto.ImageTags = new Dictionary(); + dto.ImageHashes = new Dictionary(); // Prevent implicitly captured closure var currentItem = item; @@ -732,6 +733,12 @@ namespace Emby.Server.Implementations.Dto { dto.ImageTags[image.Type] = tag; } + + var hash = image.Hash; + if (hash != null && hash.Length > 0) + { + dto.ImageHashes[tag] = image.Hash; + } } } } diff --git a/Emby.Server.Implementations/Library/LibraryManager.cs b/Emby.Server.Implementations/Library/LibraryManager.cs index 0b86b2db7e..bc35b04106 100644 --- a/Emby.Server.Implementations/Library/LibraryManager.cs +++ b/Emby.Server.Implementations/Library/LibraryManager.cs @@ -21,6 +21,7 @@ using MediaBrowser.Common.Extensions; using MediaBrowser.Common.Progress; using MediaBrowser.Controller; using MediaBrowser.Controller.Configuration; +using MediaBrowser.Controller.Drawing; using MediaBrowser.Controller.Dto; using MediaBrowser.Controller.Entities; using MediaBrowser.Controller.Entities.Audio; @@ -35,6 +36,7 @@ using MediaBrowser.Controller.Resolvers; using MediaBrowser.Controller.Sorting; using MediaBrowser.Model.Configuration; using MediaBrowser.Model.Dlna; +using MediaBrowser.Model.Drawing; using MediaBrowser.Model.Dto; using MediaBrowser.Model.Entities; using MediaBrowser.Model.IO; @@ -109,6 +111,18 @@ namespace Emby.Server.Implementations.Library /// The comparers. private IBaseItemComparer[] Comparers { get; set; } + /// + /// Gets or sets the active item repository + /// + /// The item repository. + public IItemRepository ItemRepository { get; set; } + + /// + /// Gets or sets the active image processor + /// + /// The image processor. + public IImageProcessor ImageProcessor { get; set; } + /// /// Occurs when [item added]. /// @@ -1817,7 +1831,19 @@ namespace Emby.Server.Implementations.Library public void UpdateImages(BaseItem item) { - _itemRepository.SaveImages(item); + item.ImageInfos + .Where(i => (i.Width == 0 || i.Height == 0)) + .ToList() + .ForEach(x => + { + string blurhash = ImageProcessor.GetImageHash(x.Path); + ImageDimensions size = ImageProcessor.GetImageDimensions(item, x, true); + x.Width = size.Width; + x.Height = size.Height; + x.Hash = blurhash; + }); + + ItemRepository.SaveImages(item); RegisterItem(item); } @@ -1839,7 +1865,7 @@ namespace Emby.Server.Implementations.Library item.DateLastSaved = DateTime.UtcNow; - RegisterItem(item); + UpdateImages(item); } _itemRepository.SaveItems(itemsList, cancellationToken); diff --git a/Jellyfin.Drawing.Skia/Jellyfin.Drawing.Skia.csproj b/Jellyfin.Drawing.Skia/Jellyfin.Drawing.Skia.csproj index a6e1f490ad..9f0e3a0042 100644 --- a/Jellyfin.Drawing.Skia/Jellyfin.Drawing.Skia.csproj +++ b/Jellyfin.Drawing.Skia/Jellyfin.Drawing.Skia.csproj @@ -21,6 +21,8 @@ + + diff --git a/Jellyfin.Drawing.Skia/SkiaEncoder.cs b/Jellyfin.Drawing.Skia/SkiaEncoder.cs index 5c7462ee29..1d73078638 100644 --- a/Jellyfin.Drawing.Skia/SkiaEncoder.cs +++ b/Jellyfin.Drawing.Skia/SkiaEncoder.cs @@ -1,7 +1,11 @@ using System; using System.Collections.Generic; +using System.Diagnostics.CodeAnalysis; using System.Globalization; using System.IO; +using System.Linq; +using System.Threading.Tasks; +using Blurhash.Core; using MediaBrowser.Common.Configuration; using MediaBrowser.Controller.Drawing; using MediaBrowser.Controller.Extensions; @@ -15,7 +19,7 @@ namespace Jellyfin.Drawing.Skia /// /// Image encoder that uses to manipulate images. /// - public class SkiaEncoder : IImageEncoder + public class SkiaEncoder : CoreEncoder, IImageEncoder { private static readonly HashSet _transparentImageTypes = new HashSet(StringComparer.OrdinalIgnoreCase) { ".png", ".gif", ".webp" }; @@ -229,6 +233,48 @@ namespace Jellyfin.Drawing.Skia } } + /// + /// The path is null. + /// The path is not valid. + /// The file at the specified path could not be used to generate a codec. + [SuppressMessage("Microsoft.Performance", "CA1814:PreferJaggedArraysOverMultidimensional")] + public string GetImageHash(string path) + { + if (path == null) + { + throw new ArgumentNullException(nameof(path)); + } + + if (!File.Exists(path)) + { + throw new FileNotFoundException("File not found", path); + } + + using (var bitmap = GetBitmap(path, false, false, null)) + { + if (bitmap == null) + { + throw new ArgumentOutOfRangeException($"Skia unable to read image {path}"); + } + + var width = bitmap.Width; + var height = bitmap.Height; + var pixels = new Pixel[width, height]; + Parallel.ForEach(Enumerable.Range(0, height), y => + { + for (var x = 0; x < width; x++) + { + var color = bitmap.GetPixel(x, y); + pixels[x, y].Red = MathUtils.SRgbToLinear(color.Red); + pixels[x, y].Green = MathUtils.SRgbToLinear(color.Green); + pixels[x, y].Blue = MathUtils.SRgbToLinear(color.Blue); + } + }); + + return CoreEncode(pixels, 4, 4); + } + } + private static bool HasDiacritics(string text) => !string.Equals(text, text.RemoveDiacritics(), StringComparison.Ordinal); diff --git a/MediaBrowser.Api/Images/ImageService.cs b/MediaBrowser.Api/Images/ImageService.cs index 2e9b3e6cb4..ecfe2ed763 100644 --- a/MediaBrowser.Api/Images/ImageService.cs +++ b/MediaBrowser.Api/Images/ImageService.cs @@ -323,6 +323,7 @@ namespace MediaBrowser.Api.Images { int? width = null; int? height = null; + string? blurhash = null; long length = 0; try @@ -332,7 +333,10 @@ namespace MediaBrowser.Api.Images var fileInfo = _fileSystem.GetFileInfo(info.Path); length = fileInfo.Length; - ImageDimensions size = _imageProcessor.GetImageDimensions(item, info); + blurhash = _imageProcessor.GetImageHash(info.Path); + info.Hash = blurhash; // TODO: this doesn't seem like the right thing to do + + ImageDimensions size = _imageProcessor.GetImageDimensions(item, info, true); _libraryManager.UpdateImages(item); width = size.Width; height = size.Height; @@ -358,6 +362,7 @@ namespace MediaBrowser.Api.Images ImageType = info.Type, ImageTag = _imageProcessor.GetImageCacheTag(item, info), Size = length, + Hash = blurhash, Width = width, Height = height }; diff --git a/MediaBrowser.Controller/Drawing/IImageEncoder.cs b/MediaBrowser.Controller/Drawing/IImageEncoder.cs index 88e67b6486..1d3f0d3b44 100644 --- a/MediaBrowser.Controller/Drawing/IImageEncoder.cs +++ b/MediaBrowser.Controller/Drawing/IImageEncoder.cs @@ -43,6 +43,13 @@ namespace MediaBrowser.Controller.Drawing /// The image dimensions. ImageDimensions GetImageSize(string path); + /// + /// Get the blurhash of an image. + /// + /// The filepath of the image. + /// The blurhash. + string GetImageHash(string path); + /// /// Encode an image. /// diff --git a/MediaBrowser.Controller/Drawing/IImageProcessor.cs b/MediaBrowser.Controller/Drawing/IImageProcessor.cs index 36c746624e..be5906cbc9 100644 --- a/MediaBrowser.Controller/Drawing/IImageProcessor.cs +++ b/MediaBrowser.Controller/Drawing/IImageProcessor.cs @@ -32,6 +32,13 @@ namespace MediaBrowser.Controller.Drawing /// ImageDimensions ImageDimensions GetImageDimensions(string path); + /// + /// Gets the blurhash of the image. + /// + /// Path to the image file. + /// BlurHash + String GetImageHash(string path); + /// /// Gets the dimensions of the image. /// diff --git a/MediaBrowser.Controller/Entities/BaseItem.cs b/MediaBrowser.Controller/Entities/BaseItem.cs index 7ed8fa7671..e5f6ea09d6 100644 --- a/MediaBrowser.Controller/Entities/BaseItem.cs +++ b/MediaBrowser.Controller/Entities/BaseItem.cs @@ -2222,6 +2222,7 @@ namespace MediaBrowser.Controller.Entities existingImage.DateModified = image.DateModified; existingImage.Width = image.Width; existingImage.Height = image.Height; + existingImage.Hash = image.Hash; } else { diff --git a/MediaBrowser.Controller/Entities/ItemImageInfo.cs b/MediaBrowser.Controller/Entities/ItemImageInfo.cs index fc46dec2ef..ba02971073 100644 --- a/MediaBrowser.Controller/Entities/ItemImageInfo.cs +++ b/MediaBrowser.Controller/Entities/ItemImageInfo.cs @@ -28,6 +28,12 @@ namespace MediaBrowser.Controller.Entities public int Height { get; set; } + /// + /// Gets or sets the blurhash. + /// + /// The blurhash. + public string Hash { get; set; } + [JsonIgnore] public bool IsLocalFile => Path == null || !Path.StartsWith("http", StringComparison.OrdinalIgnoreCase); } diff --git a/MediaBrowser.Model/Dto/BaseItemDto.cs b/MediaBrowser.Model/Dto/BaseItemDto.cs index 607355d8d3..8c6c9683a4 100644 --- a/MediaBrowser.Model/Dto/BaseItemDto.cs +++ b/MediaBrowser.Model/Dto/BaseItemDto.cs @@ -510,6 +510,12 @@ namespace MediaBrowser.Model.Dto /// The series thumb image tag. public string SeriesThumbImageTag { get; set; } + /// + /// Gets or sets the blurhash for the image tags. + /// + /// The blurhashes. + public Dictionary ImageHashes { get; set; } + /// /// Gets or sets the series studio. /// diff --git a/MediaBrowser.Model/Dto/ImageInfo.cs b/MediaBrowser.Model/Dto/ImageInfo.cs index 57942ac23b..39bdc09ed8 100644 --- a/MediaBrowser.Model/Dto/ImageInfo.cs +++ b/MediaBrowser.Model/Dto/ImageInfo.cs @@ -30,6 +30,12 @@ namespace MediaBrowser.Model.Dto /// The path. public string Path { get; set; } + /// + /// Gets or sets the blurhash. + /// + /// The blurhash. + public string Hash { get; set; } + /// /// Gets or sets the height. ///