From 0e2c362078c5b0babaa0fd254106452e6d67ebe8 Mon Sep 17 00:00:00 2001 From: Nick <20588554+nicknsy@users.noreply.github.com> Date: Tue, 30 May 2023 14:23:02 -0700 Subject: [PATCH] Move SkiaSharp related code to Jellyfin.Drawing and IImageEncoder --- .../Drawing/IImageEncoder.cs | 11 +++ .../MediaBrowser.Providers.csproj | 3 +- .../Trickplay/TrickplayManager.cs | 97 +++++++------------ src/Jellyfin.Drawing.Skia/SkiaEncoder.cs | 79 +++++++++++++++ src/Jellyfin.Drawing/NullImageEncoder.cs | 6 ++ 5 files changed, 131 insertions(+), 65 deletions(-) diff --git a/MediaBrowser.Controller/Drawing/IImageEncoder.cs b/MediaBrowser.Controller/Drawing/IImageEncoder.cs index e5c8ebfaf9..42c680761d 100644 --- a/MediaBrowser.Controller/Drawing/IImageEncoder.cs +++ b/MediaBrowser.Controller/Drawing/IImageEncoder.cs @@ -2,6 +2,7 @@ using System; using System.Collections.Generic; +using MediaBrowser.Model.Configuration; using MediaBrowser.Model.Drawing; namespace MediaBrowser.Controller.Drawing @@ -81,5 +82,15 @@ namespace MediaBrowser.Controller.Drawing /// The list of poster paths. /// The list of backdrop paths. void CreateSplashscreen(IReadOnlyList posters, IReadOnlyList backdrops); + + /// + /// Creates a new jpeg trickplay grid image. + /// + /// The options to use when creating the image. Width and Height are a quantity of tiles in this case, not pixels. + /// The image encode quality. + /// The width of a single trickplay image. + /// Optional height of a single trickplay image, if it is known. + /// Height of single decoded trickplay image. + int CreateTrickplayGrid(ImageCollageOptions options, int quality, int imgWidth, int? imgHeight); } } diff --git a/MediaBrowser.Providers/MediaBrowser.Providers.csproj b/MediaBrowser.Providers/MediaBrowser.Providers.csproj index c836c8ed53..7ef70f4b08 100644 --- a/MediaBrowser.Providers/MediaBrowser.Providers.csproj +++ b/MediaBrowser.Providers/MediaBrowser.Providers.csproj @@ -1,4 +1,4 @@ - + @@ -22,7 +22,6 @@ - diff --git a/MediaBrowser.Providers/Trickplay/TrickplayManager.cs b/MediaBrowser.Providers/Trickplay/TrickplayManager.cs index 2304f803ec..419adc4b03 100644 --- a/MediaBrowser.Providers/Trickplay/TrickplayManager.cs +++ b/MediaBrowser.Providers/Trickplay/TrickplayManager.cs @@ -6,6 +6,7 @@ using System.Linq; using System.Threading; using System.Threading.Tasks; using MediaBrowser.Controller.Configuration; +using MediaBrowser.Controller.Drawing; using MediaBrowser.Controller.Entities; using MediaBrowser.Controller.Library; using MediaBrowser.Controller.MediaEncoding; @@ -15,7 +16,6 @@ using MediaBrowser.Model.Configuration; using MediaBrowser.Model.Entities; using MediaBrowser.Model.IO; using Microsoft.Extensions.Logging; -using SkiaSharp; namespace MediaBrowser.Providers.Trickplay; @@ -31,6 +31,7 @@ public class TrickplayManager : ITrickplayManager private readonly EncodingHelper _encodingHelper; private readonly ILibraryManager _libraryManager; private readonly IServerConfigurationManager _config; + private readonly IImageEncoder _imageEncoder; private static readonly SemaphoreSlim _resourcePool = new(1, 1); private static readonly string[] _trickplayImgExtensions = { ".jpg" }; @@ -45,6 +46,7 @@ public class TrickplayManager : ITrickplayManager /// The encoding helper. /// The library manager. /// The server configuration manager. + /// The image encoder. public TrickplayManager( ILogger logger, IItemRepository itemRepo, @@ -52,7 +54,8 @@ public class TrickplayManager : ITrickplayManager IFileSystem fileSystem, EncodingHelper encodingHelper, ILibraryManager libraryManager, - IServerConfigurationManager config) + IServerConfigurationManager config, + IImageEncoder imageEncoder) { _logger = logger; _itemRepo = itemRepo; @@ -61,6 +64,7 @@ public class TrickplayManager : ITrickplayManager _encodingHelper = encodingHelper; _libraryManager = libraryManager; _config = config; + _imageEncoder = imageEncoder; } /// @@ -141,7 +145,8 @@ public class TrickplayManager : ITrickplayManager } var images = _fileSystem.GetFiles(imgTempDir, _trickplayImgExtensions, false, false) - .OrderBy(i => i.FullName) + .Select(i => i.FullName) + .OrderBy(i => i) .ToList(); // Create tiles @@ -185,11 +190,11 @@ public class TrickplayManager : ITrickplayManager } } - private TrickplayTilesInfo CreateTiles(List images, int width, TrickplayOptions options, string workDir, string outputDir) + private TrickplayTilesInfo CreateTiles(List images, int width, TrickplayOptions options, string workDir, string outputDir) { if (images.Count == 0) { - throw new InvalidOperationException("Can't create trickplay from 0 images."); + throw new ArgumentException("Can't create trickplay from 0 images."); } Directory.CreateDirectory(workDir); @@ -200,76 +205,42 @@ public class TrickplayManager : ITrickplayManager Interval = options.Interval, TileWidth = options.TileWidth, TileHeight = options.TileHeight, - TileCount = 0, + TileCount = images.Count, + // Set during image generation + Height = 0, Bandwidth = 0 }; - var firstImg = SKBitmap.Decode(images[0].FullName); - if (firstImg == null) - { - throw new InvalidDataException("Could not decode image data."); - } - - tilesInfo.Height = firstImg.Height; - if (tilesInfo.Width != firstImg.Width) - { - throw new InvalidOperationException("Image width does not match config width."); - } - /* - * Generate grids of trickplay image tiles + * Generate trickplay tile grids from sets of images */ - var imgNo = 0; - var i = 0; - while (i < images.Count) + var imageOptions = new ImageCollageOptions { - var tileGrid = new SKBitmap(tilesInfo.Width * tilesInfo.TileWidth, tilesInfo.Height * tilesInfo.TileHeight); + Width = tilesInfo.TileWidth, + Height = tilesInfo.TileHeight + }; - using (var canvas = new SKCanvas(tileGrid)) + var tilesPerGrid = tilesInfo.TileWidth * tilesInfo.TileHeight; + var requiredTileGrids = (int)Math.Ceiling((double)images.Count / tilesPerGrid); + + for (int i = 0; i < requiredTileGrids; i++) + { + // Set output/input paths + var tileGridPath = Path.Combine(workDir, $"{i}.jpg"); + + imageOptions.OutputPath = tileGridPath; + imageOptions.InputPaths = images.Skip(i * tilesPerGrid).Take(tilesPerGrid).ToList(); + + // Generate image and use returned height for tiles info + var height = _imageEncoder.CreateTrickplayGrid(imageOptions, options.JpegQuality, tilesInfo.Width, tilesInfo.Height != 0 ? tilesInfo.Height : null); + if (tilesInfo.Height == 0) { - for (var y = 0; y < tilesInfo.TileHeight; y++) - { - for (var x = 0; x < tilesInfo.TileWidth; x++) - { - if (i >= images.Count) - { - break; - } - - var img = SKBitmap.Decode(images[i].FullName); - if (img == null) - { - throw new InvalidDataException("Could not decode image data."); - } - - if (tilesInfo.Width != img.Width) - { - throw new InvalidOperationException("Image width does not match config width."); - } - - if (tilesInfo.Height != img.Height) - { - throw new InvalidOperationException("Image height does not match first image height."); - } - - canvas.DrawBitmap(img, x * tilesInfo.Width, y * tilesInfo.Height); - tilesInfo.TileCount++; - i++; - } - } - } - - // Output each tile grid to singular file - var tileGridPath = Path.Combine(workDir, $"{imgNo}.jpg"); - using (var stream = File.OpenWrite(tileGridPath)) - { - tileGrid.Encode(stream, SKEncodedImageFormat.Jpeg, options.JpegQuality); + tilesInfo.Height = height; } + // Update bitrate var bitrate = (int)Math.Ceiling((decimal)new FileInfo(tileGridPath).Length * 8 / tilesInfo.TileWidth / tilesInfo.TileHeight / (tilesInfo.Interval / 1000)); tilesInfo.Bandwidth = Math.Max(tilesInfo.Bandwidth, bitrate); - - imgNo++; } /* diff --git a/src/Jellyfin.Drawing.Skia/SkiaEncoder.cs b/src/Jellyfin.Drawing.Skia/SkiaEncoder.cs index 2d980db181..2facf0f370 100644 --- a/src/Jellyfin.Drawing.Skia/SkiaEncoder.cs +++ b/src/Jellyfin.Drawing.Skia/SkiaEncoder.cs @@ -2,14 +2,18 @@ using System; using System.Collections.Generic; using System.Globalization; using System.IO; +using System.Linq; +using System.Security.Cryptography.Xml; using BlurHashSharp.SkiaSharp; using Jellyfin.Extensions; using MediaBrowser.Common.Configuration; using MediaBrowser.Common.Extensions; using MediaBrowser.Controller.Drawing; +using MediaBrowser.Controller.LiveTv; using MediaBrowser.Model.Drawing; using Microsoft.Extensions.Logging; using SkiaSharp; +using static System.Net.Mime.MediaTypeNames; using SKSvg = SkiaSharp.Extended.Svg.SKSvg; namespace Jellyfin.Drawing.Skia; @@ -526,6 +530,81 @@ public class SkiaEncoder : IImageEncoder splashBuilder.GenerateSplash(posters, backdrops, outputPath); } + /// + public int CreateTrickplayGrid(ImageCollageOptions options, int quality, int imgWidth, int? imgHeight) + { + var paths = options.InputPaths; + var tileWidth = options.Width; + var tileHeight = options.Height; + + if (paths.Count < 1) + { + throw new ArgumentException("InputPaths cannot be empty."); + } + else if (paths.Count > tileWidth * tileHeight) + { + throw new ArgumentException($"InputPaths contains more images than would fit on {tileWidth}x{tileHeight} grid."); + } + + // If no height provided, use height of first image. + if (!imgHeight.HasValue) + { + using var firstImg = Decode(paths[0], false, null, out _); + + if (firstImg is null) + { + throw new InvalidDataException("Could not decode image data."); + } + + if (firstImg.Width != imgWidth) + { + throw new InvalidOperationException("Image width does not match provided width."); + } + + imgHeight = firstImg.Height; + } + + // Make horizontal strips using every provided image. + using var tileGrid = new SKBitmap(imgWidth * tileWidth, imgHeight.Value * tileHeight); + using var canvas = new SKCanvas(tileGrid); + + var imgIndex = 0; + for (var y = 0; y < tileHeight; y++) + { + for (var x = 0; x < tileWidth; x++) + { + if (imgIndex >= paths.Count) + { + break; + } + + using var img = Decode(paths[imgIndex++], false, null, out _); + + if (img is null) + { + throw new InvalidDataException("Could not decode image data."); + } + + if (img.Width != imgWidth) + { + throw new InvalidOperationException("Image width does not match provided width."); + } + + if (img.Height != imgHeight) + { + throw new InvalidOperationException("Image height does not match first image height."); + } + + canvas.DrawBitmap(img, x * imgWidth, y * imgHeight.Value); + } + } + + using var outputStream = new SKFileWStream(options.OutputPath); + tileGrid.Encode(outputStream, SKEncodedImageFormat.Jpeg, quality); + + return imgHeight.Value; + } + private void DrawIndicator(SKCanvas canvas, int imageWidth, int imageHeight, ImageProcessingOptions options) { try diff --git a/src/Jellyfin.Drawing/NullImageEncoder.cs b/src/Jellyfin.Drawing/NullImageEncoder.cs index 171128bed3..15345e1bc8 100644 --- a/src/Jellyfin.Drawing/NullImageEncoder.cs +++ b/src/Jellyfin.Drawing/NullImageEncoder.cs @@ -49,6 +49,12 @@ public class NullImageEncoder : IImageEncoder throw new NotImplementedException(); } + /// + public int CreateTrickplayGrid(ImageCollageOptions options, int quality, int imgWidth, int? imgHeight) + { + throw new NotImplementedException(); + } + /// public string GetImageBlurHash(int xComp, int yComp, string path) {