From e9f23c61c937b230b1d3bd7865a083aeb3d51657 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20M=C3=BCller?= Date: Tue, 1 Aug 2023 17:11:32 +0200 Subject: [PATCH 1/5] Fix the fLaC/flac HLS issue also for audio-only I moved the first application of the workaround out of the if block so that it also applies to audio-only streams. The workaround was extended likewise. We should first and foremost adhere to the specifications and apply workarounds afterwards for software that doesn't follow them. So I turned around the workaround to first output the fLaC variant and then the alternative flac variant. Fixes: #10066 --- Jellyfin.Api/Helpers/DynamicHlsHelper.cs | 21 +++++++++++-------- Jellyfin.Api/Helpers/HlsCodecStringHelpers.cs | 6 ++++-- 2 files changed, 16 insertions(+), 11 deletions(-) diff --git a/Jellyfin.Api/Helpers/DynamicHlsHelper.cs b/Jellyfin.Api/Helpers/DynamicHlsHelper.cs index 63667e7e69..dfcccddfc2 100644 --- a/Jellyfin.Api/Helpers/DynamicHlsHelper.cs +++ b/Jellyfin.Api/Helpers/DynamicHlsHelper.cs @@ -198,15 +198,15 @@ public class DynamicHlsHelper var basicPlaylist = AppendPlaylist(builder, state, playlistUrl, totalBitrate, subtitleGroup); + // Provide a workaround for the case issue between flac and fLaC. + var flacWaPlaylist = ApplyFlacCaseWorkaround(state, basicPlaylist.ToString()); + if (!string.IsNullOrEmpty(flacWaPlaylist)) + { + builder.Append(flacWaPlaylist); + } + if (state.VideoStream is not null && state.VideoRequest is not null) { - // Provide a workaround for the case issue between flac and fLaC. - var flacWaPlaylist = ApplyFlacCaseWorkaround(state, basicPlaylist.ToString()); - if (!string.IsNullOrEmpty(flacWaPlaylist)) - { - builder.Append(flacWaPlaylist); - } - var encodingOptions = _serverConfigurationManager.GetEncodingOptions(); // Provide SDR HEVC entrance for backward compatibility. @@ -775,8 +775,11 @@ public class DynamicHlsHelper return string.Empty; } - var newPlaylist = srcPlaylist.Replace(",flac\"", ",fLaC\"", StringComparison.Ordinal); + var newPlaylist = srcPlaylist; - return newPlaylist.Contains(",fLaC\"", StringComparison.Ordinal) ? newPlaylist : string.Empty; + newPlaylist = newPlaylist.Replace(",fLaC\"", ",flac\"", StringComparison.Ordinal); + newPlaylist = newPlaylist.Replace("\"fLaC\"", "\"flac\"", StringComparison.Ordinal); + + return string.Equals(srcPlaylist, newPlaylist, StringComparison.Ordinal) ? string.Empty : newPlaylist; } } diff --git a/Jellyfin.Api/Helpers/HlsCodecStringHelpers.cs b/Jellyfin.Api/Helpers/HlsCodecStringHelpers.cs index 9a141a16d9..9b1c52045f 100644 --- a/Jellyfin.Api/Helpers/HlsCodecStringHelpers.cs +++ b/Jellyfin.Api/Helpers/HlsCodecStringHelpers.cs @@ -5,7 +5,9 @@ using System.Text; namespace Jellyfin.Api.Helpers; /// -/// Hls Codec string helpers. +/// Helpers to generate HLS codec strings according to +/// RFC 6381 section 3.3 +/// and the MP4 Registration Authority. /// public static class HlsCodecStringHelpers { @@ -27,7 +29,7 @@ public static class HlsCodecStringHelpers /// /// Codec name for FLAC. /// - public const string FLAC = "flac"; + public const string FLAC = "fLaC"; /// /// Codec name for ALAC. From 19fb061381dd107d5e0236cf9d8b59b2e2318130 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20M=C3=BCller?= Date: Tue, 1 Aug 2023 17:22:42 +0200 Subject: [PATCH 2/5] Correct the HLS Opus codec string Apple doesn't support Opus via HLS yet, but if they ever do, they will definitely expect "Opus" instead of "opus". See https://mp4ra.org/#/codecs Fixes: #10066 --- Jellyfin.Api/Helpers/DynamicHlsHelper.cs | 28 ++++++++++--------- Jellyfin.Api/Helpers/HlsCodecStringHelpers.cs | 2 +- 2 files changed, 16 insertions(+), 14 deletions(-) diff --git a/Jellyfin.Api/Helpers/DynamicHlsHelper.cs b/Jellyfin.Api/Helpers/DynamicHlsHelper.cs index dfcccddfc2..888b667a68 100644 --- a/Jellyfin.Api/Helpers/DynamicHlsHelper.cs +++ b/Jellyfin.Api/Helpers/DynamicHlsHelper.cs @@ -198,11 +198,11 @@ public class DynamicHlsHelper var basicPlaylist = AppendPlaylist(builder, state, playlistUrl, totalBitrate, subtitleGroup); - // Provide a workaround for the case issue between flac and fLaC. - var flacWaPlaylist = ApplyFlacCaseWorkaround(state, basicPlaylist.ToString()); - if (!string.IsNullOrEmpty(flacWaPlaylist)) + // Provide a workaround for alternative codec string capitalization. + var alternativeCodecCapitalizationPlaylist = ApplyCodecCapitalizationWorkaround(state, basicPlaylist.ToString()); + if (!string.IsNullOrEmpty(alternativeCodecCapitalizationPlaylist)) { - builder.Append(flacWaPlaylist); + builder.Append(alternativeCodecCapitalizationPlaylist); } if (state.VideoStream is not null && state.VideoRequest is not null) @@ -238,11 +238,11 @@ public class DynamicHlsHelper var sdrTotalBitrate = sdrOutputAudioBitrate + sdrOutputVideoBitrate; var sdrPlaylist = AppendPlaylist(builder, state, sdrVideoUrl, sdrTotalBitrate, subtitleGroup); - // Provide a workaround for the case issue between flac and fLaC. - flacWaPlaylist = ApplyFlacCaseWorkaround(state, sdrPlaylist.ToString()); - if (!string.IsNullOrEmpty(flacWaPlaylist)) + // Provide a workaround for alternative codec string capitalization. + alternativeCodecCapitalizationPlaylist = ApplyCodecCapitalizationWorkaround(state, sdrPlaylist.ToString()); + if (!string.IsNullOrEmpty(alternativeCodecCapitalizationPlaylist)) { - builder.Append(flacWaPlaylist); + builder.Append(alternativeCodecCapitalizationPlaylist); } // Restore the video codec @@ -275,11 +275,11 @@ public class DynamicHlsHelper var newPlaylist = ReplacePlaylistCodecsField(basicPlaylist, playlistCodecsField, newPlaylistCodecsField); builder.Append(newPlaylist); - // Provide a workaround for the case issue between flac and fLaC. - flacWaPlaylist = ApplyFlacCaseWorkaround(state, newPlaylist); - if (!string.IsNullOrEmpty(flacWaPlaylist)) + // Provide a workaround for alternative codec string capitalization. + alternativeCodecCapitalizationPlaylist = ApplyCodecCapitalizationWorkaround(state, newPlaylist); + if (!string.IsNullOrEmpty(alternativeCodecCapitalizationPlaylist)) { - builder.Append(flacWaPlaylist); + builder.Append(alternativeCodecCapitalizationPlaylist); } } } @@ -768,7 +768,7 @@ public class DynamicHlsHelper StringComparison.Ordinal); } - private string ApplyFlacCaseWorkaround(StreamState state, string srcPlaylist) + private string ApplyCodecCapitalizationWorkaround(StreamState state, string srcPlaylist) { if (!string.Equals(state.ActualOutputAudioCodec, "flac", StringComparison.OrdinalIgnoreCase)) { @@ -779,6 +779,8 @@ public class DynamicHlsHelper newPlaylist = newPlaylist.Replace(",fLaC\"", ",flac\"", StringComparison.Ordinal); newPlaylist = newPlaylist.Replace("\"fLaC\"", "\"flac\"", StringComparison.Ordinal); + newPlaylist = newPlaylist.Replace(",Opus\"", ",opus\"", StringComparison.Ordinal); + newPlaylist = newPlaylist.Replace("\"Opus\"", "\"opus\"", StringComparison.Ordinal); return string.Equals(srcPlaylist, newPlaylist, StringComparison.Ordinal) ? string.Empty : newPlaylist; } diff --git a/Jellyfin.Api/Helpers/HlsCodecStringHelpers.cs b/Jellyfin.Api/Helpers/HlsCodecStringHelpers.cs index 9b1c52045f..5eec1b0ca6 100644 --- a/Jellyfin.Api/Helpers/HlsCodecStringHelpers.cs +++ b/Jellyfin.Api/Helpers/HlsCodecStringHelpers.cs @@ -39,7 +39,7 @@ public static class HlsCodecStringHelpers /// /// Codec name for OPUS. /// - public const string OPUS = "opus"; + public const string OPUS = "Opus"; /// /// Gets a MP3 codec string. From 79cff704ff4e00895a8d2b97ecbbea3e2f5f56fc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20M=C3=BCller?= Date: Tue, 1 Aug 2023 17:28:49 +0200 Subject: [PATCH 3/5] Allow flac inside mp4 for all HLS audio streams The -strict -2 setting was only added if the encoder was set to 'copy'. If 'flac' is explicitly requested, we also need to set it, so that ffmpeg doesn't abort the conversion. Fixes: #10066 --- .../Controllers/DynamicHlsController.cs | 26 +++++++++---------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/Jellyfin.Api/Controllers/DynamicHlsController.cs b/Jellyfin.Api/Controllers/DynamicHlsController.cs index ce684e457c..94093f1671 100644 --- a/Jellyfin.Api/Controllers/DynamicHlsController.cs +++ b/Jellyfin.Api/Controllers/DynamicHlsController.cs @@ -1704,18 +1704,29 @@ public class DynamicHlsController : BaseJellyfinApiController var audioCodec = _encodingHelper.GetAudioEncoder(state); + // dts, flac, opus and truehd are experimental in mp4 muxer + var strictArgs = string.Empty; + var actualOutputAudioCodec = state.ActualOutputAudioCodec; + if (string.Equals(actualOutputAudioCodec, "flac", StringComparison.OrdinalIgnoreCase) + || string.Equals(actualOutputAudioCodec, "opus", StringComparison.OrdinalIgnoreCase) + || string.Equals(actualOutputAudioCodec, "dts", StringComparison.OrdinalIgnoreCase) + || string.Equals(actualOutputAudioCodec, "truehd", StringComparison.OrdinalIgnoreCase)) + { + strictArgs = " -strict -2"; + } + if (!state.IsOutputVideo) { if (EncodingHelper.IsCopyCodec(audioCodec)) { var bitStreamArgs = EncodingHelper.GetAudioBitStreamArguments(state, state.Request.SegmentContainer, state.MediaSource.Container); - return "-acodec copy -strict -2" + bitStreamArgs; + return "-acodec copy" + bitStreamArgs + strictArgs; } var audioTranscodeParams = string.Empty; - audioTranscodeParams += "-acodec " + audioCodec; + audioTranscodeParams += "-acodec " + audioCodec + strictArgs; var audioBitrate = state.OutputAudioBitrate; var audioChannels = state.OutputAudioChannels; @@ -1747,17 +1758,6 @@ public class DynamicHlsController : BaseJellyfinApiController return audioTranscodeParams; } - // dts, flac, opus and truehd are experimental in mp4 muxer - var strictArgs = string.Empty; - var actualOutputAudioCodec = state.ActualOutputAudioCodec; - if (string.Equals(actualOutputAudioCodec, "flac", StringComparison.OrdinalIgnoreCase) - || string.Equals(actualOutputAudioCodec, "opus", StringComparison.OrdinalIgnoreCase) - || string.Equals(actualOutputAudioCodec, "dts", StringComparison.OrdinalIgnoreCase) - || string.Equals(actualOutputAudioCodec, "truehd", StringComparison.OrdinalIgnoreCase)) - { - strictArgs = " -strict -2"; - } - if (EncodingHelper.IsCopyCodec(audioCodec)) { var videoCodec = _encodingHelper.GetVideoEncoder(state, _encodingOptions); From 1635d82345a035c007daffe980cc09de17984813 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20M=C3=BCller?= Date: Sat, 16 Sep 2023 12:57:20 +0200 Subject: [PATCH 4/5] Remove workaround for codec capitalization This is not required anymore as Shaka Player now supports the correct codec strings. --- Jellyfin.Api/Helpers/DynamicHlsHelper.cs | 40 +----------------------- 1 file changed, 1 insertion(+), 39 deletions(-) diff --git a/Jellyfin.Api/Helpers/DynamicHlsHelper.cs b/Jellyfin.Api/Helpers/DynamicHlsHelper.cs index 03b723ef1b..276a09f41e 100644 --- a/Jellyfin.Api/Helpers/DynamicHlsHelper.cs +++ b/Jellyfin.Api/Helpers/DynamicHlsHelper.cs @@ -198,13 +198,6 @@ public class DynamicHlsHelper var basicPlaylist = AppendPlaylist(builder, state, playlistUrl, totalBitrate, subtitleGroup); - // Provide a workaround for alternative codec string capitalization. - var alternativeCodecCapitalizationPlaylist = ApplyCodecCapitalizationWorkaround(state, basicPlaylist.ToString()); - if (!string.IsNullOrEmpty(alternativeCodecCapitalizationPlaylist)) - { - builder.Append(alternativeCodecCapitalizationPlaylist); - } - if (state.VideoStream is not null && state.VideoRequest is not null) { var encodingOptions = _serverConfigurationManager.GetEncodingOptions(); @@ -236,14 +229,7 @@ public class DynamicHlsHelper } var sdrTotalBitrate = sdrOutputAudioBitrate + sdrOutputVideoBitrate; - var sdrPlaylist = AppendPlaylist(builder, state, sdrVideoUrl, sdrTotalBitrate, subtitleGroup); - - // Provide a workaround for alternative codec string capitalization. - alternativeCodecCapitalizationPlaylist = ApplyCodecCapitalizationWorkaround(state, sdrPlaylist.ToString()); - if (!string.IsNullOrEmpty(alternativeCodecCapitalizationPlaylist)) - { - builder.Append(alternativeCodecCapitalizationPlaylist); - } + AppendPlaylist(builder, state, sdrVideoUrl, sdrTotalBitrate, subtitleGroup); // Restore the video codec state.OutputVideoCodec = "copy"; @@ -274,13 +260,6 @@ public class DynamicHlsHelper state.VideoStream.Level = originalLevel; var newPlaylist = ReplacePlaylistCodecsField(basicPlaylist, playlistCodecsField, newPlaylistCodecsField); builder.Append(newPlaylist); - - // Provide a workaround for alternative codec string capitalization. - alternativeCodecCapitalizationPlaylist = ApplyCodecCapitalizationWorkaround(state, newPlaylist); - if (!string.IsNullOrEmpty(alternativeCodecCapitalizationPlaylist)) - { - builder.Append(alternativeCodecCapitalizationPlaylist); - } } } @@ -767,21 +746,4 @@ public class DynamicHlsHelper newValue.ToString(), StringComparison.Ordinal); } - - private string ApplyCodecCapitalizationWorkaround(StreamState state, string srcPlaylist) - { - if (!string.Equals(state.ActualOutputAudioCodec, "flac", StringComparison.OrdinalIgnoreCase)) - { - return string.Empty; - } - - var newPlaylist = srcPlaylist; - - newPlaylist = newPlaylist.Replace(",fLaC\"", ",flac\"", StringComparison.Ordinal); - newPlaylist = newPlaylist.Replace("\"fLaC\"", "\"flac\"", StringComparison.Ordinal); - newPlaylist = newPlaylist.Replace(",Opus\"", ",opus\"", StringComparison.Ordinal); - newPlaylist = newPlaylist.Replace("\"Opus\"", "\"opus\"", StringComparison.Ordinal); - - return string.Equals(srcPlaylist, newPlaylist, StringComparison.Ordinal) ? string.Empty : newPlaylist; - } } From aa073748c00fe2b0508fde88d900143acec09e36 Mon Sep 17 00:00:00 2001 From: Nyanmisaka Date: Mon, 9 Oct 2023 23:12:41 +0800 Subject: [PATCH 5/5] Drop experimental status of flac-in-MP4 for FFmpeg 6+ Signed-off-by: nyanmisaka --- Jellyfin.Api/Controllers/DynamicHlsController.cs | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/Jellyfin.Api/Controllers/DynamicHlsController.cs b/Jellyfin.Api/Controllers/DynamicHlsController.cs index 760fefbf51..44afdb6b11 100644 --- a/Jellyfin.Api/Controllers/DynamicHlsController.cs +++ b/Jellyfin.Api/Controllers/DynamicHlsController.cs @@ -45,6 +45,8 @@ public class DynamicHlsController : BaseJellyfinApiController private const string DefaultEventEncoderPreset = "superfast"; private const TranscodingJobType TranscodingJobType = MediaBrowser.Controller.MediaEncoding.TranscodingJobType.Hls; + private readonly Version _minFFmpegFlacInMp4 = new Version(6, 0); + private readonly ILibraryManager _libraryManager; private readonly IUserManager _userManager; private readonly IDlnaManager _dlnaManager; @@ -1705,13 +1707,14 @@ public class DynamicHlsController : BaseJellyfinApiController var audioCodec = _encodingHelper.GetAudioEncoder(state); var bitStreamArgs = EncodingHelper.GetAudioBitStreamArguments(state, state.Request.SegmentContainer, state.MediaSource.Container); - // dts, flac, opus and truehd are experimental in mp4 muxer + // opus, dts, truehd and flac (in FFmpeg 5 and older) are experimental in mp4 muxer var strictArgs = string.Empty; var actualOutputAudioCodec = state.ActualOutputAudioCodec; - if (string.Equals(actualOutputAudioCodec, "flac", StringComparison.OrdinalIgnoreCase) - || string.Equals(actualOutputAudioCodec, "opus", StringComparison.OrdinalIgnoreCase) + if (string.Equals(actualOutputAudioCodec, "opus", StringComparison.OrdinalIgnoreCase) || string.Equals(actualOutputAudioCodec, "dts", StringComparison.OrdinalIgnoreCase) - || string.Equals(actualOutputAudioCodec, "truehd", StringComparison.OrdinalIgnoreCase)) + || string.Equals(actualOutputAudioCodec, "truehd", StringComparison.OrdinalIgnoreCase) + || (string.Equals(actualOutputAudioCodec, "flac", StringComparison.OrdinalIgnoreCase) + && _mediaEncoder.EncoderVersion < _minFFmpegFlacInMp4)) { strictArgs = " -strict -2"; }