using System; using System.Collections; using System.Collections.Generic; using System.Diagnostics; using System.Globalization; using System.IO; using System.Linq; using System.Security.Cryptography; using System.Text; using System.Text.RegularExpressions; using System.Threading; using System.Threading.Tasks; using Jellyfin.Data.Enums; using Jellyfin.Plugin.Lyrics.Configuration; using Jellyfin.Plugin.Lyrics.Models; using MediaBrowser.Common.Configuration; using MediaBrowser.Controller.Dto; using MediaBrowser.Controller.Entities; using MediaBrowser.Controller.Entities.Audio; using MediaBrowser.Controller.Library; using MediaBrowser.Controller.Lyrics; using MediaBrowser.Model.Entities; using MediaBrowser.Model.Globalization; using MediaBrowser.Model.Lyrics; using MediaBrowser.Model.Tasks; using Microsoft.Extensions.Logging; namespace Jellyfin.Plugin.Lyrics; /// /// Task to download lyrics. /// public class LyricDownloadTask : IScheduledTask { private const int QueryPageLimit = 100; private const int DefaultMaxTracksPerRun = 2000; private const int MinimumMaxTracksPerRun = 100; private const int DefaultFailureStateTtlDays = 90; private const int MinimumFailureStateTtlDays = 1; private const int PeriodicStateSaveMutationThreshold = 200; private static readonly int[] DefaultBackoffScheduleDays = [1, 3, 7, 30]; private static readonly TimeSpan ErrorRetryDelay = TimeSpan.FromDays(1); // Matches an LRC line/word time tag like [00:12.34] or [1:02]; metadata tags ([ar:..], [length:03:21], ..) don't // match because a digit must immediately follow '['. private static readonly Regex SyncedLyricTagRegex = new(@"\[\d{1,3}:\d{2}([.:]\d{1,3})?\]", RegexOptions.Compiled); private static readonly BaseItemKind[] ItemKinds = [BaseItemKind.Audio]; private static readonly MediaType[] MediaTypes = [MediaType.Audio]; private static readonly SourceType[] SourceTypes = [SourceType.Library]; private static readonly DtoOptions DtoOptions = new(false); private readonly ILibraryManager _libraryManager; private readonly ILyricManager _lyricManager; private readonly RetryStateStore _retryStateStore; private readonly ILogger _logger; private readonly ILocalizationManager _localizationManager; /// /// Initializes a new instance of the class. /// /// Instance of the interface. /// Instance of the interface. /// Instance of the interface. /// Instance of the interface. /// Instance of the interface. /// Instance of the interface. public LyricDownloadTask( ILibraryManager libraryManager, ILyricManager lyricManager, IApplicationPaths applicationPaths, ILogger logger, ILoggerFactory loggerFactory, ILocalizationManager localizationManager) { _libraryManager = libraryManager; _lyricManager = lyricManager; _retryStateStore = new RetryStateStore(applicationPaths, loggerFactory.CreateLogger()); _logger = logger; _localizationManager = localizationManager; } /// public string Name => "Download and upgrade lyrics"; /// public string Key => "DLLyrics"; /// public string Description => "Task to download missing lyrics and upgrade plain lyrics to synced lyrics from lrclib.net"; /// public string Category => _localizationManager.GetLocalizedString("TasksLibraryCategory"); /// public async Task ExecuteAsync(IProgress progress, CancellationToken cancellationToken) { var stopwatch = Stopwatch.StartNew(); var nowUtc = DateTime.UtcNow; var configuration = GetSanitizedConfiguration(); var maxTracksPerRun = configuration.EnableRunCap ? configuration.MaxTracksPerRun : int.MaxValue; var backoffScheduleDays = configuration.BackoffScheduleDays.Length == 0 ? DefaultBackoffScheduleDays : configuration.BackoffScheduleDays; var query = new InternalItemsQuery { Recursive = true, IsVirtualItem = false, IncludeItemTypes = ItemKinds, DtoOptions = DtoOptions, MediaTypes = MediaTypes, SourceTypes = SourceTypes, Limit = QueryPageLimit }; var totalCount = _libraryManager.GetCount(query); if (totalCount == 0) { _logger.LogInformation("Lyrics task complete. No audio items found."); progress.Report(100); return; } var retryState = await _retryStateStore.LoadAsync(cancellationToken).ConfigureAwait(false); var prunedEntriesCount = PruneExpiredEntries(retryState, nowUtc, configuration.FailureStateTtlDays); var stateMutations = prunedEntriesCount; var currentIndex = NormalizeCursor(retryState.Cursor, totalCount); var visitedItemCount = 0; var processedTrackCount = 0; var capReached = false; var missingDownloadedCount = 0; var upgradedToSyncedCount = 0; var alreadySyncedSkippedCount = 0; var dbDesyncSkippedCount = 0; var plainNoSyncedFoundCount = 0; var missingNoLyricsFoundCount = 0; var backoffSkippedCount = 0; var errorsCount = 0; while (visitedItemCount < totalCount) { cancellationToken.ThrowIfCancellationRequested(); var pageStart = currentIndex - (currentIndex % QueryPageLimit); query.StartIndex = pageStart; var queryResult = _libraryManager.GetItemsResult(query); var pageItems = queryResult.Items; if (pageItems.Count == 0) { _logger.LogWarning("Lyrics task stopped early because no items were returned for StartIndex {StartIndex}.", pageStart); break; } var pageOffset = currentIndex - pageStart; if (pageOffset >= pageItems.Count) { currentIndex = (pageStart + pageItems.Count) % totalCount; continue; } for (var pageIndex = pageOffset; pageIndex < pageItems.Count && visitedItemCount < totalCount; pageIndex++) { cancellationToken.ThrowIfCancellationRequested(); var item = pageItems[pageIndex]; visitedItemCount++; currentIndex = (currentIndex + 1) % totalCount; progress.Report(100d * visitedItemCount / totalCount); if (item is not Audio audioItem) { continue; } if (processedTrackCount >= maxTracksPerRun) { capReached = true; break; } processedTrackCount++; var itemKey = audioItem.Id.ToString("N", CultureInfo.InvariantCulture); var trackSignature = ComputeTrackSignature(audioItem); var trackNowUtc = DateTime.UtcNow; if (retryState.Entries.TryGetValue(itemKey, out var retryEntry) && !string.Equals(retryEntry.TrackSignature, trackSignature, StringComparison.Ordinal)) { retryState.Entries.Remove(itemKey); retryEntry = null; stateMutations++; } if (configuration.EnableAdaptiveRetryBackoff && retryEntry is not null && retryEntry.NextRetryUtc > trackNowUtc) { backoffSkippedCount++; continue; } // Check the filesystem directly for existing lyric files to avoid re-downloading when the DB hasn't registered them yet. // Lyrics can sit next to the media file or, when "Save lyrics into media folders" is off, in the item's internal metadata folder. var lyricFileOnDisk = FindLyricFileOnDisk(audioItem); var lyricInFileFound = lyricFileOnDisk is not null; try { var existingLyrics = await _lyricManager.GetLyricsAsync(audioItem, cancellationToken).ConfigureAwait(false); if (existingLyrics is null && !lyricInFileFound) { _logger.LogDebug("Searching for lyrics for {Path}", audioItem.Path); var lyricResults = await _lyricManager.SearchLyricsAsync(audioItem, true, cancellationToken).ConfigureAwait(false); if (lyricResults.Count != 0) { _logger.LogDebug("Saving lyrics for {Path}", audioItem.Path); await _lyricManager.DownloadLyricsAsync(audioItem, lyricResults[0].Id, cancellationToken).ConfigureAwait(false); missingDownloadedCount++; stateMutations += ClearRetryState(retryState, itemKey); } else { missingNoLyricsFoundCount++; if (configuration.EnableAdaptiveRetryBackoff) { stateMutations += UpdateNoResultEntry(retryState, itemKey, trackSignature, trackNowUtc, backoffScheduleDays); } } } else if (existingLyrics is null && lyricInFileFound) { dbDesyncSkippedCount++; stateMutations += ClearRetryState(retryState, itemKey); } else if (HasSyncedLyrics(existingLyrics!)) { alreadySyncedSkippedCount++; stateMutations += ClearRetryState(retryState, itemKey); } else if (lyricFileOnDisk is not null && IsSyncedLyricFile(lyricFileOnDisk)) { // DB still reports plain, but a synced .lrc already sits on disk (core doesn't refresh the DB // after a download). Nothing to upgrade; re-downloading would loop until the next library scan. dbDesyncSkippedCount++; stateMutations += ClearRetryState(retryState, itemKey); } else { _logger.LogDebug("Checking upgrade to synced lyrics for {Path}", audioItem.Path); var lyricResults = await _lyricManager.SearchLyricsAsync(audioItem, true, cancellationToken).ConfigureAwait(false); var syncedCandidate = SelectBestSyncedCandidate(lyricResults); if (syncedCandidate is not null) { _logger.LogDebug("Upgrading to synced lyrics for {Path}", audioItem.Path); await _lyricManager.DownloadLyricsAsync(audioItem, syncedCandidate.Id, cancellationToken).ConfigureAwait(false); upgradedToSyncedCount++; stateMutations += ClearRetryState(retryState, itemKey); } else { plainNoSyncedFoundCount++; if (configuration.EnableAdaptiveRetryBackoff) { stateMutations += UpdateNoResultEntry(retryState, itemKey, trackSignature, trackNowUtc, backoffScheduleDays); } } } } catch (Exception ex) { errorsCount++; _logger.LogError(ex, "Error processing lyrics for {Path}", audioItem.Path); if (configuration.EnableAdaptiveRetryBackoff) { stateMutations += UpdateErrorEntry(retryState, itemKey, trackSignature, trackNowUtc); } } if (stateMutations >= PeriodicStateSaveMutationThreshold) { await _retryStateStore.SaveAsync(retryState, cancellationToken).ConfigureAwait(false); stateMutations = 0; } } if (capReached) { break; } } retryState.Cursor = NormalizeCursor(currentIndex, totalCount); await _retryStateStore.SaveAsync(retryState, cancellationToken).ConfigureAwait(false); if (LyricsPlugin.Instance is not null) { LyricsPlugin.Instance.Configuration.StateCursor = retryState.Cursor; } _logger.LogInformation( "Lyrics task complete in {Elapsed}. Processed tracks: {ProcessedTrackCount}, visited items: {VisitedItemCount}/{TotalCount}, cap reached: {CapReached}, missing downloaded: {MissingDownloadedCount}, upgraded to synced: {UpgradedToSyncedCount}, already synced skipped: {AlreadySyncedSkippedCount}, db-desync skipped: {DbDesyncSkippedCount}, missing with no lyrics found: {MissingNoLyricsFoundCount}, plain with no synced found: {PlainNoSyncedFoundCount}, backoff skipped: {BackoffSkippedCount}, pruned retry-state entries: {PrunedEntriesCount}, errors: {ErrorsCount}", stopwatch.Elapsed, processedTrackCount, visitedItemCount, totalCount, capReached, missingDownloadedCount, upgradedToSyncedCount, alreadySyncedSkippedCount, dbDesyncSkippedCount, missingNoLyricsFoundCount, plainNoSyncedFoundCount, backoffSkippedCount, prunedEntriesCount, errorsCount); progress.Report(100); } /// public IEnumerable GetDefaultTriggers() { return [ new TaskTriggerInfo { Type = TaskTriggerInfoType.IntervalTrigger, IntervalTicks = TimeSpan.FromHours(24).Ticks } ]; } private static bool HasSyncedLyrics(LyricDto existingLyrics) { return existingLyrics.Metadata?.IsSynced == true; } // Returns the path of an existing lyric sidecar (preferring synced .lrc over plain .txt), or null when none exists. private static string? FindLyricFileOnDisk(Audio audioItem) { if (string.IsNullOrEmpty(audioItem.Path)) { return null; } // Next to the media file (library option "Save lyrics into media folders" enabled). var lrcNextTo = Path.ChangeExtension(audioItem.Path, ".lrc"); if (File.Exists(lrcNextTo)) { return lrcNextTo; } var txtNextTo = Path.ChangeExtension(audioItem.Path, ".txt"); if (File.Exists(txtNextTo)) { return txtNextTo; } // Internal metadata folder (library option disabled): {InternalMetadataPath}/{fileNameWithoutExt}.{format} var metadataPath = audioItem.GetInternalMetadataPath(); if (string.IsNullOrEmpty(metadataPath)) { return null; } var baseName = Path.GetFileNameWithoutExtension(audioItem.Path); var lrcMeta = Path.Combine(metadataPath, baseName + ".lrc"); if (File.Exists(lrcMeta)) { return lrcMeta; } var txtMeta = Path.Combine(metadataPath, baseName + ".txt"); return File.Exists(txtMeta) ? txtMeta : null; } // Treats a lyric file as synced when any line carries an LRC time tag. Content-based so it works regardless of // extension or which tool wrote the file. On read failure, falls back to "not synced" so normal logic proceeds. private static bool IsSyncedLyricFile(string path) { try { foreach (var line in File.ReadLines(path)) { if (SyncedLyricTagRegex.IsMatch(line)) { return true; } } } catch (IOException) { } catch (UnauthorizedAccessException) { } return false; } private static PluginConfiguration GetSanitizedConfiguration() { var configuration = LyricsPlugin.Instance?.Configuration ?? new PluginConfiguration(); configuration.MaxTracksPerRun = configuration.MaxTracksPerRun <= 0 ? DefaultMaxTracksPerRun : Math.Max(configuration.MaxTracksPerRun, MinimumMaxTracksPerRun); configuration.FailureStateTtlDays = configuration.FailureStateTtlDays <= 0 ? DefaultFailureStateTtlDays : Math.Max(configuration.FailureStateTtlDays, MinimumFailureStateTtlDays); configuration.BackoffScheduleDays = (configuration.BackoffScheduleDays ?? DefaultBackoffScheduleDays) .Where(static value => value > 0) .Distinct() .ToArray(); return configuration; } private static int NormalizeCursor(int cursor, int totalCount) { if (totalCount <= 0) { return 0; } if (cursor < 0) { cursor = 0; } return cursor % totalCount; } private static int PruneExpiredEntries(LyricsRetryState state, DateTime nowUtc, int ttlDays) { var cutoffUtc = nowUtc.AddDays(-ttlDays); var keysToRemove = state.Entries .Where(static kvp => kvp.Value.LastAttemptUtc != default) .Where(kvp => kvp.Value.LastAttemptUtc < cutoffUtc) .Select(static kvp => kvp.Key) .ToArray(); foreach (var key in keysToRemove) { state.Entries.Remove(key); } return keysToRemove.Length; } private static int ClearRetryState(LyricsRetryState state, string itemKey) { return state.Entries.Remove(itemKey) ? 1 : 0; } private static int UpdateNoResultEntry( LyricsRetryState state, string itemKey, string trackSignature, DateTime nowUtc, int[] backoffScheduleDays) { if (!state.Entries.TryGetValue(itemKey, out var entry)) { entry = new LyricsRetryEntry(); } entry.TrackSignature = trackSignature; entry.ConsecutiveNoResultCount++; entry.LastAttemptUtc = nowUtc; entry.LastOutcome = LyricsRetryOutcome.NoResult; var retryIndex = Math.Min(entry.ConsecutiveNoResultCount - 1, backoffScheduleDays.Length - 1); entry.NextRetryUtc = nowUtc.AddDays(backoffScheduleDays[retryIndex]); state.Entries[itemKey] = entry; return 1; } private static int UpdateErrorEntry( LyricsRetryState state, string itemKey, string trackSignature, DateTime nowUtc) { if (!state.Entries.TryGetValue(itemKey, out var entry)) { entry = new LyricsRetryEntry(); } entry.TrackSignature = trackSignature; entry.LastAttemptUtc = nowUtc; entry.LastOutcome = LyricsRetryOutcome.Error; entry.NextRetryUtc = nowUtc.Add(ErrorRetryDelay); state.Entries[itemKey] = entry; return 1; } private static string ComputeTrackSignature(Audio audioItem) { var builder = new StringBuilder(); builder.Append(NormalizeForSignature(audioItem.Name)); builder.Append('|'); builder.Append(NormalizeForSignature(audioItem.Path)); builder.Append('|'); builder.Append(audioItem.RunTimeTicks?.ToString(CultureInfo.InvariantCulture) ?? "0"); builder.Append('|'); builder.Append(NormalizeForSignature(GetStringProperty(audioItem, "Album"))); builder.Append('|'); builder.Append(NormalizeForSignature(GetJoinedEnumerableProperty(audioItem, "Artists"))); builder.Append('|'); builder.Append(NormalizeForSignature(GetJoinedEnumerableProperty(audioItem, "AlbumArtists"))); var hash = SHA256.HashData(Encoding.UTF8.GetBytes(builder.ToString())); return Convert.ToHexString(hash); } private static string NormalizeForSignature(string? value) { return string.IsNullOrWhiteSpace(value) ? string.Empty : value.Trim().ToLowerInvariant(); } private static string? GetStringProperty(object instance, string propertyName) { return instance.GetType().GetProperty(propertyName)?.GetValue(instance) as string; } private static string GetJoinedEnumerableProperty(object instance, string propertyName) { if (instance.GetType().GetProperty(propertyName)?.GetValue(instance) is not IEnumerable values) { return string.Empty; } var normalizedValues = new List(); foreach (var value in values) { if (value is string str && !string.IsNullOrWhiteSpace(str)) { normalizedValues.Add(str.Trim()); } } return string.Join('|', normalizedValues); } private static RemoteLyricInfoDto? SelectBestSyncedCandidate(IReadOnlyList lyricResults) { return lyricResults.FirstOrDefault(static x => x.Lyrics?.Metadata?.IsSynced == true); } }