Skip to content

fix(search): read Indexer.MinimumAge and Indexer.MaximumSize, and measure age in UTC #921

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
m4bard wants to merge 1 commit into
base: canary
Choose a base branch
from
Open

fix(search): read Indexer.MinimumAge and Indexer.MaximumSize, and measure age in UTC #921

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
81 changes: 61 additions & 20 deletions listenarr.application/Search/Scoring/SearchResultScorer.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,8 @@
*/
using Microsoft.Extensions.Logging;

using System.Globalization;

namespace Listenarr.Application.Search.Scoring
{
public class SearchResultScorer
Expand Down Expand Up @@ -89,6 +91,43 @@ public async Task<QualityScore> Score(SearchResult searchResult, QualityProfile
// Detect NZB/Usenet more broadly
var isNzb = IsNzbResult(searchResult);

// The indexer is read before the size and age gates because all three depend on it.
// It also corrects isNzb from the indexer's own type, and that correction used to
// happen after the size gate had already run, so a Usenet result recognised only by
// its indexer type was size-checked despite the exemption just below.
int indexerRetention = 0;
int indexerMaximumSizeMb = 0;
int indexerMinimumAgeMinutes = 0;
if (searchResult.IndexerId.HasValue && _indexerRepository != null)
{
try
{
var idx = await _indexerRepository.GetByIdAsync(searchResult.IndexerId.Value);
if (idx != null)
{
indexerRetention = idx.Retention;
indexerMaximumSizeMb = idx.MaximumSize;
indexerMinimumAgeMinutes = idx.MinimumAge;
if (!isNzb && !string.IsNullOrWhiteSpace(idx.Type) && string.Equals(idx.Type, "Usenet", StringComparison.OrdinalIgnoreCase))
{
isNzb = true;
_logger.LogDebug("Indexer {IndexerId} type '{Type}' detected as Usenet; applying NZB/Usenet exemptions", searchResult.IndexerId.Value, idx.Type);
}
}
}
catch (Exception ex) when (ex is not OperationCanceledException && ex is not OutOfMemoryException && ex is not StackOverflowException)
{
_logger.LogDebug(ex, "Failed to fetch indexer settings for IndexerId {Id}", searchResult.IndexerId.Value);
}
}

if (indexerMaximumSizeMb > 0 && searchResult.Size > (long)indexerMaximumSizeMb * 1024 * 1024)
{
score.RejectionReasons.Add($"File too large for indexer (> {indexerMaximumSizeMb} MB)");
score.TotalScore = -1;
return score;
}

// Size checks (skip for NZB)
if (!isNzb && searchResult.Size > 0)
{
Expand All @@ -115,33 +154,35 @@ public async Task<QualityScore> Score(SearchResult searchResult, QualityProfile
return score;
}

// Age checks and indexer retention
double ageDays = 0;
int indexerRetention = 0;
if (searchResult.IndexerId.HasValue && _indexerRepository != null)

// Parsed to UTC explicitly. A bare TryParse converts a trailing Z to the host's local
// time and returns Kind=Local, and this then subtracts it from DateTime.UtcNow, so
// every age was out by the server's UTC offset: results looked older west of UTC and
// newer east of it. AssumeUniversal covers indexer dates that carry no offset at all.
if (!string.IsNullOrEmpty(searchResult.PublishedDate)
&& DateTime.TryParse(
searchResult.PublishedDate,
CultureInfo.InvariantCulture,
DateTimeStyles.AdjustToUniversal | DateTimeStyles.AssumeUniversal,
out var publishDate))
{
try
ageDays = (DateTime.UtcNow - publishDate).TotalDays;

// Usenet only, and the reason is propagation rather than preference: a post that
// has not finished propagating downloads as an incomplete or failed grab. Sonarr
// and Radarr expose the same per-indexer minimum for the same reason.
if (isNzb && indexerMinimumAgeMinutes > 0)
{
var idx = await _indexerRepository.GetByIdAsync(searchResult.IndexerId.Value);
if (idx != null)
var ageMinutes = (DateTime.UtcNow - publishDate).TotalMinutes;
if (ageMinutes < indexerMinimumAgeMinutes)
{
indexerRetention = idx.Retention;
if (!isNzb && !string.IsNullOrWhiteSpace(idx.Type) && string.Equals(idx.Type, "Usenet", StringComparison.OrdinalIgnoreCase))
{
isNzb = true;
_logger.LogDebug("Indexer {IndexerId} type '{Type}' detected as Usenet; applying NZB/Usenet exemptions", searchResult.IndexerId.Value, idx.Type);
}
score.RejectionReasons.Add($"Too new ({(int)ageMinutes} minutes < indexer minimum age {indexerMinimumAgeMinutes} minutes)");
score.TotalScore = -1;
return score;
}
}
catch (Exception ex) when (ex is not OperationCanceledException && ex is not OutOfMemoryException && ex is not StackOverflowException)
{
_logger.LogDebug(ex, "Failed to fetch indexer retention for IndexerId {Id}", searchResult.IndexerId.Value);
}
}

if (!string.IsNullOrEmpty(searchResult.PublishedDate) && DateTime.TryParse(searchResult.PublishedDate, out var publishDate))
{
ageDays = (DateTime.UtcNow - publishDate).TotalDays;
if (isNzb)
{
if (indexerRetention > 0 && ageDays > indexerRetention)
Expand Down
166 changes: 166 additions & 0 deletions tests/Features/Application/Audiobooks/Quality/QualityProfileScoringTests.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -491,6 +491,172 @@ public async Task Age_Rejection_Applied_When_Age_Exceeds_IndexerRetention()
Assert.True(score.TotalScore < 0, "Result should be rejected for age exceeding indexer retention");
}

[Fact]
public async Task Age_Is_Measured_In_Utc_Whatever_Offset_The_Indexer_Sends()
{
// The published date here is ten minutes old, written with a +09:00 offset. A bare
// DateTime.TryParse converts it to the host's local time and hands back Kind=Local,
// which is then subtracted from DateTime.UtcNow, so the age came out wrong by the
// difference between the two offsets. That made every age check depend on where the
// server was, and for a minutes-scale check like this one it decides the outcome.
var options = new Microsoft.EntityFrameworkCore.DbContextOptionsBuilder<ListenArrDbContext>().UseInMemoryDatabase(Guid.NewGuid().ToString()).Options;
using var db = new ListenArrDbContext(options);
var indexer = new Listenarr.Domain.Search.Indexer
{
Name = "OffsetIndexer",
Url = "https://offset.local",
MinimumAge = 120,
Retention = 3650,
IsEnabled = true
};
db.Indexers.Add(indexer);
db.SaveChanges();

var service = new QualityProfileService(new QualityProfileRepository(db), NullLogger<QualityProfileService>.Instance, new EfIndexerRepository(db));
var profile = new QualityProfile { MaximumAge = 3650, MinimumSeeders = 0 };

var tenMinutesAgoInTokyo = DateTimeOffset.UtcNow
.AddMinutes(-10)
.ToOffset(TimeSpan.FromHours(9))
.ToString("o");

var result = new SearchResult
{
Title = "Fresh Post From Another Timezone",
PublishedDate = tenMinutesAgoInTokyo,
DownloadType = "nzb",
IndexerId = indexer.Id
};

var score = await service.ScoreSearchResult(result, profile);

// Ten minutes is under the two hour minimum however it is written down.
Assert.Contains(score.RejectionReasons, reason => reason.Contains("Too new", StringComparison.Ordinal));
}

[Theory]
[InlineData(200, true)]
[InlineData(0, false)]
public async Task Indexer_MaximumSize_Rejects_Results_Over_The_Limit(int indexerMaximumSize, bool expectRejection)
{
// Indexer.MaximumSize had no reader. The scorer's existing size gate reads
// QualityProfile.MaximumSize, which shadows it by name. The second case is the
// control: with the indexer limit unset, the same result has to pass.
var options = new Microsoft.EntityFrameworkCore.DbContextOptionsBuilder<ListenArrDbContext>().UseInMemoryDatabase(Guid.NewGuid().ToString()).Options;
using var db = new ListenArrDbContext(options);
var indexer = new Listenarr.Domain.Search.Indexer
{
Name = "SizeCappedIndexer",
Url = "https://size.local",
MaximumSize = indexerMaximumSize,
IsEnabled = true
};
db.Indexers.Add(indexer);
db.SaveChanges();

var service = new QualityProfileService(new QualityProfileRepository(db), NullLogger<QualityProfileService>.Instance, new EfIndexerRepository(db));
var profile = new QualityProfile { MinimumSeeders = 0, MaximumAge = 3650 };

var result = new SearchResult
{
Title = "Large Result",
PublishedDate = DateTime.UtcNow.AddDays(-1).ToString("o"),
DownloadType = "torrent",
Size = 300L * 1024 * 1024,
Seeders = 10,
IndexerId = indexer.Id
};

var score = await service.ScoreSearchResult(result, profile);

if (expectRejection)
{
Assert.Contains(score.RejectionReasons, reason => reason.Contains("too large for indexer", StringComparison.OrdinalIgnoreCase));
Assert.True(score.TotalScore < 0);
}
else
{
Assert.DoesNotContain(score.RejectionReasons, reason => reason.Contains("too large for indexer", StringComparison.OrdinalIgnoreCase));
}
}

[Theory]
[InlineData(120, true)]
[InlineData(0, false)]
public async Task Indexer_MinimumAge_Rejects_Nzbs_That_Have_Not_Propagated(int minimumAgeMinutes, bool expectRejection)
{
// Indexer.MinimumAge had no reader either. It exists so a post that has not finished
// propagating is not grabbed as an incomplete download. The second case is the control.
var options = new Microsoft.EntityFrameworkCore.DbContextOptionsBuilder<ListenArrDbContext>().UseInMemoryDatabase(Guid.NewGuid().ToString()).Options;
using var db = new ListenArrDbContext(options);
var indexer = new Listenarr.Domain.Search.Indexer
{
Name = "PropagationIndexer",
Url = "https://usenet.local",
MinimumAge = minimumAgeMinutes,
Retention = 3650,
IsEnabled = true
};
db.Indexers.Add(indexer);
db.SaveChanges();

var service = new QualityProfileService(new QualityProfileRepository(db), NullLogger<QualityProfileService>.Instance, new EfIndexerRepository(db));
var profile = new QualityProfile { MaximumAge = 3650, MinimumSeeders = 0 };

var result = new SearchResult
{
Title = "Fresh Post",
PublishedDate = DateTime.UtcNow.AddMinutes(-10).ToString("o"),
DownloadType = "nzb",
IndexerId = indexer.Id
};

var score = await service.ScoreSearchResult(result, profile);

if (expectRejection)
{
Assert.Contains(score.RejectionReasons, reason => reason.Contains("Too new", StringComparison.Ordinal));
Assert.True(score.TotalScore < 0);
}
else
{
Assert.DoesNotContain(score.RejectionReasons, reason => reason.Contains("Too new", StringComparison.Ordinal));
}
}

[Fact]
public async Task Indexer_MinimumAge_Does_Not_Apply_To_Torrents()
{
// Propagation is a Usenet concern. A torrent that has just been posted is complete.
var options = new Microsoft.EntityFrameworkCore.DbContextOptionsBuilder<ListenArrDbContext>().UseInMemoryDatabase(Guid.NewGuid().ToString()).Options;
using var db = new ListenArrDbContext(options);
var indexer = new Listenarr.Domain.Search.Indexer
{
Name = "TorrentMinAge",
Url = "https://torrent.local",
MinimumAge = 120,
Type = "Torrent",
IsEnabled = true
};
db.Indexers.Add(indexer);
db.SaveChanges();

var service = new QualityProfileService(new QualityProfileRepository(db), NullLogger<QualityProfileService>.Instance, new EfIndexerRepository(db));
var profile = new QualityProfile { MaximumAge = 3650, MinimumSeeders = 0 };

var result = new SearchResult
{
Title = "Fresh Torrent",
PublishedDate = DateTime.UtcNow.AddMinutes(-10).ToString("o"),
DownloadType = "torrent",
Seeders = 10,
IndexerId = indexer.Id
};

var score = await service.ScoreSearchResult(result, profile);
Assert.DoesNotContain(score.RejectionReasons, reason => reason.Contains("Too new", StringComparison.Ordinal));
}

[Fact]
public async Task Torrent_Age_Rejection_Uses_ProfileMaxAge()
{
Expand Down