fix: raise default scan limit from 100 to 500, use 999 for initial scans

- Default scanLimit increased to 500 (was 100, missing most channel content)
- First scan (lastCheckedAt === null) uses max(scanLimit, 999) for full catalog
- Discovery timeout scales with limit: 60s base + 30s per 500 items
- Updated platform-settings-repository defaults to match
This commit is contained in:
jlightner 2026-04-03 22:07:24 +00:00
parent 4546ddb4ea
commit f494d31e60
3 changed files with 12 additions and 5 deletions

View file

@ -64,7 +64,7 @@ export async function upsertPlatformSettings(
subtitleLanguages: data.subtitleLanguages ?? null,
grabAllEnabled: data.grabAllEnabled ?? false,
grabAllOrder: data.grabAllOrder ?? 'newest',
scanLimit: data.scanLimit ?? 100,
scanLimit: data.scanLimit ?? 500,
rateLimitDelay: data.rateLimitDelay ?? 1000,
defaultMonitoringMode: data.defaultMonitoringMode ?? 'all',
createdAt: now,
@ -79,7 +79,7 @@ export async function upsertPlatformSettings(
subtitleLanguages: data.subtitleLanguages ?? null,
grabAllEnabled: data.grabAllEnabled ?? false,
grabAllOrder: data.grabAllOrder ?? 'newest',
scanLimit: data.scanLimit ?? 100,
scanLimit: data.scanLimit ?? 500,
rateLimitDelay: data.rateLimitDelay ?? 1000,
defaultMonitoringMode: data.defaultMonitoringMode ?? 'all',
updatedAt: now,
@ -114,7 +114,7 @@ function mapRow(row: typeof platformSettings.$inferSelect): PlatformSettings {
subtitleLanguages: row.subtitleLanguages,
grabAllEnabled: row.grabAllEnabled,
grabAllOrder: row.grabAllOrder as 'newest' | 'oldest',
scanLimit: row.scanLimit ?? 100,
scanLimit: row.scanLimit ?? 500,
rateLimitDelay: row.rateLimitDelay ?? 1000,
defaultMonitoringMode: (row.defaultMonitoringMode ?? 'all') as MonitoringMode,
createdAt: row.createdAt,

View file

@ -207,9 +207,14 @@ export class SchedulerService {
// 3. Load platform settings for scan limit and rate limit delay
const platformSettingsRow = await getPlatformSettings(this.db, channel.platform);
const scanLimit = platformSettingsRow?.scanLimit ?? 100;
const baseScanLimit = platformSettingsRow?.scanLimit ?? 500;
const rateLimitDelay = platformSettingsRow?.rateLimitDelay ?? 1000;
// First scan (lastCheckedAt === null) → grab full catalog up to 999
const scanLimit = channel.lastCheckedAt === null
? Math.max(baseScanLimit, 999)
: baseScanLimit;
// 4. Load existing content IDs for dedup gating
const existingIds = new Set(
await getRecentContentIds(this.db, channel.id)

View file

@ -99,6 +99,8 @@ export class YouTubeSource implements PlatformSource {
const signal = options?.signal;
// ── Phase 1: Fast discovery via --flat-playlist ──
// Timeout scales with limit: 60s base + 30s per 500 items
const discoveryTimeout = 60_000 + Math.ceil(limit / 500) * 30_000;
const flatResult = await execYtDlp(
[
'--flat-playlist',
@ -107,7 +109,7 @@ export class YouTubeSource implements PlatformSource {
`1:${limit}`,
channel.url,
],
{ timeout: 60_000 }
{ timeout: discoveryTimeout }
);
const flatEntries = parseJsonLines(flatResult.stdout);