fix: raise default scan limit from 100 to 500, use 999 for initial scans
- Default scanLimit increased to 500 (was 100, missing most channel content) - First scan (lastCheckedAt === null) uses max(scanLimit, 999) for full catalog - Discovery timeout scales with limit: 60s base + 30s per 500 items - Updated platform-settings-repository defaults to match
This commit is contained in:
parent
4546ddb4ea
commit
f494d31e60
3 changed files with 12 additions and 5 deletions
|
|
@ -64,7 +64,7 @@ export async function upsertPlatformSettings(
|
||||||
subtitleLanguages: data.subtitleLanguages ?? null,
|
subtitleLanguages: data.subtitleLanguages ?? null,
|
||||||
grabAllEnabled: data.grabAllEnabled ?? false,
|
grabAllEnabled: data.grabAllEnabled ?? false,
|
||||||
grabAllOrder: data.grabAllOrder ?? 'newest',
|
grabAllOrder: data.grabAllOrder ?? 'newest',
|
||||||
scanLimit: data.scanLimit ?? 100,
|
scanLimit: data.scanLimit ?? 500,
|
||||||
rateLimitDelay: data.rateLimitDelay ?? 1000,
|
rateLimitDelay: data.rateLimitDelay ?? 1000,
|
||||||
defaultMonitoringMode: data.defaultMonitoringMode ?? 'all',
|
defaultMonitoringMode: data.defaultMonitoringMode ?? 'all',
|
||||||
createdAt: now,
|
createdAt: now,
|
||||||
|
|
@ -79,7 +79,7 @@ export async function upsertPlatformSettings(
|
||||||
subtitleLanguages: data.subtitleLanguages ?? null,
|
subtitleLanguages: data.subtitleLanguages ?? null,
|
||||||
grabAllEnabled: data.grabAllEnabled ?? false,
|
grabAllEnabled: data.grabAllEnabled ?? false,
|
||||||
grabAllOrder: data.grabAllOrder ?? 'newest',
|
grabAllOrder: data.grabAllOrder ?? 'newest',
|
||||||
scanLimit: data.scanLimit ?? 100,
|
scanLimit: data.scanLimit ?? 500,
|
||||||
rateLimitDelay: data.rateLimitDelay ?? 1000,
|
rateLimitDelay: data.rateLimitDelay ?? 1000,
|
||||||
defaultMonitoringMode: data.defaultMonitoringMode ?? 'all',
|
defaultMonitoringMode: data.defaultMonitoringMode ?? 'all',
|
||||||
updatedAt: now,
|
updatedAt: now,
|
||||||
|
|
@ -114,7 +114,7 @@ function mapRow(row: typeof platformSettings.$inferSelect): PlatformSettings {
|
||||||
subtitleLanguages: row.subtitleLanguages,
|
subtitleLanguages: row.subtitleLanguages,
|
||||||
grabAllEnabled: row.grabAllEnabled,
|
grabAllEnabled: row.grabAllEnabled,
|
||||||
grabAllOrder: row.grabAllOrder as 'newest' | 'oldest',
|
grabAllOrder: row.grabAllOrder as 'newest' | 'oldest',
|
||||||
scanLimit: row.scanLimit ?? 100,
|
scanLimit: row.scanLimit ?? 500,
|
||||||
rateLimitDelay: row.rateLimitDelay ?? 1000,
|
rateLimitDelay: row.rateLimitDelay ?? 1000,
|
||||||
defaultMonitoringMode: (row.defaultMonitoringMode ?? 'all') as MonitoringMode,
|
defaultMonitoringMode: (row.defaultMonitoringMode ?? 'all') as MonitoringMode,
|
||||||
createdAt: row.createdAt,
|
createdAt: row.createdAt,
|
||||||
|
|
|
||||||
|
|
@ -207,9 +207,14 @@ export class SchedulerService {
|
||||||
|
|
||||||
// 3. Load platform settings for scan limit and rate limit delay
|
// 3. Load platform settings for scan limit and rate limit delay
|
||||||
const platformSettingsRow = await getPlatformSettings(this.db, channel.platform);
|
const platformSettingsRow = await getPlatformSettings(this.db, channel.platform);
|
||||||
const scanLimit = platformSettingsRow?.scanLimit ?? 100;
|
const baseScanLimit = platformSettingsRow?.scanLimit ?? 500;
|
||||||
const rateLimitDelay = platformSettingsRow?.rateLimitDelay ?? 1000;
|
const rateLimitDelay = platformSettingsRow?.rateLimitDelay ?? 1000;
|
||||||
|
|
||||||
|
// First scan (lastCheckedAt === null) → grab full catalog up to 999
|
||||||
|
const scanLimit = channel.lastCheckedAt === null
|
||||||
|
? Math.max(baseScanLimit, 999)
|
||||||
|
: baseScanLimit;
|
||||||
|
|
||||||
// 4. Load existing content IDs for dedup gating
|
// 4. Load existing content IDs for dedup gating
|
||||||
const existingIds = new Set(
|
const existingIds = new Set(
|
||||||
await getRecentContentIds(this.db, channel.id)
|
await getRecentContentIds(this.db, channel.id)
|
||||||
|
|
|
||||||
|
|
@ -99,6 +99,8 @@ export class YouTubeSource implements PlatformSource {
|
||||||
const signal = options?.signal;
|
const signal = options?.signal;
|
||||||
|
|
||||||
// ── Phase 1: Fast discovery via --flat-playlist ──
|
// ── Phase 1: Fast discovery via --flat-playlist ──
|
||||||
|
// Timeout scales with limit: 60s base + 30s per 500 items
|
||||||
|
const discoveryTimeout = 60_000 + Math.ceil(limit / 500) * 30_000;
|
||||||
const flatResult = await execYtDlp(
|
const flatResult = await execYtDlp(
|
||||||
[
|
[
|
||||||
'--flat-playlist',
|
'--flat-playlist',
|
||||||
|
|
@ -107,7 +109,7 @@ export class YouTubeSource implements PlatformSource {
|
||||||
`1:${limit}`,
|
`1:${limit}`,
|
||||||
channel.url,
|
channel.url,
|
||||||
],
|
],
|
||||||
{ timeout: 60_000 }
|
{ timeout: discoveryTimeout }
|
||||||
);
|
);
|
||||||
|
|
||||||
const flatEntries = parseJsonLines(flatResult.stdout);
|
const flatEntries = parseJsonLines(flatResult.stdout);
|
||||||
|
|
|
||||||
Loading…
Add table
Reference in a new issue