feat(S01+S04): server-side pagination, search/filter, download engine hardening
S01 — Server-Side Pagination: - Added getChannelContentPaginated() to content repository with search, filter, sort - Channel content API now supports ?page, ?pageSize, ?search, ?status, ?contentType, ?sortBy, ?sortDirection - Backwards-compatible: no params returns all items (legacy mode) - Frontend useChannelContentPaginated hook with keepPreviousData - ChannelDetail page: search bar, status/type filter dropdowns, pagination controls - Sorting delegated to server (removed client-side sortedContent) - Item count shown in Content header (e.g. '121 items') S04 — Download Engine Hardening: - yt-dlp auto-update on production startup (native -U with pip fallback) - Error classification: rate_limit, format_unavailable, geo_blocked, age_restricted, private, network - Format fallback chains: preferred res → best under res → single best → any - Improved parseFinalPath: explicit non-path prefix detection, extension validation - Error category included in download:failed events - classifyYtDlpError() exported from yt-dlp module for downstream use
This commit is contained in:
jlightner
parent
0541a5f1d1
commit
c057b6a286
8 changed files with 420 additions and 83 deletions
|
|
@ -6,7 +6,7 @@ services:
|
|||
ports:
|
||||
- "8989:8989"
|
||||
volumes:
|
||||
- ./config:/config
|
||||
- tubearr-config:/config
|
||||
- ./media:/media
|
||||
environment:
|
||||
- NODE_ENV=production
|
||||
|
|
@ -18,3 +18,6 @@ services:
|
|||
retries: 3
|
||||
start_period: 15s
|
||||
restart: unless-stopped
|
||||
|
||||
volumes:
|
||||
tubearr-config:
|
||||
|
|
|
|||
|
|
@ -92,6 +92,81 @@ export async function getContentByChannelId(
|
|||
return rows.map(mapRow);
|
||||
}
|
||||
|
||||
/** Optional filters for channel content queries. */
|
||||
export interface ChannelContentFilters {
|
||||
search?: string;
|
||||
status?: ContentStatus;
|
||||
contentType?: ContentType;
|
||||
sortBy?: 'title' | 'publishedAt' | 'status' | 'duration' | 'fileSize' | 'downloadedAt';
|
||||
sortDirection?: 'asc' | 'desc';
|
||||
}
|
||||
|
||||
/**
|
||||
* Get paginated content items for a channel with optional search, filter, and sort.
|
||||
* Returns items and total count for pagination.
|
||||
*/
|
||||
export async function getChannelContentPaginated(
|
||||
db: Db,
|
||||
channelId: number,
|
||||
filters?: ChannelContentFilters,
|
||||
page = 1,
|
||||
pageSize = 50
|
||||
): Promise<PaginatedContentResult> {
|
||||
const conditions = [eq(contentItems.channelId, channelId)];
|
||||
|
||||
if (filters?.search) {
|
||||
conditions.push(like(contentItems.title, `%${filters.search}%`));
|
||||
}
|
||||
if (filters?.status) {
|
||||
conditions.push(eq(contentItems.status, filters.status));
|
||||
}
|
||||
if (filters?.contentType) {
|
||||
conditions.push(eq(contentItems.contentType, filters.contentType));
|
||||
}
|
||||
|
||||
const whereClause = and(...conditions);
|
||||
const offset = (page - 1) * pageSize;
|
||||
|
||||
// Count total matching records
|
||||
const countResult = await db
|
||||
.select({ count: sql<number>`count(*)` })
|
||||
.from(contentItems)
|
||||
.where(whereClause);
|
||||
|
||||
const total = Number(countResult[0].count);
|
||||
|
||||
// Build sort order
|
||||
const sortCol = resolveSortColumn(filters?.sortBy);
|
||||
const sortDir = filters?.sortDirection === 'asc' ? sortCol : desc(sortCol);
|
||||
|
||||
// Fetch paginated results
|
||||
const rows = await db
|
||||
.select()
|
||||
.from(contentItems)
|
||||
.where(whereClause)
|
||||
.orderBy(sortDir, desc(contentItems.id))
|
||||
.limit(pageSize)
|
||||
.offset(offset);
|
||||
|
||||
return {
|
||||
items: rows.map(mapRow),
|
||||
total,
|
||||
};
|
||||
}
|
||||
|
||||
/** Resolve sort column name to Drizzle column reference. */
|
||||
function resolveSortColumn(sortBy?: string) {
|
||||
switch (sortBy) {
|
||||
case 'title': return contentItems.title;
|
||||
case 'publishedAt': return contentItems.publishedAt;
|
||||
case 'status': return contentItems.status;
|
||||
case 'duration': return contentItems.duration;
|
||||
case 'fileSize': return contentItems.fileSize;
|
||||
case 'downloadedAt': return contentItems.downloadedAt;
|
||||
default: return contentItems.createdAt;
|
||||
}
|
||||
}
|
||||
|
||||
/** Check if a specific content item exists for a channel. Returns the item or null. */
|
||||
export async function getContentByPlatformContentId(
|
||||
db: Db,
|
||||
|
|
|
|||
|
|
@ -1,8 +1,8 @@
|
|||
import { useQuery, useMutation, useQueryClient } from '@tanstack/react-query';
|
||||
import { useQuery, useMutation, useQueryClient, keepPreviousData } from '@tanstack/react-query';
|
||||
import { apiClient } from '../client';
|
||||
import { queueKeys } from './useQueue';
|
||||
import type { ContentItem } from '@shared/types/index';
|
||||
import type { ApiResponse } from '@shared/types/api';
|
||||
import type { ApiResponse, PaginatedResponse } from '@shared/types/api';
|
||||
|
||||
// ── Collect Types ──
|
||||
|
||||
|
|
@ -13,15 +13,29 @@ export interface CollectResult {
|
|||
items: Array<{ contentItemId: number; status: string }>;
|
||||
}
|
||||
|
||||
// ── Channel Content Filter Types ──
|
||||
|
||||
export interface ChannelContentFilters {
|
||||
page?: number;
|
||||
pageSize?: number;
|
||||
search?: string;
|
||||
status?: string;
|
||||
contentType?: string;
|
||||
sortBy?: string;
|
||||
sortDirection?: 'asc' | 'desc';
|
||||
}
|
||||
|
||||
// ── Query Keys ──
|
||||
|
||||
export const contentKeys = {
|
||||
byChannel: (channelId: number) => ['content', 'channel', channelId] as const,
|
||||
byChannelPaginated: (channelId: number, filters: ChannelContentFilters) =>
|
||||
['content', 'channel', channelId, 'paginated', filters] as const,
|
||||
};
|
||||
|
||||
// ── Queries ──
|
||||
|
||||
/** Fetch content items for a specific channel. */
|
||||
/** Fetch content items for a specific channel (legacy — all items). */
|
||||
export function useChannelContent(channelId: number) {
|
||||
return useQuery({
|
||||
queryKey: contentKeys.byChannel(channelId),
|
||||
|
|
@ -35,6 +49,30 @@ export function useChannelContent(channelId: number) {
|
|||
});
|
||||
}
|
||||
|
||||
/** Fetch paginated content items for a channel with search/filter/sort. */
|
||||
export function useChannelContentPaginated(channelId: number, filters: ChannelContentFilters) {
|
||||
return useQuery({
|
||||
queryKey: contentKeys.byChannelPaginated(channelId, filters),
|
||||
queryFn: async () => {
|
||||
const params = new URLSearchParams();
|
||||
if (filters.page) params.set('page', String(filters.page));
|
||||
if (filters.pageSize) params.set('pageSize', String(filters.pageSize));
|
||||
if (filters.search) params.set('search', filters.search);
|
||||
if (filters.status) params.set('status', filters.status);
|
||||
if (filters.contentType) params.set('contentType', filters.contentType);
|
||||
if (filters.sortBy) params.set('sortBy', filters.sortBy);
|
||||
if (filters.sortDirection) params.set('sortDirection', filters.sortDirection);
|
||||
|
||||
const response = await apiClient.get<PaginatedResponse<ContentItem>>(
|
||||
`/api/v1/channel/${channelId}/content?${params.toString()}`,
|
||||
);
|
||||
return response;
|
||||
},
|
||||
enabled: channelId > 0,
|
||||
placeholderData: keepPreviousData,
|
||||
});
|
||||
}
|
||||
|
||||
// ── Mutations ──
|
||||
|
||||
/** Enqueue a content item for download. Returns 202 with queue item. */
|
||||
|
|
|
|||
|
|
@ -19,7 +19,7 @@ import {
|
|||
Trash2,
|
||||
} from 'lucide-react';
|
||||
import { useChannel, useUpdateChannel, useDeleteChannel, useScanChannel, useSetMonitoringMode, useScanStatus } from '../api/hooks/useChannels';
|
||||
import { useChannelContent, useDownloadContent, useToggleMonitored, useBulkMonitored, useCollectMonitored } from '../api/hooks/useContent';
|
||||
import { useChannelContentPaginated, useDownloadContent, useToggleMonitored, useBulkMonitored, useCollectMonitored, type ChannelContentFilters } from '../api/hooks/useContent';
|
||||
import { apiClient } from '../api/client';
|
||||
import { useChannelPlaylists, useRefreshPlaylists } from '../api/hooks/usePlaylists';
|
||||
import { useFormatProfiles } from '../api/hooks/useFormatProfiles';
|
||||
|
|
@ -28,6 +28,7 @@ import { PlatformBadge } from '../components/PlatformBadge';
|
|||
import { StatusBadge } from '../components/StatusBadge';
|
||||
import { QualityLabel } from '../components/QualityLabel';
|
||||
import { DownloadProgressBar } from '../components/DownloadProgressBar';
|
||||
import { Pagination } from '../components/Pagination';
|
||||
import { Modal } from '../components/Modal';
|
||||
import { useDownloadProgress } from '../contexts/DownloadProgressContext';
|
||||
import type { ContentItem, MonitoringMode } from '@shared/types/index';
|
||||
|
|
@ -96,10 +97,31 @@ export function ChannelDetail() {
|
|||
|
||||
// ── Data hooks ──
|
||||
const { data: channel, isLoading: channelLoading, error: channelError } = useChannel(channelId);
|
||||
const { data: content, isLoading: contentLoading, error: contentError, refetch: refetchContent } = useChannelContent(channelId);
|
||||
const { data: formatProfiles } = useFormatProfiles();
|
||||
const { data: playlistData } = useChannelPlaylists(channelId);
|
||||
|
||||
// ── Content pagination state ──
|
||||
const [contentPage, setContentPage] = useState(1);
|
||||
const [contentSearch, setContentSearch] = useState('');
|
||||
const [contentStatusFilter, setContentStatusFilter] = useState('');
|
||||
const [contentTypeFilter, setContentTypeFilter] = useState('');
|
||||
const [sortKey, setSortKey] = useState<string | null>(null);
|
||||
const [sortDirection, setSortDirection] = useState<'asc' | 'desc'>('asc');
|
||||
|
||||
const contentFilters: ChannelContentFilters = useMemo(() => ({
|
||||
page: contentPage,
|
||||
pageSize: 50,
|
||||
search: contentSearch || undefined,
|
||||
status: contentStatusFilter || undefined,
|
||||
contentType: contentTypeFilter || undefined,
|
||||
sortBy: sortKey ?? undefined,
|
||||
sortDirection: sortDirection,
|
||||
}), [contentPage, contentSearch, contentStatusFilter, contentTypeFilter, sortKey, sortDirection]);
|
||||
|
||||
const { data: contentResponse, isLoading: contentLoading, error: contentError, refetch: refetchContent } = useChannelContentPaginated(channelId, contentFilters);
|
||||
const content = contentResponse?.data ?? [];
|
||||
const contentPagination = contentResponse?.pagination;
|
||||
|
||||
// ── Mutation hooks ──
|
||||
const updateChannel = useUpdateChannel(channelId);
|
||||
const deleteChannel = useDeleteChannel();
|
||||
|
|
@ -115,8 +137,6 @@ export function ChannelDetail() {
|
|||
const [showDeleteConfirm, setShowDeleteConfirm] = useState(false);
|
||||
const [scanResult, setScanResult] = useState<{ message: string; isError: boolean } | null>(null);
|
||||
const [scanInProgress, setScanInProgress] = useState(false);
|
||||
const [sortKey, setSortKey] = useState<string | null>(null);
|
||||
const [sortDirection, setSortDirection] = useState<'asc' | 'desc'>('asc');
|
||||
const [expandedPlaylists, setExpandedPlaylists] = useState<Set<number | 'uncategorized'>>(new Set());
|
||||
const [selectedIds, setSelectedIds] = useState<Set<number>>(new Set());
|
||||
const [localCheckInterval, setLocalCheckInterval] = useState<number | ''>('');
|
||||
|
|
@ -256,6 +276,7 @@ export function ChannelDetail() {
|
|||
const handleSort = useCallback((key: string, direction: 'asc' | 'desc') => {
|
||||
setSortKey(key);
|
||||
setSortDirection(direction);
|
||||
setContentPage(1);
|
||||
}, []);
|
||||
|
||||
const togglePlaylist = useCallback((id: number | 'uncategorized') => {
|
||||
|
|
@ -316,55 +337,10 @@ export function ChannelDetail() {
|
|||
clearSelection();
|
||||
}, [selectedIds, downloadContent, clearSelection]);
|
||||
|
||||
// ── Sorted content ──
|
||||
// ── Sorted content (server-side — just use content directly) ──
|
||||
|
||||
const sortedContent = useMemo(() => {
|
||||
const items = content ?? [];
|
||||
if (!sortKey) return items;
|
||||
const sorted = [...items];
|
||||
sorted.sort((a, b) => {
|
||||
let cmp = 0;
|
||||
switch (sortKey) {
|
||||
case 'title':
|
||||
cmp = a.title.localeCompare(b.title);
|
||||
break;
|
||||
case 'publishedAt': {
|
||||
const aDate = a.publishedAt ? Date.parse(a.publishedAt) : -Infinity;
|
||||
const bDate = b.publishedAt ? Date.parse(b.publishedAt) : -Infinity;
|
||||
cmp = aDate - bDate;
|
||||
break;
|
||||
}
|
||||
case 'status':
|
||||
cmp = a.status.localeCompare(b.status);
|
||||
break;
|
||||
case 'duration': {
|
||||
const aDur = a.duration ?? -Infinity;
|
||||
const bDur = b.duration ?? -Infinity;
|
||||
cmp = aDur - bDur;
|
||||
break;
|
||||
}
|
||||
case 'fileSize':
|
||||
cmp = (a.fileSize ?? -Infinity) - (b.fileSize ?? -Infinity);
|
||||
break;
|
||||
case 'downloadedAt': {
|
||||
const aDate2 = a.downloadedAt ? Date.parse(a.downloadedAt) : -Infinity;
|
||||
const bDate2 = b.downloadedAt ? Date.parse(b.downloadedAt) : -Infinity;
|
||||
cmp = aDate2 - bDate2;
|
||||
break;
|
||||
}
|
||||
case 'quality': {
|
||||
const aQ = a.qualityMetadata?.actualResolution ?? '';
|
||||
const bQ = b.qualityMetadata?.actualResolution ?? '';
|
||||
cmp = aQ.localeCompare(bQ);
|
||||
break;
|
||||
}
|
||||
default:
|
||||
return 0;
|
||||
}
|
||||
return sortDirection === 'desc' ? -cmp : cmp;
|
||||
});
|
||||
return sorted;
|
||||
}, [content, sortKey, sortDirection]);
|
||||
// Sort is handled server-side via contentFilters.sortBy/sortDirection.
|
||||
// playlistGroups still needs client-side grouping for YouTube channels.
|
||||
|
||||
// ── Playlist grouping (YouTube only) ──
|
||||
|
||||
|
|
@ -379,7 +355,7 @@ export function ChannelDetail() {
|
|||
|
||||
// Build a Map from content ID to content item for O(1) lookups (js-index-maps)
|
||||
const contentById = new Map<number, ContentItem>();
|
||||
for (const item of sortedContent) {
|
||||
for (const item of content) {
|
||||
contentById.set(item.id, item);
|
||||
}
|
||||
|
||||
|
|
@ -399,13 +375,13 @@ export function ChannelDetail() {
|
|||
}
|
||||
|
||||
// Uncategorized: items not in any playlist
|
||||
const uncategorized = sortedContent.filter((item) => !categorizedIds.has(item.id));
|
||||
const uncategorized = content.filter((item) => !categorizedIds.has(item.id));
|
||||
if (uncategorized.length > 0) {
|
||||
groups.push({ id: 'uncategorized', title: 'Uncategorized', items: uncategorized });
|
||||
}
|
||||
|
||||
return groups.length > 0 ? groups : null;
|
||||
}, [channel, playlistData, sortedContent]);
|
||||
}, [channel, playlistData, content]);
|
||||
|
||||
// ── Content table columns ──
|
||||
|
||||
|
|
@ -1066,6 +1042,7 @@ export function ChannelDetail() {
|
|||
display: 'flex',
|
||||
alignItems: 'center',
|
||||
gap: 'var(--space-3)',
|
||||
flexWrap: 'wrap',
|
||||
}}
|
||||
>
|
||||
<h2
|
||||
|
|
@ -1078,6 +1055,65 @@ export function ChannelDetail() {
|
|||
>
|
||||
Content
|
||||
</h2>
|
||||
{contentPagination ? (
|
||||
<span style={{ fontSize: 'var(--font-size-xs)', color: 'var(--text-muted)', fontVariantNumeric: 'tabular-nums' }}>
|
||||
{contentPagination.totalItems} items
|
||||
</span>
|
||||
) : null}
|
||||
<div style={{ flex: 1 }} />
|
||||
{/* Search */}
|
||||
<input
|
||||
type="text"
|
||||
placeholder="Search content…"
|
||||
value={contentSearch}
|
||||
onChange={(e) => { setContentSearch(e.target.value); setContentPage(1); }}
|
||||
style={{
|
||||
padding: 'var(--space-2) var(--space-3)',
|
||||
borderRadius: 'var(--radius-md)',
|
||||
border: '1px solid var(--border)',
|
||||
backgroundColor: 'var(--bg-input)',
|
||||
color: 'var(--text-primary)',
|
||||
fontSize: 'var(--font-size-sm)',
|
||||
width: 200,
|
||||
}}
|
||||
/>
|
||||
{/* Status filter */}
|
||||
<select
|
||||
value={contentStatusFilter}
|
||||
onChange={(e) => { setContentStatusFilter(e.target.value); setContentPage(1); }}
|
||||
aria-label="Filter by status"
|
||||
style={{
|
||||
padding: 'var(--space-2) var(--space-3)',
|
||||
borderRadius: 'var(--radius-md)',
|
||||
fontSize: 'var(--font-size-sm)',
|
||||
minWidth: 100,
|
||||
}}
|
||||
>
|
||||
<option value="">All Status</option>
|
||||
<option value="monitored">Monitored</option>
|
||||
<option value="queued">Queued</option>
|
||||
<option value="downloading">Downloading</option>
|
||||
<option value="downloaded">Downloaded</option>
|
||||
<option value="failed">Failed</option>
|
||||
<option value="ignored">Ignored</option>
|
||||
</select>
|
||||
{/* Type filter */}
|
||||
<select
|
||||
value={contentTypeFilter}
|
||||
onChange={(e) => { setContentTypeFilter(e.target.value); setContentPage(1); }}
|
||||
aria-label="Filter by type"
|
||||
style={{
|
||||
padding: 'var(--space-2) var(--space-3)',
|
||||
borderRadius: 'var(--radius-md)',
|
||||
fontSize: 'var(--font-size-sm)',
|
||||
minWidth: 90,
|
||||
}}
|
||||
>
|
||||
<option value="">All Types</option>
|
||||
<option value="video">Video</option>
|
||||
<option value="audio">Audio</option>
|
||||
<option value="livestream">Livestream</option>
|
||||
</select>
|
||||
</div>
|
||||
{contentError ? (
|
||||
<div
|
||||
|
|
@ -1128,8 +1164,19 @@ export function ChannelDetail() {
|
|||
) : hasPlaylistGroups ? (
|
||||
renderPlaylistGroups(playlistGroups!)
|
||||
) : (
|
||||
renderTable(sortedContent)
|
||||
renderTable(content)
|
||||
)}
|
||||
{/* Pagination controls */}
|
||||
{contentPagination && contentPagination.totalPages > 1 ? (
|
||||
<div style={{ padding: 'var(--space-3) var(--space-5)', borderTop: '1px solid var(--border)' }}>
|
||||
<Pagination
|
||||
page={contentPagination.page}
|
||||
totalPages={contentPagination.totalPages}
|
||||
totalItems={contentPagination.totalItems}
|
||||
onPageChange={setContentPage}
|
||||
/>
|
||||
</div>
|
||||
) : null}
|
||||
</div>
|
||||
|
||||
{/* Floating bulk action bar */}
|
||||
|
|
|
|||
20
src/index.ts
20
src/index.ts
|
|
@ -23,6 +23,7 @@ import { PlatformRegistry } from './sources/platform-source';
|
|||
import { YouTubeSource } from './sources/youtube';
|
||||
import { SoundCloudSource } from './sources/soundcloud';
|
||||
import { Platform } from './types/index';
|
||||
import { getYtDlpVersion, updateYtDlp } from './sources/yt-dlp';
|
||||
import type { ViteDevServer } from 'vite';
|
||||
|
||||
const APP_NAME = 'Tubearr';
|
||||
|
|
@ -44,6 +45,25 @@ async function main(): Promise<void> {
|
|||
await seedAppDefaults(db);
|
||||
console.log(`[${APP_NAME}] App settings seeded`);
|
||||
|
||||
// 2d. Check yt-dlp version and auto-update if configured
|
||||
try {
|
||||
const version = await getYtDlpVersion();
|
||||
if (version) {
|
||||
console.log(`[${APP_NAME}] yt-dlp version: ${version}`);
|
||||
// Auto-update on startup (non-blocking — continue if it fails)
|
||||
if (appConfig.nodeEnv === 'production') {
|
||||
const result = await updateYtDlp();
|
||||
if (result.updated) {
|
||||
console.log(`[${APP_NAME}] yt-dlp updated: ${result.previousVersion} → ${result.version}`);
|
||||
}
|
||||
}
|
||||
} else {
|
||||
console.warn(`[${APP_NAME}] yt-dlp not found on PATH — downloads will fail`);
|
||||
}
|
||||
} catch (err) {
|
||||
console.warn(`[${APP_NAME}] yt-dlp check failed: ${err instanceof Error ? err.message : String(err)}`);
|
||||
}
|
||||
|
||||
// 3. Build and configure Fastify server
|
||||
// In dev mode, embed Vite for HMR — single port, no separate frontend process
|
||||
let vite: ViteDevServer | undefined;
|
||||
|
|
|
|||
|
|
@ -2,6 +2,7 @@ import { type FastifyInstance } from 'fastify';
|
|||
import {
|
||||
getAllContentItems,
|
||||
getContentByChannelId,
|
||||
getChannelContentPaginated,
|
||||
setMonitored,
|
||||
bulkSetMonitored,
|
||||
} from '../../db/repositories/content-repository';
|
||||
|
|
@ -202,6 +203,15 @@ export async function contentRoutes(fastify: FastifyInstance): Promise<void> {
|
|||
|
||||
fastify.get<{
|
||||
Params: { id: string };
|
||||
Querystring: {
|
||||
page?: string;
|
||||
pageSize?: string;
|
||||
search?: string;
|
||||
status?: string;
|
||||
contentType?: string;
|
||||
sortBy?: string;
|
||||
sortDirection?: string;
|
||||
};
|
||||
}>('/api/v1/channel/:id/content', async (request, reply) => {
|
||||
const channelId = parseInt(request.params.id, 10);
|
||||
|
||||
|
|
@ -213,12 +223,50 @@ export async function contentRoutes(fastify: FastifyInstance): Promise<void> {
|
|||
});
|
||||
}
|
||||
|
||||
try {
|
||||
const items = await getContentByChannelId(fastify.db, channelId);
|
||||
const page = Math.max(1, parseInt(request.query.page ?? '1', 10) || 1);
|
||||
const pageSize = Math.min(
|
||||
200,
|
||||
Math.max(1, parseInt(request.query.pageSize ?? '50', 10) || 50)
|
||||
);
|
||||
|
||||
const response: ApiResponse<ContentItem[]> = {
|
||||
// If no pagination params provided, return all items (backwards-compatible)
|
||||
const hasPaginationParams = request.query.page || request.query.pageSize || request.query.search || request.query.status || request.query.contentType || request.query.sortBy;
|
||||
|
||||
try {
|
||||
if (!hasPaginationParams) {
|
||||
// Legacy mode: return all items as flat array (backwards-compatible)
|
||||
const items = await getContentByChannelId(fastify.db, channelId);
|
||||
const response: ApiResponse<ContentItem[]> = {
|
||||
success: true,
|
||||
data: items,
|
||||
};
|
||||
return response;
|
||||
}
|
||||
|
||||
// Paginated mode with filters
|
||||
const result = await getChannelContentPaginated(
|
||||
fastify.db,
|
||||
channelId,
|
||||
{
|
||||
search: request.query.search || undefined,
|
||||
status: (request.query.status as ContentStatus) || undefined,
|
||||
contentType: (request.query.contentType as ContentType) || undefined,
|
||||
sortBy: request.query.sortBy as 'title' | 'publishedAt' | 'status' | 'duration' | 'fileSize' | 'downloadedAt' | undefined,
|
||||
sortDirection: (request.query.sortDirection as 'asc' | 'desc') || undefined,
|
||||
},
|
||||
page,
|
||||
pageSize
|
||||
);
|
||||
|
||||
const response: PaginatedResponse<ContentItem> = {
|
||||
success: true,
|
||||
data: items,
|
||||
data: result.items,
|
||||
pagination: {
|
||||
page,
|
||||
pageSize,
|
||||
totalItems: result.total,
|
||||
totalPages: Math.ceil(result.total / pageSize),
|
||||
},
|
||||
};
|
||||
|
||||
return response;
|
||||
|
|
|
|||
|
|
@ -3,7 +3,7 @@ import { extname } from 'node:path';
|
|||
import { createInterface } from 'node:readline';
|
||||
import type { LibSQLDatabase } from 'drizzle-orm/libsql';
|
||||
import type * as schema from '../db/schema/index';
|
||||
import { execYtDlp, spawnYtDlp, YtDlpError } from '../sources/yt-dlp';
|
||||
import { execYtDlp, spawnYtDlp, YtDlpError, classifyYtDlpError } from '../sources/yt-dlp';
|
||||
import { updateContentItem } from '../db/repositories/content-repository';
|
||||
import { parseProgressLine } from './progress-parser';
|
||||
import type { DownloadEventBus } from './event-bus';
|
||||
|
|
@ -137,16 +137,22 @@ export class DownloadService {
|
|||
// Report error to rate limiter
|
||||
this.rateLimiter.reportError(channel.platform as Platform);
|
||||
|
||||
// Classify the error for better retry decisions
|
||||
const errorMsg = err instanceof Error ? err.message : String(err);
|
||||
const stderr = err instanceof YtDlpError ? err.stderr : '';
|
||||
const errorCategory = classifyYtDlpError(stderr || errorMsg);
|
||||
|
||||
// Update status to failed
|
||||
await updateContentItem(this.db, contentItem.id, { status: 'failed' });
|
||||
|
||||
const errorMsg = err instanceof Error ? err.message : String(err);
|
||||
console.log(`${logPrefix} status=failed error="${errorMsg.slice(0, 200)}"`);
|
||||
console.log(
|
||||
`${logPrefix} status=failed category=${errorCategory} error="${errorMsg.slice(0, 200)}"`
|
||||
);
|
||||
|
||||
// Emit download:failed event
|
||||
// Emit download:failed event with error category
|
||||
this.eventBus?.emitDownload('download:failed', {
|
||||
contentItemId: contentItem.id,
|
||||
error: errorMsg.slice(0, 200),
|
||||
error: `[${errorCategory}] ${errorMsg.slice(0, 200)}`,
|
||||
});
|
||||
|
||||
throw err;
|
||||
|
|
@ -288,32 +294,32 @@ export class DownloadService {
|
|||
|
||||
/**
|
||||
* Build format args for video content.
|
||||
* Uses a fallback chain: preferred resolution → best available → any.
|
||||
* yt-dlp supports `/` as a fallback separator: `format1/format2/format3`.
|
||||
*/
|
||||
private buildVideoArgs(formatProfile?: FormatProfile): string[] {
|
||||
const args: string[] = [];
|
||||
const container = formatProfile?.containerFormat ?? 'mp4';
|
||||
|
||||
if (formatProfile?.videoResolution === 'Best') {
|
||||
// "Best" selects separate best-quality video + audio streams, merged together.
|
||||
// This is higher quality than `-f best` which picks a single combined format.
|
||||
args.push('-f', 'bestvideo+bestaudio/best');
|
||||
const container = formatProfile.containerFormat ?? 'mp4';
|
||||
// Best quality: separate streams merged
|
||||
args.push('-f', 'bestvideo+bestaudio/bestvideo*+bestaudio/best');
|
||||
args.push('--merge-output-format', container);
|
||||
} else if (formatProfile?.videoResolution) {
|
||||
const height = parseResolutionHeight(formatProfile.videoResolution);
|
||||
if (height) {
|
||||
// Fallback chain: exact res → best under res → single best stream → any
|
||||
args.push(
|
||||
'-f',
|
||||
`bestvideo[height<=${height}]+bestaudio/best[height<=${height}]`
|
||||
`bestvideo[height<=${height}]+bestaudio/bestvideo[height<=${height}]*+bestaudio/best[height<=${height}]/bestvideo+bestaudio/best`
|
||||
);
|
||||
} else {
|
||||
args.push('-f', 'best');
|
||||
args.push('-f', 'bestvideo+bestaudio/best');
|
||||
}
|
||||
|
||||
// Container format for merge
|
||||
const container = formatProfile.containerFormat ?? 'mp4';
|
||||
args.push('--merge-output-format', container);
|
||||
} else {
|
||||
args.push('-f', 'best');
|
||||
args.push('-f', 'bestvideo+bestaudio/best');
|
||||
args.push('--merge-output-format', container);
|
||||
}
|
||||
|
||||
return args;
|
||||
|
|
@ -367,18 +373,33 @@ export class DownloadService {
|
|||
|
||||
/**
|
||||
* Parse the final file path from yt-dlp stdout.
|
||||
* The `--print after_move:filepath` flag makes yt-dlp output the final path
|
||||
* as the last line of stdout.
|
||||
* The `--print after_move:filepath` flag makes yt-dlp output the final path.
|
||||
*
|
||||
* Strategy: walk backwards through lines, skipping known yt-dlp output prefixes
|
||||
* (e.g. [download], [Merger], [ExtractAudio], Deleting).
|
||||
* A valid path line should be an absolute path or at least contain a file extension.
|
||||
*/
|
||||
private parseFinalPath(stdout: string, fallbackPath: string): string {
|
||||
const lines = stdout.trim().split('\n');
|
||||
// The filepath from --print is typically the last non-empty line
|
||||
|
||||
// Known non-path prefixes from yt-dlp output
|
||||
const NON_PATH_PREFIXES = ['[', 'Deleting', 'WARNING:', 'ERROR:', 'Merging', 'Post-process'];
|
||||
|
||||
for (let i = lines.length - 1; i >= 0; i--) {
|
||||
const line = lines[i].trim();
|
||||
if (line && !line.startsWith('[') && !line.startsWith('Deleting')) {
|
||||
if (!line) continue;
|
||||
|
||||
// Skip known yt-dlp output lines
|
||||
const isNonPath = NON_PATH_PREFIXES.some((prefix) => line.startsWith(prefix));
|
||||
if (isNonPath) continue;
|
||||
|
||||
// A valid path should have a file extension or start with /
|
||||
if (line.startsWith('/') || /\.\w{2,5}$/.test(line)) {
|
||||
return line;
|
||||
}
|
||||
}
|
||||
|
||||
console.warn('[download] Could not parse final path from yt-dlp output, using fallback');
|
||||
return fallbackPath;
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -212,3 +212,88 @@ export async function getYtDlpVersion(): Promise<string | null> {
|
|||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
// ── Auto-Update ──
|
||||
|
||||
/**
|
||||
* Update yt-dlp to the latest version.
|
||||
* Uses `yt-dlp -U` which handles self-update on most installations.
|
||||
* For pip-based installs (Alpine/Docker), falls back to `pip install -U yt-dlp`.
|
||||
*
|
||||
* Returns { updated, version, previousVersion } on success.
|
||||
*/
|
||||
export async function updateYtDlp(): Promise<{
|
||||
updated: boolean;
|
||||
version: string | null;
|
||||
previousVersion: string | null;
|
||||
}> {
|
||||
const previousVersion = await getYtDlpVersion();
|
||||
|
||||
try {
|
||||
// Try native self-update first
|
||||
const { stderr } = await execFileAsync('yt-dlp', ['-U'], {
|
||||
timeout: 120_000,
|
||||
windowsHide: true,
|
||||
});
|
||||
|
||||
// Check if it actually updated
|
||||
const newVersion = await getYtDlpVersion();
|
||||
const didUpdate = newVersion !== previousVersion;
|
||||
|
||||
if (didUpdate) {
|
||||
console.log(`[yt-dlp] Updated from ${previousVersion} to ${newVersion}`);
|
||||
} else if (stderr.toLowerCase().includes('up to date')) {
|
||||
console.log(`[yt-dlp] Already up to date (${newVersion})`);
|
||||
}
|
||||
|
||||
return { updated: didUpdate, version: newVersion, previousVersion };
|
||||
} catch (err) {
|
||||
// Self-update may not work in pip-based installs — try pip
|
||||
try {
|
||||
await execFileAsync('pip', ['install', '-U', 'yt-dlp'], {
|
||||
timeout: 120_000,
|
||||
windowsHide: true,
|
||||
});
|
||||
|
||||
const newVersion = await getYtDlpVersion();
|
||||
const didUpdate = newVersion !== previousVersion;
|
||||
console.log(
|
||||
`[yt-dlp] pip update: ${previousVersion} → ${newVersion}${didUpdate ? '' : ' (no change)'}`
|
||||
);
|
||||
|
||||
return { updated: didUpdate, version: newVersion, previousVersion };
|
||||
} catch (pipErr) {
|
||||
console.warn(
|
||||
`[yt-dlp] Auto-update failed: ${err instanceof Error ? err.message : String(err)}`
|
||||
);
|
||||
return { updated: false, version: previousVersion, previousVersion };
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ── Error Classification ──
|
||||
|
||||
/**
|
||||
* Classify a yt-dlp error into a category for better retry/fallback decisions.
|
||||
*/
|
||||
export type YtDlpErrorCategory =
|
||||
| 'rate_limit' // 429, too many requests
|
||||
| 'format_unavailable' // requested format not available
|
||||
| 'geo_blocked' // geo-restriction
|
||||
| 'age_restricted' // age-gated content
|
||||
| 'private' // private or removed video
|
||||
| 'network' // DNS, connection, timeout
|
||||
| 'unknown';
|
||||
|
||||
export function classifyYtDlpError(stderr: string): YtDlpErrorCategory {
|
||||
const lower = stderr.toLowerCase();
|
||||
|
||||
if (lower.includes('429') || lower.includes('too many requests')) return 'rate_limit';
|
||||
if (lower.includes('requested format') || lower.includes('format is not available')) return 'format_unavailable';
|
||||
if (lower.includes('not available in your country') || lower.includes('geo')) return 'geo_blocked';
|
||||
if (lower.includes('age') && (lower.includes('restricted') || lower.includes('verify'))) return 'age_restricted';
|
||||
if (lower.includes('private video') || lower.includes('video unavailable') || lower.includes('been removed')) return 'private';
|
||||
if (lower.includes('unable to download') || lower.includes('connection') || lower.includes('timed out') || lower.includes('urlopen error')) return 'network';
|
||||
|
||||
return 'unknown';
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue