feat(S01+S04): server-side pagination, search/filter, download engine hardening
S01 — Server-Side Pagination: - Added getChannelContentPaginated() to content repository with search, filter, sort - Channel content API now supports ?page, ?pageSize, ?search, ?status, ?contentType, ?sortBy, ?sortDirection - Backwards-compatible: no params returns all items (legacy mode) - Frontend useChannelContentPaginated hook with keepPreviousData - ChannelDetail page: search bar, status/type filter dropdowns, pagination controls - Sorting delegated to server (removed client-side sortedContent) - Item count shown in Content header (e.g. '121 items') S04 — Download Engine Hardening: - yt-dlp auto-update on production startup (native -U with pip fallback) - Error classification: rate_limit, format_unavailable, geo_blocked, age_restricted, private, network - Format fallback chains: preferred res → best under res → single best → any - Improved parseFinalPath: explicit non-path prefix detection, extension validation - Error category included in download:failed events - classifyYtDlpError() exported from yt-dlp module for downstream use
This commit is contained in:
parent
0541a5f1d1
commit
c057b6a286
8 changed files with 420 additions and 83 deletions
|
|
@ -6,7 +6,7 @@ services:
|
||||||
ports:
|
ports:
|
||||||
- "8989:8989"
|
- "8989:8989"
|
||||||
volumes:
|
volumes:
|
||||||
- ./config:/config
|
- tubearr-config:/config
|
||||||
- ./media:/media
|
- ./media:/media
|
||||||
environment:
|
environment:
|
||||||
- NODE_ENV=production
|
- NODE_ENV=production
|
||||||
|
|
@ -18,3 +18,6 @@ services:
|
||||||
retries: 3
|
retries: 3
|
||||||
start_period: 15s
|
start_period: 15s
|
||||||
restart: unless-stopped
|
restart: unless-stopped
|
||||||
|
|
||||||
|
volumes:
|
||||||
|
tubearr-config:
|
||||||
|
|
|
||||||
|
|
@ -92,6 +92,81 @@ export async function getContentByChannelId(
|
||||||
return rows.map(mapRow);
|
return rows.map(mapRow);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/** Optional filters for channel content queries. */
|
||||||
|
export interface ChannelContentFilters {
|
||||||
|
search?: string;
|
||||||
|
status?: ContentStatus;
|
||||||
|
contentType?: ContentType;
|
||||||
|
sortBy?: 'title' | 'publishedAt' | 'status' | 'duration' | 'fileSize' | 'downloadedAt';
|
||||||
|
sortDirection?: 'asc' | 'desc';
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Get paginated content items for a channel with optional search, filter, and sort.
|
||||||
|
* Returns items and total count for pagination.
|
||||||
|
*/
|
||||||
|
export async function getChannelContentPaginated(
|
||||||
|
db: Db,
|
||||||
|
channelId: number,
|
||||||
|
filters?: ChannelContentFilters,
|
||||||
|
page = 1,
|
||||||
|
pageSize = 50
|
||||||
|
): Promise<PaginatedContentResult> {
|
||||||
|
const conditions = [eq(contentItems.channelId, channelId)];
|
||||||
|
|
||||||
|
if (filters?.search) {
|
||||||
|
conditions.push(like(contentItems.title, `%${filters.search}%`));
|
||||||
|
}
|
||||||
|
if (filters?.status) {
|
||||||
|
conditions.push(eq(contentItems.status, filters.status));
|
||||||
|
}
|
||||||
|
if (filters?.contentType) {
|
||||||
|
conditions.push(eq(contentItems.contentType, filters.contentType));
|
||||||
|
}
|
||||||
|
|
||||||
|
const whereClause = and(...conditions);
|
||||||
|
const offset = (page - 1) * pageSize;
|
||||||
|
|
||||||
|
// Count total matching records
|
||||||
|
const countResult = await db
|
||||||
|
.select({ count: sql<number>`count(*)` })
|
||||||
|
.from(contentItems)
|
||||||
|
.where(whereClause);
|
||||||
|
|
||||||
|
const total = Number(countResult[0].count);
|
||||||
|
|
||||||
|
// Build sort order
|
||||||
|
const sortCol = resolveSortColumn(filters?.sortBy);
|
||||||
|
const sortDir = filters?.sortDirection === 'asc' ? sortCol : desc(sortCol);
|
||||||
|
|
||||||
|
// Fetch paginated results
|
||||||
|
const rows = await db
|
||||||
|
.select()
|
||||||
|
.from(contentItems)
|
||||||
|
.where(whereClause)
|
||||||
|
.orderBy(sortDir, desc(contentItems.id))
|
||||||
|
.limit(pageSize)
|
||||||
|
.offset(offset);
|
||||||
|
|
||||||
|
return {
|
||||||
|
items: rows.map(mapRow),
|
||||||
|
total,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Resolve sort column name to Drizzle column reference. */
|
||||||
|
function resolveSortColumn(sortBy?: string) {
|
||||||
|
switch (sortBy) {
|
||||||
|
case 'title': return contentItems.title;
|
||||||
|
case 'publishedAt': return contentItems.publishedAt;
|
||||||
|
case 'status': return contentItems.status;
|
||||||
|
case 'duration': return contentItems.duration;
|
||||||
|
case 'fileSize': return contentItems.fileSize;
|
||||||
|
case 'downloadedAt': return contentItems.downloadedAt;
|
||||||
|
default: return contentItems.createdAt;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
/** Check if a specific content item exists for a channel. Returns the item or null. */
|
/** Check if a specific content item exists for a channel. Returns the item or null. */
|
||||||
export async function getContentByPlatformContentId(
|
export async function getContentByPlatformContentId(
|
||||||
db: Db,
|
db: Db,
|
||||||
|
|
|
||||||
|
|
@ -1,8 +1,8 @@
|
||||||
import { useQuery, useMutation, useQueryClient } from '@tanstack/react-query';
|
import { useQuery, useMutation, useQueryClient, keepPreviousData } from '@tanstack/react-query';
|
||||||
import { apiClient } from '../client';
|
import { apiClient } from '../client';
|
||||||
import { queueKeys } from './useQueue';
|
import { queueKeys } from './useQueue';
|
||||||
import type { ContentItem } from '@shared/types/index';
|
import type { ContentItem } from '@shared/types/index';
|
||||||
import type { ApiResponse } from '@shared/types/api';
|
import type { ApiResponse, PaginatedResponse } from '@shared/types/api';
|
||||||
|
|
||||||
// ── Collect Types ──
|
// ── Collect Types ──
|
||||||
|
|
||||||
|
|
@ -13,15 +13,29 @@ export interface CollectResult {
|
||||||
items: Array<{ contentItemId: number; status: string }>;
|
items: Array<{ contentItemId: number; status: string }>;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// ── Channel Content Filter Types ──
|
||||||
|
|
||||||
|
export interface ChannelContentFilters {
|
||||||
|
page?: number;
|
||||||
|
pageSize?: number;
|
||||||
|
search?: string;
|
||||||
|
status?: string;
|
||||||
|
contentType?: string;
|
||||||
|
sortBy?: string;
|
||||||
|
sortDirection?: 'asc' | 'desc';
|
||||||
|
}
|
||||||
|
|
||||||
// ── Query Keys ──
|
// ── Query Keys ──
|
||||||
|
|
||||||
export const contentKeys = {
|
export const contentKeys = {
|
||||||
byChannel: (channelId: number) => ['content', 'channel', channelId] as const,
|
byChannel: (channelId: number) => ['content', 'channel', channelId] as const,
|
||||||
|
byChannelPaginated: (channelId: number, filters: ChannelContentFilters) =>
|
||||||
|
['content', 'channel', channelId, 'paginated', filters] as const,
|
||||||
};
|
};
|
||||||
|
|
||||||
// ── Queries ──
|
// ── Queries ──
|
||||||
|
|
||||||
/** Fetch content items for a specific channel. */
|
/** Fetch content items for a specific channel (legacy — all items). */
|
||||||
export function useChannelContent(channelId: number) {
|
export function useChannelContent(channelId: number) {
|
||||||
return useQuery({
|
return useQuery({
|
||||||
queryKey: contentKeys.byChannel(channelId),
|
queryKey: contentKeys.byChannel(channelId),
|
||||||
|
|
@ -35,6 +49,30 @@ export function useChannelContent(channelId: number) {
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/** Fetch paginated content items for a channel with search/filter/sort. */
|
||||||
|
export function useChannelContentPaginated(channelId: number, filters: ChannelContentFilters) {
|
||||||
|
return useQuery({
|
||||||
|
queryKey: contentKeys.byChannelPaginated(channelId, filters),
|
||||||
|
queryFn: async () => {
|
||||||
|
const params = new URLSearchParams();
|
||||||
|
if (filters.page) params.set('page', String(filters.page));
|
||||||
|
if (filters.pageSize) params.set('pageSize', String(filters.pageSize));
|
||||||
|
if (filters.search) params.set('search', filters.search);
|
||||||
|
if (filters.status) params.set('status', filters.status);
|
||||||
|
if (filters.contentType) params.set('contentType', filters.contentType);
|
||||||
|
if (filters.sortBy) params.set('sortBy', filters.sortBy);
|
||||||
|
if (filters.sortDirection) params.set('sortDirection', filters.sortDirection);
|
||||||
|
|
||||||
|
const response = await apiClient.get<PaginatedResponse<ContentItem>>(
|
||||||
|
`/api/v1/channel/${channelId}/content?${params.toString()}`,
|
||||||
|
);
|
||||||
|
return response;
|
||||||
|
},
|
||||||
|
enabled: channelId > 0,
|
||||||
|
placeholderData: keepPreviousData,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
// ── Mutations ──
|
// ── Mutations ──
|
||||||
|
|
||||||
/** Enqueue a content item for download. Returns 202 with queue item. */
|
/** Enqueue a content item for download. Returns 202 with queue item. */
|
||||||
|
|
|
||||||
|
|
@ -19,7 +19,7 @@ import {
|
||||||
Trash2,
|
Trash2,
|
||||||
} from 'lucide-react';
|
} from 'lucide-react';
|
||||||
import { useChannel, useUpdateChannel, useDeleteChannel, useScanChannel, useSetMonitoringMode, useScanStatus } from '../api/hooks/useChannels';
|
import { useChannel, useUpdateChannel, useDeleteChannel, useScanChannel, useSetMonitoringMode, useScanStatus } from '../api/hooks/useChannels';
|
||||||
import { useChannelContent, useDownloadContent, useToggleMonitored, useBulkMonitored, useCollectMonitored } from '../api/hooks/useContent';
|
import { useChannelContentPaginated, useDownloadContent, useToggleMonitored, useBulkMonitored, useCollectMonitored, type ChannelContentFilters } from '../api/hooks/useContent';
|
||||||
import { apiClient } from '../api/client';
|
import { apiClient } from '../api/client';
|
||||||
import { useChannelPlaylists, useRefreshPlaylists } from '../api/hooks/usePlaylists';
|
import { useChannelPlaylists, useRefreshPlaylists } from '../api/hooks/usePlaylists';
|
||||||
import { useFormatProfiles } from '../api/hooks/useFormatProfiles';
|
import { useFormatProfiles } from '../api/hooks/useFormatProfiles';
|
||||||
|
|
@ -28,6 +28,7 @@ import { PlatformBadge } from '../components/PlatformBadge';
|
||||||
import { StatusBadge } from '../components/StatusBadge';
|
import { StatusBadge } from '../components/StatusBadge';
|
||||||
import { QualityLabel } from '../components/QualityLabel';
|
import { QualityLabel } from '../components/QualityLabel';
|
||||||
import { DownloadProgressBar } from '../components/DownloadProgressBar';
|
import { DownloadProgressBar } from '../components/DownloadProgressBar';
|
||||||
|
import { Pagination } from '../components/Pagination';
|
||||||
import { Modal } from '../components/Modal';
|
import { Modal } from '../components/Modal';
|
||||||
import { useDownloadProgress } from '../contexts/DownloadProgressContext';
|
import { useDownloadProgress } from '../contexts/DownloadProgressContext';
|
||||||
import type { ContentItem, MonitoringMode } from '@shared/types/index';
|
import type { ContentItem, MonitoringMode } from '@shared/types/index';
|
||||||
|
|
@ -96,10 +97,31 @@ export function ChannelDetail() {
|
||||||
|
|
||||||
// ── Data hooks ──
|
// ── Data hooks ──
|
||||||
const { data: channel, isLoading: channelLoading, error: channelError } = useChannel(channelId);
|
const { data: channel, isLoading: channelLoading, error: channelError } = useChannel(channelId);
|
||||||
const { data: content, isLoading: contentLoading, error: contentError, refetch: refetchContent } = useChannelContent(channelId);
|
|
||||||
const { data: formatProfiles } = useFormatProfiles();
|
const { data: formatProfiles } = useFormatProfiles();
|
||||||
const { data: playlistData } = useChannelPlaylists(channelId);
|
const { data: playlistData } = useChannelPlaylists(channelId);
|
||||||
|
|
||||||
|
// ── Content pagination state ──
|
||||||
|
const [contentPage, setContentPage] = useState(1);
|
||||||
|
const [contentSearch, setContentSearch] = useState('');
|
||||||
|
const [contentStatusFilter, setContentStatusFilter] = useState('');
|
||||||
|
const [contentTypeFilter, setContentTypeFilter] = useState('');
|
||||||
|
const [sortKey, setSortKey] = useState<string | null>(null);
|
||||||
|
const [sortDirection, setSortDirection] = useState<'asc' | 'desc'>('asc');
|
||||||
|
|
||||||
|
const contentFilters: ChannelContentFilters = useMemo(() => ({
|
||||||
|
page: contentPage,
|
||||||
|
pageSize: 50,
|
||||||
|
search: contentSearch || undefined,
|
||||||
|
status: contentStatusFilter || undefined,
|
||||||
|
contentType: contentTypeFilter || undefined,
|
||||||
|
sortBy: sortKey ?? undefined,
|
||||||
|
sortDirection: sortDirection,
|
||||||
|
}), [contentPage, contentSearch, contentStatusFilter, contentTypeFilter, sortKey, sortDirection]);
|
||||||
|
|
||||||
|
const { data: contentResponse, isLoading: contentLoading, error: contentError, refetch: refetchContent } = useChannelContentPaginated(channelId, contentFilters);
|
||||||
|
const content = contentResponse?.data ?? [];
|
||||||
|
const contentPagination = contentResponse?.pagination;
|
||||||
|
|
||||||
// ── Mutation hooks ──
|
// ── Mutation hooks ──
|
||||||
const updateChannel = useUpdateChannel(channelId);
|
const updateChannel = useUpdateChannel(channelId);
|
||||||
const deleteChannel = useDeleteChannel();
|
const deleteChannel = useDeleteChannel();
|
||||||
|
|
@ -115,8 +137,6 @@ export function ChannelDetail() {
|
||||||
const [showDeleteConfirm, setShowDeleteConfirm] = useState(false);
|
const [showDeleteConfirm, setShowDeleteConfirm] = useState(false);
|
||||||
const [scanResult, setScanResult] = useState<{ message: string; isError: boolean } | null>(null);
|
const [scanResult, setScanResult] = useState<{ message: string; isError: boolean } | null>(null);
|
||||||
const [scanInProgress, setScanInProgress] = useState(false);
|
const [scanInProgress, setScanInProgress] = useState(false);
|
||||||
const [sortKey, setSortKey] = useState<string | null>(null);
|
|
||||||
const [sortDirection, setSortDirection] = useState<'asc' | 'desc'>('asc');
|
|
||||||
const [expandedPlaylists, setExpandedPlaylists] = useState<Set<number | 'uncategorized'>>(new Set());
|
const [expandedPlaylists, setExpandedPlaylists] = useState<Set<number | 'uncategorized'>>(new Set());
|
||||||
const [selectedIds, setSelectedIds] = useState<Set<number>>(new Set());
|
const [selectedIds, setSelectedIds] = useState<Set<number>>(new Set());
|
||||||
const [localCheckInterval, setLocalCheckInterval] = useState<number | ''>('');
|
const [localCheckInterval, setLocalCheckInterval] = useState<number | ''>('');
|
||||||
|
|
@ -256,6 +276,7 @@ export function ChannelDetail() {
|
||||||
const handleSort = useCallback((key: string, direction: 'asc' | 'desc') => {
|
const handleSort = useCallback((key: string, direction: 'asc' | 'desc') => {
|
||||||
setSortKey(key);
|
setSortKey(key);
|
||||||
setSortDirection(direction);
|
setSortDirection(direction);
|
||||||
|
setContentPage(1);
|
||||||
}, []);
|
}, []);
|
||||||
|
|
||||||
const togglePlaylist = useCallback((id: number | 'uncategorized') => {
|
const togglePlaylist = useCallback((id: number | 'uncategorized') => {
|
||||||
|
|
@ -316,55 +337,10 @@ export function ChannelDetail() {
|
||||||
clearSelection();
|
clearSelection();
|
||||||
}, [selectedIds, downloadContent, clearSelection]);
|
}, [selectedIds, downloadContent, clearSelection]);
|
||||||
|
|
||||||
// ── Sorted content ──
|
// ── Sorted content (server-side — just use content directly) ──
|
||||||
|
|
||||||
const sortedContent = useMemo(() => {
|
// Sort is handled server-side via contentFilters.sortBy/sortDirection.
|
||||||
const items = content ?? [];
|
// playlistGroups still needs client-side grouping for YouTube channels.
|
||||||
if (!sortKey) return items;
|
|
||||||
const sorted = [...items];
|
|
||||||
sorted.sort((a, b) => {
|
|
||||||
let cmp = 0;
|
|
||||||
switch (sortKey) {
|
|
||||||
case 'title':
|
|
||||||
cmp = a.title.localeCompare(b.title);
|
|
||||||
break;
|
|
||||||
case 'publishedAt': {
|
|
||||||
const aDate = a.publishedAt ? Date.parse(a.publishedAt) : -Infinity;
|
|
||||||
const bDate = b.publishedAt ? Date.parse(b.publishedAt) : -Infinity;
|
|
||||||
cmp = aDate - bDate;
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
case 'status':
|
|
||||||
cmp = a.status.localeCompare(b.status);
|
|
||||||
break;
|
|
||||||
case 'duration': {
|
|
||||||
const aDur = a.duration ?? -Infinity;
|
|
||||||
const bDur = b.duration ?? -Infinity;
|
|
||||||
cmp = aDur - bDur;
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
case 'fileSize':
|
|
||||||
cmp = (a.fileSize ?? -Infinity) - (b.fileSize ?? -Infinity);
|
|
||||||
break;
|
|
||||||
case 'downloadedAt': {
|
|
||||||
const aDate2 = a.downloadedAt ? Date.parse(a.downloadedAt) : -Infinity;
|
|
||||||
const bDate2 = b.downloadedAt ? Date.parse(b.downloadedAt) : -Infinity;
|
|
||||||
cmp = aDate2 - bDate2;
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
case 'quality': {
|
|
||||||
const aQ = a.qualityMetadata?.actualResolution ?? '';
|
|
||||||
const bQ = b.qualityMetadata?.actualResolution ?? '';
|
|
||||||
cmp = aQ.localeCompare(bQ);
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
default:
|
|
||||||
return 0;
|
|
||||||
}
|
|
||||||
return sortDirection === 'desc' ? -cmp : cmp;
|
|
||||||
});
|
|
||||||
return sorted;
|
|
||||||
}, [content, sortKey, sortDirection]);
|
|
||||||
|
|
||||||
// ── Playlist grouping (YouTube only) ──
|
// ── Playlist grouping (YouTube only) ──
|
||||||
|
|
||||||
|
|
@ -379,7 +355,7 @@ export function ChannelDetail() {
|
||||||
|
|
||||||
// Build a Map from content ID to content item for O(1) lookups (js-index-maps)
|
// Build a Map from content ID to content item for O(1) lookups (js-index-maps)
|
||||||
const contentById = new Map<number, ContentItem>();
|
const contentById = new Map<number, ContentItem>();
|
||||||
for (const item of sortedContent) {
|
for (const item of content) {
|
||||||
contentById.set(item.id, item);
|
contentById.set(item.id, item);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -399,13 +375,13 @@ export function ChannelDetail() {
|
||||||
}
|
}
|
||||||
|
|
||||||
// Uncategorized: items not in any playlist
|
// Uncategorized: items not in any playlist
|
||||||
const uncategorized = sortedContent.filter((item) => !categorizedIds.has(item.id));
|
const uncategorized = content.filter((item) => !categorizedIds.has(item.id));
|
||||||
if (uncategorized.length > 0) {
|
if (uncategorized.length > 0) {
|
||||||
groups.push({ id: 'uncategorized', title: 'Uncategorized', items: uncategorized });
|
groups.push({ id: 'uncategorized', title: 'Uncategorized', items: uncategorized });
|
||||||
}
|
}
|
||||||
|
|
||||||
return groups.length > 0 ? groups : null;
|
return groups.length > 0 ? groups : null;
|
||||||
}, [channel, playlistData, sortedContent]);
|
}, [channel, playlistData, content]);
|
||||||
|
|
||||||
// ── Content table columns ──
|
// ── Content table columns ──
|
||||||
|
|
||||||
|
|
@ -1066,6 +1042,7 @@ export function ChannelDetail() {
|
||||||
display: 'flex',
|
display: 'flex',
|
||||||
alignItems: 'center',
|
alignItems: 'center',
|
||||||
gap: 'var(--space-3)',
|
gap: 'var(--space-3)',
|
||||||
|
flexWrap: 'wrap',
|
||||||
}}
|
}}
|
||||||
>
|
>
|
||||||
<h2
|
<h2
|
||||||
|
|
@ -1078,6 +1055,65 @@ export function ChannelDetail() {
|
||||||
>
|
>
|
||||||
Content
|
Content
|
||||||
</h2>
|
</h2>
|
||||||
|
{contentPagination ? (
|
||||||
|
<span style={{ fontSize: 'var(--font-size-xs)', color: 'var(--text-muted)', fontVariantNumeric: 'tabular-nums' }}>
|
||||||
|
{contentPagination.totalItems} items
|
||||||
|
</span>
|
||||||
|
) : null}
|
||||||
|
<div style={{ flex: 1 }} />
|
||||||
|
{/* Search */}
|
||||||
|
<input
|
||||||
|
type="text"
|
||||||
|
placeholder="Search content…"
|
||||||
|
value={contentSearch}
|
||||||
|
onChange={(e) => { setContentSearch(e.target.value); setContentPage(1); }}
|
||||||
|
style={{
|
||||||
|
padding: 'var(--space-2) var(--space-3)',
|
||||||
|
borderRadius: 'var(--radius-md)',
|
||||||
|
border: '1px solid var(--border)',
|
||||||
|
backgroundColor: 'var(--bg-input)',
|
||||||
|
color: 'var(--text-primary)',
|
||||||
|
fontSize: 'var(--font-size-sm)',
|
||||||
|
width: 200,
|
||||||
|
}}
|
||||||
|
/>
|
||||||
|
{/* Status filter */}
|
||||||
|
<select
|
||||||
|
value={contentStatusFilter}
|
||||||
|
onChange={(e) => { setContentStatusFilter(e.target.value); setContentPage(1); }}
|
||||||
|
aria-label="Filter by status"
|
||||||
|
style={{
|
||||||
|
padding: 'var(--space-2) var(--space-3)',
|
||||||
|
borderRadius: 'var(--radius-md)',
|
||||||
|
fontSize: 'var(--font-size-sm)',
|
||||||
|
minWidth: 100,
|
||||||
|
}}
|
||||||
|
>
|
||||||
|
<option value="">All Status</option>
|
||||||
|
<option value="monitored">Monitored</option>
|
||||||
|
<option value="queued">Queued</option>
|
||||||
|
<option value="downloading">Downloading</option>
|
||||||
|
<option value="downloaded">Downloaded</option>
|
||||||
|
<option value="failed">Failed</option>
|
||||||
|
<option value="ignored">Ignored</option>
|
||||||
|
</select>
|
||||||
|
{/* Type filter */}
|
||||||
|
<select
|
||||||
|
value={contentTypeFilter}
|
||||||
|
onChange={(e) => { setContentTypeFilter(e.target.value); setContentPage(1); }}
|
||||||
|
aria-label="Filter by type"
|
||||||
|
style={{
|
||||||
|
padding: 'var(--space-2) var(--space-3)',
|
||||||
|
borderRadius: 'var(--radius-md)',
|
||||||
|
fontSize: 'var(--font-size-sm)',
|
||||||
|
minWidth: 90,
|
||||||
|
}}
|
||||||
|
>
|
||||||
|
<option value="">All Types</option>
|
||||||
|
<option value="video">Video</option>
|
||||||
|
<option value="audio">Audio</option>
|
||||||
|
<option value="livestream">Livestream</option>
|
||||||
|
</select>
|
||||||
</div>
|
</div>
|
||||||
{contentError ? (
|
{contentError ? (
|
||||||
<div
|
<div
|
||||||
|
|
@ -1128,8 +1164,19 @@ export function ChannelDetail() {
|
||||||
) : hasPlaylistGroups ? (
|
) : hasPlaylistGroups ? (
|
||||||
renderPlaylistGroups(playlistGroups!)
|
renderPlaylistGroups(playlistGroups!)
|
||||||
) : (
|
) : (
|
||||||
renderTable(sortedContent)
|
renderTable(content)
|
||||||
)}
|
)}
|
||||||
|
{/* Pagination controls */}
|
||||||
|
{contentPagination && contentPagination.totalPages > 1 ? (
|
||||||
|
<div style={{ padding: 'var(--space-3) var(--space-5)', borderTop: '1px solid var(--border)' }}>
|
||||||
|
<Pagination
|
||||||
|
page={contentPagination.page}
|
||||||
|
totalPages={contentPagination.totalPages}
|
||||||
|
totalItems={contentPagination.totalItems}
|
||||||
|
onPageChange={setContentPage}
|
||||||
|
/>
|
||||||
|
</div>
|
||||||
|
) : null}
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
{/* Floating bulk action bar */}
|
{/* Floating bulk action bar */}
|
||||||
|
|
|
||||||
20
src/index.ts
20
src/index.ts
|
|
@ -23,6 +23,7 @@ import { PlatformRegistry } from './sources/platform-source';
|
||||||
import { YouTubeSource } from './sources/youtube';
|
import { YouTubeSource } from './sources/youtube';
|
||||||
import { SoundCloudSource } from './sources/soundcloud';
|
import { SoundCloudSource } from './sources/soundcloud';
|
||||||
import { Platform } from './types/index';
|
import { Platform } from './types/index';
|
||||||
|
import { getYtDlpVersion, updateYtDlp } from './sources/yt-dlp';
|
||||||
import type { ViteDevServer } from 'vite';
|
import type { ViteDevServer } from 'vite';
|
||||||
|
|
||||||
const APP_NAME = 'Tubearr';
|
const APP_NAME = 'Tubearr';
|
||||||
|
|
@ -44,6 +45,25 @@ async function main(): Promise<void> {
|
||||||
await seedAppDefaults(db);
|
await seedAppDefaults(db);
|
||||||
console.log(`[${APP_NAME}] App settings seeded`);
|
console.log(`[${APP_NAME}] App settings seeded`);
|
||||||
|
|
||||||
|
// 2d. Check yt-dlp version and auto-update if configured
|
||||||
|
try {
|
||||||
|
const version = await getYtDlpVersion();
|
||||||
|
if (version) {
|
||||||
|
console.log(`[${APP_NAME}] yt-dlp version: ${version}`);
|
||||||
|
// Auto-update on startup (non-blocking — continue if it fails)
|
||||||
|
if (appConfig.nodeEnv === 'production') {
|
||||||
|
const result = await updateYtDlp();
|
||||||
|
if (result.updated) {
|
||||||
|
console.log(`[${APP_NAME}] yt-dlp updated: ${result.previousVersion} → ${result.version}`);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
console.warn(`[${APP_NAME}] yt-dlp not found on PATH — downloads will fail`);
|
||||||
|
}
|
||||||
|
} catch (err) {
|
||||||
|
console.warn(`[${APP_NAME}] yt-dlp check failed: ${err instanceof Error ? err.message : String(err)}`);
|
||||||
|
}
|
||||||
|
|
||||||
// 3. Build and configure Fastify server
|
// 3. Build and configure Fastify server
|
||||||
// In dev mode, embed Vite for HMR — single port, no separate frontend process
|
// In dev mode, embed Vite for HMR — single port, no separate frontend process
|
||||||
let vite: ViteDevServer | undefined;
|
let vite: ViteDevServer | undefined;
|
||||||
|
|
|
||||||
|
|
@ -2,6 +2,7 @@ import { type FastifyInstance } from 'fastify';
|
||||||
import {
|
import {
|
||||||
getAllContentItems,
|
getAllContentItems,
|
||||||
getContentByChannelId,
|
getContentByChannelId,
|
||||||
|
getChannelContentPaginated,
|
||||||
setMonitored,
|
setMonitored,
|
||||||
bulkSetMonitored,
|
bulkSetMonitored,
|
||||||
} from '../../db/repositories/content-repository';
|
} from '../../db/repositories/content-repository';
|
||||||
|
|
@ -202,6 +203,15 @@ export async function contentRoutes(fastify: FastifyInstance): Promise<void> {
|
||||||
|
|
||||||
fastify.get<{
|
fastify.get<{
|
||||||
Params: { id: string };
|
Params: { id: string };
|
||||||
|
Querystring: {
|
||||||
|
page?: string;
|
||||||
|
pageSize?: string;
|
||||||
|
search?: string;
|
||||||
|
status?: string;
|
||||||
|
contentType?: string;
|
||||||
|
sortBy?: string;
|
||||||
|
sortDirection?: string;
|
||||||
|
};
|
||||||
}>('/api/v1/channel/:id/content', async (request, reply) => {
|
}>('/api/v1/channel/:id/content', async (request, reply) => {
|
||||||
const channelId = parseInt(request.params.id, 10);
|
const channelId = parseInt(request.params.id, 10);
|
||||||
|
|
||||||
|
|
@ -213,12 +223,50 @@ export async function contentRoutes(fastify: FastifyInstance): Promise<void> {
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
try {
|
const page = Math.max(1, parseInt(request.query.page ?? '1', 10) || 1);
|
||||||
const items = await getContentByChannelId(fastify.db, channelId);
|
const pageSize = Math.min(
|
||||||
|
200,
|
||||||
|
Math.max(1, parseInt(request.query.pageSize ?? '50', 10) || 50)
|
||||||
|
);
|
||||||
|
|
||||||
const response: ApiResponse<ContentItem[]> = {
|
// If no pagination params provided, return all items (backwards-compatible)
|
||||||
|
const hasPaginationParams = request.query.page || request.query.pageSize || request.query.search || request.query.status || request.query.contentType || request.query.sortBy;
|
||||||
|
|
||||||
|
try {
|
||||||
|
if (!hasPaginationParams) {
|
||||||
|
// Legacy mode: return all items as flat array (backwards-compatible)
|
||||||
|
const items = await getContentByChannelId(fastify.db, channelId);
|
||||||
|
const response: ApiResponse<ContentItem[]> = {
|
||||||
|
success: true,
|
||||||
|
data: items,
|
||||||
|
};
|
||||||
|
return response;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Paginated mode with filters
|
||||||
|
const result = await getChannelContentPaginated(
|
||||||
|
fastify.db,
|
||||||
|
channelId,
|
||||||
|
{
|
||||||
|
search: request.query.search || undefined,
|
||||||
|
status: (request.query.status as ContentStatus) || undefined,
|
||||||
|
contentType: (request.query.contentType as ContentType) || undefined,
|
||||||
|
sortBy: request.query.sortBy as 'title' | 'publishedAt' | 'status' | 'duration' | 'fileSize' | 'downloadedAt' | undefined,
|
||||||
|
sortDirection: (request.query.sortDirection as 'asc' | 'desc') || undefined,
|
||||||
|
},
|
||||||
|
page,
|
||||||
|
pageSize
|
||||||
|
);
|
||||||
|
|
||||||
|
const response: PaginatedResponse<ContentItem> = {
|
||||||
success: true,
|
success: true,
|
||||||
data: items,
|
data: result.items,
|
||||||
|
pagination: {
|
||||||
|
page,
|
||||||
|
pageSize,
|
||||||
|
totalItems: result.total,
|
||||||
|
totalPages: Math.ceil(result.total / pageSize),
|
||||||
|
},
|
||||||
};
|
};
|
||||||
|
|
||||||
return response;
|
return response;
|
||||||
|
|
|
||||||
|
|
@ -3,7 +3,7 @@ import { extname } from 'node:path';
|
||||||
import { createInterface } from 'node:readline';
|
import { createInterface } from 'node:readline';
|
||||||
import type { LibSQLDatabase } from 'drizzle-orm/libsql';
|
import type { LibSQLDatabase } from 'drizzle-orm/libsql';
|
||||||
import type * as schema from '../db/schema/index';
|
import type * as schema from '../db/schema/index';
|
||||||
import { execYtDlp, spawnYtDlp, YtDlpError } from '../sources/yt-dlp';
|
import { execYtDlp, spawnYtDlp, YtDlpError, classifyYtDlpError } from '../sources/yt-dlp';
|
||||||
import { updateContentItem } from '../db/repositories/content-repository';
|
import { updateContentItem } from '../db/repositories/content-repository';
|
||||||
import { parseProgressLine } from './progress-parser';
|
import { parseProgressLine } from './progress-parser';
|
||||||
import type { DownloadEventBus } from './event-bus';
|
import type { DownloadEventBus } from './event-bus';
|
||||||
|
|
@ -137,16 +137,22 @@ export class DownloadService {
|
||||||
// Report error to rate limiter
|
// Report error to rate limiter
|
||||||
this.rateLimiter.reportError(channel.platform as Platform);
|
this.rateLimiter.reportError(channel.platform as Platform);
|
||||||
|
|
||||||
|
// Classify the error for better retry decisions
|
||||||
|
const errorMsg = err instanceof Error ? err.message : String(err);
|
||||||
|
const stderr = err instanceof YtDlpError ? err.stderr : '';
|
||||||
|
const errorCategory = classifyYtDlpError(stderr || errorMsg);
|
||||||
|
|
||||||
// Update status to failed
|
// Update status to failed
|
||||||
await updateContentItem(this.db, contentItem.id, { status: 'failed' });
|
await updateContentItem(this.db, contentItem.id, { status: 'failed' });
|
||||||
|
|
||||||
const errorMsg = err instanceof Error ? err.message : String(err);
|
console.log(
|
||||||
console.log(`${logPrefix} status=failed error="${errorMsg.slice(0, 200)}"`);
|
`${logPrefix} status=failed category=${errorCategory} error="${errorMsg.slice(0, 200)}"`
|
||||||
|
);
|
||||||
|
|
||||||
// Emit download:failed event
|
// Emit download:failed event with error category
|
||||||
this.eventBus?.emitDownload('download:failed', {
|
this.eventBus?.emitDownload('download:failed', {
|
||||||
contentItemId: contentItem.id,
|
contentItemId: contentItem.id,
|
||||||
error: errorMsg.slice(0, 200),
|
error: `[${errorCategory}] ${errorMsg.slice(0, 200)}`,
|
||||||
});
|
});
|
||||||
|
|
||||||
throw err;
|
throw err;
|
||||||
|
|
@ -288,32 +294,32 @@ export class DownloadService {
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Build format args for video content.
|
* Build format args for video content.
|
||||||
|
* Uses a fallback chain: preferred resolution → best available → any.
|
||||||
|
* yt-dlp supports `/` as a fallback separator: `format1/format2/format3`.
|
||||||
*/
|
*/
|
||||||
private buildVideoArgs(formatProfile?: FormatProfile): string[] {
|
private buildVideoArgs(formatProfile?: FormatProfile): string[] {
|
||||||
const args: string[] = [];
|
const args: string[] = [];
|
||||||
|
const container = formatProfile?.containerFormat ?? 'mp4';
|
||||||
|
|
||||||
if (formatProfile?.videoResolution === 'Best') {
|
if (formatProfile?.videoResolution === 'Best') {
|
||||||
// "Best" selects separate best-quality video + audio streams, merged together.
|
// Best quality: separate streams merged
|
||||||
// This is higher quality than `-f best` which picks a single combined format.
|
args.push('-f', 'bestvideo+bestaudio/bestvideo*+bestaudio/best');
|
||||||
args.push('-f', 'bestvideo+bestaudio/best');
|
|
||||||
const container = formatProfile.containerFormat ?? 'mp4';
|
|
||||||
args.push('--merge-output-format', container);
|
args.push('--merge-output-format', container);
|
||||||
} else if (formatProfile?.videoResolution) {
|
} else if (formatProfile?.videoResolution) {
|
||||||
const height = parseResolutionHeight(formatProfile.videoResolution);
|
const height = parseResolutionHeight(formatProfile.videoResolution);
|
||||||
if (height) {
|
if (height) {
|
||||||
|
// Fallback chain: exact res → best under res → single best stream → any
|
||||||
args.push(
|
args.push(
|
||||||
'-f',
|
'-f',
|
||||||
`bestvideo[height<=${height}]+bestaudio/best[height<=${height}]`
|
`bestvideo[height<=${height}]+bestaudio/bestvideo[height<=${height}]*+bestaudio/best[height<=${height}]/bestvideo+bestaudio/best`
|
||||||
);
|
);
|
||||||
} else {
|
} else {
|
||||||
args.push('-f', 'best');
|
args.push('-f', 'bestvideo+bestaudio/best');
|
||||||
}
|
}
|
||||||
|
|
||||||
// Container format for merge
|
|
||||||
const container = formatProfile.containerFormat ?? 'mp4';
|
|
||||||
args.push('--merge-output-format', container);
|
args.push('--merge-output-format', container);
|
||||||
} else {
|
} else {
|
||||||
args.push('-f', 'best');
|
args.push('-f', 'bestvideo+bestaudio/best');
|
||||||
|
args.push('--merge-output-format', container);
|
||||||
}
|
}
|
||||||
|
|
||||||
return args;
|
return args;
|
||||||
|
|
@ -367,18 +373,33 @@ export class DownloadService {
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Parse the final file path from yt-dlp stdout.
|
* Parse the final file path from yt-dlp stdout.
|
||||||
* The `--print after_move:filepath` flag makes yt-dlp output the final path
|
* The `--print after_move:filepath` flag makes yt-dlp output the final path.
|
||||||
* as the last line of stdout.
|
*
|
||||||
|
* Strategy: walk backwards through lines, skipping known yt-dlp output prefixes
|
||||||
|
* (e.g. [download], [Merger], [ExtractAudio], Deleting).
|
||||||
|
* A valid path line should be an absolute path or at least contain a file extension.
|
||||||
*/
|
*/
|
||||||
private parseFinalPath(stdout: string, fallbackPath: string): string {
|
private parseFinalPath(stdout: string, fallbackPath: string): string {
|
||||||
const lines = stdout.trim().split('\n');
|
const lines = stdout.trim().split('\n');
|
||||||
// The filepath from --print is typically the last non-empty line
|
|
||||||
|
// Known non-path prefixes from yt-dlp output
|
||||||
|
const NON_PATH_PREFIXES = ['[', 'Deleting', 'WARNING:', 'ERROR:', 'Merging', 'Post-process'];
|
||||||
|
|
||||||
for (let i = lines.length - 1; i >= 0; i--) {
|
for (let i = lines.length - 1; i >= 0; i--) {
|
||||||
const line = lines[i].trim();
|
const line = lines[i].trim();
|
||||||
if (line && !line.startsWith('[') && !line.startsWith('Deleting')) {
|
if (!line) continue;
|
||||||
|
|
||||||
|
// Skip known yt-dlp output lines
|
||||||
|
const isNonPath = NON_PATH_PREFIXES.some((prefix) => line.startsWith(prefix));
|
||||||
|
if (isNonPath) continue;
|
||||||
|
|
||||||
|
// A valid path should have a file extension or start with /
|
||||||
|
if (line.startsWith('/') || /\.\w{2,5}$/.test(line)) {
|
||||||
return line;
|
return line;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
console.warn('[download] Could not parse final path from yt-dlp output, using fallback');
|
||||||
return fallbackPath;
|
return fallbackPath;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -212,3 +212,88 @@ export async function getYtDlpVersion(): Promise<string | null> {
|
||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// ── Auto-Update ──
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Update yt-dlp to the latest version.
|
||||||
|
* Uses `yt-dlp -U` which handles self-update on most installations.
|
||||||
|
* For pip-based installs (Alpine/Docker), falls back to `pip install -U yt-dlp`.
|
||||||
|
*
|
||||||
|
* Returns { updated, version, previousVersion } on success.
|
||||||
|
*/
|
||||||
|
export async function updateYtDlp(): Promise<{
|
||||||
|
updated: boolean;
|
||||||
|
version: string | null;
|
||||||
|
previousVersion: string | null;
|
||||||
|
}> {
|
||||||
|
const previousVersion = await getYtDlpVersion();
|
||||||
|
|
||||||
|
try {
|
||||||
|
// Try native self-update first
|
||||||
|
const { stderr } = await execFileAsync('yt-dlp', ['-U'], {
|
||||||
|
timeout: 120_000,
|
||||||
|
windowsHide: true,
|
||||||
|
});
|
||||||
|
|
||||||
|
// Check if it actually updated
|
||||||
|
const newVersion = await getYtDlpVersion();
|
||||||
|
const didUpdate = newVersion !== previousVersion;
|
||||||
|
|
||||||
|
if (didUpdate) {
|
||||||
|
console.log(`[yt-dlp] Updated from ${previousVersion} to ${newVersion}`);
|
||||||
|
} else if (stderr.toLowerCase().includes('up to date')) {
|
||||||
|
console.log(`[yt-dlp] Already up to date (${newVersion})`);
|
||||||
|
}
|
||||||
|
|
||||||
|
return { updated: didUpdate, version: newVersion, previousVersion };
|
||||||
|
} catch (err) {
|
||||||
|
// Self-update may not work in pip-based installs — try pip
|
||||||
|
try {
|
||||||
|
await execFileAsync('pip', ['install', '-U', 'yt-dlp'], {
|
||||||
|
timeout: 120_000,
|
||||||
|
windowsHide: true,
|
||||||
|
});
|
||||||
|
|
||||||
|
const newVersion = await getYtDlpVersion();
|
||||||
|
const didUpdate = newVersion !== previousVersion;
|
||||||
|
console.log(
|
||||||
|
`[yt-dlp] pip update: ${previousVersion} → ${newVersion}${didUpdate ? '' : ' (no change)'}`
|
||||||
|
);
|
||||||
|
|
||||||
|
return { updated: didUpdate, version: newVersion, previousVersion };
|
||||||
|
} catch (pipErr) {
|
||||||
|
console.warn(
|
||||||
|
`[yt-dlp] Auto-update failed: ${err instanceof Error ? err.message : String(err)}`
|
||||||
|
);
|
||||||
|
return { updated: false, version: previousVersion, previousVersion };
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// ── Error Classification ──
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Classify a yt-dlp error into a category for better retry/fallback decisions.
|
||||||
|
*/
|
||||||
|
export type YtDlpErrorCategory =
|
||||||
|
| 'rate_limit' // 429, too many requests
|
||||||
|
| 'format_unavailable' // requested format not available
|
||||||
|
| 'geo_blocked' // geo-restriction
|
||||||
|
| 'age_restricted' // age-gated content
|
||||||
|
| 'private' // private or removed video
|
||||||
|
| 'network' // DNS, connection, timeout
|
||||||
|
| 'unknown';
|
||||||
|
|
||||||
|
export function classifyYtDlpError(stderr: string): YtDlpErrorCategory {
|
||||||
|
const lower = stderr.toLowerCase();
|
||||||
|
|
||||||
|
if (lower.includes('429') || lower.includes('too many requests')) return 'rate_limit';
|
||||||
|
if (lower.includes('requested format') || lower.includes('format is not available')) return 'format_unavailable';
|
||||||
|
if (lower.includes('not available in your country') || lower.includes('geo')) return 'geo_blocked';
|
||||||
|
if (lower.includes('age') && (lower.includes('restricted') || lower.includes('verify'))) return 'age_restricted';
|
||||||
|
if (lower.includes('private video') || lower.includes('video unavailable') || lower.includes('been removed')) return 'private';
|
||||||
|
if (lower.includes('unable to download') || lower.includes('connection') || lower.includes('timed out') || lower.includes('urlopen error')) return 'network';
|
||||||
|
|
||||||
|
return 'unknown';
|
||||||
|
}
|
||||||
|
|
|
||||||
Loading…
Add table
Reference in a new issue