diff --git a/CLAUDE.md b/CLAUDE.md index b27a01b1..10a206c5 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -38,7 +38,7 @@ For multi-part requests (e.g., "review this PR AND explain WebSocket handling"), - `modules/playlistConstants.js`: backend playlist fetch/batch limits, preview default, and shared refresh-conflict text. Client feature defaults live in `components/PlaylistPage/playlistConstants.ts`. - First-setup recovery persists `playlists.auto_download_setup_error`: legacy scheduled incomplete snapshots stay enabled and retry with a visible waiting notice; oversized playlists are disabled with an explanation. Recovery refreshes the cache without setting a baseline or clearing requests and cannot become a successful idle sweep. Successful setup clears the error. Subscribe/restore returns 201 with a warning for expected following failures, disables auto-download for size/completeness failures, and preserves settings on a concurrent refresh. Warnings travel through the subscription hook and router state to the page. - Playlist batch/following endpoints: `GET /api/playlists/:playlistId/download-preview` previews publication/position choices; `POST /api/playlists/:playlistId/download-batch` queues explicit ids; `POST /api/playlists/:playlistId/following` sets up/resumes following or resets the starting point while preserving pause state. All require auth and an enabled playlist. Swagger annotations in `routes/playlists.js` describe validation and errors. -- `modules/download/`: download orchestration (`downloadExecutor`, `ytdlpCommandBuilder`, `DownloadProgressMonitor`, `tempPathManager`, `videoMetadataProcessor`, `customArgsParser`, `cookiePlayerClients` (pure: the yt-dlp `player_client` list (`default,mweb,web_safari`) added to video downloads and single-video metadata fetches whenever a cookies file is in play, because YouTube's SABR-only experiment strips URLs from the logged-in default clients and leaves only 360p itag 18; folded into any user `youtube:` extractor-args token since yt-dlp keeps only the last `--extractor-args` per extractor key, and a user-supplied `player_client` wins), `ytdlpStderrSignals` (pure stderr-line classifiers shared by `YtdlpOutputRouter` and `downloadJobFinalizer`: `containsHttp403` ignores yt-dlp's "require a GVS PO Token ... may yield HTTP Error 403" advisory, which prints on every free-account cookie run, and `isSabrRestriction` matches the account-level SABR experiment warning; the router classifies complete stderr lines only, holding a split line until its newline and flushing the remainder from `dispose()`, after which output is logged but never classified or broadcast; `downloadExecutor` finalizes on the child's `close` event (stdio drained), not `exit`, with a 5 s `STDIO_DRAIN_TIMEOUT_MS` fallback for pipes held open by an orphaned ffmpeg or post-process child), `ytdlpValidator`, `downloadRunTracker` (aggregates the many jobs of one channel+playlist sweep into a single end-of-run summary + notification), `ytdlpEnvBuilder` (yt-dlp process env construction), `DownloadTimeoutController` (per-run activity/absolute timeouts + graceful shutdown), `YtdlpErrorTracker` (per-run ERROR-line classification and failure bookkeeping; a `WARNING: Unable to download video subtitles` line un-fails the current video, because yt-dlp prints the subtitle downloader's ERROR first and then still downloads the video under `--ignore-errors`), `downloadStatusResolver` (pure job-status/final-state decision functions), `downloadResultProcessor` (post-run URL resolution, success/failure partitioning, archive reconciliation), `downloadCleanup` (in-progress video and .part/fragment file cleanup), `downloadCompletionEffects` (post-completion side effects: temp-file/tracking cleanup, poster backfill, playlist hooks, Plex refresh, next-job kickoff), `downloadJobFinalizer` (terminal status persistence, final WebSocket broadcast, run-tracker reporting, notification dispatch), `YtdlpOutputRouter` (per-run stdout/stderr routing + throttled progress emission, plus a 25s heartbeat that rebroadcasts the monitor snapshot while yt-dlp is silent (large merges, audio extraction) so the activity page and its REST seed stay live through quiet post-processing), `transient403RetryPlanner` (pure decision logic for auto-retrying videos that failed with a transient mid-stream HTTP 403; the finalizer enqueues an "Auto-retry" URL-list job via a callback injected from downloadModule, capped by the `downloadAutoRetryCount` config, and handed-off failures are tagged `autoRetryQueued` so run summaries report the post-retry outcome), `failureAdvisor` (pure registry mapping known persistent-failure signatures (403 with/without cookies, bot check) to plain-language diagnoses; the finalizer stamps a `diagnosisKey` on reportable failed videos and emits a deduped `diagnoses` list that flows into job data, run summaries, the Download History UI, and notification "Likely cause" lines), `failedVideoEnricher` (best-effort DB backfill of title/channel onto failed-video records whose download died before metadata existed, sourced from videos, channelvideos + channels, and playlistvideos; called once by the finalizer right after result partitioning, and the client falls back to a linked YouTube ID when enrichment finds nothing)). +- `modules/download/`: download orchestration (`downloadExecutor`, `ytdlpCommandBuilder`, `DownloadProgressMonitor`, `tempPathManager`, `videoMetadataProcessor`, `customArgsParser`, `cookiePlayerClients` (pure: the yt-dlp `player_client` list (`default,mweb,web_safari`) added to video downloads and single-video metadata fetches whenever a cookies file is in play, because YouTube's SABR-only experiment strips URLs from the logged-in default clients and leaves only 360p itag 18; folded into any user `youtube:` extractor-args token since yt-dlp keeps only the last `--extractor-args` per extractor key, and a user-supplied `player_client` wins), `ytdlpStderrSignals` (pure stderr-line classifiers shared by `YtdlpOutputRouter` and `downloadJobFinalizer`: `containsHttp403` ignores yt-dlp's "require a GVS PO Token ... may yield HTTP Error 403" advisory, which prints on every free-account cookie run, and `isSabrRestriction` matches the account-level SABR experiment warning; the router classifies complete stderr lines only, holding a split line until its newline and flushing the remainder from `dispose()`, after which output is logged but never classified or broadcast; `downloadExecutor` finalizes on the child's `close` event (stdio drained), not `exit`, with a 5 s `STDIO_DRAIN_TIMEOUT_MS` fallback for pipes held open by an orphaned ffmpeg or post-process child), `ytdlpValidator`, `downloadRunTracker` (aggregates the many jobs of one channel+playlist sweep into a single end-of-run summary + notification), `ytdlpEnvBuilder` (yt-dlp process env construction), `DownloadTimeoutController` (per-run activity/absolute timeouts + graceful shutdown), `YtdlpErrorTracker` (per-run ERROR-line classification and failure bookkeeping; a `WARNING: Unable to download video subtitles` line un-fails the current video, because yt-dlp prints the subtitle downloader's ERROR first and then still downloads the video under `--ignore-errors`), `downloadStatusResolver` (pure job-status/final-state decision functions), `downloadResultProcessor` (post-run URL resolution, success/failure partitioning, archive reconciliation), `downloadCleanup` (in-progress video and .part/fragment file cleanup), `downloadCompletionEffects` (post-completion side effects: temp-file/tracking cleanup, poster backfill, playlist hooks, Plex refresh, next-job kickoff), `downloadJobFinalizer` (terminal status persistence, final WebSocket broadcast, run-tracker reporting, notification dispatch), `YtdlpOutputRouter` (per-run stdout/stderr routing + throttled progress emission, plus a 25s heartbeat that rebroadcasts the monitor snapshot while yt-dlp is silent (large merges, audio extraction) so the activity page and its REST seed stay live through quiet post-processing), `transient403RetryPlanner` (pure decision logic for auto-retrying eligible download failures, including transient mid-stream HTTP 403 errors and cookie-specific `Video unavailable` failures; the finalizer enqueues an "Auto-retry" URL-list job via a callback injected from downloadModule, capped by the `downloadAutoRetryCount` config, and handed-off failures are tagged `autoRetryQueued` so run summaries report the post-retry outcome), `failureAdvisor` (pure registry mapping known persistent-failure signatures (403 with/without cookies, bot check) to plain-language diagnoses; the finalizer stamps a `diagnosisKey` on reportable failed videos and emits a deduped `diagnoses` list that flows into job data, run summaries, the Download History UI, and notification "Likely cause" lines), `failedVideoEnricher` (best-effort DB backfill of title/channel onto failed-video records whose download died before metadata existed, sourced from videos, channelvideos + channels, and playlistvideos; called once by the finalizer right after result partitioning, and the client falls back to a linked YouTube ID when enrichment finds nothing)). - `modules/filesystem/`: path/file abstraction (`pathBuilder`, `directoryManager`, `fileOperations`, `sanitizer`, `constants`). Good example of the sub-module aggregator pattern. - `modules/channel/`: channel domain decomposed from the former 2,851-line channelModule (`fetchRegistry` shared in-flight-fetch guard, `channelYtdlpExecutor` yt-dlp listing runner, `channelIdentity`, `tabState`, `channelMappers`, `videoEntryParser`, `channelVideoWriter`/`channelVideoQuery` channel-video persistence and querying, `channelThumbnails` (avatar + banner caching, poster/backdrop backfill via `backfillChannelImages`), `channelBackdropBackfill` (on `writeBackdropImages` enable: fetches missing channel banners and backfills backdrop.jpg into channel folders), `apiFallbackLogger`, `channelMetadataFetcher`, `tabManager`, `channelProvisioning` (getChannelInfo/upsertChannel), `channelCatalog` (list/enable/disable), `autoDownloadScheduler` (cron + channels.txt generation), `channelVideoFetcher`, `channelVideosService` (getChannelVideos/fetchAllChannelVideos)); `channelModule.js` remains the public facade. - `modules/notifications/`: multi-service notifications via Apprise (`serviceRegistry`, `formatters/`, `senders/`). Good example of a pluggable service registry. diff --git a/client/src/components/Configuration/sections/DownloadPerformanceSection.tsx b/client/src/components/Configuration/sections/DownloadPerformanceSection.tsx index a84c1dca..b4aa148c 100644 --- a/client/src/components/Configuration/sections/DownloadPerformanceSection.tsx +++ b/client/src/components/Configuration/sections/DownloadPerformanceSection.tsx @@ -123,7 +123,7 @@ export const DownloadPerformanceSection: React.FC3 auto-retries - Re-run videos that fail with a transient HTTP 403 in a fresh download job + Re-run videos after retryable download failures in a fresh download job diff --git a/client/src/components/Configuration/sections/__tests__/DownloadPerformanceSection.test.tsx b/client/src/components/Configuration/sections/__tests__/DownloadPerformanceSection.test.tsx index 6a308a38..b6a2594e 100644 --- a/client/src/components/Configuration/sections/__tests__/DownloadPerformanceSection.test.tsx +++ b/client/src/components/Configuration/sections/__tests__/DownloadPerformanceSection.test.tsx @@ -371,7 +371,7 @@ describe('DownloadPerformanceSection Component', () => { expect(screen.getAllByText('Auto-Retry Failed Videos').length).toBeGreaterThan(0); expect( - screen.getByText('Re-run videos that fail with a transient HTTP 403 in a fresh download job') + screen.getByText('Re-run videos after retryable download failures in a fresh download job') ).toBeInTheDocument(); }); diff --git a/client/src/components/DownloadManager/DownloadProgress.tsx b/client/src/components/DownloadManager/DownloadProgress.tsx index 468ac545..6cda537d 100644 --- a/client/src/components/DownloadManager/DownloadProgress.tsx +++ b/client/src/components/DownloadManager/DownloadProgress.tsx @@ -497,10 +497,13 @@ const DownloadProgress: React.FC = ({ { expect(mockNavigate).toHaveBeenCalledWith('/settings/cookies'); }); + test('does not offer cookie settings when the no-cookies fallback fails', async () => { + renderWithContext( + + ); + + const processCallback = getProcessCallback(); + + await act(async () => { + processCallback({ + error: true, + text: 'Bot detection encountered during the no-cookies fallback. The fallback also failed, so this video may be genuinely unavailable.', + errorCode: 'NO_COOKIES_FALLBACK_FAILED', + }); + }); + + await waitFor(() => { + expect(screen.getByText('Download Failed')).toBeInTheDocument(); + }); + + expect( + screen.getByText(/Bot detection encountered during the no-cookies fallback/) + ).toBeInTheDocument(); + expect( + screen.queryByRole('button', { name: 'Go to Settings' }) + ).not.toBeInTheDocument(); + }); + + test('does not offer cookie settings for a recoverable no-cookies fallback 403', async () => { + renderWithContext( + + ); + + const processCallback = getProcessCallback(); + + await act(async () => { + processCallback({ + error: true, + text: 'HTTP 403 detected during the no-cookies fallback. If this retry fails, the video may be genuinely unavailable.', + errorCode: 'NO_COOKIES_FALLBACK_403', + }); + }); + + await waitFor(() => { + expect(screen.getByText('Download Failed')).toBeInTheDocument(); + }); + + expect( + screen.getByText(/HTTP 403 detected during the no-cookies fallback/) + ).toBeInTheDocument(); + expect( + screen.queryByRole('button', { name: 'Go to Settings' }) + ).not.toBeInTheDocument(); + }); + test('displays generic error without settings button', async () => { renderWithContext( { } }); }); -}); \ No newline at end of file +}); diff --git a/docs/CONFIG.md b/docs/CONFIG.md index a49651dd..08bdf7a0 100644 --- a/docs/CONFIG.md +++ b/docs/CONFIG.md @@ -611,9 +611,9 @@ The old `discordWebhookUrl` and `notificationService` fields are automatically r - **Config Key**: `downloadAutoRetryCount` - **Type**: `number` - **Default**: `1` -- **Description**: Number of times a video that fails with a transient HTTP 403 is automatically re-queued in a fresh download job +- **Description**: Number of times a video with a retryable download failure is automatically re-queued in a fresh download job - **Options**: `0`, `1`, `2`, `3` (the values offered in the UI; `0` disables auto-retry) -- **Note**: YouTube sometimes rejects an already-issued stream URL mid-download with HTTP 403. yt-dlp's own retries (`downloadRetryCount`) re-request the same rejected URL and cannot recover; only a fresh yt-dlp run with a fresh extraction can. When a video fails with the 403 signature, Youtarr queues an "Auto-retry" job for just that video. Permanent failures (members-only, terminated channels, bot detection) are never auto-retried. +- **Note**: Auto-retry currently handles transient HTTP 403 failures by starting a fresh yt-dlp extraction, and cookie-specific `Video unavailable` failures by retrying anonymously without cookies. Permanent failures (members-only, terminated channels, bot detection) are never auto-retried. ### Enable Stall Detection - **Config Key**: `enableStallDetection` diff --git a/docs/YOUTARR_VS_ALTERNATIVES.md b/docs/YOUTARR_VS_ALTERNATIVES.md index 8f841ca7..8e7e166a 100644 --- a/docs/YOUTARR_VS_ALTERNATIVES.md +++ b/docs/YOUTARR_VS_ALTERNATIVES.md @@ -87,7 +87,7 @@ Legend: ✅ supported, ❌ not supported, ⚠️ partial/caveat, "unclear" = not | REST/GraphQL API | ✅ 40+ endpoints, Swagger UI at `/swagger`, API keys with rate-limiting | ❌ Open feature request | ✅ REST + OpenAPI docs at `/api/docs/` | | Multi-user / RBAC | ❌ Single admin + API keys | ❌ Single HTTP basic-auth user | ⚠️ Superuser/staff/read-only roles; library is still shared across users | | Apprise notifications | ✅ | ✅ | ✅ | -| Cookies (private/unlisted/members-only) | ✅ Persistent `cookies.txt` for downloads + separate one-time cookies file for bulk import (auto-deleted after use) [^1]; transient 403 failures retry automatically, persistent ones get a plain-language explanation | ✅ Three modes: Disabled / When Needed / All Ops | ✅ Plus PO-token provider support for bot-detection bypass | +| Cookies (private/unlisted/members-only) | ✅ Persistent `cookies.txt` for downloads + separate one-time cookies file for bulk import (auto-deleted after use) [^1]; retryable download failures retry automatically, persistent ones get a plain-language explanation | ✅ Three modes: Disabled / When Needed / All Ops | ✅ Plus PO-token provider support for bot-detection bypass | | Backup / restore | ✅ In-app backup & restore scripts | ⚠️ File-level (copy `/config`) | ⚠️ Documented manual multi-step procedure | | Responsive / mobile-friendly UI | ✅ Responsive; mobile screenshots | ⚠️ Likely responsive (Tailwind); not explicitly claimed | ✅ | | Live-stream / premiere handling | ⚠️ Streams in per-tab controls; in-progress stream handling not supported | ✅ Per-source shorts/livestream rules | ✅ Streams as a separate per-channel tab; disable via page size 0 | diff --git a/server/modules/__tests__/downloadModule.test.js b/server/modules/__tests__/downloadModule.test.js index 821fcbbc..eea9b519 100644 --- a/server/modules/__tests__/downloadModule.test.js +++ b/server/modules/__tests__/downloadModule.test.js @@ -17,13 +17,15 @@ jest.mock('../configModule', () => { }; mockConfigModule.on = jest.fn(); mockConfigModule.getDefaultSubfolder = jest.fn().mockReturnValue(null); + mockConfigModule.getCookiesPath = jest.fn().mockReturnValue('/cookies/file.txt'); return mockConfigModule; }); jest.mock('../jobModule', () => ({ addOrUpdateJob: jest.fn(), updateJob: jest.fn(), - getJob: jest.fn().mockReturnValue({ status: 'Pending' }) + getJob: jest.fn().mockReturnValue({ status: 'Pending' }), + startNextJob: jest.fn().mockResolvedValue() })); jest.mock('../download/downloadExecutor'); @@ -1028,13 +1030,73 @@ describe('DownloadModule', () => { body: expect.objectContaining({ urls: ['https://www.youtube.com/watch?v=abc123def45'], overrideSettings: { resolution: '720' }, - jobLabel: 'Auto-retry: 1 video (HTTP 403)', + jobLabel: 'Auto-retry: 1 video', autoRetryAttempt: 1, runId: 'run-1', }), }); }); + it('marks cookie-specific retries as anonymous', async () => { + await downloadModule.enqueueAutoRetryJob({ + retryVideos: [{ + ...retryVideo, + anonymousRetry: true, + }], + autoRetryAttempt: 1, + runId: 'run-cookie', + sourceJobData: {}, + }); + + expect(doSpecificDownloadsSpy).toHaveBeenCalledWith({ + body: expect.objectContaining({ + urls: ['https://www.youtube.com/watch?v=abc123def45'], + anonymousRetry: true, + autoRetryAttempt: 1, + runId: 'run-cookie', + }), + }); + }); + + it('splits mixed authenticated and anonymous retries into separate jobs', async () => { + const anonymousVideo = { + youtubeId: 'zzz999xxx11', + url: 'https://www.youtube.com/watch?v=zzz999xxx11', + anonymousRetry: true, + }; + + await downloadModule.enqueueAutoRetryJob({ + retryVideos: [ + { ...retryVideo, anonymousRetry: false }, + anonymousVideo, + ], + autoRetryAttempt: 1, + runId: 'run-mixed', + sourceJobData: {}, + }); + + expect(doSpecificDownloadsSpy).toHaveBeenCalledTimes(2); + + const bodies = doSpecificDownloadsSpy.mock.calls.map(([{ body }]) => body); + + expect(bodies).toEqual(expect.arrayContaining([ + expect.objectContaining({ + urls: ['https://www.youtube.com/watch?v=abc123def45'], + jobLabel: 'Auto-retry: 1 video', + anonymousRetry: false, + autoRetryAttempt: 1, + runId: 'run-mixed', + }), + expect.objectContaining({ + urls: ['https://www.youtube.com/watch?v=zzz999xxx11'], + jobLabel: 'Auto-retry: 1 video (no cookies)', + anonymousRetry: true, + autoRetryAttempt: 1, + runId: 'run-mixed', + }), + ])); + }); + it('resolves owning channels for unmapped videos and passes channelId when unique', async () => { ChannelVideoMock.findAll.mockResolvedValue([ { youtube_id: 'abc123def45', channel_id: 'UC-owner' }, @@ -1295,7 +1357,7 @@ describe('DownloadModule', () => { }), false ); - expect(YtdlpCommandBuilderMock.getBaseCommandArgsForManualDownload).toHaveBeenCalledWith('1080', false, null, false); + expect(YtdlpCommandBuilderMock.getBaseCommandArgsForManualDownload).toHaveBeenCalledWith('1080', false, null, false, { cookiesEnabled: true }); expect(mockDownloadExecutor.doDownload).toHaveBeenCalledWith( expect.arrayContaining([ '--format', 'best[height<=1080]', @@ -1310,7 +1372,17 @@ describe('DownloadModule', () => { ['https://youtube.com/watch?v=abc123', 'https://youtube.com/watch?v=def456'], false, false, - { subfolderOverride: null, subfolderFallback: null, ratingOverride: undefined, ratingFallback: null, skipVideoFolder: false, ownerChannelId: null, ownerChannelMap: null } + expect.objectContaining({ + subfolderOverride: null, + subfolderFallback: null, + ratingOverride: undefined, + ratingFallback: null, + skipVideoFolder: false, + ownerChannelId: null, + ownerChannelMap: null, + cookiesEnabled: true, + anonymousRetry: false + }) ); }); @@ -1341,7 +1413,17 @@ describe('DownloadModule', () => { ['https://youtube.com/watch?v=xyz789'], false, false, - { subfolderOverride: null, subfolderFallback: null, ratingOverride: undefined, ratingFallback: null, skipVideoFolder: false, ownerChannelId: null, ownerChannelMap: null } + expect.objectContaining({ + subfolderOverride: null, + subfolderFallback: null, + ratingOverride: undefined, + ratingFallback: null, + skipVideoFolder: false, + ownerChannelId: null, + ownerChannelMap: null, + cookiesEnabled: true, + anonymousRetry: false + }) ); }); @@ -1367,7 +1449,17 @@ describe('DownloadModule', () => { ['-abc123', 'https://youtube.com/watch?v=def456'], false, false, - { subfolderOverride: null, subfolderFallback: null, ratingOverride: undefined, ratingFallback: null, skipVideoFolder: false, ownerChannelId: null, ownerChannelMap: null } + expect.objectContaining({ + subfolderOverride: null, + subfolderFallback: null, + ratingOverride: undefined, + ratingFallback: null, + skipVideoFolder: false, + ownerChannelId: null, + ownerChannelMap: null, + cookiesEnabled: true, + anonymousRetry: false + }) ); }); @@ -1384,7 +1476,61 @@ describe('DownloadModule', () => { await downloadModule.doSpecificDownloads(request); - expect(YtdlpCommandBuilderMock.getBaseCommandArgsForManualDownload).toHaveBeenCalledWith('480', false, null, false); + expect(YtdlpCommandBuilderMock.getBaseCommandArgsForManualDownload).toHaveBeenCalledWith( + '480', + false, + null, + false, + { cookiesEnabled: true } + ); + }); + + it('disables cookies at command-build and execution time for anonymous auto-retry jobs', async () => { + jobModuleMock.getJob.mockReturnValue({ status: 'In Progress' }); + + await downloadModule.doSpecificDownloads({ + body: { + urls: ['https://youtube.com/watch?v=cookieRetry'], + anonymousRetry: true, + }, + }); + + expect( + YtdlpCommandBuilderMock.getBaseCommandArgsForManualDownload + ).toHaveBeenLastCalledWith( + '1080', + false, + null, + false, + { cookiesEnabled: false } + ); + + let directives = mockDownloadExecutor.doDownload.mock.calls[0][7]; + expect(directives.cookiesEnabled).toBe(false); + expect(directives.anonymousRetry).toBe(true); + + YtdlpCommandBuilderMock.getBaseCommandArgsForManualDownload.mockClear(); + mockDownloadExecutor.doDownload.mockClear(); + + await downloadModule.doSpecificDownloads({ + body: { + urls: ['https://youtube.com/watch?v=normalRetry'], + }, + }); + + expect( + YtdlpCommandBuilderMock.getBaseCommandArgsForManualDownload + ).toHaveBeenLastCalledWith( + '1080', + false, + null, + false, + { cookiesEnabled: true } + ); + + directives = mockDownloadExecutor.doDownload.mock.calls[0][7]; + expect(directives.cookiesEnabled).toBe(true); + expect(directives.anonymousRetry).toBe(false); }); it('should respect channel-level quality override when present', async () => { @@ -1404,7 +1550,7 @@ describe('DownloadModule', () => { where: { channel_id: 'UC123456' }, attributes: ['video_quality', 'audio_format', 'skip_video_folder'] }); - expect(YtdlpCommandBuilderMock.getBaseCommandArgsForManualDownload).toHaveBeenCalledWith('720', false, null, false); + expect(YtdlpCommandBuilderMock.getBaseCommandArgsForManualDownload).toHaveBeenCalledWith('720', false, null, false, { cookiesEnabled: true }); }); it('should respect channel-level audio_format when no override provided', async () => { @@ -1420,7 +1566,7 @@ describe('DownloadModule', () => { await downloadModule.doSpecificDownloads(request); - expect(YtdlpCommandBuilderMock.getBaseCommandArgsForManualDownload).toHaveBeenCalledWith('720', false, 'mp3_only', false); + expect(YtdlpCommandBuilderMock.getBaseCommandArgsForManualDownload).toHaveBeenCalledWith('720', false, 'mp3_only', false, { cookiesEnabled: true }); }); it('should prioritize override audioFormat over channel audio_format', async () => { @@ -1439,7 +1585,7 @@ describe('DownloadModule', () => { await downloadModule.doSpecificDownloads(request); - expect(YtdlpCommandBuilderMock.getBaseCommandArgsForManualDownload).toHaveBeenCalledWith('720', false, 'video_mp3', false); + expect(YtdlpCommandBuilderMock.getBaseCommandArgsForManualDownload).toHaveBeenCalledWith('720', false, 'video_mp3', false, { cookiesEnabled: true }); }); it('should allow null audioFormat override to bypass channel mp3_only setting', async () => { @@ -1458,7 +1604,7 @@ describe('DownloadModule', () => { await downloadModule.doSpecificDownloads(request); - expect(YtdlpCommandBuilderMock.getBaseCommandArgsForManualDownload).toHaveBeenCalledWith('720', false, null, false); + expect(YtdlpCommandBuilderMock.getBaseCommandArgsForManualDownload).toHaveBeenCalledWith('720', false, null, false, { cookiesEnabled: true }); }); it('should handle allowRedownload override setting', async () => { @@ -1475,7 +1621,7 @@ describe('DownloadModule', () => { await downloadModule.doSpecificDownloads(request); - expect(YtdlpCommandBuilderMock.getBaseCommandArgsForManualDownload).toHaveBeenCalledWith('720', true, null, false); + expect(YtdlpCommandBuilderMock.getBaseCommandArgsForManualDownload).toHaveBeenCalledWith('720', true, null, false, { cookiesEnabled: true }); expect(mockDownloadExecutor.doDownload).toHaveBeenCalledWith( expect.arrayContaining([ '--format', 'best[height<=720]', @@ -1489,7 +1635,17 @@ describe('DownloadModule', () => { ['https://youtube.com/watch?v=test1', 'https://youtube.com/watch?v=test2'], true, false, - { subfolderOverride: null, subfolderFallback: null, ratingOverride: undefined, ratingFallback: null, skipVideoFolder: false, ownerChannelId: null, ownerChannelMap: null } + expect.objectContaining({ + subfolderOverride: null, + subfolderFallback: null, + ratingOverride: undefined, + ratingFallback: null, + skipVideoFolder: false, + ownerChannelId: null, + ownerChannelMap: null, + cookiesEnabled: true, + anonymousRetry: false + }) ); // Verify that --download-archive is NOT in the arguments when allowRedownload is true const callArgs = mockDownloadExecutor.doDownload.mock.calls[0][0]; @@ -1510,7 +1666,7 @@ describe('DownloadModule', () => { await downloadModule.doSpecificDownloads(request); - expect(YtdlpCommandBuilderMock.getBaseCommandArgsForManualDownload).toHaveBeenCalledWith('480', false, null, false); + expect(YtdlpCommandBuilderMock.getBaseCommandArgsForManualDownload).toHaveBeenCalledWith('480', false, null, false, { cookiesEnabled: true }); expect(mockDownloadExecutor.doDownload).toHaveBeenCalledWith( expect.arrayContaining([ '--format', 'best[height<=480]', @@ -1524,7 +1680,17 @@ describe('DownloadModule', () => { ['https://youtube.com/watch?v=test'], false, false, - { subfolderOverride: null, subfolderFallback: null, ratingOverride: undefined, ratingFallback: null, skipVideoFolder: false, ownerChannelId: null, ownerChannelMap: null } + expect.objectContaining({ + subfolderOverride: null, + subfolderFallback: null, + ratingOverride: undefined, + ratingFallback: null, + skipVideoFolder: false, + ownerChannelId: null, + ownerChannelMap: null, + cookiesEnabled: true, + anonymousRetry: false + }) ); }); @@ -1541,7 +1707,7 @@ describe('DownloadModule', () => { await downloadModule.doSpecificDownloads(request); - expect(YtdlpCommandBuilderMock.getBaseCommandArgsForManualDownload).toHaveBeenCalledWith('1080', false, null, false); + expect(YtdlpCommandBuilderMock.getBaseCommandArgsForManualDownload).toHaveBeenCalledWith('1080', false, null, false, { cookiesEnabled: true }); expect(mockDownloadExecutor.doDownload).toHaveBeenCalledWith( expect.arrayContaining([ '--download-archive', './config/complete.list', @@ -1553,7 +1719,17 @@ describe('DownloadModule', () => { ['https://youtube.com/watch?v=default'], false, false, - { subfolderOverride: null, subfolderFallback: null, ratingOverride: undefined, ratingFallback: null, skipVideoFolder: false, ownerChannelId: null, ownerChannelMap: null } + expect.objectContaining({ + subfolderOverride: null, + subfolderFallback: null, + ratingOverride: undefined, + ratingFallback: null, + skipVideoFolder: false, + ownerChannelId: null, + ownerChannelMap: null, + cookiesEnabled: true, + anonymousRetry: false + }) ); }); @@ -1581,7 +1757,17 @@ describe('DownloadModule', () => { ['https://youtube.com/watch?v=test'], false, false, - { subfolderOverride: 'Movies', subfolderFallback: null, ratingOverride: undefined, ratingFallback: null, skipVideoFolder: false, ownerChannelId: null, ownerChannelMap: null } + expect.objectContaining({ + subfolderOverride: 'Movies', + subfolderFallback: null, + ratingOverride: undefined, + ratingFallback: null, + skipVideoFolder: false, + ownerChannelId: null, + ownerChannelMap: null, + cookiesEnabled: true, + anonymousRetry: false + }) ); }); @@ -1607,7 +1793,17 @@ describe('DownloadModule', () => { ['https://youtube.com/watch?v=test'], false, false, - { subfolderOverride: null, subfolderFallback: null, ratingOverride: undefined, ratingFallback: null, skipVideoFolder: false, ownerChannelId: null, ownerChannelMap: null } + expect.objectContaining({ + subfolderOverride: null, + subfolderFallback: null, + ratingOverride: undefined, + ratingFallback: null, + skipVideoFolder: false, + ownerChannelId: null, + ownerChannelMap: null, + cookiesEnabled: true, + anonymousRetry: false + }) ); }); @@ -1632,7 +1828,17 @@ describe('DownloadModule', () => { ['https://youtube.com/watch?v=test'], false, false, - { subfolderOverride: '', subfolderFallback: null, ratingOverride: undefined, ratingFallback: null, skipVideoFolder: false, ownerChannelId: null, ownerChannelMap: null } + expect.objectContaining({ + subfolderOverride: '', + subfolderFallback: null, + ratingOverride: undefined, + ratingFallback: null, + skipVideoFolder: false, + ownerChannelId: null, + ownerChannelMap: null, + cookiesEnabled: true, + anonymousRetry: false + }) ); }); @@ -1649,7 +1855,7 @@ describe('DownloadModule', () => { await downloadModule.doSpecificDownloads(request); - expect(YtdlpCommandBuilderMock.getBaseCommandArgsForManualDownload).toHaveBeenCalledWith('1080', false, null, true); + expect(YtdlpCommandBuilderMock.getBaseCommandArgsForManualDownload).toHaveBeenCalledWith('1080', false, null, true, { cookiesEnabled: true }); expect(mockDownloadExecutor.doDownload).toHaveBeenCalledWith( expect.any(Array), mockJobId, @@ -1658,7 +1864,17 @@ describe('DownloadModule', () => { ['https://youtube.com/watch?v=test'], false, false, - { subfolderOverride: null, subfolderFallback: null, ratingOverride: undefined, ratingFallback: null, skipVideoFolder: true, ownerChannelId: null, ownerChannelMap: null } + expect.objectContaining({ + subfolderOverride: null, + subfolderFallback: null, + ratingOverride: undefined, + ratingFallback: null, + skipVideoFolder: true, + ownerChannelId: null, + ownerChannelMap: null, + cookiesEnabled: true, + anonymousRetry: false + }) ); }); @@ -1675,7 +1891,7 @@ describe('DownloadModule', () => { await downloadModule.doSpecificDownloads(request); - expect(YtdlpCommandBuilderMock.getBaseCommandArgsForManualDownload).toHaveBeenCalledWith('1080', false, null, true); + expect(YtdlpCommandBuilderMock.getBaseCommandArgsForManualDownload).toHaveBeenCalledWith('1080', false, null, true, { cookiesEnabled: true }); expect(mockDownloadExecutor.doDownload).toHaveBeenCalledWith( expect.any(Array), mockJobId, @@ -1684,7 +1900,17 @@ describe('DownloadModule', () => { ['https://youtube.com/watch?v=test'], false, false, - { subfolderOverride: null, subfolderFallback: null, ratingOverride: undefined, ratingFallback: null, skipVideoFolder: true, ownerChannelId: 'UC123456', ownerChannelMap: null } + expect.objectContaining({ + subfolderOverride: null, + subfolderFallback: null, + ratingOverride: undefined, + ratingFallback: null, + skipVideoFolder: true, + ownerChannelId: 'UC123456', + ownerChannelMap: null, + cookiesEnabled: true, + anonymousRetry: false + }) ); }); @@ -1704,7 +1930,7 @@ describe('DownloadModule', () => { await downloadModule.doSpecificDownloads(request); - expect(YtdlpCommandBuilderMock.getBaseCommandArgsForManualDownload).toHaveBeenCalledWith('1080', false, null, false); + expect(YtdlpCommandBuilderMock.getBaseCommandArgsForManualDownload).toHaveBeenCalledWith('1080', false, null, false, { cookiesEnabled: true }); expect(mockDownloadExecutor.doDownload).toHaveBeenCalledWith( expect.any(Array), mockJobId, @@ -1713,7 +1939,17 @@ describe('DownloadModule', () => { ['https://youtube.com/watch?v=test'], false, false, - { subfolderOverride: null, subfolderFallback: null, ratingOverride: undefined, ratingFallback: null, skipVideoFolder: false, ownerChannelId: 'UC123456', ownerChannelMap: null } + expect.objectContaining({ + subfolderOverride: null, + subfolderFallback: null, + ratingOverride: undefined, + ratingFallback: null, + skipVideoFolder: false, + ownerChannelId: 'UC123456', + ownerChannelMap: null, + cookiesEnabled: true, + anonymousRetry: false + }) ); }); @@ -1732,7 +1968,7 @@ describe('DownloadModule', () => { await downloadModule.doSpecificDownloads(request); - expect(YtdlpCommandBuilderMock.getBaseCommandArgsForManualDownload).toHaveBeenCalledWith('1080', false, null, true); + expect(YtdlpCommandBuilderMock.getBaseCommandArgsForManualDownload).toHaveBeenCalledWith('1080', false, null, true, { cookiesEnabled: true }); expect(mockDownloadExecutor.doDownload).toHaveBeenCalledWith( expect.any(Array), mockJobId, @@ -1741,7 +1977,17 @@ describe('DownloadModule', () => { ['https://youtube.com/watch?v=test'], false, false, - { subfolderOverride: null, subfolderFallback: null, ratingOverride: undefined, ratingFallback: null, skipVideoFolder: true, ownerChannelId: 'UC123456', ownerChannelMap: null } + expect.objectContaining({ + subfolderOverride: null, + subfolderFallback: null, + ratingOverride: undefined, + ratingFallback: null, + skipVideoFolder: true, + ownerChannelId: 'UC123456', + ownerChannelMap: null, + cookiesEnabled: true, + anonymousRetry: false + }) ); }); @@ -1760,7 +2006,7 @@ describe('DownloadModule', () => { await downloadModule.doSpecificDownloads(request); - expect(YtdlpCommandBuilderMock.getBaseCommandArgsForManualDownload).toHaveBeenCalledWith('1080', false, null, false); + expect(YtdlpCommandBuilderMock.getBaseCommandArgsForManualDownload).toHaveBeenCalledWith('1080', false, null, false, { cookiesEnabled: true }); expect(mockDownloadExecutor.doDownload).toHaveBeenCalledWith( expect.any(Array), mockJobId, @@ -1769,7 +2015,17 @@ describe('DownloadModule', () => { ['https://youtube.com/watch?v=test'], false, false, - { subfolderOverride: null, subfolderFallback: null, ratingOverride: undefined, ratingFallback: null, skipVideoFolder: false, ownerChannelId: 'UC123456', ownerChannelMap: null } + expect.objectContaining({ + subfolderOverride: null, + subfolderFallback: null, + ratingOverride: undefined, + ratingFallback: null, + skipVideoFolder: false, + ownerChannelId: 'UC123456', + ownerChannelMap: null, + cookiesEnabled: true, + anonymousRetry: false + }) ); }); @@ -1827,7 +2083,17 @@ describe('DownloadModule', () => { [], false, false, - { subfolderOverride: null, subfolderFallback: null, ratingOverride: undefined, ratingFallback: null, skipVideoFolder: false, ownerChannelId: null, ownerChannelMap: null } + expect.objectContaining({ + subfolderOverride: null, + subfolderFallback: null, + ratingOverride: undefined, + ratingFallback: null, + skipVideoFolder: false, + ownerChannelId: null, + ownerChannelMap: null, + cookiesEnabled: true, + anonymousRetry: false + }) ); }); @@ -1843,7 +2109,13 @@ describe('DownloadModule', () => { }); // Template always nested: 4th arg (skipVideoFolder) is false expect(YtdlpCommandBuilderMock.getBaseCommandArgsForManualDownload) - .toHaveBeenCalledWith(expect.anything(), false, null, false); + .toHaveBeenCalledWith( + expect.anything(), + false, + null, + false, + { cookiesEnabled: true } + ); // Executor options (8th positional arg) carry the per-video directives const options = mockDownloadExecutor.doDownload.mock.calls[0][7]; expect(options.structurePerVideo).toBe(true); diff --git a/server/modules/download/YtdlpOutputRouter.js b/server/modules/download/YtdlpOutputRouter.js index 841f1289..dac8dc01 100644 --- a/server/modules/download/YtdlpOutputRouter.js +++ b/server/modules/download/YtdlpOutputRouter.js @@ -17,7 +17,16 @@ const PROGRESS_THROTTLE_MS = 250; const PROGRESS_HEARTBEAT_MS = 25 * 1000; class YtdlpOutputRouter { - constructor({ jobId, config, monitor, errorTracker, timeoutController, cookiesEnabled = false, heartbeatIntervalMs = PROGRESS_HEARTBEAT_MS }) { + constructor({ + jobId, + config, + monitor, + errorTracker, + timeoutController, + cookiesEnabled = false, + anonymousRetry = false, + heartbeatIntervalMs = PROGRESS_HEARTBEAT_MS, + }) { this.jobId = jobId; this.config = config; this.monitor = monitor; @@ -26,6 +35,9 @@ class YtdlpOutputRouter { // Branches the mid-run 403/bot hints: with cookies enabled, "set cookies" // is exactly the wrong advice (stale cookies are the usual cause). this.cookiesEnabled = cookiesEnabled; + // Distinguishes an intentional no-cookies fallback from a normal run + // where the user has not configured cookies. + this.anonymousRetry = anonymousRetry; // Per-run detection state, read by the executor/finalizer after exit this.partialDestinations = new Set(); this.stderrBuffer = ''; @@ -247,9 +259,11 @@ class YtdlpOutputRouter { // Check for bot detection message (handle different quote types and patterns) if (line.includes('Sign in to confirm') && line.includes('not a bot')) { this.botDetected = true; - const botMessage = this.cookiesEnabled - ? 'Bot detection encountered even though cookies are configured - they are likely expired or rotated. Re-export fresh cookies from your browser and upload them again.' - : 'Bot detection encountered. Please set cookies in your Configuration or try different cookies to resolve this issue.'; + const botMessage = this.anonymousRetry + ? 'Bot detection encountered during the no-cookies fallback. The fallback also failed, so this video may be genuinely unavailable.' + : this.cookiesEnabled + ? 'Bot detection encountered even though cookies are configured - they are likely expired or rotated. Re-export fresh cookies from your browser and upload them again.' + : 'Bot detection encountered. Please set cookies in your Configuration or try different cookies to resolve this issue.'; MessageEmitter.emitMessage( 'broadcast', null, @@ -269,9 +283,11 @@ class YtdlpOutputRouter { return; } this.cookiesSuggestionEmitted = true; - const message = this.cookiesEnabled - ? 'HTTP 403 detected while using your uploaded cookies. If the download fails, try re-exporting fresh cookies from your browser, or disable cookies in Settings -> Cookies.' - : 'HTTP 403 detected: YouTube may be blocking requests. If download fails, try setting cookies in Configuration.'; + const message = this.anonymousRetry + ? 'HTTP 403 detected during the no-cookies fallback. If this retry fails, the video may be genuinely unavailable.' + : this.cookiesEnabled + ? 'HTTP 403 detected while using your uploaded cookies. If the download fails, try re-exporting fresh cookies from your browser, or disable cookies in Settings -> Cookies.' + : 'HTTP 403 detected: YouTube may be blocking requests. If download fails, try setting cookies in Configuration.'; // Don't set monitor.hasError here - let the final exit code determine success/failure // 403s on HLS fragments are often recoverable and don't indicate actual failure MessageEmitter.emitMessage( @@ -283,7 +299,11 @@ class YtdlpOutputRouter { text: message, progress: this.monitor.snapshot('warning'), warning: true, - errorCode: this.cookiesEnabled ? 'COOKIES_MAY_BE_STALE' : 'COOKIES_RECOMMENDED' + errorCode: this.anonymousRetry + ? 'NO_COOKIES_FALLBACK_403' + : this.cookiesEnabled + ? 'COOKIES_MAY_BE_STALE' + : 'COOKIES_RECOMMENDED' } ); } diff --git a/server/modules/download/__tests__/YtdlpOutputRouter.test.js b/server/modules/download/__tests__/YtdlpOutputRouter.test.js index 59c27282..fafd0fb7 100644 --- a/server/modules/download/__tests__/YtdlpOutputRouter.test.js +++ b/server/modules/download/__tests__/YtdlpOutputRouter.test.js @@ -407,6 +407,65 @@ describe('YtdlpOutputRouter', () => { }); }); + describe('context-aware hints during anonymous retry', () => { + let anonymousRouter; + + beforeEach(() => { + anonymousRouter = new YtdlpOutputRouter({ + jobId: 'job-123', + config: { enableStallDetection: false }, + monitor: makeMonitor(), + errorTracker: makeErrorTracker(), + timeoutController: makeTimeoutController(), + cookiesEnabled: false, + anonymousRetry: true + }); + }); + + afterEach(() => { + if (anonymousRouter.progressFlushTimer) { + clearTimeout(anonymousRouter.progressFlushTimer); + anonymousRouter.progressFlushTimer = null; + } + }); + + it('reports a recoverable 403 during the no-cookies fallback without recommending cookies', () => { + anonymousRouter.handleStderrChunk('HTTP Error 403: Forbidden\n'); + + const call = MessageEmitter.emitMessage.mock.calls.find( + (c) => c[4] && c[4].errorCode === 'NO_COOKIES_FALLBACK_403' + ); + + expect(call).toBeDefined(); + expect(call[4].text).toMatch(/no-cookies fallback/i); + expect(call[4].text).toMatch(/if this retry fails/i); + expect(call[4].text).toMatch(/genuinely unavailable/i); + expect(call[4].text).not.toMatch(/fallback also failed/i); + expect(call[4].text).not.toMatch(/set.*cookies|enable.*cookies|re-export/i); + expect( + MessageEmitter.emitMessage.mock.calls.some( + (c) => c[4] && ( + c[4].errorCode === 'COOKIES_RECOMMENDED' || + c[4].errorCode === 'COOKIES_MAY_BE_STALE' + ) + ) + ).toBe(false); + }); + + it('reports bot detection as a failed no-cookies fallback instead of recommending cookies', () => { + anonymousRouter.handleStderrChunk('Sign in to confirm you\'re not a bot\n'); + + const call = MessageEmitter.emitMessage.mock.calls.find( + (c) => c[4] && c[4].progress && c[4].progress.state === 'bot_detected' + ); + + expect(call).toBeDefined(); + expect(call[4].text).toMatch(/no-cookies fallback/i); + expect(call[4].text).toMatch(/genuinely unavailable/i); + expect(call[4].text).not.toMatch(/set.*cookies|enable.*cookies|re-export/i); + }); + }); + describe('isImportantMessage', () => { it('should identify download destination messages as important', () => { const line = '[download] Destination: /output/Channel - Title [abc123].mp4'; diff --git a/server/modules/download/__tests__/downloadExecutor.test.js b/server/modules/download/__tests__/downloadExecutor.test.js index 2b60c6f1..4ff0c33a 100644 --- a/server/modules/download/__tests__/downloadExecutor.test.js +++ b/server/modules/download/__tests__/downloadExecutor.test.js @@ -153,6 +153,7 @@ describe('DownloadExecutor', () => { configModule.getConfig.mockReturnValue({ enableStallDetection: false, }); + configModule.getCookiesPath.mockReturnValue(null); // Setup filesystem module mocks with sensible defaults filesystem.isMainVideoFile.mockReturnValue(true); @@ -593,7 +594,82 @@ describe('DownloadExecutor', () => { expect(enqueueAutoRetry).toHaveBeenCalledWith({ retryVideos: [{ youtubeId: 'abc123def45', - url: 'https://www.youtube.com/watch?v=abc123def45' + url: 'https://www.youtube.com/watch?v=abc123def45', + anonymousRetry: false + }], + autoRetryAttempt: 1, + runId: null, + sourceJobData: {} + }); + }); + + it('does not requeue cookie-specific Video unavailable when execution explicitly disabled cookies', async () => { + const enqueueAutoRetry = jest.fn().mockResolvedValue(); + const retryExecutor = new DownloadExecutor({ enqueueAutoRetry }); + + // Model a Youtarr installation that has cookies configured globally. + configModule.getCookiesPath.mockReturnValue('/cookies/file.txt'); + + setTimeout(() => { + mockProcess.stdout.emit( + 'data', + Buffer.from('[youtube] Extracting URL: https://www.youtube.com/watch?v=abc123def45\n') + ); + mockProcess.stderr.emit( + 'data', + Buffer.from('ERROR: [youtube] abc123def45: Video unavailable\n') + ); + endProcess(1, null); + }, 10); + + await retryExecutor.doDownload( + mockArgs, + mockJobId, + mockJobType, + 0, + null, + false, + false, + { cookiesEnabled: false } + ); + + expect(enqueueAutoRetry).not.toHaveBeenCalled(); + }); + + it('requeues cookie-specific Video unavailable anonymously when execution uses cookies', async () => { + const enqueueAutoRetry = jest.fn().mockResolvedValue(); + const retryExecutor = new DownloadExecutor({ enqueueAutoRetry }); + + configModule.getCookiesPath.mockReturnValue('/cookies/file.txt'); + + setTimeout(() => { + mockProcess.stdout.emit( + 'data', + Buffer.from('[youtube] Extracting URL: https://www.youtube.com/watch?v=abc123def45\n') + ); + mockProcess.stderr.emit( + 'data', + Buffer.from('ERROR: [youtube] abc123def45: Video unavailable\n') + ); + endProcess(1, null); + }, 10); + + await retryExecutor.doDownload( + mockArgs, + mockJobId, + mockJobType, + 0, + null, + false, + false, + { cookiesEnabled: true } + ); + + expect(enqueueAutoRetry).toHaveBeenCalledWith({ + retryVideos: [{ + youtubeId: 'abc123def45', + url: 'https://www.youtube.com/watch?v=abc123def45', + anonymousRetry: true }], autoRetryAttempt: 1, runId: null, diff --git a/server/modules/download/__tests__/downloadJobFinalizer.test.js b/server/modules/download/__tests__/downloadJobFinalizer.test.js index d08a0a0e..fcef8746 100644 --- a/server/modules/download/__tests__/downloadJobFinalizer.test.js +++ b/server/modules/download/__tests__/downloadJobFinalizer.test.js @@ -123,6 +123,7 @@ const makeContext = (overrides = {}) => ({ runId: null, tempChannelsFile: null, onTempChannelsFileCleaned: jest.fn(), + anonymousRetry: false, ...overrides }); @@ -186,6 +187,39 @@ describe('downloadJobFinalizer', () => { expect(finalCall[4].text).not.toMatch(/set cookies/i); }); + it('uses no-cookies fallback messaging when bot detection hits an anonymous retry', async () => { + configModule.getCookiesPath.mockReturnValue('/app/config/cookies.user.txt'); + + await finalizeDownloadJob(makeContext({ + code: 1, + anonymousRetry: true, + cookiesEnabled: false, + router: makeRouter({ botDetected: true }) + })); + + expect(jobModule.updateJob).toHaveBeenCalledWith(mockJobId, expect.objectContaining({ + status: 'Error', + error: 'NO_COOKIES_FALLBACK_FAILED', + output: expect.stringMatching(/no-cookies fallback/i), + notes: expect.stringMatching(/no-cookies fallback/i) + })); + + const [, fields] = jobModule.updateJob.mock.calls.find( + ([id, update]) => id === mockJobId && update.error === 'NO_COOKIES_FALLBACK_FAILED' + ); + expect(fields.output).toMatch(/genuinely unavailable/i); + expect(fields.notes).toMatch(/genuinely unavailable/i); + expect(fields.output).not.toMatch(/set.*cookies|enable.*cookies|re-export/i); + expect(fields.notes).not.toMatch(/set.*cookies|enable.*cookies|re-export/i); + + const finalCall = MessageEmitter.emitMessage.mock.calls.find( + (call) => call[4] && call[4].text && call[4].text.startsWith('Download failed') + ); + expect(finalCall).toBeDefined(); + expect(finalCall[4].text).toMatch(/no-cookies fallback/i); + expect(finalCall[4].text).not.toMatch(/set.*cookies|enable.*cookies|re-export/i); + }); + it('marks a subtitle-only exit 1 Complete with Warnings without calling the videos failed', async () => { downloadResultProcessor.resolveUrlsToProcess.mockReturnValue(['https://youtu.be/abc123def45']); downloadResultProcessor.partitionDownloadResults.mockReturnValue({ @@ -386,7 +420,7 @@ describe('downloadJobFinalizer', () => { expect(jobModule.startNextJob).toHaveBeenCalled(); }); - describe('auto-retry of transient 403 failures', () => { + describe('auto-retry of retryable download failures', () => { const make403Failure = (overrides = {}) => ({ youtubeId: 'vid403aaaa1', title: 'Failing Video', @@ -414,7 +448,8 @@ describe('downloadJobFinalizer', () => { expect(enqueueAutoRetry).toHaveBeenCalledWith({ retryVideos: [{ youtubeId: 'vid403aaaa1', - url: 'https://www.youtube.com/watch?v=vid403aaaa1' + url: 'https://www.youtube.com/watch?v=vid403aaaa1', + anonymousRetry: false }], autoRetryAttempt: 1, runId: 'run-9', @@ -463,6 +498,46 @@ describe('downloadJobFinalizer', () => { })); }); + it('requeues cookie-specific Video unavailable anonymously when cookies are enabled', async () => { + const failure = make403Failure({ + error: 'Video unavailable', + }); + primeFailure(failure); + configModule.getCookiesPath.mockReturnValue('/app/config/cookies.user.txt'); + const enqueueAutoRetry = jest.fn().mockResolvedValue(); + + await finalizeDownloadJob(makeContext({ + code: 1, + enqueueAutoRetry, + })); + + expect(enqueueAutoRetry).toHaveBeenCalledWith(expect.objectContaining({ + retryVideos: [{ + youtubeId: 'vid403aaaa1', + url: 'https://www.youtube.com/watch?v=vid403aaaa1', + anonymousRetry: true, + }], + })); + expect(failure.autoRetryQueued).toBe(true); + }); + + it('does not requeue cookie-specific Video unavailable after cookies were already disabled', async () => { + const failure = make403Failure({ + error: 'Video unavailable', + }); + primeFailure(failure); + const enqueueAutoRetry = jest.fn(); + + await finalizeDownloadJob(makeContext({ + code: 1, + enqueueAutoRetry, + cookiesEnabled: false, + })); + + expect(enqueueAutoRetry).not.toHaveBeenCalled(); + expect(failure.autoRetryQueued).toBeUndefined(); + }); + it('does not enqueue when bot detection fired', async () => { primeFailure(make403Failure()); const enqueueAutoRetry = jest.fn(); @@ -530,7 +605,7 @@ describe('downloadJobFinalizer', () => { expect(logger.error).toHaveBeenCalledWith( { err: expect.any(Error), jobId: mockJobId }, - 'Failed to enqueue auto-retry for transient 403 failures' + 'Failed to enqueue auto-retry for retryable download failures' ); expect(failure.autoRetryQueued).toBeUndefined(); const finalCall = MessageEmitter.emitMessage.mock.calls.find( diff --git a/server/modules/download/__tests__/downloadStatusResolver.test.js b/server/modules/download/__tests__/downloadStatusResolver.test.js index 852b6c5d..62ccb284 100644 --- a/server/modules/download/__tests__/downloadStatusResolver.test.js +++ b/server/modules/download/__tests__/downloadStatusResolver.test.js @@ -141,6 +141,22 @@ describe('downloadStatusResolver', () => { expect(result.notes).toBe('YouTube denied access (HTTP 403). Configure cookies in Settings -> Cookies to resolve this issue.'); }); + it('uses no-cookies fallback advice for HTTP 403 during an anonymous retry', () => { + const result = describeNonZeroExit({ + ...baseInput, + videoCount: 2, + httpForbiddenDetected: true, + cookiesEnabled: false, + anonymousRetry: true + }); + + expect(result.errorCode).toBe('NO_COOKIES_FALLBACK_FAILED'); + expect(result.output).toBe('2 videos. Error: YouTube returned HTTP 403 (Forbidden)'); + expect(result.notes).toMatch(/no-cookies fallback/i); + expect(result.notes).toMatch(/genuinely unavailable/i); + expect(result.notes).not.toMatch(/configure.*cookies|set.*cookies|enable.*cookies|re-export/i); + }); + it('recommends refreshing or disabling cookies when HTTP 403 was detected with cookies enabled', () => { const result = describeNonZeroExit({ ...baseInput, @@ -503,6 +519,22 @@ describe('downloadStatusResolver', () => { expect(result.finalText).toBe('Download failed: Bot detection encountered. Please set cookies in your Configuration or try different cookies to resolve this issue.'); }); + it('uses no-cookies fallback messaging for bot detection during an anonymous retry', () => { + const result = resolveFinalPresentation({ + ...baseInput, + code: 1, + botDetected: true, + cookiesEnabled: false, + anonymousRetry: true + }); + + expect(result.finalState).toBe('failed'); + expect(result.finalErrorCode).toBe('NO_COOKIES_FALLBACK_FAILED'); + expect(result.finalText).toMatch(/no-cookies fallback/i); + expect(result.finalText).toMatch(/genuinely unavailable/i); + expect(result.finalText).not.toMatch(/set.*cookies|enable.*cookies|re-export/i); + }); + it('uses stale-cookie messaging for bot detection when cookies are enabled', () => { const result = resolveFinalPresentation({ ...baseInput, diff --git a/server/modules/download/__tests__/failureAdvisor.test.js b/server/modules/download/__tests__/failureAdvisor.test.js index 5bac5adf..587eb8f3 100644 --- a/server/modules/download/__tests__/failureAdvisor.test.js +++ b/server/modules/download/__tests__/failureAdvisor.test.js @@ -13,6 +13,7 @@ const failedVideo = (overrides = {}) => ({ const context = (overrides = {}) => ({ cookiesEnabled: false, + anonymousRetry: false, httpForbiddenDetected: false, botDetected: false, ...overrides, @@ -40,6 +41,20 @@ describe('failureAdvisor', () => { expect(diagnoses[0].message).toMatch(/uploading YouTube cookies/i); }); + it('diagnoses a 403 during the no-cookies fallback without recommending cookies', () => { + const videos = [failedVideo()]; + const diagnoses = adviseFailures(videos, context({ + cookiesEnabled: false, + anonymousRetry: true, + })); + + expect(videos[0].diagnosisKey).toBe('http-403-anonymous-retry'); + expect(diagnoses[0].key).toBe('http-403-anonymous-retry'); + expect(diagnoses[0].message).toMatch(/no-cookies fallback/i); + expect(diagnoses[0].message).toMatch(/genuinely unavailable/i); + expect(diagnoses[0].message).not.toMatch(/upload.*cookies|set.*cookies|enable.*cookies|re-export/i); + }); + it('matches fragment-shaped failures only when the run-level 403 flag is set', () => { const fragmentVideo = () => failedVideo({ error: 'fragment 3 not found, unable to continue' }); @@ -71,6 +86,20 @@ describe('failureAdvisor', () => { expect(diagnoses[0].message).toMatch(/upload youtube cookies/i); }); + it('diagnoses a bot check during the no-cookies fallback without recommending cookies', () => { + const videos = [failedVideo({ error: 'Sign in to confirm you\'re not a bot' })]; + const diagnoses = adviseFailures(videos, context({ + cookiesEnabled: false, + anonymousRetry: true, + })); + + expect(videos[0].diagnosisKey).toBe('bot-check-anonymous-retry'); + expect(diagnoses[0].key).toBe('bot-check-anonymous-retry'); + expect(diagnoses[0].message).toMatch(/no-cookies fallback/i); + expect(diagnoses[0].message).toMatch(/genuinely unavailable/i); + expect(diagnoses[0].message).not.toMatch(/upload.*cookies|set.*cookies|enable.*cookies|re-export/i); + }); + it('applies bot-check advice to download failures when bot detection fired run-wide', () => { const videos = [failedVideo({ error: 'Unable to extract video data' })]; adviseFailures(videos, context({ botDetected: true })); diff --git a/server/modules/download/__tests__/jobTypes.test.js b/server/modules/download/__tests__/jobTypes.test.js index c51cb11e..4a90481c 100644 --- a/server/modules/download/__tests__/jobTypes.test.js +++ b/server/modules/download/__tests__/jobTypes.test.js @@ -99,8 +99,15 @@ describe('jobTypes', () => { describe('auto-retry jobs', () => { it('labels jobs with the video count and pluralizes', () => { - expect(autoRetryJobLabel(1)).toBe(`${AUTO_RETRY_LABEL_PREFIX}1 video (HTTP 403)`); - expect(autoRetryJobLabel(2)).toBe(`${AUTO_RETRY_LABEL_PREFIX}2 videos (HTTP 403)`); + expect(autoRetryJobLabel(1)).toBe(`${AUTO_RETRY_LABEL_PREFIX}1 video`); + expect(autoRetryJobLabel(2)).toBe(`${AUTO_RETRY_LABEL_PREFIX}2 videos`); + }); + + it('marks anonymous auto-retry jobs as no-cookies', () => { + expect(autoRetryJobLabel(1, { anonymous: true })) + .toBe(`${AUTO_RETRY_LABEL_PREFIX}1 video (no cookies)`); + expect(autoRetryJobLabel(2, { anonymous: true })) + .toBe(`${AUTO_RETRY_LABEL_PREFIX}2 videos (no cookies)`); }); it('treats auto-retry jobs as specific URL-list download jobs', () => { diff --git a/server/modules/download/__tests__/transient403RetryPlanner.test.js b/server/modules/download/__tests__/transient403RetryPlanner.test.js index 177ea728..4434626c 100644 --- a/server/modules/download/__tests__/transient403RetryPlanner.test.js +++ b/server/modules/download/__tests__/transient403RetryPlanner.test.js @@ -3,6 +3,7 @@ const { planAutoRetry, isTransient403Failure, + isCookieVideoUnavailableFailure, resolveRetryCount, MAX_AUTO_RETRY_COUNT, DEFAULT_AUTO_RETRY_COUNT, @@ -52,6 +53,56 @@ describe('transient403RetryPlanner', () => { }); }); + describe('isCookieVideoUnavailableFailure', () => { + it('matches Video unavailable only when cookies are enabled', () => { + const unavailable = failedVideo({ error: 'Video unavailable' }); + + expect(isCookieVideoUnavailableFailure( + unavailable, + { cookiesEnabled: true } + )).toBe(true); + + expect(isCookieVideoUnavailableFailure( + unavailable, + { cookiesEnabled: false } + )).toBe(false); + }); + + it('matches the yt-dlp prefixed Video unavailable format', () => { + const unavailable = failedVideo({ + error: '[youtube] abc123def45: Video unavailable', + }); + + expect(isCookieVideoUnavailableFailure( + unavailable, + { cookiesEnabled: true } + )).toBe(true); + }); + + it('does not match Video unavailable with a genuine sub-reason', () => { + const unavailable = failedVideo({ + error: 'Video unavailable. This video is private', + }); + + expect(isCookieVideoUnavailableFailure( + unavailable, + { cookiesEnabled: true } + )).toBe(false); + }); + + it('does not classify unrelated failures as cookie-specific', () => { + expect(isCookieVideoUnavailableFailure( + failedVideo({ error: 'Postprocessing failed' }), + { cookiesEnabled: true } + )).toBe(false); + + expect(isCookieVideoUnavailableFailure( + undefined, + { cookiesEnabled: true } + )).toBe(false); + }); + }); + describe('resolveRetryCount', () => { it('falls back to the default when the config value is missing or invalid', () => { expect(resolveRetryCount(undefined)).toBe(DEFAULT_AUTO_RETRY_COUNT); @@ -75,6 +126,7 @@ describe('transient403RetryPlanner', () => { retryVideos: [{ youtubeId: 'abc123def45', url: 'https://www.youtube.com/watch?v=abc123def45', + anonymousRetry: false, }], nextAttempt: 1, }); @@ -108,6 +160,75 @@ describe('transient403RetryPlanner', () => { expect(plan).toBeNull(); }); + it('marks cookie-specific Video unavailable failures for anonymous retry', () => { + const plan = planAutoRetry({ + failedVideosList: [failedVideo({ error: 'Video unavailable' })], + cookiesEnabled: true, + }); + + expect(plan).toEqual({ + retryVideos: [{ + youtubeId: 'abc123def45', + url: 'https://www.youtube.com/watch?v=abc123def45', + anonymousRetry: true, + }], + nextAttempt: 1, + }); + }); + + it('does not retry Video unavailable anonymously when cookies are not enabled', () => { + const plan = planAutoRetry({ + failedVideosList: [failedVideo({ error: 'Video unavailable' })], + cookiesEnabled: false, + }); + + expect(plan).toBeNull(); + }); + + it('keeps ordinary 403 retries authenticated when cookies are enabled', () => { + const plan = planAutoRetry({ + failedVideosList: [failedVideo()], + cookiesEnabled: true, + }); + + expect(plan.retryVideos[0].anonymousRetry).toBe(false); + }); + + it('keeps subsequent 403 retries anonymous when the source job was anonymous', () => { + const plan = planAutoRetry({ + failedVideosList: [failedVideo()], + sourceJobData: { + autoRetryAttempt: 1, + anonymousRetry: true, + }, + maxAttempts: 3, + cookiesEnabled: false, + }); + + expect(plan).toEqual({ + retryVideos: [{ + youtubeId: 'abc123def45', + url: 'https://www.youtube.com/watch?v=abc123def45', + anonymousRetry: true, + }], + nextAttempt: 2, + }); + }); + + it('does not retry bare Video unavailable again when the source job was anonymous', () => { + const plan = planAutoRetry({ + failedVideosList: [failedVideo({ error: 'Video unavailable' })], + sourceJobData: { + autoRetryAttempt: 1, + anonymousRetry: true, + }, + maxAttempts: 3, + cookiesEnabled: false, + }); + + expect(plan).toBeNull(); + }); + it('returns null when bot detection fired', () => { expect(planAutoRetry({ failedVideosList: [failedVideo()], botDetected: true })).toBeNull(); }); diff --git a/server/modules/download/__tests__/ytdlpCommandBuilder.test.js b/server/modules/download/__tests__/ytdlpCommandBuilder.test.js index 34896bb3..3161ac5d 100644 --- a/server/modules/download/__tests__/ytdlpCommandBuilder.test.js +++ b/server/modules/download/__tests__/ytdlpCommandBuilder.test.js @@ -736,6 +736,21 @@ describe('YtdlpCommandBuilder', () => { expect(result[cookiesIndex + 1]).toBe('/path/to/cookies.txt'); }); + it('should omit cookies when cookiesEnabled is false', () => { + configModule.getCookiesPath.mockReturnValue('/cookies/file.txt'); + + const result = YtdlpCommandBuilder.getBaseCommandArgsForManualDownload( + undefined, + false, + null, + false, + { cookiesEnabled: false } + ); + + expect(result).not.toContain('--cookies'); + expect(result).not.toContain('/cookies/file.txt'); + }); + it('should include sponsorblock args when configured', () => { mockConfig.sponsorblockEnabled = true; mockConfig.sponsorblockCategories = { sponsor: true }; diff --git a/server/modules/download/downloadExecutor.js b/server/modules/download/downloadExecutor.js index d7d5ed9a..824c90e6 100644 --- a/server/modules/download/downloadExecutor.js +++ b/server/modules/download/downloadExecutor.js @@ -24,7 +24,7 @@ const STDIO_DRAIN_TIMEOUT_MS = 5 * 1000; class DownloadExecutor { // enqueueAutoRetry is injected by downloadModule so the finalizer can queue - // transient-403 retry jobs without a require cycle back into downloadModule. + // auto-retry jobs without a require cycle back into downloadModule. constructor({ enqueueAutoRetry = null } = {}) { this.enqueueAutoRetry = enqueueAutoRetry; this.tempChannelsFile = null; @@ -252,13 +252,19 @@ class DownloadExecutor { }, }); + const cookiesEnabled = + postProcessDirectives.cookiesEnabled ?? + Boolean(configModule.getCookiesPath()); + const anonymousRetry = postProcessDirectives.anonymousRetry === true; + const router = new YtdlpOutputRouter({ jobId, config, monitor, errorTracker, timeoutController, - cookiesEnabled: Boolean(configModule.getCookiesPath()), + cookiesEnabled, + anonymousRetry, heartbeatIntervalMs: this.progressHeartbeatMs, }); @@ -337,6 +343,8 @@ class DownloadExecutor { tempChannelsFile: this.tempChannelsFile, onTempChannelsFileCleaned: () => { this.tempChannelsFile = null; }, enqueueAutoRetry: this.enqueueAutoRetry, + cookiesEnabled, + anonymousRetry, }); resolve(); } catch (err) { diff --git a/server/modules/download/downloadJobFinalizer.js b/server/modules/download/downloadJobFinalizer.js index 2629d519..9d107763 100644 --- a/server/modules/download/downloadJobFinalizer.js +++ b/server/modules/download/downloadJobFinalizer.js @@ -146,6 +146,8 @@ async function finalizeDownloadJob({ tempChannelsFile, onTempChannelsFileCleaned, enqueueAutoRetry = null, + cookiesEnabled = Boolean(configModule.getCookiesPath()), + anonymousRetry = false, }) { // True once the job's terminal status has been persisted; the catch // below must not overwrite it with 'Error' for failures that happen @@ -199,7 +201,7 @@ async function finalizeDownloadJob({ videoActivity.finish(jobId, video.youtubeId); } - // Auto-retry transient 403 failures. Enqueue while this job is still + // Auto-retry eligible download failures. Enqueue while this job is still // In Progress so the retry queues as Pending behind it, and read job data // now, before the terminal update replaces it. Handed-off failures are // tagged so run summaries and notifications report the post-retry outcome @@ -215,6 +217,7 @@ async function finalizeDownloadJob({ wasTerminated, sourceJobData, maxAttempts: configModule.getConfig().downloadAutoRetryCount, + cookiesEnabled, }); if (retryPlan) { try { @@ -233,10 +236,10 @@ async function finalizeDownloadJob({ autoRetryQueuedCount = retryPlan.retryVideos.length; logger.info( { jobId, count: autoRetryQueuedCount, attempt: retryPlan.nextAttempt }, - 'Queued auto-retry job for transient 403 failures' + 'Queued auto-retry job for retryable download failures' ); } catch (err) { - logger.error({ err, jobId }, 'Failed to enqueue auto-retry for transient 403 failures'); + logger.error({ err, jobId }, 'Failed to enqueue auto-retry for retryable download failures'); } } } @@ -246,9 +249,8 @@ async function finalizeDownloadJob({ const reportableFailedVideos = failedVideosList.filter((video) => !video.autoRetryQueued); // Cookie state drives both the failure diagnoses and the cookie-related - // terminal messages below: with cookies enabled, "set cookies" advice is - // exactly backwards (stale cookies are the usual cause). - const cookiesEnabled = Boolean(configModule.getCookiesPath()); + // terminal messages below. Use the effective state supplied by the + // executor so anonymous retries remain anonymous throughout finalization. // Reportable failures are final by construction (the auto-retry already // failed or was never possible), so diagnose them. A diagnosis failure @@ -257,6 +259,7 @@ async function finalizeDownloadJob({ try { diagnoses = failureAdvisor.adviseFailures(reportableFailedVideos, { cookiesEnabled, + anonymousRetry, httpForbiddenDetected, botDetected, }); @@ -296,9 +299,11 @@ async function finalizeDownloadJob({ if (botDetected) { status = 'Error'; - output = cookiesEnabled - ? 'Bot detection encountered even though cookies are configured - they are likely expired or rotated.' - : 'Bot detection encountered. Please set cookies in your Configuration.'; + output = anonymousRetry + ? 'Bot detection encountered during the no-cookies fallback. The fallback also failed, so this video may be genuinely unavailable.' + : cookiesEnabled + ? 'Bot detection encountered even though cookies are configured - they are likely expired or rotated.' + : 'Bot detection encountered. Please set cookies in your Configuration.'; await persistCompletedVideosBeforeTerminalUpdate(jobId, videoData, failedVideosList); await jobModule.updateJob(jobId, { @@ -306,12 +311,14 @@ async function finalizeDownloadJob({ endDate: Date.now(), output: output, data: dataPayload, - notes: cookiesEnabled - ? 'YouTube requires authentication and your uploaded cookies appear stale. Re-export fresh cookies from your browser and upload them in Settings -> Cookies.' - : 'YouTube requires authentication. Enable cookies in Configuration to resolve this issue.', - error: 'COOKIES_REQUIRED' + notes: anonymousRetry + ? 'The no-cookies fallback was also rejected by YouTube. This video may be genuinely unavailable.' + : cookiesEnabled + ? 'YouTube requires authentication and your uploaded cookies appear stale. Re-export fresh cookies from your browser and upload them in Settings -> Cookies.' + : 'YouTube requires authentication. Enable cookies in Configuration to resolve this issue.', + error: anonymousRetry ? 'NO_COOKIES_FALLBACK_FAILED' : 'COOKIES_REQUIRED' }); - jobErrorCode = 'COOKIES_REQUIRED'; + jobErrorCode = anonymousRetry ? 'NO_COOKIES_FALLBACK_FAILED' : 'COOKIES_REQUIRED'; } else if (timeoutController.shutdownInProgress || timeoutController.shutdownReason || wasManuallyTerminated) { // Handle timeout/graceful shutdown or manual termination await downloadCleanup.cleanupInProgressVideos(jobId); @@ -350,6 +357,7 @@ async function finalizeDownloadJob({ terminatedChannelCount: errorTracker.terminatedChannelIds.size, httpForbiddenDetected, cookiesEnabled, + anonymousRetry, flags, failureDetails, subtitleFailureCount: errorTracker.subtitleFailureCount || 0 @@ -475,6 +483,7 @@ async function finalizeDownloadJob({ unexpectedErrorCount: errorTracker.unexpectedErrorCount, httpForbiddenDetected, cookiesEnabled, + anonymousRetry, autoRetryQueuedCount, subtitleFailureCount: errorTracker.subtitleFailureCount || 0 }); diff --git a/server/modules/download/downloadStatusResolver.js b/server/modules/download/downloadStatusResolver.js index 0f4da09f..3d83f480 100644 --- a/server/modules/download/downloadStatusResolver.js +++ b/server/modules/download/downloadStatusResolver.js @@ -56,6 +56,7 @@ function describeNonZeroExit({ terminatedChannelCount, httpForbiddenDetected, cookiesEnabled = false, + anonymousRetry = false, flags, failureDetails, subtitleFailureCount = 0 @@ -81,10 +82,14 @@ function describeNonZeroExit({ // already enabled, "configure cookies" is the wrong advice: stale or // rotated cookies are the usual cause. output = `${videoCount} videos. Error: YouTube returned HTTP 403 (Forbidden)`; - notes = cookiesEnabled - ? 'YouTube denied access (HTTP 403) while using your uploaded cookies. Re-export fresh cookies from your browser, or disable cookies in Settings -> Cookies.' - : 'YouTube denied access (HTTP 403). Configure cookies in Settings -> Cookies to resolve this issue.'; - errorCode = 'COOKIES_RECOMMENDED'; + notes = anonymousRetry + ? 'The no-cookies fallback was also blocked by YouTube. This video may be genuinely unavailable.' + : cookiesEnabled + ? 'YouTube denied access (HTTP 403) while using your uploaded cookies. Re-export fresh cookies from your browser, or disable cookies in Settings -> Cookies.' + : 'YouTube denied access (HTTP 403). Configure cookies in Settings -> Cookies to resolve this issue.'; + errorCode = anonymousRetry + ? 'NO_COOKIES_FALLBACK_FAILED' + : 'COOKIES_RECOMMENDED'; } else { // Failed with other error output = `${videoCount} videos. Error: Command exited with code ${code}`; @@ -133,6 +138,7 @@ function resolveFinalPresentation({ unexpectedErrorCount, httpForbiddenDetected, cookiesEnabled = false, + anonymousRetry = false, // Failures already handed off to a queued auto-retry job. Affects only the // presented text; state derivation still counts them as failures. autoRetryQueuedCount = 0, @@ -192,12 +198,16 @@ function resolveFinalPresentation({ finalText = `Download terminated: ${terminationReason}. ${completedCount} video${completedCount !== 1 ? 's' : ''} completed successfully.`; } else if (botDetected) { finalState = 'failed'; - finalErrorCode = 'COOKIES_REQUIRED'; - // With cookies already enabled, "set cookies" is the wrong advice: stale - // or rotated cookies are the usual cause. - finalText = cookiesEnabled - ? 'Download failed: Bot detection encountered even though cookies are configured - they are likely expired or rotated. Re-export fresh cookies from your browser and upload them in Settings -> Cookies.' - : 'Download failed: Bot detection encountered. Please set cookies in your Configuration or try different cookies to resolve this issue.'; + finalErrorCode = anonymousRetry + ? 'NO_COOKIES_FALLBACK_FAILED' + : 'COOKIES_REQUIRED'; + // Anonymous retries intentionally suppress configured cookies, so they + // need a third diagnostic state rather than ordinary "set cookies" advice. + finalText = anonymousRetry + ? 'Download failed: Bot detection encountered during the no-cookies fallback. The fallback also failed, so this video may be genuinely unavailable.' + : cookiesEnabled + ? 'Download failed: Bot detection encountered even though cookies are configured - they are likely expired or rotated. Re-export fresh cookies from your browser and upload them in Settings -> Cookies.' + : 'Download failed: Bot detection encountered. Please set cookies in your Configuration or try different cookies to resolve this issue.'; } else if (monitorHasError && finalState === 'complete' && !flags.hasOnlyHandledErrors) { // Don't let a recognized termination flip the state back to error // via DownloadProgressMonitor's broad hasError flag. diff --git a/server/modules/download/failureAdvisor.js b/server/modules/download/failureAdvisor.js index eea2dff9..a1fde21d 100644 --- a/server/modules/download/failureAdvisor.js +++ b/server/modules/download/failureAdvisor.js @@ -31,6 +31,12 @@ const ADVICE = { 'cookies from your browser (Settings -> Cookies) often ' + 'resolves it.', }, + 'http-403-anonymous-retry': { + title: 'No-cookies fallback also failed', + message: + 'The no-cookies fallback was also blocked by YouTube. This video may ' + + 'be genuinely unavailable.', + }, 'bot-check-cookies-enabled': { title: 'YouTube bot check despite cookies', message: @@ -45,6 +51,12 @@ const ADVICE = { 'from your browser in Settings -> Cookies to resolve ' + 'this.', }, + 'bot-check-anonymous-retry': { + title: 'No-cookies fallback also failed', + message: + 'YouTube rejected the no-cookies fallback with a bot check. This video ' + + 'may be genuinely unavailable.', + }, }; // Ordered registry; first match wins. Bot-check outranks http-403 because a @@ -58,13 +70,21 @@ const REGISTRY = [ return context.botDetected === true && DOWNLOAD_FAILURE_PATTERN.test(error); }, keyFor: (context) => - context.cookiesEnabled ? 'bot-check-cookies-enabled' : 'bot-check-cookies-disabled', + context.anonymousRetry + ? 'bot-check-anonymous-retry' + : context.cookiesEnabled + ? 'bot-check-cookies-enabled' + : 'bot-check-cookies-disabled', }, { matches: (video, context) => isTransient403Failure(video, { httpForbiddenDetected: context.httpForbiddenDetected === true }), keyFor: (context) => - context.cookiesEnabled ? 'http-403-cookies-enabled' : 'http-403-cookies-disabled', + context.anonymousRetry + ? 'http-403-anonymous-retry' + : context.cookiesEnabled + ? 'http-403-cookies-enabled' + : 'http-403-cookies-disabled', }, ]; diff --git a/server/modules/download/jobTypes.js b/server/modules/download/jobTypes.js index 9cb14fbc..92981201 100644 --- a/server/modules/download/jobTypes.js +++ b/server/modules/download/jobTypes.js @@ -24,7 +24,7 @@ const PLAYLIST_RETRY_LABEL_PREFIX = 'Playlist Retry: '; // already-running guard match that substring, and a download-all job is a // URL-list job, not a channel/tab sweep. const CHANNEL_DOWNLOAD_ALL_LABEL_PREFIX = 'Channel Download All: '; -// Auto-retry jobs enqueued after transient-403 failures. Always a URL-list +// Auto-retry jobs enqueued after retryable download failures. Always a URL-list // job. Like the download-all label, must NOT contain CHANNEL_DOWNLOAD_LABEL // even when retrying channel-sweep failures. const AUTO_RETRY_LABEL_PREFIX = 'Auto-retry: '; @@ -68,9 +68,10 @@ function channelDownloadAllJobLabel(channel) { return `${CHANNEL_DOWNLOAD_ALL_LABEL_PREFIX}${channel.title || channel.channel_id}`; } -// Build the activity-view job label for a transient-403 auto-retry job. -function autoRetryJobLabel(videoCount) { - return `${AUTO_RETRY_LABEL_PREFIX}${videoCount} video${videoCount !== 1 ? 's' : ''} (HTTP 403)`; +// Build the activity-view job label for an auto-retry job. +function autoRetryJobLabel(videoCount, { anonymous = false } = {}) { + const suffix = anonymous ? ' (no cookies)' : ''; + return `${AUTO_RETRY_LABEL_PREFIX}${videoCount} video${videoCount !== 1 ? 's' : ''}${suffix}`; } // True for a channel download-all job. These jobs are exempt from the diff --git a/server/modules/download/transient403RetryPlanner.js b/server/modules/download/transient403RetryPlanner.js index 3444b1d6..01661d42 100644 --- a/server/modules/download/transient403RetryPlanner.js +++ b/server/modules/download/transient403RetryPlanner.js @@ -12,6 +12,12 @@ const DEFAULT_AUTO_RETRY_COUNT = 1; const HTTP_403_PATTERN = /http error 403|403[:\s]+forbidden/i; +// Some videos can return "Video unavailable" only when authenticated +// cookies are used, while anonymous extraction succeeds through a different +// YouTube client. These are eligible for an anonymous retry. +const COOKIE_VIDEO_UNAVAILABLE_PATTERN = + /^\s*(?:\[youtube\]\s+[^:]+:\s*)?Video unavailable\s*$/i; + // Fragment-based 403 failures surface the 403 only in WARNING lines; the final // ERROR line just reports the data/fragment failure. When the run-level 403 // flag is set, treat these error shapes as 403-caused. @@ -54,6 +60,18 @@ function isTransient403Failure(failedVideo, { httpForbiddenDetected = false } = return false; } +function isCookieVideoUnavailableFailure( + failedVideo, + { cookiesEnabled = false } = {} +) { + if (!cookiesEnabled) return false; + + const error = String((failedVideo && failedVideo.error) || ''); + // The end anchor is deliberate: an appended sub-reason indicates a real + // unavailability rather than the cookie-specific failure handled here. + return COOKIE_VIDEO_UNAVAILABLE_PATTERN.test(error); +} + // Decides whether (and for which videos) to enqueue an auto-retry job. // botDetected means a retry can't help (cookies are required); sourceJobData // carries the attempt counter. Returns { retryVideos, nextAttempt } or null. @@ -64,22 +82,38 @@ function planAutoRetry({ wasTerminated = false, sourceJobData = {}, maxAttempts, + cookiesEnabled = false, } = {}) { if (botDetected || wasTerminated) return null; if (!Array.isArray(failedVideosList) || failedVideosList.length === 0) return null; const budget = resolveRetryCount(maxAttempts); const attempt = Number(readJobDataValue(sourceJobData, 'autoRetryAttempt')) || 0; + const sourceWasAnonymous = readJobDataValue(sourceJobData, 'anonymousRetry') === true; if (attempt >= budget) return null; const retryVideos = failedVideosList .filter((video) => video && video.youtubeId) - .filter((video) => isTransient403Failure(video, { httpForbiddenDetected })) - .map((video) => ({ + .map((video) => { + const cookieVideoUnavailable = isCookieVideoUnavailableFailure( + video, + { cookiesEnabled } + ); + const transient403 = isTransient403Failure(video, { httpForbiddenDetected }); + + return { + video, + retryable: cookieVideoUnavailable || transient403, + anonymousRetry: cookieVideoUnavailable || (sourceWasAnonymous && transient403), + }; + }) + .filter(({ retryable }) => retryable) + .map(({ video, anonymousRetry }) => ({ youtubeId: video.youtubeId, // Channel-sweep failures never enter the archive diff, so their url is // usually null; reconstruct it from the youtube id. url: video.url || `https://www.youtube.com/watch?v=${video.youtubeId}`, + anonymousRetry, })); if (retryVideos.length === 0) return null; @@ -90,6 +124,7 @@ function planAutoRetry({ module.exports = { planAutoRetry, isTransient403Failure, + isCookieVideoUnavailableFailure, resolveRetryCount, MAX_AUTO_RETRY_COUNT, DEFAULT_AUTO_RETRY_COUNT, diff --git a/server/modules/download/ytdlpCommandBuilder.js b/server/modules/download/ytdlpCommandBuilder.js index 229dd827..77bba795 100644 --- a/server/modules/download/ytdlpCommandBuilder.js +++ b/server/modules/download/ytdlpCommandBuilder.js @@ -206,7 +206,7 @@ class YtdlpCommandBuilder { * @returns {string[]} */ static buildCommonArgs(config, options = {}) { - const { skipSleepRequests = false } = options; + const { skipSleepRequests = false, cookiesEnabled = true } = options; const args = []; // IP family (replaces previously-hardcoded -4) @@ -239,7 +239,7 @@ class YtdlpCommandBuilder { // Cookies const cookiesPath = configModule.getCookiesPath(); - if (cookiesPath) { + if (cookiesEnabled && cookiesPath) { args.push('--cookies', cookiesPath); } @@ -294,9 +294,11 @@ class YtdlpCommandBuilder { * @param {Object} config * @returns {string[]} */ - static buildVideoExtractionCustomArgs(config) { + static buildVideoExtractionCustomArgs(config, options = {}) { const customArgs = this.buildCustomArgs(config); - if (!configModule.getCookiesPath()) { + const { cookiesEnabled = true } = options; + + if (!cookiesEnabled || !configModule.getCookiesPath()) { return customArgs; } return mergeCookiePlayerClients(customArgs); @@ -609,8 +611,15 @@ class YtdlpCommandBuilder { * @param {boolean} skipVideoFolder - If true, skip the video subfolder level (flat structure) * @returns {string[]} - Array of yt-dlp command arguments */ - static getBaseCommandArgsForManualDownload(resolution, allowRedownload = false, audioFormat = null, skipVideoFolder = false) { + static getBaseCommandArgsForManualDownload( + resolution, + allowRedownload = false, + audioFormat = null, + skipVideoFolder = false, + options = {} + ) { const config = configModule.getConfig(); + const { cookiesEnabled = true } = options; const res = resolution || config.preferredResolution || '1080'; const videoCodec = config.videoCodec || 'default'; @@ -619,7 +628,7 @@ class YtdlpCommandBuilder { // Start with common args (includes -4, proxy, sleep-requests, cookies) const args = [ - ...this.buildCommonArgs(config), + ...this.buildCommonArgs(config, { cookiesEnabled }), '--windows-filenames', // Sanitize filenames for Windows/Plex compatibility '--ffmpeg-location', configModule.ffmpegPath, '--socket-timeout', String(config.downloadSocketTimeoutSeconds || 30), @@ -677,7 +686,7 @@ class YtdlpCommandBuilder { // Custom user args MUST be appended last so yt-dlp's last-wins semantics // let users override managed defaults like --retries / --fragment-retries. // URL operands are appended later by downloadModule. - args.push(...this.buildVideoExtractionCustomArgs(config)); + args.push(...this.buildVideoExtractionCustomArgs(config, { cookiesEnabled })); return args; } diff --git a/server/modules/downloadModule.js b/server/modules/downloadModule.js index 95d196d8..5037711e 100644 --- a/server/modules/downloadModule.js +++ b/server/modules/downloadModule.js @@ -104,8 +104,8 @@ class DownloadModule { } /** - * Enqueue a follow-up URL-list job for videos that failed with a transient - * HTTP 403. Owning channels are resolved so channel-tier settings (quality, + * Enqueue a follow-up URL-list job for videos with retryable download + * failures. Owning channels are resolved so channel-tier settings (quality, * audio format, subfolder routing) apply on the retry; the source job's * overrideSettings carry dialog-picked options through for manual and * playlist downloads. @@ -118,6 +118,34 @@ class DownloadModule { async enqueueAutoRetryJob({ retryVideos, autoRetryAttempt, runId, sourceJobData = {} }) { if (!Array.isArray(retryVideos) || retryVideos.length === 0) return; + // Keep normal transient-403 retries authenticated, while videos that + // specifically failed with cookie-induced "Video unavailable" are retried + // anonymously. Never mix the two modes in one yt-dlp invocation. + const anonymousVideos = retryVideos.filter( + (video) => video && video.anonymousRetry === true + ); + const authenticatedVideos = retryVideos.filter( + (video) => !video || video.anonymousRetry !== true + ); + + if (anonymousVideos.length > 0 && authenticatedVideos.length > 0) { + await this.enqueueAutoRetryJob({ + retryVideos: authenticatedVideos, + autoRetryAttempt, + runId, + sourceJobData, + }); + + await this.enqueueAutoRetryJob({ + retryVideos: anonymousVideos, + autoRetryAttempt, + runId, + sourceJobData, + }); + + return; + } + const ownerChannelMap = { ...(this.getJobDataValue(sourceJobData, 'ownerChannelMap') || {}) }; const unmappedIds = retryVideos .map((video) => video.youtubeId) @@ -147,11 +175,16 @@ class DownloadModule { || (mappedChannelIds.size === 1 ? [...mappedChannelIds][0] : null); const effectiveQuality = this.getJobDataValue(sourceJobData, 'effectiveQuality') || null; + const anonymousRetry = retryVideos.every( + (video) => video && video.anonymousRetry === true + ); + const body = { urls: retryVideos.map((video) => video.url), overrideSettings: { ...this.getOverrideSettings(sourceJobData) }, - jobLabel: autoRetryJobLabel(retryVideos.length), + jobLabel: autoRetryJobLabel(retryVideos.length, { anonymous: anonymousRetry }), autoRetryAttempt, + anonymousRetry, }; if (Object.keys(ownerChannelMap).length > 0) body.ownerChannelMap = ownerChannelMap; if (channelId) body.channelId = channelId; @@ -161,7 +194,7 @@ class DownloadModule { logger.info( { videoCount: retryVideos.length, autoRetryAttempt, channelId, runId }, - 'Enqueueing auto-retry job for transient 403 failures' + 'Enqueueing auto-retry job for retryable download failures' ); await this.doSpecificDownloads({ body }); } @@ -712,7 +745,22 @@ class DownloadModule { // For manual downloads, we don't apply duration filters but still exclude members-only // Subfolder override is passed to post-processor via environment variable // Pass audioFormat for MP3 downloads - const args = YtdlpCommandBuilder.getBaseCommandArgsForManualDownload(resolution, allowRedownload, audioFormat, skipVideoFolder); + const anonymousRetry = Boolean(this.getJobDataValue(jobData, 'anonymousRetry')); + const cookiesEnabled = Boolean(configModule.getCookiesPath()) && !anonymousRetry; + const args = YtdlpCommandBuilder.getBaseCommandArgsForManualDownload( + resolution, + allowRedownload, + audioFormat, + skipVideoFolder, + { cookiesEnabled } + ); + + if (anonymousRetry) { + logger.info( + { urls }, + 'Retrying cookie-specific Video unavailable failure without cookies' + ); + } // Check if any URLs are for videos marked as ignored, and remove them from archive // This allows users to manually download videos they've marked to ignore for channel downloads @@ -785,6 +833,8 @@ class DownloadModule { // the resolution priority in videoDownloadPostProcessFiles.js. ownerChannelId: channelId || null, ownerChannelMap: this.getJobDataValue(jobData, 'ownerChannelMap') || null, + cookiesEnabled, + anonymousRetry, } )).catch(async err => { // Covers failures before the executor installs its own process