Fix podcast episode downloads: ffmpeg muxer bug and dead-URL retry storm
_extract_audio wrote transcoded audio to a .tmp temp path, so ffmpeg couldn't guess a muxer from the filename and aborted on every video episode whose source was still reachable. Pass -f mp3 explicitly instead of relying on the extension. Separately, a permanently 404'd episode was retried on every startup and every 6-hour poll forever, since nothing remembered past failures. Track failed downloads per show folder with a 7-day backoff before retrying. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
@@ -2,6 +2,8 @@
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from pathlib import Path
|
||||
|
||||
import httpx2
|
||||
@@ -491,9 +493,12 @@ async def test_download_episode_extracts_audio_via_ffmpeg(
|
||||
async def communicate(self) -> tuple[bytes, bytes]:
|
||||
return b"", b""
|
||||
|
||||
captured_args: list[object] = []
|
||||
|
||||
async def fake_exec(*args: object, **kwargs: object) -> _SucceedingProcess:
|
||||
# ffmpeg's destination path is the last positional argument; write to it here
|
||||
# to stand in for what the real binary would have produced.
|
||||
captured_args.extend(args)
|
||||
Path(str(args[-1])).write_bytes(b"extracted-audio")
|
||||
return _SucceedingProcess()
|
||||
|
||||
@@ -510,3 +515,118 @@ async def test_download_episode_extracts_audio_via_ffmpeg(
|
||||
|
||||
assert (folder / "20250101 - Video Only.mp3").read_bytes() == b"extracted-audio"
|
||||
assert list(folder.glob(".downloading-*.tmp")) == []
|
||||
# The destination path ends in ".tmp", not ".mp3" - ffmpeg can't guess the muxer
|
||||
# from that extension, so the format must be passed explicitly.
|
||||
assert captured_args[captured_args.index("-f") + 1] == "mp3"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_sync_show_does_not_retry_a_failed_episode_within_the_backoff(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
folder = tmp_path / "show"
|
||||
folder.mkdir()
|
||||
enclosure_requests: list[str] = []
|
||||
|
||||
def handler(request: httpx2.Request) -> httpx2.Response:
|
||||
url = str(request.url)
|
||||
if url == "http://example.com/feed.xml":
|
||||
return httpx2.Response(200, content=_FEED)
|
||||
enclosure_requests.append(url)
|
||||
return httpx2.Response(404)
|
||||
|
||||
async with httpx2.AsyncClient(transport=httpx2.MockTransport(handler)) as client:
|
||||
await sync_show(client, folder, "http://example.com/feed.xml")
|
||||
await sync_show(client, folder, "http://example.com/feed.xml")
|
||||
|
||||
# Both episodes 404 on the first pass and get remembered; the second pass must not
|
||||
# re-request either enclosure while the backoff is still in effect.
|
||||
assert len(enclosure_requests) == 2
|
||||
assert (folder / podcast_feeds._FAILED_DOWNLOADS_FILENAME).exists()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_sync_show_retries_a_failed_episode_once_the_backoff_expires(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
folder = tmp_path / "show"
|
||||
folder.mkdir()
|
||||
enclosure_requests: list[str] = []
|
||||
|
||||
def handler(request: httpx2.Request) -> httpx2.Response:
|
||||
url = str(request.url)
|
||||
if url == "http://example.com/feed.xml":
|
||||
return httpx2.Response(200, content=_FEED)
|
||||
enclosure_requests.append(url)
|
||||
return httpx2.Response(404)
|
||||
|
||||
async with httpx2.AsyncClient(transport=httpx2.MockTransport(handler)) as client:
|
||||
await sync_show(client, folder, "http://example.com/feed.xml")
|
||||
|
||||
# Backdate both failure records past the backoff window, simulating that a
|
||||
# week has gone by since the first attempt.
|
||||
record_path = folder / podcast_feeds._FAILED_DOWNLOADS_FILENAME
|
||||
stale = datetime.now(UTC) - podcast_feeds._RETRY_BACKOFF - timedelta(seconds=1)
|
||||
record = json.loads(record_path.read_text(encoding="utf-8"))
|
||||
record_path.write_text(
|
||||
json.dumps({filename: stale.isoformat() for filename in record}), encoding="utf-8"
|
||||
)
|
||||
|
||||
await sync_show(client, folder, "http://example.com/feed.xml")
|
||||
|
||||
assert len(enclosure_requests) == 4
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_sync_show_clears_the_failure_record_once_an_episode_succeeds(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
folder = tmp_path / "show"
|
||||
folder.mkdir()
|
||||
should_fail = True
|
||||
|
||||
def handler(request: httpx2.Request) -> httpx2.Response:
|
||||
url = str(request.url)
|
||||
if url == "http://example.com/feed.xml":
|
||||
return httpx2.Response(200, content=_FEED)
|
||||
if should_fail:
|
||||
return httpx2.Response(404)
|
||||
return httpx2.Response(200, content=b"audio-bytes")
|
||||
|
||||
async with httpx2.AsyncClient(transport=httpx2.MockTransport(handler)) as client:
|
||||
await sync_show(client, folder, "http://example.com/feed.xml")
|
||||
assert (folder / podcast_feeds._FAILED_DOWNLOADS_FILENAME).exists()
|
||||
|
||||
should_fail = False
|
||||
# Backdate the record so the retry isn't just skipped by the backoff window.
|
||||
record_path = folder / podcast_feeds._FAILED_DOWNLOADS_FILENAME
|
||||
stale = datetime.now(UTC) - podcast_feeds._RETRY_BACKOFF - timedelta(seconds=1)
|
||||
record = json.loads(record_path.read_text(encoding="utf-8"))
|
||||
record_path.write_text(
|
||||
json.dumps({filename: stale.isoformat() for filename in record}), encoding="utf-8"
|
||||
)
|
||||
|
||||
changed = await sync_show(client, folder, "http://example.com/feed.xml")
|
||||
|
||||
assert changed is True
|
||||
assert not (folder / podcast_feeds._FAILED_DOWNLOADS_FILENAME).exists()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_sync_show_survives_a_corrupt_failure_record(tmp_path: Path) -> None:
|
||||
folder = tmp_path / "show"
|
||||
folder.mkdir()
|
||||
(folder / podcast_feeds._FAILED_DOWNLOADS_FILENAME).write_text(
|
||||
"not valid json", encoding="utf-8"
|
||||
)
|
||||
|
||||
def handler(request: httpx2.Request) -> httpx2.Response:
|
||||
if str(request.url) == "http://example.com/feed.xml":
|
||||
return httpx2.Response(200, content=_FEED)
|
||||
return httpx2.Response(200, content=b"audio-bytes")
|
||||
|
||||
async with httpx2.AsyncClient(transport=httpx2.MockTransport(handler)) as client:
|
||||
changed = await sync_show(client, folder, "http://example.com/feed.xml")
|
||||
|
||||
assert changed is True
|
||||
assert (folder / "20250101 - Episode One.mp3").exists()
|
||||
|
||||
Reference in New Issue
Block a user