Martin Bauer
df89acd9a8
_extract_audio wrote transcoded audio to a .tmp temp path, so ffmpeg couldn't guess a muxer from the filename and aborted on every video episode whose source was still reachable. Pass -f mp3 explicitly instead of relying on the extension. Separately, a permanently 404'd episode was retried on every startup and every 6-hour poll forever, since nothing remembered past failures. Track failed downloads per show folder with a 7-day backoff before retrying. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
633 lines
23 KiB
Python
633 lines
23 KiB
Python
"""Podcast feed polling: finding shows, parsing feeds, and downloading only what's new."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
from datetime import UTC, datetime, timedelta
|
|
from pathlib import Path
|
|
|
|
import httpx2
|
|
import pytest
|
|
|
|
from musicmouse.library import podcast_feeds
|
|
from musicmouse.library.podcast_feeds import (
|
|
AudioExtractionError,
|
|
Episode,
|
|
_entry_enclosure,
|
|
download_episode,
|
|
episode_cover_filename,
|
|
episode_filename,
|
|
find_feed_shows,
|
|
missing_episodes,
|
|
parse_feed,
|
|
resolve_episode_cover,
|
|
sync_all_shows,
|
|
sync_show,
|
|
)
|
|
|
|
_FEED = b"""<?xml version="1.0"?>
|
|
<rss version="2.0">
|
|
<channel>
|
|
<title>Test Show</title>
|
|
<item>
|
|
<title>Episode One</title>
|
|
<pubDate>Wed, 01 Jan 2025 08:00:00 GMT</pubDate>
|
|
<enclosure url="http://example.com/ep1.mp3" length="100" type="audio/mpeg" />
|
|
<guid>ep1</guid>
|
|
</item>
|
|
<item>
|
|
<title>No Audio Here</title>
|
|
<pubDate>Thu, 02 Jan 2025 08:00:00 GMT</pubDate>
|
|
<guid>ep2</guid>
|
|
</item>
|
|
<item>
|
|
<title>Episode Two: Special/Chars?</title>
|
|
<pubDate>Fri, 03 Jan 2025 08:00:00 GMT</pubDate>
|
|
<enclosure url="http://example.com/redirect?id=2" length="200" type="audio/mpeg" />
|
|
<guid>ep3</guid>
|
|
</item>
|
|
</channel>
|
|
</rss>
|
|
"""
|
|
|
|
_FEED_WITH_COVERS = b"""<?xml version="1.0"?>
|
|
<rss version="2.0" xmlns:itunes="http://www.itunes.com/dtds/podcast-1.0.dtd">
|
|
<channel>
|
|
<title>Cover Show</title>
|
|
<link>http://example.com/show</link>
|
|
<itunes:image href="http://example.com/channel.jpg" />
|
|
<item>
|
|
<title>Unique Cover</title>
|
|
<link>http://example.com/episodes/unique</link>
|
|
<pubDate>Wed, 01 Jan 2025 08:00:00 GMT</pubDate>
|
|
<itunes:image href="http://example.com/unique.jpg" />
|
|
<enclosure url="http://example.com/ep1.mp3" length="100" type="audio/mpeg" />
|
|
<guid>ep1</guid>
|
|
</item>
|
|
<item>
|
|
<title>Same As Channel</title>
|
|
<link>http://example.com/show</link>
|
|
<pubDate>Thu, 02 Jan 2025 08:00:00 GMT</pubDate>
|
|
<itunes:image href="http://example.com/channel.jpg" />
|
|
<enclosure url="http://example.com/ep2.mp3" length="100" type="audio/mpeg" />
|
|
<guid>ep2</guid>
|
|
</item>
|
|
</channel>
|
|
</rss>
|
|
"""
|
|
|
|
_VIDEO_FEED = b"""<?xml version="1.0"?>
|
|
<rss version="2.0">
|
|
<channel>
|
|
<title>Video Show</title>
|
|
<item>
|
|
<title>Video Only</title>
|
|
<pubDate>Wed, 01 Jan 2025 08:00:00 GMT</pubDate>
|
|
<enclosure url="http://example.com/ep1.mp4" length="100" type="video/mpeg" />
|
|
<guid>ep1</guid>
|
|
</item>
|
|
</channel>
|
|
</rss>
|
|
"""
|
|
|
|
|
|
def _root_with_show(
|
|
tmp_path: Path, *, feed_url: str | None = "http://example.com/feed.xml"
|
|
) -> Path:
|
|
root = tmp_path / "music"
|
|
show = root / "Kinderpodcasts" / "Test Show"
|
|
show.mkdir(parents=True)
|
|
if feed_url is not None:
|
|
(show / "feed.txt").write_text(f"{feed_url}\n")
|
|
(root / "Kinderpodcasts" / "No Feed Show").mkdir(parents=True)
|
|
# A non-episode section must never be scanned for feed markers.
|
|
music = root / "Musik" / "Some Album"
|
|
music.mkdir(parents=True)
|
|
(music / "feed.txt").write_text("http://example.com/should-be-ignored.xml\n")
|
|
return root
|
|
|
|
|
|
def test_find_feed_shows_only_picks_up_episode_sections_with_a_marker(tmp_path: Path) -> None:
|
|
root = _root_with_show(tmp_path)
|
|
found = find_feed_shows(root)
|
|
assert found == [
|
|
("Kinderpodcasts", root / "Kinderpodcasts" / "Test Show", "http://example.com/feed.xml")
|
|
]
|
|
|
|
|
|
def test_find_feed_shows_skips_an_empty_marker(tmp_path: Path) -> None:
|
|
root = tmp_path / "music"
|
|
show = root / "Kinderpodcasts" / "Empty"
|
|
show.mkdir(parents=True)
|
|
(show / "feed.txt").write_text("\n\n")
|
|
assert find_feed_shows(root) == []
|
|
|
|
|
|
def test_parse_feed_keeps_only_entries_with_an_audio_enclosure() -> None:
|
|
episodes = parse_feed(_FEED)
|
|
assert [episode.title for episode in episodes] == ["Episode One", "Episode Two: Special/Chars?"]
|
|
assert episodes[0].enclosure_url == "http://example.com/ep1.mp3"
|
|
assert episodes[0].published.year == 2025
|
|
assert episodes[0].published.month == 1
|
|
assert episodes[0].published.day == 1
|
|
|
|
|
|
def test_episode_filename_matches_the_hand_placed_convention() -> None:
|
|
episode = parse_feed(_FEED)[0]
|
|
assert episode_filename(
|
|
episode.published, episode.title, episode.enclosure_type, episode.enclosure_url
|
|
) == "20250101 - Episode One.mp3"
|
|
|
|
|
|
def test_episode_filename_sanitizes_illegal_characters_and_falls_back_on_type() -> None:
|
|
episode = parse_feed(_FEED)[1]
|
|
# No file suffix on the (redirect-style) URL, so the extension comes from the
|
|
# enclosure's content type instead.
|
|
name = episode_filename(
|
|
episode.published, episode.title, episode.enclosure_type, episode.enclosure_url
|
|
)
|
|
# Illegal characters become "_" rather than being dropped, matching the
|
|
# convention every episode already on disk (downloaded by hand via the
|
|
# `podcast-dl` CLI) was named with.
|
|
assert name == "20250103 - Episode Two_ Special_Chars_.mp3"
|
|
|
|
|
|
def test_episode_filename_matches_the_podcast_dl_convention_on_a_real_title() -> None:
|
|
# Pinned against a real file already in the library: `podcast-dl` replaced the
|
|
# "?" with "_" rather than dropping it, so ours must too or the whole back
|
|
# catalog would look "new" the moment a feed marker is added for that show.
|
|
published = parse_feed(_FEED)[0].published
|
|
name = episode_filename(published, "Wie funktioniert Bogenschießen?", "audio/mpeg", "x.mp3")
|
|
assert name == "20250101 - Wie funktioniert Bogenschießen_.mp3"
|
|
|
|
|
|
def test_missing_episodes_skips_files_already_on_disk(tmp_path: Path) -> None:
|
|
folder = tmp_path / "show"
|
|
folder.mkdir()
|
|
episodes = parse_feed(_FEED)
|
|
(folder / "20250101 - Episode One.mp3").write_bytes(b"already here")
|
|
|
|
pending = missing_episodes(folder, episodes)
|
|
|
|
assert [episode.title for episode, _ in pending] == ["Episode Two: Special/Chars?"]
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_download_episode_writes_via_a_temp_file(tmp_path: Path) -> None:
|
|
folder = tmp_path / "show"
|
|
folder.mkdir()
|
|
episode = Episode(
|
|
title="Episode One",
|
|
published=parse_feed(_FEED)[0].published,
|
|
enclosure_url="http://example.com/ep1.mp3",
|
|
enclosure_type="audio/mpeg",
|
|
)
|
|
|
|
def handler(request: httpx2.Request) -> httpx2.Response:
|
|
return httpx2.Response(200, content=b"audio-bytes")
|
|
|
|
async with httpx2.AsyncClient(transport=httpx2.MockTransport(handler)) as client:
|
|
await download_episode(client, folder, episode, "20250101 - Episode One.mp3")
|
|
|
|
assert (folder / "20250101 - Episode One.mp3").read_bytes() == b"audio-bytes"
|
|
assert list(folder.glob(".downloading-*.tmp")) == []
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_sync_show_downloads_new_episodes_and_is_a_noop_on_rerun(tmp_path: Path) -> None:
|
|
folder = tmp_path / "show"
|
|
folder.mkdir()
|
|
|
|
def handler(request: httpx2.Request) -> httpx2.Response:
|
|
if str(request.url) == "http://example.com/feed.xml":
|
|
return httpx2.Response(200, content=_FEED)
|
|
return httpx2.Response(200, content=b"audio-bytes")
|
|
|
|
async with httpx2.AsyncClient(transport=httpx2.MockTransport(handler)) as client:
|
|
changed_first = await sync_show(client, folder, "http://example.com/feed.xml")
|
|
changed_second = await sync_show(client, folder, "http://example.com/feed.xml")
|
|
|
|
assert changed_first is True
|
|
assert changed_second is False
|
|
assert (folder / "20250101 - Episode One.mp3").exists()
|
|
assert (folder / "20250103 - Episode Two_ Special_Chars_.mp3").exists()
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_sync_show_survives_an_unreachable_feed(tmp_path: Path) -> None:
|
|
folder = tmp_path / "show"
|
|
folder.mkdir()
|
|
|
|
def handler(request: httpx2.Request) -> httpx2.Response:
|
|
raise httpx2.ConnectError("mock: connection refused")
|
|
|
|
async with httpx2.AsyncClient(transport=httpx2.MockTransport(handler)) as client:
|
|
changed = await sync_show(client, folder, "http://example.com/feed.xml")
|
|
|
|
assert changed is False
|
|
assert list(folder.iterdir()) == []
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_sync_show_survives_a_broken_enclosure(tmp_path: Path) -> None:
|
|
folder = tmp_path / "show"
|
|
folder.mkdir()
|
|
|
|
def handler(request: httpx2.Request) -> httpx2.Response:
|
|
if str(request.url) == "http://example.com/feed.xml":
|
|
return httpx2.Response(200, content=_FEED)
|
|
return httpx2.Response(404)
|
|
|
|
async with httpx2.AsyncClient(transport=httpx2.MockTransport(handler)) as client:
|
|
changed = await sync_show(client, folder, "http://example.com/feed.xml")
|
|
|
|
# Both enclosures 404, so nothing landed - but the pass still completed cleanly.
|
|
assert changed is False
|
|
assert list(folder.glob("*.mp3")) == []
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_sync_all_shows_keeps_going_past_one_broken_show(tmp_path: Path) -> None:
|
|
root = tmp_path / "music"
|
|
good = root / "Kinderpodcasts" / "Good Show"
|
|
good.mkdir(parents=True)
|
|
(good / "feed.txt").write_text("http://example.com/feed.xml\n")
|
|
bad = root / "Kinderpodcasts" / "Bad Show"
|
|
bad.mkdir(parents=True)
|
|
(bad / "feed.txt").write_text("http://example.com/broken.xml\n")
|
|
|
|
def handler(request: httpx2.Request) -> httpx2.Response:
|
|
url = str(request.url)
|
|
if url == "http://example.com/feed.xml":
|
|
return httpx2.Response(200, content=_FEED)
|
|
if url == "http://example.com/broken.xml":
|
|
raise httpx2.ConnectError("mock: connection refused")
|
|
return httpx2.Response(200, content=b"audio-bytes")
|
|
|
|
async with httpx2.AsyncClient(transport=httpx2.MockTransport(handler)) as client:
|
|
changed = await sync_all_shows(client, root)
|
|
|
|
assert changed is True
|
|
assert list(good.glob("*.mp3"))
|
|
assert list(bad.glob("*.mp3")) == []
|
|
|
|
|
|
# ------------------------------------------------------------------- per-episode art
|
|
|
|
|
|
def test_parse_feed_extracts_cover_and_link_only_when_distinct_from_the_channel() -> None:
|
|
unique, same = parse_feed(_FEED_WITH_COVERS)
|
|
assert unique.cover_url == "http://example.com/unique.jpg"
|
|
assert unique.link == "http://example.com/episodes/unique"
|
|
# This item's image/link both just repeat the channel's own - not real per-episode
|
|
# art, so both come out None rather than a false positive.
|
|
assert same.cover_url is None
|
|
assert same.link is None
|
|
|
|
|
|
def test_episode_cover_filename_shares_the_audio_stem_and_takes_the_url_extension() -> None:
|
|
episode = parse_feed(_FEED_WITH_COVERS)[0]
|
|
name = episode_cover_filename(episode.published, episode.title, episode.cover_url)
|
|
assert name == "20250101 - Unique Cover.jpg"
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_resolve_episode_cover_prefers_the_feed_image_without_a_request() -> None:
|
|
episode = parse_feed(_FEED_WITH_COVERS)[0]
|
|
|
|
def handler(request: httpx2.Request) -> httpx2.Response:
|
|
raise AssertionError("must not fetch the episode page when cover_url is already set")
|
|
|
|
async with httpx2.AsyncClient(transport=httpx2.MockTransport(handler)) as client:
|
|
cover = await resolve_episode_cover(client, episode)
|
|
|
|
assert cover == "http://example.com/unique.jpg"
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_resolve_episode_cover_falls_back_to_og_image_on_the_linked_page() -> None:
|
|
episode = Episode(
|
|
title="No Feed Image",
|
|
published=parse_feed(_FEED)[0].published,
|
|
enclosure_url="http://example.com/ep.mp3",
|
|
enclosure_type="audio/mpeg",
|
|
link="http://example.com/episode-page",
|
|
)
|
|
|
|
def handler(request: httpx2.Request) -> httpx2.Response:
|
|
return httpx2.Response(
|
|
200,
|
|
content=b'<html><head>'
|
|
b'<meta property="og:image" content="http://example.com/scraped.jpg"/>'
|
|
b"</head></html>",
|
|
)
|
|
|
|
async with httpx2.AsyncClient(transport=httpx2.MockTransport(handler)) as client:
|
|
cover = await resolve_episode_cover(client, episode)
|
|
|
|
assert cover == "http://example.com/scraped.jpg"
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_resolve_episode_cover_returns_none_when_the_link_is_unreachable() -> None:
|
|
episode = Episode(
|
|
title="Broken Link",
|
|
published=parse_feed(_FEED)[0].published,
|
|
enclosure_url="http://example.com/ep.mp3",
|
|
enclosure_type="audio/mpeg",
|
|
link="http://example.com/missing",
|
|
)
|
|
|
|
async with httpx2.AsyncClient(
|
|
transport=httpx2.MockTransport(lambda _: httpx2.Response(404))
|
|
) as client:
|
|
cover = await resolve_episode_cover(client, episode)
|
|
|
|
assert cover is None
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_resolve_episode_cover_returns_none_without_a_cover_or_link() -> None:
|
|
episode = Episode(
|
|
title="Nothing",
|
|
published=parse_feed(_FEED)[0].published,
|
|
enclosure_url="http://example.com/ep.mp3",
|
|
enclosure_type="audio/mpeg",
|
|
)
|
|
|
|
async def handler(request: httpx2.Request) -> httpx2.Response: # pragma: no cover
|
|
raise AssertionError("must not make a request with neither a cover nor a link")
|
|
|
|
async with httpx2.AsyncClient(transport=httpx2.MockTransport(handler)) as client:
|
|
cover = await resolve_episode_cover(client, episode)
|
|
|
|
assert cover is None
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_sync_show_downloads_the_episode_cover_alongside_the_audio(tmp_path: Path) -> None:
|
|
folder = tmp_path / "show"
|
|
folder.mkdir()
|
|
|
|
def handler(request: httpx2.Request) -> httpx2.Response:
|
|
url = str(request.url)
|
|
if url == "http://example.com/feed.xml":
|
|
return httpx2.Response(200, content=_FEED_WITH_COVERS)
|
|
if url == "http://example.com/unique.jpg":
|
|
return httpx2.Response(200, content=b"cover-bytes")
|
|
return httpx2.Response(200, content=b"audio-bytes")
|
|
|
|
async with httpx2.AsyncClient(transport=httpx2.MockTransport(handler)) as client:
|
|
changed = await sync_show(client, folder, "http://example.com/feed.xml")
|
|
|
|
assert changed is True
|
|
assert (folder / "20250101 - Unique Cover.jpg").read_bytes() == b"cover-bytes"
|
|
# No cover file for the episode whose image just repeats the channel's.
|
|
assert not (folder / "20250102 - Same As Channel.jpg").exists()
|
|
assert list(folder.glob(".downloading-*.tmp")) == []
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_sync_show_survives_a_failed_cover_download(tmp_path: Path) -> None:
|
|
folder = tmp_path / "show"
|
|
folder.mkdir()
|
|
|
|
def handler(request: httpx2.Request) -> httpx2.Response:
|
|
url = str(request.url)
|
|
if url == "http://example.com/feed.xml":
|
|
return httpx2.Response(200, content=_FEED_WITH_COVERS)
|
|
if url == "http://example.com/unique.jpg":
|
|
return httpx2.Response(404)
|
|
return httpx2.Response(200, content=b"audio-bytes")
|
|
|
|
async with httpx2.AsyncClient(transport=httpx2.MockTransport(handler)) as client:
|
|
changed = await sync_show(client, folder, "http://example.com/feed.xml")
|
|
|
|
# The episode itself still downloaded fine - only its cover failed.
|
|
assert changed is True
|
|
assert (folder / "20250101 - Unique Cover.mp3").exists()
|
|
assert not (folder / "20250101 - Unique Cover.jpg").exists()
|
|
|
|
|
|
# ---------------------------------------------------------------------- video shows
|
|
|
|
|
|
def test_entry_enclosure_accepts_video_when_no_audio_is_offered() -> None:
|
|
entry = {"enclosures": [{"href": "http://example.com/ep.mp4", "type": "video/mpeg"}]}
|
|
assert _entry_enclosure(entry) == ("http://example.com/ep.mp4", "video/mpeg")
|
|
|
|
|
|
def test_entry_enclosure_prefers_audio_over_video_when_both_are_offered() -> None:
|
|
entry = {
|
|
"enclosures": [
|
|
{"href": "http://example.com/ep.mp4", "type": "video/mpeg"},
|
|
{"href": "http://example.com/ep.mp3", "type": "audio/mpeg"},
|
|
]
|
|
}
|
|
assert _entry_enclosure(entry) == ("http://example.com/ep.mp3", "audio/mpeg")
|
|
|
|
|
|
def test_parse_feed_keeps_a_video_only_episode() -> None:
|
|
episodes = parse_feed(_VIDEO_FEED)
|
|
assert episodes[0].enclosure_url == "http://example.com/ep1.mp4"
|
|
assert episodes[0].enclosure_type == "video/mpeg"
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_download_episode_raises_when_ffmpeg_is_not_installed(
|
|
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
|
|
) -> None:
|
|
monkeypatch.setattr(podcast_feeds.shutil, "which", lambda _: None)
|
|
folder = tmp_path / "show"
|
|
folder.mkdir()
|
|
episode = parse_feed(_VIDEO_FEED)[0]
|
|
|
|
async with httpx2.AsyncClient(
|
|
transport=httpx2.MockTransport(lambda _: httpx2.Response(200))
|
|
) as client:
|
|
with pytest.raises(AudioExtractionError):
|
|
await download_episode(client, folder, episode, "20250101 - Video Only.mp3")
|
|
|
|
assert list(folder.glob(".downloading-*.tmp")) == []
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_download_episode_raises_when_ffmpeg_exits_nonzero(
|
|
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
|
|
) -> None:
|
|
monkeypatch.setattr(podcast_feeds.shutil, "which", lambda _: "/usr/bin/ffmpeg")
|
|
|
|
class _FailingProcess:
|
|
returncode = 1
|
|
|
|
async def communicate(self) -> tuple[bytes, bytes]:
|
|
return b"", b"boom"
|
|
|
|
async def fake_exec(*args: object, **kwargs: object) -> _FailingProcess:
|
|
return _FailingProcess()
|
|
|
|
monkeypatch.setattr(podcast_feeds.asyncio, "create_subprocess_exec", fake_exec)
|
|
|
|
folder = tmp_path / "show"
|
|
folder.mkdir()
|
|
episode = parse_feed(_VIDEO_FEED)[0]
|
|
|
|
async with httpx2.AsyncClient(
|
|
transport=httpx2.MockTransport(lambda _: httpx2.Response(200))
|
|
) as client:
|
|
with pytest.raises(AudioExtractionError, match="boom"):
|
|
await download_episode(client, folder, episode, "20250101 - Video Only.mp3")
|
|
|
|
assert list(folder.glob(".downloading-*.tmp")) == []
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_download_episode_extracts_audio_via_ffmpeg(
|
|
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
|
|
) -> None:
|
|
monkeypatch.setattr(podcast_feeds.shutil, "which", lambda _: "/usr/bin/ffmpeg")
|
|
|
|
class _SucceedingProcess:
|
|
returncode = 0
|
|
|
|
async def communicate(self) -> tuple[bytes, bytes]:
|
|
return b"", b""
|
|
|
|
captured_args: list[object] = []
|
|
|
|
async def fake_exec(*args: object, **kwargs: object) -> _SucceedingProcess:
|
|
# ffmpeg's destination path is the last positional argument; write to it here
|
|
# to stand in for what the real binary would have produced.
|
|
captured_args.extend(args)
|
|
Path(str(args[-1])).write_bytes(b"extracted-audio")
|
|
return _SucceedingProcess()
|
|
|
|
monkeypatch.setattr(podcast_feeds.asyncio, "create_subprocess_exec", fake_exec)
|
|
|
|
folder = tmp_path / "show"
|
|
folder.mkdir()
|
|
episode = parse_feed(_VIDEO_FEED)[0]
|
|
|
|
async with httpx2.AsyncClient(
|
|
transport=httpx2.MockTransport(lambda _: httpx2.Response(200))
|
|
) as client:
|
|
await download_episode(client, folder, episode, "20250101 - Video Only.mp3")
|
|
|
|
assert (folder / "20250101 - Video Only.mp3").read_bytes() == b"extracted-audio"
|
|
assert list(folder.glob(".downloading-*.tmp")) == []
|
|
# The destination path ends in ".tmp", not ".mp3" - ffmpeg can't guess the muxer
|
|
# from that extension, so the format must be passed explicitly.
|
|
assert captured_args[captured_args.index("-f") + 1] == "mp3"
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_sync_show_does_not_retry_a_failed_episode_within_the_backoff(
|
|
tmp_path: Path,
|
|
) -> None:
|
|
folder = tmp_path / "show"
|
|
folder.mkdir()
|
|
enclosure_requests: list[str] = []
|
|
|
|
def handler(request: httpx2.Request) -> httpx2.Response:
|
|
url = str(request.url)
|
|
if url == "http://example.com/feed.xml":
|
|
return httpx2.Response(200, content=_FEED)
|
|
enclosure_requests.append(url)
|
|
return httpx2.Response(404)
|
|
|
|
async with httpx2.AsyncClient(transport=httpx2.MockTransport(handler)) as client:
|
|
await sync_show(client, folder, "http://example.com/feed.xml")
|
|
await sync_show(client, folder, "http://example.com/feed.xml")
|
|
|
|
# Both episodes 404 on the first pass and get remembered; the second pass must not
|
|
# re-request either enclosure while the backoff is still in effect.
|
|
assert len(enclosure_requests) == 2
|
|
assert (folder / podcast_feeds._FAILED_DOWNLOADS_FILENAME).exists()
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_sync_show_retries_a_failed_episode_once_the_backoff_expires(
|
|
tmp_path: Path,
|
|
) -> None:
|
|
folder = tmp_path / "show"
|
|
folder.mkdir()
|
|
enclosure_requests: list[str] = []
|
|
|
|
def handler(request: httpx2.Request) -> httpx2.Response:
|
|
url = str(request.url)
|
|
if url == "http://example.com/feed.xml":
|
|
return httpx2.Response(200, content=_FEED)
|
|
enclosure_requests.append(url)
|
|
return httpx2.Response(404)
|
|
|
|
async with httpx2.AsyncClient(transport=httpx2.MockTransport(handler)) as client:
|
|
await sync_show(client, folder, "http://example.com/feed.xml")
|
|
|
|
# Backdate both failure records past the backoff window, simulating that a
|
|
# week has gone by since the first attempt.
|
|
record_path = folder / podcast_feeds._FAILED_DOWNLOADS_FILENAME
|
|
stale = datetime.now(UTC) - podcast_feeds._RETRY_BACKOFF - timedelta(seconds=1)
|
|
record = json.loads(record_path.read_text(encoding="utf-8"))
|
|
record_path.write_text(
|
|
json.dumps({filename: stale.isoformat() for filename in record}), encoding="utf-8"
|
|
)
|
|
|
|
await sync_show(client, folder, "http://example.com/feed.xml")
|
|
|
|
assert len(enclosure_requests) == 4
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_sync_show_clears_the_failure_record_once_an_episode_succeeds(
|
|
tmp_path: Path,
|
|
) -> None:
|
|
folder = tmp_path / "show"
|
|
folder.mkdir()
|
|
should_fail = True
|
|
|
|
def handler(request: httpx2.Request) -> httpx2.Response:
|
|
url = str(request.url)
|
|
if url == "http://example.com/feed.xml":
|
|
return httpx2.Response(200, content=_FEED)
|
|
if should_fail:
|
|
return httpx2.Response(404)
|
|
return httpx2.Response(200, content=b"audio-bytes")
|
|
|
|
async with httpx2.AsyncClient(transport=httpx2.MockTransport(handler)) as client:
|
|
await sync_show(client, folder, "http://example.com/feed.xml")
|
|
assert (folder / podcast_feeds._FAILED_DOWNLOADS_FILENAME).exists()
|
|
|
|
should_fail = False
|
|
# Backdate the record so the retry isn't just skipped by the backoff window.
|
|
record_path = folder / podcast_feeds._FAILED_DOWNLOADS_FILENAME
|
|
stale = datetime.now(UTC) - podcast_feeds._RETRY_BACKOFF - timedelta(seconds=1)
|
|
record = json.loads(record_path.read_text(encoding="utf-8"))
|
|
record_path.write_text(
|
|
json.dumps({filename: stale.isoformat() for filename in record}), encoding="utf-8"
|
|
)
|
|
|
|
changed = await sync_show(client, folder, "http://example.com/feed.xml")
|
|
|
|
assert changed is True
|
|
assert not (folder / podcast_feeds._FAILED_DOWNLOADS_FILENAME).exists()
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_sync_show_survives_a_corrupt_failure_record(tmp_path: Path) -> None:
|
|
folder = tmp_path / "show"
|
|
folder.mkdir()
|
|
(folder / podcast_feeds._FAILED_DOWNLOADS_FILENAME).write_text(
|
|
"not valid json", encoding="utf-8"
|
|
)
|
|
|
|
def handler(request: httpx2.Request) -> httpx2.Response:
|
|
if str(request.url) == "http://example.com/feed.xml":
|
|
return httpx2.Response(200, content=_FEED)
|
|
return httpx2.Response(200, content=b"audio-bytes")
|
|
|
|
async with httpx2.AsyncClient(transport=httpx2.MockTransport(handler)) as client:
|
|
changed = await sync_show(client, folder, "http://example.com/feed.xml")
|
|
|
|
assert changed is True
|
|
assert (folder / "20250101 - Episode One.mp3").exists()
|