Martin Bauer
5af5c54c62
A podcast's episode cover was only fetched right after the feed sync downloaded that episode, so anything placed by hand (or downloaded before the og:image fallback existed) never got one - in Wissen macht Ah only the three newest episodes had their own art. The sync now fetches the cover of every on-disk episode in the feed window that lacks a sidecar image. One that can't be found or fetched is recorded in .missing-covers.json and left alone for the same 7-day backoff as failed downloads, so a feed without per-episode art doesn't cost a page fetch per episode every poll. The scanner's per-episode fingerprint now includes the sidecar cover, so one added after the episode was first scanned invalidates its cached entry instead of being ignored until the index is rebuilt. Episodes without a sidecar fingerprint exactly as before, so existing caches stay valid. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
857 lines
31 KiB
Python
857 lines
31 KiB
Python
"""Podcast feed polling: finding shows, parsing feeds, and downloading only what's new."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
from datetime import UTC, datetime, timedelta
|
|
from pathlib import Path
|
|
|
|
import httpx2
|
|
import pytest
|
|
|
|
from musicmouse.library import podcast_feeds
|
|
from musicmouse.library.podcast_feeds import (
|
|
AudioExtractionError,
|
|
Episode,
|
|
_entry_enclosure,
|
|
download_episode,
|
|
episode_cover_filename,
|
|
episode_filename,
|
|
find_feed_shows,
|
|
missing_episodes,
|
|
newest_episodes,
|
|
parse_feed,
|
|
prune_show,
|
|
resolve_episode_cover,
|
|
sync_all_shows,
|
|
sync_show,
|
|
)
|
|
|
|
_FEED = b"""<?xml version="1.0"?>
|
|
<rss version="2.0">
|
|
<channel>
|
|
<title>Test Show</title>
|
|
<item>
|
|
<title>Episode One</title>
|
|
<pubDate>Wed, 01 Jan 2025 08:00:00 GMT</pubDate>
|
|
<enclosure url="http://example.com/ep1.mp3" length="100" type="audio/mpeg" />
|
|
<guid>ep1</guid>
|
|
</item>
|
|
<item>
|
|
<title>No Audio Here</title>
|
|
<pubDate>Thu, 02 Jan 2025 08:00:00 GMT</pubDate>
|
|
<guid>ep2</guid>
|
|
</item>
|
|
<item>
|
|
<title>Episode Two: Special/Chars?</title>
|
|
<pubDate>Fri, 03 Jan 2025 08:00:00 GMT</pubDate>
|
|
<enclosure url="http://example.com/redirect?id=2" length="200" type="audio/mpeg" />
|
|
<guid>ep3</guid>
|
|
</item>
|
|
</channel>
|
|
</rss>
|
|
"""
|
|
|
|
_FEED_WITH_COVERS = b"""<?xml version="1.0"?>
|
|
<rss version="2.0" xmlns:itunes="http://www.itunes.com/dtds/podcast-1.0.dtd">
|
|
<channel>
|
|
<title>Cover Show</title>
|
|
<link>http://example.com/show</link>
|
|
<itunes:image href="http://example.com/channel.jpg" />
|
|
<item>
|
|
<title>Unique Cover</title>
|
|
<link>http://example.com/episodes/unique</link>
|
|
<pubDate>Wed, 01 Jan 2025 08:00:00 GMT</pubDate>
|
|
<itunes:image href="http://example.com/unique.jpg" />
|
|
<enclosure url="http://example.com/ep1.mp3" length="100" type="audio/mpeg" />
|
|
<guid>ep1</guid>
|
|
</item>
|
|
<item>
|
|
<title>Same As Channel</title>
|
|
<link>http://example.com/show</link>
|
|
<pubDate>Thu, 02 Jan 2025 08:00:00 GMT</pubDate>
|
|
<itunes:image href="http://example.com/channel.jpg" />
|
|
<enclosure url="http://example.com/ep2.mp3" length="100" type="audio/mpeg" />
|
|
<guid>ep2</guid>
|
|
</item>
|
|
</channel>
|
|
</rss>
|
|
"""
|
|
|
|
_VIDEO_FEED = b"""<?xml version="1.0"?>
|
|
<rss version="2.0">
|
|
<channel>
|
|
<title>Video Show</title>
|
|
<item>
|
|
<title>Video Only</title>
|
|
<pubDate>Wed, 01 Jan 2025 08:00:00 GMT</pubDate>
|
|
<enclosure url="http://example.com/ep1.mp4" length="100" type="video/mpeg" />
|
|
<guid>ep1</guid>
|
|
</item>
|
|
</channel>
|
|
</rss>
|
|
"""
|
|
|
|
|
|
def _root_with_show(
|
|
tmp_path: Path, *, feed_url: str | None = "http://example.com/feed.xml"
|
|
) -> Path:
|
|
root = tmp_path / "music"
|
|
show = root / "Kinderpodcasts" / "Test Show"
|
|
show.mkdir(parents=True)
|
|
if feed_url is not None:
|
|
(show / "feed.txt").write_text(f"{feed_url}\n")
|
|
(root / "Kinderpodcasts" / "No Feed Show").mkdir(parents=True)
|
|
# A non-episode section must never be scanned for feed markers.
|
|
music = root / "Musik" / "Some Album"
|
|
music.mkdir(parents=True)
|
|
(music / "feed.txt").write_text("http://example.com/should-be-ignored.xml\n")
|
|
return root
|
|
|
|
|
|
def test_find_feed_shows_only_picks_up_episode_sections_with_a_marker(tmp_path: Path) -> None:
|
|
root = _root_with_show(tmp_path)
|
|
found = find_feed_shows(root)
|
|
assert found == [
|
|
("Kinderpodcasts", root / "Kinderpodcasts" / "Test Show", "http://example.com/feed.xml")
|
|
]
|
|
|
|
|
|
def test_find_feed_shows_skips_an_empty_marker(tmp_path: Path) -> None:
|
|
root = tmp_path / "music"
|
|
show = root / "Kinderpodcasts" / "Empty"
|
|
show.mkdir(parents=True)
|
|
(show / "feed.txt").write_text("\n\n")
|
|
assert find_feed_shows(root) == []
|
|
|
|
|
|
def test_parse_feed_keeps_only_entries_with_an_audio_enclosure() -> None:
|
|
episodes = parse_feed(_FEED)
|
|
assert [episode.title for episode in episodes] == ["Episode One", "Episode Two: Special/Chars?"]
|
|
assert episodes[0].enclosure_url == "http://example.com/ep1.mp3"
|
|
assert episodes[0].published.year == 2025
|
|
assert episodes[0].published.month == 1
|
|
assert episodes[0].published.day == 1
|
|
|
|
|
|
def test_episode_filename_matches_the_hand_placed_convention() -> None:
|
|
episode = parse_feed(_FEED)[0]
|
|
assert episode_filename(
|
|
episode.published, episode.title, episode.enclosure_type, episode.enclosure_url
|
|
) == "20250101 - Episode One.mp3"
|
|
|
|
|
|
def test_episode_filename_sanitizes_illegal_characters_and_falls_back_on_type() -> None:
|
|
episode = parse_feed(_FEED)[1]
|
|
# No file suffix on the (redirect-style) URL, so the extension comes from the
|
|
# enclosure's content type instead.
|
|
name = episode_filename(
|
|
episode.published, episode.title, episode.enclosure_type, episode.enclosure_url
|
|
)
|
|
# Illegal characters become "_" rather than being dropped, matching the
|
|
# convention every episode already on disk (downloaded by hand via the
|
|
# `podcast-dl` CLI) was named with.
|
|
assert name == "20250103 - Episode Two_ Special_Chars_.mp3"
|
|
|
|
|
|
def test_episode_filename_matches_the_podcast_dl_convention_on_a_real_title() -> None:
|
|
# Pinned against a real file already in the library: `podcast-dl` replaced the
|
|
# "?" with "_" rather than dropping it, so ours must too or the whole back
|
|
# catalog would look "new" the moment a feed marker is added for that show.
|
|
published = parse_feed(_FEED)[0].published
|
|
name = episode_filename(published, "Wie funktioniert Bogenschießen?", "audio/mpeg", "x.mp3")
|
|
assert name == "20250101 - Wie funktioniert Bogenschießen_.mp3"
|
|
|
|
|
|
def test_missing_episodes_skips_files_already_on_disk(tmp_path: Path) -> None:
|
|
folder = tmp_path / "show"
|
|
folder.mkdir()
|
|
episodes = parse_feed(_FEED)
|
|
(folder / "20250101 - Episode One.mp3").write_bytes(b"already here")
|
|
|
|
pending = missing_episodes(folder, episodes)
|
|
|
|
assert [episode.title for episode, _ in pending] == ["Episode Two: Special/Chars?"]
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_download_episode_writes_via_a_temp_file(tmp_path: Path) -> None:
|
|
folder = tmp_path / "show"
|
|
folder.mkdir()
|
|
episode = Episode(
|
|
title="Episode One",
|
|
published=parse_feed(_FEED)[0].published,
|
|
enclosure_url="http://example.com/ep1.mp3",
|
|
enclosure_type="audio/mpeg",
|
|
)
|
|
|
|
def handler(request: httpx2.Request) -> httpx2.Response:
|
|
return httpx2.Response(200, content=b"audio-bytes")
|
|
|
|
async with httpx2.AsyncClient(transport=httpx2.MockTransport(handler)) as client:
|
|
await download_episode(client, folder, episode, "20250101 - Episode One.mp3")
|
|
|
|
assert (folder / "20250101 - Episode One.mp3").read_bytes() == b"audio-bytes"
|
|
assert list(folder.glob(".downloading-*.tmp")) == []
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_sync_show_downloads_new_episodes_and_is_a_noop_on_rerun(tmp_path: Path) -> None:
|
|
folder = tmp_path / "show"
|
|
folder.mkdir()
|
|
|
|
def handler(request: httpx2.Request) -> httpx2.Response:
|
|
if str(request.url) == "http://example.com/feed.xml":
|
|
return httpx2.Response(200, content=_FEED)
|
|
return httpx2.Response(200, content=b"audio-bytes")
|
|
|
|
async with httpx2.AsyncClient(transport=httpx2.MockTransport(handler)) as client:
|
|
changed_first = await sync_show(client, folder, "http://example.com/feed.xml")
|
|
changed_second = await sync_show(client, folder, "http://example.com/feed.xml")
|
|
|
|
assert changed_first is True
|
|
assert changed_second is False
|
|
assert (folder / "20250101 - Episode One.mp3").exists()
|
|
assert (folder / "20250103 - Episode Two_ Special_Chars_.mp3").exists()
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_sync_show_survives_an_unreachable_feed(tmp_path: Path) -> None:
|
|
folder = tmp_path / "show"
|
|
folder.mkdir()
|
|
|
|
def handler(request: httpx2.Request) -> httpx2.Response:
|
|
raise httpx2.ConnectError("mock: connection refused")
|
|
|
|
async with httpx2.AsyncClient(transport=httpx2.MockTransport(handler)) as client:
|
|
changed = await sync_show(client, folder, "http://example.com/feed.xml")
|
|
|
|
assert changed is False
|
|
assert list(folder.iterdir()) == []
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_sync_show_survives_a_broken_enclosure(tmp_path: Path) -> None:
|
|
folder = tmp_path / "show"
|
|
folder.mkdir()
|
|
|
|
def handler(request: httpx2.Request) -> httpx2.Response:
|
|
if str(request.url) == "http://example.com/feed.xml":
|
|
return httpx2.Response(200, content=_FEED)
|
|
return httpx2.Response(404)
|
|
|
|
async with httpx2.AsyncClient(transport=httpx2.MockTransport(handler)) as client:
|
|
changed = await sync_show(client, folder, "http://example.com/feed.xml")
|
|
|
|
# Both enclosures 404, so nothing landed - but the pass still completed cleanly.
|
|
assert changed is False
|
|
assert list(folder.glob("*.mp3")) == []
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_sync_all_shows_keeps_going_past_one_broken_show(tmp_path: Path) -> None:
|
|
root = tmp_path / "music"
|
|
good = root / "Kinderpodcasts" / "Good Show"
|
|
good.mkdir(parents=True)
|
|
(good / "feed.txt").write_text("http://example.com/feed.xml\n")
|
|
bad = root / "Kinderpodcasts" / "Bad Show"
|
|
bad.mkdir(parents=True)
|
|
(bad / "feed.txt").write_text("http://example.com/broken.xml\n")
|
|
|
|
def handler(request: httpx2.Request) -> httpx2.Response:
|
|
url = str(request.url)
|
|
if url == "http://example.com/feed.xml":
|
|
return httpx2.Response(200, content=_FEED)
|
|
if url == "http://example.com/broken.xml":
|
|
raise httpx2.ConnectError("mock: connection refused")
|
|
return httpx2.Response(200, content=b"audio-bytes")
|
|
|
|
async with httpx2.AsyncClient(transport=httpx2.MockTransport(handler)) as client:
|
|
changed = await sync_all_shows(client, root)
|
|
|
|
assert changed is True
|
|
assert list(good.glob("*.mp3"))
|
|
assert list(bad.glob("*.mp3")) == []
|
|
|
|
|
|
# ------------------------------------------------------------------- per-episode art
|
|
|
|
|
|
def test_parse_feed_extracts_cover_and_link_only_when_distinct_from_the_channel() -> None:
|
|
unique, same = parse_feed(_FEED_WITH_COVERS)
|
|
assert unique.cover_url == "http://example.com/unique.jpg"
|
|
assert unique.link == "http://example.com/episodes/unique"
|
|
# This item's image/link both just repeat the channel's own - not real per-episode
|
|
# art, so both come out None rather than a false positive.
|
|
assert same.cover_url is None
|
|
assert same.link is None
|
|
|
|
|
|
def test_episode_cover_filename_shares_the_audio_stem_and_takes_the_url_extension() -> None:
|
|
episode = parse_feed(_FEED_WITH_COVERS)[0]
|
|
name = episode_cover_filename(episode.published, episode.title, episode.cover_url)
|
|
assert name == "20250101 - Unique Cover.jpg"
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_resolve_episode_cover_prefers_the_feed_image_without_a_request() -> None:
|
|
episode = parse_feed(_FEED_WITH_COVERS)[0]
|
|
|
|
def handler(request: httpx2.Request) -> httpx2.Response:
|
|
raise AssertionError("must not fetch the episode page when cover_url is already set")
|
|
|
|
async with httpx2.AsyncClient(transport=httpx2.MockTransport(handler)) as client:
|
|
cover = await resolve_episode_cover(client, episode)
|
|
|
|
assert cover == "http://example.com/unique.jpg"
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_resolve_episode_cover_falls_back_to_og_image_on_the_linked_page() -> None:
|
|
episode = Episode(
|
|
title="No Feed Image",
|
|
published=parse_feed(_FEED)[0].published,
|
|
enclosure_url="http://example.com/ep.mp3",
|
|
enclosure_type="audio/mpeg",
|
|
link="http://example.com/episode-page",
|
|
)
|
|
|
|
def handler(request: httpx2.Request) -> httpx2.Response:
|
|
return httpx2.Response(
|
|
200,
|
|
content=b'<html><head>'
|
|
b'<meta property="og:image" content="http://example.com/scraped.jpg"/>'
|
|
b"</head></html>",
|
|
)
|
|
|
|
async with httpx2.AsyncClient(transport=httpx2.MockTransport(handler)) as client:
|
|
cover = await resolve_episode_cover(client, episode)
|
|
|
|
assert cover == "http://example.com/scraped.jpg"
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_resolve_episode_cover_returns_none_when_the_link_is_unreachable() -> None:
|
|
episode = Episode(
|
|
title="Broken Link",
|
|
published=parse_feed(_FEED)[0].published,
|
|
enclosure_url="http://example.com/ep.mp3",
|
|
enclosure_type="audio/mpeg",
|
|
link="http://example.com/missing",
|
|
)
|
|
|
|
async with httpx2.AsyncClient(
|
|
transport=httpx2.MockTransport(lambda _: httpx2.Response(404))
|
|
) as client:
|
|
cover = await resolve_episode_cover(client, episode)
|
|
|
|
assert cover is None
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_resolve_episode_cover_returns_none_without_a_cover_or_link() -> None:
|
|
episode = Episode(
|
|
title="Nothing",
|
|
published=parse_feed(_FEED)[0].published,
|
|
enclosure_url="http://example.com/ep.mp3",
|
|
enclosure_type="audio/mpeg",
|
|
)
|
|
|
|
async def handler(request: httpx2.Request) -> httpx2.Response: # pragma: no cover
|
|
raise AssertionError("must not make a request with neither a cover nor a link")
|
|
|
|
async with httpx2.AsyncClient(transport=httpx2.MockTransport(handler)) as client:
|
|
cover = await resolve_episode_cover(client, episode)
|
|
|
|
assert cover is None
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_sync_show_downloads_the_episode_cover_alongside_the_audio(tmp_path: Path) -> None:
|
|
folder = tmp_path / "show"
|
|
folder.mkdir()
|
|
|
|
def handler(request: httpx2.Request) -> httpx2.Response:
|
|
url = str(request.url)
|
|
if url == "http://example.com/feed.xml":
|
|
return httpx2.Response(200, content=_FEED_WITH_COVERS)
|
|
if url == "http://example.com/unique.jpg":
|
|
return httpx2.Response(200, content=b"cover-bytes")
|
|
return httpx2.Response(200, content=b"audio-bytes")
|
|
|
|
async with httpx2.AsyncClient(transport=httpx2.MockTransport(handler)) as client:
|
|
changed = await sync_show(client, folder, "http://example.com/feed.xml")
|
|
|
|
assert changed is True
|
|
assert (folder / "20250101 - Unique Cover.jpg").read_bytes() == b"cover-bytes"
|
|
# No cover file for the episode whose image just repeats the channel's.
|
|
assert not (folder / "20250102 - Same As Channel.jpg").exists()
|
|
assert list(folder.glob(".downloading-*.tmp")) == []
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_sync_show_survives_a_failed_cover_download(tmp_path: Path) -> None:
|
|
folder = tmp_path / "show"
|
|
folder.mkdir()
|
|
|
|
def handler(request: httpx2.Request) -> httpx2.Response:
|
|
url = str(request.url)
|
|
if url == "http://example.com/feed.xml":
|
|
return httpx2.Response(200, content=_FEED_WITH_COVERS)
|
|
if url == "http://example.com/unique.jpg":
|
|
return httpx2.Response(404)
|
|
return httpx2.Response(200, content=b"audio-bytes")
|
|
|
|
async with httpx2.AsyncClient(transport=httpx2.MockTransport(handler)) as client:
|
|
changed = await sync_show(client, folder, "http://example.com/feed.xml")
|
|
|
|
# The episode itself still downloaded fine - only its cover failed.
|
|
assert changed is True
|
|
assert (folder / "20250101 - Unique Cover.mp3").exists()
|
|
assert not (folder / "20250101 - Unique Cover.jpg").exists()
|
|
|
|
|
|
_FEED_WITH_LINKS = b"""<?xml version="1.0"?>
|
|
<rss version="2.0">
|
|
<channel>
|
|
<title>Link Show</title>
|
|
<link>http://example.com/show</link>
|
|
<item>
|
|
<title>Old One</title>
|
|
<link>http://example.com/episodes/old</link>
|
|
<pubDate>Wed, 01 Jan 2025 08:00:00 GMT</pubDate>
|
|
<enclosure url="http://example.com/ep1.mp3" length="100" type="audio/mpeg" />
|
|
<guid>ep1</guid>
|
|
</item>
|
|
</channel>
|
|
</rss>
|
|
"""
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_sync_show_backfills_the_cover_of_an_episode_already_on_disk(
|
|
tmp_path: Path,
|
|
) -> None:
|
|
# Placed by hand (or downloaded before covers were fetched): the audio is there,
|
|
# so nothing is downloaded - but its cover still has to be fetched.
|
|
folder = tmp_path / "show"
|
|
folder.mkdir()
|
|
(folder / "20250101 - Old One.mp3").write_bytes(b"audio-bytes")
|
|
requests: list[str] = []
|
|
|
|
def handler(request: httpx2.Request) -> httpx2.Response:
|
|
url = str(request.url)
|
|
requests.append(url)
|
|
if url == "http://example.com/feed.xml":
|
|
return httpx2.Response(200, content=_FEED_WITH_LINKS)
|
|
if url == "http://example.com/episodes/old":
|
|
html = '<meta property="og:image" content="http://example.com/old.png">'
|
|
return httpx2.Response(200, text=html)
|
|
if url == "http://example.com/old.png":
|
|
return httpx2.Response(200, content=b"cover-bytes")
|
|
return httpx2.Response(500)
|
|
|
|
async with httpx2.AsyncClient(transport=httpx2.MockTransport(handler)) as client:
|
|
changed = await sync_show(client, folder, "http://example.com/feed.xml")
|
|
requests.clear()
|
|
changed_again = await sync_show(client, folder, "http://example.com/feed.xml")
|
|
|
|
assert changed is True
|
|
assert (folder / "20250101 - Old One.png").read_bytes() == b"cover-bytes"
|
|
# Once the sidecar exists, a later poll doesn't go looking for it again.
|
|
assert changed_again is False
|
|
assert requests == ["http://example.com/feed.xml"]
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_sync_show_does_not_look_for_a_missing_cover_again_within_the_backoff(
|
|
tmp_path: Path,
|
|
) -> None:
|
|
folder = tmp_path / "show"
|
|
folder.mkdir()
|
|
(folder / "20250101 - Old One.mp3").write_bytes(b"audio-bytes")
|
|
page_requests: list[str] = []
|
|
|
|
def handler(request: httpx2.Request) -> httpx2.Response:
|
|
url = str(request.url)
|
|
if url == "http://example.com/feed.xml":
|
|
return httpx2.Response(200, content=_FEED_WITH_LINKS)
|
|
page_requests.append(url)
|
|
return httpx2.Response(200, text="<html>no og:image here</html>")
|
|
|
|
async with httpx2.AsyncClient(transport=httpx2.MockTransport(handler)) as client:
|
|
changed = await sync_show(client, folder, "http://example.com/feed.xml")
|
|
await sync_show(client, folder, "http://example.com/feed.xml")
|
|
|
|
assert changed is False
|
|
assert page_requests == ["http://example.com/episodes/old"]
|
|
assert (folder / podcast_feeds._MISSING_COVERS_FILENAME).exists()
|
|
|
|
|
|
# ---------------------------------------------------------------------- video shows
|
|
|
|
|
|
def test_entry_enclosure_accepts_video_when_no_audio_is_offered() -> None:
|
|
entry = {"enclosures": [{"href": "http://example.com/ep.mp4", "type": "video/mpeg"}]}
|
|
assert _entry_enclosure(entry) == ("http://example.com/ep.mp4", "video/mpeg")
|
|
|
|
|
|
def test_entry_enclosure_prefers_audio_over_video_when_both_are_offered() -> None:
|
|
entry = {
|
|
"enclosures": [
|
|
{"href": "http://example.com/ep.mp4", "type": "video/mpeg"},
|
|
{"href": "http://example.com/ep.mp3", "type": "audio/mpeg"},
|
|
]
|
|
}
|
|
assert _entry_enclosure(entry) == ("http://example.com/ep.mp3", "audio/mpeg")
|
|
|
|
|
|
def test_parse_feed_keeps_a_video_only_episode() -> None:
|
|
episodes = parse_feed(_VIDEO_FEED)
|
|
assert episodes[0].enclosure_url == "http://example.com/ep1.mp4"
|
|
assert episodes[0].enclosure_type == "video/mpeg"
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_download_episode_raises_when_ffmpeg_is_not_installed(
|
|
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
|
|
) -> None:
|
|
monkeypatch.setattr(podcast_feeds.shutil, "which", lambda _: None)
|
|
folder = tmp_path / "show"
|
|
folder.mkdir()
|
|
episode = parse_feed(_VIDEO_FEED)[0]
|
|
|
|
async with httpx2.AsyncClient(
|
|
transport=httpx2.MockTransport(lambda _: httpx2.Response(200))
|
|
) as client:
|
|
with pytest.raises(AudioExtractionError):
|
|
await download_episode(client, folder, episode, "20250101 - Video Only.mp3")
|
|
|
|
assert list(folder.glob(".downloading-*.tmp")) == []
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_download_episode_raises_when_ffmpeg_exits_nonzero(
|
|
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
|
|
) -> None:
|
|
monkeypatch.setattr(podcast_feeds.shutil, "which", lambda _: "/usr/bin/ffmpeg")
|
|
|
|
class _FailingProcess:
|
|
returncode = 1
|
|
|
|
async def communicate(self) -> tuple[bytes, bytes]:
|
|
return b"", b"boom"
|
|
|
|
async def fake_exec(*args: object, **kwargs: object) -> _FailingProcess:
|
|
return _FailingProcess()
|
|
|
|
monkeypatch.setattr(podcast_feeds.asyncio, "create_subprocess_exec", fake_exec)
|
|
|
|
folder = tmp_path / "show"
|
|
folder.mkdir()
|
|
episode = parse_feed(_VIDEO_FEED)[0]
|
|
|
|
async with httpx2.AsyncClient(
|
|
transport=httpx2.MockTransport(lambda _: httpx2.Response(200))
|
|
) as client:
|
|
with pytest.raises(AudioExtractionError, match="boom"):
|
|
await download_episode(client, folder, episode, "20250101 - Video Only.mp3")
|
|
|
|
assert list(folder.glob(".downloading-*.tmp")) == []
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_download_episode_extracts_audio_via_ffmpeg(
|
|
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
|
|
) -> None:
|
|
monkeypatch.setattr(podcast_feeds.shutil, "which", lambda _: "/usr/bin/ffmpeg")
|
|
|
|
class _SucceedingProcess:
|
|
returncode = 0
|
|
|
|
async def communicate(self) -> tuple[bytes, bytes]:
|
|
return b"", b""
|
|
|
|
captured_args: list[object] = []
|
|
|
|
async def fake_exec(*args: object, **kwargs: object) -> _SucceedingProcess:
|
|
# ffmpeg's destination path is the last positional argument; write to it here
|
|
# to stand in for what the real binary would have produced.
|
|
captured_args.extend(args)
|
|
Path(str(args[-1])).write_bytes(b"extracted-audio")
|
|
return _SucceedingProcess()
|
|
|
|
monkeypatch.setattr(podcast_feeds.asyncio, "create_subprocess_exec", fake_exec)
|
|
|
|
folder = tmp_path / "show"
|
|
folder.mkdir()
|
|
episode = parse_feed(_VIDEO_FEED)[0]
|
|
|
|
async with httpx2.AsyncClient(
|
|
transport=httpx2.MockTransport(lambda _: httpx2.Response(200))
|
|
) as client:
|
|
await download_episode(client, folder, episode, "20250101 - Video Only.mp3")
|
|
|
|
assert (folder / "20250101 - Video Only.mp3").read_bytes() == b"extracted-audio"
|
|
assert list(folder.glob(".downloading-*.tmp")) == []
|
|
# The destination path ends in ".tmp", not ".mp3" - ffmpeg can't guess the muxer
|
|
# from that extension, so the format must be passed explicitly.
|
|
assert captured_args[captured_args.index("-f") + 1] == "mp3"
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_sync_show_does_not_retry_a_failed_episode_within_the_backoff(
|
|
tmp_path: Path,
|
|
) -> None:
|
|
folder = tmp_path / "show"
|
|
folder.mkdir()
|
|
enclosure_requests: list[str] = []
|
|
|
|
def handler(request: httpx2.Request) -> httpx2.Response:
|
|
url = str(request.url)
|
|
if url == "http://example.com/feed.xml":
|
|
return httpx2.Response(200, content=_FEED)
|
|
enclosure_requests.append(url)
|
|
return httpx2.Response(404)
|
|
|
|
async with httpx2.AsyncClient(transport=httpx2.MockTransport(handler)) as client:
|
|
await sync_show(client, folder, "http://example.com/feed.xml")
|
|
await sync_show(client, folder, "http://example.com/feed.xml")
|
|
|
|
# Both episodes 404 on the first pass and get remembered; the second pass must not
|
|
# re-request either enclosure while the backoff is still in effect.
|
|
assert len(enclosure_requests) == 2
|
|
assert (folder / podcast_feeds._FAILED_DOWNLOADS_FILENAME).exists()
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_sync_show_retries_a_failed_episode_once_the_backoff_expires(
|
|
tmp_path: Path,
|
|
) -> None:
|
|
folder = tmp_path / "show"
|
|
folder.mkdir()
|
|
enclosure_requests: list[str] = []
|
|
|
|
def handler(request: httpx2.Request) -> httpx2.Response:
|
|
url = str(request.url)
|
|
if url == "http://example.com/feed.xml":
|
|
return httpx2.Response(200, content=_FEED)
|
|
enclosure_requests.append(url)
|
|
return httpx2.Response(404)
|
|
|
|
async with httpx2.AsyncClient(transport=httpx2.MockTransport(handler)) as client:
|
|
await sync_show(client, folder, "http://example.com/feed.xml")
|
|
|
|
# Backdate both failure records past the backoff window, simulating that a
|
|
# week has gone by since the first attempt.
|
|
record_path = folder / podcast_feeds._FAILED_DOWNLOADS_FILENAME
|
|
stale = datetime.now(UTC) - podcast_feeds._RETRY_BACKOFF - timedelta(seconds=1)
|
|
record = json.loads(record_path.read_text(encoding="utf-8"))
|
|
record_path.write_text(
|
|
json.dumps({filename: stale.isoformat() for filename in record}), encoding="utf-8"
|
|
)
|
|
|
|
await sync_show(client, folder, "http://example.com/feed.xml")
|
|
|
|
assert len(enclosure_requests) == 4
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_sync_show_clears_the_failure_record_once_an_episode_succeeds(
|
|
tmp_path: Path,
|
|
) -> None:
|
|
folder = tmp_path / "show"
|
|
folder.mkdir()
|
|
should_fail = True
|
|
|
|
def handler(request: httpx2.Request) -> httpx2.Response:
|
|
url = str(request.url)
|
|
if url == "http://example.com/feed.xml":
|
|
return httpx2.Response(200, content=_FEED)
|
|
if should_fail:
|
|
return httpx2.Response(404)
|
|
return httpx2.Response(200, content=b"audio-bytes")
|
|
|
|
async with httpx2.AsyncClient(transport=httpx2.MockTransport(handler)) as client:
|
|
await sync_show(client, folder, "http://example.com/feed.xml")
|
|
assert (folder / podcast_feeds._FAILED_DOWNLOADS_FILENAME).exists()
|
|
|
|
should_fail = False
|
|
# Backdate the record so the retry isn't just skipped by the backoff window.
|
|
record_path = folder / podcast_feeds._FAILED_DOWNLOADS_FILENAME
|
|
stale = datetime.now(UTC) - podcast_feeds._RETRY_BACKOFF - timedelta(seconds=1)
|
|
record = json.loads(record_path.read_text(encoding="utf-8"))
|
|
record_path.write_text(
|
|
json.dumps({filename: stale.isoformat() for filename in record}), encoding="utf-8"
|
|
)
|
|
|
|
changed = await sync_show(client, folder, "http://example.com/feed.xml")
|
|
|
|
assert changed is True
|
|
assert not (folder / podcast_feeds._FAILED_DOWNLOADS_FILENAME).exists()
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_sync_show_survives_a_corrupt_failure_record(tmp_path: Path) -> None:
|
|
folder = tmp_path / "show"
|
|
folder.mkdir()
|
|
(folder / podcast_feeds._FAILED_DOWNLOADS_FILENAME).write_text(
|
|
"not valid json", encoding="utf-8"
|
|
)
|
|
|
|
def handler(request: httpx2.Request) -> httpx2.Response:
|
|
if str(request.url) == "http://example.com/feed.xml":
|
|
return httpx2.Response(200, content=_FEED)
|
|
return httpx2.Response(200, content=b"audio-bytes")
|
|
|
|
async with httpx2.AsyncClient(transport=httpx2.MockTransport(handler)) as client:
|
|
changed = await sync_show(client, folder, "http://example.com/feed.xml")
|
|
|
|
assert changed is True
|
|
assert (folder / "20250101 - Episode One.mp3").exists()
|
|
|
|
|
|
# --------------------------------------------------------------- episode retention
|
|
|
|
|
|
def _episode(day: int, *, title: str | None = None) -> Episode:
|
|
return Episode(
|
|
title=title or f"Episode {day}",
|
|
published=datetime(2025, 1, day, tzinfo=UTC),
|
|
enclosure_url=f"http://example.com/ep{day}.mp3",
|
|
enclosure_type="audio/mpeg",
|
|
)
|
|
|
|
|
|
def _write_episode(folder: Path, name: str) -> Path:
|
|
path = folder / name
|
|
path.write_bytes(b"audio")
|
|
return path
|
|
|
|
|
|
def test_newest_episodes_takes_the_most_recent_and_orders_them_newest_first() -> None:
|
|
episodes = [_episode(1), _episode(5), _episode(3)]
|
|
assert [e.published.day for e in newest_episodes(episodes, 2)] == [5, 3]
|
|
|
|
|
|
def test_newest_episodes_keeps_everything_when_the_limit_is_none() -> None:
|
|
episodes = [_episode(1), _episode(5), _episode(3)]
|
|
assert len(newest_episodes(episodes, None)) == 3
|
|
|
|
|
|
def test_prune_show_deletes_only_the_oldest_beyond_the_limit(tmp_path: Path) -> None:
|
|
folder = tmp_path / "show"
|
|
folder.mkdir()
|
|
for day in range(1, 6):
|
|
_write_episode(folder, f"2025010{day} - Episode {day}.mp3")
|
|
|
|
assert prune_show(folder, 2) == 3
|
|
|
|
remaining = sorted(path.name for path in folder.iterdir())
|
|
assert remaining == ["20250104 - Episode 4.mp3", "20250105 - Episode 5.mp3"]
|
|
|
|
|
|
def test_prune_show_is_a_noop_below_the_limit(tmp_path: Path) -> None:
|
|
folder = tmp_path / "show"
|
|
folder.mkdir()
|
|
_write_episode(folder, "20250101 - Episode 1.mp3")
|
|
assert prune_show(folder, 50) == 0
|
|
assert prune_show(folder, None) == 0
|
|
assert (folder / "20250101 - Episode 1.mp3").exists()
|
|
|
|
|
|
def test_prune_show_takes_the_sidecar_cover_with_the_episode(tmp_path: Path) -> None:
|
|
folder = tmp_path / "show"
|
|
folder.mkdir()
|
|
_write_episode(folder, "20250101 - Old.mp3")
|
|
(folder / "20250101 - Old.jpg").write_bytes(b"art")
|
|
_write_episode(folder, "20250102 - New.mp3")
|
|
(folder / "20250102 - New.jpg").write_bytes(b"art")
|
|
|
|
assert prune_show(folder, 1) == 1
|
|
|
|
assert not (folder / "20250101 - Old.jpg").exists()
|
|
assert (folder / "20250102 - New.jpg").exists()
|
|
|
|
|
|
def test_prune_show_leaves_everything_it_did_not_name(tmp_path: Path) -> None:
|
|
"""A folder holds more than episodes, and a hand-placed file is not ours to delete."""
|
|
folder = tmp_path / "show"
|
|
folder.mkdir()
|
|
for day in range(1, 4):
|
|
_write_episode(folder, f"2025010{day} - Episode {day}.mp3")
|
|
(folder / "feed.txt").write_text("http://example.com/feed.xml\n")
|
|
(folder / "folder.jpg").write_bytes(b"art")
|
|
(folder / ".failed-downloads.json").write_text("{}")
|
|
_write_episode(folder, "Grandpa's tape.mp3")
|
|
|
|
prune_show(folder, 1)
|
|
|
|
assert (folder / "feed.txt").exists()
|
|
assert (folder / "folder.jpg").exists()
|
|
assert (folder / ".failed-downloads.json").exists()
|
|
assert (folder / "Grandpa's tape.mp3").exists()
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_sync_show_prunes_to_the_limit_after_downloading(tmp_path: Path) -> None:
|
|
folder = tmp_path / "show"
|
|
folder.mkdir()
|
|
|
|
def handler(request: httpx2.Request) -> httpx2.Response:
|
|
if str(request.url) == "http://example.com/feed.xml":
|
|
return httpx2.Response(200, content=_FEED)
|
|
return httpx2.Response(200, content=b"audio-bytes")
|
|
|
|
async with httpx2.AsyncClient(transport=httpx2.MockTransport(handler)) as client:
|
|
changed = await sync_show(client, folder, "http://example.com/feed.xml", keep=1)
|
|
|
|
assert changed is True
|
|
episodes = sorted(path.name for path in folder.iterdir() if path.suffix == ".mp3")
|
|
assert episodes == ["20250103 - Episode Two_ Special_Chars_.mp3"]
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_sync_show_does_not_redownload_what_it_pruned(tmp_path: Path) -> None:
|
|
"""The whole point of capping downloads as well as deletions.
|
|
|
|
Prune to the newest N but fetch everything the feed offers, and each poll would
|
|
re-download the episodes the previous one deleted - forever, at full size.
|
|
"""
|
|
folder = tmp_path / "show"
|
|
folder.mkdir()
|
|
downloads: list[str] = []
|
|
|
|
def handler(request: httpx2.Request) -> httpx2.Response:
|
|
if str(request.url) == "http://example.com/feed.xml":
|
|
return httpx2.Response(200, content=_FEED)
|
|
downloads.append(str(request.url))
|
|
return httpx2.Response(200, content=b"audio-bytes")
|
|
|
|
async with httpx2.AsyncClient(transport=httpx2.MockTransport(handler)) as client:
|
|
await sync_show(client, folder, "http://example.com/feed.xml", keep=1)
|
|
first_pass = len(downloads)
|
|
changed_again = await sync_show(client, folder, "http://example.com/feed.xml", keep=1)
|
|
|
|
assert changed_again is False
|
|
assert len(downloads) == first_pass
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_sync_show_reports_a_change_when_it_only_pruned(tmp_path: Path) -> None:
|
|
"""A deletion needs a rescan as much as a download does, or the library goes on
|
|
offering tracks whose files are gone."""
|
|
folder = tmp_path / "show"
|
|
folder.mkdir()
|
|
_write_episode(folder, "20250101 - Episode One.mp3")
|
|
_write_episode(folder, "20250103 - Episode Two_ Special_Chars_.mp3")
|
|
|
|
def handler(request: httpx2.Request) -> httpx2.Response:
|
|
return httpx2.Response(200, content=_FEED)
|
|
|
|
async with httpx2.AsyncClient(transport=httpx2.MockTransport(handler)) as client:
|
|
changed = await sync_show(client, folder, "http://example.com/feed.xml", keep=1)
|
|
|
|
assert changed is True
|
|
assert not (folder / "20250101 - Episode One.mp3").exists()
|