Martin Bauer
fa3d92189c
A show that has published for years is unbounded: GEOlino Spezial alone is 358 episodes and 5.8 GB, and the device it syncs onto is a 30 GB SD card that also holds the rest of the library. Nothing stopped the 6-hourly poll from eventually filling it. Cap each show's folder at general.podcast_episode_limit (default 50, null to keep everything), pruning the oldest past that after each sync pass. The same limit caps what is downloaded, and it has to be one number for both. Prune to the newest N but keep fetching everything the feed offers, and every poll would re-download exactly the episodes the previous one deleted - forever, at full size, since missing_episodes() decides purely from what is on disk. There is a test for that specific loop. Pruning only touches files named the way this module names them (YYYYMMDD - Title.ext), so feed.txt, folder.jpg, the failed-download record and anything placed by hand are all left alone; a parse that fails means "not ours", not "delete it". An episode's sidecar cover goes with it. A pass that only deleted still reports a change, because the library needs the rescan just as much as it does after a download. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
780 lines
28 KiB
Python
780 lines
28 KiB
Python
"""Podcast feed polling: finding shows, parsing feeds, and downloading only what's new."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
from datetime import UTC, datetime, timedelta
|
|
from pathlib import Path
|
|
|
|
import httpx2
|
|
import pytest
|
|
|
|
from musicmouse.library import podcast_feeds
|
|
from musicmouse.library.podcast_feeds import (
|
|
AudioExtractionError,
|
|
Episode,
|
|
_entry_enclosure,
|
|
download_episode,
|
|
episode_cover_filename,
|
|
episode_filename,
|
|
find_feed_shows,
|
|
missing_episodes,
|
|
newest_episodes,
|
|
parse_feed,
|
|
prune_show,
|
|
resolve_episode_cover,
|
|
sync_all_shows,
|
|
sync_show,
|
|
)
|
|
|
|
_FEED = b"""<?xml version="1.0"?>
|
|
<rss version="2.0">
|
|
<channel>
|
|
<title>Test Show</title>
|
|
<item>
|
|
<title>Episode One</title>
|
|
<pubDate>Wed, 01 Jan 2025 08:00:00 GMT</pubDate>
|
|
<enclosure url="http://example.com/ep1.mp3" length="100" type="audio/mpeg" />
|
|
<guid>ep1</guid>
|
|
</item>
|
|
<item>
|
|
<title>No Audio Here</title>
|
|
<pubDate>Thu, 02 Jan 2025 08:00:00 GMT</pubDate>
|
|
<guid>ep2</guid>
|
|
</item>
|
|
<item>
|
|
<title>Episode Two: Special/Chars?</title>
|
|
<pubDate>Fri, 03 Jan 2025 08:00:00 GMT</pubDate>
|
|
<enclosure url="http://example.com/redirect?id=2" length="200" type="audio/mpeg" />
|
|
<guid>ep3</guid>
|
|
</item>
|
|
</channel>
|
|
</rss>
|
|
"""
|
|
|
|
_FEED_WITH_COVERS = b"""<?xml version="1.0"?>
|
|
<rss version="2.0" xmlns:itunes="http://www.itunes.com/dtds/podcast-1.0.dtd">
|
|
<channel>
|
|
<title>Cover Show</title>
|
|
<link>http://example.com/show</link>
|
|
<itunes:image href="http://example.com/channel.jpg" />
|
|
<item>
|
|
<title>Unique Cover</title>
|
|
<link>http://example.com/episodes/unique</link>
|
|
<pubDate>Wed, 01 Jan 2025 08:00:00 GMT</pubDate>
|
|
<itunes:image href="http://example.com/unique.jpg" />
|
|
<enclosure url="http://example.com/ep1.mp3" length="100" type="audio/mpeg" />
|
|
<guid>ep1</guid>
|
|
</item>
|
|
<item>
|
|
<title>Same As Channel</title>
|
|
<link>http://example.com/show</link>
|
|
<pubDate>Thu, 02 Jan 2025 08:00:00 GMT</pubDate>
|
|
<itunes:image href="http://example.com/channel.jpg" />
|
|
<enclosure url="http://example.com/ep2.mp3" length="100" type="audio/mpeg" />
|
|
<guid>ep2</guid>
|
|
</item>
|
|
</channel>
|
|
</rss>
|
|
"""
|
|
|
|
_VIDEO_FEED = b"""<?xml version="1.0"?>
|
|
<rss version="2.0">
|
|
<channel>
|
|
<title>Video Show</title>
|
|
<item>
|
|
<title>Video Only</title>
|
|
<pubDate>Wed, 01 Jan 2025 08:00:00 GMT</pubDate>
|
|
<enclosure url="http://example.com/ep1.mp4" length="100" type="video/mpeg" />
|
|
<guid>ep1</guid>
|
|
</item>
|
|
</channel>
|
|
</rss>
|
|
"""
|
|
|
|
|
|
def _root_with_show(
|
|
tmp_path: Path, *, feed_url: str | None = "http://example.com/feed.xml"
|
|
) -> Path:
|
|
root = tmp_path / "music"
|
|
show = root / "Kinderpodcasts" / "Test Show"
|
|
show.mkdir(parents=True)
|
|
if feed_url is not None:
|
|
(show / "feed.txt").write_text(f"{feed_url}\n")
|
|
(root / "Kinderpodcasts" / "No Feed Show").mkdir(parents=True)
|
|
# A non-episode section must never be scanned for feed markers.
|
|
music = root / "Musik" / "Some Album"
|
|
music.mkdir(parents=True)
|
|
(music / "feed.txt").write_text("http://example.com/should-be-ignored.xml\n")
|
|
return root
|
|
|
|
|
|
def test_find_feed_shows_only_picks_up_episode_sections_with_a_marker(tmp_path: Path) -> None:
|
|
root = _root_with_show(tmp_path)
|
|
found = find_feed_shows(root)
|
|
assert found == [
|
|
("Kinderpodcasts", root / "Kinderpodcasts" / "Test Show", "http://example.com/feed.xml")
|
|
]
|
|
|
|
|
|
def test_find_feed_shows_skips_an_empty_marker(tmp_path: Path) -> None:
|
|
root = tmp_path / "music"
|
|
show = root / "Kinderpodcasts" / "Empty"
|
|
show.mkdir(parents=True)
|
|
(show / "feed.txt").write_text("\n\n")
|
|
assert find_feed_shows(root) == []
|
|
|
|
|
|
def test_parse_feed_keeps_only_entries_with_an_audio_enclosure() -> None:
|
|
episodes = parse_feed(_FEED)
|
|
assert [episode.title for episode in episodes] == ["Episode One", "Episode Two: Special/Chars?"]
|
|
assert episodes[0].enclosure_url == "http://example.com/ep1.mp3"
|
|
assert episodes[0].published.year == 2025
|
|
assert episodes[0].published.month == 1
|
|
assert episodes[0].published.day == 1
|
|
|
|
|
|
def test_episode_filename_matches_the_hand_placed_convention() -> None:
|
|
episode = parse_feed(_FEED)[0]
|
|
assert episode_filename(
|
|
episode.published, episode.title, episode.enclosure_type, episode.enclosure_url
|
|
) == "20250101 - Episode One.mp3"
|
|
|
|
|
|
def test_episode_filename_sanitizes_illegal_characters_and_falls_back_on_type() -> None:
|
|
episode = parse_feed(_FEED)[1]
|
|
# No file suffix on the (redirect-style) URL, so the extension comes from the
|
|
# enclosure's content type instead.
|
|
name = episode_filename(
|
|
episode.published, episode.title, episode.enclosure_type, episode.enclosure_url
|
|
)
|
|
# Illegal characters become "_" rather than being dropped, matching the
|
|
# convention every episode already on disk (downloaded by hand via the
|
|
# `podcast-dl` CLI) was named with.
|
|
assert name == "20250103 - Episode Two_ Special_Chars_.mp3"
|
|
|
|
|
|
def test_episode_filename_matches_the_podcast_dl_convention_on_a_real_title() -> None:
|
|
# Pinned against a real file already in the library: `podcast-dl` replaced the
|
|
# "?" with "_" rather than dropping it, so ours must too or the whole back
|
|
# catalog would look "new" the moment a feed marker is added for that show.
|
|
published = parse_feed(_FEED)[0].published
|
|
name = episode_filename(published, "Wie funktioniert Bogenschießen?", "audio/mpeg", "x.mp3")
|
|
assert name == "20250101 - Wie funktioniert Bogenschießen_.mp3"
|
|
|
|
|
|
def test_missing_episodes_skips_files_already_on_disk(tmp_path: Path) -> None:
|
|
folder = tmp_path / "show"
|
|
folder.mkdir()
|
|
episodes = parse_feed(_FEED)
|
|
(folder / "20250101 - Episode One.mp3").write_bytes(b"already here")
|
|
|
|
pending = missing_episodes(folder, episodes)
|
|
|
|
assert [episode.title for episode, _ in pending] == ["Episode Two: Special/Chars?"]
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_download_episode_writes_via_a_temp_file(tmp_path: Path) -> None:
|
|
folder = tmp_path / "show"
|
|
folder.mkdir()
|
|
episode = Episode(
|
|
title="Episode One",
|
|
published=parse_feed(_FEED)[0].published,
|
|
enclosure_url="http://example.com/ep1.mp3",
|
|
enclosure_type="audio/mpeg",
|
|
)
|
|
|
|
def handler(request: httpx2.Request) -> httpx2.Response:
|
|
return httpx2.Response(200, content=b"audio-bytes")
|
|
|
|
async with httpx2.AsyncClient(transport=httpx2.MockTransport(handler)) as client:
|
|
await download_episode(client, folder, episode, "20250101 - Episode One.mp3")
|
|
|
|
assert (folder / "20250101 - Episode One.mp3").read_bytes() == b"audio-bytes"
|
|
assert list(folder.glob(".downloading-*.tmp")) == []
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_sync_show_downloads_new_episodes_and_is_a_noop_on_rerun(tmp_path: Path) -> None:
|
|
folder = tmp_path / "show"
|
|
folder.mkdir()
|
|
|
|
def handler(request: httpx2.Request) -> httpx2.Response:
|
|
if str(request.url) == "http://example.com/feed.xml":
|
|
return httpx2.Response(200, content=_FEED)
|
|
return httpx2.Response(200, content=b"audio-bytes")
|
|
|
|
async with httpx2.AsyncClient(transport=httpx2.MockTransport(handler)) as client:
|
|
changed_first = await sync_show(client, folder, "http://example.com/feed.xml")
|
|
changed_second = await sync_show(client, folder, "http://example.com/feed.xml")
|
|
|
|
assert changed_first is True
|
|
assert changed_second is False
|
|
assert (folder / "20250101 - Episode One.mp3").exists()
|
|
assert (folder / "20250103 - Episode Two_ Special_Chars_.mp3").exists()
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_sync_show_survives_an_unreachable_feed(tmp_path: Path) -> None:
|
|
folder = tmp_path / "show"
|
|
folder.mkdir()
|
|
|
|
def handler(request: httpx2.Request) -> httpx2.Response:
|
|
raise httpx2.ConnectError("mock: connection refused")
|
|
|
|
async with httpx2.AsyncClient(transport=httpx2.MockTransport(handler)) as client:
|
|
changed = await sync_show(client, folder, "http://example.com/feed.xml")
|
|
|
|
assert changed is False
|
|
assert list(folder.iterdir()) == []
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_sync_show_survives_a_broken_enclosure(tmp_path: Path) -> None:
|
|
folder = tmp_path / "show"
|
|
folder.mkdir()
|
|
|
|
def handler(request: httpx2.Request) -> httpx2.Response:
|
|
if str(request.url) == "http://example.com/feed.xml":
|
|
return httpx2.Response(200, content=_FEED)
|
|
return httpx2.Response(404)
|
|
|
|
async with httpx2.AsyncClient(transport=httpx2.MockTransport(handler)) as client:
|
|
changed = await sync_show(client, folder, "http://example.com/feed.xml")
|
|
|
|
# Both enclosures 404, so nothing landed - but the pass still completed cleanly.
|
|
assert changed is False
|
|
assert list(folder.glob("*.mp3")) == []
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_sync_all_shows_keeps_going_past_one_broken_show(tmp_path: Path) -> None:
|
|
root = tmp_path / "music"
|
|
good = root / "Kinderpodcasts" / "Good Show"
|
|
good.mkdir(parents=True)
|
|
(good / "feed.txt").write_text("http://example.com/feed.xml\n")
|
|
bad = root / "Kinderpodcasts" / "Bad Show"
|
|
bad.mkdir(parents=True)
|
|
(bad / "feed.txt").write_text("http://example.com/broken.xml\n")
|
|
|
|
def handler(request: httpx2.Request) -> httpx2.Response:
|
|
url = str(request.url)
|
|
if url == "http://example.com/feed.xml":
|
|
return httpx2.Response(200, content=_FEED)
|
|
if url == "http://example.com/broken.xml":
|
|
raise httpx2.ConnectError("mock: connection refused")
|
|
return httpx2.Response(200, content=b"audio-bytes")
|
|
|
|
async with httpx2.AsyncClient(transport=httpx2.MockTransport(handler)) as client:
|
|
changed = await sync_all_shows(client, root)
|
|
|
|
assert changed is True
|
|
assert list(good.glob("*.mp3"))
|
|
assert list(bad.glob("*.mp3")) == []
|
|
|
|
|
|
# ------------------------------------------------------------------- per-episode art
|
|
|
|
|
|
def test_parse_feed_extracts_cover_and_link_only_when_distinct_from_the_channel() -> None:
|
|
unique, same = parse_feed(_FEED_WITH_COVERS)
|
|
assert unique.cover_url == "http://example.com/unique.jpg"
|
|
assert unique.link == "http://example.com/episodes/unique"
|
|
# This item's image/link both just repeat the channel's own - not real per-episode
|
|
# art, so both come out None rather than a false positive.
|
|
assert same.cover_url is None
|
|
assert same.link is None
|
|
|
|
|
|
def test_episode_cover_filename_shares_the_audio_stem_and_takes_the_url_extension() -> None:
|
|
episode = parse_feed(_FEED_WITH_COVERS)[0]
|
|
name = episode_cover_filename(episode.published, episode.title, episode.cover_url)
|
|
assert name == "20250101 - Unique Cover.jpg"
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_resolve_episode_cover_prefers_the_feed_image_without_a_request() -> None:
|
|
episode = parse_feed(_FEED_WITH_COVERS)[0]
|
|
|
|
def handler(request: httpx2.Request) -> httpx2.Response:
|
|
raise AssertionError("must not fetch the episode page when cover_url is already set")
|
|
|
|
async with httpx2.AsyncClient(transport=httpx2.MockTransport(handler)) as client:
|
|
cover = await resolve_episode_cover(client, episode)
|
|
|
|
assert cover == "http://example.com/unique.jpg"
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_resolve_episode_cover_falls_back_to_og_image_on_the_linked_page() -> None:
|
|
episode = Episode(
|
|
title="No Feed Image",
|
|
published=parse_feed(_FEED)[0].published,
|
|
enclosure_url="http://example.com/ep.mp3",
|
|
enclosure_type="audio/mpeg",
|
|
link="http://example.com/episode-page",
|
|
)
|
|
|
|
def handler(request: httpx2.Request) -> httpx2.Response:
|
|
return httpx2.Response(
|
|
200,
|
|
content=b'<html><head>'
|
|
b'<meta property="og:image" content="http://example.com/scraped.jpg"/>'
|
|
b"</head></html>",
|
|
)
|
|
|
|
async with httpx2.AsyncClient(transport=httpx2.MockTransport(handler)) as client:
|
|
cover = await resolve_episode_cover(client, episode)
|
|
|
|
assert cover == "http://example.com/scraped.jpg"
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_resolve_episode_cover_returns_none_when_the_link_is_unreachable() -> None:
|
|
episode = Episode(
|
|
title="Broken Link",
|
|
published=parse_feed(_FEED)[0].published,
|
|
enclosure_url="http://example.com/ep.mp3",
|
|
enclosure_type="audio/mpeg",
|
|
link="http://example.com/missing",
|
|
)
|
|
|
|
async with httpx2.AsyncClient(
|
|
transport=httpx2.MockTransport(lambda _: httpx2.Response(404))
|
|
) as client:
|
|
cover = await resolve_episode_cover(client, episode)
|
|
|
|
assert cover is None
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_resolve_episode_cover_returns_none_without_a_cover_or_link() -> None:
|
|
episode = Episode(
|
|
title="Nothing",
|
|
published=parse_feed(_FEED)[0].published,
|
|
enclosure_url="http://example.com/ep.mp3",
|
|
enclosure_type="audio/mpeg",
|
|
)
|
|
|
|
async def handler(request: httpx2.Request) -> httpx2.Response: # pragma: no cover
|
|
raise AssertionError("must not make a request with neither a cover nor a link")
|
|
|
|
async with httpx2.AsyncClient(transport=httpx2.MockTransport(handler)) as client:
|
|
cover = await resolve_episode_cover(client, episode)
|
|
|
|
assert cover is None
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_sync_show_downloads_the_episode_cover_alongside_the_audio(tmp_path: Path) -> None:
|
|
folder = tmp_path / "show"
|
|
folder.mkdir()
|
|
|
|
def handler(request: httpx2.Request) -> httpx2.Response:
|
|
url = str(request.url)
|
|
if url == "http://example.com/feed.xml":
|
|
return httpx2.Response(200, content=_FEED_WITH_COVERS)
|
|
if url == "http://example.com/unique.jpg":
|
|
return httpx2.Response(200, content=b"cover-bytes")
|
|
return httpx2.Response(200, content=b"audio-bytes")
|
|
|
|
async with httpx2.AsyncClient(transport=httpx2.MockTransport(handler)) as client:
|
|
changed = await sync_show(client, folder, "http://example.com/feed.xml")
|
|
|
|
assert changed is True
|
|
assert (folder / "20250101 - Unique Cover.jpg").read_bytes() == b"cover-bytes"
|
|
# No cover file for the episode whose image just repeats the channel's.
|
|
assert not (folder / "20250102 - Same As Channel.jpg").exists()
|
|
assert list(folder.glob(".downloading-*.tmp")) == []
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_sync_show_survives_a_failed_cover_download(tmp_path: Path) -> None:
|
|
folder = tmp_path / "show"
|
|
folder.mkdir()
|
|
|
|
def handler(request: httpx2.Request) -> httpx2.Response:
|
|
url = str(request.url)
|
|
if url == "http://example.com/feed.xml":
|
|
return httpx2.Response(200, content=_FEED_WITH_COVERS)
|
|
if url == "http://example.com/unique.jpg":
|
|
return httpx2.Response(404)
|
|
return httpx2.Response(200, content=b"audio-bytes")
|
|
|
|
async with httpx2.AsyncClient(transport=httpx2.MockTransport(handler)) as client:
|
|
changed = await sync_show(client, folder, "http://example.com/feed.xml")
|
|
|
|
# The episode itself still downloaded fine - only its cover failed.
|
|
assert changed is True
|
|
assert (folder / "20250101 - Unique Cover.mp3").exists()
|
|
assert not (folder / "20250101 - Unique Cover.jpg").exists()
|
|
|
|
|
|
# ---------------------------------------------------------------------- video shows
|
|
|
|
|
|
def test_entry_enclosure_accepts_video_when_no_audio_is_offered() -> None:
|
|
entry = {"enclosures": [{"href": "http://example.com/ep.mp4", "type": "video/mpeg"}]}
|
|
assert _entry_enclosure(entry) == ("http://example.com/ep.mp4", "video/mpeg")
|
|
|
|
|
|
def test_entry_enclosure_prefers_audio_over_video_when_both_are_offered() -> None:
|
|
entry = {
|
|
"enclosures": [
|
|
{"href": "http://example.com/ep.mp4", "type": "video/mpeg"},
|
|
{"href": "http://example.com/ep.mp3", "type": "audio/mpeg"},
|
|
]
|
|
}
|
|
assert _entry_enclosure(entry) == ("http://example.com/ep.mp3", "audio/mpeg")
|
|
|
|
|
|
def test_parse_feed_keeps_a_video_only_episode() -> None:
|
|
episodes = parse_feed(_VIDEO_FEED)
|
|
assert episodes[0].enclosure_url == "http://example.com/ep1.mp4"
|
|
assert episodes[0].enclosure_type == "video/mpeg"
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_download_episode_raises_when_ffmpeg_is_not_installed(
|
|
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
|
|
) -> None:
|
|
monkeypatch.setattr(podcast_feeds.shutil, "which", lambda _: None)
|
|
folder = tmp_path / "show"
|
|
folder.mkdir()
|
|
episode = parse_feed(_VIDEO_FEED)[0]
|
|
|
|
async with httpx2.AsyncClient(
|
|
transport=httpx2.MockTransport(lambda _: httpx2.Response(200))
|
|
) as client:
|
|
with pytest.raises(AudioExtractionError):
|
|
await download_episode(client, folder, episode, "20250101 - Video Only.mp3")
|
|
|
|
assert list(folder.glob(".downloading-*.tmp")) == []
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_download_episode_raises_when_ffmpeg_exits_nonzero(
|
|
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
|
|
) -> None:
|
|
monkeypatch.setattr(podcast_feeds.shutil, "which", lambda _: "/usr/bin/ffmpeg")
|
|
|
|
class _FailingProcess:
|
|
returncode = 1
|
|
|
|
async def communicate(self) -> tuple[bytes, bytes]:
|
|
return b"", b"boom"
|
|
|
|
async def fake_exec(*args: object, **kwargs: object) -> _FailingProcess:
|
|
return _FailingProcess()
|
|
|
|
monkeypatch.setattr(podcast_feeds.asyncio, "create_subprocess_exec", fake_exec)
|
|
|
|
folder = tmp_path / "show"
|
|
folder.mkdir()
|
|
episode = parse_feed(_VIDEO_FEED)[0]
|
|
|
|
async with httpx2.AsyncClient(
|
|
transport=httpx2.MockTransport(lambda _: httpx2.Response(200))
|
|
) as client:
|
|
with pytest.raises(AudioExtractionError, match="boom"):
|
|
await download_episode(client, folder, episode, "20250101 - Video Only.mp3")
|
|
|
|
assert list(folder.glob(".downloading-*.tmp")) == []
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_download_episode_extracts_audio_via_ffmpeg(
|
|
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
|
|
) -> None:
|
|
monkeypatch.setattr(podcast_feeds.shutil, "which", lambda _: "/usr/bin/ffmpeg")
|
|
|
|
class _SucceedingProcess:
|
|
returncode = 0
|
|
|
|
async def communicate(self) -> tuple[bytes, bytes]:
|
|
return b"", b""
|
|
|
|
captured_args: list[object] = []
|
|
|
|
async def fake_exec(*args: object, **kwargs: object) -> _SucceedingProcess:
|
|
# ffmpeg's destination path is the last positional argument; write to it here
|
|
# to stand in for what the real binary would have produced.
|
|
captured_args.extend(args)
|
|
Path(str(args[-1])).write_bytes(b"extracted-audio")
|
|
return _SucceedingProcess()
|
|
|
|
monkeypatch.setattr(podcast_feeds.asyncio, "create_subprocess_exec", fake_exec)
|
|
|
|
folder = tmp_path / "show"
|
|
folder.mkdir()
|
|
episode = parse_feed(_VIDEO_FEED)[0]
|
|
|
|
async with httpx2.AsyncClient(
|
|
transport=httpx2.MockTransport(lambda _: httpx2.Response(200))
|
|
) as client:
|
|
await download_episode(client, folder, episode, "20250101 - Video Only.mp3")
|
|
|
|
assert (folder / "20250101 - Video Only.mp3").read_bytes() == b"extracted-audio"
|
|
assert list(folder.glob(".downloading-*.tmp")) == []
|
|
# The destination path ends in ".tmp", not ".mp3" - ffmpeg can't guess the muxer
|
|
# from that extension, so the format must be passed explicitly.
|
|
assert captured_args[captured_args.index("-f") + 1] == "mp3"
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_sync_show_does_not_retry_a_failed_episode_within_the_backoff(
|
|
tmp_path: Path,
|
|
) -> None:
|
|
folder = tmp_path / "show"
|
|
folder.mkdir()
|
|
enclosure_requests: list[str] = []
|
|
|
|
def handler(request: httpx2.Request) -> httpx2.Response:
|
|
url = str(request.url)
|
|
if url == "http://example.com/feed.xml":
|
|
return httpx2.Response(200, content=_FEED)
|
|
enclosure_requests.append(url)
|
|
return httpx2.Response(404)
|
|
|
|
async with httpx2.AsyncClient(transport=httpx2.MockTransport(handler)) as client:
|
|
await sync_show(client, folder, "http://example.com/feed.xml")
|
|
await sync_show(client, folder, "http://example.com/feed.xml")
|
|
|
|
# Both episodes 404 on the first pass and get remembered; the second pass must not
|
|
# re-request either enclosure while the backoff is still in effect.
|
|
assert len(enclosure_requests) == 2
|
|
assert (folder / podcast_feeds._FAILED_DOWNLOADS_FILENAME).exists()
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_sync_show_retries_a_failed_episode_once_the_backoff_expires(
|
|
tmp_path: Path,
|
|
) -> None:
|
|
folder = tmp_path / "show"
|
|
folder.mkdir()
|
|
enclosure_requests: list[str] = []
|
|
|
|
def handler(request: httpx2.Request) -> httpx2.Response:
|
|
url = str(request.url)
|
|
if url == "http://example.com/feed.xml":
|
|
return httpx2.Response(200, content=_FEED)
|
|
enclosure_requests.append(url)
|
|
return httpx2.Response(404)
|
|
|
|
async with httpx2.AsyncClient(transport=httpx2.MockTransport(handler)) as client:
|
|
await sync_show(client, folder, "http://example.com/feed.xml")
|
|
|
|
# Backdate both failure records past the backoff window, simulating that a
|
|
# week has gone by since the first attempt.
|
|
record_path = folder / podcast_feeds._FAILED_DOWNLOADS_FILENAME
|
|
stale = datetime.now(UTC) - podcast_feeds._RETRY_BACKOFF - timedelta(seconds=1)
|
|
record = json.loads(record_path.read_text(encoding="utf-8"))
|
|
record_path.write_text(
|
|
json.dumps({filename: stale.isoformat() for filename in record}), encoding="utf-8"
|
|
)
|
|
|
|
await sync_show(client, folder, "http://example.com/feed.xml")
|
|
|
|
assert len(enclosure_requests) == 4
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_sync_show_clears_the_failure_record_once_an_episode_succeeds(
|
|
tmp_path: Path,
|
|
) -> None:
|
|
folder = tmp_path / "show"
|
|
folder.mkdir()
|
|
should_fail = True
|
|
|
|
def handler(request: httpx2.Request) -> httpx2.Response:
|
|
url = str(request.url)
|
|
if url == "http://example.com/feed.xml":
|
|
return httpx2.Response(200, content=_FEED)
|
|
if should_fail:
|
|
return httpx2.Response(404)
|
|
return httpx2.Response(200, content=b"audio-bytes")
|
|
|
|
async with httpx2.AsyncClient(transport=httpx2.MockTransport(handler)) as client:
|
|
await sync_show(client, folder, "http://example.com/feed.xml")
|
|
assert (folder / podcast_feeds._FAILED_DOWNLOADS_FILENAME).exists()
|
|
|
|
should_fail = False
|
|
# Backdate the record so the retry isn't just skipped by the backoff window.
|
|
record_path = folder / podcast_feeds._FAILED_DOWNLOADS_FILENAME
|
|
stale = datetime.now(UTC) - podcast_feeds._RETRY_BACKOFF - timedelta(seconds=1)
|
|
record = json.loads(record_path.read_text(encoding="utf-8"))
|
|
record_path.write_text(
|
|
json.dumps({filename: stale.isoformat() for filename in record}), encoding="utf-8"
|
|
)
|
|
|
|
changed = await sync_show(client, folder, "http://example.com/feed.xml")
|
|
|
|
assert changed is True
|
|
assert not (folder / podcast_feeds._FAILED_DOWNLOADS_FILENAME).exists()
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_sync_show_survives_a_corrupt_failure_record(tmp_path: Path) -> None:
|
|
folder = tmp_path / "show"
|
|
folder.mkdir()
|
|
(folder / podcast_feeds._FAILED_DOWNLOADS_FILENAME).write_text(
|
|
"not valid json", encoding="utf-8"
|
|
)
|
|
|
|
def handler(request: httpx2.Request) -> httpx2.Response:
|
|
if str(request.url) == "http://example.com/feed.xml":
|
|
return httpx2.Response(200, content=_FEED)
|
|
return httpx2.Response(200, content=b"audio-bytes")
|
|
|
|
async with httpx2.AsyncClient(transport=httpx2.MockTransport(handler)) as client:
|
|
changed = await sync_show(client, folder, "http://example.com/feed.xml")
|
|
|
|
assert changed is True
|
|
assert (folder / "20250101 - Episode One.mp3").exists()
|
|
|
|
|
|
# --------------------------------------------------------------- episode retention
|
|
|
|
|
|
def _episode(day: int, *, title: str | None = None) -> Episode:
|
|
return Episode(
|
|
title=title or f"Episode {day}",
|
|
published=datetime(2025, 1, day, tzinfo=UTC),
|
|
enclosure_url=f"http://example.com/ep{day}.mp3",
|
|
enclosure_type="audio/mpeg",
|
|
)
|
|
|
|
|
|
def _write_episode(folder: Path, name: str) -> Path:
|
|
path = folder / name
|
|
path.write_bytes(b"audio")
|
|
return path
|
|
|
|
|
|
def test_newest_episodes_takes_the_most_recent_and_orders_them_newest_first() -> None:
|
|
episodes = [_episode(1), _episode(5), _episode(3)]
|
|
assert [e.published.day for e in newest_episodes(episodes, 2)] == [5, 3]
|
|
|
|
|
|
def test_newest_episodes_keeps_everything_when_the_limit_is_none() -> None:
|
|
episodes = [_episode(1), _episode(5), _episode(3)]
|
|
assert len(newest_episodes(episodes, None)) == 3
|
|
|
|
|
|
def test_prune_show_deletes_only_the_oldest_beyond_the_limit(tmp_path: Path) -> None:
|
|
folder = tmp_path / "show"
|
|
folder.mkdir()
|
|
for day in range(1, 6):
|
|
_write_episode(folder, f"2025010{day} - Episode {day}.mp3")
|
|
|
|
assert prune_show(folder, 2) == 3
|
|
|
|
remaining = sorted(path.name for path in folder.iterdir())
|
|
assert remaining == ["20250104 - Episode 4.mp3", "20250105 - Episode 5.mp3"]
|
|
|
|
|
|
def test_prune_show_is_a_noop_below_the_limit(tmp_path: Path) -> None:
|
|
folder = tmp_path / "show"
|
|
folder.mkdir()
|
|
_write_episode(folder, "20250101 - Episode 1.mp3")
|
|
assert prune_show(folder, 50) == 0
|
|
assert prune_show(folder, None) == 0
|
|
assert (folder / "20250101 - Episode 1.mp3").exists()
|
|
|
|
|
|
def test_prune_show_takes_the_sidecar_cover_with_the_episode(tmp_path: Path) -> None:
|
|
folder = tmp_path / "show"
|
|
folder.mkdir()
|
|
_write_episode(folder, "20250101 - Old.mp3")
|
|
(folder / "20250101 - Old.jpg").write_bytes(b"art")
|
|
_write_episode(folder, "20250102 - New.mp3")
|
|
(folder / "20250102 - New.jpg").write_bytes(b"art")
|
|
|
|
assert prune_show(folder, 1) == 1
|
|
|
|
assert not (folder / "20250101 - Old.jpg").exists()
|
|
assert (folder / "20250102 - New.jpg").exists()
|
|
|
|
|
|
def test_prune_show_leaves_everything_it_did_not_name(tmp_path: Path) -> None:
|
|
"""A folder holds more than episodes, and a hand-placed file is not ours to delete."""
|
|
folder = tmp_path / "show"
|
|
folder.mkdir()
|
|
for day in range(1, 4):
|
|
_write_episode(folder, f"2025010{day} - Episode {day}.mp3")
|
|
(folder / "feed.txt").write_text("http://example.com/feed.xml\n")
|
|
(folder / "folder.jpg").write_bytes(b"art")
|
|
(folder / ".failed-downloads.json").write_text("{}")
|
|
_write_episode(folder, "Grandpa's tape.mp3")
|
|
|
|
prune_show(folder, 1)
|
|
|
|
assert (folder / "feed.txt").exists()
|
|
assert (folder / "folder.jpg").exists()
|
|
assert (folder / ".failed-downloads.json").exists()
|
|
assert (folder / "Grandpa's tape.mp3").exists()
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_sync_show_prunes_to_the_limit_after_downloading(tmp_path: Path) -> None:
|
|
folder = tmp_path / "show"
|
|
folder.mkdir()
|
|
|
|
def handler(request: httpx2.Request) -> httpx2.Response:
|
|
if str(request.url) == "http://example.com/feed.xml":
|
|
return httpx2.Response(200, content=_FEED)
|
|
return httpx2.Response(200, content=b"audio-bytes")
|
|
|
|
async with httpx2.AsyncClient(transport=httpx2.MockTransport(handler)) as client:
|
|
changed = await sync_show(client, folder, "http://example.com/feed.xml", keep=1)
|
|
|
|
assert changed is True
|
|
episodes = sorted(path.name for path in folder.iterdir() if path.suffix == ".mp3")
|
|
assert episodes == ["20250103 - Episode Two_ Special_Chars_.mp3"]
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_sync_show_does_not_redownload_what_it_pruned(tmp_path: Path) -> None:
|
|
"""The whole point of capping downloads as well as deletions.
|
|
|
|
Prune to the newest N but fetch everything the feed offers, and each poll would
|
|
re-download the episodes the previous one deleted - forever, at full size.
|
|
"""
|
|
folder = tmp_path / "show"
|
|
folder.mkdir()
|
|
downloads: list[str] = []
|
|
|
|
def handler(request: httpx2.Request) -> httpx2.Response:
|
|
if str(request.url) == "http://example.com/feed.xml":
|
|
return httpx2.Response(200, content=_FEED)
|
|
downloads.append(str(request.url))
|
|
return httpx2.Response(200, content=b"audio-bytes")
|
|
|
|
async with httpx2.AsyncClient(transport=httpx2.MockTransport(handler)) as client:
|
|
await sync_show(client, folder, "http://example.com/feed.xml", keep=1)
|
|
first_pass = len(downloads)
|
|
changed_again = await sync_show(client, folder, "http://example.com/feed.xml", keep=1)
|
|
|
|
assert changed_again is False
|
|
assert len(downloads) == first_pass
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_sync_show_reports_a_change_when_it_only_pruned(tmp_path: Path) -> None:
|
|
"""A deletion needs a rescan as much as a download does, or the library goes on
|
|
offering tracks whose files are gone."""
|
|
folder = tmp_path / "show"
|
|
folder.mkdir()
|
|
_write_episode(folder, "20250101 - Episode One.mp3")
|
|
_write_episode(folder, "20250103 - Episode Two_ Special_Chars_.mp3")
|
|
|
|
def handler(request: httpx2.Request) -> httpx2.Response:
|
|
return httpx2.Response(200, content=_FEED)
|
|
|
|
async with httpx2.AsyncClient(transport=httpx2.MockTransport(handler)) as client:
|
|
changed = await sync_show(client, folder, "http://example.com/feed.xml", keep=1)
|
|
|
|
assert changed is True
|
|
assert not (folder / "20250101 - Episode One.mp3").exists()
|