"""The library scan: what ends up in the index, and what stays out of it.""" from __future__ import annotations import asyncio import contextlib import json import os import threading from collections.abc import Callable from dataclasses import replace from pathlib import Path import pytest from musicmouse.config import DEFAULT_AUDIO_EXTENSIONS, load_config from musicmouse.library import Album, Analyzer, MusicLibrary, default_worker_count from musicmouse.library.analysis import ANALYZER_VERSION, BeatGrid, TrackAnalysis, TrackCurves from musicmouse.library.cache import LibraryCache from musicmouse.library.colors import colors_from_id from musicmouse.library.models import album_id, track_key from tests.conftest import VALID_CONFIG, write_config, write_track EXTENSIONS = frozenset(DEFAULT_AUDIO_EXTENSIONS) async def build(config_dir: Path, *, analyzer: Analyzer | None = None) -> MusicLibrary: config = load_config(write_config(config_dir, VALID_CONFIG)) return await MusicLibrary.build( config.general.library.root, config.general.library.cache, EXTENSIONS, analyzer=analyzer, figure_kinds=config.figure_kinds, ) class FakeAnalyzer: """An `Analyzer` that does no DSP, so the worker's own behaviour - which tracks it touches, how it reacts to `is_busy`, what a raising analyzer does to the batch - can be tested without librosa.""" def __init__(self, *, version: int = ANALYZER_VERSION, fails: set[str] | None = None) -> None: self.version = version self._fails = fails or set() self.calls: list[Path] = [] def analyze(self, path: Path) -> tuple[TrackAnalysis, BeatGrid | None, TrackCurves | None]: self.calls.append(path) if path.name in self._fails: raise RuntimeError(f"boom: {path.name}") analysis = TrackAnalysis(version=self.version, tempo=100.0, energy=0.5) curves = TrackCurves(hop_seconds=1.0, energy=(0.5,), valence=(0.5,), drive=(0.5,)) return analysis, BeatGrid((0.1,), (1.0,)), curves class PidAnalyzer(FakeAnalyzer): """Reports *which process* analyzed each track, as `tempo`. The point of a pool is that this is never the process running the test - and a class at module scope is also the only kind of analyzer a worker can be handed, since it has to survive being pickled over to one. """ def analyze(self, path: Path) -> tuple[TrackAnalysis, BeatGrid | None, TrackCurves | None]: analysis, grid, curves = super().analyze(path) return replace(analysis, tempo=float(os.getpid())), grid, curves def album_named(library: MusicLibrary, title: str) -> Album: return next(album for album in library.albums if album.title == title) # ------------------------------------------------------------------------ structure async def test_every_section_is_scanned(config_dir: Path) -> None: library = await build(config_dir) assert {album.title for album in library.albums} == { "Fuchs", "Eule", "Kinderparty Lieder", "Conni in den Bergen", "Alt", "Neu", } async def test_figure_playlists_are_keyed_by_figure_name(config_dir: Path) -> None: library = await build(config_dir) playlists = library.figure_playlists() assert set(playlists) == {"fuchs", "eule"} assert len(playlists["fuchs"]) == 3 assert [track.path.name for track in playlists["fuchs"].tracks] == [ "00 - track.mp3", "01 - track.mp3", "02 - track.mp3", ] # The playlist carries the album it came from, which is how a front-end answers # "what is playing?" without keeping its own copy of that state. assert playlists["fuchs"].album_id == album_named(library, "Fuchs").id async def test_music_takes_its_title_and_artist_from_the_tags(config_dir: Path) -> None: album = album_named(await build(config_dir), "Kinderparty Lieder") assert album.kind == "music" assert album.artist == "Kinderparty" # Music groups by artist, so it has no series. assert album.series is None assert album.category == "Kinderparty" async def test_audiobooks_group_by_the_name_before_the_comma(config_dir: Path) -> None: album = album_named(await build(config_dir), "Conni in den Bergen") assert album.kind == "book" # album_artist is a credit list; only the first name is the character. assert album.artist == "Conni, Julia Boehme" assert album.series == "Conni" assert album.category == "Conni" async def test_podcast_episodes_are_their_own_albums_grouped_by_the_show( config_dir: Path, ) -> None: """One podcast mp3 behaves like a whole audiobook: its own album, its own cover, rather than a chapter buried in one album named after the entire feed.""" library = await build(config_dir) episodes = [a for a in library.albums if a.category == "Wissen macht Ah"] assert {a.title for a in episodes} == {"Alt", "Neu"} # The tags say album="Wissen macht Ah! - Podcast" and artist= - # neither groups usefully, so the folder (the show) wins for artist/series, same # as it always did; only the *title* now comes from the episode's own tags. assert all(a.kind == "book" for a in episodes) assert all(a.artist == "Wissen macht Ah" for a in episodes) assert all(a.series == "Wissen macht Ah" for a in episodes) assert all(len(a.tracks) == 1 for a in episodes) async def test_podcast_episodes_are_newest_first(config_dir: Path) -> None: library = await build(config_dir) episodes = [a for a in library.albums if a.category == "Wissen macht Ah"] assert [a.title for a in episodes] == ["Neu", "Alt"] async def test_latest_episode_finds_the_newest_episode_of_a_show(config_dir: Path) -> None: library = await build(config_dir) latest = library.latest_episode("Wissen macht Ah") assert latest is not None assert latest.title == "Neu" async def test_latest_episode_is_none_for_an_unknown_show(config_dir: Path) -> None: library = await build(config_dir) assert library.latest_episode("no such show") is None async def test_latest_episode_ignores_series_that_are_not_episode_unit(config_dir: Path) -> None: """A music/book section groups by `series` too (an audiobook's character), but it is a folder-unit shelf, not an episode-unit one - `latest_episode` must not treat an audiobook as if it had "episodes".""" library = await build(config_dir) audiobook = album_named(library, "Conni in den Bergen") assert audiobook.series is not None assert library.latest_episode(audiobook.series) is None async def test_other_sections_keep_filename_order(config_dir: Path) -> None: album = album_named(await build(config_dir), "Kinderparty Lieder") assert [track.path.name for track in album.tracks] == [ "00 - lied.mp3", "01 - lied.mp3", ] async def test_figures_are_marked_and_others_are_not(config_dir: Path) -> None: library = await build(config_dir) assert album_named(library, "Fuchs").figure == "fuchs" assert album_named(library, "Kinderparty Lieder").figure is None async def test_durations_come_from_the_files(config_dir: Path) -> None: album = album_named(await build(config_dir), "Eule") assert all(track.duration > 0 for track in album.tracks) assert album.duration == pytest.approx(sum(t.duration for t in album.tracks)) # -------------------------------------------------------------------- what is skipped async def test_scratch_files_never_reach_a_playlist(config_dir: Path) -> None: library = await build(config_dir) episodes = [a for a in library.albums if a.category == "Wissen macht Ah"] names = {episode.tracks[0].path.name for episode in episodes} assert "archive.json" not in names assert not any(name.startswith(".") for name in names) assert len(episodes) == 2 async def test_unknown_top_level_folders_are_ignored(config_dir: Path) -> None: stray = config_dir / "music" / "Sonstiges" / "Irgendwas" write_track(stray / "01 - x.mp3", title="X") library = await build(config_dir) assert not any(album.section == "Sonstiges" for album in library.albums) async def test_a_missing_section_warns_rather_than_failing( config_dir: Path, caplog: pytest.LogCaptureFixture ) -> None: import shutil shutil.rmtree(config_dir / "music" / "Musik") library = await build(config_dir) assert "Musik" in caplog.text assert len(library.albums) == 5 async def test_an_empty_album_folder_is_not_an_album(config_dir: Path) -> None: (config_dir / "music" / "Musik" / "Leer").mkdir() library = await build(config_dir) assert not any(album.title == "Leer" for album in library.albums) # ------------------------------------------------------------------------- colours async def test_every_album_has_three_colours(config_dir: Path) -> None: library = await build(config_dir) for album in library.albums: assert len(album.colors) == 3 assert all(colour.startswith("#") and len(colour) == 7 for colour in album.colors) async def test_colours_are_synthesised_when_there_is_no_cover(config_dir: Path) -> None: album = album_named(await build(config_dir), "Fuchs") assert album.cover is None assert album.colors == colors_from_id(album.id) async def test_a_cover_file_is_found_and_used(config_dir: Path) -> None: from PIL import Image folder = config_dir / "music" / "Musik" / "Kinderparty - Kinderparty Lieder" Image.new("RGB", (32, 32), (200, 40, 30)).save(folder / "cover.jpg") album = album_named(await build(config_dir), "Kinderparty Lieder") assert album.cover == folder / "cover.jpg" # A solid red cover has one usable colour; the rest fall back to the synthesised # palette rather than repeating it. assert album.colors[0] != colors_from_id(album.id)[0] async def test_an_episodes_sidecar_cover_wins_over_the_shows_shared_one(config_dir: Path) -> None: from PIL import Image # Downloaded by `podcast_feeds.py` when a feed has real per-episode art (see its # `episode_cover_filename`) - same stem as the episode's audio file. podcast = config_dir / "music" / "Kinderpodcasts" / "Wissen macht Ah" Image.new("RGB", (32, 32), (10, 20, 30)).save(podcast / "cover.jpg") Image.new("RGB", (32, 32), (200, 40, 30)).save(podcast / "20260101 - Neu.jpg") library = await build(config_dir) assert album_named(library, "Neu").cover == podcast / "20260101 - Neu.jpg" # No sidecar for this one, so it still falls back to the show's shared cover. assert album_named(library, "Alt").cover == podcast / "cover.jpg" # --------------------------------------------------------------------------- cache async def test_a_second_build_reuses_the_index(config_dir: Path) -> None: first = await build(config_dir) second = await build(config_dir) assert {a.id for a in first.albums} == {a.id for a in second.albums} assert (config_dir / ".cache" / "index.json").is_file() async def test_a_changed_folder_is_rescanned(config_dir: Path) -> None: library = await build(config_dir) assert len(album_named(library, "Eule").tracks) == 2 write_track(config_dir / "music" / "Figuren" / "eule" / "99 - neu.mp3", title="Neu") library = await build(config_dir) assert len(album_named(library, "Eule").tracks) == 3 async def test_a_new_episode_only_costs_scanning_that_one_file(config_dir: Path) -> None: """The point of fingerprinting a podcast per episode rather than per folder: a show's other, untouched episodes come straight out of the cache, ids and all.""" first = await build(config_dir) before = {a.id: a for a in first.albums if a.category == "Wissen macht Ah"} assert len(before) == 2 write_track( config_dir / "music" / "Kinderpodcasts" / "Wissen macht Ah" / "20260201 - Neuer.mp3", title="Neuer", album="Wissen macht Ah! - Podcast", albumartist="Ein Name, Noch Einer, Und Einer", ) second = await build(config_dir) after = {a.id: a for a in second.albums if a.category == "Wissen macht Ah"} assert len(after) == 3 # The two pre-existing episodes kept their identity across the rescan. assert set(before) <= set(after) async def test_a_rescan_leaves_analysis_alone(config_dir: Path) -> None: """The whole reason the cache is a directory rather than one file. "Kinderparty Lieder" rather than "Eule": folding analysis back into the index is restricted to music (see `_ANALYZED_KINDS`), since only music is ever analyzed - a book's `analysis/` file, if one somehow existed, is not something a rescan need resurface. """ library = await build(config_dir) album = album_named(library, "Kinderparty Lieder") from musicmouse.library.models import track_key key = track_key(album.tracks[0].path) library.cache.store_analysis(key, TrackAnalysis(version=ANALYZER_VERSION, tempo=128.0)) library.cache.store_beats(key, BeatGrid((0.5, 1.0), (1.0, 0.5))) curve = TrackCurves(hop_seconds=1.0, energy=(0.4, 0.6), valence=(0.5, 0.5), drive=(0.3, 0.7)) library.cache.store_curve(key, curve) # Force a full rescan by dropping the cheap part of the cache. (config_dir / ".cache" / "index.json").unlink() library = await build(config_dir) album = album_named(library, "Kinderparty Lieder") assert album.tracks[0].analysis is not None assert album.tracks[0].analysis.tempo == 128.0 grid = library.beats(album.id, 0) assert grid is not None assert grid.times == (0.5, 1.0) curve = library.curve(album.id, 0) assert curve is not None assert curve.energy == (0.4, 0.6) async def test_a_rescan_does_not_fold_analysis_into_a_non_music_album( config_dir: Path, ) -> None: """The other half of the restriction above: a book track's `analysis/` file (however it got there) is not folded into the index, so a rescan never stats hundreds of book/podcast files that can never have one under normal operation.""" library = await build(config_dir) album = album_named(library, "Eule") from musicmouse.library.models import track_key key = track_key(album.tracks[0].path) library.cache.store_analysis(key, TrackAnalysis(version=ANALYZER_VERSION, tempo=128.0)) (config_dir / ".cache" / "index.json").unlink() library = await build(config_dir) album = album_named(library, "Eule") assert album.tracks[0].analysis is None async def test_analysis_survives_an_analyzer_that_grew_a_field(tmp_path: Path) -> None: cache = LibraryCache(tmp_path / "cache") cache.prepare() (cache.analysis / "abc.json").write_text( json.dumps({"version": 1, "tempo": 90.0, "danceability": 0.7}) ) analysis = cache.load_analysis("abc") assert analysis is not None assert analysis.tempo == 90.0 async def test_a_corrupt_index_is_rebuilt_rather_than_fatal(config_dir: Path) -> None: await build(config_dir) (config_dir / ".cache" / "index.json").write_text("{ not json") library = await build(config_dir) assert len(library.albums) == 6 def test_album_ids_are_stable_and_path_derived(tmp_path: Path) -> None: first = album_id(tmp_path, tmp_path / "Musik" / "Ein Album") second = album_id(tmp_path, tmp_path / "Musik" / "Ein Album") other = album_id(tmp_path, tmp_path / "Musik" / "Ein Anderes") assert first == second assert first != other async def test_an_album_nested_under_an_extra_grouping_folder_is_still_found( config_dir: Path, ) -> None: """The bug this fixed: an artist folder that groups its books one level deeper than usual (an age-range folder, say) has no audio directly in it, so the scanner used to treat it as an empty album and skip the whole artist rather than looking further down for the actual album folders.""" for index in range(2): write_track( config_dir / "music" / "Hörbücher" / "Petzi" / "ab 5" / "Petzi und der Wal" / f"{index:02d} - teil.mp3", title=f"Teil {index}", album="Petzi und der Wal", albumartist="Petzi", ) library = await build(config_dir) album = album_named(library, "Petzi und der Wal") assert album.kind == "book" assert len(album.tracks) == 2 # -------------------------------------------------------------------- figure kinds async def test_a_figure_is_music_unless_the_config_says_otherwise( config_dir: Path, ) -> None: """A figure folder is named after the figurine, so nothing on disk says what it is.""" library = await build(config_dir) assert album_named(library, "Fuchs").kind == "music" assert album_named(library, "Eule").kind == "book" async def test_a_book_figure_groups_with_the_audiobooks(config_dir: Path) -> None: """The bug this fixed: a figure holding an audiobook landed in its artist's shelf as music, so the category showed "4 Alben" and was drawn like one.""" for index in range(2): write_track( config_dir / "music" / "Figuren" / "eule" / f"{index:02d} - track.mp3", title=f"Teil {index}", album="Conni in den Bergen", albumartist="Conni, Julia Boehme", ) library = await build(config_dir) figure = album_named(library, "Eule") assert figure.kind == "book" # Books group by series, which a music album does not have at all. assert figure.series == "Conni" assert figure.category == "Conni" conni = [album for album in library.albums if album.category == "Conni"] assert len(conni) == 2 assert {album.kind for album in conni} == {"book"} async def test_a_music_figure_has_no_series(config_dir: Path) -> None: figure = album_named(await build(config_dir), "Fuchs") assert figure.series is None assert figure.category == "Die Tiere" async def test_a_figures_kind_survives_the_cache(config_dir: Path) -> None: """It cannot be re-derived on load: the section says "Figuren", not what is in it. The index used to rebuild `kind` from the section, so a book figure came back as music on the second boot - and the browse view drew it square. """ first = await build(config_dir) assert album_named(first, "Eule").kind == "book" assert (config_dir / ".cache" / "index.json").is_file() # Nothing on disk changed, so this run is served entirely from the index. second = await build(config_dir) eule = album_named(second, "Eule") assert eule.kind == "book" assert eule.series is not None assert album_named(second, "Fuchs").kind == "music" assert album_named(second, "Fuchs").series is None # -------------------------------------------------------------------- analysis worker async def _wait_for(predicate: Callable[[], bool], *, timeout: float = 2.0) -> None: """Poll until `predicate()` is true, rather than guessing a sleep duration.""" async def poll() -> None: while not predicate(): await asyncio.sleep(0.01) await asyncio.wait_for(poll(), timeout=timeout) async def _cancel(task: asyncio.Task[object]) -> None: task.cancel() with contextlib.suppress(asyncio.CancelledError): await task async def test_analyze_pending_only_touches_music_albums(config_dir: Path) -> None: """Books and podcasts are the majority of a real library and none of them get a background - see `_ANALYZED_KINDS`.""" analyzer = FakeAnalyzer() library = await build(config_dir, analyzer=analyzer) music_track_count = sum(len(a.tracks) for a in library.albums if a.kind == "music") assert music_track_count == 5 # fuchs (3) + Kinderparty Lieder (2) done = await library.analyze_pending() assert done == 5 assert len(analyzer.calls) == 5 touched = { next(a for a in library.albums if p in {t.path for t in a.tracks}).kind for p in analyzer.calls } assert touched == {"music"} async def test_analyze_pending_persists_curves_alongside_beats(config_dir: Path) -> None: """`MusicLibrary.curve()` mirrors `.beats()` - both are per-track detail fetched for the one track currently playing, written together every pass.""" analyzer = FakeAnalyzer() library = await build(config_dir, analyzer=analyzer) album = next(a for a in library.albums if a.kind == "music") await library.analyze_pending() curve = library.curve(album.id, 0) assert curve is not None assert curve.hop_seconds == 1.0 assert curve.energy == (0.5,) assert library.beats(album.id, 0) is not None async def test_analyze_pending_does_not_repeat_work_already_cached(config_dir: Path) -> None: analyzer = FakeAnalyzer() library = await build(config_dir, analyzer=analyzer) await library.analyze_pending() again = await library.analyze_pending() assert again == 0 assert len(analyzer.calls) == 5 async def test_a_failing_track_is_marked_attempted_and_not_retried(config_dir: Path) -> None: """A corrupt file, a DRM'd one, or one the analyzer just chokes on must not abort the batch, and must not be retried on every single future pass either.""" analyzer = FakeAnalyzer(fails={"00 - lied.mp3"}) library = await build(config_dir, analyzer=analyzer) done = await library.analyze_pending() assert done == 5 # the failure still counts as "handled" kinderparty = album_named(library, "Kinderparty Lieder") failed = next(t for t in kinderparty.tracks if t.path.name == "00 - lied.mp3") cached = library.cache.load_analysis(track_key(failed.path)) assert cached is not None assert cached.version == ANALYZER_VERSION assert cached.tempo is None # attempted, not computed again = await library.analyze_pending() assert again == 0 assert len(analyzer.calls) == 5 async def test_is_busy_pauses_the_pass_until_it_clears( config_dir: Path, monkeypatch: pytest.MonkeyPatch ) -> None: import musicmouse.library as library_module monkeypatch.setattr(library_module, "_BUSY_POLL_SECONDS", 0.01) analyzer = FakeAnalyzer() library = await build(config_dir, analyzer=analyzer) busy = True task = asyncio.create_task(library.analyze_pending(is_busy=lambda: busy)) try: await asyncio.sleep(0.05) assert analyzer.calls == [] # never got past the busy check busy = False await asyncio.wait_for(task, timeout=2) finally: if not task.done(): await _cancel(task) assert len(analyzer.calls) == 5 async def test_run_analysis_does_the_work_refresh_left_pending(config_dir: Path) -> None: """The trigger this feature actually depends on: `refresh()` - at startup, or from a parent's "Bibliothek neu einlesen" - ends by requesting analysis, and `run_analysis` is what turns that request into finished work, without a second `refresh()` needed to see it.""" analyzer = FakeAnalyzer() library = await build(config_dir, analyzer=analyzer) # refresh() already requested batches: list[int] = [] async def on_batch() -> None: batches.append(len(analyzer.calls)) worker = asyncio.create_task(library.run_analysis(on_batch=on_batch)) try: await _wait_for(lambda: bool(batches)) finally: await _cancel(worker) assert batches[-1] == 5 kinderparty = album_named(library, "Kinderparty Lieder") assert kinderparty.tracks[0].analysis is not None assert kinderparty.tracks[0].analysis.tempo == 100.0 async def test_requests_raised_during_a_pass_coalesce_into_one_more_pass( config_dir: Path, ) -> None: """Calling `request_analysis` three times while a pass is already running must not queue three more passes - just one, right after the current one finishes.""" release = threading.Event() class BlockingOnceAnalyzer(FakeAnalyzer): def __init__(self) -> None: super().__init__() self._blocked = False def analyze(self, path: Path) -> tuple[TrackAnalysis, BeatGrid | None, TrackCurves | None]: if not self._blocked: self._blocked = True release.wait(timeout=2) return super().analyze(path) analyzer = BlockingOnceAnalyzer() library = await build(config_dir, analyzer=analyzer) pass_count = 0 original = library.analyze_pending async def counting(**kwargs: object) -> int: nonlocal pass_count pass_count += 1 return await original(**kwargs) # type: ignore[arg-type] library.analyze_pending = counting # type: ignore[method-assign] worker = asyncio.create_task(library.run_analysis()) try: await _wait_for(lambda: pass_count >= 1) # the first pass has started await asyncio.sleep(0.05) # ... and is now blocked inside the analyzer library.request_analysis() library.request_analysis() library.request_analysis() release.set() await _wait_for(lambda: pass_count >= 2) # Give the coalesced pass a moment to actually run - everything is already # cached, so it finds nothing pending and returns without incrementing further. await asyncio.sleep(0.1) finally: await _cancel(worker) assert pass_count == 2 assert len(analyzer.calls) == 5 # ------------------------------------------------------------------ worker processes async def test_a_parallel_pass_analyzes_every_track_in_worker_processes( config_dir: Path, ) -> None: """The whole point of `analysis_workers`: a machine with cores gets to use them. Two workers rather than the real default, because what is being checked is that the work left this process at all - not how fast a five-track fixture goes. """ library = await build(config_dir, analyzer=PidAnalyzer(fails={"00 - lied.mp3"})) done = await library.analyze_pending(workers=2) assert done == 5 analyzed = [ track.analysis for album in library.albums if album.kind == "music" for track in album.tracks ] assert all(a is not None and a.version == ANALYZER_VERSION for a in analyzed) # The one file the analyzer chokes on is recorded as attempted, exactly as on the # in-process path - a raise in a worker must not abort the other four. failed = [a for a in analyzed if a is not None and a.tempo is None] assert len(failed) == 1 workers = {a.tempo for a in analyzed if a is not None and a.tempo is not None} assert workers assert os.getpid() not in workers async def test_a_parallel_pass_starts_no_workers_while_playback_is_busy( config_dir: Path, monkeypatch: pytest.MonkeyPatch ) -> None: """Being busy holds the pass *before* the pool exists, so a device playing an album is not also hosting a set of idle analyzer processes.""" import musicmouse.library as library_module monkeypatch.setattr(library_module, "_BUSY_POLL_SECONDS", 0.01) library = await build(config_dir, analyzer=PidAnalyzer()) busy = True task = asyncio.create_task(library.analyze_pending(workers=2, is_busy=lambda: busy)) try: await asyncio.sleep(0.2) assert list(library.cache.analysis.glob("*.json")) == [] busy = False assert await asyncio.wait_for(task, timeout=30) == 5 finally: if not task.done(): await _cancel(task) async def test_a_parallel_pass_is_cancellable_mid_flight(config_dir: Path) -> None: """Shutdown cancels this task like any other, and it must not hang waiting for worker processes to finish the tracks they are on.""" library = await build(config_dir, analyzer=PidAnalyzer()) task = asyncio.create_task(library.analyze_pending(workers=2)) await asyncio.sleep(0.1) # long enough to have workers starting up await asyncio.wait_for(_cancel(task), timeout=10) def test_the_default_worker_count_leaves_a_core_for_everything_else() -> None: count = default_worker_count() assert 1 <= count <= max(1, (os.cpu_count() or 1) - 1)