Martin Bauer
69119bb72a
- podcast_feeds.py: fetch per-episode cover art from a feed's itunes:image (when it's genuinely distinct from the channel image) or, failing that, from the og:image on the episode's own linked page; extract audio from video-only enclosures via ffmpeg; match podcast-dl's filename convention (illegal characters become "_" instead of being dropped) so enabling feed.txt on an already-downloaded show doesn't re-download its back catalog - scanner.py: _cover_for_episode now checks for a same-stem sidecar cover image before falling back to embedded ID3 art and the shared folder cover - scanner.py/__init__.py: fixed a bug where an album folder nested one level deeper than usual (an age-range grouping folder, say) was mistaken for an empty album and skipped; added periodic progress logging for long scans and analysis passes Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
329 lines
13 KiB
Python
329 lines
13 KiB
Python
"""The music collection: what exists, what it is called, and what colour it is.
|
|
|
|
The scan is the only part of this backend that touches hundreds of files, so it runs
|
|
in a worker thread and its results are cached. Everything the rest of the app sees is
|
|
plain immutable data.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import asyncio
|
|
import contextlib
|
|
import logging
|
|
import os
|
|
import time
|
|
from collections.abc import Awaitable, Callable, Collection, Mapping
|
|
from dataclasses import replace
|
|
from pathlib import Path
|
|
from typing import Final
|
|
|
|
from musicmouse.library.analysis import (
|
|
ANALYZER_VERSION,
|
|
Analyzer,
|
|
BeatGrid,
|
|
NullAnalyzer,
|
|
TrackAnalysis,
|
|
TrackCurves,
|
|
)
|
|
from musicmouse.library.cache import Fingerprint, LibraryCache
|
|
from musicmouse.library.models import Album, LibraryTrack, album_id, track_key
|
|
from musicmouse.library.scanner import scan_library
|
|
from musicmouse.library.sections import SECTIONS, AlbumKind
|
|
from musicmouse.media import Playlist
|
|
|
|
_log = logging.getLogger(__name__)
|
|
|
|
__all__ = [
|
|
"ANALYZER_VERSION",
|
|
"SECTIONS",
|
|
"Album",
|
|
"Analyzer",
|
|
"BeatGrid",
|
|
"LibraryCache",
|
|
"LibraryTrack",
|
|
"MusicLibrary",
|
|
"NullAnalyzer",
|
|
"TrackCurves",
|
|
"album_id",
|
|
"track_key",
|
|
]
|
|
|
|
#: The only kind worth spending DSP on: an audiobook chapter or a podcast episode is
|
|
#: tens of minutes of narration with no musical mood to extract, and there are far more
|
|
#: of them in a typical library than there are songs.
|
|
_ANALYZED_KINDS: Final[tuple[AlbumKind, ...]] = ("music",)
|
|
|
|
#: How often the background worker checks whether it may resume after `is_busy()` said
|
|
#: no. A track's own analysis is a couple of seconds of CPU, so overshooting by this
|
|
#: much when playback stops is not worth polling harder for.
|
|
_BUSY_POLL_SECONDS: Final = 2.0
|
|
|
|
#: Analysis is folded back into the live index and persisted this often during a long
|
|
#: run, so a first-time pass over an unanalyzed library shows up gradually in open
|
|
#: browser tabs rather than only after the whole thing finishes.
|
|
_PUBLISH_BATCH_SIZE: Final = 25
|
|
|
|
#: How often a long analysis pass reports where it is, so a first-time run over a large
|
|
#: library - minutes of DSP per track - doesn't sit silent with nothing on the console
|
|
#: to say it is still going.
|
|
_PROGRESS_INTERVAL_SECONDS: Final = 5.0
|
|
|
|
|
|
def _analyze_one(
|
|
analyzer: Analyzer, path: Path
|
|
) -> tuple[TrackAnalysis, BeatGrid | None, TrackCurves | None]:
|
|
"""Runs in a worker thread. Lowers this thread's own scheduling priority first.
|
|
|
|
On Linux, ``os.nice`` affects only the calling thread, not the whole process - so
|
|
this makes idle-time analysis yield CPU to anything else without touching threads
|
|
used for other work. Niceness only ever increases and clamps at the OS maximum
|
|
(19), so calling this repeatedly on a reused pool thread is harmless.
|
|
"""
|
|
with contextlib.suppress(OSError):
|
|
os.nice(1)
|
|
return analyzer.analyze(path)
|
|
|
|
|
|
class MusicLibrary:
|
|
"""An immutable index of albums, rebuilt wholesale rather than mutated in place."""
|
|
|
|
def __init__(
|
|
self,
|
|
root: Path,
|
|
cache: LibraryCache,
|
|
extensions: frozenset[str],
|
|
*,
|
|
analyzer: Analyzer | None = None,
|
|
figure_kinds: Mapping[str, AlbumKind] | None = None,
|
|
) -> None:
|
|
self.root = root
|
|
self.cache = cache
|
|
self.extensions = extensions
|
|
self.analyzer: Analyzer = analyzer or NullAnalyzer()
|
|
#: What each figure holds. The only thing a folder name cannot say.
|
|
self.figure_kinds: Mapping[str, AlbumKind] = figure_kinds or {}
|
|
self._entries: dict[str, tuple[Album, Fingerprint]] = {}
|
|
#: Set by `request_analysis`, consumed by `run_analysis`. An `Event` rather than
|
|
#: a queue: a request raised while one is already pending or running just
|
|
#: coalesces into it, which is exactly what "look again" should mean here.
|
|
self._analysis_requested = asyncio.Event()
|
|
|
|
# -------------------------------------------------------------------- reading
|
|
|
|
@property
|
|
def albums(self) -> list[Album]:
|
|
return [album for album, _ in self._entries.values()]
|
|
|
|
def get(self, identifier: str | None) -> Album | None:
|
|
entry = self._entries.get(identifier) if identifier else None
|
|
return entry[0] if entry else None
|
|
|
|
def figure_playlists(self) -> dict[str, Playlist]:
|
|
"""One playlist per figure folder, keyed by figure name.
|
|
|
|
The figure path and the web path must hand the player the *same* object for the
|
|
same album: ``reactions.playback.play_figure`` resumes on an identity check.
|
|
"""
|
|
return {
|
|
album.figure: album.to_playlist()
|
|
for album, _ in self._entries.values()
|
|
if album.figure is not None
|
|
}
|
|
|
|
def latest_episode(self, series: str) -> Album | None:
|
|
"""The newest episode-unit album of a podcast show, by filename.
|
|
|
|
Episode files are named ``YYYYMMDD - Title``, so filename order is
|
|
chronological - the same fact ``Kinderpodcasts``' ``order="newest_first"``
|
|
already relies on at scan time. ``None`` if the show is unknown or empty.
|
|
"""
|
|
candidates = [
|
|
album
|
|
for album in self.albums
|
|
if album.series == series
|
|
and (section := SECTIONS.get(album.section)) is not None
|
|
and section.album_unit == "episode"
|
|
]
|
|
if not candidates:
|
|
return None
|
|
return max(
|
|
candidates, key=lambda album: album.tracks[0].path.name if album.tracks else ""
|
|
)
|
|
|
|
def beats(self, identifier: str, index: int) -> BeatGrid | None:
|
|
album = self.get(identifier)
|
|
if album is None or not 0 <= index < len(album.tracks):
|
|
return None
|
|
return self.cache.load_beats(track_key(album.tracks[index].path))
|
|
|
|
def curve(self, identifier: str, index: int) -> TrackCurves | None:
|
|
album = self.get(identifier)
|
|
if album is None or not 0 <= index < len(album.tracks):
|
|
return None
|
|
return self.cache.load_curve(track_key(album.tracks[index].path))
|
|
|
|
# -------------------------------------------------------------------- writing
|
|
|
|
async def refresh(self) -> None:
|
|
"""Rescan from disk. Blocking work happens off the loop; the swap is atomic.
|
|
|
|
Ends by waking the background analyzer: a rescan is exactly when new tracks -
|
|
the only ones analysis can be pending for - enter the index, whether that is
|
|
the startup scan or a parent tapping "Bibliothek neu einlesen".
|
|
"""
|
|
known = dict(self._entries)
|
|
entries = await asyncio.to_thread(
|
|
scan_library,
|
|
self.root,
|
|
self.extensions,
|
|
self.cache,
|
|
known=known,
|
|
figure_kinds=self.figure_kinds,
|
|
)
|
|
self._entries = await asyncio.to_thread(self._with_analysis, entries)
|
|
await asyncio.to_thread(self.cache.store_index, self._entries)
|
|
self.request_analysis()
|
|
|
|
def _with_analysis(
|
|
self,
|
|
entries: dict[str, tuple[Album, Fingerprint]],
|
|
*,
|
|
kinds: Collection[AlbumKind] = _ANALYZED_KINDS,
|
|
) -> dict[str, tuple[Album, Fingerprint]]:
|
|
"""Fold cached analysis results into the freshly scanned index.
|
|
|
|
Restricted to `kinds` (music by default): stat-ing hundreds of book and podcast
|
|
tracks whose analysis can never exist would be pure waste. Skipped entirely
|
|
while ``analysis/`` is empty, which is the normal case until an analyzer has
|
|
actually been run.
|
|
"""
|
|
if not any(self.cache.analysis.glob("*.json")):
|
|
return entries
|
|
out: dict[str, tuple[Album, Fingerprint]] = {}
|
|
for identifier, (album, fingerprint) in entries.items():
|
|
if album.kind not in kinds:
|
|
out[identifier] = (album, fingerprint)
|
|
continue
|
|
tracks = tuple(
|
|
replace(track, analysis=self.cache.load_analysis(track_key(track.path)))
|
|
for track in album.tracks
|
|
)
|
|
out[identifier] = (replace(album, tracks=tracks), fingerprint)
|
|
return out
|
|
|
|
@classmethod
|
|
async def build(
|
|
cls,
|
|
root: Path,
|
|
cache_dir: Path,
|
|
extensions: frozenset[str],
|
|
*,
|
|
analyzer: Analyzer | None = None,
|
|
figure_kinds: Mapping[str, AlbumKind] | None = None,
|
|
) -> MusicLibrary:
|
|
cache = LibraryCache(cache_dir)
|
|
library = cls(root, cache, extensions, analyzer=analyzer, figure_kinds=figure_kinds)
|
|
library._entries = await asyncio.to_thread(cache.load_index)
|
|
await library.refresh()
|
|
return library
|
|
|
|
# ------------------------------------------------------------------- analysis
|
|
|
|
def request_analysis(self) -> None:
|
|
"""Wake the background worker to look for tracks with no current analysis.
|
|
|
|
Idempotent, and safe to call before `run_analysis` has even started a first
|
|
time - the request just waits on the event.
|
|
"""
|
|
self._analysis_requested.set()
|
|
|
|
async def run_analysis(
|
|
self,
|
|
*,
|
|
is_busy: Callable[[], bool] = lambda: False,
|
|
on_batch: Callable[[], Awaitable[None]] | None = None,
|
|
) -> None:
|
|
"""Analyze pending tracks whenever `request_analysis` wakes this up.
|
|
|
|
A long-running task, cancelled at shutdown alongside every other one. Loops
|
|
forever so a request raised *during* a pass (a rescan mid-analysis) starts
|
|
another pass right after, rather than being lost.
|
|
"""
|
|
while True:
|
|
await self._analysis_requested.wait()
|
|
self._analysis_requested.clear()
|
|
await self.analyze_pending(is_busy=is_busy, on_batch=on_batch)
|
|
|
|
async def analyze_pending(
|
|
self,
|
|
*,
|
|
kinds: Collection[AlbumKind] = _ANALYZED_KINDS,
|
|
is_busy: Callable[[], bool] = lambda: False,
|
|
on_batch: Callable[[], Awaitable[None]] | None = None,
|
|
batch_size: int = _PUBLISH_BATCH_SIZE,
|
|
) -> int:
|
|
"""Run the analyzer over tracks of `kinds` that have no current result.
|
|
|
|
Restricted to music by default - see `_ANALYZED_KINDS`. Checked before every
|
|
track, `is_busy()` pauses the whole pass rather than one file: analysis must
|
|
never compete with audio decoding for CPU, and a children's player is idle most
|
|
of the day, so the pass simply resumes next time it is. A track the analyzer
|
|
fails on (corrupt file, DRM, zero length) is still recorded as attempted - with
|
|
every scalar left `None` - so it is never retried forever and the frontend falls
|
|
back to the un-analyzed baseline for it. Results are folded into the live index
|
|
and persisted every `batch_size` tracks, so a long first run is visible in open
|
|
browser tabs as it goes rather than only once it finishes.
|
|
"""
|
|
analyzer = self.analyzer
|
|
if analyzer.version < ANALYZER_VERSION:
|
|
return 0
|
|
|
|
done = 0
|
|
last_report = time.monotonic()
|
|
for album in self.albums:
|
|
if album.kind not in kinds:
|
|
continue
|
|
for track in album.tracks:
|
|
while is_busy():
|
|
await asyncio.sleep(_BUSY_POLL_SECONDS)
|
|
key = track_key(track.path)
|
|
cached = self.cache.load_analysis(key)
|
|
if cached is not None and cached.version >= analyzer.version:
|
|
continue
|
|
now = time.monotonic()
|
|
if now - last_report >= _PROGRESS_INTERVAL_SECONDS:
|
|
_log.info("Analyzing library: %d tracks done so far, now on %s", done, track.path)
|
|
last_report = now
|
|
try:
|
|
analysis, grid, curve = await asyncio.to_thread(
|
|
_analyze_one, analyzer, track.path
|
|
)
|
|
except Exception:
|
|
_log.warning(
|
|
"Analyzer raised on %s; marking it attempted so it is not retried forever",
|
|
track.path,
|
|
exc_info=True,
|
|
)
|
|
analysis, grid, curve = TrackAnalysis(version=analyzer.version), None, None
|
|
if grid is not None:
|
|
self.cache.store_beats(key, grid)
|
|
if curve is not None:
|
|
self.cache.store_curve(key, curve)
|
|
self.cache.store_analysis(key, analysis)
|
|
done += 1
|
|
if done % batch_size == 0:
|
|
await self._publish_analysis(kinds, on_batch)
|
|
if done % batch_size:
|
|
await self._publish_analysis(kinds, on_batch)
|
|
if done:
|
|
_log.info("Analyzed %d tracks", done)
|
|
return done
|
|
|
|
async def _publish_analysis(
|
|
self, kinds: Collection[AlbumKind], on_batch: Callable[[], Awaitable[None]] | None
|
|
) -> None:
|
|
self._entries = await asyncio.to_thread(self._with_analysis, self._entries, kinds=kinds)
|
|
await asyncio.to_thread(self.cache.store_index, self._entries)
|
|
if on_batch is not None:
|
|
await on_batch()
|