Martin Bauer
fd6283718c
A first pass over an unanalysed library is hours of librosa, and there was no reason for a desktop to spend them one core at a time. Analysis now runs in a process pool sized by `general.library.analysis_workers`, defaulting to one per core bar one (capped at 8) when the key is absent. Two consequences worth knowing before touching this: whatever an `Analyzer` returns has to be picklable, and an analyzer that keeps state on its own instance - a test double counting calls - only behaves as written with `analysis_workers=1`. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
418 lines
18 KiB
Python
418 lines
18 KiB
Python
"""The music collection: what exists, what it is called, and what colour it is.
|
|
|
|
The scan is the only part of this backend that touches hundreds of files, so it runs
|
|
in a worker thread and its results are cached. Everything the rest of the app sees is
|
|
plain immutable data.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import asyncio
|
|
import logging
|
|
import time
|
|
from collections import deque
|
|
from collections.abc import Awaitable, Callable, Collection, Mapping
|
|
from concurrent.futures import BrokenExecutor, Executor
|
|
from dataclasses import replace
|
|
from pathlib import Path
|
|
from typing import Any, Final
|
|
|
|
from musicmouse.library.analysis import (
|
|
ANALYZER_VERSION,
|
|
Analyzer,
|
|
BeatGrid,
|
|
NullAnalyzer,
|
|
TrackAnalysis,
|
|
TrackCurves,
|
|
)
|
|
from musicmouse.library.cache import Fingerprint, LibraryCache
|
|
from musicmouse.library.models import Album, LibraryTrack, album_id, track_key
|
|
from musicmouse.library.scanner import scan_library
|
|
from musicmouse.library.sections import SECTIONS, AlbumKind
|
|
from musicmouse.library.workers import analysis_pool, analyze_one, default_worker_count
|
|
from musicmouse.media import Playlist
|
|
|
|
_log = logging.getLogger(__name__)
|
|
|
|
__all__ = [
|
|
"ANALYZER_VERSION",
|
|
"SECTIONS",
|
|
"Album",
|
|
"Analyzer",
|
|
"BeatGrid",
|
|
"LibraryCache",
|
|
"LibraryTrack",
|
|
"MusicLibrary",
|
|
"NullAnalyzer",
|
|
"TrackCurves",
|
|
"album_id",
|
|
"default_worker_count",
|
|
"track_key",
|
|
]
|
|
|
|
#: The only kind worth spending DSP on: an audiobook chapter or a podcast episode is
|
|
#: tens of minutes of narration with no musical mood to extract, and there are far more
|
|
#: of them in a typical library than there are songs.
|
|
_ANALYZED_KINDS: Final[tuple[AlbumKind, ...]] = ("music",)
|
|
|
|
#: How often the background worker checks whether it may resume after `is_busy()` said
|
|
#: no. A track's own analysis is a couple of seconds of CPU, so overshooting by this
|
|
#: much when playback stops is not worth polling harder for.
|
|
_BUSY_POLL_SECONDS: Final = 2.0
|
|
|
|
#: Analysis is folded back into the live index and persisted this often during a long
|
|
#: run, so a first-time pass over an unanalyzed library shows up gradually in open
|
|
#: browser tabs rather than only after the whole thing finishes.
|
|
_PUBLISH_BATCH_SIZE: Final = 25
|
|
|
|
#: How often a long analysis pass reports where it is, so a first-time run over a large
|
|
#: library - minutes of DSP per track - doesn't sit silent with nothing on the console
|
|
#: to say it is still going.
|
|
_PROGRESS_INTERVAL_SECONDS: Final = 5.0
|
|
|
|
|
|
def _discard(future: asyncio.Future[Any]) -> None:
|
|
"""Drop a result nobody is going to read, without leaving a warning behind."""
|
|
if not future.done():
|
|
future.cancel()
|
|
elif not future.cancelled():
|
|
future.exception()
|
|
|
|
|
|
class MusicLibrary:
|
|
"""An immutable index of albums, rebuilt wholesale rather than mutated in place."""
|
|
|
|
def __init__(
|
|
self,
|
|
root: Path,
|
|
cache: LibraryCache,
|
|
extensions: frozenset[str],
|
|
*,
|
|
analyzer: Analyzer | None = None,
|
|
analysis_workers: int = 1,
|
|
figure_kinds: Mapping[str, AlbumKind] | None = None,
|
|
) -> None:
|
|
self.root = root
|
|
self.cache = cache
|
|
self.extensions = extensions
|
|
self.analyzer: Analyzer = analyzer or NullAnalyzer()
|
|
#: How many tracks background analysis may work on at once. The default of 1
|
|
#: keeps the analyzer in this process, where an analyzer that holds state
|
|
#: still behaves as written; the app passes `default_worker_count()`, which is
|
|
#: what makes a first-time pass finish in hours rather than days on a desktop.
|
|
#: See `musicmouse.library.workers`.
|
|
self.analysis_workers = analysis_workers
|
|
#: What each figure holds. The only thing a folder name cannot say.
|
|
self.figure_kinds: Mapping[str, AlbumKind] = figure_kinds or {}
|
|
self._entries: dict[str, tuple[Album, Fingerprint]] = {}
|
|
#: Set by `request_analysis`, consumed by `run_analysis`. An `Event` rather than
|
|
#: a queue: a request raised while one is already pending or running just
|
|
#: coalesces into it, which is exactly what "look again" should mean here.
|
|
self._analysis_requested = asyncio.Event()
|
|
|
|
# -------------------------------------------------------------------- reading
|
|
|
|
@property
|
|
def albums(self) -> list[Album]:
|
|
return [album for album, _ in self._entries.values()]
|
|
|
|
def get(self, identifier: str | None) -> Album | None:
|
|
entry = self._entries.get(identifier) if identifier else None
|
|
return entry[0] if entry else None
|
|
|
|
def figure_playlists(self) -> dict[str, Playlist]:
|
|
"""One playlist per figure folder, keyed by figure name.
|
|
|
|
The figure path and the web path must hand the player the *same* object for the
|
|
same album: ``reactions.playback.play_figure`` resumes on an identity check.
|
|
"""
|
|
return {
|
|
album.figure: album.to_playlist()
|
|
for album, _ in self._entries.values()
|
|
if album.figure is not None
|
|
}
|
|
|
|
def latest_episode(self, series: str) -> Album | None:
|
|
"""The newest episode-unit album of a podcast show, by filename.
|
|
|
|
Episode files are named ``YYYYMMDD - Title``, so filename order is
|
|
chronological - the same fact ``Kinderpodcasts``' ``order="newest_first"``
|
|
already relies on at scan time. ``None`` if the show is unknown or empty.
|
|
"""
|
|
candidates = [
|
|
album
|
|
for album in self.albums
|
|
if album.series == series
|
|
and (section := SECTIONS.get(album.section)) is not None
|
|
and section.album_unit == "episode"
|
|
]
|
|
if not candidates:
|
|
return None
|
|
return max(candidates, key=lambda album: album.tracks[0].path.name if album.tracks else "")
|
|
|
|
def beats(self, identifier: str, index: int) -> BeatGrid | None:
|
|
album = self.get(identifier)
|
|
if album is None or not 0 <= index < len(album.tracks):
|
|
return None
|
|
return self.cache.load_beats(track_key(album.tracks[index].path))
|
|
|
|
def curve(self, identifier: str, index: int) -> TrackCurves | None:
|
|
album = self.get(identifier)
|
|
if album is None or not 0 <= index < len(album.tracks):
|
|
return None
|
|
return self.cache.load_curve(track_key(album.tracks[index].path))
|
|
|
|
# -------------------------------------------------------------------- writing
|
|
|
|
async def refresh(self) -> None:
|
|
"""Rescan from disk. Blocking work happens off the loop; the swap is atomic.
|
|
|
|
Ends by waking the background analyzer: a rescan is exactly when new tracks -
|
|
the only ones analysis can be pending for - enter the index, whether that is
|
|
the startup scan or a parent tapping "Bibliothek neu einlesen".
|
|
"""
|
|
known = dict(self._entries)
|
|
entries = await asyncio.to_thread(
|
|
scan_library,
|
|
self.root,
|
|
self.extensions,
|
|
self.cache,
|
|
known=known,
|
|
figure_kinds=self.figure_kinds,
|
|
)
|
|
self._entries = await asyncio.to_thread(self._with_analysis, entries)
|
|
await asyncio.to_thread(self.cache.store_index, self._entries)
|
|
self.request_analysis()
|
|
|
|
def _with_analysis(
|
|
self,
|
|
entries: dict[str, tuple[Album, Fingerprint]],
|
|
*,
|
|
kinds: Collection[AlbumKind] = _ANALYZED_KINDS,
|
|
) -> dict[str, tuple[Album, Fingerprint]]:
|
|
"""Fold cached analysis results into the freshly scanned index.
|
|
|
|
Restricted to `kinds` (music by default): stat-ing hundreds of book and podcast
|
|
tracks whose analysis can never exist would be pure waste. Skipped entirely
|
|
while ``analysis/`` is empty, which is the normal case until an analyzer has
|
|
actually been run.
|
|
"""
|
|
if not any(self.cache.analysis.glob("*.json")):
|
|
return entries
|
|
out: dict[str, tuple[Album, Fingerprint]] = {}
|
|
for identifier, (album, fingerprint) in entries.items():
|
|
if album.kind not in kinds:
|
|
out[identifier] = (album, fingerprint)
|
|
continue
|
|
tracks = tuple(
|
|
replace(track, analysis=self.cache.load_analysis(track_key(track.path)))
|
|
for track in album.tracks
|
|
)
|
|
out[identifier] = (replace(album, tracks=tracks), fingerprint)
|
|
return out
|
|
|
|
@classmethod
|
|
async def build(
|
|
cls,
|
|
root: Path,
|
|
cache_dir: Path,
|
|
extensions: frozenset[str],
|
|
*,
|
|
analyzer: Analyzer | None = None,
|
|
analysis_workers: int = 1,
|
|
figure_kinds: Mapping[str, AlbumKind] | None = None,
|
|
) -> MusicLibrary:
|
|
cache = LibraryCache(cache_dir)
|
|
library = cls(
|
|
root,
|
|
cache,
|
|
extensions,
|
|
analyzer=analyzer,
|
|
analysis_workers=analysis_workers,
|
|
figure_kinds=figure_kinds,
|
|
)
|
|
library._entries = await asyncio.to_thread(cache.load_index)
|
|
await library.refresh()
|
|
return library
|
|
|
|
# ------------------------------------------------------------------- analysis
|
|
|
|
def request_analysis(self) -> None:
|
|
"""Wake the background worker to look for tracks with no current analysis.
|
|
|
|
Idempotent, and safe to call before `run_analysis` has even started a first
|
|
time - the request just waits on the event.
|
|
"""
|
|
self._analysis_requested.set()
|
|
|
|
async def run_analysis(
|
|
self,
|
|
*,
|
|
is_busy: Callable[[], bool] = lambda: False,
|
|
on_batch: Callable[[], Awaitable[None]] | None = None,
|
|
) -> None:
|
|
"""Analyze pending tracks whenever `request_analysis` wakes this up.
|
|
|
|
A long-running task, cancelled at shutdown alongside every other one. Loops
|
|
forever so a request raised *during* a pass (a rescan mid-analysis) starts
|
|
another pass right after, rather than being lost.
|
|
"""
|
|
while True:
|
|
await self._analysis_requested.wait()
|
|
self._analysis_requested.clear()
|
|
await self.analyze_pending(is_busy=is_busy, on_batch=on_batch)
|
|
|
|
async def analyze_pending(
|
|
self,
|
|
*,
|
|
kinds: Collection[AlbumKind] = _ANALYZED_KINDS,
|
|
is_busy: Callable[[], bool] = lambda: False,
|
|
on_batch: Callable[[], Awaitable[None]] | None = None,
|
|
batch_size: int = _PUBLISH_BATCH_SIZE,
|
|
workers: int | None = None,
|
|
) -> int:
|
|
"""Run the analyzer over tracks of `kinds` that have no current result.
|
|
|
|
Restricted to music by default - see `_ANALYZED_KINDS`. Up to `workers` tracks
|
|
are analyzed at once (`self.analysis_workers` when not given), each in its own
|
|
worker process - see `musicmouse.library.workers` for why processes and how the
|
|
machine is kept usable while they run. Checked before every track is handed out,
|
|
`is_busy()` pauses the whole pass rather than one file: analysis must never
|
|
compete with audio decoding for CPU, and a children's player is idle most of the
|
|
day, so the pass simply resumes next time it is. Tracks already in flight when
|
|
it goes busy are allowed to finish - their results are already paid for - and
|
|
the workers are then shut down for the duration rather than sitting idle with
|
|
a librosa apiece resident on a machine that is now playing music.
|
|
|
|
A track the analyzer fails on (corrupt file, DRM, zero length) is still recorded
|
|
as attempted - with every scalar left `None` - so it is never retried forever and
|
|
the frontend falls back to the un-analyzed baseline for it. A worker *process*
|
|
dying, though, says nothing about the track it was on, so that ends the pass
|
|
without recording anything: the next one picks the same tracks up again.
|
|
|
|
Results are folded into the live index and persisted every `batch_size` tracks,
|
|
so a long first run is visible in open browser tabs as it goes rather than only
|
|
once it finishes.
|
|
"""
|
|
analyzer = self.analyzer
|
|
if analyzer.version < ANALYZER_VERSION:
|
|
return 0
|
|
pending = deque(self._pending_tracks(kinds, analyzer.version))
|
|
if not pending:
|
|
return 0
|
|
|
|
workers = self.analysis_workers if workers is None else max(1, workers)
|
|
total = len(pending)
|
|
_log.info("Analyzing %d tracks with %d worker(s)", total, workers)
|
|
|
|
loop = asyncio.get_running_loop()
|
|
in_flight: dict[asyncio.Future[Any], tuple[str, Path]] = {}
|
|
done = 0
|
|
last_report = time.monotonic()
|
|
try:
|
|
while pending:
|
|
while is_busy():
|
|
# Waited out *between* pools, so a half-hour album is not played
|
|
# with a houseful of idle worker processes holding onto librosa.
|
|
await asyncio.sleep(_BUSY_POLL_SECONDS)
|
|
with analysis_pool(workers) as pool:
|
|
try:
|
|
while in_flight or (pending and not is_busy()):
|
|
while pending and len(in_flight) < workers and not is_busy():
|
|
key, path = pending.popleft()
|
|
in_flight[self._submit(pool, loop, path)] = (key, path)
|
|
if not in_flight:
|
|
break # gone busy: drop the pool and wait above
|
|
finished, _ = await asyncio.wait(
|
|
in_flight, return_when=asyncio.FIRST_COMPLETED
|
|
)
|
|
for future in finished:
|
|
key, path = in_flight.pop(future)
|
|
self._store_result(key, path, future)
|
|
done += 1
|
|
if done % batch_size == 0:
|
|
await self._publish_analysis(kinds, on_batch)
|
|
now = time.monotonic()
|
|
if now - last_report >= _PROGRESS_INTERVAL_SECONDS:
|
|
_log.info("Analyzing library: %d/%d tracks done", done, total)
|
|
last_report = now
|
|
finally:
|
|
# Nothing is left to read these - without this, a pass that ends
|
|
# early (cancelled at shutdown, or a dead pool) leaves "exception
|
|
# was never retrieved" behind for every track still in flight.
|
|
for future in in_flight:
|
|
_discard(future)
|
|
in_flight.clear()
|
|
except BrokenExecutor:
|
|
_log.error(
|
|
"An analysis worker process died (out of memory?) after %d of %d tracks; "
|
|
"stopping this pass. The rest are retried on the next one.",
|
|
done,
|
|
total,
|
|
)
|
|
if done % batch_size:
|
|
await self._publish_analysis(kinds, on_batch)
|
|
if done:
|
|
_log.info("Analyzed %d tracks", done)
|
|
return done
|
|
|
|
def _pending_tracks(self, kinds: Collection[AlbumKind], version: int) -> list[tuple[str, Path]]:
|
|
"""The whole pass's worklist, as (cache key, path), worked out up front.
|
|
|
|
Up front rather than per track, because a pool has to have the next file ready
|
|
the moment a worker frees up, and because it is what lets the progress line say
|
|
"37/412". Deduplicated by cache key: the same file can sit in two albums, and
|
|
two workers analyzing it at once would be pure waste.
|
|
"""
|
|
worklist: list[tuple[str, Path]] = []
|
|
seen: set[str] = set()
|
|
for album in self.albums:
|
|
if album.kind not in kinds:
|
|
continue
|
|
for track in album.tracks:
|
|
key = track_key(track.path)
|
|
if key in seen:
|
|
continue
|
|
seen.add(key)
|
|
cached = self.cache.load_analysis(key)
|
|
if cached is not None and cached.version >= version:
|
|
continue
|
|
worklist.append((key, track.path))
|
|
return worklist
|
|
|
|
def _submit(
|
|
self, pool: Executor | None, loop: asyncio.AbstractEventLoop, path: Path
|
|
) -> asyncio.Future[Any]:
|
|
"""Start one track, in a worker process or - with no pool - on a thread here."""
|
|
if pool is None:
|
|
return asyncio.ensure_future(asyncio.to_thread(analyze_one, self.analyzer, path))
|
|
return loop.run_in_executor(pool, analyze_one, self.analyzer, path)
|
|
|
|
def _store_result(self, key: str, path: Path, future: asyncio.Future[Any]) -> None:
|
|
"""Persist one finished track. Re-raises only what ends the whole pass."""
|
|
error = future.exception()
|
|
if isinstance(error, BrokenExecutor):
|
|
raise error
|
|
if error is not None:
|
|
_log.warning(
|
|
"Analyzer raised on %s; marking it attempted so it is not retried forever",
|
|
path,
|
|
exc_info=error,
|
|
)
|
|
analysis, grid, curve = TrackAnalysis(version=self.analyzer.version), None, None
|
|
else:
|
|
analysis, grid, curve = future.result()
|
|
if grid is not None:
|
|
self.cache.store_beats(key, grid)
|
|
if curve is not None:
|
|
self.cache.store_curve(key, curve)
|
|
self.cache.store_analysis(key, analysis)
|
|
|
|
async def _publish_analysis(
|
|
self, kinds: Collection[AlbumKind], on_batch: Callable[[], Awaitable[None]] | None
|
|
) -> None:
|
|
self._entries = await asyncio.to_thread(self._with_analysis, self._entries, kinds=kinds)
|
|
await asyncio.to_thread(self.cache.store_index, self._entries)
|
|
if on_batch is not None:
|
|
await on_batch()
|