363 lines
15 KiB
Python
363 lines
15 KiB
Python
"""Where scan results live between runs.
|
|
|
|
A directory rather than a single file, because the three kinds of content cost wildly
|
|
different amounts to produce::
|
|
|
|
<cache_dir>/
|
|
├── index.json cheap: tags and structure. Thrown away freely.
|
|
├── covers/<album_id>.thumb.jpg the same art at THUMB_COVER_PX, for cards and rows
|
|
├── covers/<album_id>.jpg medium: the album's art - out of an ID3 APIC
|
|
│ frame, or copied from a cover.jpg in the
|
|
│ folder - downscaled to MAX_COVER_PX on
|
|
│ the way in. An album's `cover` always
|
|
│ points here and never into the library,
|
|
│ so nothing can serve full-size art by
|
|
│ accident.
|
|
└── analysis/<track_key>.json expensive: minutes of DSP per track
|
|
analysis/<track_key>.beats.json
|
|
analysis/<track_key>.curve.json
|
|
|
|
That split is the whole point. A rescan must be free to rebuild ``index.json`` without
|
|
destroying analysis, so everything expensive is keyed by a *content* key (see
|
|
:func:`~musicmouse.library.models.track_key`) rather than by album id - renaming a
|
|
folder or re-sorting a section then costs nothing.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import io
|
|
import json
|
|
import logging
|
|
import os
|
|
from dataclasses import dataclass
|
|
from pathlib import Path
|
|
from typing import Any, Final, cast
|
|
|
|
from PIL import Image, UnidentifiedImageError
|
|
|
|
from musicmouse.library.analysis import BeatGrid, TrackAnalysis, TrackCurves
|
|
from musicmouse.library.models import Album, LibraryTrack
|
|
from musicmouse.library.sections import SECTIONS, AlbumKind
|
|
|
|
_log = logging.getLogger(__name__)
|
|
|
|
__all__ = ["Fingerprint", "LibraryCache"]
|
|
|
|
#: Bump whenever the scanner's *derivation* changes - how a title, artist or series is
|
|
#: worked out - not just when the JSON shape does. A cached entry is reused whenever its
|
|
#: files are untouched, so otherwise a change to that logic is invisible until somebody
|
|
#: edits their music folder.
|
|
# 5: every album's `cover` now points into this cache, downscaled, including the ones
|
|
# that used to point straight at a `cover.jpg` in the library folder.
|
|
# 6: MAX_COVER_PX dropped from 640 to 384. This is what applies it. `shrink_cover` never
|
|
# scales art *up*, so a cover already written at some size is only ever rewritten
|
|
# smaller - which means a cache holding 640 px files cannot be brought to 384 px by
|
|
# re-reading them, only by going back to the original art. Discarding the index does
|
|
# exactly that: every album is scanned again and `store_cover` runs on the real art.
|
|
#
|
|
# Bumping this is cheap by design - see the module docstring.
|
|
_INDEX_VERSION = 6
|
|
|
|
|
|
@dataclass(frozen=True, slots=True)
|
|
class Fingerprint:
|
|
"""What makes a folder's cached entry still valid: its files, sizes and mtimes."""
|
|
|
|
files: tuple[tuple[str, int, int], ...]
|
|
|
|
@classmethod
|
|
def of(cls, paths: list[Path]) -> Fingerprint:
|
|
entries: list[tuple[str, int, int]] = []
|
|
for path in paths:
|
|
stat = path.stat()
|
|
entries.append((path.name, stat.st_size, int(stat.st_mtime)))
|
|
return cls(tuple(entries))
|
|
|
|
def to_json(self) -> list[list[Any]]:
|
|
return [list(entry) for entry in self.files]
|
|
|
|
@classmethod
|
|
def from_json(cls, data: list[list[Any]]) -> Fingerprint:
|
|
return cls(tuple((str(n), int(s), int(m)) for n, s, m in data))
|
|
|
|
|
|
def _write_atomic(path: Path, payload: str) -> None:
|
|
"""Write via a sibling temp file so a crash never leaves a half-written cache."""
|
|
temp = path.with_name(f"{path.name}.tmp{os.getpid()}")
|
|
temp.write_text(payload, encoding="utf-8")
|
|
temp.replace(path)
|
|
|
|
|
|
class LibraryCache:
|
|
def __init__(self, directory: Path) -> None:
|
|
self.directory = directory
|
|
self.covers = directory / "covers"
|
|
self.analysis = directory / "analysis"
|
|
|
|
def prepare(self) -> None:
|
|
for folder in (self.directory, self.covers, self.analysis):
|
|
folder.mkdir(parents=True, exist_ok=True)
|
|
|
|
# -------------------------------------------------------------------- covers
|
|
|
|
def cover_path(self, album_id: str) -> Path:
|
|
return self.covers / f"{album_id}.jpg"
|
|
|
|
def thumb_path(self, album_id: str) -> Path:
|
|
return self.covers / f"{album_id}.thumb.jpg"
|
|
|
|
def store_cover(self, album_id: str, data: bytes) -> Path:
|
|
path = self.cover_path(album_id)
|
|
# The thumb comes from the original art, not from the 384 px copy: one lossy
|
|
# generation fewer, and `Image.draft` lets JPEG decode at a fraction of the size.
|
|
path.write_bytes(shrink_cover(data))
|
|
self.thumb_path(album_id).write_bytes(
|
|
shrink_cover(data, max_px=THUMB_COVER_PX, quality=_THUMB_JPEG_QUALITY)
|
|
)
|
|
return path
|
|
|
|
def _refresh_thumb(self, cover: Path) -> bool:
|
|
"""Make ``<id>.thumb.jpg`` exist and be newer than the cover it came from."""
|
|
thumb = cover.with_name(f"{cover.stem}.thumb.jpg")
|
|
try:
|
|
if thumb.exists() and thumb.stat().st_mtime_ns >= cover.stat().st_mtime_ns:
|
|
return False
|
|
thumb.write_bytes(
|
|
shrink_cover(cover.read_bytes(), max_px=THUMB_COVER_PX, quality=_THUMB_JPEG_QUALITY)
|
|
)
|
|
except OSError: # pragma: no cover - a cache we cannot write is not fatal
|
|
return False
|
|
return True
|
|
|
|
def shrink_stored_covers(self) -> int:
|
|
"""Rewrite any already-stored cover that predates :data:`MAX_COVER_PX`.
|
|
|
|
Needed because :func:`~musicmouse.library.scanner.scan_library` reuses an album
|
|
whose fingerprint is unchanged *without* re-reading its tags, so a cover written
|
|
by an older version would otherwise never be touched again. Reading a JPEG's
|
|
dimensions only parses its header, so once every file is within the limit this
|
|
costs one small read per album and nothing else.
|
|
|
|
Also backfills any missing (or stale) thumbnail. Returns the number of full-size
|
|
files actually rewritten.
|
|
"""
|
|
rewritten = 0
|
|
for path in sorted(self.covers.glob("*.jpg")):
|
|
if path.name.endswith(".thumb.jpg"):
|
|
continue
|
|
try:
|
|
with Image.open(path) as image:
|
|
oversized = max(image.size) > MAX_COVER_PX
|
|
except (OSError, UnidentifiedImageError):
|
|
continue
|
|
if oversized:
|
|
try:
|
|
shrunk = shrink_cover(path.read_bytes())
|
|
path.write_bytes(shrunk)
|
|
except OSError: # pragma: no cover - a cache we cannot write is not fatal
|
|
continue
|
|
rewritten += 1
|
|
# Covers stored before thumbnails existed get theirs here, without a rescan.
|
|
self._refresh_thumb(path)
|
|
return rewritten
|
|
|
|
# ------------------------------------------------------------------ analysis
|
|
|
|
def load_analysis(self, key: str) -> TrackAnalysis | None:
|
|
path = self.analysis / f"{key}.json"
|
|
try:
|
|
return TrackAnalysis.from_json(json.loads(path.read_text(encoding="utf-8")))
|
|
except (OSError, ValueError):
|
|
return None
|
|
|
|
def store_analysis(self, key: str, analysis: TrackAnalysis) -> None:
|
|
_write_atomic(self.analysis / f"{key}.json", json.dumps(analysis.to_json()))
|
|
|
|
def load_beats(self, key: str) -> BeatGrid | None:
|
|
path = self.analysis / f"{key}.beats.json"
|
|
try:
|
|
return BeatGrid.from_json(json.loads(path.read_text(encoding="utf-8")))
|
|
except (OSError, ValueError, KeyError):
|
|
return None
|
|
|
|
def store_beats(self, key: str, grid: BeatGrid) -> None:
|
|
_write_atomic(self.analysis / f"{key}.beats.json", json.dumps(grid.to_json()))
|
|
|
|
def load_curve(self, key: str) -> TrackCurves | None:
|
|
path = self.analysis / f"{key}.curve.json"
|
|
try:
|
|
return TrackCurves.from_json(json.loads(path.read_text(encoding="utf-8")))
|
|
except (OSError, ValueError, KeyError):
|
|
return None
|
|
|
|
def store_curve(self, key: str, curve: TrackCurves) -> None:
|
|
_write_atomic(self.analysis / f"{key}.curve.json", json.dumps(curve.to_json()))
|
|
|
|
# --------------------------------------------------------------------- index
|
|
|
|
def load_index(self) -> dict[str, tuple[Album, Fingerprint]]:
|
|
path = self.directory / "index.json"
|
|
try:
|
|
raw = json.loads(path.read_text(encoding="utf-8"))
|
|
except (OSError, ValueError):
|
|
return {}
|
|
if raw.get("version") != _INDEX_VERSION:
|
|
_log.info("Library index is from an older version; rescanning from scratch")
|
|
return {}
|
|
|
|
out: dict[str, tuple[Album, Fingerprint]] = {}
|
|
for entry in raw.get("albums", []):
|
|
try:
|
|
out[entry["id"]] = (_album_from_json(entry), Fingerprint.from_json(entry["files"]))
|
|
except (KeyError, TypeError, ValueError):
|
|
_log.debug("Dropping unreadable index entry %r", entry.get("id"))
|
|
return out
|
|
|
|
def store_index(self, albums: dict[str, tuple[Album, Fingerprint]]) -> None:
|
|
payload = {
|
|
"version": _INDEX_VERSION,
|
|
"albums": [
|
|
_album_to_json(album) | {"files": fingerprint.to_json()}
|
|
for album, fingerprint in albums.values()
|
|
],
|
|
}
|
|
self.prepare()
|
|
_write_atomic(self.directory / "index.json", json.dumps(payload, ensure_ascii=False))
|
|
|
|
|
|
# ------------------------------------------------------------------ serialisation
|
|
|
|
|
|
def _album_to_json(album: Album) -> dict[str, Any]:
|
|
return {
|
|
"id": album.id,
|
|
"section": album.section,
|
|
"kind": album.kind,
|
|
"title": album.title,
|
|
"artist": album.artist,
|
|
"series": album.series,
|
|
"figure": album.figure,
|
|
"colors": list(album.colors),
|
|
"folder": str(album.folder),
|
|
"cover": str(album.cover) if album.cover else None,
|
|
"tracks": [
|
|
{
|
|
"path": str(track.path),
|
|
"title": track.title,
|
|
"duration": track.duration,
|
|
"analysis": track.analysis.to_json() if track.analysis else None,
|
|
}
|
|
for track in album.tracks
|
|
],
|
|
}
|
|
|
|
|
|
def _album_from_json(data: dict[str, Any]) -> Album:
|
|
section = str(data["section"])
|
|
if section not in SECTIONS:
|
|
raise ValueError(f"unknown section {section!r}")
|
|
# Read back what was stored rather than re-deriving it from the section: a figure's
|
|
# kind comes from the config, so the section cannot answer for it.
|
|
kind = str(data["kind"])
|
|
if kind not in ("music", "book"):
|
|
raise ValueError(f"unknown kind {kind!r}")
|
|
red, green, blue = data["colors"]
|
|
return Album(
|
|
id=str(data["id"]),
|
|
section=section,
|
|
kind=cast("AlbumKind", kind),
|
|
title=str(data["title"]),
|
|
artist=str(data["artist"]),
|
|
series=data["series"],
|
|
figure=data["figure"],
|
|
colors=(str(red), str(green), str(blue)),
|
|
folder=Path(data["folder"]),
|
|
cover=Path(data["cover"]) if data["cover"] else None,
|
|
tracks=tuple(
|
|
LibraryTrack(
|
|
path=Path(track["path"]),
|
|
title=str(track["title"]),
|
|
duration=float(track["duration"]),
|
|
analysis=(
|
|
TrackAnalysis.from_json(track["analysis"]) if track.get("analysis") else None
|
|
),
|
|
)
|
|
for track in data["tracks"]
|
|
),
|
|
)
|
|
|
|
|
|
# ---------------------------------------------------------------------------- covers
|
|
|
|
#: Longest edge kept for cached album art, in pixels.
|
|
#:
|
|
#: The art that comes out of an ID3 APIC frame is sized for a record sleeve, not for a
|
|
#: screen: a real library here averaged 3000x3000 and 580 kB per cover, 140 MB for 284
|
|
#: albums. The browser was decoding nine megapixels - about 36 MB of bitmap - for every
|
|
#: cover it painted, to show it in a 185 px card on a Pi with 2 GB of RAM.
|
|
#:
|
|
#: 340 is the largest any of this app's screens asks for (the play view; the browse grid
|
|
#: asks for 180 and the player bar for 56), so this only has to beat 340 to be lossless
|
|
#: where it shows.
|
|
#:
|
|
#: It was 640 first, for headroom on a tablet at devicePixelRatio 2, and that headroom
|
|
#: turned out to be expensive on the device that actually runs this. Measured on
|
|
#: musicdolphin, typing "conni" over a 343-album library, keydown to painted, worst
|
|
#: keystroke of three runs: 3.4-9.1 s at 640 px against 0.2 s at 384 px. Decode cost
|
|
#: goes with pixel count, and a browse grid paints a card 132 px wide.
|
|
#:
|
|
#: A tablet at devicePixelRatio 2 therefore gets a slightly soft cover on the play view
|
|
#: and nowhere else. That is the trade: a visible sharpness margin nobody asked for, for
|
|
#: a search box that answers a keystroke in a frame.
|
|
MAX_COVER_PX: Final = 384
|
|
|
|
#: Re-encode quality. At these dimensions the difference from 95 is invisible and the
|
|
#: file is a third of the size.
|
|
_COVER_JPEG_QUALITY: Final = 85
|
|
|
|
#: Longest edge of the card/row thumbnail. A grid card is about 217 CSS px wide on a
|
|
#: 1080p kiosk (five columns in the 1180 px grid), so 256 covers it at devicePixelRatio 1
|
|
#: with a little to spare; only the play view asks for the full-size file. Decode cost
|
|
#: and texture upload go with pixel count, and this is 44% of the 384 px file's.
|
|
THUMB_COVER_PX: Final = 256
|
|
|
|
#: Baseline JPEG, 4:2:0 (see :func:`shrink_cover`). libjpeg-turbo is the fastest decoder
|
|
#: a browser on a Pi has, so JPEG stays the format; only the dimensions shrink.
|
|
_THUMB_JPEG_QUALITY: Final = 80
|
|
|
|
|
|
def shrink_cover(
|
|
data: bytes, max_px: int = MAX_COVER_PX, quality: int = _COVER_JPEG_QUALITY
|
|
) -> bytes:
|
|
"""Downscale cover art to ``max_px`` (default :data:`MAX_COVER_PX`) on its longest edge.
|
|
|
|
Art that is already small enough is returned untouched rather than re-encoded, so
|
|
repeated scans never degrade it. Anything Pillow cannot read is passed through
|
|
unchanged: a cover that is too big is a performance problem, a cover that is missing
|
|
is a visible one.
|
|
|
|
The output is always a baseline (non-progressive) 4:2:0 JPEG: the cheapest layout
|
|
for a browser to decode.
|
|
"""
|
|
try:
|
|
with Image.open(io.BytesIO(data)) as image:
|
|
if max(image.size) <= max_px:
|
|
return data
|
|
# JPEG can decode straight to 1/2, 1/4 or 1/8 size; a no-op for other formats.
|
|
image.draft("RGB", (max_px, max_px))
|
|
image = image.convert("RGB")
|
|
# `thumbnail` keeps the aspect ratio and never scales up.
|
|
image.thumbnail((max_px, max_px), Image.Resampling.LANCZOS)
|
|
buffer = io.BytesIO()
|
|
image.save(
|
|
buffer,
|
|
format="JPEG",
|
|
quality=quality,
|
|
optimize=True,
|
|
progressive=False,
|
|
subsampling="4:2:0",
|
|
)
|
|
return buffer.getvalue()
|
|
except (OSError, UnidentifiedImageError, ValueError):
|
|
return data
|