Add librosa beat/mood analysis and an Ambience background driven by it

The background worker now runs a real librosa analyzer (tempo, beat grid,
per-second energy/valence curves) instead of only the null baseline, kept
behind build_analyzer() so a plain checkout without the analysis extra still
runs fine. The web player reads that per-track analysis and drives a new
animated "Ambience" background (bubbles, colour, current) that reacts to the
beat and the mood curve as the track plays, plus a debug overlay for tuning
it. Also adds a one-off script to backfill podcast cover art from iTunes.
This commit is contained in:
2026-09-10 22:42:51 +02:00
parent 8aed3b022b
commit 2e0e6ad199
24 changed files with 2789 additions and 60 deletions

2
.gitignore vendored
View File

@@ -10,3 +10,5 @@ __pycache__
.pytest_cache .pytest_cache
.mypy_cache .mypy_cache
.ruff_cache .ruff_cache
.envrc
.direnv

View File

@@ -17,7 +17,7 @@ import asyncio
import contextlib import contextlib
import logging import logging
import sys import sys
from collections.abc import Coroutine from collections.abc import Awaitable, Callable, Coroutine
from pathlib import Path from pathlib import Path
from typing import Any from typing import Any
@@ -31,6 +31,7 @@ from musicmouse.devices.null_transport import NullTransport
from musicmouse.devices.player import Player, VlcPlayer from musicmouse.devices.player import Player, VlcPlayer
from musicmouse.devices.serial_link import SerialLink from musicmouse.devices.serial_link import SerialLink
from musicmouse.library import MusicLibrary from musicmouse.library import MusicLibrary
from musicmouse.library.analysis import build_analyzer
from musicmouse.reactions import register_all from musicmouse.reactions import register_all
from musicmouse.services.base import Service from musicmouse.services.base import Service
from musicmouse.services.mqtt import MqttService, build_entities from musicmouse.services.mqtt import MqttService, build_entities
@@ -43,9 +44,7 @@ def parse_args(argv: list[str] | None = None) -> argparse.Namespace:
parser = argparse.ArgumentParser( parser = argparse.ArgumentParser(
prog="musicmouse", description="Host backend for the MusicMouse RFID music player." prog="musicmouse", description="Host backend for the MusicMouse RFID music player."
) )
parser.add_argument( parser.add_argument("-c", "--config", type=Path, required=True, help="path to config.yml")
"-c", "--config", type=Path, required=True, help="path to config.yml"
)
parser.add_argument( parser.add_argument(
"-s", "-s",
"--simulate", "--simulate",
@@ -127,9 +126,7 @@ async def run_real(config: Config, config_path: Path, *, hardware: bool = True)
clock=clock, clock=clock,
) )
mouse = MusicMouseDevice(bus, link, config.tag_map, port=general.serial_port) mouse = MusicMouseDevice(bus, link, config.tag_map, port=general.serial_port)
link.attach( link.attach(mouse.feed, on_connect=mouse.on_connected, on_disconnect=mouse.on_disconnected)
mouse.feed, on_connect=mouse.on_connected, on_disconnect=mouse.on_disconnected
)
else: else:
mouse = MusicMouseDevice(bus, NullTransport(), config.tag_map, port="none") mouse = MusicMouseDevice(bus, NullTransport(), config.tag_map, port="none")
@@ -137,9 +134,7 @@ async def run_real(config: Config, config_path: Path, *, hardware: bool = True)
library = await build_library(config) library = await build_library(config)
app = _build_app(config, bus, mouse, player, library, clock=clock) app = _build_app(config, bus, mouse, player, library, clock=clock)
services = _build_services( services = _build_services(app, mouse, player, clock=clock, config_path=config_path)
app, mouse, player, clock=clock, config_path=config_path
)
_log.info( _log.info(
"MusicMouse %s starting: %d figures, %d albums, serial %s, audio %s, mqtt %s", "MusicMouse %s starting: %d figures, %d albums, serial %s, audio %s, mqtt %s",
@@ -157,6 +152,15 @@ async def run_real(config: Config, config_path: Path, *, hardware: bool = True)
[ [
*([link.run()] if link else []), *([link.run()] if link else []),
player.run(), player.run(),
# `is_busy` reads `player.is_playing` directly rather than the library
# knowing about playback at all: analysis must never compete with audio
# decoding for CPU, and this device is otherwise idle most of the day, so
# the pass simply resumes once it is. `on_batch` is unset unless the web
# front-end is on - nothing else has a use for the notification.
library.run_analysis(
is_busy=lambda: player.is_playing,
on_batch=_analysis_batch_hook(services),
),
*(service.run() for service in services), *(service.run() for service in services),
] ]
) )
@@ -184,6 +188,15 @@ async def run_simulated(config: Config, config_path: Path, script: Path | None)
sim.app, sim.app.mouse, sim.player, clock=RealClock(), config_path=config_path sim.app, sim.app.mouse, sim.player, clock=RealClock(), config_path=config_path
) )
tasks = [asyncio.create_task(service.run(), name=service.name) for service in services] tasks = [asyncio.create_task(service.run(), name=service.name) for service in services]
tasks.append(
asyncio.create_task(
sim.app.library.run_analysis(
is_busy=lambda: sim.player.is_playing,
on_batch=_analysis_batch_hook(services),
),
name="library-analysis",
)
)
try: try:
if script is not None: if script is not None:
await run_script_file(sim, script) await run_script_file(sim, script)
@@ -250,6 +263,7 @@ async def build_library(config: Config) -> MusicLibrary:
library_config.root, library_config.root,
library_config.cache, library_config.cache,
frozenset(config.general.audio_extensions), frozenset(config.general.audio_extensions),
analyzer=build_analyzer(),
figure_kinds=config.figure_kinds, figure_kinds=config.figure_kinds,
) )
@@ -279,6 +293,16 @@ def _build_app(
return app return app
def _analysis_batch_hook(services: list[Service]) -> Callable[[], Awaitable[None]] | None:
"""The web front-end's own hub, if it is running - so open tabs refetch the library
as background analysis lands, instead of only after a manual reload. `None` when
there is no web service, which `MusicLibrary.analyze_pending` treats as "nobody to
tell".
"""
web_service = next((s for s in services if isinstance(s, WebService)), None)
return web_service.hub.broadcast_library if web_service else None
def _build_services( def _build_services(
app: App, app: App,
mouse: MusicMouseDevice, mouse: MusicMouseDevice,

View File

@@ -8,12 +8,22 @@ plain immutable data.
from __future__ import annotations from __future__ import annotations
import asyncio import asyncio
import contextlib
import logging import logging
from collections.abc import Mapping import os
from collections.abc import Awaitable, Callable, Collection, Mapping
from dataclasses import replace from dataclasses import replace
from pathlib import Path from pathlib import Path
from typing import Final
from musicmouse.library.analysis import ANALYZER_VERSION, Analyzer, BeatGrid, NullAnalyzer from musicmouse.library.analysis import (
ANALYZER_VERSION,
Analyzer,
BeatGrid,
NullAnalyzer,
TrackAnalysis,
TrackCurves,
)
from musicmouse.library.cache import Fingerprint, LibraryCache from musicmouse.library.cache import Fingerprint, LibraryCache
from musicmouse.library.models import Album, LibraryTrack, album_id, track_key from musicmouse.library.models import Album, LibraryTrack, album_id, track_key
from musicmouse.library.scanner import scan_library from musicmouse.library.scanner import scan_library
@@ -32,10 +42,41 @@ __all__ = [
"LibraryTrack", "LibraryTrack",
"MusicLibrary", "MusicLibrary",
"NullAnalyzer", "NullAnalyzer",
"TrackCurves",
"album_id", "album_id",
"track_key", "track_key",
] ]
#: The only kind worth spending DSP on: an audiobook chapter or a podcast episode is
#: tens of minutes of narration with no musical mood to extract, and there are far more
#: of them in a typical library than there are songs.
_ANALYZED_KINDS: Final[tuple[AlbumKind, ...]] = ("music",)
#: How often the background worker checks whether it may resume after `is_busy()` said
#: no. A track's own analysis is a couple of seconds of CPU, so overshooting by this
#: much when playback stops is not worth polling harder for.
_BUSY_POLL_SECONDS: Final = 2.0
#: Analysis is folded back into the live index and persisted this often during a long
#: run, so a first-time pass over an unanalyzed library shows up gradually in open
#: browser tabs rather than only after the whole thing finishes.
_PUBLISH_BATCH_SIZE: Final = 25
def _analyze_one(
analyzer: Analyzer, path: Path
) -> tuple[TrackAnalysis, BeatGrid | None, TrackCurves | None]:
"""Runs in a worker thread. Lowers this thread's own scheduling priority first.
On Linux, ``os.nice`` affects only the calling thread, not the whole process - so
this makes idle-time analysis yield CPU to anything else without touching threads
used for other work. Niceness only ever increases and clamps at the OS maximum
(19), so calling this repeatedly on a reused pool thread is harmless.
"""
with contextlib.suppress(OSError):
os.nice(1)
return analyzer.analyze(path)
class MusicLibrary: class MusicLibrary:
"""An immutable index of albums, rebuilt wholesale rather than mutated in place.""" """An immutable index of albums, rebuilt wholesale rather than mutated in place."""
@@ -56,6 +97,10 @@ class MusicLibrary:
#: What each figure holds. The only thing a folder name cannot say. #: What each figure holds. The only thing a folder name cannot say.
self.figure_kinds: Mapping[str, AlbumKind] = figure_kinds or {} self.figure_kinds: Mapping[str, AlbumKind] = figure_kinds or {}
self._entries: dict[str, tuple[Album, Fingerprint]] = {} self._entries: dict[str, tuple[Album, Fingerprint]] = {}
#: Set by `request_analysis`, consumed by `run_analysis`. An `Event` rather than
#: a queue: a request raised while one is already pending or running just
#: coalesces into it, which is exactly what "look again" should mean here.
self._analysis_requested = asyncio.Event()
# -------------------------------------------------------------------- reading # -------------------------------------------------------------------- reading
@@ -85,10 +130,21 @@ class MusicLibrary:
return None return None
return self.cache.load_beats(track_key(album.tracks[index].path)) return self.cache.load_beats(track_key(album.tracks[index].path))
def curve(self, identifier: str, index: int) -> TrackCurves | None:
album = self.get(identifier)
if album is None or not 0 <= index < len(album.tracks):
return None
return self.cache.load_curve(track_key(album.tracks[index].path))
# -------------------------------------------------------------------- writing # -------------------------------------------------------------------- writing
async def refresh(self) -> None: async def refresh(self) -> None:
"""Rescan from disk. Blocking work happens off the loop; the swap is atomic.""" """Rescan from disk. Blocking work happens off the loop; the swap is atomic.
Ends by waking the background analyzer: a rescan is exactly when new tracks -
the only ones analysis can be pending for - enter the index, whether that is
the startup scan or a parent tapping "Bibliothek neu einlesen".
"""
known = dict(self._entries) known = dict(self._entries)
entries = await asyncio.to_thread( entries = await asyncio.to_thread(
scan_library, scan_library,
@@ -100,19 +156,28 @@ class MusicLibrary:
) )
self._entries = await asyncio.to_thread(self._with_analysis, entries) self._entries = await asyncio.to_thread(self._with_analysis, entries)
await asyncio.to_thread(self.cache.store_index, self._entries) await asyncio.to_thread(self.cache.store_index, self._entries)
self.request_analysis()
def _with_analysis( def _with_analysis(
self, entries: dict[str, tuple[Album, Fingerprint]] self,
entries: dict[str, tuple[Album, Fingerprint]],
*,
kinds: Collection[AlbumKind] = _ANALYZED_KINDS,
) -> dict[str, tuple[Album, Fingerprint]]: ) -> dict[str, tuple[Album, Fingerprint]]:
"""Fold cached analysis results into the freshly scanned index. """Fold cached analysis results into the freshly scanned index.
Skipped entirely while ``analysis/`` is empty, which is the normal case until an Restricted to `kinds` (music by default): stat-ing hundreds of book and podcast
analyzer has actually been run - no point stat-ing 900 files that cannot exist. tracks whose analysis can never exist would be pure waste. Skipped entirely
while ``analysis/`` is empty, which is the normal case until an analyzer has
actually been run.
""" """
if not any(self.cache.analysis.glob("*.json")): if not any(self.cache.analysis.glob("*.json")):
return entries return entries
out: dict[str, tuple[Album, Fingerprint]] = {} out: dict[str, tuple[Album, Fingerprint]] = {}
for identifier, (album, fingerprint) in entries.items(): for identifier, (album, fingerprint) in entries.items():
if album.kind not in kinds:
out[identifier] = (album, fingerprint)
continue
tracks = tuple( tracks = tuple(
replace(track, analysis=self.cache.load_analysis(track_key(track.path))) replace(track, analysis=self.cache.load_analysis(track_key(track.path)))
for track in album.tracks for track in album.tracks
@@ -136,12 +201,52 @@ class MusicLibrary:
await library.refresh() await library.refresh()
return library return library
async def analyze_pending(self) -> int: # ------------------------------------------------------------------- analysis
"""Run the analyzer over tracks that have no current result.
Never called during startup: the index is what playback needs, and analysis is def request_analysis(self) -> None:
minutes of DSP per track. Results are keyed by file content, so they can equally """Wake the background worker to look for tracks with no current analysis.
well be produced on a faster machine and the ``analysis/`` folder copied over.
Idempotent, and safe to call before `run_analysis` has even started a first
time - the request just waits on the event.
"""
self._analysis_requested.set()
async def run_analysis(
self,
*,
is_busy: Callable[[], bool] = lambda: False,
on_batch: Callable[[], Awaitable[None]] | None = None,
) -> None:
"""Analyze pending tracks whenever `request_analysis` wakes this up.
A long-running task, cancelled at shutdown alongside every other one. Loops
forever so a request raised *during* a pass (a rescan mid-analysis) starts
another pass right after, rather than being lost.
"""
while True:
await self._analysis_requested.wait()
self._analysis_requested.clear()
await self.analyze_pending(is_busy=is_busy, on_batch=on_batch)
async def analyze_pending(
self,
*,
kinds: Collection[AlbumKind] = _ANALYZED_KINDS,
is_busy: Callable[[], bool] = lambda: False,
on_batch: Callable[[], Awaitable[None]] | None = None,
batch_size: int = _PUBLISH_BATCH_SIZE,
) -> int:
"""Run the analyzer over tracks of `kinds` that have no current result.
Restricted to music by default - see `_ANALYZED_KINDS`. Checked before every
track, `is_busy()` pauses the whole pass rather than one file: analysis must
never compete with audio decoding for CPU, and a children's player is idle most
of the day, so the pass simply resumes next time it is. A track the analyzer
fails on (corrupt file, DRM, zero length) is still recorded as attempted - with
every scalar left `None` - so it is never retried forever and the frontend falls
back to the un-analyzed baseline for it. Results are folded into the live index
and persisted every `batch_size` tracks, so a long first run is visible in open
browser tabs as it goes rather than only once it finishes.
""" """
analyzer = self.analyzer analyzer = self.analyzer
if analyzer.version < ANALYZER_VERSION: if analyzer.version < ANALYZER_VERSION:
@@ -149,16 +254,44 @@ class MusicLibrary:
done = 0 done = 0
for album in self.albums: for album in self.albums:
if album.kind not in kinds:
continue
for track in album.tracks: for track in album.tracks:
while is_busy():
await asyncio.sleep(_BUSY_POLL_SECONDS)
key = track_key(track.path) key = track_key(track.path)
cached = self.cache.load_analysis(key) cached = self.cache.load_analysis(key)
if cached is not None and cached.version >= analyzer.version: if cached is not None and cached.version >= analyzer.version:
continue continue
analysis, grid = await asyncio.to_thread(analyzer.analyze, track.path) try:
analysis, grid, curve = await asyncio.to_thread(
_analyze_one, analyzer, track.path
)
except Exception:
_log.warning(
"Analyzer raised on %s; marking it attempted so it is not retried forever",
track.path,
exc_info=True,
)
analysis, grid, curve = TrackAnalysis(version=analyzer.version), None, None
if grid is not None: if grid is not None:
self.cache.store_beats(key, grid) self.cache.store_beats(key, grid)
if curve is not None:
self.cache.store_curve(key, curve)
self.cache.store_analysis(key, analysis) self.cache.store_analysis(key, analysis)
done += 1 done += 1
if done % batch_size == 0:
await self._publish_analysis(kinds, on_batch)
if done % batch_size:
await self._publish_analysis(kinds, on_batch)
if done: if done:
_log.info("Analyzed %d tracks", done) _log.info("Analyzed %d tracks", done)
return done return done
async def _publish_analysis(
self, kinds: Collection[AlbumKind], on_batch: Callable[[], Awaitable[None]] | None
) -> None:
self._entries = await asyncio.to_thread(self._with_analysis, self._entries, kinds=kinds)
await asyncio.to_thread(self.cache.store_index, self._entries)
if on_batch is not None:
await on_batch()

View File

@@ -1,16 +1,20 @@
"""Offline audio analysis: the seam, not the implementation. """Offline audio analysis, and the seam that lets it be optional.
Nothing here computes anything yet. What it does is fix the shape of the results so :class:`LibrosaAnalyzer` (``musicmouse.library.librosa_analyzer``) does the real work,
that the analyzer can arrive later without touching the scanner, the cache format, the kept in its own module behind :func:`build_analyzer` so that importing *this* module -
API contract or the frontend's data flow. which the scanner, the cache and the web API all do - never pulls in librosa or numpy.
:class:`MusicLibrary` runs whichever analyzer it is given in the background, between
tracks, so the reactive background is a thing a library grows into rather than a
migration.
Two rules hold the design together: Two rules hold the result shape together:
* **Scalars travel with the index, time series do not.** A 25-minute podcast at 120 BPM * **Scalars travel with the index, time series do not.** A 25-minute podcast at 120 BPM
has ~3000 beats; 900 tracks of that inside ``GET /api/library`` would be tens of has ~3000 beats; 900 tracks of that inside ``GET /api/library`` would be tens of
megabytes. :class:`TrackAnalysis` is a handful of floats and rides along; the beat megabytes. :class:`TrackAnalysis` is a handful of floats and rides along; the beat
grid lives in its own file and is fetched for the one track that is playing. grid and the per-second :class:`TrackCurves` each live in their own file and are
* **Every field is optional with a default.** Adding ``danceability`` later needs no fetched only for the one track that is playing.
* **Every field is optional with a default.** Adding a new scalar later needs no
migration and no cache wipe: unknown keys on disk are dropped on load, missing ones migration and no cache wipe: unknown keys on disk are dropped on load, missing ones
fall back to the default. Only :data:`ANALYZER_VERSION` moving past what a file fall back to the default. Only :data:`ANALYZER_VERSION` moving past what a file
records marks that file stale. records marks that file stale.
@@ -18,21 +22,28 @@ Two rules hold the design together:
from __future__ import annotations from __future__ import annotations
import logging
from dataclasses import asdict, dataclass, fields from dataclasses import asdict, dataclass, fields
from pathlib import Path from pathlib import Path
from typing import Any, Final, Protocol from typing import Any, Final, Protocol
_log = logging.getLogger(__name__)
__all__ = [ __all__ = [
"ANALYZER_VERSION", "ANALYZER_VERSION",
"Analyzer", "Analyzer",
"BeatGrid", "BeatGrid",
"NullAnalyzer", "NullAnalyzer",
"TrackAnalysis", "TrackAnalysis",
"TrackCurves",
"build_analyzer",
] ]
#: Bumped when an analyzer's output changes meaning. Cached results recorded under a #: Bumped when an analyzer's output changes meaning. Cached results recorded under a
#: lower version are recomputed; results at or above it are left alone. #: lower version are recomputed; results at or above it are left alone. 2: energy/
ANALYZER_VERSION: Final = 1 #: valence scalars became mean-of-curve instead of whole-track-percentile/heuristic,
#: and every track needs a new TrackCurves artifact producing.
ANALYZER_VERSION: Final = 2
@dataclass(frozen=True, slots=True) @dataclass(frozen=True, slots=True)
@@ -48,6 +59,10 @@ class TrackAnalysis:
valence: float | None = None valence: float | None = None
#: 0..1 spectral centroid. Drives the background hue. #: 0..1 spectral centroid. Drives the background hue.
brightness: float | None = None brightness: float | None = None
#: 0..1 confidence that :attr:`tempo` is an audible, steady beat rather than an
#: artifact of free-tempo or spoken-word material. Below-threshold tracks should
#: not be pulsed on the beat even though a grid exists for them.
pulse: float | None = None
#: Whether a beat grid for this track exists on disk. #: Whether a beat grid for this track exists on disk.
beats: bool = False beats: bool = False
@@ -85,22 +100,80 @@ class BeatGrid:
return cls(tuple(flat[0::2]), tuple(flat[1::2])) return cls(tuple(flat[0::2]), tuple(flat[1::2]))
@dataclass(frozen=True, slots=True)
class TrackCurves:
"""Per-second samples of the things that vary *within* a track. Regularly
sampled, so just a hop and equal-length arrays, no per-sample timestamps -
contrast :class:`BeatGrid`'s event-based irregular times.
``energy``/``valence`` drive colour; ``drive`` (rhythmic intensity) modulates the
water current's magnitude. Tempo is deliberately absent: measured against a real
library it is flat to within a few percent inside a track, so a per-second tempo
curve would carry estimator noise (including occasional octave errors) and
nothing else. Whole-track tempo keeps setting the current's base magnitude;
``drive`` is the signal that actually varies.
"""
hop_seconds: float
energy: tuple[float, ...]
valence: tuple[float, ...]
drive: tuple[float, ...]
def to_json(self) -> dict[str, Any]:
return {
"hop_seconds": self.hop_seconds,
"energy": list(self.energy),
"valence": list(self.valence),
"drive": list(self.drive),
}
@classmethod
def from_json(cls, data: dict[str, Any]) -> TrackCurves:
return cls(
hop_seconds=float(data["hop_seconds"]),
energy=tuple(data["energy"]),
valence=tuple(data["valence"]),
drive=tuple(data["drive"]),
)
class Analyzer(Protocol): class Analyzer(Protocol):
"""Turns one audio file into cacheable analysis results. """Turns one audio file into cacheable analysis results.
Implementations are CPU-bound and run off the event loop. The real one will live Implementations are CPU-bound and run off the event loop.
behind an optional dependency group so the device never installs numpy to play music.
""" """
version: int version: int
def analyze(self, path: Path) -> tuple[TrackAnalysis, BeatGrid | None]: ... def analyze(self, path: Path) -> tuple[TrackAnalysis, BeatGrid | None, TrackCurves | None]: ...
class NullAnalyzer: class NullAnalyzer:
"""Analyzes nothing. Keeps the wiring exercised until a real analyzer lands.""" """Analyzes nothing. What :func:`build_analyzer` falls back to without librosa."""
version = 0 version = 0
def analyze(self, path: Path) -> tuple[TrackAnalysis, BeatGrid | None]: # noqa: ARG002 def analyze(
return TrackAnalysis(), None self,
path: Path, # noqa: ARG002
) -> tuple[TrackAnalysis, BeatGrid | None, TrackCurves | None]:
return TrackAnalysis(), None, None
def build_analyzer() -> Analyzer:
"""The real analyzer if its optional dependency group is installed, else a no-op.
``NullAnalyzer.version`` is ``0``, which is below :data:`ANALYZER_VERSION`, so
:meth:`~musicmouse.library.MusicLibrary.analyze_pending` returns immediately and the
background stays at its static baseline - not a crash, not a degraded mode, just the
feature switched off until ``pip install -e '.[analysis]'`` turns it on.
"""
try:
from musicmouse.library.librosa_analyzer import LibrosaAnalyzer
except ImportError:
_log.info(
"librosa is not installed: the reactive background is off. "
"Install the 'analysis' extra to enable it."
)
return NullAnalyzer()
return LibrosaAnalyzer()

View File

@@ -8,6 +8,7 @@ different amounts to produce::
├── covers/<album_id>.jpg medium: art pulled out of an ID3 APIC frame ├── covers/<album_id>.jpg medium: art pulled out of an ID3 APIC frame
└── analysis/<track_key>.json expensive: minutes of DSP per track └── analysis/<track_key>.json expensive: minutes of DSP per track
analysis/<track_key>.beats.json analysis/<track_key>.beats.json
analysis/<track_key>.curve.json
That split is the whole point. A rescan must be free to rebuild ``index.json`` without That split is the whole point. A rescan must be free to rebuild ``index.json`` without
destroying analysis, so everything expensive is keyed by a *content* key (see destroying analysis, so everything expensive is keyed by a *content* key (see
@@ -24,7 +25,7 @@ from dataclasses import dataclass
from pathlib import Path from pathlib import Path
from typing import Any, cast from typing import Any, cast
from musicmouse.library.analysis import BeatGrid, TrackAnalysis from musicmouse.library.analysis import BeatGrid, TrackAnalysis, TrackCurves
from musicmouse.library.models import Album, LibraryTrack from musicmouse.library.models import Album, LibraryTrack
from musicmouse.library.sections import SECTIONS, AlbumKind from musicmouse.library.sections import SECTIONS, AlbumKind
@@ -110,6 +111,16 @@ class LibraryCache:
def store_beats(self, key: str, grid: BeatGrid) -> None: def store_beats(self, key: str, grid: BeatGrid) -> None:
_write_atomic(self.analysis / f"{key}.beats.json", json.dumps(grid.to_json())) _write_atomic(self.analysis / f"{key}.beats.json", json.dumps(grid.to_json()))
def load_curve(self, key: str) -> TrackCurves | None:
path = self.analysis / f"{key}.curve.json"
try:
return TrackCurves.from_json(json.loads(path.read_text(encoding="utf-8")))
except (OSError, ValueError, KeyError):
return None
def store_curve(self, key: str, curve: TrackCurves) -> None:
_write_atomic(self.analysis / f"{key}.curve.json", json.dumps(curve.to_json()))
# --------------------------------------------------------------------- index # --------------------------------------------------------------------- index
def load_index(self) -> dict[str, tuple[Album, Fingerprint]]: def load_index(self) -> dict[str, tuple[Album, Fingerprint]]:

View File

@@ -0,0 +1,318 @@
"""A concrete :class:`~musicmouse.library.analysis.Analyzer` built on librosa.
Only reached through :func:`musicmouse.library.analysis.build_analyzer`, so nothing
else in the app ever imports librosa or numpy: a device without the ``analysis`` extra
installed never executes this module at all.
Every scalar here is a signal-processing proxy, not a measurement of how a track
actually feels - ``valence`` most of all, see its section below. Good enough to drive
an ambient background; not a music information retrieval research result.
"""
from __future__ import annotations
import logging
import math
import warnings
from collections.abc import Callable
from pathlib import Path
import librosa
import numpy as np
from musicmouse.library.analysis import ANALYZER_VERSION, BeatGrid, TrackAnalysis, TrackCurves
_log = logging.getLogger(__name__)
__all__ = ["LibrosaAnalyzer"]
#: librosa's own default. Ample for everything below - the highest band that matters,
#: the brightness ceiling, sits well under this rate's 11025 Hz Nyquist frequency.
_SAMPLE_RATE = 22050
_HOP_LENGTH = 512
#: `energy`: the 80th-percentile RMS frame, in dB, mapped floor..ceil to 0..1. -32 dB is
#: a quiet passage, -8 dB is a hot, compressed master. A percentile rather than the mean
#: or max so a quiet intro or a gap between phrases doesn't drag a loud track down, and
#: one clipped peak doesn't blow it out.
_ENERGY_DB_FLOOR = -32.0
_ENERGY_DB_CEIL = -8.0
#: `brightness`: the median spectral centroid (robust to a single loud transient), in
#: Hz, log-mapped floor..ceil to 0..1. 300 Hz is a dark, bass/vocal-heavy mix; 4000 Hz is
#: bright, sparkly production.
_BRIGHTNESS_HZ_FLOOR = 300.0
_BRIGHTNESS_HZ_CEIL = 4000.0
#: `valence`'s tempo term: 60 BPM reads as a lullaby, 150 BPM as a romp.
_TEMPO_BPM_FLOOR = 60.0
_TEMPO_BPM_CEIL = 150.0
#: Key is a global property of a track; the middle minute is its most representative
#: one and this keeps the most expensive feature (chroma) off the tail of a long track.
_CHROMA_EXCERPT_SECONDS = 120.0
#: Seconds per curve sample. Fixed at analysis time, not user-facing - changing this
#: needs a re-analysis and an ANALYZER_VERSION bump, unlike the frontend's own
#: unrelated "curve sample interval" debug slider, which just smooths already-fetched
#: samples for live preview.
_ANALYSIS_HOP_SECONDS = 1.0
#: `drive`'s blend of onset activity (steadier, measured 33-55% relative spread across
#: a track) and local pulse strength (more dynamic but spikier, 59-189%) - weighted
#: toward activity so a busy chorus reads clearly without the current twitching on
#: every transient.
_DRIVE_ACTIVITY_WEIGHT = 0.6
_DRIVE_PLP_WEIGHT = 0.4
#: Krumhansl-Kessler key profiles (Krumhansl & Kessler 1982), starting from C. Every
#: other key is scored by rotating these twelve weights, not by transposing the audio.
_MAJOR_PROFILE = np.array([6.35, 2.23, 3.48, 2.33, 4.38, 4.09, 2.52, 5.19, 2.39, 3.66, 2.29, 2.88])
_MINOR_PROFILE = np.array([6.33, 2.68, 3.52, 5.38, 2.60, 3.53, 2.54, 4.75, 3.98, 2.69, 3.34, 3.17])
def _clip01(value: float) -> float:
return max(0.0, min(1.0, value))
def _normalize(value: float, floor: float, ceil: float) -> float:
"""Linear map ``floor..ceil`` to ``0..1``, clipped at both ends."""
return _clip01((value - floor) / (ceil - floor))
def _center_excerpt(y: np.ndarray, sr: float, seconds: float) -> np.ndarray:
max_samples = int(seconds * sr)
if y.size <= max_samples:
return y
start = (y.size - max_samples) // 2
return y[start : start + max_samples]
def _frames_per_window(sr: float, hop_length: int, window_seconds: float) -> int:
return max(1, round(window_seconds * sr / hop_length))
def _windowed(
values: np.ndarray, frames_per_window: int, reduce: Callable[[np.ndarray], float]
) -> np.ndarray:
"""Buckets an already-computed per-frame array (RMS, spectral centroid, onset
envelope, PLP - anything on the STFT hop grid) into `frames_per_window`-wide
windows, applying `reduce` to each. The last window is short rather than dropped,
so a track's tail is never silently excluded from its own curve."""
n = max(1, math.ceil(values.size / frames_per_window))
out = np.empty(n)
for i in range(n):
w = values[i * frames_per_window : (i + 1) * frames_per_window]
out[i] = reduce(w if w.size else values[-1:])
return out
def _energy_window(w: np.ndarray) -> float:
"""The same 80th-percentile+dB statistic as the whole-track `energy` scalar,
applied to one window."""
db = librosa.amplitude_to_db(np.array([np.percentile(w, 80)]), ref=1.0)[0]
return _normalize(float(db), _ENERGY_DB_FLOOR, _ENERGY_DB_CEIL)
def _brightness_window(w: np.ndarray) -> float:
"""The same median+log statistic as the whole-track `brightness` scalar, applied
to one window."""
hz = max(float(np.median(w)), 1.0) # guard log2(0)
floor, ceil = math.log2(_BRIGHTNESS_HZ_FLOOR), math.log2(_BRIGHTNESS_HZ_CEIL)
return _normalize(math.log2(hz), floor, ceil)
def _norm95(values: np.ndarray) -> np.ndarray:
"""Scale so the array's 95th percentile maps to 1.0 - a robust max that one loud
transient can't blow out, the same reasoning as the beat grid's strengths."""
ceiling = float(np.percentile(values, 95))
return values / ceiling if ceiling > 0 else np.zeros_like(values)
def _majorness(chroma_mean: np.ndarray) -> float:
"""Best major key-profile correlation minus best minor one, over all 12 rotations.
Positive means the track's pitch-class distribution fits a major key better than
any minor one; negative the other way round. `np.corrcoef` is undefined for a
perfectly flat chroma vector (silence, pure noise) - `nan_to_num` turns that into
"no signal either way" rather than raising.
"""
def best_fit(profile: np.ndarray) -> float:
return max(
float(np.nan_to_num(np.corrcoef(chroma_mean, np.roll(profile, i))[0, 1]))
for i in range(12)
)
return best_fit(_MAJOR_PROFILE) - best_fit(_MINOR_PROFILE)
class LibrosaAnalyzer:
"""Turns one music file into :class:`TrackAnalysis` plus an optional beat grid
and :class:`TrackCurves`."""
version = ANALYZER_VERSION
def analyze(self, path: Path) -> tuple[TrackAnalysis, BeatGrid | None, TrackCurves | None]:
"""Never raises: a file this can't make sense of is analyzed as "nothing".
A corrupt file, a DRM'd one, or a zero-length one must not abort a batch of
hundreds - the fallback records `version` so :meth:`MusicLibrary.analyze_pending`
does not retry it forever, while every scalar stays `None` so the frontend falls
back to the un-analyzed baseline look rather than something half-computed.
"""
try:
y, sr = librosa.load(path, sr=_SAMPLE_RATE, mono=True)
except Exception:
_log.warning("Could not decode %s; leaving it unanalyzed", path, exc_info=True)
return TrackAnalysis(version=self.version), None, None
if y.size == 0:
_log.warning("%s decoded to no audio; leaving it unanalyzed", path)
return TrackAnalysis(version=self.version), None, None
try:
return self._analyze(y, sr)
except Exception:
_log.warning("Analysis failed for %s; leaving it unanalyzed", path, exc_info=True)
return TrackAnalysis(version=self.version), None, None
def _analyze(
self, y: np.ndarray, sr: float
) -> tuple[TrackAnalysis, BeatGrid | None, TrackCurves | None]:
# One onset envelope feeds tempo, the beat grid's strengths, `pulse` and
# `drive` - the single most expensive shared computation, so it is done once.
onset_env = librosa.onset.onset_strength(y=y, sr=sr, hop_length=_HOP_LENGTH)
tempo_raw, beat_frames = librosa.beat.beat_track(
onset_envelope=onset_env, sr=sr, hop_length=_HOP_LENGTH
)
tempo_bpm = float(np.atleast_1d(tempo_raw)[0])
grid = self._beat_grid(onset_env, beat_frames, sr)
frames_per_window = _frames_per_window(sr, _HOP_LENGTH, _ANALYSIS_HOP_SECONDS)
rms = librosa.feature.rms(y=y, hop_length=_HOP_LENGTH)[0]
energy_curve = _windowed(rms, frames_per_window, _energy_window)
energy = float(np.mean(energy_curve))
centroid = librosa.feature.spectral_centroid(y=y, sr=sr, hop_length=_HOP_LENGTH)[0]
# Scalar `brightness` stays the whole-track median exactly as before this
# refactor - it no longer drives anything on the frontend (only its curve
# feeds `valence` below), so its own meaning is deliberately left unchanged.
brightness = _brightness_window(centroid)
brightness_curve = _windowed(centroid, frames_per_window, _brightness_window)
majorness_norm, tempo_norm = self._valence_terms(y, sr, tempo_bpm)
valence_curve = np.clip(
0.5 * majorness_norm + 0.3 * brightness_curve + 0.2 * tempo_norm, 0.0, 1.0
)
valence = float(np.mean(valence_curve))
pulse = self._pulse(onset_env, sr, tempo_bpm)
drive_curve = self._drive_curve(onset_env, sr, frames_per_window)
curves = TrackCurves(
hop_seconds=_ANALYSIS_HOP_SECONDS,
energy=tuple(float(v) for v in energy_curve),
valence=tuple(float(v) for v in valence_curve),
drive=tuple(float(v) for v in drive_curve),
)
analysis = TrackAnalysis(
version=self.version,
tempo=tempo_bpm,
energy=energy,
valence=valence,
brightness=brightness,
pulse=pulse,
beats=grid is not None,
)
return analysis, grid, curves
def _beat_grid(
self, onset_env: np.ndarray, beat_frames: np.ndarray, sr: float
) -> BeatGrid | None:
if beat_frames.size == 0:
return None
times = librosa.frames_to_time(beat_frames, sr=sr, hop_length=_HOP_LENGTH)
raw_strengths = onset_env[np.clip(beat_frames, 0, onset_env.size - 1)]
# The 95th percentile rather than the max, so one loud crash does not flatten
# every other beat's strength toward zero.
scale = float(np.percentile(raw_strengths, 95))
strengths = raw_strengths / scale if scale > 0 else np.zeros_like(raw_strengths)
return BeatGrid(
tuple(float(t) for t in times),
tuple(_clip01(float(s)) for s in strengths),
)
def _valence_terms(self, y: np.ndarray, sr: float, tempo_bpm: float) -> tuple[float, float]:
"""majorness_norm, tempo_norm - the two whole-track-constant terms of the
valence formula. See the module docstring for why valence is a heuristic, not
a measurement.
Weighted for a *children's* library specifically: mode (major/minor) is the
strongest and most legible cue in this repertoire - a minor-key children's song
is almost always deliberately sad or spooky, unlike in pop where mode is a much
weaker signal. Tempo adds romp-vs-lullaby. Both stay whole-track constants:
key is a global property of a track, and (unlike brightness) chroma-based key
detection is too expensive and too noisy over a short window to be worth
computing per second for what is now only a secondary, "nudge" contribution to
color - see `TrackCurves.valence`.
"""
excerpt = _center_excerpt(y, sr, _CHROMA_EXCERPT_SECONDS)
chroma = librosa.feature.chroma_cqt(y=excerpt, sr=sr, hop_length=_HOP_LENGTH)
majorness = _majorness(chroma.mean(axis=1))
majorness_norm = _clip01((majorness + 1.0) / 2.0)
tempo_norm = _normalize(tempo_bpm, _TEMPO_BPM_FLOOR, _TEMPO_BPM_CEIL)
return majorness_norm, tempo_norm
def _drive_curve(self, onset_env: np.ndarray, sr: float, frames_per_window: int) -> np.ndarray:
"""Rhythmic intensity per window, 0..1, normalised *within the track*.
This - not tempo - is what the frontend's water current breathes with while a
track plays. Measured against a real library, local tempo is flat to within a
few percent inside a track (recorded children's music is played to a click);
the residual "variation" a per-second tempo curve would show is mostly
estimator noise, including occasional octave errors. Onset activity and local
pulse strength (PLP) both genuinely vary within a track (33-55% and 59-189%
relative spread respectively) and don't carry that failure mode.
Per-track normalisation (each component scaled by its own 95th percentile) is
deliberate: absolute "how energetic is this song" is already carried by
whole-track `tempo` (the current's base magnitude) and by `energy` (colour).
This curve is for relative shape within the track - the intro is calmer than
the chorus - so every track uses its own full 0..1 range rather than a
uniformly quiet song sitting flat near zero throughout.
"""
activity = _windowed(onset_env, frames_per_window, lambda w: float(np.mean(w)))
plp = librosa.beat.plp(onset_envelope=onset_env, sr=sr, hop_length=_HOP_LENGTH)
pulse_curve = _windowed(plp, frames_per_window, lambda w: float(np.mean(w)))
activity_term = _DRIVE_ACTIVITY_WEIGHT * _norm95(activity)
pulse_term = _DRIVE_PLP_WEIGHT * _norm95(pulse_curve)
return np.clip(activity_term + pulse_term, 0.0, 1.0)
def _pulse(self, onset_env: np.ndarray, sr: float, tempo_bpm: float) -> float:
"""0..1 confidence that `tempo` is an audible, steady beat.
The onset envelope's autocorrelation at the beat period, relative to its value
at lag zero: a track that truly pulses at `tempo` has a strong echo of itself
one beat later; free-tempo or spoken-word material does not, even though
`beat_track` always returns *some* grid for it.
"""
if tempo_bpm <= 0 or onset_env.size < 2:
return 0.0
period_frames = round((60.0 / tempo_bpm) * sr / _HOP_LENGTH)
if not 0 < period_frames < onset_env.size:
return 0.0
with warnings.catch_warnings():
# A known-spurious warning from numba's complex-magnitude dufunc under
# this numba/numpy/librosa combination (confirmed: the input here has no
# NaN/Inf, and the result is a normal finite float) - narrowly silenced
# rather than left to spam the log once per track analyzed.
warnings.filterwarnings(
"ignore", message="invalid value encountered in cast", category=RuntimeWarning
)
ac = librosa.autocorrelate(onset_env)
if ac[0] <= 0:
return 0.0
return _clip01(float(ac[period_frames] / ac[0]))

View File

@@ -34,7 +34,6 @@ from musicmouse.events import (
from musicmouse.services.web.hub import StateHub from musicmouse.services.web.hub import StateHub
from musicmouse.services.web.schemas import ( from musicmouse.services.web.schemas import (
AlbumOut, AlbumOut,
BeatsOut,
HaConfigOut, HaConfigOut,
HaDeviceOut, HaDeviceOut,
LibraryOut, LibraryOut,
@@ -43,6 +42,8 @@ from musicmouse.services.web.schemas import (
SeekIn, SeekIn,
SettingsIn, SettingsIn,
SettingsOut, SettingsOut,
TrackCurvesOut,
TrackDetailOut,
VolumeIn, VolumeIn,
) )
from musicmouse.services.web.settings import ( from musicmouse.services.web.settings import (
@@ -86,11 +87,16 @@ def build_router(
) )
@router.get("/tracks/{album_id}/{index}/analysis") @router.get("/tracks/{album_id}/{index}/analysis")
def get_track_analysis(album_id: str, index: int) -> BeatsOut: def get_track_analysis(album_id: str, index: int) -> TrackDetailOut:
grid = app.library.beats(album_id, index) grid = app.library.beats(album_id, index)
if grid is None: curve = app.library.curve(album_id, index)
if grid is None and curve is None:
raise HTTPException(status_code=404, detail="not analyzed") raise HTTPException(status_code=404, detail="not analyzed")
return BeatsOut(times=list(grid.times), strengths=list(grid.strengths)) return TrackDetailOut(
times=list(grid.times) if grid else [],
strengths=list(grid.strengths) if grid else [],
curve=TrackCurvesOut(**curve.to_json()) if curve else None,
)
@router.post("/library/refresh", status_code=202) @router.post("/library/refresh", status_code=202)
async def refresh_library() -> Response: async def refresh_library() -> Response:

View File

@@ -15,7 +15,6 @@ from musicmouse.library.analysis import TrackAnalysis
__all__ = [ __all__ = [
"AlbumOut", "AlbumOut",
"BeatsOut",
"HaConfigOut", "HaConfigOut",
"HaDeviceOut", "HaDeviceOut",
"LibraryOut", "LibraryOut",
@@ -24,6 +23,8 @@ __all__ = [
"SeekIn", "SeekIn",
"SettingsIn", "SettingsIn",
"SettingsOut", "SettingsOut",
"TrackCurvesOut",
"TrackDetailOut",
"TrackOut", "TrackOut",
"VolumeIn", "VolumeIn",
] ]
@@ -34,6 +35,8 @@ class AnalysisOut(BaseModel):
energy: float | None = None energy: float | None = None
valence: float | None = None valence: float | None = None
brightness: float | None = None brightness: float | None = None
#: 0..1 confidence that `tempo` is an audible, steady beat - see `TrackAnalysis`.
pulse: float | None = None
beats: bool = False beats: bool = False
@classmethod @classmethod
@@ -45,6 +48,7 @@ class AnalysisOut(BaseModel):
energy=analysis.energy, energy=analysis.energy,
valence=analysis.valence, valence=analysis.valence,
brightness=analysis.brightness, brightness=analysis.brightness,
pulse=analysis.pulse,
beats=analysis.beats, beats=analysis.beats,
) )
@@ -99,9 +103,23 @@ class LibraryOut(BaseModel):
albums: list[AlbumOut] albums: list[AlbumOut]
class BeatsOut(BaseModel): class TrackCurvesOut(BaseModel):
hop_seconds: float
energy: list[float]
valence: list[float]
drive: list[float]
class TrackDetailOut(BaseModel):
"""Beats + mood/drive curves for the one track currently playing - fetched
together since both are per-track detail the library payload never carries.
``curve`` is ``None`` only when the whole track failed analysis; ``times``/
``strengths`` are empty (not ``None``) for a track with no reliable beat, since a
free-tempo or spoken-word track still has real energy/valence/drive curves."""
times: list[float] times: list[float]
strengths: list[float] strengths: list[float]
curve: TrackCurvesOut | None = None
class ConnectionOut(BaseModel): class ConnectionOut(BaseModel):

View File

@@ -24,6 +24,10 @@ dependencies = [
[project.optional-dependencies] [project.optional-dependencies]
dev = ["mypy>=1.10", "pytest-asyncio>=0.23", "pytest>=8.0", "ruff>=0.5"] dev = ["mypy>=1.10", "pytest-asyncio>=0.23", "pytest>=8.0", "ruff>=0.5"]
# Off by default: the reactive background is nice, not required, and librosa/numpy are
# a real install cost on a Pi. Missing this group means `build_analyzer()` falls back
# to `NullAnalyzer` and the background stays at its static baseline - never a crash.
analysis = ["librosa>=1.0", "numpy>=2.0"]
[project.scripts] [project.scripts]
musicmouse = "musicmouse.__main__:main" musicmouse = "musicmouse.__main__:main"
@@ -62,5 +66,7 @@ files = ["musicmouse"]
warn_unreachable = true warn_unreachable = true
[[tool.mypy.overrides]] [[tool.mypy.overrides]]
module = ["vlc", "serial_asyncio", "ruamel.*"] # numpy ships its own types, but only when the `analysis` extra is installed - a plain
# checkout must still type-check clean, so it needs the same treatment as librosa.
module = ["vlc", "serial_asyncio", "ruamel.*", "librosa.*", "numpy"]
ignore_missing_imports = true ignore_missing_imports = true

View File

@@ -0,0 +1,115 @@
#!/usr/bin/env python3
"""One-off: backfill cover art for the podcast shows already in the library.
The scanner now reads a podcast episode's own embedded art first, and a show folder's
``cover.jpg`` second (see ``musicmouse.library.scanner._cover_for_episode``) - but
episodes downloaded before today rarely carry per-episode art, and none of the shows
have a folder-level cover on disk yet. This script fills that gap once, by hand, so it
is not something the running app does on its own.
For every ``Kinderpodcasts/<show>`` folder that has no ``cover.jpg``/``.jpeg``/``.png``
already, it looks the show up on the public iTunes Search API and saves the top match's
artwork as ``cover.jpg`` in that folder. A show it cannot match confidently is left
alone and logged, rather than guessed at - check those by hand afterwards.
This makes real network requests to a third-party service and writes into the real
music library, so it is meant to be run and reviewed by a person, not called from the
app:
python scripts/fetch_podcast_covers.py --config /path/to/config.yml
python scripts/fetch_podcast_covers.py --config /path/to/config.yml --dry-run
"""
from __future__ import annotations
import argparse
import json
import logging
import sys
import urllib.error
import urllib.parse
import urllib.request
from pathlib import Path
sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
from musicmouse.config import load_config
_log = logging.getLogger("fetch_podcast_covers")
#: Mirrors scanner.py's `_COVER_NAMES` - what counts as "already has a cover".
_COVER_NAMES = ("cover.jpg", "cover.jpeg", "cover.png", "folder.jpg")
_ITUNES_SEARCH = "https://itunes.apple.com/search"
_TIMEOUT = 10.0
def _has_cover(folder: Path) -> bool:
return any((folder / name).is_file() for name in _COVER_NAMES)
def _find_artwork_url(show_name: str) -> str | None:
query = urllib.parse.urlencode({"media": "podcast", "term": show_name, "limit": 1})
try:
with urllib.request.urlopen(f"{_ITUNES_SEARCH}?{query}", timeout=_TIMEOUT) as response:
body = json.loads(response.read())
except (urllib.error.URLError, TimeoutError, json.JSONDecodeError) as error:
_log.warning("Lookup for %r failed: %s", show_name, error)
return None
results = body.get("results") or []
if not results:
return None
# iTunes serves a 100x100 thumbnail by default; ask for something worth showing.
artwork = results[0].get("artworkUrl100")
return artwork.replace("100x100", "600x600") if artwork else None
def backfill(root: Path, *, dry_run: bool) -> None:
podcasts_root = root / "Kinderpodcasts"
if not podcasts_root.is_dir():
_log.warning("No Kinderpodcasts folder at %s", podcasts_root)
return
for folder in sorted(podcasts_root.iterdir()):
if not folder.is_dir() or folder.name.startswith("."):
continue
if _has_cover(folder):
_log.info("%-40s already has a cover, skipping", folder.name)
continue
url = _find_artwork_url(folder.name)
if url is None:
_log.warning("%-40s no confident match - fetch this one by hand", folder.name)
continue
_log.info("%-40s -> %s", folder.name, url)
if dry_run:
continue
try:
with urllib.request.urlopen(url, timeout=_TIMEOUT) as response:
data = response.read()
except (urllib.error.URLError, TimeoutError) as error:
_log.warning("%-40s download failed: %s", folder.name, error)
continue
(folder / "cover.jpg").write_bytes(data)
def main(argv: list[str] | None = None) -> int:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("-c", "--config", type=Path, required=True, help="path to config.yml")
parser.add_argument(
"--dry-run",
action="store_true",
help="print what would be fetched without downloading or writing anything",
)
args = parser.parse_args(argv)
logging.basicConfig(level=logging.INFO, format="%(message)s")
config = load_config(args.config)
backfill(config.general.library.root, dry_run=args.dry_run)
return 0
if __name__ == "__main__":
sys.exit(main())

View File

@@ -2,32 +2,56 @@
from __future__ import annotations from __future__ import annotations
import asyncio
import contextlib
import json import json
import threading
from collections.abc import Callable
from pathlib import Path from pathlib import Path
import pytest import pytest
from musicmouse.config import DEFAULT_AUDIO_EXTENSIONS, load_config from musicmouse.config import DEFAULT_AUDIO_EXTENSIONS, load_config
from musicmouse.library import Album, MusicLibrary from musicmouse.library import Album, Analyzer, MusicLibrary
from musicmouse.library.analysis import ANALYZER_VERSION, BeatGrid, TrackAnalysis from musicmouse.library.analysis import ANALYZER_VERSION, BeatGrid, TrackAnalysis, TrackCurves
from musicmouse.library.cache import LibraryCache from musicmouse.library.cache import LibraryCache
from musicmouse.library.colors import colors_from_id from musicmouse.library.colors import colors_from_id
from musicmouse.library.models import album_id from musicmouse.library.models import album_id, track_key
from tests.conftest import VALID_CONFIG, write_config, write_track from tests.conftest import VALID_CONFIG, write_config, write_track
EXTENSIONS = frozenset(DEFAULT_AUDIO_EXTENSIONS) EXTENSIONS = frozenset(DEFAULT_AUDIO_EXTENSIONS)
async def build(config_dir: Path) -> MusicLibrary: async def build(config_dir: Path, *, analyzer: Analyzer | None = None) -> MusicLibrary:
config = load_config(write_config(config_dir, VALID_CONFIG)) config = load_config(write_config(config_dir, VALID_CONFIG))
return await MusicLibrary.build( return await MusicLibrary.build(
config.general.library.root, config.general.library.root,
config.general.library.cache, config.general.library.cache,
EXTENSIONS, EXTENSIONS,
analyzer=analyzer,
figure_kinds=config.figure_kinds, figure_kinds=config.figure_kinds,
) )
class FakeAnalyzer:
"""An `Analyzer` that does no DSP, so the worker's own behaviour - which tracks it
touches, how it reacts to `is_busy`, what a raising analyzer does to the batch - can
be tested without librosa."""
def __init__(self, *, version: int = ANALYZER_VERSION, fails: set[str] | None = None) -> None:
self.version = version
self._fails = fails or set()
self.calls: list[Path] = []
def analyze(self, path: Path) -> tuple[TrackAnalysis, BeatGrid | None, TrackCurves | None]:
self.calls.append(path)
if path.name in self._fails:
raise RuntimeError(f"boom: {path.name}")
analysis = TrackAnalysis(version=self.version, tempo=100.0, energy=0.5)
curves = TrackCurves(hop_seconds=1.0, energy=(0.5,), valence=(0.5,), drive=(0.5,))
return analysis, BeatGrid((0.1,), (1.0,)), curves
def album_named(library: MusicLibrary, title: str) -> Album: def album_named(library: MusicLibrary, title: str) -> Album:
return next(album for album in library.albums if album.title == title) return next(album for album in library.albums if album.title == title)
@@ -233,25 +257,56 @@ async def test_a_new_episode_only_costs_scanning_that_one_file(config_dir: Path)
async def test_a_rescan_leaves_analysis_alone(config_dir: Path) -> None: async def test_a_rescan_leaves_analysis_alone(config_dir: Path) -> None:
"""The whole reason the cache is a directory rather than one file.""" """The whole reason the cache is a directory rather than one file.
"Kinderparty Lieder" rather than "Eule": folding analysis back into the index is
restricted to music (see `_ANALYZED_KINDS`), since only music is ever analyzed - a
book's `analysis/` file, if one somehow existed, is not something a rescan need
resurface.
"""
library = await build(config_dir)
album = album_named(library, "Kinderparty Lieder")
from musicmouse.library.models import track_key
key = track_key(album.tracks[0].path)
library.cache.store_analysis(key, TrackAnalysis(version=ANALYZER_VERSION, tempo=128.0))
library.cache.store_beats(key, BeatGrid((0.5, 1.0), (1.0, 0.5)))
curve = TrackCurves(hop_seconds=1.0, energy=(0.4, 0.6), valence=(0.5, 0.5), drive=(0.3, 0.7))
library.cache.store_curve(key, curve)
# Force a full rescan by dropping the cheap part of the cache.
(config_dir / ".cache" / "index.json").unlink()
library = await build(config_dir)
album = album_named(library, "Kinderparty Lieder")
assert album.tracks[0].analysis is not None
assert album.tracks[0].analysis.tempo == 128.0
grid = library.beats(album.id, 0)
assert grid is not None
assert grid.times == (0.5, 1.0)
curve = library.curve(album.id, 0)
assert curve is not None
assert curve.energy == (0.4, 0.6)
async def test_a_rescan_does_not_fold_analysis_into_a_non_music_album(
config_dir: Path,
) -> None:
"""The other half of the restriction above: a book track's `analysis/` file (however
it got there) is not folded into the index, so a rescan never stats hundreds of
book/podcast files that can never have one under normal operation."""
library = await build(config_dir) library = await build(config_dir)
album = album_named(library, "Eule") album = album_named(library, "Eule")
from musicmouse.library.models import track_key from musicmouse.library.models import track_key
key = track_key(album.tracks[0].path) key = track_key(album.tracks[0].path)
library.cache.store_analysis(key, TrackAnalysis(version=ANALYZER_VERSION, tempo=128.0)) library.cache.store_analysis(key, TrackAnalysis(version=ANALYZER_VERSION, tempo=128.0))
library.cache.store_beats(key, BeatGrid((0.5, 1.0), (1.0, 0.5)))
# Force a full rescan by dropping the cheap part of the cache.
(config_dir / ".cache" / "index.json").unlink() (config_dir / ".cache" / "index.json").unlink()
library = await build(config_dir) library = await build(config_dir)
album = album_named(library, "Eule") album = album_named(library, "Eule")
assert album.tracks[0].analysis is not None assert album.tracks[0].analysis is None
assert album.tracks[0].analysis.tempo == 128.0
grid = library.beats(album.id, 0)
assert grid is not None
assert grid.times == (0.5, 1.0)
async def test_analysis_survives_an_analyzer_that_grew_a_field(tmp_path: Path) -> None: async def test_analysis_survives_an_analyzer_that_grew_a_field(tmp_path: Path) -> None:
@@ -340,3 +395,189 @@ async def test_a_figures_kind_survives_the_cache(config_dir: Path) -> None:
assert eule.series is not None assert eule.series is not None
assert album_named(second, "Fuchs").kind == "music" assert album_named(second, "Fuchs").kind == "music"
assert album_named(second, "Fuchs").series is None assert album_named(second, "Fuchs").series is None
# -------------------------------------------------------------------- analysis worker
async def _wait_for(predicate: Callable[[], bool], *, timeout: float = 2.0) -> None:
"""Poll until `predicate()` is true, rather than guessing a sleep duration."""
async def poll() -> None:
while not predicate():
await asyncio.sleep(0.01)
await asyncio.wait_for(poll(), timeout=timeout)
async def _cancel(task: asyncio.Task[object]) -> None:
task.cancel()
with contextlib.suppress(asyncio.CancelledError):
await task
async def test_analyze_pending_only_touches_music_albums(config_dir: Path) -> None:
"""Books and podcasts are the majority of a real library and none of them get a
background - see `_ANALYZED_KINDS`."""
analyzer = FakeAnalyzer()
library = await build(config_dir, analyzer=analyzer)
music_track_count = sum(len(a.tracks) for a in library.albums if a.kind == "music")
assert music_track_count == 5 # fuchs (3) + Kinderparty Lieder (2)
done = await library.analyze_pending()
assert done == 5
assert len(analyzer.calls) == 5
touched = {
next(a for a in library.albums if p in {t.path for t in a.tracks}).kind
for p in analyzer.calls
}
assert touched == {"music"}
async def test_analyze_pending_persists_curves_alongside_beats(config_dir: Path) -> None:
"""`MusicLibrary.curve()` mirrors `.beats()` - both are per-track detail fetched
for the one track currently playing, written together every pass."""
analyzer = FakeAnalyzer()
library = await build(config_dir, analyzer=analyzer)
album = next(a for a in library.albums if a.kind == "music")
await library.analyze_pending()
curve = library.curve(album.id, 0)
assert curve is not None
assert curve.hop_seconds == 1.0
assert curve.energy == (0.5,)
assert library.beats(album.id, 0) is not None
async def test_analyze_pending_does_not_repeat_work_already_cached(config_dir: Path) -> None:
analyzer = FakeAnalyzer()
library = await build(config_dir, analyzer=analyzer)
await library.analyze_pending()
again = await library.analyze_pending()
assert again == 0
assert len(analyzer.calls) == 5
async def test_a_failing_track_is_marked_attempted_and_not_retried(config_dir: Path) -> None:
"""A corrupt file, a DRM'd one, or one the analyzer just chokes on must not abort
the batch, and must not be retried on every single future pass either."""
analyzer = FakeAnalyzer(fails={"00 - lied.mp3"})
library = await build(config_dir, analyzer=analyzer)
done = await library.analyze_pending()
assert done == 5 # the failure still counts as "handled"
kinderparty = album_named(library, "Kinderparty Lieder")
failed = next(t for t in kinderparty.tracks if t.path.name == "00 - lied.mp3")
cached = library.cache.load_analysis(track_key(failed.path))
assert cached is not None
assert cached.version == ANALYZER_VERSION
assert cached.tempo is None # attempted, not computed
again = await library.analyze_pending()
assert again == 0
assert len(analyzer.calls) == 5
async def test_is_busy_pauses_the_pass_until_it_clears(
config_dir: Path, monkeypatch: pytest.MonkeyPatch
) -> None:
import musicmouse.library as library_module
monkeypatch.setattr(library_module, "_BUSY_POLL_SECONDS", 0.01)
analyzer = FakeAnalyzer()
library = await build(config_dir, analyzer=analyzer)
busy = True
task = asyncio.create_task(library.analyze_pending(is_busy=lambda: busy))
try:
await asyncio.sleep(0.05)
assert analyzer.calls == [] # never got past the busy check
busy = False
await asyncio.wait_for(task, timeout=2)
finally:
if not task.done():
await _cancel(task)
assert len(analyzer.calls) == 5
async def test_run_analysis_does_the_work_refresh_left_pending(config_dir: Path) -> None:
"""The trigger this feature actually depends on: `refresh()` - at startup, or from
a parent's "Bibliothek neu einlesen" - ends by requesting analysis, and
`run_analysis` is what turns that request into finished work, without a second
`refresh()` needed to see it."""
analyzer = FakeAnalyzer()
library = await build(config_dir, analyzer=analyzer) # refresh() already requested
batches: list[int] = []
async def on_batch() -> None:
batches.append(len(analyzer.calls))
worker = asyncio.create_task(library.run_analysis(on_batch=on_batch))
try:
await _wait_for(lambda: bool(batches))
finally:
await _cancel(worker)
assert batches[-1] == 5
kinderparty = album_named(library, "Kinderparty Lieder")
assert kinderparty.tracks[0].analysis is not None
assert kinderparty.tracks[0].analysis.tempo == 100.0
async def test_requests_raised_during_a_pass_coalesce_into_one_more_pass(
config_dir: Path,
) -> None:
"""Calling `request_analysis` three times while a pass is already running must not
queue three more passes - just one, right after the current one finishes."""
release = threading.Event()
class BlockingOnceAnalyzer(FakeAnalyzer):
def __init__(self) -> None:
super().__init__()
self._blocked = False
def analyze(self, path: Path) -> tuple[TrackAnalysis, BeatGrid | None, TrackCurves | None]:
if not self._blocked:
self._blocked = True
release.wait(timeout=2)
return super().analyze(path)
analyzer = BlockingOnceAnalyzer()
library = await build(config_dir, analyzer=analyzer)
pass_count = 0
original = library.analyze_pending
async def counting(**kwargs: object) -> int:
nonlocal pass_count
pass_count += 1
return await original(**kwargs) # type: ignore[arg-type]
library.analyze_pending = counting # type: ignore[method-assign]
worker = asyncio.create_task(library.run_analysis())
try:
await _wait_for(lambda: pass_count >= 1) # the first pass has started
await asyncio.sleep(0.05) # ... and is now blocked inside the analyzer
library.request_analysis()
library.request_analysis()
library.request_analysis()
release.set()
await _wait_for(lambda: pass_count >= 2)
# Give the coalesced pass a moment to actually run - everything is already
# cached, so it finds nothing pending and returns without incrementing further.
await asyncio.sleep(0.1)
finally:
await _cancel(worker)
assert pass_count == 2
assert len(analyzer.calls) == 5

View File

@@ -0,0 +1,199 @@
"""`LibrosaAnalyzer` against synthesised signals, not shipped audio fixtures.
Skipped wholesale when the ``analysis`` extra is not installed - exactly the state a
checkout of this repo is in until someone runs ``pip install -e '.[analysis]'``, and
what :func:`musicmouse.library.analysis.build_analyzer` falls back to `NullAnalyzer` for.
Warnings become errors project-wide (see ``pytest.ini_options.filterwarnings`` in
``pyproject.toml``), and numba emits a benign one on its first JIT compile per process -
so this module, alone, turns that off rather than loosening the project-wide setting.
"""
from __future__ import annotations
from pathlib import Path
import pytest
librosa = pytest.importorskip("librosa")
np = pytest.importorskip("numpy")
sf = pytest.importorskip("soundfile")
from musicmouse.library.librosa_analyzer import LibrosaAnalyzer # noqa: E402
pytestmark = pytest.mark.filterwarnings("ignore")
_SR = 22050
def _click_train(bpm: float, seconds: float = 20.0, sr: int = _SR) -> np.ndarray:
"""A metronome: short decaying clicks exactly `bpm` apart, easy for a beat tracker
to lock onto - real music is messier, but this makes a known-answer test possible.
"""
y = np.zeros(int(seconds * sr), dtype=np.float32)
period = 60.0 / bpm
click = np.hanning(200).astype(np.float32)
t = 0.0
while t < seconds:
i = int(t * sr)
n = min(len(click), len(y) - i)
if n > 0:
y[i : i + n] += click[:n]
t += period
return y
def _sine(freq: float, seconds: float = 12.0, sr: int = _SR, amp: float = 0.5) -> np.ndarray:
t = np.linspace(0, seconds, int(seconds * sr), endpoint=False)
return (amp * np.sin(2 * np.pi * freq * t)).astype(np.float32)
def _write(tmp_path: Path, name: str, y: np.ndarray, sr: int = _SR) -> Path:
path = tmp_path / name
sf.write(path, y, sr)
return path
def _close_to_bpm(tempo: float, target: float, tolerance: float = 8.0) -> bool:
"""Beat trackers routinely report a tempo at half or double the "true" one - both
are the same beat grid, just every-other-click or twice-per-click. Any of the three
counts as a correct detection."""
candidates = (target, target / 2, target * 2)
return any(abs(tempo - candidate) <= tolerance for candidate in candidates)
def test_tempo_from_a_click_train(tmp_path: Path) -> None:
path = _write(tmp_path, "clicks.wav", _click_train(120.0))
analysis, grid, curves = LibrosaAnalyzer().analyze(path)
assert analysis.tempo is not None
assert _close_to_bpm(analysis.tempo, 120.0)
assert grid is not None
assert len(grid.times) > 10
assert curves is not None
def test_brightness_orders_a_high_tone_above_a_low_one(tmp_path: Path) -> None:
low = _write(tmp_path, "low.wav", _sine(300.0))
high = _write(tmp_path, "high.wav", _sine(3000.0))
analyzer = LibrosaAnalyzer()
low_analysis, _, _ = analyzer.analyze(low)
high_analysis, _, _ = analyzer.analyze(high)
assert low_analysis.brightness is not None
assert high_analysis.brightness is not None
assert high_analysis.brightness > low_analysis.brightness
def test_energy_orders_a_loud_signal_above_a_quiet_one(tmp_path: Path) -> None:
loud = _write(tmp_path, "loud.wav", _sine(440.0, amp=0.9))
quiet = _write(tmp_path, "quiet.wav", _sine(440.0, amp=0.09))
analyzer = LibrosaAnalyzer()
loud_analysis, _, _ = analyzer.analyze(loud)
quiet_analysis, _, _ = analyzer.analyze(quiet)
assert loud_analysis.energy is not None
assert quiet_analysis.energy is not None
assert loud_analysis.energy > quiet_analysis.energy
def test_pulse_is_higher_for_a_steady_beat_than_a_plain_tone(tmp_path: Path) -> None:
"""What `pulse` is for: telling a track that actually pulses apart from one that
merely has *a* tempo number attached to it, like a sustained tone or narration."""
clicks = _write(tmp_path, "clicks.wav", _click_train(120.0))
tone = _write(tmp_path, "tone.wav", _sine(440.0))
analyzer = LibrosaAnalyzer()
click_analysis, _, _ = analyzer.analyze(clicks)
tone_analysis, _, _ = analyzer.analyze(tone)
assert click_analysis.pulse is not None
assert tone_analysis.pulse is not None
assert click_analysis.pulse > tone_analysis.pulse
def test_a_corrupt_file_is_analyzed_as_nothing_rather_than_raising(tmp_path: Path) -> None:
path = tmp_path / "corrupt.mp3"
path.write_bytes(b"not an audio file")
analysis, grid, curves = LibrosaAnalyzer().analyze(path)
assert analysis.version == LibrosaAnalyzer.version
assert analysis.tempo is None
assert analysis.energy is None
assert analysis.valence is None
assert analysis.brightness is None
assert analysis.pulse is None
assert analysis.beats is False
assert grid is None
assert curves is None
def test_a_zero_length_file_is_analyzed_as_nothing_rather_than_raising(tmp_path: Path) -> None:
path = _write(tmp_path, "empty.wav", np.zeros(0, dtype=np.float32))
analysis, grid, curves = LibrosaAnalyzer().analyze(path)
assert analysis.version == LibrosaAnalyzer.version
assert analysis.tempo is None
assert grid is None
assert curves is None
def test_curves_are_produced_alongside_the_scalars(tmp_path: Path) -> None:
path = _write(tmp_path, "clicks.wav", _click_train(120.0))
_, _, curves = LibrosaAnalyzer().analyze(path)
assert curves is not None
assert curves.hop_seconds == 1.0
assert len(curves.energy) == len(curves.valence) == len(curves.drive) >= 1
def test_whole_track_scalars_are_the_mean_of_their_curves(tmp_path: Path) -> None:
path = _write(tmp_path, "clicks.wav", _click_train(120.0))
analysis, _, curves = LibrosaAnalyzer().analyze(path)
assert curves is not None
assert analysis.energy == pytest.approx(sum(curves.energy) / len(curves.energy))
assert analysis.valence == pytest.approx(sum(curves.valence) / len(curves.valence))
def test_energy_curve_tracks_a_loud_then_quiet_signal(tmp_path: Path) -> None:
loud = _sine(440.0, seconds=10.0, amp=0.9)
quiet = _sine(440.0, seconds=10.0, amp=0.09)
path = _write(tmp_path, "loud_then_quiet.wav", np.concatenate([loud, quiet]))
_, _, curves = LibrosaAnalyzer().analyze(path)
assert curves is not None
half = len(curves.energy) // 2
first_half_mean = sum(curves.energy[:half]) / half
second_half_mean = sum(curves.energy[half:]) / (len(curves.energy) - half)
assert first_half_mean > second_half_mean
def test_drive_is_higher_for_a_steady_beat_than_a_plain_tone(tmp_path: Path) -> None:
"""What `drive` is for: a track's rhythmic intensity, the signal that actually
varies within a track (unlike tempo, which is flat to within a few percent inside
a real recording) - see `LibrosaAnalyzer._drive_curve`."""
clicks = _write(tmp_path, "clicks.wav", _click_train(120.0))
tone = _write(tmp_path, "tone.wav", _sine(440.0))
analyzer = LibrosaAnalyzer()
_, _, click_curves = analyzer.analyze(clicks)
_, _, tone_curves = analyzer.analyze(tone)
assert click_curves is not None
assert tone_curves is not None
click_mean = sum(click_curves.drive) / len(click_curves.drive)
tone_mean = sum(tone_curves.drive) / len(tone_curves.drive)
assert click_mean > tone_mean
def test_a_clip_shorter_than_one_hop_still_produces_a_one_sample_curve(tmp_path: Path) -> None:
path = _write(tmp_path, "short.wav", _sine(440.0, seconds=0.3))
_, _, curves = LibrosaAnalyzer().analyze(path)
assert curves is not None
assert len(curves.energy) == 1

View File

@@ -17,6 +17,8 @@ import pytest
from fastapi import FastAPI from fastapi import FastAPI
from musicmouse.config import WebConfig, load_config from musicmouse.config import WebConfig, load_config
from musicmouse.library.analysis import BeatGrid, TrackCurves
from musicmouse.library.models import track_key
from musicmouse.services.web.service import build_app from musicmouse.services.web.service import build_app
from musicmouse.simulator.harness import Simulation, build_simulation from musicmouse.simulator.harness import Simulation, build_simulation
from tests.conftest import VALID_CONFIG, write_config from tests.conftest import VALID_CONFIG, write_config
@@ -96,6 +98,50 @@ async def test_unanalyzed_tracks_report_no_analysis(client: Client) -> None:
assert (await client.get(f"/api/tracks/{album['id']}/0/analysis")).status_code == 404 assert (await client.get(f"/api/tracks/{album['id']}/0/analysis")).status_code == 404
async def test_an_analyzed_track_reports_its_beats_and_curves(
client: Client, sim: Simulation
) -> None:
album = await album_by_title(client, "Eule")
library_album = sim.app.library.get(album["id"])
assert library_album is not None
key = track_key(library_album.tracks[0].path)
sim.app.library.cache.store_beats(key, BeatGrid((0.5, 1.0), (1.0, 0.5)))
sim.app.library.cache.store_curve(
key, TrackCurves(hop_seconds=1.0, energy=(0.4, 0.6), valence=(0.5, 0.5), drive=(0.3, 0.7))
)
response = await client.get(f"/api/tracks/{album['id']}/0/analysis")
assert response.status_code == 200
body = response.json()
assert body["times"] == [0.5, 1.0]
assert body["curve"]["energy"] == [0.4, 0.6]
assert body["curve"]["valence"] == [0.5, 0.5]
assert body["curve"]["drive"] == [0.3, 0.7]
async def test_a_track_with_a_curve_but_no_reliable_beat_still_reports_200(
client: Client, sim: Simulation
) -> None:
"""A free-tempo/spoken-word track has no usable beat grid, but energy/valence/
drive are computed regardless - the endpoint must not 404 just because `beats`
came back empty."""
album = await album_by_title(client, "Eule")
library_album = sim.app.library.get(album["id"])
assert library_album is not None
key = track_key(library_album.tracks[0].path)
sim.app.library.cache.store_curve(
key, TrackCurves(hop_seconds=1.0, energy=(0.5,), valence=(0.5,), drive=(0.5,))
)
response = await client.get(f"/api/tracks/{album['id']}/0/analysis")
assert response.status_code == 200
body = response.json()
assert body["times"] == []
assert body["curve"]["energy"] == [0.5]
# -------------------------------------------------------------------------- state # -------------------------------------------------------------------------- state

View File

@@ -11,9 +11,10 @@ import { useCallback, useEffect, useMemo, useRef, useState } from "react";
import { api } from "./api/client"; import { api } from "./api/client";
import type { Album, HaConfig } from "./api/types"; import type { Album, HaConfig } from "./api/types";
import { AlbumModal } from "./components/AlbumModal"; import { AlbumModal } from "./components/AlbumModal";
import { Ambience, type AmbienceDebugSnapshot } from "./components/Ambience";
import { AmbienceDebugOverlay } from "./components/AmbienceDebugOverlay";
import { AppHeader } from "./components/AppHeader"; import { AppHeader } from "./components/AppHeader";
import { BrowseView } from "./components/BrowseView"; import { BrowseView } from "./components/BrowseView";
import { Bubbles } from "./components/Bubbles";
import { HelpOverlay } from "./components/HelpOverlay"; import { HelpOverlay } from "./components/HelpOverlay";
import { ParentPanel } from "./components/ParentPanel"; import { ParentPanel } from "./components/ParentPanel";
import { PlayerBar } from "./components/PlayerBar"; import { PlayerBar } from "./components/PlayerBar";
@@ -22,6 +23,8 @@ import { RoomView } from "./components/RoomView";
import { useGridColumns } from "./hooks/useGridColumns"; import { useGridColumns } from "./hooks/useGridColumns";
import { useLibrary } from "./hooks/useLibrary"; import { useLibrary } from "./hooks/useLibrary";
import { usePlayerState } from "./hooks/usePlayerState"; import { usePlayerState } from "./hooks/usePlayerState";
import { DEFAULT_MANUAL_CONTROL, DEFAULT_TUNABLES } from "./lib/ambienceTunables";
import type { AmbienceTunables, ManualControl } from "./lib/ambienceTunables";
import type { Action, UiState } from "./lib/keyboard"; import type { Action, UiState } from "./lib/keyboard";
import { handleKey, initialUiState } from "./lib/keyboard"; import { handleKey, initialUiState } from "./lib/keyboard";
import { playPop } from "./lib/pop"; import { playPop } from "./lib/pop";
@@ -43,6 +46,15 @@ export function App() {
const [parentMode, setParentMode] = useState( const [parentMode, setParentMode] = useState(
() => new URLSearchParams(location.search).get("parentMode") === "1", () => new URLSearchParams(location.search).get("parentMode") === "1",
); );
const [debugDynamicUI] = useState(
() => new URLSearchParams(location.search).get("debugDynamicUI") === "1",
);
const [debugSnapshot, setDebugSnapshot] = useState<AmbienceDebugSnapshot | null>(null);
// Exists regardless of `debugDynamicUI` (so `Ambience` always has consistent,
// harmless defaults - `manual.enabled` defaults to false) - only the overlay, and
// thus the ability to change either, is gated behind the query param.
const [tunables, setTunables] = useState<AmbienceTunables>(DEFAULT_TUNABLES);
const [manual, setManual] = useState<ManualControl>(DEFAULT_MANUAL_CONTROL);
// undefined: not yet resolved (hide the nav pill to avoid a flash). null: confirmed // undefined: not yet resolved (hide the nav pill to avoid a flash). null: confirmed
// absent - the room page is a separate opt-in feature, off by default. // absent - the room page is a separate opt-in feature, off by default.
const [haConfig, setHaConfig] = useState<HaConfig | null | undefined>(undefined); const [haConfig, setHaConfig] = useState<HaConfig | null | undefined>(undefined);
@@ -228,7 +240,15 @@ export function App() {
return ( return (
<div className="stage"> <div className="stage">
<Bubbles /> {state && (
<Ambience
album={currentAlbum}
state={state}
tunables={tunables}
manual={manual}
onDebugFrame={debugDynamicUI ? setDebugSnapshot : undefined}
/>
)}
{ui.view === "browse" && state && ( {ui.view === "browse" && state && (
<div <div
@@ -346,6 +366,17 @@ export function App() {
{parentMode && <ParentPanel onClose={() => setParentMode(false)} />} {parentMode && <ParentPanel onClose={() => setParentMode(false)} />}
{(!state || library.loading) && <Splash error={library.error} />} {(!state || library.loading) && <Splash error={library.error} />}
{debugDynamicUI && debugSnapshot && (
<AmbienceDebugOverlay
snapshot={debugSnapshot}
tunables={tunables}
onTunablesChange={setTunables}
manual={manual}
onManualChange={setManual}
playing={state?.playing ?? false}
/>
)}
</div> </div>
); );
} }

View File

@@ -1,6 +1,6 @@
/** Every call the UI makes. Commands are fire-and-forget: the websocket reports back. */ /** Every call the UI makes. Commands are fire-and-forget: the websocket reports back. */
import type { Album, HaConfig, HaEntityState, PlayerState, Settings } from "./types"; import type { Album, HaConfig, HaEntityState, PlayerState, Settings, TrackDetail } from "./types";
async function request<T>(path: string, init?: RequestInit): Promise<T> { async function request<T>(path: string, init?: RequestInit): Promise<T> {
const response = await fetch(`/api${path}`, { const response = await fetch(`/api${path}`, {
@@ -47,6 +47,14 @@ async function fetchHaStates(entityIds: string[]): Promise<Record<string, HaEnti
return byId; return byId;
} }
/** `null` means "not analyzed" (the backend's expected 404 for this), not an error -
* the ambient background just falls back to its un-analyzed baseline for that track. */
async function fetchTrackDetail(albumId: string, trackIndex: number): Promise<TrackDetail | null> {
const response = await fetch(`/api/tracks/${albumId}/${trackIndex}/analysis`);
if (!response.ok) return null;
return (await response.json()) as TrackDetail;
}
export const api = { export const api = {
library: () => request<{ albums: Album[] }>("/library").then((body) => body.albums), library: () => request<{ albums: Album[] }>("/library").then((body) => body.albums),
state: () => request<PlayerState>("/state"), state: () => request<PlayerState>("/state"),
@@ -70,6 +78,8 @@ export const api = {
haStates: fetchHaStates, haStates: fetchHaStates,
haCallService: (domain: string, service: string, body: Record<string, unknown>) => haCallService: (domain: string, service: string, body: Record<string, unknown>) =>
post(`/ha/services/${domain}/${service}`, body), post(`/ha/services/${domain}/${service}`, body),
trackDetail: fetchTrackDetail,
}; };
export const coverUrl = (albumId: string) => `/api/albums/${albumId}/cover`; export const coverUrl = (albumId: string) => `/api/albums/${albumId}/cover`;

View File

@@ -7,9 +7,33 @@ export interface TrackAnalysis {
energy: number | null; energy: number | null;
valence: number | null; valence: number | null;
brightness: number | null; brightness: number | null;
/** 0..1 confidence that `tempo` is an audible, steady beat rather than an artifact
* of free-tempo or spoken-word material - below some threshold, don't pulse on it. */
pulse: number | null;
beats: boolean; beats: boolean;
} }
/** Per-second energy/valence/drive samples - the parts of the mood/rhythm formulas
* that vary *within* a track. Regularly sampled, so just a hop and equal-length
* arrays, no per-sample timestamps. Mirrors `TrackCurvesOut`. */
export interface TrackCurves {
hop_seconds: number;
energy: number[];
valence: number[];
drive: number[];
}
/** A track's beat grid plus its mood/drive curves, fetched together from
* `GET /api/tracks/{album_id}/{index}/analysis` - only for the one track that is
* playing, never inside the library payload. `curve` is `null` only when the whole
* track failed analysis; `times`/`strengths` are empty (not absent) for a track with
* no reliable beat, since energy/valence/drive are still real. Mirrors `TrackDetailOut`. */
export interface TrackDetail {
times: number[];
strengths: number[];
curve: TrackCurves | null;
}
export interface Track { export interface Track {
title: string; title: string;
/** Seconds, read from the file's tags at scan time. */ /** Seconds, read from the file's tags at scan time. */

View File

@@ -0,0 +1,472 @@
/** The background: a gradient plus a bubble field, both painted on one canvas.
*
* Replaces the old fixed 21-`<div>` field (`bubbleRise` in app.css) with a real
* particle loop, because "spawn frequency" and "current" need a rate and a
* magnitude, not a fixed set of `animation-duration`s. The gradient moves to canvas
* too, for the same reason: CSS cannot interpolate `linear-gradient` colour stops
* between tracks, a canvas can just ease toward a new target every frame.
*
* `energy`/`valence`/`drive` are now sampled from a per-second curve at the current
* playback position, so - unlike the original version of this component - the
* mapping from analysis to visuals can no longer run once per track. It runs every
* frame instead (`ambienceFromSampled` below), fed by curve samples smoothed with an
* adjustable time constant (`tunables.curveSmoothingTau`).
*
* One `requestAnimationFrame` loop, started once and never restarted - everything it
* reads (`playing`, tunables, manual control, the per-track base, the beat/curve
* detail, the playback clock) lives in refs, so an album or track change, or a debug
* slider moving, never costs a re-render of this component, let alone a dropped
* frame. This mirrors `usePlaybackClock`'s anchor trick (a `{position, at}` pair
* re-seeded from the server, read against `performance.now()`) but reads it from
* inside the loop instead of `useState`, which is what keeps 60fps from touching
* React at all.
*/
import { useEffect, useRef } from "react";
import type { Album, PlayerState, TrackAnalysis } from "../api/types";
import { useTrackDetail } from "../hooks/useTrackDetail";
import {
ambienceBaseFor,
ambienceColorAt,
currentMagnitudeAt,
currentVelocityAt,
effectiveCurve,
oklchString,
sampleCurveAt,
spawnRateAt,
tempoToMagnitude,
tempoToRise,
type Ambience as AmbienceTarget,
type AmbienceBase,
type Oklch,
} from "../lib/ambience";
import type { AmbienceTunables, ManualControl } from "../lib/ambienceTunables";
import { groupOf, type Group } from "../lib/search";
/** Everything `?debugDynamicUI=1` wants to see: the raw analysis, what it maps to,
* and where the eased, on-screen look currently sits relative to that target. */
export interface AmbienceDebugSnapshot {
group: Group | "idle";
trackTitle: string | null;
playing: boolean;
position: number;
analysis: TrackAnalysis | null;
target: AmbienceTarget;
current: AmbienceTarget;
sampledEnergy: number;
sampledValence: number;
sampledDrive: number;
/** The fully beat-kicked current magnitude actually driving bubbles this frame -
* distinct from `current.currentMagnitude`, which is pre-kick. */
currentMagnitude: number;
kick: number;
beatsTrusted: boolean;
beatGridLoaded: boolean;
beatCount: number;
beatCursor: number;
bubbleCount: number;
/** Whether a real fetched curve is in use, vs. the flat scalar-derived fallback. */
curveLoaded: boolean;
/** Whether manual control is *actually* driving the scene right now - distinct from
* `manual.enabled`, which can stay checked across a play/pause cycle. */
usingManual: boolean;
}
interface Props {
album: Album | null;
state: PlayerState;
tunables: AmbienceTunables;
manual: ManualControl;
/** Wired up only behind `?debugDynamicUI=1` - see `AmbienceDebugOverlay`. Throttled
* internally, so passing this costs nothing close to every frame's React render. */
onDebugFrame?: (snapshot: AmbienceDebugSnapshot) => void;
}
//: The debug overlay doesn't need 60fps, and re-rendering React that often just to
//: feed it would defeat the whole point of keeping this loop ref-only.
const DEBUG_EMIT_INTERVAL_MS = 200;
//: Below this, `pulse` says the detected tempo is not a trustworthy, audible beat -
//: moved here from the fetch hook, since it now gates *use* (does this beat crossing
//: drive the kick), not *fetch* (the curve is wanted regardless of beat quality).
const PULSE_THRESHOLD = 0.3;
interface Bubble {
/** Horizontal position in CSS pixels - stateful, unlike the old sine-wander's fixed
* anchor: a persistent one-directional current has to be integrated over time. */
x: number;
/** Pixels risen since spawn. */
risen: number;
/** Size at spawn. */
baseSize: number;
/** Current painted size - grows from `baseSize` as the bubble rises. */
size: number;
baseAlpha: number;
}
//: How quickly the painted look eases toward the per-frame target - an exponential
//: time constant, so this is "how many seconds to close most of the gap", not a hard
//: crossfade duration. ~0.5s reaches ~95% of the way there in about 1.5s. Distinct
//: from `tunables.curveSmoothingTau`, which smooths the sampled signal *feeding*
//: the target, not the ease from `current` toward it - the two compound in series.
const CROSSFADE_TAU_SECONDS = 0.5;
//: How quickly a beat's kick fades back out.
const KICK_DECAY_TAU_SECONDS = 0.12;
//: Extra bubbles/second at the moment of a full-strength (1.0) beat - kept exactly as
//: before this refactor; only the current-magnitude boost is new.
const BEAT_BURST_RATE = 6;
//: Today's rise speed, roughly: the old CSS took a bubble ~10s to cross 120vh.
const BASE_RISE_FRACTION_PER_SECOND = 0.12;
const BUBBLE_MIN_SIZE = 8;
const BUBBLE_MAX_SIZE = 24;
//: The canvas frame clamps to this, so a tab returning from the background after
//: minutes away doesn't dump a giant `dt` into the physics in one jump.
const MAX_FRAME_SECONDS = 0.05;
function clamp01(value: number): number {
return Math.max(0, Math.min(1, value));
}
function lerp(a: number, b: number, t: number): number {
return a + (b - a) * t;
}
function lerpColor(a: Oklch, b: Oklch, t: number): Oklch {
return { l: lerp(a.l, b.l, t), c: lerp(a.c, b.c, t), h: lerp(a.h, b.h, t) };
}
function lerpAmbience(a: AmbienceTarget, b: AmbienceTarget, t: number): AmbienceTarget {
return {
gradient: [
lerpColor(a.gradient[0], b.gradient[0], t),
lerpColor(a.gradient[1], b.gradient[1], t),
lerpColor(a.gradient[2], b.gradient[2], t),
],
tint: lerpColor(a.tint, b.tint, t),
spawnRate: lerp(a.spawnRate, b.spawnRate, t),
rise: lerp(a.rise, b.rise, t),
currentMagnitude: lerp(a.currentMagnitude, b.currentMagnitude, t),
};
}
/** First index whose beat time is `>= position` - a binary search, since a track's
* grid can run to several thousand entries and this runs every frame. */
function beatCursorFor(times: number[], position: number): number {
let lo = 0;
let hi = times.length;
while (lo < hi) {
const mid = (lo + hi) >>> 1;
if (times[mid]! < position) lo = mid + 1;
else hi = mid;
}
return lo;
}
function spawnBubble(width: number): Bubble {
const baseSize = BUBBLE_MIN_SIZE + Math.random() * (BUBBLE_MAX_SIZE - BUBBLE_MIN_SIZE);
return {
x: Math.random() * width,
risen: 0,
baseSize,
size: baseSize,
baseAlpha: 0.25 + Math.random() * 0.2,
};
}
/** Turns already-smoothed energy/valence/drive into a full `Ambience` - shared
* between the mount-time seed (unsmoothed, snapped straight to the first sample) and
* every subsequent frame (smoothed with `tunables.curveSmoothingTau`), so the two
* paths can never disagree on what a given (energy, valence, drive) triple means. */
function ambienceFromSampled(
base: AmbienceBase,
usingManual: boolean,
manual: ManualControl,
tunables: AmbienceTunables,
energy: number,
valence: number,
drive: number,
): AmbienceTarget {
if (!usingManual && !base.music) {
return { ...base.fixed!, rise: base.rise, currentMagnitude: 0 };
}
const rise = usingManual ? tempoToRise(manual.tempo) : base.rise;
const baseMagnitude = usingManual ? tempoToMagnitude(manual.tempo) : base.baseCurrentMagnitude;
const { gradient, tint } = ambienceColorAt(energy, valence, tunables);
const magnitude = currentMagnitudeAt(baseMagnitude, drive, tunables.driveCurrentGain);
return { gradient, tint, spawnRate: spawnRateAt(energy), rise, currentMagnitude: magnitude };
}
export function Ambience({ album, state, tunables, manual, onDebugFrame }: Props) {
const canvasRef = useRef<HTMLCanvasElement | null>(null);
const detail = useTrackDetail(album, state);
const playingRef = useRef(state.playing);
playingRef.current = state.playing;
const detailRef = useRef(detail);
detailRef.current = detail;
const tunablesRef = useRef(tunables);
tunablesRef.current = tunables;
const manualRef = useRef(manual);
manualRef.current = manual;
const onDebugFrameRef = useRef(onDebugFrame);
onDebugFrameRef.current = onDebugFrame;
const groupRef = useRef<Group | "idle">(album ? groupOf(album) : "idle");
groupRef.current = album ? groupOf(album) : "idle";
const trackTitleRef = useRef(state.track_title);
trackTitleRef.current = state.track_title;
// The same anchor trick as `usePlaybackClock`, but read from inside the rAF loop
// rather than through `useState` - see the module docstring.
const clockAnchor = useRef({ position: state.position, at: performance.now() });
useEffect(() => {
clockAnchor.current = { position: state.position, at: performance.now() };
}, [state.position]);
const trackAnalysis = album?.tracks[state.track_index]?.analysis ?? null;
const analysisRef = useRef(trackAnalysis);
analysisRef.current = trackAnalysis;
const baseRef = useRef(ambienceBaseFor(album, trackAnalysis));
useEffect(() => {
baseRef.current = ambienceBaseFor(album, trackAnalysis);
// eslint-disable-next-line react-hooks/exhaustive-deps
}, [album, trackAnalysis]);
useEffect(() => {
const canvas = canvasRef.current;
const ctx = canvas?.getContext("2d");
if (!canvas || !ctx) return;
const size = { width: 0, height: 0 };
// The CSS `.ambience` rule pins this canvas to 100% of `.stage`, independent of
// its own width/height attributes - required so this handler's own writes below
// can't feed back into the size it just measured (see the comment there). The
// clamp is a second line of defence against the same failure mode from anywhere
// else: an unclamped size can hit the browser's canvas allocation limit and throw.
const MAX_BACKING_STORE_PX = 4096;
const resize = () => {
const dpr = window.devicePixelRatio || 1;
const rect = canvas.getBoundingClientRect();
size.width = rect.width;
size.height = rect.height;
canvas.width = Math.min(MAX_BACKING_STORE_PX, Math.max(1, Math.round(rect.width * dpr)));
canvas.height = Math.min(MAX_BACKING_STORE_PX, Math.max(1, Math.round(rect.height * dpr)));
// Scale factors from the (possibly clamped) backing store, not raw `dpr`, so
// drawing in CSS-pixel units still lands correctly even when clamped.
const scaleX = canvas.width / Math.max(1, rect.width);
const scaleY = canvas.height / Math.max(1, rect.height);
ctx.setTransform(scaleX, 0, 0, scaleY, 0, 0);
};
resize();
const observer = new ResizeObserver(resize);
observer.observe(canvas);
const media = matchMedia("(prefers-reduced-motion: reduce)");
let reducedMotion = media.matches;
const onMotionChange = () => {
reducedMotion = media.matches;
};
media.addEventListener("change", onMotionChange);
let bubbles: Bubble[] = [];
let spawnAccumulator = 0;
let beatCursor = 0;
let kick = 0;
let manualBeatTimer = 0;
let lastFrame = performance.now();
let lastDebugEmit = 0;
let raf = 0;
// Seeded from the first real sample, unsmoothed, so the first paint doesn't flash
// from a hardcoded baseline toward wherever the track's mood actually starts.
let smoothedEnergy = 0.5;
let smoothedValence = 0.5;
let smoothedDrive = 1.0;
{
const base0 = baseRef.current;
const manual0 = manualRef.current;
const usingManual0 = manual0.enabled && !playingRef.current;
if (usingManual0) {
smoothedEnergy = manual0.energy;
smoothedValence = manual0.valence;
smoothedDrive = manual0.drive;
} else if (base0.music) {
const curve0 = effectiveCurve(analysisRef.current, detailRef.current);
const sampled0 = sampleCurveAt(curve0, clockAnchor.current.position);
smoothedEnergy = sampled0.energy;
smoothedValence = sampled0.valence;
smoothedDrive = sampled0.drive;
}
}
let current: AmbienceTarget = ambienceFromSampled(
baseRef.current,
manualRef.current.enabled && !playingRef.current,
manualRef.current,
tunablesRef.current,
smoothedEnergy,
smoothedValence,
smoothedDrive,
);
const tick = (now: number) => {
const dt = Math.min(MAX_FRAME_SECONDS, (now - lastFrame) / 1000);
lastFrame = now;
const base = baseRef.current;
const tunables = tunablesRef.current;
const manual = manualRef.current;
const playing = playingRef.current;
// Real playback always wins the instant it starts - manual mode only ever has
// an effect while genuinely nothing is playing, so the two inputs can never
// fight over the same frame.
const usingManual = manual.enabled && !playing;
const effectivePlaying = playing || usingManual; // manual mode still animates
const anchor = clockAnchor.current;
const position = playing ? anchor.position + (now - anchor.at) / 1000 : anchor.position;
const sampled = usingManual
? { energy: manual.energy, valence: manual.valence, drive: manual.drive }
: sampleCurveAt(effectiveCurve(analysisRef.current, detailRef.current), position);
const smoothing =
tunables.curveSmoothingTau <= 0 ? 1 : 1 - Math.exp(-dt / tunables.curveSmoothingTau);
smoothedEnergy = lerp(smoothedEnergy, sampled.energy, smoothing);
smoothedValence = lerp(smoothedValence, sampled.valence, smoothing);
smoothedDrive = lerp(smoothedDrive, sampled.drive, smoothing);
const target = ambienceFromSampled(
base,
usingManual,
manual,
tunables,
smoothedEnergy,
smoothedValence,
smoothedDrive,
);
current = lerpAmbience(current, target, 1 - Math.exp(-dt / CROSSFADE_TAU_SECONDS));
const analysis = analysisRef.current;
const beatsTrusted =
analysis !== null && analysis.beats && (analysis.pulse ?? 0) >= PULSE_THRESHOLD;
const grid = beatsTrusted && !usingManual ? detailRef.current : null;
if (usingManual) {
// A continuous fake metronome at the manual tempo, instead of walking a real
// grid - reuses the same `kick` variable and decay as real beats, so every
// downstream effect (spawn burst, current boost) is identical either way.
manualBeatTimer -= dt;
if (manualBeatTimer <= 0) {
kick = Math.max(kick, manual.beatStrength);
manualBeatTimer = 60 / Math.max(1, manual.tempo);
}
} else if (grid && grid.times.length > 0) {
// Playback normally only moves forward, so the common case is just advancing
// the cursor with the `while` below. A backward seek is the one thing that
// can break that invariant - detected by the previously consumed beat now
// being back in the future - and needs a real re-search.
if (beatCursor > 0 && grid.times[beatCursor - 1]! > position) {
beatCursor = beatCursorFor(grid.times, position);
}
while (beatCursor < grid.times.length && grid.times[beatCursor]! <= position) {
kick = Math.max(kick, grid.strengths[beatCursor] ?? 0);
beatCursor++;
}
} else {
beatCursor = 0;
}
kick *= Math.exp(-dt / KICK_DECAY_TAU_SECONDS);
const kickedMagnitude = current.currentMagnitude * (1 + kick * tunables.beatCurrentFactor);
if (effectivePlaying) {
spawnAccumulator += (current.spawnRate + kick * BEAT_BURST_RATE) * dt;
while (spawnAccumulator >= 1) {
spawnAccumulator -= 1;
bubbles.push(spawnBubble(size.width));
}
}
// Not playing (and not in manual mode): no new spawns, but bubbles already in
// flight keep rising and draining away rather than freezing mid-air.
const riseSpeed = current.rise * size.height * BASE_RISE_FRACTION_PER_SECOND;
bubbles = bubbles.filter((b) => {
const heightFraction = clamp01(b.risen / size.height);
const velocity = currentVelocityAt(heightFraction, kickedMagnitude);
b.x = (((b.x + velocity * dt) % size.width) + size.width) % size.width;
b.risen += riseSpeed * dt;
b.size = b.baseSize * (1 + tunables.bubbleGrowthRate * heightFraction);
return b.risen < size.height + BUBBLE_MAX_SIZE * 2;
});
ctx.clearRect(0, 0, size.width, size.height);
const [top, mid, deep] = current.gradient;
const gradient = ctx.createLinearGradient(0, 0, 0, size.height);
gradient.addColorStop(0, oklchString(top));
gradient.addColorStop(0.45, oklchString(mid));
gradient.addColorStop(1, oklchString(deep));
ctx.fillStyle = gradient;
ctx.fillRect(0, 0, size.width, size.height);
if (!reducedMotion) {
for (const b of bubbles) {
const y = size.height + BUBBLE_MAX_SIZE - b.risen;
const topThird = size.height / 3;
const fade = y < topThird ? Math.max(0, y / topThird) : 1;
ctx.beginPath();
ctx.fillStyle = oklchString(current.tint, b.baseAlpha * fade);
ctx.arc(b.x, y, b.size / 2, 0, Math.PI * 2);
ctx.fill();
}
}
const debugFrame = onDebugFrameRef.current;
if (debugFrame && now - lastDebugEmit >= DEBUG_EMIT_INTERVAL_MS) {
lastDebugEmit = now;
debugFrame({
group: groupRef.current,
trackTitle: trackTitleRef.current,
playing,
position,
analysis,
target,
current,
sampledEnergy: smoothedEnergy,
sampledValence: smoothedValence,
sampledDrive: smoothedDrive,
currentMagnitude: kickedMagnitude,
kick,
beatsTrusted,
beatGridLoaded: grid !== null,
beatCount: grid?.times.length ?? 0,
beatCursor,
bubbleCount: bubbles.length,
curveLoaded: detailRef.current?.curve != null,
usingManual,
});
}
raf = requestAnimationFrame(tick);
};
raf = requestAnimationFrame(tick);
return () => {
cancelAnimationFrame(raf);
observer.disconnect();
media.removeEventListener("change", onMotionChange);
};
}, []);
return (
<canvas
ref={canvasRef}
className="ambience"
aria-hidden="true"
style={{ position: "absolute", inset: 0, zIndex: 0, pointerEvents: "none" }}
/>
);
}

View File

@@ -0,0 +1,329 @@
/** `?debugDynamicUI=1`: every number the reactive background is currently using, laid
* out so a "why does this track look wrong" question can be answered by eye - the raw
* analysis, what it's sampled to right now, what that maps to, and where the eased
* on-screen look currently sits relative to that target (the crossfade, beat kicks
* and so on are otherwise invisible except as motion). Also the live controls: 5
* always-on tunable sliders, and a manual-control mode that takes over the whole
* pipeline whenever nothing is actually playing - a playground for developing a feel
* for the effect without needing music playing. Not part of the app for anyone who
* didn't ask for it: this only ever mounts behind the query param, in `App.tsx`.
*/
import type { AmbienceDebugSnapshot } from "./Ambience";
import { oklchString, type Ambience, type Oklch } from "../lib/ambience";
import type { AmbienceTunables, ManualControl } from "../lib/ambienceTunables";
interface Props {
snapshot: AmbienceDebugSnapshot;
tunables: AmbienceTunables;
onTunablesChange: (tunables: AmbienceTunables) => void;
manual: ManualControl;
onManualChange: (manual: ManualControl) => void;
playing: boolean;
}
const fmt = (value: number | null, digits = 2): string =>
value === null ? "—" : value.toFixed(digits);
function Swatch({ color }: { color: Oklch }) {
return (
<span
style={{
display: "inline-block",
width: 14,
height: 14,
borderRadius: 4,
background: oklchString(color),
border: "1px solid oklch(100% 0 0 / .4)",
verticalAlign: "middle",
}}
/>
);
}
function Field({ label, value }: { label: string; value: string }) {
return (
<span style={{ whiteSpace: "nowrap" }}>
<span style={{ opacity: 0.6 }}>{label}</span> <b>{value}</b>
</span>
);
}
function RowLabel({ children }: { children: string }) {
return (
<span style={{ opacity: 0.6, width: 58, display: "inline-block" }}>{children}</span>
);
}
function AmbienceRow({ label, ambience }: { label: string; ambience: Ambience }) {
const [top] = ambience.gradient;
return (
<div style={{ display: "flex", gap: 14, alignItems: "center", flexWrap: "wrap" }}>
<RowLabel>{label}</RowLabel>
<Swatch color={top} />
<Field label="hue" value={fmt(top.h, 0)} />
<Field label="chroma" value={fmt(top.c, 3)} />
<Field label="lightness" value={fmt(top.l, 0)} />
<span>
<span style={{ opacity: 0.6 }}>tint</span> <Swatch color={ambience.tint} />
</span>
<Field label="spawn/s" value={fmt(ambience.spawnRate)} />
<Field label="current" value={fmt(ambience.currentMagnitude, 1)} />
<Field label="rise" value={fmt(ambience.rise)} />
</div>
);
}
function TunableInput({
label,
value,
min,
max,
step,
onChange,
}: {
label: string;
value: number;
min: number;
max: number;
step: number;
onChange: (value: number) => void;
}) {
return (
<label style={{ display: "inline-flex", alignItems: "center", gap: 6, whiteSpace: "nowrap" }}>
<span style={{ opacity: 0.6 }}>{label}</span>
<input
type="range"
min={min}
max={max}
step={step}
value={value}
onChange={(e) => onChange(Number(e.target.value))}
style={{ width: 70 }}
/>
<b style={{ width: 32, display: "inline-block" }}>{value.toFixed(2)}</b>
</label>
);
}
function TunableRow({
tunables,
onChange,
}: {
tunables: AmbienceTunables;
onChange: (tunables: AmbienceTunables) => void;
}) {
const set = <K extends keyof AmbienceTunables>(key: K, value: number) =>
onChange({ ...tunables, [key]: value });
return (
<div
style={{
display: "flex",
gap: 14,
flexWrap: "wrap",
alignItems: "center",
pointerEvents: "auto",
}}
>
<RowLabel>tunables</RowLabel>
<TunableInput
label="energy→color"
value={tunables.energyColorGain}
min={0}
max={2}
step={0.05}
onChange={(v) => set("energyColorGain", v)}
/>
<TunableInput
label="beat→current"
value={tunables.beatCurrentFactor}
min={0}
max={2}
step={0.05}
onChange={(v) => set("beatCurrentFactor", v)}
/>
<TunableInput
label="drive→current"
value={tunables.driveCurrentGain}
min={0}
max={2}
step={0.05}
onChange={(v) => set("driveCurrentGain", v)}
/>
<TunableInput
label="bubble growth"
value={tunables.bubbleGrowthRate}
min={0}
max={1}
step={0.02}
onChange={(v) => set("bubbleGrowthRate", v)}
/>
<TunableInput
label="curve τ (s)"
value={tunables.curveSmoothingTau}
min={0}
max={2}
step={0.05}
onChange={(v) => set("curveSmoothingTau", v)}
/>
</div>
);
}
function ManualControlRow({
manual,
onChange,
playing,
active,
}: {
manual: ManualControl;
onChange: (manual: ManualControl) => void;
playing: boolean;
active: boolean;
}) {
const set = <K extends keyof ManualControl>(key: K, value: ManualControl[K]) =>
onChange({ ...manual, [key]: value });
return (
<div
style={{
display: "flex",
gap: 14,
flexWrap: "wrap",
alignItems: "center",
pointerEvents: "auto",
}}
>
<span style={{ opacity: 0.6, width: 58, display: "inline-block" }}>
manual {active && <span style={{ color: "oklch(80% 0.14 340)" }}></span>}
</span>
<label style={{ display: "inline-flex", alignItems: "center", gap: 6 }}>
<input
type="checkbox"
checked={manual.enabled}
onChange={(e) => set("enabled", e.target.checked)}
/>
enabled{playing ? " (available while paused)" : ""}
</label>
<TunableInput
label="energy"
value={manual.energy}
min={0}
max={1}
step={0.02}
onChange={(v) => set("energy", v)}
/>
<TunableInput
label="valence"
value={manual.valence}
min={0}
max={1}
step={0.02}
onChange={(v) => set("valence", v)}
/>
<TunableInput
label="tempo"
value={manual.tempo}
min={40}
max={200}
step={1}
onChange={(v) => set("tempo", v)}
/>
<TunableInput
label="drive"
value={manual.drive}
min={0}
max={1}
step={0.02}
onChange={(v) => set("drive", v)}
/>
<TunableInput
label="beat strength"
value={manual.beatStrength}
min={0}
max={1}
step={0.02}
onChange={(v) => set("beatStrength", v)}
/>
</div>
);
}
export function AmbienceDebugOverlay({
snapshot,
tunables,
onTunablesChange,
manual,
onManualChange,
playing,
}: Props) {
const { analysis } = snapshot;
return (
<div
style={{
position: "fixed",
left: 0,
right: 0,
bottom: 0,
zIndex: 1000,
background: "oklch(15% 0.02 210 / .92)",
color: "oklch(95% 0.01 210)",
font: "12px/1.5 ui-monospace, Menlo, monospace",
padding: "8px 14px",
display: "flex",
flexDirection: "column",
gap: 4,
pointerEvents: "none",
textShadow: "0 1px 2px oklch(0% 0 0 / .6)",
}}
>
<div style={{ display: "flex", gap: 14, flexWrap: "wrap", alignItems: "center" }}>
<b style={{ color: "oklch(80% 0.14 340)" }}>debugDynamicUI</b>
<Field label="group" value={snapshot.group} />
<Field label="playing" value={snapshot.playing ? "yes" : "no"} />
<Field label="position" value={`${fmt(snapshot.position, 1)}s`} />
<Field label="track" value={snapshot.trackTitle ?? "—"} />
</div>
<div style={{ display: "flex", gap: 14, flexWrap: "wrap", alignItems: "center" }}>
<RowLabel>analysis</RowLabel>
<Field label="tempo" value={analysis ? `${fmt(analysis.tempo, 0)} bpm` : "—"} />
<Field label="energy" value={fmt(analysis?.energy ?? null)} />
<Field label="valence" value={fmt(analysis?.valence ?? null)} />
<Field label="brightness" value={fmt(analysis?.brightness ?? null)} />
<Field label="pulse" value={fmt(analysis?.pulse ?? null)} />
<Field label="beats" value={analysis ? (analysis.beats ? "yes" : "no") : "—"} />
</div>
<div style={{ display: "flex", gap: 14, flexWrap: "wrap", alignItems: "center" }}>
<RowLabel>sampled</RowLabel>
<Field label="energy" value={fmt(snapshot.sampledEnergy)} />
<Field label="valence" value={fmt(snapshot.sampledValence)} />
<Field label="drive" value={fmt(snapshot.sampledDrive)} />
<Field label="curve" value={snapshot.curveLoaded ? "loaded" : "flat fallback"} />
</div>
<AmbienceRow label="target" ambience={snapshot.target} />
<AmbienceRow label="current" ambience={snapshot.current} />
<div style={{ display: "flex", gap: 14, flexWrap: "wrap", alignItems: "center" }}>
<RowLabel>beats</RowLabel>
<Field label="kick" value={fmt(snapshot.kick)} />
<Field label="trusted" value={snapshot.beatsTrusted ? "yes" : "no"} />
<Field label="loaded" value={snapshot.beatGridLoaded ? "yes" : "no"} />
<Field label="count" value={String(snapshot.beatCount)} />
<Field label="cursor" value={String(snapshot.beatCursor)} />
<Field label="kicked current" value={fmt(snapshot.currentMagnitude, 1)} />
<Field label="bubbles" value={String(snapshot.bubbleCount)} />
</div>
<TunableRow tunables={tunables} onChange={onTunablesChange} />
<ManualControlRow
manual={manual}
onChange={onManualChange}
playing={playing}
active={snapshot.usingManual}
/>
</div>
);
}

View File

@@ -73,6 +73,9 @@ export function PlayView({
pointerEvents: "none", pointerEvents: "none",
opacity: 0.92, opacity: 0.92,
animation: "dolphinSwim 62s linear infinite", animation: "dolphinSwim 62s linear infinite",
// Freezes in its current pose rather than snapping back to frame 0 -
// `animation: "none"` would restart it, `animationPlayState` just pauses.
animationPlayState: state.playing ? "running" : "paused",
filter: "drop-shadow(0 10px 26px oklch(10% 0.05 210 / .45))", filter: "drop-shadow(0 10px 26px oklch(10% 0.05 210 / .45))",
}} }}
/> />

View File

@@ -0,0 +1,44 @@
/** The current track's beat grid and mood/drive curves, fetched lazily - never inside
* the library payload, and never for a track that has no analysis of any kind.
*
* Fetched whenever the current track is music and has *any* analysis - unlike the
* beat grid's own trustworthiness gate (a spoken-word track or a free-tempo lullaby
* always produces *some* grid from the beat tracker, and pulsing the background to a
* beat nobody can actually hear looks broken), which is a decision about whether the
* already-fetched beats are used, not whether they're fetched at all - see the
* `PULSE_THRESHOLD` gate in `Ambience.tsx`. The energy/valence/drive curves this also
* carries have no such caveat: they drive color and current for every analyzed music
* track regardless of beat quality.
*/
import { useEffect, useState } from "react";
import { api } from "../api/client";
import type { Album, PlayerState, TrackDetail } from "../api/types";
import { groupOf } from "../lib/search";
export function useTrackDetail(album: Album | null, state: PlayerState): TrackDetail | null {
const [detail, setDetail] = useState<TrackDetail | null>(null);
const albumId = state.album_id;
const trackIndex = state.track_index;
const analysis = album?.tracks[trackIndex]?.analysis ?? null;
const eligible =
album !== null && albumId !== null && groupOf(album) === "music" && analysis !== null;
useEffect(() => {
// Cleared up front, not just on failure - a track change must not keep the
// previous track's detail visible while the new one is still in flight.
setDetail(null);
if (!eligible || albumId === null) return;
let cancelled = false;
void api.trackDetail(albumId, trackIndex).then((result) => {
if (!cancelled) setDetail(result);
});
return () => {
cancelled = true;
};
}, [eligible, albumId, trackIndex]);
return detail;
}

View File

@@ -0,0 +1,243 @@
import { describe, expect, it } from "vitest";
import type { Album, TrackAnalysis } from "../../api/types";
import {
ambienceBaseFor,
ambienceColorAt,
currentMagnitudeAt,
currentVelocityAt,
effectiveCurve,
IDLE_AMBIENCE,
sampleCurveAt,
spawnRateAt,
tempoToMagnitude,
tempoToRise,
} from "../ambience";
import { DEFAULT_TUNABLES } from "../ambienceTunables";
function album(over: Partial<Album> = {}): Album {
return {
id: "a",
section: "Musik",
kind: "music",
title: "Album",
artist: "Kinderparty",
series: null,
figure: null,
category: "Kinderparty",
colors: ["#111111", "#222222", "#333333"],
has_cover: false,
duration: 120,
tracks: [{ title: "Lied", duration: 60, analysis: null }],
...over,
};
}
function analysis(over: Partial<TrackAnalysis> = {}): TrackAnalysis {
return {
tempo: null,
energy: null,
valence: null,
brightness: null,
pulse: null,
beats: false,
...over,
};
}
describe("ambienceBaseFor", () => {
it("is the idle sea with no bubbles when nothing is loaded", () => {
const base = ambienceBaseFor(null, null);
expect(base.music).toBe(false);
expect(base.fixed).toEqual(IDLE_AMBIENCE);
});
it("gives music, audiobooks and podcasts their own, distinct hue families", () => {
const musicHue = IDLE_AMBIENCE.gradient[0].h; // music's baseline hue
const audiobook = ambienceBaseFor(album({ kind: "book", section: "Hörbücher" }), null);
const podcast = ambienceBaseFor(album({ kind: "book", section: "Kinderpodcasts" }), null);
const hues = [musicHue, audiobook.fixed!.gradient[0].h, podcast.fixed!.gradient[0].h];
expect(new Set(hues).size).toBe(3); // all three distinct
expect(musicHue).toBe(210);
expect(audiobook.fixed!.gradient[0].h).toBe(80);
expect(podcast.fixed!.gradient[0].h).toBe(300);
});
it("never applies the mood mapping outside music, even if a book track somehow has one", () => {
const bookAlbum = album({ kind: "book", section: "Hörbücher" });
const withoutAnalysis = ambienceBaseFor(bookAlbum, null);
const withAnalysis = ambienceBaseFor(
bookAlbum,
analysis({ valence: 1, energy: 1, brightness: 1, tempo: 160 }),
);
expect(withAnalysis).toEqual(withoutAnalysis);
expect(withAnalysis.baseCurrentMagnitude).toBe(0); // effects only ever touch music
});
it("gives podcasts a slower spawn rate than music's baseline drift", () => {
const musicBaselineSpawnRate = spawnRateAt(0.5);
const podcast = ambienceBaseFor(album({ kind: "book", section: "Kinderpodcasts" }), null);
expect(podcast.fixed!.spawnRate).toBeLessThan(musicBaselineSpawnRate);
});
it("un-analyzed music falls back to the same tempo baseline as tempoToMagnitude/Rise", () => {
const musicAlbum = album({ kind: "music", section: "Musik" });
const base = ambienceBaseFor(musicAlbum, null);
expect(base.music).toBe(true);
expect(base.fixed).toBeNull();
expect(base.baseCurrentMagnitude).toBe(tempoToMagnitude(null));
expect(base.rise).toBe(tempoToRise(null));
});
it("treats a Figuren album by its kind, not its section", () => {
const figureBook = album({ kind: "book", section: "Figuren", figure: "eule" });
const podcast = ambienceBaseFor(album({ kind: "book", section: "Kinderpodcasts" }), null);
const figureBase = ambienceBaseFor(figureBook, null);
expect(figureBase.fixed).not.toEqual(podcast.fixed);
});
});
describe("tempoToMagnitude / tempoToRise", () => {
it("has a stronger base current and rises faster for an up-tempo track than a slow one", () => {
expect(tempoToMagnitude(160)).toBeGreaterThan(tempoToMagnitude(60));
expect(tempoToRise(160)).toBeGreaterThan(tempoToRise(60));
});
});
describe("effectiveCurve", () => {
it("falls back to the baseline per scalar, not as an all-or-nothing object", () => {
// A track the analyzer failed on: every field `null`, not a missing object.
const attempted = effectiveCurve(analysis(), null);
expect(attempted.energy).toEqual([0.5]);
expect(attempted.valence).toEqual([0.5]);
expect(attempted.drive).toEqual([1.0]); // no-op multiplier, not a damping 0.5
// Only `valence` present: it should carry through independently of energy's baseline.
const partial = effectiveCurve(analysis({ valence: 1 }), null);
expect(partial.energy).toEqual([0.5]);
expect(partial.valence).toEqual([1]);
});
it("prefers a fetched curve over the scalar fallback", () => {
const curve = { hop_seconds: 1, energy: [0.9], valence: [0.1], drive: [0.7] };
const result = effectiveCurve(analysis({ energy: 0.5 }), { times: [], strengths: [], curve });
expect(result).toBe(curve);
});
});
describe("ambienceColorAt", () => {
it("equals the un-analyzed baseline exactly at energy=valence=0.5", () => {
const result = ambienceColorAt(0.5, 0.5, DEFAULT_TUNABLES);
expect(result.gradient).toEqual(IDLE_AMBIENCE.gradient);
expect(result.tint).toEqual(IDLE_AMBIENCE.tint);
});
it("collapses to the baseline when energyColorGain is 0, regardless of energy/valence", () => {
const tunables = { ...DEFAULT_TUNABLES, energyColorGain: 0 };
const result = ambienceColorAt(0, 0.5, tunables);
expect(result.gradient[0].c).toBe(IDLE_AMBIENCE.gradient[0].c);
expect(result.gradient[0].l).toBe(IDLE_AMBIENCE.gradient[0].l);
});
it("energy alone drives chroma/lightness, not hue - the primary, noticeable driver", () => {
const low = ambienceColorAt(0, 0.5, DEFAULT_TUNABLES);
const high = ambienceColorAt(1, 0.5, DEFAULT_TUNABLES);
expect(high.gradient[0].h).toBe(low.gradient[0].h);
expect(high.gradient[0].c).toBeGreaterThan(low.gradient[0].c);
expect(high.gradient[0].l).toBeGreaterThan(low.gradient[0].l);
});
it("valence alone drives hue, not chroma/lightness, with a smaller swing than energy's", () => {
const low = ambienceColorAt(0.5, 0, DEFAULT_TUNABLES);
const high = ambienceColorAt(0.5, 1, DEFAULT_TUNABLES);
expect(high.gradient[0].c).toBe(low.gradient[0].c);
expect(high.gradient[0].l).toBe(low.gradient[0].l);
const hueSwing = Math.abs(high.gradient[0].h - low.gradient[0].h);
expect(hueSwing).toBeGreaterThan(0);
expect(hueSwing).toBeLessThan(30); // a nudge, not the primary colour driver
});
it("keeps the valence hue nudge inside a water-plausible range at both extremes", () => {
const sad = ambienceColorAt(0.5, 0, DEFAULT_TUNABLES);
const happy = ambienceColorAt(0.5, 1, DEFAULT_TUNABLES);
for (const hue of [sad.gradient[0].h, happy.gradient[0].h]) {
expect(hue).toBeGreaterThanOrEqual(160); // never drifts as far as orange
expect(hue).toBeLessThanOrEqual(260);
}
// Sad (dark, low valence) reads colder/more violet than happy (bright, high valence).
expect(sad.gradient[0].h).toBeGreaterThan(happy.gradient[0].h);
});
});
describe("spawnRateAt", () => {
it("increases monotonically with energy", () => {
expect(spawnRateAt(1)).toBeGreaterThan(spawnRateAt(0.5));
expect(spawnRateAt(0.5)).toBeGreaterThan(spawnRateAt(0));
});
});
describe("currentVelocityAt", () => {
it("is ~0 in the bottom third, +magnitude in the middle, -magnitude in the top", () => {
expect(currentVelocityAt(0.15, 100)).toBeCloseTo(0, 0);
expect(currentVelocityAt(0.5, 100)).toBeCloseTo(100, 0);
expect(currentVelocityAt(0.85, 100)).toBeCloseTo(-100, 0);
});
it("blends smoothly across both band boundaries - no jump between adjacent samples", () => {
const samples = Array.from({ length: 200 }, (_, i) => currentVelocityAt(i / 199, 100));
for (let i = 1; i < samples.length; i++) {
expect(Math.abs(samples[i]! - samples[i - 1]!)).toBeLessThan(15);
}
});
});
describe("sampleCurveAt", () => {
const curve = { hop_seconds: 1, energy: [0.2, 0.8], valence: [0.4, 0.6], drive: [0, 1] };
it("interpolates linearly between samples at a known midpoint", () => {
const result = sampleCurveAt(curve, 0.5);
expect(result.energy).toBeCloseTo(0.5);
expect(result.valence).toBeCloseTo(0.5);
expect(result.drive).toBeCloseTo(0.5);
});
it("holds flat before the first and past the last sample", () => {
expect(sampleCurveAt(curve, -5).energy).toBe(0.2);
expect(sampleCurveAt(curve, 500).energy).toBe(0.8);
});
it("a length-1 curve returns that value everywhere", () => {
const flat = { hop_seconds: 1, energy: [0.42], valence: [0.42], drive: [0.42] };
expect(sampleCurveAt(flat, 0).energy).toBe(0.42);
expect(sampleCurveAt(flat, 99).energy).toBe(0.42);
});
});
describe("currentMagnitudeAt", () => {
it("returns exactly the base magnitude at drive=1, driveCurrentGain=1", () => {
expect(currentMagnitudeAt(100, 1, 1)).toBe(100);
});
it("never drops to 0 at drive=0 - damps toward a floor instead", () => {
const result = currentMagnitudeAt(100, 0, 1);
expect(result).toBeGreaterThan(0);
expect(result).toBeLessThan(100);
});
it("increases monotonically with drive", () => {
expect(currentMagnitudeAt(100, 1, 1)).toBeGreaterThan(currentMagnitudeAt(100, 0.5, 1));
expect(currentMagnitudeAt(100, 0.5, 1)).toBeGreaterThan(currentMagnitudeAt(100, 0, 1));
});
it("driveCurrentGain=0 collapses to the floor regardless of drive", () => {
expect(currentMagnitudeAt(100, 0.2, 0)).toBe(currentMagnitudeAt(100, 0.9, 0));
});
});

296
web/src/lib/ambience.ts Normal file
View File

@@ -0,0 +1,296 @@
/** What the background should look like for whatever is loaded and playing.
*
* Pure mapping, no DOM and no animation: `Ambience.tsx` reads this every frame and
* eases the canvas toward it. Split into a per-track part (`ambienceBaseFor` - group
* palette, tempo-derived rise and current-magnitude base, computed once on track
* change) and per-frame pure functions (`ambienceColorAt`, `spawnRateAt`,
* `currentVelocityAt`, `sampleCurveAt`, `currentMagnitudeAt`) - because `energy` and
* `valence` are now sampled from a per-second curve at the current playback position,
* so the visual result can no longer be a single value computed once per track.
*/
import type { Album, TrackAnalysis, TrackCurves, TrackDetail } from "../api/types";
import type { AmbienceTunables } from "./ambienceTunables";
import { groupOf } from "./search";
export interface Oklch {
/** 0..100, percent lightness. */
l: number;
c: number;
/** Degrees. */
h: number;
}
export interface Ambience {
/** Three gradient stops, top to deep - the vertical falloff every hue family shares. */
gradient: [Oklch, Oklch, Oklch];
/** Bubble fill colour. */
tint: Oklch;
/** Bubbles spawned per second. */
spawnRate: number;
/** Rise speed, viewport-heights per second. */
rise: number;
/** U≈V magnitude (px/s) of the 3-band water current - tempo x drive-derived,
* before the beat kick multiplies it further in `Ambience.tsx`. 0 for non-music. */
currentMagnitude: number;
}
export function oklchString(color: Oklch, alpha = 1): string {
return `oklch(${color.l}% ${color.c} ${color.h} / ${alpha})`;
}
const clamp01 = (value: number): number => Math.max(0, Math.min(1, value));
const lerp = (a: number, b: number, t: number): number => a + (b - a) * t;
//: Today's sea, unchanged - the Dolphin Beats identity, and music's baseline hue.
const MUSIC_HUE = 210;
//: Matches the book-paper tint `cardBackground` already uses for audiobooks
//: (`oklch(93% 0.055 88)` in `lib/covers.ts`).
const AUDIOBOOK_HUE = 80;
//: The one unused family, so all three shelves are instantly distinguishable at a glance.
const PODCAST_HUE = 300;
const BASE_CHROMA = 0.07;
const BASE_TOP_LIGHTNESS = 55;
const DEEP_LIGHTNESS = 20;
const DEEP_CHROMA = 0.045;
/** The vertical falloff every hue family shares - only the hue and the top stop's
* chroma/lightness change; the deep stop always stays dark so text contrast never
* degrades, no matter how bright a mood gets. */
function gradientForHue(hue: number, chroma: number, topLightness: number): [Oklch, Oklch, Oklch] {
return [
{ l: topLightness, c: chroma, h: hue },
{ l: topLightness - 17, c: Math.max(0, chroma - 0.01), h: hue },
{ l: DEEP_LIGHTNESS, c: DEEP_CHROMA, h: hue },
];
}
//: Roughly today's density: 21 fixed bubbles over a ~10s mean period.
const MUSIC_SPAWN_BASELINE = 2.0;
//: Energy's swing on top of the baseline - matches the old energy-to-spawn-rate range.
const MUSIC_SPAWN_ENERGY_SWING = 2.6;
//: Audiobooks and podcasts: a slow ambient drift, not a field.
const AMBIENT_SPAWN_RATE = 0.5;
const BUBBLE_TINT: Oklch = { l: 90, c: 0.02, h: MUSIC_HUE };
//: Tint chroma tracks gradient chroma proportionally, so a result with every scalar
//: `null` (the un-analyzed baseline, or a track the analyzer failed on) computes back
//: to exactly `BUBBLE_TINT.c` rather than a near-miss from an independent constant.
const TINT_CHROMA_RATIO = BUBBLE_TINT.c / BASE_CHROMA;
const TEMPO_BPM_FLOOR = 60;
const TEMPO_BPM_CEIL = 160;
const RISE_AT_FLOOR = 0.6;
const RISE_AT_CEIL = 1.6;
//: U≈V base magnitude range, px/s - tuned so bubbles visibly drift sideways over a
//: few seconds without looking like they're flying off screen.
const CURRENT_MAGNITUDE_MIN = 15;
const CURRENT_MAGNITUDE_MAX = 55;
//: Fallback for unanalyzed music (`tempo === null`) - a small non-zero value, so an
//: unanalyzed track's water isn't dead-still, mirroring the old baseline turbulence.
const CURRENT_MAGNITUDE_BASELINE = 25;
/** tempo → U≈V base magnitude (px/s), before drive or any beat kick modulate it
* further. `null` (unanalyzed music) falls back to `CURRENT_MAGNITUDE_BASELINE`. */
export function tempoToMagnitude(tempo: number | null): number {
if (tempo === null) return CURRENT_MAGNITUDE_BASELINE;
const tempoNorm = clamp01((tempo - TEMPO_BPM_FLOOR) / (TEMPO_BPM_CEIL - TEMPO_BPM_FLOOR));
return lerp(CURRENT_MAGNITUDE_MIN, CURRENT_MAGNITUDE_MAX, tempoNorm);
}
/** tempo → rise speed multiplier. `null` (unanalyzed music) falls back to `1`, the
* same baseline `IDLE_AMBIENCE` uses. */
export function tempoToRise(tempo: number | null): number {
if (tempo === null) return 1;
const tempoNorm = clamp01((tempo - TEMPO_BPM_FLOOR) / (TEMPO_BPM_CEIL - TEMPO_BPM_FLOOR));
return lerp(RISE_AT_FLOOR, RISE_AT_CEIL, tempoNorm);
}
/** Nothing loaded: today's sea, no bubbles. */
export const IDLE_AMBIENCE: Ambience = {
gradient: gradientForHue(MUSIC_HUE, BASE_CHROMA, BASE_TOP_LIGHTNESS),
tint: BUBBLE_TINT,
spawnRate: 0,
rise: 1,
currentMagnitude: 0,
};
function fixedAmbience(hue: number): {
gradient: [Oklch, Oklch, Oklch];
tint: Oklch;
spawnRate: number;
} {
return {
gradient: gradientForHue(hue, BASE_CHROMA, BASE_TOP_LIGHTNESS),
tint: { l: 90, c: 0.02, h: hue },
spawnRate: AMBIENT_SPAWN_RATE,
};
}
const AUDIOBOOK_FIXED = fixedAmbience(AUDIOBOOK_HUE);
const PODCAST_FIXED = fixedAmbience(PODCAST_HUE);
/** The per-track constant part of the picture: group palette (non-music), and
* tempo-derived rise/current-magnitude bases (music). Computed once on track change -
* everything time-varying (colour, current strength) is sampled per-frame instead,
* see `ambienceColorAt`/`currentMagnitudeAt`. */
export interface AmbienceBase {
music: boolean;
/** Complete, computed once, for non-music/idle. Null for music, whose gradient/tint/
* spawnRate instead come from `ambienceColorAt`/`spawnRateAt` every frame. */
fixed: { gradient: [Oklch, Oklch, Oklch]; tint: Oklch; spawnRate: number } | null;
rise: number;
/** Tempo-only base, *before* the per-frame `drive` modulation in
* `currentMagnitudeAt` - distinct from `Ambience.currentMagnitude`, which is the
* fully-modulated value actually used for rendering. Named differently on purpose
* so the two are never confused. */
baseCurrentMagnitude: number;
}
/** The background's per-track basis for whatever `album` is loaded - not necessarily
* playing; the caller decides whether to actually animate from `state.playing`.
* `analysis` is ignored for anything but music, even if a book or podcast track
* somehow carries one - the mood/current treatment is a music-only effect. */
export function ambienceBaseFor(
album: Album | null,
analysis: TrackAnalysis | null,
): AmbienceBase {
if (album === null) {
return {
music: false,
fixed: IDLE_AMBIENCE,
rise: IDLE_AMBIENCE.rise,
baseCurrentMagnitude: 0,
};
}
const group = groupOf(album);
if (group === "audiobooks" || group === "podcasts") {
const fixed = group === "audiobooks" ? AUDIOBOOK_FIXED : PODCAST_FIXED;
return { music: false, fixed, rise: 0.6, baseCurrentMagnitude: 0 };
}
const tempo = analysis?.tempo ?? null;
return {
music: true,
fixed: null,
rise: tempoToRise(tempo),
baseCurrentMagnitude: tempoToMagnitude(tempo),
};
}
//: valence 0 -> dark blue-violet, valence 1 -> bright green-teal, centred on
//: MUSIC_HUE - a narrower swing than the pre-refactor 85 degrees, since valence is
//: now a secondary nudge rather than the primary colour driver. Clamped implicitly by
//: its own small amplitude: at any valence 0..1 the result stays well inside a
//: water-plausible range, never drifting toward orange.
const VALENCE_HUE_NUDGE = 24;
//: Energy's swing on chroma/lightness - wide, so the effect reads as "noticeable" per
//: the design brief, in contrast to valence's now-narrow hue nudge.
const CHROMA_ENERGY_SWING = 0.11;
const LIGHTNESS_ENERGY_SWING = 33;
/** hue from valence only (secondary, smaller nudge); chroma+lightness from energy
* only (primary, "noticeable" driver). Both centred on the un-analyzed baseline
* (energy=valence=0.5) so that value reproduces today's flat look exactly, at any
* `energyColorGain`. */
export function ambienceColorAt(
energy: number,
valence: number,
tunables: AmbienceTunables,
): { gradient: [Oklch, Oklch, Oklch]; tint: Oklch } {
const hue = MUSIC_HUE - VALENCE_HUE_NUDGE * (clamp01(valence) - 0.5);
const gain = tunables.energyColorGain;
const energyDelta = clamp01(energy) - 0.5;
const chroma = Math.max(0, BASE_CHROMA + CHROMA_ENERGY_SWING * gain * energyDelta);
const topLightness = BASE_TOP_LIGHTNESS + LIGHTNESS_ENERGY_SWING * gain * energyDelta;
return {
gradient: gradientForHue(hue, chroma, topLightness),
tint: { l: 90, c: chroma * TINT_CHROMA_RATIO, h: hue },
};
}
/** energy → bubble spawn rate, monotonically increasing. */
export function spawnRateAt(energy: number): number {
return MUSIC_SPAWN_BASELINE + MUSIC_SPAWN_ENERGY_SWING * (clamp01(energy) - 0.5);
}
//: The 3-band water current's boundaries and the width of the smooth blend across
//: each one - plain constants, not exposed in the debug UI.
const BAND_LOW = 1 / 3;
const BAND_HIGH = 2 / 3;
const BAND_BLEND = 0.12;
function smoothstep(edge0: number, edge1: number, x: number): number {
const t = clamp01((x - edge0) / (edge1 - edge0));
return t * t * (3 - 2 * t);
}
/** Bottom third ~0, middle third +magnitude, top third -magnitude, smoothly blended
* across each boundary via smoothstep rather than a hard cutoff - a bubble crossing
* 1/3 or 2/3 of the way up changes direction continuously, not with a visible kink.
* heightFraction: 0 = water floor (just spawned), 1 = surface. */
export function currentVelocityAt(heightFraction: number, magnitude: number): number {
const risingEdge = smoothstep(BAND_LOW - BAND_BLEND, BAND_LOW + BAND_BLEND, heightFraction);
const fallingEdge = smoothstep(BAND_HIGH - BAND_BLEND, BAND_HIGH + BAND_BLEND, heightFraction);
return magnitude * (risingEdge - 2 * fallingEdge); // 0 → +magnitude → -magnitude
}
/** Linear interpolation between a `TrackCurves`'s ~1/s samples at a playback
* position; holds flat before the first / past the last sample. A length-1 curve
* (the flat-fallback case) returns that one value everywhere - no special-casing
* needed. */
export function sampleCurveAt(
curve: TrackCurves,
positionSeconds: number,
): { energy: number; valence: number; drive: number } {
const { hop_seconds, energy, valence, drive } = curve;
const n = energy.length;
const fractionalIndex = positionSeconds / hop_seconds;
const i0 = Math.max(0, Math.min(n - 1, Math.floor(fractionalIndex)));
const i1 = Math.min(n - 1, i0 + 1);
const t = i1 === i0 ? 0 : clamp01(fractionalIndex - i0);
return {
energy: lerp(energy[i0]!, energy[i1]!, t),
valence: lerp(valence[i0]!, valence[i1]!, t),
drive: lerp(drive[i0]!, drive[i1]!, t),
};
}
//: The floor `currentMagnitudeAt` damps toward at drive=0 - never fully still, since
//: a quiet passage is still music, not silence.
const DRIVE_MAGNITUDE_FLOOR = 0.4;
/** base (tempo-derived, whole-track) magnitude x a drive multiplier, so a quiet verse
* damps the current toward `DRIVE_MAGNITUDE_FLOOR` (never fully still) and a driving
* chorus can reach - or with `driveCurrentGain` > 1, exceed - the full base magnitude. */
export function currentMagnitudeAt(
baseMagnitude: number,
drive: number,
driveCurrentGain: number,
): number {
const driveMultiplier =
DRIVE_MAGNITUDE_FLOOR + (1 - DRIVE_MAGNITUDE_FLOOR) * clamp01(drive * driveCurrentGain);
return baseMagnitude * driveMultiplier;
}
/** The curve to sample from: the real one once fetched, or a length-1 flat fallback
* built from the whole-track scalars - available synchronously from the library
* payload, so there is no "flash of neutral baseline" while the real curve is still
* in flight. `drive` defaults to `1.0` (not `0.5`): with no curve loaded yet,
* `currentMagnitudeAt` should reproduce today's pre-refactor behaviour - base
* magnitude alone, no drive damping - rather than silently halving it. */
export function effectiveCurve(
analysis: TrackAnalysis | null,
detail: TrackDetail | null,
): TrackCurves {
if (detail?.curve) return detail.curve;
return {
hop_seconds: 1,
energy: [analysis?.energy ?? 0.5],
valence: [analysis?.valence ?? 0.5],
drive: [1.0],
};
}

View File

@@ -0,0 +1,68 @@
/** The 5 parameters `?debugDynamicUI=1` can adjust live, session-only (plain React
* state in App.tsx, no persistence). Every other constant this refactor introduces
* lives in `lib/ambience.ts` / `components/Ambience.tsx` as a plain named constant -
* that pair of files plus this one's `DEFAULT_TUNABLES` is the complete answer to
* "what do I edit to make a debug-slider tweak permanent."
*/
export interface AmbienceTunables {
/** Multiplies energy's swing on chroma/lightness. 1.0 = default. */
energyColorGain: number;
/** Multiplies how much a beat crossing boosts current magnitude, on top of the
* always-present tempo x drive-derived base. 0 = no effect. */
beatCurrentFactor: number;
/** Multiplies the sampled `drive` curve before it modulates current magnitude (see
* `currentMagnitudeAt`). 1.0 = default; 0 = current ignores drive entirely and sits
* at `DRIVE_MAGNITUDE_FLOOR` x base magnitude regardless of the music. */
driveCurrentGain: number;
/** How much bigger a bubble gets by the time it's risen a full screen height, e.g.
* 0.4 = 40% larger at the top. */
bubbleGrowthRate: number;
/** Seconds: exponential smoothing time constant for how quickly the curve-sampled
* energy/valence/drive feeding colour and current track the underlying (already
* linearly-interpolated) curve - a frontend preview knob, distinct from the
* backend's fixed analysis-time sampling rate (`_ANALYSIS_HOP_SECONDS` in
* `librosa_analyzer.py`, which needs a re-analysis to change). 0 = no extra
* smoothing. */
curveSmoothingTau: number;
}
export const DEFAULT_TUNABLES: AmbienceTunables = {
energyColorGain: 1.0,
beatCurrentFactor: 0.6,
driveCurrentGain: 1.0,
bubbleGrowthRate: 0.4,
curveSmoothingTau: 0.4,
};
/** Drives the whole visual pipeline from these values instead of real playback data -
* a "playground mode" for developing a feel for the effect without needing music
* playing. Only takes effect while nothing is actually playing (`!state.playing`) -
* real playback always wins the instant it starts, so there's never a fight between
* the two. Session-only, same as `AmbienceTunables`; distinct from it conceptually:
* tunables are *parameters* (how strongly the system reacts), this is *input*
* (pretending to be the music). */
export interface ManualControl {
enabled: boolean;
energy: number; // 0..1
valence: number; // 0..1
tempo: number; // bpm - also stands in for the current's base magnitude and rise
/** 0..1 - stands in for the sampled `drive` curve, since there's no real track to
* derive rhythmic intensity from. 1.0 default = full current magnitude, matching
* the flat-curve fallback's own default (see `effectiveCurve`). */
drive: number;
/** Strength of each synthetic beat pulse, auto-generated at the `tempo` above while
* manual mode is active - a continuous fake metronome rather than a one-shot
* trigger button, so the beat-driven effects (spawn burst + current kick) can be
* watched continuously rather than poked one click at a time. */
beatStrength: number; // 0..1
}
export const DEFAULT_MANUAL_CONTROL: ManualControl = {
enabled: false,
energy: 0.5,
valence: 0.5,
tempo: 120,
drive: 1.0,
beatStrength: 0.8,
};

View File

@@ -87,6 +87,9 @@ button {
); );
} }
/* Still used by the room-control page's own `<Bubbles>` field (`RoomView.tsx`) - the
main player's bubbles moved to `Ambience.tsx`'s canvas, which needs real spawn and
turbulence control that a fixed set of CSS animations cannot give it. */
.bubble { .bubble {
position: absolute; position: absolute;
bottom: -40px; bottom: -40px;
@@ -97,6 +100,20 @@ button {
pointer-events: none; pointer-events: none;
} }
.ambience {
display: block;
/* A canvas is a replaced element: without an explicit size here, its CSS layout box
falls back to its own width/height *attributes* - the very thing the resize
handler sets from this box's measured size times devicePixelRatio. On any display
where dpr != 1, that is a feedback loop: each resize multiplies the attribute
(and thus, without this rule, the box) by dpr again, diverging within a few
frames until the canvas exceeds the browser's max size and throws. Pinning the
box to 100% of `.stage` breaks the loop - layout no longer depends on the
attribute at all. */
width: 100%;
height: 100%;
}
.pill { .pill {
border: none; border: none;
cursor: pointer; cursor: pointer;