Martin Bauer
2e0e6ad199
The background worker now runs a real librosa analyzer (tempo, beat grid, per-second energy/valence curves) instead of only the null baseline, kept behind build_analyzer() so a plain checkout without the analysis extra still runs fine. The web player reads that per-track analysis and drives a new animated "Ambience" background (bubbles, colour, current) that reacts to the beat and the mood curve as the track plays, plus a debug overlay for tuning it. Also adds a one-off script to backfill podcast cover art from iTunes.
473 lines
18 KiB
TypeScript
473 lines
18 KiB
TypeScript
/** The background: a gradient plus a bubble field, both painted on one canvas.
|
|
*
|
|
* Replaces the old fixed 21-`<div>` field (`bubbleRise` in app.css) with a real
|
|
* particle loop, because "spawn frequency" and "current" need a rate and a
|
|
* magnitude, not a fixed set of `animation-duration`s. The gradient moves to canvas
|
|
* too, for the same reason: CSS cannot interpolate `linear-gradient` colour stops
|
|
* between tracks, a canvas can just ease toward a new target every frame.
|
|
*
|
|
* `energy`/`valence`/`drive` are now sampled from a per-second curve at the current
|
|
* playback position, so - unlike the original version of this component - the
|
|
* mapping from analysis to visuals can no longer run once per track. It runs every
|
|
* frame instead (`ambienceFromSampled` below), fed by curve samples smoothed with an
|
|
* adjustable time constant (`tunables.curveSmoothingTau`).
|
|
*
|
|
* One `requestAnimationFrame` loop, started once and never restarted - everything it
|
|
* reads (`playing`, tunables, manual control, the per-track base, the beat/curve
|
|
* detail, the playback clock) lives in refs, so an album or track change, or a debug
|
|
* slider moving, never costs a re-render of this component, let alone a dropped
|
|
* frame. This mirrors `usePlaybackClock`'s anchor trick (a `{position, at}` pair
|
|
* re-seeded from the server, read against `performance.now()`) but reads it from
|
|
* inside the loop instead of `useState`, which is what keeps 60fps from touching
|
|
* React at all.
|
|
*/
|
|
|
|
import { useEffect, useRef } from "react";
|
|
import type { Album, PlayerState, TrackAnalysis } from "../api/types";
|
|
import { useTrackDetail } from "../hooks/useTrackDetail";
|
|
import {
|
|
ambienceBaseFor,
|
|
ambienceColorAt,
|
|
currentMagnitudeAt,
|
|
currentVelocityAt,
|
|
effectiveCurve,
|
|
oklchString,
|
|
sampleCurveAt,
|
|
spawnRateAt,
|
|
tempoToMagnitude,
|
|
tempoToRise,
|
|
type Ambience as AmbienceTarget,
|
|
type AmbienceBase,
|
|
type Oklch,
|
|
} from "../lib/ambience";
|
|
import type { AmbienceTunables, ManualControl } from "../lib/ambienceTunables";
|
|
import { groupOf, type Group } from "../lib/search";
|
|
|
|
/** Everything `?debugDynamicUI=1` wants to see: the raw analysis, what it maps to,
|
|
* and where the eased, on-screen look currently sits relative to that target. */
|
|
export interface AmbienceDebugSnapshot {
|
|
group: Group | "idle";
|
|
trackTitle: string | null;
|
|
playing: boolean;
|
|
position: number;
|
|
analysis: TrackAnalysis | null;
|
|
target: AmbienceTarget;
|
|
current: AmbienceTarget;
|
|
sampledEnergy: number;
|
|
sampledValence: number;
|
|
sampledDrive: number;
|
|
/** The fully beat-kicked current magnitude actually driving bubbles this frame -
|
|
* distinct from `current.currentMagnitude`, which is pre-kick. */
|
|
currentMagnitude: number;
|
|
kick: number;
|
|
beatsTrusted: boolean;
|
|
beatGridLoaded: boolean;
|
|
beatCount: number;
|
|
beatCursor: number;
|
|
bubbleCount: number;
|
|
/** Whether a real fetched curve is in use, vs. the flat scalar-derived fallback. */
|
|
curveLoaded: boolean;
|
|
/** Whether manual control is *actually* driving the scene right now - distinct from
|
|
* `manual.enabled`, which can stay checked across a play/pause cycle. */
|
|
usingManual: boolean;
|
|
}
|
|
|
|
interface Props {
|
|
album: Album | null;
|
|
state: PlayerState;
|
|
tunables: AmbienceTunables;
|
|
manual: ManualControl;
|
|
/** Wired up only behind `?debugDynamicUI=1` - see `AmbienceDebugOverlay`. Throttled
|
|
* internally, so passing this costs nothing close to every frame's React render. */
|
|
onDebugFrame?: (snapshot: AmbienceDebugSnapshot) => void;
|
|
}
|
|
|
|
//: The debug overlay doesn't need 60fps, and re-rendering React that often just to
|
|
//: feed it would defeat the whole point of keeping this loop ref-only.
|
|
const DEBUG_EMIT_INTERVAL_MS = 200;
|
|
|
|
//: Below this, `pulse` says the detected tempo is not a trustworthy, audible beat -
|
|
//: moved here from the fetch hook, since it now gates *use* (does this beat crossing
|
|
//: drive the kick), not *fetch* (the curve is wanted regardless of beat quality).
|
|
const PULSE_THRESHOLD = 0.3;
|
|
|
|
interface Bubble {
|
|
/** Horizontal position in CSS pixels - stateful, unlike the old sine-wander's fixed
|
|
* anchor: a persistent one-directional current has to be integrated over time. */
|
|
x: number;
|
|
/** Pixels risen since spawn. */
|
|
risen: number;
|
|
/** Size at spawn. */
|
|
baseSize: number;
|
|
/** Current painted size - grows from `baseSize` as the bubble rises. */
|
|
size: number;
|
|
baseAlpha: number;
|
|
}
|
|
|
|
//: How quickly the painted look eases toward the per-frame target - an exponential
|
|
//: time constant, so this is "how many seconds to close most of the gap", not a hard
|
|
//: crossfade duration. ~0.5s reaches ~95% of the way there in about 1.5s. Distinct
|
|
//: from `tunables.curveSmoothingTau`, which smooths the sampled signal *feeding*
|
|
//: the target, not the ease from `current` toward it - the two compound in series.
|
|
const CROSSFADE_TAU_SECONDS = 0.5;
|
|
//: How quickly a beat's kick fades back out.
|
|
const KICK_DECAY_TAU_SECONDS = 0.12;
|
|
//: Extra bubbles/second at the moment of a full-strength (1.0) beat - kept exactly as
|
|
//: before this refactor; only the current-magnitude boost is new.
|
|
const BEAT_BURST_RATE = 6;
|
|
//: Today's rise speed, roughly: the old CSS took a bubble ~10s to cross 120vh.
|
|
const BASE_RISE_FRACTION_PER_SECOND = 0.12;
|
|
const BUBBLE_MIN_SIZE = 8;
|
|
const BUBBLE_MAX_SIZE = 24;
|
|
//: The canvas frame clamps to this, so a tab returning from the background after
|
|
//: minutes away doesn't dump a giant `dt` into the physics in one jump.
|
|
const MAX_FRAME_SECONDS = 0.05;
|
|
|
|
function clamp01(value: number): number {
|
|
return Math.max(0, Math.min(1, value));
|
|
}
|
|
|
|
function lerp(a: number, b: number, t: number): number {
|
|
return a + (b - a) * t;
|
|
}
|
|
|
|
function lerpColor(a: Oklch, b: Oklch, t: number): Oklch {
|
|
return { l: lerp(a.l, b.l, t), c: lerp(a.c, b.c, t), h: lerp(a.h, b.h, t) };
|
|
}
|
|
|
|
function lerpAmbience(a: AmbienceTarget, b: AmbienceTarget, t: number): AmbienceTarget {
|
|
return {
|
|
gradient: [
|
|
lerpColor(a.gradient[0], b.gradient[0], t),
|
|
lerpColor(a.gradient[1], b.gradient[1], t),
|
|
lerpColor(a.gradient[2], b.gradient[2], t),
|
|
],
|
|
tint: lerpColor(a.tint, b.tint, t),
|
|
spawnRate: lerp(a.spawnRate, b.spawnRate, t),
|
|
rise: lerp(a.rise, b.rise, t),
|
|
currentMagnitude: lerp(a.currentMagnitude, b.currentMagnitude, t),
|
|
};
|
|
}
|
|
|
|
/** First index whose beat time is `>= position` - a binary search, since a track's
|
|
* grid can run to several thousand entries and this runs every frame. */
|
|
function beatCursorFor(times: number[], position: number): number {
|
|
let lo = 0;
|
|
let hi = times.length;
|
|
while (lo < hi) {
|
|
const mid = (lo + hi) >>> 1;
|
|
if (times[mid]! < position) lo = mid + 1;
|
|
else hi = mid;
|
|
}
|
|
return lo;
|
|
}
|
|
|
|
function spawnBubble(width: number): Bubble {
|
|
const baseSize = BUBBLE_MIN_SIZE + Math.random() * (BUBBLE_MAX_SIZE - BUBBLE_MIN_SIZE);
|
|
return {
|
|
x: Math.random() * width,
|
|
risen: 0,
|
|
baseSize,
|
|
size: baseSize,
|
|
baseAlpha: 0.25 + Math.random() * 0.2,
|
|
};
|
|
}
|
|
|
|
/** Turns already-smoothed energy/valence/drive into a full `Ambience` - shared
|
|
* between the mount-time seed (unsmoothed, snapped straight to the first sample) and
|
|
* every subsequent frame (smoothed with `tunables.curveSmoothingTau`), so the two
|
|
* paths can never disagree on what a given (energy, valence, drive) triple means. */
|
|
function ambienceFromSampled(
|
|
base: AmbienceBase,
|
|
usingManual: boolean,
|
|
manual: ManualControl,
|
|
tunables: AmbienceTunables,
|
|
energy: number,
|
|
valence: number,
|
|
drive: number,
|
|
): AmbienceTarget {
|
|
if (!usingManual && !base.music) {
|
|
return { ...base.fixed!, rise: base.rise, currentMagnitude: 0 };
|
|
}
|
|
const rise = usingManual ? tempoToRise(manual.tempo) : base.rise;
|
|
const baseMagnitude = usingManual ? tempoToMagnitude(manual.tempo) : base.baseCurrentMagnitude;
|
|
const { gradient, tint } = ambienceColorAt(energy, valence, tunables);
|
|
const magnitude = currentMagnitudeAt(baseMagnitude, drive, tunables.driveCurrentGain);
|
|
return { gradient, tint, spawnRate: spawnRateAt(energy), rise, currentMagnitude: magnitude };
|
|
}
|
|
|
|
export function Ambience({ album, state, tunables, manual, onDebugFrame }: Props) {
|
|
const canvasRef = useRef<HTMLCanvasElement | null>(null);
|
|
const detail = useTrackDetail(album, state);
|
|
|
|
const playingRef = useRef(state.playing);
|
|
playingRef.current = state.playing;
|
|
|
|
const detailRef = useRef(detail);
|
|
detailRef.current = detail;
|
|
|
|
const tunablesRef = useRef(tunables);
|
|
tunablesRef.current = tunables;
|
|
|
|
const manualRef = useRef(manual);
|
|
manualRef.current = manual;
|
|
|
|
const onDebugFrameRef = useRef(onDebugFrame);
|
|
onDebugFrameRef.current = onDebugFrame;
|
|
|
|
const groupRef = useRef<Group | "idle">(album ? groupOf(album) : "idle");
|
|
groupRef.current = album ? groupOf(album) : "idle";
|
|
|
|
const trackTitleRef = useRef(state.track_title);
|
|
trackTitleRef.current = state.track_title;
|
|
|
|
// The same anchor trick as `usePlaybackClock`, but read from inside the rAF loop
|
|
// rather than through `useState` - see the module docstring.
|
|
const clockAnchor = useRef({ position: state.position, at: performance.now() });
|
|
useEffect(() => {
|
|
clockAnchor.current = { position: state.position, at: performance.now() };
|
|
}, [state.position]);
|
|
|
|
const trackAnalysis = album?.tracks[state.track_index]?.analysis ?? null;
|
|
const analysisRef = useRef(trackAnalysis);
|
|
analysisRef.current = trackAnalysis;
|
|
const baseRef = useRef(ambienceBaseFor(album, trackAnalysis));
|
|
useEffect(() => {
|
|
baseRef.current = ambienceBaseFor(album, trackAnalysis);
|
|
// eslint-disable-next-line react-hooks/exhaustive-deps
|
|
}, [album, trackAnalysis]);
|
|
|
|
useEffect(() => {
|
|
const canvas = canvasRef.current;
|
|
const ctx = canvas?.getContext("2d");
|
|
if (!canvas || !ctx) return;
|
|
|
|
const size = { width: 0, height: 0 };
|
|
// The CSS `.ambience` rule pins this canvas to 100% of `.stage`, independent of
|
|
// its own width/height attributes - required so this handler's own writes below
|
|
// can't feed back into the size it just measured (see the comment there). The
|
|
// clamp is a second line of defence against the same failure mode from anywhere
|
|
// else: an unclamped size can hit the browser's canvas allocation limit and throw.
|
|
const MAX_BACKING_STORE_PX = 4096;
|
|
const resize = () => {
|
|
const dpr = window.devicePixelRatio || 1;
|
|
const rect = canvas.getBoundingClientRect();
|
|
size.width = rect.width;
|
|
size.height = rect.height;
|
|
canvas.width = Math.min(MAX_BACKING_STORE_PX, Math.max(1, Math.round(rect.width * dpr)));
|
|
canvas.height = Math.min(MAX_BACKING_STORE_PX, Math.max(1, Math.round(rect.height * dpr)));
|
|
// Scale factors from the (possibly clamped) backing store, not raw `dpr`, so
|
|
// drawing in CSS-pixel units still lands correctly even when clamped.
|
|
const scaleX = canvas.width / Math.max(1, rect.width);
|
|
const scaleY = canvas.height / Math.max(1, rect.height);
|
|
ctx.setTransform(scaleX, 0, 0, scaleY, 0, 0);
|
|
};
|
|
resize();
|
|
const observer = new ResizeObserver(resize);
|
|
observer.observe(canvas);
|
|
|
|
const media = matchMedia("(prefers-reduced-motion: reduce)");
|
|
let reducedMotion = media.matches;
|
|
const onMotionChange = () => {
|
|
reducedMotion = media.matches;
|
|
};
|
|
media.addEventListener("change", onMotionChange);
|
|
|
|
let bubbles: Bubble[] = [];
|
|
let spawnAccumulator = 0;
|
|
let beatCursor = 0;
|
|
let kick = 0;
|
|
let manualBeatTimer = 0;
|
|
let lastFrame = performance.now();
|
|
let lastDebugEmit = 0;
|
|
let raf = 0;
|
|
|
|
// Seeded from the first real sample, unsmoothed, so the first paint doesn't flash
|
|
// from a hardcoded baseline toward wherever the track's mood actually starts.
|
|
let smoothedEnergy = 0.5;
|
|
let smoothedValence = 0.5;
|
|
let smoothedDrive = 1.0;
|
|
{
|
|
const base0 = baseRef.current;
|
|
const manual0 = manualRef.current;
|
|
const usingManual0 = manual0.enabled && !playingRef.current;
|
|
if (usingManual0) {
|
|
smoothedEnergy = manual0.energy;
|
|
smoothedValence = manual0.valence;
|
|
smoothedDrive = manual0.drive;
|
|
} else if (base0.music) {
|
|
const curve0 = effectiveCurve(analysisRef.current, detailRef.current);
|
|
const sampled0 = sampleCurveAt(curve0, clockAnchor.current.position);
|
|
smoothedEnergy = sampled0.energy;
|
|
smoothedValence = sampled0.valence;
|
|
smoothedDrive = sampled0.drive;
|
|
}
|
|
}
|
|
let current: AmbienceTarget = ambienceFromSampled(
|
|
baseRef.current,
|
|
manualRef.current.enabled && !playingRef.current,
|
|
manualRef.current,
|
|
tunablesRef.current,
|
|
smoothedEnergy,
|
|
smoothedValence,
|
|
smoothedDrive,
|
|
);
|
|
|
|
const tick = (now: number) => {
|
|
const dt = Math.min(MAX_FRAME_SECONDS, (now - lastFrame) / 1000);
|
|
lastFrame = now;
|
|
|
|
const base = baseRef.current;
|
|
const tunables = tunablesRef.current;
|
|
const manual = manualRef.current;
|
|
const playing = playingRef.current;
|
|
// Real playback always wins the instant it starts - manual mode only ever has
|
|
// an effect while genuinely nothing is playing, so the two inputs can never
|
|
// fight over the same frame.
|
|
const usingManual = manual.enabled && !playing;
|
|
const effectivePlaying = playing || usingManual; // manual mode still animates
|
|
|
|
const anchor = clockAnchor.current;
|
|
const position = playing ? anchor.position + (now - anchor.at) / 1000 : anchor.position;
|
|
|
|
const sampled = usingManual
|
|
? { energy: manual.energy, valence: manual.valence, drive: manual.drive }
|
|
: sampleCurveAt(effectiveCurve(analysisRef.current, detailRef.current), position);
|
|
const smoothing =
|
|
tunables.curveSmoothingTau <= 0 ? 1 : 1 - Math.exp(-dt / tunables.curveSmoothingTau);
|
|
smoothedEnergy = lerp(smoothedEnergy, sampled.energy, smoothing);
|
|
smoothedValence = lerp(smoothedValence, sampled.valence, smoothing);
|
|
smoothedDrive = lerp(smoothedDrive, sampled.drive, smoothing);
|
|
|
|
const target = ambienceFromSampled(
|
|
base,
|
|
usingManual,
|
|
manual,
|
|
tunables,
|
|
smoothedEnergy,
|
|
smoothedValence,
|
|
smoothedDrive,
|
|
);
|
|
current = lerpAmbience(current, target, 1 - Math.exp(-dt / CROSSFADE_TAU_SECONDS));
|
|
|
|
const analysis = analysisRef.current;
|
|
const beatsTrusted =
|
|
analysis !== null && analysis.beats && (analysis.pulse ?? 0) >= PULSE_THRESHOLD;
|
|
const grid = beatsTrusted && !usingManual ? detailRef.current : null;
|
|
|
|
if (usingManual) {
|
|
// A continuous fake metronome at the manual tempo, instead of walking a real
|
|
// grid - reuses the same `kick` variable and decay as real beats, so every
|
|
// downstream effect (spawn burst, current boost) is identical either way.
|
|
manualBeatTimer -= dt;
|
|
if (manualBeatTimer <= 0) {
|
|
kick = Math.max(kick, manual.beatStrength);
|
|
manualBeatTimer = 60 / Math.max(1, manual.tempo);
|
|
}
|
|
} else if (grid && grid.times.length > 0) {
|
|
// Playback normally only moves forward, so the common case is just advancing
|
|
// the cursor with the `while` below. A backward seek is the one thing that
|
|
// can break that invariant - detected by the previously consumed beat now
|
|
// being back in the future - and needs a real re-search.
|
|
if (beatCursor > 0 && grid.times[beatCursor - 1]! > position) {
|
|
beatCursor = beatCursorFor(grid.times, position);
|
|
}
|
|
while (beatCursor < grid.times.length && grid.times[beatCursor]! <= position) {
|
|
kick = Math.max(kick, grid.strengths[beatCursor] ?? 0);
|
|
beatCursor++;
|
|
}
|
|
} else {
|
|
beatCursor = 0;
|
|
}
|
|
kick *= Math.exp(-dt / KICK_DECAY_TAU_SECONDS);
|
|
|
|
const kickedMagnitude = current.currentMagnitude * (1 + kick * tunables.beatCurrentFactor);
|
|
|
|
if (effectivePlaying) {
|
|
spawnAccumulator += (current.spawnRate + kick * BEAT_BURST_RATE) * dt;
|
|
while (spawnAccumulator >= 1) {
|
|
spawnAccumulator -= 1;
|
|
bubbles.push(spawnBubble(size.width));
|
|
}
|
|
}
|
|
// Not playing (and not in manual mode): no new spawns, but bubbles already in
|
|
// flight keep rising and draining away rather than freezing mid-air.
|
|
|
|
const riseSpeed = current.rise * size.height * BASE_RISE_FRACTION_PER_SECOND;
|
|
bubbles = bubbles.filter((b) => {
|
|
const heightFraction = clamp01(b.risen / size.height);
|
|
const velocity = currentVelocityAt(heightFraction, kickedMagnitude);
|
|
b.x = (((b.x + velocity * dt) % size.width) + size.width) % size.width;
|
|
b.risen += riseSpeed * dt;
|
|
b.size = b.baseSize * (1 + tunables.bubbleGrowthRate * heightFraction);
|
|
return b.risen < size.height + BUBBLE_MAX_SIZE * 2;
|
|
});
|
|
|
|
ctx.clearRect(0, 0, size.width, size.height);
|
|
const [top, mid, deep] = current.gradient;
|
|
const gradient = ctx.createLinearGradient(0, 0, 0, size.height);
|
|
gradient.addColorStop(0, oklchString(top));
|
|
gradient.addColorStop(0.45, oklchString(mid));
|
|
gradient.addColorStop(1, oklchString(deep));
|
|
ctx.fillStyle = gradient;
|
|
ctx.fillRect(0, 0, size.width, size.height);
|
|
|
|
if (!reducedMotion) {
|
|
for (const b of bubbles) {
|
|
const y = size.height + BUBBLE_MAX_SIZE - b.risen;
|
|
const topThird = size.height / 3;
|
|
const fade = y < topThird ? Math.max(0, y / topThird) : 1;
|
|
ctx.beginPath();
|
|
ctx.fillStyle = oklchString(current.tint, b.baseAlpha * fade);
|
|
ctx.arc(b.x, y, b.size / 2, 0, Math.PI * 2);
|
|
ctx.fill();
|
|
}
|
|
}
|
|
|
|
const debugFrame = onDebugFrameRef.current;
|
|
if (debugFrame && now - lastDebugEmit >= DEBUG_EMIT_INTERVAL_MS) {
|
|
lastDebugEmit = now;
|
|
debugFrame({
|
|
group: groupRef.current,
|
|
trackTitle: trackTitleRef.current,
|
|
playing,
|
|
position,
|
|
analysis,
|
|
target,
|
|
current,
|
|
sampledEnergy: smoothedEnergy,
|
|
sampledValence: smoothedValence,
|
|
sampledDrive: smoothedDrive,
|
|
currentMagnitude: kickedMagnitude,
|
|
kick,
|
|
beatsTrusted,
|
|
beatGridLoaded: grid !== null,
|
|
beatCount: grid?.times.length ?? 0,
|
|
beatCursor,
|
|
bubbleCount: bubbles.length,
|
|
curveLoaded: detailRef.current?.curve != null,
|
|
usingManual,
|
|
});
|
|
}
|
|
|
|
raf = requestAnimationFrame(tick);
|
|
};
|
|
raf = requestAnimationFrame(tick);
|
|
|
|
return () => {
|
|
cancelAnimationFrame(raf);
|
|
observer.disconnect();
|
|
media.removeEventListener("change", onMotionChange);
|
|
};
|
|
}, []);
|
|
|
|
return (
|
|
<canvas
|
|
ref={canvasRef}
|
|
className="ambience"
|
|
aria-hidden="true"
|
|
style={{ position: "absolute", inset: 0, zIndex: 0, pointerEvents: "none" }}
|
|
/>
|
|
);
|
|
}
|