Add librosa beat/mood analysis and an Ambience background driven by it
The background worker now runs a real librosa analyzer (tempo, beat grid, per-second energy/valence curves) instead of only the null baseline, kept behind build_analyzer() so a plain checkout without the analysis extra still runs fine. The web player reads that per-track analysis and drives a new animated "Ambience" background (bubbles, colour, current) that reacts to the beat and the mood curve as the track plays, plus a debug overlay for tuning it. Also adds a one-off script to backfill podcast cover art from iTunes.
This commit is contained in:
472
web/src/components/Ambience.tsx
Normal file
472
web/src/components/Ambience.tsx
Normal file
@@ -0,0 +1,472 @@
|
||||
/** The background: a gradient plus a bubble field, both painted on one canvas.
|
||||
*
|
||||
* Replaces the old fixed 21-`<div>` field (`bubbleRise` in app.css) with a real
|
||||
* particle loop, because "spawn frequency" and "current" need a rate and a
|
||||
* magnitude, not a fixed set of `animation-duration`s. The gradient moves to canvas
|
||||
* too, for the same reason: CSS cannot interpolate `linear-gradient` colour stops
|
||||
* between tracks, a canvas can just ease toward a new target every frame.
|
||||
*
|
||||
* `energy`/`valence`/`drive` are now sampled from a per-second curve at the current
|
||||
* playback position, so - unlike the original version of this component - the
|
||||
* mapping from analysis to visuals can no longer run once per track. It runs every
|
||||
* frame instead (`ambienceFromSampled` below), fed by curve samples smoothed with an
|
||||
* adjustable time constant (`tunables.curveSmoothingTau`).
|
||||
*
|
||||
* One `requestAnimationFrame` loop, started once and never restarted - everything it
|
||||
* reads (`playing`, tunables, manual control, the per-track base, the beat/curve
|
||||
* detail, the playback clock) lives in refs, so an album or track change, or a debug
|
||||
* slider moving, never costs a re-render of this component, let alone a dropped
|
||||
* frame. This mirrors `usePlaybackClock`'s anchor trick (a `{position, at}` pair
|
||||
* re-seeded from the server, read against `performance.now()`) but reads it from
|
||||
* inside the loop instead of `useState`, which is what keeps 60fps from touching
|
||||
* React at all.
|
||||
*/
|
||||
|
||||
import { useEffect, useRef } from "react";
|
||||
import type { Album, PlayerState, TrackAnalysis } from "../api/types";
|
||||
import { useTrackDetail } from "../hooks/useTrackDetail";
|
||||
import {
|
||||
ambienceBaseFor,
|
||||
ambienceColorAt,
|
||||
currentMagnitudeAt,
|
||||
currentVelocityAt,
|
||||
effectiveCurve,
|
||||
oklchString,
|
||||
sampleCurveAt,
|
||||
spawnRateAt,
|
||||
tempoToMagnitude,
|
||||
tempoToRise,
|
||||
type Ambience as AmbienceTarget,
|
||||
type AmbienceBase,
|
||||
type Oklch,
|
||||
} from "../lib/ambience";
|
||||
import type { AmbienceTunables, ManualControl } from "../lib/ambienceTunables";
|
||||
import { groupOf, type Group } from "../lib/search";
|
||||
|
||||
/** Everything `?debugDynamicUI=1` wants to see: the raw analysis, what it maps to,
|
||||
* and where the eased, on-screen look currently sits relative to that target. */
|
||||
export interface AmbienceDebugSnapshot {
|
||||
group: Group | "idle";
|
||||
trackTitle: string | null;
|
||||
playing: boolean;
|
||||
position: number;
|
||||
analysis: TrackAnalysis | null;
|
||||
target: AmbienceTarget;
|
||||
current: AmbienceTarget;
|
||||
sampledEnergy: number;
|
||||
sampledValence: number;
|
||||
sampledDrive: number;
|
||||
/** The fully beat-kicked current magnitude actually driving bubbles this frame -
|
||||
* distinct from `current.currentMagnitude`, which is pre-kick. */
|
||||
currentMagnitude: number;
|
||||
kick: number;
|
||||
beatsTrusted: boolean;
|
||||
beatGridLoaded: boolean;
|
||||
beatCount: number;
|
||||
beatCursor: number;
|
||||
bubbleCount: number;
|
||||
/** Whether a real fetched curve is in use, vs. the flat scalar-derived fallback. */
|
||||
curveLoaded: boolean;
|
||||
/** Whether manual control is *actually* driving the scene right now - distinct from
|
||||
* `manual.enabled`, which can stay checked across a play/pause cycle. */
|
||||
usingManual: boolean;
|
||||
}
|
||||
|
||||
interface Props {
|
||||
album: Album | null;
|
||||
state: PlayerState;
|
||||
tunables: AmbienceTunables;
|
||||
manual: ManualControl;
|
||||
/** Wired up only behind `?debugDynamicUI=1` - see `AmbienceDebugOverlay`. Throttled
|
||||
* internally, so passing this costs nothing close to every frame's React render. */
|
||||
onDebugFrame?: (snapshot: AmbienceDebugSnapshot) => void;
|
||||
}
|
||||
|
||||
//: The debug overlay doesn't need 60fps, and re-rendering React that often just to
|
||||
//: feed it would defeat the whole point of keeping this loop ref-only.
|
||||
const DEBUG_EMIT_INTERVAL_MS = 200;
|
||||
|
||||
//: Below this, `pulse` says the detected tempo is not a trustworthy, audible beat -
|
||||
//: moved here from the fetch hook, since it now gates *use* (does this beat crossing
|
||||
//: drive the kick), not *fetch* (the curve is wanted regardless of beat quality).
|
||||
const PULSE_THRESHOLD = 0.3;
|
||||
|
||||
interface Bubble {
|
||||
/** Horizontal position in CSS pixels - stateful, unlike the old sine-wander's fixed
|
||||
* anchor: a persistent one-directional current has to be integrated over time. */
|
||||
x: number;
|
||||
/** Pixels risen since spawn. */
|
||||
risen: number;
|
||||
/** Size at spawn. */
|
||||
baseSize: number;
|
||||
/** Current painted size - grows from `baseSize` as the bubble rises. */
|
||||
size: number;
|
||||
baseAlpha: number;
|
||||
}
|
||||
|
||||
//: How quickly the painted look eases toward the per-frame target - an exponential
|
||||
//: time constant, so this is "how many seconds to close most of the gap", not a hard
|
||||
//: crossfade duration. ~0.5s reaches ~95% of the way there in about 1.5s. Distinct
|
||||
//: from `tunables.curveSmoothingTau`, which smooths the sampled signal *feeding*
|
||||
//: the target, not the ease from `current` toward it - the two compound in series.
|
||||
const CROSSFADE_TAU_SECONDS = 0.5;
|
||||
//: How quickly a beat's kick fades back out.
|
||||
const KICK_DECAY_TAU_SECONDS = 0.12;
|
||||
//: Extra bubbles/second at the moment of a full-strength (1.0) beat - kept exactly as
|
||||
//: before this refactor; only the current-magnitude boost is new.
|
||||
const BEAT_BURST_RATE = 6;
|
||||
//: Today's rise speed, roughly: the old CSS took a bubble ~10s to cross 120vh.
|
||||
const BASE_RISE_FRACTION_PER_SECOND = 0.12;
|
||||
const BUBBLE_MIN_SIZE = 8;
|
||||
const BUBBLE_MAX_SIZE = 24;
|
||||
//: The canvas frame clamps to this, so a tab returning from the background after
|
||||
//: minutes away doesn't dump a giant `dt` into the physics in one jump.
|
||||
const MAX_FRAME_SECONDS = 0.05;
|
||||
|
||||
function clamp01(value: number): number {
|
||||
return Math.max(0, Math.min(1, value));
|
||||
}
|
||||
|
||||
function lerp(a: number, b: number, t: number): number {
|
||||
return a + (b - a) * t;
|
||||
}
|
||||
|
||||
function lerpColor(a: Oklch, b: Oklch, t: number): Oklch {
|
||||
return { l: lerp(a.l, b.l, t), c: lerp(a.c, b.c, t), h: lerp(a.h, b.h, t) };
|
||||
}
|
||||
|
||||
function lerpAmbience(a: AmbienceTarget, b: AmbienceTarget, t: number): AmbienceTarget {
|
||||
return {
|
||||
gradient: [
|
||||
lerpColor(a.gradient[0], b.gradient[0], t),
|
||||
lerpColor(a.gradient[1], b.gradient[1], t),
|
||||
lerpColor(a.gradient[2], b.gradient[2], t),
|
||||
],
|
||||
tint: lerpColor(a.tint, b.tint, t),
|
||||
spawnRate: lerp(a.spawnRate, b.spawnRate, t),
|
||||
rise: lerp(a.rise, b.rise, t),
|
||||
currentMagnitude: lerp(a.currentMagnitude, b.currentMagnitude, t),
|
||||
};
|
||||
}
|
||||
|
||||
/** First index whose beat time is `>= position` - a binary search, since a track's
|
||||
* grid can run to several thousand entries and this runs every frame. */
|
||||
function beatCursorFor(times: number[], position: number): number {
|
||||
let lo = 0;
|
||||
let hi = times.length;
|
||||
while (lo < hi) {
|
||||
const mid = (lo + hi) >>> 1;
|
||||
if (times[mid]! < position) lo = mid + 1;
|
||||
else hi = mid;
|
||||
}
|
||||
return lo;
|
||||
}
|
||||
|
||||
function spawnBubble(width: number): Bubble {
|
||||
const baseSize = BUBBLE_MIN_SIZE + Math.random() * (BUBBLE_MAX_SIZE - BUBBLE_MIN_SIZE);
|
||||
return {
|
||||
x: Math.random() * width,
|
||||
risen: 0,
|
||||
baseSize,
|
||||
size: baseSize,
|
||||
baseAlpha: 0.25 + Math.random() * 0.2,
|
||||
};
|
||||
}
|
||||
|
||||
/** Turns already-smoothed energy/valence/drive into a full `Ambience` - shared
|
||||
* between the mount-time seed (unsmoothed, snapped straight to the first sample) and
|
||||
* every subsequent frame (smoothed with `tunables.curveSmoothingTau`), so the two
|
||||
* paths can never disagree on what a given (energy, valence, drive) triple means. */
|
||||
function ambienceFromSampled(
|
||||
base: AmbienceBase,
|
||||
usingManual: boolean,
|
||||
manual: ManualControl,
|
||||
tunables: AmbienceTunables,
|
||||
energy: number,
|
||||
valence: number,
|
||||
drive: number,
|
||||
): AmbienceTarget {
|
||||
if (!usingManual && !base.music) {
|
||||
return { ...base.fixed!, rise: base.rise, currentMagnitude: 0 };
|
||||
}
|
||||
const rise = usingManual ? tempoToRise(manual.tempo) : base.rise;
|
||||
const baseMagnitude = usingManual ? tempoToMagnitude(manual.tempo) : base.baseCurrentMagnitude;
|
||||
const { gradient, tint } = ambienceColorAt(energy, valence, tunables);
|
||||
const magnitude = currentMagnitudeAt(baseMagnitude, drive, tunables.driveCurrentGain);
|
||||
return { gradient, tint, spawnRate: spawnRateAt(energy), rise, currentMagnitude: magnitude };
|
||||
}
|
||||
|
||||
export function Ambience({ album, state, tunables, manual, onDebugFrame }: Props) {
|
||||
const canvasRef = useRef<HTMLCanvasElement | null>(null);
|
||||
const detail = useTrackDetail(album, state);
|
||||
|
||||
const playingRef = useRef(state.playing);
|
||||
playingRef.current = state.playing;
|
||||
|
||||
const detailRef = useRef(detail);
|
||||
detailRef.current = detail;
|
||||
|
||||
const tunablesRef = useRef(tunables);
|
||||
tunablesRef.current = tunables;
|
||||
|
||||
const manualRef = useRef(manual);
|
||||
manualRef.current = manual;
|
||||
|
||||
const onDebugFrameRef = useRef(onDebugFrame);
|
||||
onDebugFrameRef.current = onDebugFrame;
|
||||
|
||||
const groupRef = useRef<Group | "idle">(album ? groupOf(album) : "idle");
|
||||
groupRef.current = album ? groupOf(album) : "idle";
|
||||
|
||||
const trackTitleRef = useRef(state.track_title);
|
||||
trackTitleRef.current = state.track_title;
|
||||
|
||||
// The same anchor trick as `usePlaybackClock`, but read from inside the rAF loop
|
||||
// rather than through `useState` - see the module docstring.
|
||||
const clockAnchor = useRef({ position: state.position, at: performance.now() });
|
||||
useEffect(() => {
|
||||
clockAnchor.current = { position: state.position, at: performance.now() };
|
||||
}, [state.position]);
|
||||
|
||||
const trackAnalysis = album?.tracks[state.track_index]?.analysis ?? null;
|
||||
const analysisRef = useRef(trackAnalysis);
|
||||
analysisRef.current = trackAnalysis;
|
||||
const baseRef = useRef(ambienceBaseFor(album, trackAnalysis));
|
||||
useEffect(() => {
|
||||
baseRef.current = ambienceBaseFor(album, trackAnalysis);
|
||||
// eslint-disable-next-line react-hooks/exhaustive-deps
|
||||
}, [album, trackAnalysis]);
|
||||
|
||||
useEffect(() => {
|
||||
const canvas = canvasRef.current;
|
||||
const ctx = canvas?.getContext("2d");
|
||||
if (!canvas || !ctx) return;
|
||||
|
||||
const size = { width: 0, height: 0 };
|
||||
// The CSS `.ambience` rule pins this canvas to 100% of `.stage`, independent of
|
||||
// its own width/height attributes - required so this handler's own writes below
|
||||
// can't feed back into the size it just measured (see the comment there). The
|
||||
// clamp is a second line of defence against the same failure mode from anywhere
|
||||
// else: an unclamped size can hit the browser's canvas allocation limit and throw.
|
||||
const MAX_BACKING_STORE_PX = 4096;
|
||||
const resize = () => {
|
||||
const dpr = window.devicePixelRatio || 1;
|
||||
const rect = canvas.getBoundingClientRect();
|
||||
size.width = rect.width;
|
||||
size.height = rect.height;
|
||||
canvas.width = Math.min(MAX_BACKING_STORE_PX, Math.max(1, Math.round(rect.width * dpr)));
|
||||
canvas.height = Math.min(MAX_BACKING_STORE_PX, Math.max(1, Math.round(rect.height * dpr)));
|
||||
// Scale factors from the (possibly clamped) backing store, not raw `dpr`, so
|
||||
// drawing in CSS-pixel units still lands correctly even when clamped.
|
||||
const scaleX = canvas.width / Math.max(1, rect.width);
|
||||
const scaleY = canvas.height / Math.max(1, rect.height);
|
||||
ctx.setTransform(scaleX, 0, 0, scaleY, 0, 0);
|
||||
};
|
||||
resize();
|
||||
const observer = new ResizeObserver(resize);
|
||||
observer.observe(canvas);
|
||||
|
||||
const media = matchMedia("(prefers-reduced-motion: reduce)");
|
||||
let reducedMotion = media.matches;
|
||||
const onMotionChange = () => {
|
||||
reducedMotion = media.matches;
|
||||
};
|
||||
media.addEventListener("change", onMotionChange);
|
||||
|
||||
let bubbles: Bubble[] = [];
|
||||
let spawnAccumulator = 0;
|
||||
let beatCursor = 0;
|
||||
let kick = 0;
|
||||
let manualBeatTimer = 0;
|
||||
let lastFrame = performance.now();
|
||||
let lastDebugEmit = 0;
|
||||
let raf = 0;
|
||||
|
||||
// Seeded from the first real sample, unsmoothed, so the first paint doesn't flash
|
||||
// from a hardcoded baseline toward wherever the track's mood actually starts.
|
||||
let smoothedEnergy = 0.5;
|
||||
let smoothedValence = 0.5;
|
||||
let smoothedDrive = 1.0;
|
||||
{
|
||||
const base0 = baseRef.current;
|
||||
const manual0 = manualRef.current;
|
||||
const usingManual0 = manual0.enabled && !playingRef.current;
|
||||
if (usingManual0) {
|
||||
smoothedEnergy = manual0.energy;
|
||||
smoothedValence = manual0.valence;
|
||||
smoothedDrive = manual0.drive;
|
||||
} else if (base0.music) {
|
||||
const curve0 = effectiveCurve(analysisRef.current, detailRef.current);
|
||||
const sampled0 = sampleCurveAt(curve0, clockAnchor.current.position);
|
||||
smoothedEnergy = sampled0.energy;
|
||||
smoothedValence = sampled0.valence;
|
||||
smoothedDrive = sampled0.drive;
|
||||
}
|
||||
}
|
||||
let current: AmbienceTarget = ambienceFromSampled(
|
||||
baseRef.current,
|
||||
manualRef.current.enabled && !playingRef.current,
|
||||
manualRef.current,
|
||||
tunablesRef.current,
|
||||
smoothedEnergy,
|
||||
smoothedValence,
|
||||
smoothedDrive,
|
||||
);
|
||||
|
||||
const tick = (now: number) => {
|
||||
const dt = Math.min(MAX_FRAME_SECONDS, (now - lastFrame) / 1000);
|
||||
lastFrame = now;
|
||||
|
||||
const base = baseRef.current;
|
||||
const tunables = tunablesRef.current;
|
||||
const manual = manualRef.current;
|
||||
const playing = playingRef.current;
|
||||
// Real playback always wins the instant it starts - manual mode only ever has
|
||||
// an effect while genuinely nothing is playing, so the two inputs can never
|
||||
// fight over the same frame.
|
||||
const usingManual = manual.enabled && !playing;
|
||||
const effectivePlaying = playing || usingManual; // manual mode still animates
|
||||
|
||||
const anchor = clockAnchor.current;
|
||||
const position = playing ? anchor.position + (now - anchor.at) / 1000 : anchor.position;
|
||||
|
||||
const sampled = usingManual
|
||||
? { energy: manual.energy, valence: manual.valence, drive: manual.drive }
|
||||
: sampleCurveAt(effectiveCurve(analysisRef.current, detailRef.current), position);
|
||||
const smoothing =
|
||||
tunables.curveSmoothingTau <= 0 ? 1 : 1 - Math.exp(-dt / tunables.curveSmoothingTau);
|
||||
smoothedEnergy = lerp(smoothedEnergy, sampled.energy, smoothing);
|
||||
smoothedValence = lerp(smoothedValence, sampled.valence, smoothing);
|
||||
smoothedDrive = lerp(smoothedDrive, sampled.drive, smoothing);
|
||||
|
||||
const target = ambienceFromSampled(
|
||||
base,
|
||||
usingManual,
|
||||
manual,
|
||||
tunables,
|
||||
smoothedEnergy,
|
||||
smoothedValence,
|
||||
smoothedDrive,
|
||||
);
|
||||
current = lerpAmbience(current, target, 1 - Math.exp(-dt / CROSSFADE_TAU_SECONDS));
|
||||
|
||||
const analysis = analysisRef.current;
|
||||
const beatsTrusted =
|
||||
analysis !== null && analysis.beats && (analysis.pulse ?? 0) >= PULSE_THRESHOLD;
|
||||
const grid = beatsTrusted && !usingManual ? detailRef.current : null;
|
||||
|
||||
if (usingManual) {
|
||||
// A continuous fake metronome at the manual tempo, instead of walking a real
|
||||
// grid - reuses the same `kick` variable and decay as real beats, so every
|
||||
// downstream effect (spawn burst, current boost) is identical either way.
|
||||
manualBeatTimer -= dt;
|
||||
if (manualBeatTimer <= 0) {
|
||||
kick = Math.max(kick, manual.beatStrength);
|
||||
manualBeatTimer = 60 / Math.max(1, manual.tempo);
|
||||
}
|
||||
} else if (grid && grid.times.length > 0) {
|
||||
// Playback normally only moves forward, so the common case is just advancing
|
||||
// the cursor with the `while` below. A backward seek is the one thing that
|
||||
// can break that invariant - detected by the previously consumed beat now
|
||||
// being back in the future - and needs a real re-search.
|
||||
if (beatCursor > 0 && grid.times[beatCursor - 1]! > position) {
|
||||
beatCursor = beatCursorFor(grid.times, position);
|
||||
}
|
||||
while (beatCursor < grid.times.length && grid.times[beatCursor]! <= position) {
|
||||
kick = Math.max(kick, grid.strengths[beatCursor] ?? 0);
|
||||
beatCursor++;
|
||||
}
|
||||
} else {
|
||||
beatCursor = 0;
|
||||
}
|
||||
kick *= Math.exp(-dt / KICK_DECAY_TAU_SECONDS);
|
||||
|
||||
const kickedMagnitude = current.currentMagnitude * (1 + kick * tunables.beatCurrentFactor);
|
||||
|
||||
if (effectivePlaying) {
|
||||
spawnAccumulator += (current.spawnRate + kick * BEAT_BURST_RATE) * dt;
|
||||
while (spawnAccumulator >= 1) {
|
||||
spawnAccumulator -= 1;
|
||||
bubbles.push(spawnBubble(size.width));
|
||||
}
|
||||
}
|
||||
// Not playing (and not in manual mode): no new spawns, but bubbles already in
|
||||
// flight keep rising and draining away rather than freezing mid-air.
|
||||
|
||||
const riseSpeed = current.rise * size.height * BASE_RISE_FRACTION_PER_SECOND;
|
||||
bubbles = bubbles.filter((b) => {
|
||||
const heightFraction = clamp01(b.risen / size.height);
|
||||
const velocity = currentVelocityAt(heightFraction, kickedMagnitude);
|
||||
b.x = (((b.x + velocity * dt) % size.width) + size.width) % size.width;
|
||||
b.risen += riseSpeed * dt;
|
||||
b.size = b.baseSize * (1 + tunables.bubbleGrowthRate * heightFraction);
|
||||
return b.risen < size.height + BUBBLE_MAX_SIZE * 2;
|
||||
});
|
||||
|
||||
ctx.clearRect(0, 0, size.width, size.height);
|
||||
const [top, mid, deep] = current.gradient;
|
||||
const gradient = ctx.createLinearGradient(0, 0, 0, size.height);
|
||||
gradient.addColorStop(0, oklchString(top));
|
||||
gradient.addColorStop(0.45, oklchString(mid));
|
||||
gradient.addColorStop(1, oklchString(deep));
|
||||
ctx.fillStyle = gradient;
|
||||
ctx.fillRect(0, 0, size.width, size.height);
|
||||
|
||||
if (!reducedMotion) {
|
||||
for (const b of bubbles) {
|
||||
const y = size.height + BUBBLE_MAX_SIZE - b.risen;
|
||||
const topThird = size.height / 3;
|
||||
const fade = y < topThird ? Math.max(0, y / topThird) : 1;
|
||||
ctx.beginPath();
|
||||
ctx.fillStyle = oklchString(current.tint, b.baseAlpha * fade);
|
||||
ctx.arc(b.x, y, b.size / 2, 0, Math.PI * 2);
|
||||
ctx.fill();
|
||||
}
|
||||
}
|
||||
|
||||
const debugFrame = onDebugFrameRef.current;
|
||||
if (debugFrame && now - lastDebugEmit >= DEBUG_EMIT_INTERVAL_MS) {
|
||||
lastDebugEmit = now;
|
||||
debugFrame({
|
||||
group: groupRef.current,
|
||||
trackTitle: trackTitleRef.current,
|
||||
playing,
|
||||
position,
|
||||
analysis,
|
||||
target,
|
||||
current,
|
||||
sampledEnergy: smoothedEnergy,
|
||||
sampledValence: smoothedValence,
|
||||
sampledDrive: smoothedDrive,
|
||||
currentMagnitude: kickedMagnitude,
|
||||
kick,
|
||||
beatsTrusted,
|
||||
beatGridLoaded: grid !== null,
|
||||
beatCount: grid?.times.length ?? 0,
|
||||
beatCursor,
|
||||
bubbleCount: bubbles.length,
|
||||
curveLoaded: detailRef.current?.curve != null,
|
||||
usingManual,
|
||||
});
|
||||
}
|
||||
|
||||
raf = requestAnimationFrame(tick);
|
||||
};
|
||||
raf = requestAnimationFrame(tick);
|
||||
|
||||
return () => {
|
||||
cancelAnimationFrame(raf);
|
||||
observer.disconnect();
|
||||
media.removeEventListener("change", onMotionChange);
|
||||
};
|
||||
}, []);
|
||||
|
||||
return (
|
||||
<canvas
|
||||
ref={canvasRef}
|
||||
className="ambience"
|
||||
aria-hidden="true"
|
||||
style={{ position: "absolute", inset: 0, zIndex: 0, pointerEvents: "none" }}
|
||||
/>
|
||||
);
|
||||
}
|
||||
Reference in New Issue
Block a user