diff --git a/python-backend/musicmouse/library/cache.py b/python-backend/musicmouse/library/cache.py index 1af58c3..953e5e6 100644 --- a/python-backend/musicmouse/library/cache.py +++ b/python-backend/musicmouse/library/cache.py @@ -5,7 +5,8 @@ different amounts to produce:: / ├── index.json cheap: tags and structure. Thrown away freely. - ├── covers/.jpg medium: art pulled out of an ID3 APIC frame + ├── covers/.jpg medium: art pulled out of an ID3 APIC frame, + │ downscaled to MAX_COVER_PX on the way in └── analysis/.json expensive: minutes of DSP per track analysis/.beats.json analysis/.curve.json @@ -18,12 +19,15 @@ folder or re-sorting a section then costs nothing. from __future__ import annotations +import io import json import logging import os from dataclasses import dataclass from pathlib import Path -from typing import Any, cast +from typing import Any, Final, cast + +from PIL import Image, UnidentifiedImageError from musicmouse.library.analysis import BeatGrid, TrackAnalysis, TrackCurves from musicmouse.library.models import Album, LibraryTrack @@ -86,9 +90,37 @@ class LibraryCache: def store_cover(self, album_id: str, data: bytes) -> Path: path = self.cover_path(album_id) - path.write_bytes(data) + path.write_bytes(shrink_cover(data)) return path + def shrink_stored_covers(self) -> int: + """Rewrite any already-stored cover that predates :data:`MAX_COVER_PX`. + + Needed because :func:`~musicmouse.library.scanner.scan_library` reuses an album + whose fingerprint is unchanged *without* re-reading its tags, so a cover written + by an older version would otherwise never be touched again. Reading a JPEG's + dimensions only parses its header, so once every file is within the limit this + costs one small read per album and nothing else. + + Returns the number of files actually rewritten. + """ + rewritten = 0 + for path in sorted(self.covers.glob("*.jpg")): + try: + with Image.open(path) as image: + oversized = max(image.size) > MAX_COVER_PX + except (OSError, UnidentifiedImageError): + continue + if not oversized: + continue + try: + shrunk = shrink_cover(path.read_bytes()) + path.write_bytes(shrunk) + except OSError: # pragma: no cover - a cache we cannot write is not fatal + continue + rewritten += 1 + return rewritten + # ------------------------------------------------------------------ analysis def load_analysis(self, key: str) -> TrackAnalysis | None: @@ -213,3 +245,44 @@ def _album_from_json(data: dict[str, Any]) -> Album: for track in data["tracks"] ), ) + + +# ---------------------------------------------------------------------------- covers + +#: Longest edge kept for cached album art, in pixels. +#: +#: The art that comes out of an ID3 APIC frame is sized for a record sleeve, not for a +#: screen: a real library here averaged 3000x3000 and 580 kB per cover, 140 MB for 284 +#: albums. The browser was decoding nine megapixels - about 36 MB of bitmap - for every +#: cover it painted, to show it in a 185 px card on a Pi with 2 GB of RAM. +#: +#: 340 is the largest any of this app's screens asks for (the play view; the browse grid +#: asks for 180 and the player bar for 56), so 640 still leaves room for a tablet at +#: devicePixelRatio 2 and cuts the decode by about twenty times. +MAX_COVER_PX: Final = 640 + +#: Re-encode quality. At these dimensions the difference from 95 is invisible and the +#: file is a third of the size. +_COVER_JPEG_QUALITY: Final = 85 + + +def shrink_cover(data: bytes) -> bytes: + """Downscale cover art to :data:`MAX_COVER_PX` on its longest edge. + + Art that is already small enough is returned untouched rather than re-encoded, so + repeated scans never degrade it. Anything Pillow cannot read is passed through + unchanged: a cover that is too big is a performance problem, a cover that is missing + is a visible one. + """ + try: + with Image.open(io.BytesIO(data)) as image: + if max(image.size) <= MAX_COVER_PX: + return data + # `thumbnail` keeps the aspect ratio and never scales up. + image = image.convert("RGB") + image.thumbnail((MAX_COVER_PX, MAX_COVER_PX), Image.Resampling.LANCZOS) + buffer = io.BytesIO() + image.save(buffer, format="JPEG", quality=_COVER_JPEG_QUALITY, optimize=True) + return buffer.getvalue() + except (OSError, UnidentifiedImageError, ValueError): + return data diff --git a/python-backend/musicmouse/library/scanner.py b/python-backend/musicmouse/library/scanner.py index 451524f..d7c188c 100644 --- a/python-backend/musicmouse/library/scanner.py +++ b/python-backend/musicmouse/library/scanner.py @@ -295,6 +295,13 @@ def scan_library( figure name to what it holds, which is the one thing the folders cannot say. """ cache.prepare() + # Covers written before MAX_COVER_PX existed are still whatever size the tag held, + # and the reuse path below means an unchanged album never rewrites its own. One + # pass here catches them; after the first run every file is already small and this + # is 300-odd header reads. + shrunk = cache.shrink_stored_covers() + if shrunk: + _log.info("Downscaled %d oversized cover(s) in the cache", shrunk) known = known or {} out: dict[str, tuple[Album, Fingerprint]] = {} last_report = time.monotonic() diff --git a/python-backend/tests/test_library.py b/python-backend/tests/test_library.py index 7334a11..5e3d303 100644 --- a/python-backend/tests/test_library.py +++ b/python-backend/tests/test_library.py @@ -4,6 +4,7 @@ from __future__ import annotations import asyncio import contextlib +import io import json import os import threading @@ -726,3 +727,76 @@ async def test_a_parallel_pass_is_cancellable_mid_flight(config_dir: Path) -> No def test_the_default_worker_count_leaves_a_core_for_everything_else() -> None: count = default_worker_count() assert 1 <= count <= max(1, (os.cpu_count() or 1) - 1) + + +class TestCoverDownscaling: + """Album art arrives sized for a record sleeve; screens want a fraction of that.""" + + @staticmethod + def _jpeg(size: tuple[int, int]) -> bytes: + from PIL import Image + + buffer = io.BytesIO() + Image.new("RGB", size, (120, 40, 200)).save(buffer, format="JPEG") + return buffer.getvalue() + + def test_oversized_art_is_shrunk_to_the_long_edge(self) -> None: + from PIL import Image + + from musicmouse.library.cache import MAX_COVER_PX, shrink_cover + + original = self._jpeg((3000, 2000)) + shrunk = shrink_cover(original) + with Image.open(io.BytesIO(shrunk)) as image: + width, height = image.size + assert max(width, height) == MAX_COVER_PX + # Aspect ratio survives, to within the rounding of a whole pixel. + assert abs(width / height - 3000 / 2000) < 0.01 + assert len(shrunk) < len(original) + + def test_art_already_small_enough_is_returned_untouched(self) -> None: + """Byte-identical, not merely similar: repeated scans must not re-encode art + over and over, each pass losing a little more to JPEG.""" + from musicmouse.library.cache import shrink_cover + + original = self._jpeg((300, 300)) + assert shrink_cover(original) is original + + def test_unreadable_art_is_passed_through_rather_than_dropped(self) -> None: + """A cover too big is a performance problem; a cover missing is a visible one.""" + from musicmouse.library.cache import shrink_cover + + assert shrink_cover(b"not an image at all") == b"not an image at all" + + def test_store_cover_shrinks_on_the_way_in(self, tmp_path: Path) -> None: + from PIL import Image + + from musicmouse.library.cache import MAX_COVER_PX, LibraryCache + + cache = LibraryCache(tmp_path) + cache.prepare() + path = cache.store_cover("abc123", self._jpeg((2400, 2400))) + with Image.open(path) as image: + assert max(image.size) == MAX_COVER_PX + + def test_covers_written_by_an_older_version_are_migrated(self, tmp_path: Path) -> None: + """The scanner reuses an unchanged album without re-reading its tags, so a + cover stored before the limit existed would otherwise never be rewritten.""" + from PIL import Image + + from musicmouse.library.cache import MAX_COVER_PX, LibraryCache + + cache = LibraryCache(tmp_path) + cache.prepare() + stale = cache.cover_path("old") + stale.write_bytes(self._jpeg((3000, 3000))) + fresh = cache.cover_path("new") + fresh.write_bytes(self._jpeg((320, 320))) + fresh_before = fresh.read_bytes() + + assert cache.shrink_stored_covers() == 1 + with Image.open(stale) as image: + assert max(image.size) == MAX_COVER_PX + assert fresh.read_bytes() == fresh_before + # Second pass has nothing left to do - the cheap steady state. + assert cache.shrink_stored_covers() == 0 diff --git a/web/README.md b/web/README.md index 1af825b..fd72465 100644 --- a/web/README.md +++ b/web/README.md @@ -95,34 +95,35 @@ seeking is the one thing a real player can do that the mockup could not. ## The Raspberry Pi profile -`?pi=1` loads the same app with the ambient canvas painting a quarter of the pixels. It -exists for musicdolphin - a Pi 4 driving a 1920x1080 kiosk screen - where Firefox -rasterizes 2D canvas in the content process, and the canvas repaints the whole screen -every frame whether or not anything is playing. +`?pi=1` loads the same app with nothing on screen animating itself. It exists for +musicdolphin - a Pi 4 driving a 1920x1080 kiosk screen. -Everything in `src/lib/lowPower.ts` was measured on the device. Sum of the Firefox -process tree, idle on the browse screen, 15 s average: +| | Full | `?pi=1` | +|---|---|---| +| Ambient canvas (gradient + bubbles) | on | **not rendered** | +| Decorative CSS loops, view transitions | on | off | +| Typing game bubbles, next-key pulse | on | off | +| Typing game pets | swimming | placed, held still | +| `backdrop-filter` on cards and list rows | on | off | +| `backdrop-filter` on panels | on | **on** | +| Progress-bar re-renders | every animation frame | 10 per second | -| | idle CPU | -|---|---| -| full app | 53.5% | -| + ambient canvas at half resolution | **25.0%** | -| + 30 fps cap on top of that | 25.0% | -| + decorative CSS animation loops off | 24.6% | -| + `backdrop-filter` glass blur off | **118.1%** | +Two of those rows are worth knowing the reasons for, because both are the opposite of +what they look like. -Two things to take from that table. The canvas resolution is the whole win, and -**turning off the backdrop blur makes it five times worse** - the blur is what promotes -each glass panel to its own compositing layer, so a canvas frame underneath repaints -only the canvas. Without it the canvas and the album grid above it share one layer and -every frame repaints all of it. The most expensive-looking CSS in the app is what keeps -the rest of it cheap; `SHOW_GLASS_BLUR` stays on everywhere, and so do the decorative -loops, which measured as noise. +**The panel blur stays on.** Dropping it measured *five times worse* on the device - +118% of a core against 25%. `backdrop-filter` is what promotes each glass panel to its +own compositing layer; without it the panels and everything under them collapse into +one layer that repaints wholesale. The card blur goes because that is a different +problem: three hundred of them in one grid, not one per panel. -So the profile is one real change plus three caps that only bite while something is -playing, and are therefore not in the table: 30 fps on the canvas, 60 bubbles alive at -once, and ten progress-bar re-renders a second instead of sixty (`usePlaybackClock` -pushes a React `setState` per animation frame, into both `PlayView` and `PlayerBar`). +**The canvas goes away rather than getting cheaper.** Half resolution and 30 fps took +it from 53.5% of a core to 25.0%, and 25% was still not good enough to use. A +`requestAnimationFrame` loop repainting the viewport is a floor you cannot get under +while it runs at all. `.stage` keeps its own static CSS gradient, so there is still a +sea behind everything - it just no longer moves or follows the track. +`AMBIENCE_QUALITY` in `src/lib/lowPower.ts` is kept accurate for whoever turns the +canvas back on. The flag is read once at module load from the URL and nowhere else - no persistence, no auto-detection. That is what makes "is this the profile or the hardware?" answerable by diff --git a/web/src/components/tippen/AquariumCreatures.tsx b/web/src/components/tippen/AquariumCreatures.tsx index 1898591..83fc09e 100644 --- a/web/src/components/tippen/AquariumCreatures.tsx +++ b/web/src/components/tippen/AquariumCreatures.tsx @@ -14,6 +14,7 @@ import { useEffect, useRef } from "react"; +import { LOW_POWER } from "../../lib/lowPower"; import { creatureById, createSwimmer, pose, stepSwimmer } from "../../lib/tippen/aquarium"; import type { CreatureId, Swimmer } from "../../lib/tippen/aquarium"; @@ -37,8 +38,11 @@ export function AquariumCreatures({ creatures, opacity }: Props) { const present = useRef>(new Set(creatures)); useEffect(() => { - // Reduced motion: every pet is still placed and shown, it just holds still. - const reducedMotion = window.matchMedia("(prefers-reduced-motion: reduce)").matches; + // Reduced motion: every pet is still placed and shown, it just holds still. `?pi=1` + // takes the same door: on that device the point is that nothing runs a frame loop, + // and the pets are the reward - they should be *there*, they just need not swim. + const reducedMotion = + LOW_POWER || window.matchMedia("(prefers-reduced-motion: reduce)").matches; let frame = 0; let lastTime = performance.now(); diff --git a/web/src/components/tippen/Stage.tsx b/web/src/components/tippen/Stage.tsx index bbd8a29..42330ae 100644 --- a/web/src/components/tippen/Stage.tsx +++ b/web/src/components/tippen/Stage.tsx @@ -1,5 +1,7 @@ /** The gradient stage every screen sits on - the app's one full-height container. - * `data-blur` is read by app.css to drop the backdrop filters wholesale. + * `data-blur` is read by app.css to drop the backdrop filters wholesale, and `data-anim` + * by tippen.css to stop the one CSS animation loop that is not a component's to switch + * off - the pulse on the next key to press. * * The pets swim here rather than on the home screen, so they stay with her on the lesson * map and during a run too. */ @@ -7,6 +9,7 @@ import type { ReactNode } from "react"; import type { CreatureId } from "../../lib/tippen/aquarium"; +import { LOW_POWER } from "../../lib/lowPower"; import { SHOW_AQUARIUM_CREATURES, SHOW_BUBBLES, SHOW_GLASS_BLUR } from "../../lib/tippen/theme"; import { AquariumCreatures } from "./AquariumCreatures"; import { Bubbles } from "./Bubbles"; @@ -21,7 +24,11 @@ interface Props { export function Stage({ children, creatures, dimmed }: Props) { return ( -
+
{SHOW_AQUARIUM_CREATURES && } {SHOW_BUBBLES && } {children} diff --git a/web/src/lib/lowPower.ts b/web/src/lib/lowPower.ts index 207035a..9402ba5 100644 --- a/web/src/lib/lowPower.ts +++ b/web/src/lib/lowPower.ts @@ -24,9 +24,18 @@ * grid included. The most expensive-looking CSS in the app is the thing keeping the * rest of it cheap. Do not "optimize" it away without re-running the numbers above. * - * So the profile is one real change - paint a quarter of the pixels - plus three caps - * that only bite while something is playing (bubbles spawn, curves are sampled per - * frame, the progress bar re-renders) and that are therefore not in the table above. + * That got it from 53.5% to 25%, and 25% was still not enough to use. The profile now + * goes further and stops the animation instead of thinning it: under `?pi=1` the + * ambient canvas is not rendered at all (`SHOW_AMBIENCE`), no bubbles rise anywhere, + * the decorative CSS loops are off, the view transitions are off, and the typing game's + * pets are placed but held still. A requestAnimationFrame loop that repaints the whole + * viewport is a floor you cannot get under while it runs at all, and on a Pi 4 that + * floor is too high. With it gone the browser has nothing to do between one keypress + * and the next. `.stage` keeps its own static CSS gradient, so the screen still has a + * sea behind it - it just no longer moves or follows the track. + * + * What stays: both blurs on the panels (see the table - dropping those was five times + * worse), and the card blur off, which is a count problem rather than a blur problem. * * Read once at module load, from the URL and nowhere else: no persistence, no * auto-detection. The kiosk points Firefox at `http://localhost:8080/?pi=1` (see the @@ -67,6 +76,10 @@ export interface AmbienceQuality { maxBubbles: number; } +/** Unreachable under `?pi=1` as things stand, because `SHOW_AMBIENCE` switches the + * canvas off there entirely. Kept, and kept accurate, because it is the setting that + * matters the moment anyone turns the canvas back on for a weak device - the half-scale + * backing store is what took it from 53.5% to 25%. */ export const AMBIENCE_QUALITY: AmbienceQuality = LOW_POWER ? { resolutionScale: 0.5, maxBackingStorePx: 1280, maxFps: 30, maxBubbles: 60 } : { resolutionScale: 1, maxBackingStorePx: 4096, maxFps: 0, maxBubbles: 0 }; diff --git a/web/src/lib/theme.ts b/web/src/lib/theme.ts index 90f6261..b9e059c 100644 --- a/web/src/lib/theme.ts +++ b/web/src/lib/theme.ts @@ -33,7 +33,7 @@ export const ROW_SPACING = 25; /** Fade the album/category grid in (with a slight upward slide) when a group or * category is entered, and the album modal in when it opens. One animation per * container, not per card, so cost stays flat regardless of grid size. */ -export const ANIMATE_VIEW_TRANSITIONS = true; +export const ANIMATE_VIEW_TRANSITIONS = !LOW_POWER; /** Frost the glass panels/cards/rows with a real backdrop blur. * @@ -45,13 +45,16 @@ export const ANIMATE_VIEW_TRANSITIONS = true; export const SHOW_GLASS_BLUR = true; /** Render the play view's animated canvas background (gradient + bubbles, both - * redrawn every frame at 60fps). Off falls back to `.stage`'s own static CSS - * gradient, which is still underneath the canvas either way. + * redrawn every frame). Off falls back to `.stage`'s own static CSS gradient, which is + * still underneath the canvas either way - so the screen keeps a background, it just + * stops being a per-track one and stops moving. * - * Deliberately still on under `?pi=1`: it is the app's whole look, and - * `AMBIENCE_QUALITY` in lib/lowPower.ts makes it affordable (quarter of the pixels, - * half the frames) rather than making it go away. */ -export const SHOW_AMBIENCE = true; + * Off under `?pi=1`. Halving its resolution and its frame rate first (see + * `AMBIENCE_QUALITY`) was not enough on the device this exists for: a requestAnimationFrame + * loop that repaints the full viewport is a floor you cannot get under while it runs at + * all, and on a Pi 4 that floor is too high. Nothing else in the app needs a frame loop, + * so with this off the browser has nothing to do between one keypress and the next. */ +export const SHOW_AMBIENCE = !LOW_POWER; /** Frost the *repeated* glass surfaces - every album card in the grid, every row in a * track list - as opposed to the handful of panels wrapped around them. @@ -65,10 +68,11 @@ export const SHOW_AMBIENCE = true; export const SHOW_CARD_BLUR = !LOW_POWER; /** Run the purely decorative CSS animation loops: the room page's bubble field and - * the dolphin mascot's bob/swim. Left on even under `?pi=1`: switching them off on the - * Pi moved the idle figure by 0.4 percentage points, which is noise, and they are the - * kind of detail this app is for. */ -export const SHOW_DECORATIVE_ANIMATIONS = true; + * the dolphin mascot's bob/swim. Off under `?pi=1`. On their own they measured as + * noise, but "nothing on this screen moves by itself" is a property worth having + * outright rather than a sum of small wins - a compositor with no animation to service + * has nothing to wake up for. */ +export const SHOW_DECORATIVE_ANIMATIONS = !LOW_POWER; // ------------------------------------------------------------------ colors -- diff --git a/web/src/lib/tippen/theme.ts b/web/src/lib/tippen/theme.ts index 2b3e9af..00c89b7 100644 --- a/web/src/lib/tippen/theme.ts +++ b/web/src/lib/tippen/theme.ts @@ -4,6 +4,8 @@ * through components. The hue lives in styles/app.css because CSS is where it is used; * it is repeated here only for the canvas, which cannot read a custom property. */ +import { LOW_POWER } from "../lowPower"; + /** The turquoise lagoon. Music is 210, Hörbücher 55, "Mein Zimmer" 300. */ export const HUE = 175; @@ -14,15 +16,20 @@ export const HUE = 175; export const SHOW_GLASS_BLUR = true; /** The decorative rising bubbles behind everything. A dozen elements on a CSS transform - * loop; measured as noise on the Pi, so they stay. */ -export const SHOW_BUBBLES = true; + * loop - off under `?pi=1`, like every other loop in the app. */ +export const SHOW_BUBBLES = !LOW_POWER; /** The earned pets swimming behind every screen. The reward that is always in view - off - * only to rule it out when chasing a performance problem. */ + * only to rule it out when chasing a performance problem. + * + * Stays on under `?pi=1`, but the pets hold still there: `AquariumCreatures` already has + * that exact mode for `prefers-reduced-motion`, where every pet is placed and shown and + * simply does not swim. Taking the animation away is the point; taking away what she + * earned is not. */ export const SHOW_AQUARIUM_CREATURES = true; /** Fade screens in on entry. */ -export const ANIMATE_VIEW_TRANSITIONS = true; +export const ANIMATE_VIEW_TRANSITIONS = !LOW_POWER; /** Show the on-screen keyboard with the finger colours. "auto" fades it out key by key * as each one is mastered - the scaffold that removes itself, which is the whole point diff --git a/web/src/styles/tippen.css b/web/src/styles/tippen.css index 113e033..852fda6 100644 --- a/web/src/styles/tippen.css +++ b/web/src/styles/tippen.css @@ -214,6 +214,12 @@ animation: keyPulse 1.3s ease-out infinite; } +/* `?pi=1` (`data-anim="off"`, set in Stage.tsx): the key still lifts and lightens, it + just stops breathing. The last animation loop left on that device. */ +.tp-stage[data-anim="off"] .tp-kb-key[data-next="true"] { + animation: none; +} + .tp-kb-key[data-home="true"]::after { /* The tactile bump on F and J, drawn so it can be pointed at on screen too. */ content: "";