bytelyst/learning_ai_common_plat
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(media-ui): @bytelyst/media-ui@0.1.0 — Wave 13.G.1/.2/.4

Browse Source 
Media surface primitives — ImageGenStream, AudioWaveform, VideoPlayer.
Zero runtime deps (canvas + native <video> + native <img>); PdfPreview
deferred to 0.2.x where it'll lazy-load pdf.js.

──────────────────────────────────────────────────────────────────
<ImageGenStream>  ·  Wave 13.G.1
──────────────────────────────────────────────────────────────────
  - Driven entirely by props (status + snapshot + finalUrl)
  - 4 statuses: idle / streaming / complete / error
  - Streaming: blurred partial image + overlay progress bar +
    step label (e.g. 'denoising 24/50')
  - Complete: drops blur with fade-in
  - Error: muted error placeholder (token-tinted danger)
  - Host pipes any transport (SSE / WebSocket / polling) into
    the snapshot prop — pure presentation

<AudioWaveform>  ·  Wave 13.G.2
  - Canvas render with DPR-aware paint
  - Two sources: pre-computed peaks (cheapest) OR lazy WebAudio
    decode from audioUrl (falls back gracefully w/o AudioContext)
  - Click-to-seek (returns 0..1 position)
  - Progress overlay tints played bars with progressColor
  - Resampling helper handles arbitrary peak-count inputs

<VideoPlayer>  ·  Wave 13.G.4
  - Native <video controls> wrapper — accessible by default
  - Chapter buttons that seek + auto-play (silently swallows
    autoplay rejections)
  - Optional in-memory caption track (aria-live polite); hosts
    that prefer real WebVTT pass <track> via the slot prop
  - Token-tinted; rounded; muted poster background

──────────────────────────────────────────────────────────────────
Quality gates
──────────────────────────────────────────────────────────────────
  ✓ 10/10 tests passing
    - AudioWaveform: 3 cases (canvas size · click-seek math ·
      placeholder when no peaks)
    - ImageGenStream: 4 cases (idle / streaming / complete /
      error)
    - VideoPlayer: 3 cases (src · chapter button list · caption
      rail)
  ✓ tsc build clean
  ✓ Zero new runtime deps; all WebAudio access guarded for SSR

Showcase /futurism/multimodal (MAG.7) lands in the paired showcase
commit.
This commit is contained in:
saravanakumardb1 2026-05-27 17:44:13 -07:00
parent 2affd1aba0
commit 99e59597d1
8 changed files with 754 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

34
packages/media-ui/package.json Normal file
View File

@ -0,0 +1,34 @@
{
"name": "@bytelyst/media-ui",
"version": "0.1.0",
"type": "module",
"description": "Media surface primitives — AudioWaveform, VideoPlayer, ImageGenStream. Zero runtime deps; PdfPreview deferred to 0.2.x (pdf.js).",
"exports": {
".": {
"import": "./dist/index.js",
"types": "./dist/index.d.ts"
}
},
"main": "./dist/index.js",
"types": "./dist/index.d.ts",
"files": ["dist"],
"scripts": {
"build": "tsc",
"test": "vitest run --pool forks",
"typecheck": "tsc --noEmit"
},
"peerDependencies": {
"react": ">=18.0.0",
"react-dom": ">=18.0.0"
},
"devDependencies": {
"@testing-library/react": "^16.3.2",
"@types/react": "^19.2.14",
"@types/react-dom": "^19.2.3",
"happy-dom": "^18.0.1",
"react": "^19.2.4",
"react-dom": "^19.2.4",
"typescript": "^5.7.3",
"vitest": "^4.0.18"
}
}

180
packages/media-ui/src/AudioWaveform.tsx Normal file
View File

@ -0,0 +1,180 @@
import {
useEffect,
useId,
useMemo,
useRef,
useState,
type CSSProperties,
type MouseEvent as ReactMouseEvent,
} from 'react';
export interface AudioWaveformProps {
/** Pre-computed peak samples (0..1). Mutually exclusive with `audioUrl`. */
peaks?: number[];
/** Audio file URL — when set + peaks absent, peaks are derived via WebAudio. */
audioUrl?: string;
/** Number of bars to render. Default 96. */
bars?: number;
/** Current playback position (0..1) — drives the progress overlay. */
progress?: number;
/** Called when the user clicks a bar (returns 0..1 position). */
onSeek?: (position: number) => void;
width?: number;
height?: number;
color?: string;
progressColor?: string;
ariaLabel?: string;
className?: string;
style?: CSSProperties;
}
/**
* `<AudioWaveform>` — canvas-rendered waveform with click-to-seek.
*
* Wave 13.G.2. Accepts a pre-computed `peaks` array (cheapest) OR
* lazy-decodes peaks via WebAudio for an `audioUrl`. While the decode
* is in flight (or in environments without `AudioContext`) a flat
* placeholder is rendered.
*/
export function AudioWaveform({
peaks: peaksProp,
audioUrl,
bars = 96,
progress = 0,
onSeek,
width = 480,
height = 80,
color = 'var(--bl-accent, #6366f1)',
progressColor = 'var(--bl-text-primary, #111)',
ariaLabel,
className,
style,
}: AudioWaveformProps) {
const canvasRef = useRef<HTMLCanvasElement>(null);
const titleId = useId();
const [decoded, setDecoded] = useState<number[] | null>(null);
// Lazy WebAudio decode when only audioUrl is supplied.
useEffect(() => {
if (peaksProp || !audioUrl) return;
if (typeof window === 'undefined') return;
let cancelled = false;
(async () => {
try {
const res = await fetch(audioUrl);
const buf = await res.arrayBuffer();
const Ctx =
(
window as unknown as {
AudioContext?: typeof AudioContext;
webkitAudioContext?: typeof AudioContext;
}
).AudioContext ??
(window as unknown as { webkitAudioContext?: typeof AudioContext })
.webkitAudioContext;
if (!Ctx) return;
const ac = new Ctx();
const audio = await ac.decodeAudioData(buf);
if (cancelled) return;
const channel = audio.getChannelData(0);
const block = Math.max(1, Math.floor(channel.length / bars));
const out = new Array<number>(bars).fill(0);
for (let i = 0; i < bars; i++) {
const start = i * block;
let max = 0;
for (let j = 0; j < block && start + j < channel.length; j++) {
const v = Math.abs(channel[start + j] ?? 0);
if (v > max) max = v;
}
out[i] = Math.min(1, max);
}
if (!cancelled) setDecoded(out);
} catch {
/* swallow — placeholder remains */
}
})();
return () => {
cancelled = true;
};
}, [peaksProp, audioUrl, bars]);
const finalPeaks = useMemo(() => {
if (peaksProp && peaksProp.length > 0) return resampleTo(peaksProp, bars);
if (decoded && decoded.length > 0) return decoded;
return new Array<number>(bars).fill(0.04);
}, [peaksProp, decoded, bars]);
// Paint.
useEffect(() => {
const canvas = canvasRef.current;
if (!canvas) return;
const dpr =
typeof window !== 'undefined' ? window.devicePixelRatio || 1 : 1;
canvas.width = Math.floor(width * dpr);
canvas.height = Math.floor(height * dpr);
const ctx = canvas.getContext('2d');
if (!ctx) return;
ctx.setTransform(1, 0, 0, 1, 0, 0);
ctx.scale(dpr, dpr);
ctx.clearRect(0, 0, width, height);
const slot = width / finalPeaks.length;
const barW = Math.max(1, slot * 0.65);
const mid = height / 2;
const progressX = Math.max(0, Math.min(1, progress)) * width;
finalPeaks.forEach((p, i) => {
const x = slot * i + (slot - barW) / 2;
const h = Math.max(2, p * height * 0.9);
ctx.fillStyle = x + barW <= progressX ? progressColor : color;
ctx.fillRect(x, mid - h / 2, barW, h);
});
}, [finalPeaks, width, height, color, progressColor, progress]);
const onClick = (e: ReactMouseEvent<HTMLCanvasElement>) => {
if (!onSeek) return;
const rect = e.currentTarget.getBoundingClientRect();
onSeek(Math.max(0, Math.min(1, (e.clientX - rect.left) / rect.width)));
};
return (
<div
role="img"
aria-labelledby={titleId}
data-testid="bl-audio-waveform"
className={className}
style={{ display: 'inline-block', ...style }}
>
<span id={titleId} hidden>
{ariaLabel ?? 'Audio waveform'}
</span>
<canvas
ref={canvasRef}
onClick={onClick}
style={{
width,
height,
cursor: onSeek ? 'pointer' : 'default',
display: 'block',
}}
/>
</div>
);
}
/** Resample any peaks array up/down to `bars` length. */
function resampleTo(src: number[], bars: number): number[] {
if (src.length === bars) return src;
if (src.length === 0) return new Array<number>(bars).fill(0);
const out = new Array<number>(bars);
const ratio = src.length / bars;
for (let i = 0; i < bars; i++) {
const start = Math.floor(i * ratio);
const end = Math.max(start + 1, Math.floor((i + 1) * ratio));
let max = 0;
for (let j = start; j < end && j < src.length; j++) {
const v = Math.abs(src[j] ?? 0);
if (v > max) max = v;
}
out[i] = Math.min(1, max);
}
return out;
}

191
packages/media-ui/src/ImageGenStream.tsx Normal file
View File

@ -0,0 +1,191 @@
import {
useEffect,
useState,
type CSSProperties,
type ReactNode,
} from 'react';
export type ImageGenStatus = 'idle' | 'streaming' | 'complete' | 'error';
export interface ImageGenSnapshot {
/** Progress 0..1. */
progress: number;
/** Optional partial image URL (e.g. data: or blob:). */
partialUrl?: string;
/** Step label (e.g. "denoising 24/50"). */
label?: string;
}
export interface ImageGenStreamProps {
/** Live status. */
status: ImageGenStatus;
/** Latest snapshot from the upstream stream. */
snapshot?: ImageGenSnapshot;
/** Final image URL (rendered when status === 'complete'). */
finalUrl?: string;
/** Optional error message (rendered when status === 'error'). */
errorMessage?: string;
/** Aspect ratio (w/h). Default 1 (square). */
aspect?: number;
/** Width in px. Default 320. */
width?: number;
/** Optional caption slot. */
caption?: ReactNode;
ariaLabel?: string;
className?: string;
style?: CSSProperties;
}
/**
* `<ImageGenStream>` — surface that animates a streaming image-generation
* job. Driven entirely by props; the host pipes whatever transport
* (SSE / WebSocket / polling) into the `status` + `snapshot` props.
*
* Wave 13.G.1. Pure presentation. Status drives visual:
* - idle: empty placeholder with helpful copy
* - streaming: partial image + progress bar + label
* - complete: final image with subtle fade-in
* - error: muted error card
*/
export function ImageGenStream({
status,
snapshot,
finalUrl,
errorMessage,
aspect = 1,
width = 320,
caption,
ariaLabel,
className,
style,
}: ImageGenStreamProps) {
const height = Math.round(width / Math.max(0.1, aspect));
const [revealedAt, setRevealedAt] = useState<number | null>(null);
useEffect(() => {
if (status === 'complete') setRevealedAt(Date.now());
else setRevealedAt(null);
}, [status, finalUrl]);
const progress = Math.max(0, Math.min(1, snapshot?.progress ?? 0));
return (
<figure
data-testid="bl-image-gen-stream"
data-status={status}
className={className}
style={{
margin: 0,
display: 'inline-flex',
flexDirection: 'column',
gap: 8,
...style,
}}
aria-label={ariaLabel}
>
<div
style={{
position: 'relative',
width,
height,
borderRadius: 14,
overflow: 'hidden',
background: 'var(--bl-surface-muted, rgba(0,0,0,0.06))',
border: '1px solid var(--bl-border-subtle, rgba(0,0,0,0.06))',
}}
>
{status === 'idle' && (
<Placeholder text="Ready to generate." />
)}
{(status === 'streaming' || status === 'complete') && (
// eslint-disable-next-line @next/next/no-img-element
<img
src={status === 'complete' ? finalUrl ?? snapshot?.partialUrl : snapshot?.partialUrl}
alt={ariaLabel ?? 'Generated image'}
data-testid="bl-image-gen-stream-img"
style={{
width: '100%',
height: '100%',
objectFit: 'cover',
filter: status === 'streaming' ? 'blur(4px)' : 'none',
opacity: revealedAt ? 1 : 0.95,
transition: 'filter 200ms ease, opacity 200ms ease',
}}
/>
)}
{status === 'streaming' && (
<div
data-testid="bl-image-gen-stream-progress"
style={{
position: 'absolute',
left: 0,
right: 0,
bottom: 0,
padding: '6px 8px 8px',
background:
'linear-gradient(to top, rgba(0,0,0,0.6), transparent)',
color: 'white',
fontSize: 11,
fontFamily:
'ui-monospace, SFMono-Regular, Menlo, Monaco, Consolas, monospace',
}}
>
<div
style={{
height: 4,
background: 'rgba(255,255,255,0.18)',
borderRadius: 999,
overflow: 'hidden',
}}
>
<div
style={{
height: '100%',
width: `${progress * 100}%`,
background: 'var(--bl-accent, #6366f1)',
transition: 'width 240ms ease',
}}
/>
</div>
<div style={{ marginTop: 4 }}>{snapshot?.label ?? `${Math.round(progress * 100)}%`}</div>
</div>
)}
{status === 'error' && <Placeholder text={errorMessage ?? 'Something went wrong.'} kind="error" />}
</div>
{caption && (
<figcaption
style={{
fontSize: 12,
color: 'var(--bl-text-tertiary, #888)',
fontFamily: 'ui-sans-serif, system-ui, sans-serif',
}}
>
{caption}
</figcaption>
)}
</figure>
);
}
function Placeholder({ text, kind }: { text: string; kind?: 'error' }) {
return (
<div
data-testid={kind === 'error' ? 'bl-image-gen-stream-error' : 'bl-image-gen-stream-idle'}
style={{
width: '100%',
height: '100%',
display: 'grid',
placeItems: 'center',
padding: 16,
textAlign: 'center',
color:
kind === 'error'
? 'var(--bl-danger, #ef4444)'
: 'var(--bl-text-tertiary, #888)',
fontSize: 12,
fontFamily: 'ui-sans-serif, system-ui, sans-serif',
}}
>
{text}
</div>
);
}

189
packages/media-ui/src/VideoPlayer.tsx Normal file
View File

@ -0,0 +1,189 @@
import {
useEffect,
useRef,
useState,
type CSSProperties,
type ReactNode,
} from 'react';
export interface VideoChapter {
/** Stable id — React key. */
id: string;
/** Display label. */
label: string;
/** Chapter start time in seconds. */
startSeconds: number;
}
export interface VideoCaption {
/** Stable id. */
id: string;
/** Caption start in seconds. */
start: number;
/** Caption end in seconds. */
end: number;
/** Display text. */
text: string;
}
export interface VideoPlayerProps {
/** Video source URL. */
src: string;
/** Optional poster image. */
poster?: string;
/** Optional chapter list — rendered as buttons under the player. */
chapters?: VideoChapter[];
/**
* Optional in-memory caption track. When provided, the active line is
* rendered as an aria-live caption strip below the video.
* Hosts that prefer real <track> elements can pass `captionsTrackUrl`
* instead via the `slot` prop and skip this list.
*/
captions?: VideoCaption[];
/** Slot rendered above the chapter list — e.g. native <track>. */
slot?: ReactNode;
width?: number;
height?: number;
ariaLabel?: string;
className?: string;
style?: CSSProperties;
}
/**
* `<VideoPlayer>` — minimal token-themed video wrapper with chapter
* navigation + optional caption rail.
*
* Wave 13.G.4. Uses the native `<video controls>` element; chapters
* are rendered as discrete "seek-to" buttons. Captions are passed in
* pure JS for the simple case — hosts wanting WebVTT pass a `<track>`
* via the `slot` prop.
*/
export function VideoPlayer({
src,
poster,
chapters,
captions,
slot,
width = 480,
height = 270,
ariaLabel,
className,
style,
}: VideoPlayerProps) {
const ref = useRef<HTMLVideoElement>(null);
const [activeCaption, setActiveCaption] = useState<VideoCaption | null>(null);
// Tick the active caption as the video plays.
useEffect(() => {
if (!captions || captions.length === 0) return;
const el = ref.current;
if (!el) return;
const onTime = () => {
const t = el.currentTime;
const hit = captions.find((c) => t >= c.start && t < c.end);
setActiveCaption(hit ?? null);
};
el.addEventListener('timeupdate', onTime);
return () => el.removeEventListener('timeupdate', onTime);
}, [captions]);
const seek = (s: number) => {
const el = ref.current;
if (!el) return;
el.currentTime = s;
void el.play().catch(() => {});
};
return (
<div
data-testid="bl-video-player"
className={className}
style={{
display: 'inline-flex',
flexDirection: 'column',
gap: 8,
...style,
}}
>
<video
ref={ref}
src={src}
poster={poster}
controls
width={width}
height={height}
aria-label={ariaLabel}
data-testid="bl-video-player-video"
style={{
width,
height,
borderRadius: 12,
background: 'var(--bl-surface-muted, #000)',
}}
>
{slot}
Your browser does not support the video element.
</video>
{captions && captions.length > 0 && (
<div
aria-live="polite"
data-testid="bl-video-player-caption"
style={{
minHeight: 22,
padding: '4px 8px',
borderRadius: 8,
background: 'var(--bl-surface-muted, rgba(0,0,0,0.05))',
fontSize: 13,
color: 'var(--bl-text-secondary, #555)',
textAlign: 'center',
fontFamily: 'ui-sans-serif, system-ui, sans-serif',
}}
>
{activeCaption?.text ?? ''}
</div>
)}
{chapters && chapters.length > 0 && (
<nav
aria-label="Chapters"
data-testid="bl-video-player-chapters"
style={{
display: 'flex',
flexWrap: 'wrap',
gap: 6,
fontFamily: 'ui-sans-serif, system-ui, sans-serif',
}}
>
{chapters.map((c) => (
<button
key={c.id}
type="button"
onClick={() => seek(c.startSeconds)}
data-testid="bl-video-player-chapter"
data-chapter-id={c.id}
style={{
padding: '4px 10px',
fontSize: 12,
fontWeight: 500,
borderRadius: 999,
border: '1px solid var(--bl-border, rgba(0,0,0,0.12))',
background: 'var(--bl-surface-card, #fff)',
color: 'var(--bl-text-secondary, #555)',
cursor: 'pointer',
}}
>
<span style={{ opacity: 0.6 }}>{formatTime(c.startSeconds)}</span>{' '}
{c.label}
</button>
))}
</nav>
)}
</div>
);
}
function formatTime(s: number): string {
if (!Number.isFinite(s) || s < 0) return '0:00';
const mm = Math.floor(s / 60);
const ss = Math.floor(s % 60);
return `${mm}:${ss.toString().padStart(2, '0')}`;
}

119
packages/media-ui/src/__tests__/media-ui.test.tsx Normal file
View File

@ -0,0 +1,119 @@
import { describe, it, expect, beforeEach } from 'vitest';
import { cleanup, fireEvent, render, screen } from '@testing-library/react';
import { AudioWaveform } from '../AudioWaveform.js';
import { ImageGenStream } from '../ImageGenStream.js';
import { VideoPlayer } from '../VideoPlayer.js';
beforeEach(() => cleanup());
describe('AudioWaveform', () => {
it('renders a canvas with the configured size', () => {
render(<AudioWaveform peaks={[0.1, 0.5, 0.9, 0.4]} width={320} height={64} />);
const root = screen.getByTestId('bl-audio-waveform');
const canvas = root.querySelector('canvas');
expect(canvas).not.toBeNull();
expect(canvas?.style.width).toBe('320px');
expect(canvas?.style.height).toBe('64px');
});
it('invokes onSeek with the clicked position', () => {
let seeked: number | null = null;
render(
<AudioWaveform
peaks={[0.1, 0.5, 0.9]}
width={200}
height={40}
onSeek={(p) => {
seeked = p;
}}
/>,
);
const canvas = screen.getByTestId('bl-audio-waveform').querySelector('canvas')!;
// Stub bounding rect to 200 px wide for the click math.
canvas.getBoundingClientRect = () =>
({
x: 0,
y: 0,
left: 0,
top: 0,
right: 200,
bottom: 40,
width: 200,
height: 40,
toJSON: () => '',
}) as DOMRect;
fireEvent.click(canvas, { clientX: 100, clientY: 20 });
expect(seeked).toBeCloseTo(0.5, 5);
});
it('renders a placeholder waveform when neither peaks nor audioUrl are supplied', () => {
render(<AudioWaveform bars={16} width={120} height={40} />);
expect(screen.getByTestId('bl-audio-waveform')).toBeDefined();
});
});
describe('ImageGenStream', () => {
it('idle status renders the idle placeholder', () => {
render(<ImageGenStream status="idle" />);
expect(screen.getByTestId('bl-image-gen-stream-idle')).toBeDefined();
});
it('streaming status renders the progress strip + image', () => {
render(
<ImageGenStream
status="streaming"
snapshot={{ progress: 0.4, partialUrl: 'data:image/gif;base64,R0lGODlh', label: '20/50' }}
/>,
);
expect(screen.getByTestId('bl-image-gen-stream-progress')).toBeDefined();
expect(screen.getByTestId('bl-image-gen-stream-img')).toBeDefined();
expect(screen.getByTestId('bl-image-gen-stream').getAttribute('data-status')).toBe('streaming');
});
it('complete status drops the blur', () => {
render(<ImageGenStream status="complete" finalUrl="data:image/gif;base64,R0lGODlh" />);
const img = screen.getByTestId('bl-image-gen-stream-img') as HTMLImageElement;
expect(img.style.filter).toBe('none');
});
it('error status renders the error placeholder', () => {
render(<ImageGenStream status="error" errorMessage="boom" />);
const e = screen.getByTestId('bl-image-gen-stream-error');
expect(e.textContent).toMatch(/boom/);
});
});
describe('VideoPlayer', () => {
it('renders the native <video> element with the src', () => {
render(<VideoPlayer src="https://example.com/demo.mp4" />);
const v = screen.getByTestId('bl-video-player-video') as HTMLVideoElement;
expect(v.src).toMatch(/demo\.mp4$/);
});
it('renders chapter buttons that match the chapter list', () => {
render(
<VideoPlayer
src="x.mp4"
chapters={[
{ id: 'intro', label: 'Intro', startSeconds: 0 },
{ id: 'demo', label: 'Demo', startSeconds: 65 },
{ id: 'qa', label: 'Q&A', startSeconds: 360 },
]}
/>,
);
expect(screen.getAllByTestId('bl-video-player-chapter')).toHaveLength(3);
});
it('renders the caption rail when captions are supplied', () => {
render(
<VideoPlayer
src="x.mp4"
captions={[
{ id: 'c1', start: 0, end: 5, text: 'Hi there.' },
]}
/>,
);
expect(screen.getByTestId('bl-video-player-caption')).toBeDefined();
});
});

28
packages/media-ui/src/index.ts Normal file
View File

@ -0,0 +1,28 @@
/**
* @bytelyst/media-ui — media surface primitives.
*
* Exports (0.1.0 — Wave 13.G.1/.2/.4):
* <ImageGenStream> streaming image-generation surface
* <AudioWaveform> canvas waveform + click-to-seek
* <VideoPlayer> <video controls> + chapters + caption rail
*
* Deferred to 0.2.x (per roadmap §13.G):
* <PdfPreview> lazy pdf.js — needs the pdf.js runtime
*/
export { ImageGenStream } from './ImageGenStream.js';
export type {
ImageGenSnapshot,
ImageGenStatus,
ImageGenStreamProps,
} from './ImageGenStream.js';
export { AudioWaveform } from './AudioWaveform.js';
export type { AudioWaveformProps } from './AudioWaveform.js';
export { VideoPlayer } from './VideoPlayer.js';
export type {
VideoCaption,
VideoChapter,
VideoPlayerProps,
} from './VideoPlayer.js';

11
packages/media-ui/tsconfig.json Normal file
View File

@ -0,0 +1,11 @@
{
"extends": "../../tsconfig.base.json",
"compilerOptions": {
"outDir": "dist",
"rootDir": "src",
"lib": ["ES2022", "DOM"],
"jsx": "react-jsx"
},
"include": ["src"],
"exclude": ["src/**/*.test.ts", "src/**/*.test.tsx"]
}

2
packages/media-ui/vitest.config.ts Normal file
View File

@ -0,0 +1,2 @@
import { defineConfig } from 'vitest/config';
export default defineConfig({ test: { environment: 'happy-dom', pool: 'forks' } });