From 8287a6302655b31f52a4a59819d79e8e81d8a95d Mon Sep 17 00:00:00 2001 From: Vanilagy <1696106+Vanilagy@users.noreply.github.com> Date: Wed, 4 Jun 2025 10:31:20 +0200 Subject: [PATCH] Add procedural generation example --- docs/examples.md | 6 +- docs/index.md | 2 +- examples/file-compression/index.html | 2 +- examples/media-player/index.html | 2 +- examples/metadata-extraction/index.html | 2 +- examples/procedural-generation/index.html | 61 +++ .../procedural-generation.ts | 504 ++++++++++++++++++ 7 files changed, 573 insertions(+), 6 deletions(-) create mode 100644 examples/procedural-generation/index.html create mode 100644 examples/procedural-generation/procedural-generation.ts diff --git a/docs/examples.md b/docs/examples.md index c5d7354..b83a28e 100644 --- a/docs/examples.md +++ b/docs/examples.md @@ -22,8 +22,10 @@ features: details: Convert an input file to a highly-compressed MP4 file. link: /examples/file-compression target: _self - - title: Feature B - details: Lorem ipsum dolor sit amet, consectetur adipiscing elit + - title: Procedural video generation + details: Generate a video file as fast as the hardware allows + link: /examples/procedural-generation + target: _self - title: Feature C details: Lorem ipsum dolor sit amet, consectetur adipiscing elit --- diff --git a/docs/index.md b/docs/index.md index c47a43b..8a7767e 100644 --- a/docs/index.md +++ b/docs/index.md @@ -42,6 +42,6 @@ Examples: ✅ Rendering thumbnails for a video / extracting a wave form? ✅ Media player ✅ Conversion demo: File compressor -- Procedurally generated video (faster than real time) +✅ Procedurally generated video (faster than real time) - Video generated from live sources - Live streaming demo? \ No newline at end of file diff --git a/examples/file-compression/index.html b/examples/file-compression/index.html index 8e667c5..fb1ae13 100644 --- a/examples/file-compression/index.html +++ b/examples/file-compression/index.html @@ -15,7 +15,7 @@

Select or drop a media file, and Mediakit will convert it to a heavily-compressed MP4 file.

- diff --git a/examples/media-player/index.html b/examples/media-player/index.html index bba598c..ea21eb6 100644 --- a/examples/media-player/index.html +++ b/examples/media-player/index.html @@ -15,7 +15,7 @@

Select or drop a media file, and a fully custom, Mediakit-powered player will appear.

- diff --git a/examples/metadata-extraction/index.html b/examples/metadata-extraction/index.html index e1847bf..d3f5f70 100644 --- a/examples/metadata-extraction/index.html +++ b/examples/metadata-extraction/index.html @@ -15,7 +15,7 @@

Select or drop a media file, and Mediakit will start extracting various metadata about that file.

- diff --git a/examples/procedural-generation/index.html b/examples/procedural-generation/index.html new file mode 100644 index 0000000..627e898 --- /dev/null +++ b/examples/procedural-generation/index.html @@ -0,0 +1,61 @@ + + + + + + + Procedural Generation example | Mediakit + + + + + + +

Procedural generation example

+

Using Mediakit, this page will procedurally generate a video of musical bouncing balls as fast as possible.

+ +
+
+ + +

+
+ +
+ + +

+
+ + +
+ + + +

+ + + + + + + +
+ +

Mediakit

+
+ + + +

View source code

+ + + \ No newline at end of file diff --git a/examples/procedural-generation/procedural-generation.ts b/examples/procedural-generation/procedural-generation.ts new file mode 100644 index 0000000..37b970c --- /dev/null +++ b/examples/procedural-generation/procedural-generation.ts @@ -0,0 +1,504 @@ +import { + Output, + BufferTarget, + Mp4OutputFormat, + CanvasSource, + AudioBufferSource, + QUALITY_HIGH, +} from 'mediakit'; + +const durationSlider = document.querySelector('#duration-slider') as HTMLInputElement; +const durationValue = document.querySelector('#duration-value') as HTMLParagraphElement; +const ballsSlider = document.querySelector('#balls-slider') as HTMLInputElement; +const ballsValue = document.querySelector('#balls-value') as HTMLParagraphElement; +const renderButton = document.querySelector('#render-button') as HTMLButtonElement; +const horizontalRule = document.querySelector('hr') as HTMLHRElement; +const progressBarContainer = document.querySelector('#progress-bar-container') as HTMLDivElement; +const progressBar = document.querySelector('#progress-bar') as HTMLDivElement; +const progressText = document.querySelector('#progress-text') as HTMLParagraphElement; +const resultVideo = document.querySelector('#result-video') as HTMLVideoElement; +const videoInfo = document.querySelector('#video-info') as HTMLParagraphElement; +const errorElement = document.querySelector('#error-element') as HTMLParagraphElement; + +// We render using OffscreenCanvas, but a canvas element would also do +const renderCanvas = new OffscreenCanvas(1280, 720); +const renderCtx = renderCanvas.getContext('2d', { alpha: false })!; + +// Stuff we need for rendering audio +let offlineAudioContext: OfflineAudioContext; +let offlineGlobalGainNode: GainNode; +let offlineDryGain: GainNode; +let offlineWetGain: GainNode; +let offlineReverbConvolver: ConvolverNode; + +// Scales are defined as semitone offsets from A440 +const scaleProgression = [ + [-24, -12, -10, -8, -7, -5, -3, -1, 0, 2, 4, 5, 7, 9, 11, 12], // Ab13#11 + [-22, -12, -10, -8, -7, -5, -3, -1, 0, 2, 4, 5, 7, 9, 11, 14], // Bb13#11 + [-29, -17, -15, -13, -12, -10, -8, -5, -3, -1, 0, 2, 4, 5, 7, 16], // Dbmaj13#11 + [-26, -14, -12, -10, -9, -7, -5, -2, 0, 2, 3, 5, 7, 9, 10, 12], // Fmaj9/13 +]; +const scaleHues = [ + 215, + 151, + 273, + 335, +]; + +const wallWidth = 10; +const frameRate = 60; +const sampleRate = 48000; + +let balls: Ball[] = []; +let currentScaleIndex = 0; +let collisionCount = 0; +let collisionsPerScale = 0; + +/** === MAIN VIDEO FILE GENERATION LOGIC === */ + +const generateVideo = async () => { + let progressInterval = -1; + + try { + // Let's set some DOM state + renderButton.disabled = true; + renderButton.textContent = 'Generating...'; + horizontalRule.style.display = ''; + progressBarContainer.style.display = ''; + progressText.style.display = ''; + progressText.textContent = 'Initializing...'; + resultVideo.style.display = 'none'; + videoInfo.style.display = 'none'; + errorElement.textContent = ''; + + const duration = Number(durationSlider.value); + const totalFrames = duration * frameRate; + + // Let's init the scene + initScene(duration); + + // Create a new output file + const output = new Output({ + target: new BufferTarget(), // Stored in memory + format: new Mp4OutputFormat(), + }); + + // For video, we use a CanvasSource for convenience, as we're rendering to a canvas + const canvasSource = new CanvasSource(renderCanvas, { + codec: 'avc', + bitrate: QUALITY_HIGH, + }); + output.addVideoTrack(canvasSource); + + // For audio, we use ArrayBufferSource, because we'll be creating an ArrayBuffer with OfflineAudioContext + const audioBufferSource = new AudioBufferSource({ + codec: 'aac', + bitrate: QUALITY_HIGH, + }); + output.addAudioTrack(audioBufferSource); + + await output.start(); + + let currentFrame = 0; + + // Start an interval that updates the progress bar + progressInterval = window.setInterval(() => { + const videoProgress = currentFrame / totalFrames; + const overallProgress = videoProgress * 0.9; // 90% when 100% of the video has been rendered + progressBar.style.width = `${overallProgress * 100}%`; + + if (currentFrame === totalFrames) { + progressText.textContent = 'Rendering audio...'; + } else { + progressText.textContent = `Rendering frame ${currentFrame}/${totalFrames}`; + } + }, 1000 / 60); + + // Now, let's crank through all frames in a tight loop and render them as fast as possible + for (currentFrame; currentFrame < totalFrames; currentFrame++) { + const currentTime = currentFrame / frameRate; + + // Update the scene + updateScene(currentTime); + + // Add the current state of the canvas as a frame to the video. Using `await` here is crucial to + // automatically slow down the rendering loop when the encoder can't keep up. + await canvasSource.add(currentTime, 1 / frameRate); + } + + // Signal to the output that no more video frames are coming (not necessary, but recommended) + void canvasSource.close(); + + // Let's render the audio. Ideally, the audio is rendered before the video (or concurrently to it), but for + // simplicity, we're rendering it after we've cranked through all frames. + const audioBuffer = await offlineAudioContext.startRendering(); + await audioBufferSource.add(audioBuffer); + void audioBufferSource.close(); + + clearInterval(progressInterval); + + // Finalize the file + progressText.textContent = 'Finalizing file...'; + progressBar.style.width = '95%'; + await output.finalize(); + + // The file is now ready! + + progressBar.style.width = '100%'; + progressBarContainer.style.display = 'none'; + progressText.style.display = 'none'; + resultVideo.style.display = ''; + videoInfo.style.display = ''; + + // Display and play the resulting media file + const videoBlob = new Blob([output.target.buffer!], { type: 'video/mp4' }); + resultVideo.src = URL.createObjectURL(videoBlob); + void resultVideo.play(); + + const fileSizeMiB = (videoBlob.size / (1024 * 1024)).toPrecision(3); + videoInfo.textContent = `File size: ${fileSizeMiB} MiB`; + } catch (error) { + clearInterval(progressInterval); + errorElement.textContent = String(error); + progressBarContainer.style.display = 'none'; + progressText.style.display = 'none'; + } finally { + renderButton.disabled = false; + renderButton.textContent = 'Generate video'; + } +}; + +/** === SCENE SIMULATION LOGIC === */ + +const initScene = (duration: number) => { + offlineAudioContext = new OfflineAudioContext(2, duration * sampleRate, sampleRate); + + // Create reverb effect + offlineReverbConvolver = offlineAudioContext.createConvolver(); + offlineReverbConvolver.buffer = createReverbImpulse(5); + + offlineGlobalGainNode = offlineAudioContext.createGain(); + offlineDryGain = offlineAudioContext.createGain(); + offlineWetGain = offlineAudioContext.createGain(); + + offlineGlobalGainNode.connect(offlineDryGain); + offlineGlobalGainNode.connect(offlineReverbConvolver); + offlineReverbConvolver.connect(offlineWetGain); + offlineDryGain.connect(offlineAudioContext.destination); + offlineWetGain.connect(offlineAudioContext.destination); + + offlineGlobalGainNode.gain.setValueAtTime(0.8, 0); + offlineDryGain.gain.setValueAtTime(0.5, 0); + offlineWetGain.gain.setValueAtTime(0.5, 0); + + // Let's init the balls + const numBalls = Number(ballsSlider.value); + balls = []; + collisionsPerScale = Math.max(10, Math.ceil(numBalls ** 1.5)); + collisionCount = 0; + currentScaleIndex = 0; + + for (let i = 0; i < numBalls; i++) { + const scaleIndex = Math.floor(Math.random() * scaleProgression[0]!.length); + const tempFreq = getFrequencyFromScaleIndex(scaleIndex); + const baseFreq = semitoneToFreq(0); + const radius = 30 * baseFreq / tempFreq; + + // Let's find a new spot for this ball where it doesn't overlap with any other ball + let x: number, y: number; + let attempts = 0; + do { + x = radius + Math.random() * (renderCanvas.width - 2 * radius); + y = radius + Math.random() * (renderCanvas.height - 2 * radius); + attempts++; + } while (attempts < 100 && balls.some((ball) => { + const dx = x - ball.x; + const dy = y - ball.y; + const distanceSquared = dx * dx + dy * dy; + return distanceSquared < (radius + ball.radius + 10) ** 2; + })); + + balls.push(new Ball(x, y, scaleIndex)); + } +}; + +const updateScene = (currentTime: number) => { + renderCtx.clearRect(0, 0, renderCanvas.width, renderCanvas.height); + + // Draw the walls + renderCtx.beginPath(); + renderCtx.rect(0, 0, wallWidth, renderCanvas.height); + renderCtx.rect(renderCanvas.width - wallWidth, 0, wallWidth, renderCanvas.height); + renderCtx.rect(0, 0, renderCanvas.width, wallWidth); + renderCtx.rect(0, renderCanvas.height - wallWidth, renderCanvas.width, wallWidth); + renderCtx.fillStyle = '#27272a'; + renderCtx.fill(); + + // Update balls + for (const ball of balls) { + ball.update(currentTime, renderCanvas.width, renderCanvas.height); + } + + // Check collisions between all pairs of balls + for (let i = 0; i < balls.length - 1; i++) { + for (let j = i + 1; j < balls.length; j++) { + const ballI = balls[i]!; + const ballJ = balls[j]!; + + if (ballI.checkCollision(ballJ)) { + ballI.collideWith(ballJ); + ballI.lastHitTime = currentTime; + ballJ.lastHitTime = currentTime; + ballI.scheduleSound(currentTime); + ballJ.scheduleSound(currentTime); + + collisionCount++; + } + } + } + + // Check if it's time to change scale + if (collisionCount >= collisionsPerScale) { + currentScaleIndex = (currentScaleIndex + 1) % scaleProgression.length; + collisionCount = 0; + } + + // Draw balls + for (const ball of balls) { + ball.draw(renderCtx, currentTime); + } +}; + +const ballHitAnimationDuration = 0.2; + +class Ball { + x: number; + y: number; + scaleIndex: number; + vx: number; + vy: number; + lastHitTime: number; + radius: number; + + constructor(x: number, y: number, scaleIndex: number) { + this.x = x; + this.y = y; + this.scaleIndex = scaleIndex; + this.vx = (Math.random() - 0.5) * 700; // Random velocity + this.vy = (Math.random() - 0.5) * 700; + this.lastHitTime = -Infinity; + + const baseRadius = 40; + this.radius = lerp( + 2 * baseRadius, + 0.3 * baseRadius, + (this.scaleIndex / (scaleProgression[0]!.length - 1)) ** 0.5, + ); + } + + getColorFromScale() { + // Lower indices (bass notes) are darker, higher are brighter + const hue = scaleHues[currentScaleIndex]; + const lightness = 25 + (this.scaleIndex / 15) * 50; + return `hsl(${hue}, 50%, ${lightness}%)`; + } + + update(currentTime: number, canvasWidth: number, canvasHeight: number) { + // Integrate + this.x += this.vx / frameRate; + this.y += this.vy / frameRate; + + let wallHit = false; + + // Wall collisions + if (this.x - this.radius <= wallWidth || this.x + this.radius >= canvasWidth - wallWidth) { + this.vx = -this.vx; + this.x = Math.max(wallWidth + this.radius, Math.min(canvasWidth - wallWidth - this.radius, this.x)); + wallHit = true; + collisionCount++; + } + if (this.y - this.radius <= wallWidth || this.y + this.radius >= canvasHeight - wallWidth) { + this.vy = -this.vy; + this.y = Math.max(wallWidth + this.radius, Math.min(canvasHeight - wallWidth - this.radius, this.y)); + wallHit = true; + collisionCount++; + } + + if (wallHit) { + this.lastHitTime = currentTime; + this.scheduleSound(currentTime); + } + } + + draw(ctx: OffscreenCanvasRenderingContext2D, currentTime: number) { + const timeSinceHit = currentTime - this.lastHitTime; + const progress = clamp(timeSinceHit / ballHitAnimationDuration, 0, 1); + + const radius = this.radius * (1 + (1 - progress) * 0.1); + + ctx.beginPath(); + ctx.arc(this.x, this.y, radius, 0, Math.PI * 2); + + const color = this.getColorFromScale(); + + ctx.fillStyle = color; + ctx.globalAlpha = 0.333; + ctx.fill(); + + const lineWidth = 0.15 * radius; + ctx.beginPath(); + ctx.arc(this.x, this.y, radius - lineWidth / 2, 0, Math.PI * 2); + + ctx.globalAlpha = 1; + ctx.strokeStyle = color; + ctx.lineWidth = lineWidth; + ctx.stroke(); + + if (progress < 1) { + const intensity = 1 - progress; + ctx.globalAlpha = intensity * 0.8; + ctx.strokeStyle = 'white'; + ctx.stroke(); + } + + ctx.globalAlpha = 1; + } + + checkCollision(other: Ball) { + const dx = this.x - other.x; + const dy = this.y - other.y; + const distance = Math.sqrt(dx * dx + dy * dy); + return distance < (this.radius + other.radius); + } + + collideWith(other: Ball) { + const dx = other.x - this.x; + const dy = other.y - this.y; + const distance = Math.hypot(dx, dy); + + const nx = dx / distance; + const ny = dy / distance; + + const rvx = other.vx - this.vx; + const rvy = other.vy - this.vy; + + // Relative velocity along collision normal + const speed = rvx * nx + rvy * ny; + + if (speed > 0) { + // Objects separating + return; + } + + const thisMass = this.radius ** 2; + const otherMass = other.radius ** 2; + + const impulse = 2 * speed / (thisMass + otherMass); + + this.vx += impulse * otherMass * nx; + this.vy += impulse * otherMass * ny; + other.vx -= impulse * thisMass * nx; + other.vy -= impulse * thisMass * ny; + + const overlap = (this.radius + other.radius) - distance; + if (overlap > 0) { + // Separate balls + const separateX = nx * overlap * 0.5; + const separateY = ny * overlap * 0.5; + + this.x -= separateX; + this.y -= separateY; + other.x += separateX; + other.y += separateY; + } + } + + scheduleSound(currentTime: number) { + const oscillator = offlineAudioContext.createOscillator(); + const gainNode = offlineAudioContext.createGain(); + const pannerNode = offlineAudioContext.createStereoPanner(); + + // Calculate pan based on x position + const panValue = ((this.x / renderCanvas.width) - 0.5) * Math.SQRT2; + + oscillator.connect(gainNode); + gainNode.connect(pannerNode); + pannerNode.connect(offlineGlobalGainNode); + + const frequency = getFrequencyFromScaleIndex(this.scaleIndex); + oscillator.frequency.setValueAtTime(frequency, currentTime); + oscillator.type = 'sine'; + pannerNode.pan.setValueAtTime(panValue, currentTime); + + // Create a nice envelope + gainNode.gain.setValueAtTime(0, currentTime); + gainNode.gain.linearRampToValueAtTime(0.08, currentTime + 0.01); + gainNode.gain.exponentialRampToValueAtTime(0.001, currentTime + 0.4); + gainNode.gain.linearRampToValueAtTime(0, currentTime + 0.5); + + oscillator.start(currentTime); + oscillator.stop(currentTime + 0.6); + } +} + +/** === UTILS === */ + +const clamp = (value: number, min: number, max: number) => { + return Math.min(Math.max(value, min), max); +}; + +const lerp = (a: number, b: number, t: number) => { + return a + (b - a) * t; +}; + +const semitoneToFreq = (semitones: number) => { + return 440 * Math.pow(2, semitones / 12); // Equal temperament +}; + +const getCurrentScale = () => { + return scaleProgression[currentScaleIndex]!; +}; + +const getFrequencyFromScaleIndex = (scaleIndex: number) => { + const currentScale = getCurrentScale(); + return semitoneToFreq(currentScale[scaleIndex]!); +}; + +const createReverbImpulse = (duration: number) => { + const length = sampleRate * duration; + const impulse = offlineAudioContext.createBuffer(2, length, sampleRate); + + for (let channel = 0; channel < 2; channel++) { + const channelData = impulse.getChannelData(channel); + for (let i = 0; i < length; i++) { + const decayFactor = Math.pow(0.001, i / length); + + // Add multiple delay lines for rich reverb + let sample = 0; + sample += (Math.random() * 2 - 1) * decayFactor; + sample += (Math.random() * 2 - 1) * decayFactor * 0.7; + sample += (Math.random() * 2 - 1) * decayFactor * 0.5; + channelData[i] = sample * 0.5; // Scale down + } + } + + return impulse; +}; + +/** === DOM LOGIC === */ + +// Update slider displays +const updateSliderDisplays = () => { + durationValue.textContent = `${durationSlider.value} seconds`; + ballsValue.textContent = `${ballsSlider.value} ${ballsSlider.value === '1' ? 'ball' : 'balls'}`; +}; + +durationSlider.addEventListener('input', updateSliderDisplays); +ballsSlider.addEventListener('input', updateSliderDisplays); + +// Initialize displays +updateSliderDisplays(); + +renderButton.addEventListener('click', () => { + void generateVideo(); +});