From 6dc4c2d49907ddfea0a63de14e214f0e8da1ab25 Mon Sep 17 00:00:00 2001 From: Vanilagy <1696106+Vanilagy@users.noreply.github.com> Date: Thu, 16 Apr 2026 16:19:08 +0200 Subject: [PATCH] More doc adjustments, some type adjustments --- docs/guide/quick-start.md | 49 ++++++++++++- docs/guide/reading-hls.md | 48 ++++++++++++ docs/guide/writing-hls.md | 57 ++++++++++++++- src/conversion.ts | 13 +--- src/encode.ts | 150 +++++++++++++++++++++----------------- src/index.ts | 2 + 6 files changed, 238 insertions(+), 81 deletions(-) diff --git a/docs/guide/quick-start.md b/docs/guide/quick-start.md index a38eb38..b507fe9 100644 --- a/docs/guide/quick-start.md +++ b/docs/guide/quick-start.md @@ -625,4 +625,51 @@ const conversion = await Conversion.init({ await conversion.execute(); // Conversion is complete -``` \ No newline at end of file +``` + +## Reading HLS playlists + +```ts +import { createInputFrom, HLS_FORMATS, desc } from 'mediabunny'; + +const input = createInputFrom('https://example.com/master.m3u8', HLS_FORMATS); + +// Get all tracks +const tracks = await input.getTracks(); + +// Get video tracks by quality +const sortedVideoTracks = await input.getVideoTracks({ + sortBy: async track => desc(await track.getDisplayHeight()), +}); + +// Select a quality +const bestVideoTrack = sortedVideoTracks[0]!; +// Get a matching audio track +const matchingAudioTrack = await bestVideoTrack.getPrimaryPairableAudioTrack(); + +// HLS tracks can be read like any other Mediabunny InputTrack +// ... + +const isLive = await bestVideoTrack.isLive(); +if (isLive) { + // Poll some data using the refresh interval, for example duration + let currentDuration: number | null = null; + const poll = async () => { + currentDuration = await bestVideoTrack.getDurationFromMetadata({ + skipLiveWait: true, + }); + + const refreshInterval = await bestVideoTrack.getLiveRefreshInterval(); + if (refreshInterval === null) { + return; // No longer live + } + + setTimeout(poll, 1000 * refreshInterval); + }; + await poll(); +} +``` + +::: info +See [Reading HLS](./reading-hls) for an in-depth guide. +::: \ No newline at end of file diff --git a/docs/guide/reading-hls.md b/docs/guide/reading-hls.md index f1d4efc..86ffa2c 100644 --- a/docs/guide/reading-hls.md +++ b/docs/guide/reading-hls.md @@ -46,6 +46,50 @@ Since Mediabunny lazy-loads all information, retrieving the list of all tracks i If you point Mediabunny directly at a media playlist, then the list of tracks is deduced by the tracks present in the first segment of the playlist. +### Track queries + +Mediabunny offers an [`InputTrackQuery`](../api/InputTrackQuery) system to aid with finding tracks in complex many-track inputs such as the ones common with HLS. + +Here's a holistic example that constructs an HLS quality ladder and then selects audio tracks based on language: + +```ts +import { desc, prefer } from 'mediabunny'; + +// Get video tracks sorted by their resolution (highest first) +const videoTracks = await input.getVideoTracks({ + sortBy: async track => [ + desc(await track.getDisplayHeight()), + // Tracks with matching resolution are sorted by bitrate + desc(await track.getBitrate()), + ], + // Filter out #EXT-X-I-FRAME-STREAM-INF tracks + filter: async track => !(await track.hasOnlyKeyPackets()), +}); +const availableQualities = videoTracks.length; + +// Get the best quality video track +const video = videoTracks[0]; + +// Get the audio track that accompanies this video track +const matchingAudioTrack = await video.getPrimaryPairableAudioTrack(); + +// Get all audio tracks +const availableAudioTracks = await video.getPairableAudioTracks(); +const availableLanguages = await Promise.all( + availableAudioTracks.map(track => track.getLanguageCode()), +); + +// Get the Spanish audio track +const matchingSpanishAudio = await video.getPrimaryPairableAudioTrack({ + filter: async track => await track.getLanguageCode() === 'es', +}); +// Get the Spanish audio track if one exists, otherwise get +// the next best thing +const matchingAudioPreferSpanish = await video.getPrimaryPairableAudioTrack({ + sortBy: async track => prefer(await track.getLanguageCode() === 'es'), +}); +``` + ### Reading track metadata For general reading of track metadata, see [Reading media files](./reading-media-files). Some notable metadata mappings for HLS are: @@ -291,3 +335,7 @@ const playbackStartTime = currentDuration! - fac * refreshInterval!; You can lower `fac` to move playback closer to the live edge (1.5 works fine too), although you should not lower it below 1. You can make `fac` larger to increase resilience against flaky internet or an unreliable media producer. + +## Subtitles + +Reading subtitles from HLS playlists is not currently supported. Sorry! \ No newline at end of file diff --git a/docs/guide/writing-hls.md b/docs/guide/writing-hls.md index 14657b2..d3a5080 100644 --- a/docs/guide/writing-hls.md +++ b/docs/guide/writing-hls.md @@ -173,6 +173,57 @@ await output.start(); Then, add media data like normal. Since Mediabunny takes care of segmentation automatically, it is required to perform [Packet buffering](./writing-media-files#packet-buffering) internally. Therefore, try writing media data in a quasi-interleaved fashion to keep memory usage bounded. +### Multiple resolutions + +A common pattern is to offer the same content in multiple resolutions and bitrates. This requires encoding the content multiple times, sometimes with additional downscaling. Mediabunny's [media sources](./media-sources) make this pattern a breeze. + +Let's suppose we have a 1080p main video stream. We want to provide a 1080p, 720p, 480p, and 360p variant in the HLS playlist. For that, use this pattern: +```ts +const source1080p = new VideoSampleSource({ + codec: 'avc', + bitrate: QUALITY_VERY_HIGH, +}); +const source720p = new VideoSampleSource({ + codec: 'avc', + bitrate: QUALITY_HIGH, + transform: { + // Frames will be automatically resized to 720p before being encoded + height: 720, + }, +}); +const source480p = new VideoSampleSource({ + codec: 'avc', + bitrate: QUALITY_MEDIUM, + transform: { + height: 480, + }, +}); +const source360p = new VideoSampleSource({ + codec: 'avc', + bitrate: QUALITY_LOW, + transform: { + height: 360, + }, +}); + +const sources = [source1080p, source720p, source480p, source360p]; +for (const source of sources) { + output.addVideoTrack(source); +} + +await output.start(); + +// Then, when adding a new frame: +const sample = // ... +for (const source of sources) { + await source.add(sample); +} +``` + +You can extend this pattern to offer content in multiple codecs as well. + +For the full list of transformation options, see [`VideoTransformOptions`](../api/VideoTransformOptions) and [`AudioTransformOptions`](../api/AudioTransformOptions). + ### Track metadata Often you'll want to provide additional [track metadata](../api/BaseTrackMetadata) when dealing with multiple tracks. For example: @@ -577,4 +628,8 @@ Whenever a segment is popped off the playlist in this fashion, the `onSegmentPop ::: info `onSegmentPopped` is not called when `singleFilePerPlaylist` is enabled. -::: \ No newline at end of file +::: + +## Subtitles + +Writing subtitles to HLS playlists is not currently supported. Sorry! \ No newline at end of file diff --git a/src/conversion.ts b/src/conversion.ts index 5f765ab..d6c05b9 100644 --- a/src/conversion.ts +++ b/src/conversion.ts @@ -50,7 +50,7 @@ import { } from './misc'; import { Output, OutputTrackGroup, TrackType } from './output'; import { Mp4OutputFormat } from './output-format'; -import { AudioSample, clampCropRectangle, validateCropRectangle, VideoSample } from './sample'; +import { AudioSample, clampCropRectangle, CropRectangle, validateCropRectangle, VideoSample } from './sample'; import { MetadataTags, validateMetadataTags } from './metadata'; import { NullTarget } from './target'; import { AudioResampler } from './resample'; @@ -183,16 +183,7 @@ export type ConversionVideoOptions = { * Specifies the rectangular region of the input video to crop to. The crop region will automatically be clamped to * the dimensions of the input video track. Cropping is performed after rotation but before resizing. */ - crop?: { - /** The distance in pixels from the left edge of the source frame to the left edge of the crop rectangle. */ - left: number; - /** The distance in pixels from the top edge of the source frame to the top edge of the crop rectangle. */ - top: number; - /** The width in pixels of the crop rectangle. */ - width: number; - /** The height in pixels of the crop rectangle. */ - height: number; - }; + crop?: CropRectangle; /** * The desired frame rate of the output video, in hertz. If not specified, the original input frame rate will * be used (which may be variable). diff --git a/src/encode.ts b/src/encode.ts index cd27631..ad4381d 100644 --- a/src/encode.ts +++ b/src/encode.ts @@ -65,59 +65,7 @@ export type VideoEncodingConfig = { /** * Optional transformations to apply to the video frames before they are passed to the encoder. */ - transform?: { - /** - * The width in pixels to resize the frames to. If height is not set, it will be deduced - * automatically based on aspect ratio. - */ - width?: number; - /** - * The height in pixels to resize the frames to. If width is not set, it will be deduced - * automatically based on aspect ratio. - */ - height?: number; - /** - * The fitting algorithm in case both width and height are set. - * - * - `'fill'` will stretch the image to fill the entire box, potentially altering aspect ratio. - * - `'contain'` will contain the entire image within the box while preserving aspect ratio. This may lead to - * letterboxing. - * - `'cover'` will scale the image until the entire box is filled, while preserving aspect ratio. - * - * To avoid ambiguity, this field must not be set when `sizeChangeBehavior` is `'fill'`, `'contain'` or - * `'deny'`, since `sizeChangeBehavior` already determines the fitting algorithm. - */ - fit?: 'fill' | 'contain' | 'cover'; - /** - * The clockwise rotation by which to rotate the frames. Rotation is applied before resizing. - */ - rotate?: Rotation; - /** - * Specifies the rectangular region of the frames to crop to. The crop region will automatically be - * clamped to the dimensions of the frame. Cropping is performed after rotation but before resizing. - */ - crop?: CropRectangle; - /** - * The frame rate in hertz to normalize the video frame stream to. - */ - frameRate?: number; - /** - * Allows for custom user-defined processing of video frames, e.g. for applying overlays, color transformations, - * or timestamp modifications. Will be called for each video frame after transformations and frame rate - * corrections. - * - * Must return a {@link VideoSample} or a `CanvasImageSource`, an array of them, or `null` for dropping the - * frame. When non-timestamped data is returned, the timestamp and duration from the input sample will be used. - */ - process?: (sample: VideoSample) => MaybePromise< - CanvasImageSource | VideoSample | (CanvasImageSource | VideoSample)[] | null - >; - /** - * Forces every video frame through the transformation step even if no transformation properties are defined. - * This can be used, for example, to bake rotation into the encoded video frames. - */ - force?: boolean; - }; + transform?: VideoTransformOptions; /** Called for each successfully encoded packet. Both the packet and the encoding metadata are passed. */ onEncodedPacket?: (packet: EncodedPacket, meta: EncodedVideoChunkMetadata | undefined) => unknown; @@ -128,6 +76,65 @@ export type VideoEncodingConfig = { onEncoderConfig?: (config: VideoEncoderConfig) => unknown; } & VideoEncodingAdditionalOptions; +/** + * Options for transforming video frames before encoding. + * @group Encoding + * @public + */ +export type VideoTransformOptions = { + /** + * The width in pixels to resize the frames to. If height is not set, it will be deduced + * automatically based on aspect ratio. + */ + width?: number; + /** + * The height in pixels to resize the frames to. If width is not set, it will be deduced + * automatically based on aspect ratio. + */ + height?: number; + /** + * The fitting algorithm in case both width and height are set. + * + * - `'fill'` will stretch the image to fill the entire box, potentially altering aspect ratio. + * - `'contain'` will contain the entire image within the box while preserving aspect ratio. This may lead to + * letterboxing. + * - `'cover'` will scale the image until the entire box is filled, while preserving aspect ratio. + * + * To avoid ambiguity, this field must not be set when `sizeChangeBehavior` is `'fill'`, `'contain'` or + * `'deny'`, since `sizeChangeBehavior` already determines the fitting algorithm. + */ + fit?: 'fill' | 'contain' | 'cover'; + /** + * The clockwise rotation by which to rotate the frames. Rotation is applied before resizing. + */ + rotate?: Rotation; + /** + * Specifies the rectangular region of the frames to crop to. The crop region will automatically be + * clamped to the dimensions of the frame. Cropping is performed after rotation but before resizing. + */ + crop?: CropRectangle; + /** + * The frame rate in hertz to normalize the video frame stream to. + */ + frameRate?: number; + /** + * Allows for custom user-defined processing of video frames, e.g. for applying overlays, color transformations, + * or timestamp modifications. Will be called for each video frame after transformations and frame rate + * corrections. + * + * Must return a {@link VideoSample} or a `CanvasImageSource`, an array of them, or `null` for dropping the + * frame. When non-timestamped data is returned, the timestamp and duration from the input sample will be used. + */ + process?: (sample: VideoSample) => MaybePromise< + CanvasImageSource | VideoSample | (CanvasImageSource | VideoSample)[] | null + >; + /** + * Forces every video frame through the transformation step even if no transformation properties are defined. + * This can be used, for example, to bake rotation into the encoded video frames. + */ + force?: boolean; +}; + export const validateVideoEncodingConfig = (config: VideoEncodingConfig) => { if (!config || typeof config !== 'object') { throw new TypeError('Encoding config must be an object.'); @@ -358,21 +365,7 @@ export type AudioEncodingConfig = { /** * Optional transformations to apply to the audio samples before they are passed to the encoder. */ - transform?: { - /** The desired number of output channels to up/downmix to. */ - numberOfChannels?: number; - /** The desired output sample rate in hertz to resample to. */ - sampleRate?: number; - /** - * Allows for custom user-defined processing of audio samples, e.g. for applying audio effects or timestamp - * modifications. Called for each audio sample after resampling and remixing. - * - * Must return an {@link AudioSample}, an array of them, or `null` for dropping the sample. - */ - process?: (sample: AudioSample) => MaybePromise< - AudioSample | AudioSample[] | null - >; - }; + transform?: AudioTransformOptions; /** Called for each successfully encoded packet. Both the packet and the encoding metadata are passed. */ onEncodedPacket?: (packet: EncodedPacket, meta: EncodedAudioChunkMetadata | undefined) => unknown; @@ -383,6 +376,27 @@ export type AudioEncodingConfig = { onEncoderConfig?: (config: AudioEncoderConfig) => unknown; } & AudioEncodingAdditionalOptions; +/** + * Options for transforming audio samples before encoding. + * @group Encoding + * @public + */ +export type AudioTransformOptions = { + /** The desired number of output channels to up/downmix to. */ + numberOfChannels?: number; + /** The desired output sample rate in hertz to resample to. */ + sampleRate?: number; + /** + * Allows for custom user-defined processing of audio samples, e.g. for applying audio effects or timestamp + * modifications. Called for each audio sample after resampling and remixing. + * + * Must return an {@link AudioSample}, an array of them, or `null` for dropping the sample. + */ + process?: (sample: AudioSample) => MaybePromise< + AudioSample | AudioSample[] | null + >; +}; + export const validateAudioEncodingConfig = (config: AudioEncodingConfig) => { if (!config || typeof config !== 'object') { throw new TypeError('Encoding config must be an object.'); diff --git a/src/index.ts b/src/index.ts index 2a99be0..971361a 100644 --- a/src/index.ts +++ b/src/index.ts @@ -103,8 +103,10 @@ export { export { VideoEncodingConfig, VideoEncodingAdditionalOptions, + VideoTransformOptions, AudioEncodingConfig, AudioEncodingAdditionalOptions, + AudioTransformOptions, canEncode, canEncodeVideo, canEncodeAudio,