arcodange/mediabunny
1
0
Fork 0
mirror of https://github.com/arcodange-org/mediabunny.git synced 2026-10-01 12:53:49 +02:00
Code Issues Packages Projects Releases Wiki Activity

EncodedVideoSample/EncodedAudioSample -> EncodedPacket

Browse Source
This commit is contained in:
David Payr
2025-02-22 19:54:59 +01:00
parent 5d1866653c
commit c2b20ccda5
24 changed files with 650 additions and 628 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/isobmff/isobmff-boxes.ts
+10 -8
View File
@@ -287,14 +287,16 @@ export const ftyp = (details: {
]);
}
if (details.fragmented) return box('ftyp', [
ascii('iso5'), // Major brand
u32(minorVersion), // Minor version
// Compatible brands
ascii('iso5'),
ascii('iso6'),
ascii('mp41'),
]);
if (details.fragmented) {
return box('ftyp', [
ascii('iso5'), // Major brand
u32(minorVersion), // Minor version
// Compatible brands
ascii('iso5'),
ascii('iso6'),
ascii('mp41'),
]);
}
return box('ftyp', [
ascii('isom'), // Major brand
src/isobmff/isobmff-demuxer.ts
+48 -80
View File
@@ -21,7 +21,7 @@ import {
InputVideoTrack,
InputVideoTrackBacking,
} from '../input-track';
import { SampleRetrievalOptions } from '../media-sink';
import { PacketRetrievalOptions } from '../media-sink';
import {
assert,
COLOR_PRIMARIES_MAP_INVERSE,
@@ -40,8 +40,8 @@ import {
isIso639Dash2LanguageCode,
roundToMultiple,
} from '../misc';
import { EncodedPacket, PLACEHOLDER_DATA } from '../packet';
import { Reader } from '../reader';
import { EncodedAudioSample, EncodedVideoSample, PLACEHOLDER_DATA, SampleType } from '../sample';
import { IsobmffReader, MAX_BOX_HEADER_SIZE } from './isobmff-reader';
type InternalTrack = {
@@ -1768,11 +1768,9 @@ export class IsobmffDemuxer extends Demuxer {
}
}
abstract class IsobmffTrackBacking<
Sample extends EncodedVideoSample | EncodedAudioSample,
> implements InputTrackBacking {
sampleToSampleIndex = new WeakMap<Sample, number>();
sampleToFragmentLocation = new WeakMap<Sample, {
abstract class IsobmffTrackBacking implements InputTrackBacking {
packetToSampleIndex = new WeakMap<EncodedPacket, number>();
packetToFragmentLocation = new WeakMap<EncodedPacket, {
fragment: Fragment;
sampleIndex: number;
}>();
@@ -1796,25 +1794,16 @@ abstract class IsobmffTrackBacking<
}
async computeDuration() {
const lastSample = await this.getSample(Infinity, { metadataOnly: true });
return (lastSample?.timestamp ?? 0) + (lastSample?.duration ?? 0);
const lastPacket = await this.getPacket(Infinity, { metadataOnly: true });
return (lastPacket?.timestamp ?? 0) + (lastPacket?.duration ?? 0);
}
async getFirstTimestamp() {
const firstSample = await this.getFirstSample({ metadataOnly: true });
return firstSample?.timestamp ?? 0;
const firstPacket = await this.getFirstPacket({ metadataOnly: true });
return firstPacket?.timestamp ?? 0;
}
abstract createSample(
data: Uint8Array,
byteLength: number,
type: SampleType,
timestamp: number,
duration: number,
sequenceNumber: number
): Sample;
async getFirstSample(options: SampleRetrievalOptions) {
async getFirstPacket(options: PacketRetrievalOptions) {
if (this.internalTrack.demuxer.isFragmented) {
return this.performFragmentedLookup(
() => {
@@ -1852,7 +1841,7 @@ abstract class IsobmffTrackBacking<
);
}
return this.fetchSampleForSampleIndex(0, options);
return this.fetchPacketForSampleIndex(0, options);
}
private mapTimestampIntoTimescale(timestamp: number) {
@@ -1862,7 +1851,7 @@ abstract class IsobmffTrackBacking<
return roundToPrecision(timestamp * this.internalTrack.timescale, 14) + this.internalTrack.editListOffset;
}
async getSample(timestamp: number, options: SampleRetrievalOptions) {
async getPacket(timestamp: number, options: PacketRetrievalOptions) {
const timestampInTimescale = this.mapTimestampIntoTimescale(timestamp);
if (this.internalTrack.demuxer.isFragmented) {
@@ -1875,15 +1864,15 @@ abstract class IsobmffTrackBacking<
} else {
const sampleTable = this.internalTrack.demuxer.getSampleTableForTrack(this.internalTrack);
const sampleIndex = getSampleIndexForTimestamp(sampleTable, timestampInTimescale);
return this.fetchSampleForSampleIndex(sampleIndex, options);
return this.fetchPacketForSampleIndex(sampleIndex, options);
}
}
async getNextSample(sample: Sample, options: SampleRetrievalOptions) {
async getNextPacket(packet: EncodedPacket, options: PacketRetrievalOptions) {
if (this.internalTrack.demuxer.isFragmented) {
const locationInFragment = this.sampleToFragmentLocation.get(sample);
const locationInFragment = this.packetToFragmentLocation.get(packet);
if (locationInFragment === undefined) {
throw new Error('Sample was not created from this track.');
throw new Error('Packet was not created from this track.');
}
const trackData = locationInFragment.fragment.trackData.get(this.internalTrack.id)!;
@@ -1941,14 +1930,14 @@ abstract class IsobmffTrackBacking<
);
}
const sampleIndex = this.sampleToSampleIndex.get(sample);
const sampleIndex = this.packetToSampleIndex.get(packet);
if (sampleIndex === undefined) {
throw new Error('Sample was not created from this track.');
throw new Error('Packet was not created from this track.');
}
return this.fetchSampleForSampleIndex(sampleIndex + 1, options);
return this.fetchPacketForSampleIndex(sampleIndex + 1, options);
}
async getKeySample(timestamp: number, options: SampleRetrievalOptions) {
async getKeyPacket(timestamp: number, options: PacketRetrievalOptions) {
const timestampInTimescale = this.mapTimestampIntoTimescale(timestamp);
if (this.internalTrack.demuxer.isFragmented) {
@@ -1965,14 +1954,14 @@ abstract class IsobmffTrackBacking<
const keyFrameSampleIndex = sampleIndex === -1
? -1
: getRelevantKeyframeIndexForSample(sampleTable, sampleIndex);
return this.fetchSampleForSampleIndex(keyFrameSampleIndex, options);
return this.fetchPacketForSampleIndex(keyFrameSampleIndex, options);
}
async getNextKeySample(sample: Sample, options: SampleRetrievalOptions) {
async getNextKeyPacket(packet: EncodedPacket, options: PacketRetrievalOptions) {
if (this.internalTrack.demuxer.isFragmented) {
const locationInFragment = this.sampleToFragmentLocation.get(sample);
const locationInFragment = this.packetToFragmentLocation.get(packet);
if (locationInFragment === undefined) {
throw new Error('Sample was not created from this track.');
throw new Error('Packet was not created from this track.');
}
const trackData = locationInFragment.fragment.trackData.get(this.internalTrack.id)!;
@@ -2039,16 +2028,16 @@ abstract class IsobmffTrackBacking<
);
}
const sampleIndex = this.sampleToSampleIndex.get(sample);
const sampleIndex = this.packetToSampleIndex.get(packet);
if (sampleIndex === undefined) {
throw new Error('Sample was not created from this track.');
throw new Error('Packet was not created from this track.');
}
const sampleTable = this.internalTrack.demuxer.getSampleTableForTrack(this.internalTrack);
const nextKeyFrameSampleIndex = getNextKeyframeIndexForSample(sampleTable, sampleIndex);
return this.fetchSampleForSampleIndex(nextKeyFrameSampleIndex, options);
return this.fetchPacketForSampleIndex(nextKeyFrameSampleIndex, options);
}
private async fetchSampleForSampleIndex(sampleIndex: number, options: SampleRetrievalOptions) {
private async fetchPacketForSampleIndex(sampleIndex: number, options: PacketRetrievalOptions) {
if (sampleIndex === -1) {
return null;
}
@@ -2076,21 +2065,21 @@ abstract class IsobmffTrackBacking<
const timestamp = (sampleInfo.presentationTimestamp - this.internalTrack.editListOffset)
/ this.internalTrack.timescale;
const duration = sampleInfo.duration / this.internalTrack.timescale;
const sample = this.createSample(
const packet = new EncodedPacket(
data,
sampleInfo.sampleSize,
sampleInfo.isKeyFrame ? 'key' : 'delta',
timestamp,
duration,
sampleIndex,
sampleInfo.sampleSize,
);
this.sampleToSampleIndex.set(sample, sampleIndex);
this.packetToSampleIndex.set(packet, sampleIndex);
return sample;
return packet;
}
private async fetchSampleInFragment(fragment: Fragment, sampleIndex: number, options: SampleRetrievalOptions) {
private async fetchPacketInFragment(fragment: Fragment, sampleIndex: number, options: PacketRetrievalOptions) {
if (sampleIndex === -1) {
return null;
}
@@ -2113,18 +2102,18 @@ abstract class IsobmffTrackBacking<
const timestamp = (fragmentSample.presentationTimestamp - this.internalTrack.editListOffset)
/ this.internalTrack.timescale;
const duration = fragmentSample.duration / this.internalTrack.timescale;
const sample = this.createSample(
const packet = new EncodedPacket(
data,
fragmentSample.byteSize,
fragmentSample.isKeyFrame ? 'key' : 'delta',
timestamp,
duration,
fragment.moofOffset + sampleIndex,
fragmentSample.byteSize,
);
this.sampleToFragmentLocation.set(sample, { fragment, sampleIndex });
this.packetToFragmentLocation.set(packet, { fragment, sampleIndex });
return sample;
return packet;
}
private findSampleInFragmentsForTimestamp(timestampInTimescale: number) {
@@ -2183,13 +2172,13 @@ abstract class IsobmffTrackBacking<
return { fragmentIndex, sampleIndex, correctSampleFound };
}
/** Looks for a sample in the fragments while trying to load as few fragments as possible to retrieve it. */
/** Looks for a packet in the fragments while trying to load as few fragments as possible to retrieve it. */
private async performFragmentedLookup(
getBestMatch: () => { fragmentIndex: number; sampleIndex: number; correctSampleFound: boolean },
searchTimestamp: number,
latestTimestamp: number,
options: SampleRetrievalOptions,
): Promise<Sample | null> {
options: PacketRetrievalOptions,
): Promise<EncodedPacket | null> {
const demuxer = this.internalTrack.demuxer;
const release = await demuxer.fragmentLookupMutex.acquire(); // The algorithm requires exclusivity
@@ -2198,7 +2187,7 @@ abstract class IsobmffTrackBacking<
if (correctSampleFound) {
// The correct sample already exists, easy path.
const fragment = this.internalTrack.fragments[fragmentIndex]!;
return this.fetchSampleInFragment(fragment, sampleIndex, options);
return this.fetchPacketInFragment(fragment, sampleIndex, options);
}
const metadataReader = demuxer.metadataReader;
@@ -2284,7 +2273,7 @@ abstract class IsobmffTrackBacking<
const { fragmentIndex, sampleIndex, correctSampleFound } = getBestMatch();
if (correctSampleFound) {
const fragment = this.internalTrack.fragments[fragmentIndex]!;
return this.fetchSampleInFragment(fragment, sampleIndex, options);
return this.fetchPacketInFragment(fragment, sampleIndex, options);
}
if (fragmentIndex !== -1) {
bestFragmentIndex = fragmentIndex;
@@ -2295,11 +2284,11 @@ abstract class IsobmffTrackBacking<
metadataReader.pos = startPos + boxInfo.totalSize;
}
let result: Sample | null = null;
let result: EncodedPacket | null = null;
const bestFragment = bestFragmentIndex !== -1 ? this.internalTrack.fragments[bestFragmentIndex]! : null;
if (bestFragment) {
// If we finished looping but didn't find a perfect match, still return the best match we found
result = await this.fetchSampleInFragment(bestFragment, bestSampleIndex, options);
result = await this.fetchPacketInFragment(bestFragment, bestSampleIndex, options);
}
// Catch faulty lookup table entries
@@ -2318,7 +2307,7 @@ abstract class IsobmffTrackBacking<
}
}
class IsobmffVideoTrackBacking extends IsobmffTrackBacking<EncodedVideoSample> implements InputVideoTrackBacking {
class IsobmffVideoTrackBacking extends IsobmffTrackBacking implements InputVideoTrackBacking {
override internalTrack: InternalVideoTrack;
decoderConfigPromise: Promise<VideoDecoderConfig> | null = null;
@@ -2359,8 +2348,8 @@ class IsobmffVideoTrackBacking extends IsobmffTrackBacking<EncodedVideoSample> i
return this.decoderConfigPromise ??= (async (): Promise<VideoDecoderConfig> => {
if (this.internalTrack.info.codec === 'vp9' && !this.internalTrack.info.vp9CodecInfo) {
const firstSample = await this.getFirstSample({});
this.internalTrack.info.vp9CodecInfo = firstSample && extractVp9CodecInfoFromFrame(firstSample.data);
const firstPacket = await this.getFirstPacket({});
this.internalTrack.info.vp9CodecInfo = firstPacket && extractVp9CodecInfoFromFrame(firstPacket.data);
}
return {
@@ -2372,20 +2361,9 @@ class IsobmffVideoTrackBacking extends IsobmffTrackBacking<EncodedVideoSample> i
};
})();
}
createSample(
data: Uint8Array,
byteLength: number,
type: SampleType,
timestamp: number,
duration: number,
sequenceNumber: number,
) {
return new EncodedVideoSample(data, type, timestamp, duration, sequenceNumber, byteLength);
}
}
class IsobmffAudioTrackBacking extends IsobmffTrackBacking<EncodedAudioSample> implements InputAudioTrackBacking {
class IsobmffAudioTrackBacking extends IsobmffTrackBacking implements InputAudioTrackBacking {
override internalTrack: InternalAudioTrack;
decoderConfig: AudioDecoderConfig | null = null;
@@ -2418,16 +2396,6 @@ class IsobmffAudioTrackBacking extends IsobmffTrackBacking<EncodedAudioSample> i
description: this.internalTrack.info.codecDescription ?? undefined,
};
}
createSample(
data: Uint8Array,
byteLength: number,
type: SampleType,
timestamp: number,
duration: number,
) {
return new EncodedAudioSample(data, type, timestamp, duration, byteLength);
}
}
const getSampleIndexForTimestamp = (sampleTable: SampleTable, timescaleUnits: number) => {
src/isobmff/isobmff-muxer.ts
+15 -15
View File
@@ -13,8 +13,8 @@ import {
validateSubtitleMetadata,
validateVideoChunkMetadata,
} from '../codec';
import { EncodedAudioSample, EncodedVideoSample, SampleType } from '../sample';
import { BufferTarget } from '../target';
import { EncodedPacket, PacketType } from '../packet';
export const GLOBAL_TIMESCALE = 1000;
const TIMESTAMP_OFFSET = 2_082_844_800; // Seconds between Jan 1 1904 and Jan 1 1970
@@ -25,7 +25,7 @@ export type Sample = {
duration: number;
data: Uint8Array | null;
size: number;
type: SampleType;
type: PacketType;
timescaleUnitsToNextSample: number;
};
@@ -290,7 +290,7 @@ export class IsobmffMuxer extends Muxer {
return newTrackData;
}
async addEncodedVideoSample(track: OutputVideoTrack, sample: EncodedVideoSample, meta?: EncodedVideoChunkMetadata) {
async addEncodedVideoPacket(track: OutputVideoTrack, packet: EncodedPacket, meta?: EncodedVideoChunkMetadata) {
const release = await this.mutex.acquire();
try {
@@ -298,15 +298,15 @@ export class IsobmffMuxer extends Muxer {
const timestamp = this.validateAndNormalizeTimestamp(
trackData.track,
sample.timestamp,
sample.type === 'key',
packet.timestamp,
packet.type === 'key',
);
const internalSample = this.createSampleForTrack(
trackData,
sample.data,
packet.data,
timestamp,
sample.duration,
sample.type,
packet.duration,
packet.type,
);
await this.registerSample(trackData, internalSample);
@@ -315,7 +315,7 @@ export class IsobmffMuxer extends Muxer {
}
}
async addEncodedAudioSample(track: OutputAudioTrack, sample: EncodedAudioSample, meta?: EncodedAudioChunkMetadata) {
async addEncodedAudioPacket(track: OutputAudioTrack, packet: EncodedPacket, meta?: EncodedAudioChunkMetadata) {
const release = await this.mutex.acquire();
try {
@@ -323,15 +323,15 @@ export class IsobmffMuxer extends Muxer {
const timestamp = this.validateAndNormalizeTimestamp(
trackData.track,
sample.timestamp,
sample.type === 'key',
packet.timestamp,
packet.type === 'key',
);
const internalSample = this.createSampleForTrack(
trackData,
sample.data,
packet.data,
timestamp,
sample.duration,
sample.type,
packet.duration,
packet.type,
);
if (trackData.requiresPcmTransformation) {
@@ -492,7 +492,7 @@ export class IsobmffMuxer extends Muxer {
data: Uint8Array,
timestamp: number,
duration: number,
type: SampleType,
type: PacketType,
) {
const sample: Sample = {
timestamp,