diff --git a/dev/demux.html b/dev/demux.html index ac9fba2..580bc38 100644 --- a/dev/demux.html +++ b/dev/demux.html @@ -17,9 +17,17 @@ source: new Mediabunny.BlobSource(file), }); - const track = await input.getPrimaryAudioTrack(); + const track = await input.getPrimaryVideoTrack(); const packetSink = new Mediabunny.EncodedPacketSink(track); + const first = await packetSink.getFirstPacket({ verifyKeyPackets: true }); + const second = await packetSink.getNextPacket(first, { verifyKeyPackets: true }); + const third = await packetSink.getNextPacket(second, { verifyKeyPackets: true }); + const fourth = await packetSink.getNextPacket(third, { verifyKeyPackets: true }); + console.log(first, second, third, fourth); + + /* + for await (const packet of packetSink.packets()) { //console.log(packet); break; @@ -39,6 +47,7 @@ break; } } + */ /* const track = await input.getPrimaryAudioTrack(); diff --git a/package-lock.json b/package-lock.json index 0135c8e..a419287 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "mediabunny", - "version": "1.48.0", + "version": "1.48.1", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "mediabunny", - "version": "1.48.0", + "version": "1.48.1", "license": "MPL-2.0", "workspaces": [ ".", @@ -12864,7 +12864,7 @@ }, "packages/aac-encoder": { "name": "@mediabunny/aac-encoder", - "version": "1.48.0", + "version": "1.48.1", "license": "MPL-2.0", "devDependencies": { "@types/emscripten": "^1.40.1" @@ -12879,7 +12879,7 @@ }, "packages/ac3": { "name": "@mediabunny/ac3", - "version": "1.48.0", + "version": "1.48.1", "license": "MPL-2.0", "devDependencies": { "@types/emscripten": "^1.40.1" @@ -12894,7 +12894,7 @@ }, "packages/flac-encoder": { "name": "@mediabunny/flac-encoder", - "version": "1.48.0", + "version": "1.48.1", "license": "MPL-2.0", "devDependencies": { "@types/emscripten": "^1.40.1" @@ -12909,7 +12909,7 @@ }, "packages/mp3-encoder": { "name": "@mediabunny/mp3-encoder", - "version": "1.48.0", + "version": "1.48.1", "license": "MPL-2.0", "devDependencies": { "@types/emscripten": "^1.40.1" @@ -12924,7 +12924,7 @@ }, "packages/server": { "name": "@mediabunny/server", - "version": "1.48.0", + "version": "1.48.1", "license": "MPL-2.0", "dependencies": { "node-av": "^6.0.0" diff --git a/package.json b/package.json index f11dfc9..7f4b035 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "mediabunny", "author": "Vanilagy", - "version": "1.48.0", + "version": "1.48.1", "description": "Pure TypeScript media toolkit for reading, writing, and converting media files, directly in the browser.", "type": "module", "workspaces": [ diff --git a/packages/aac-encoder/package.json b/packages/aac-encoder/package.json index 15c659a..acf1dd0 100644 --- a/packages/aac-encoder/package.json +++ b/packages/aac-encoder/package.json @@ -1,7 +1,7 @@ { "name": "@mediabunny/aac-encoder", "author": "Vanilagy", - "version": "1.48.0", + "version": "1.48.1", "description": "AAC encoder extension for Mediabunny, based on FFmpeg.", "main": "./dist/bundles/mediabunny-aac-encoder.mjs", "module": "./dist/bundles/mediabunny-aac-encoder.mjs", diff --git a/packages/ac3/package.json b/packages/ac3/package.json index 101a84b..35a7e5b 100644 --- a/packages/ac3/package.json +++ b/packages/ac3/package.json @@ -1,7 +1,7 @@ { "name": "@mediabunny/ac3", "author": "Vanilagy", - "version": "1.48.0", + "version": "1.48.1", "description": "AC-3 and E-AC-3 (Dolby Digital) decoder and encoder extension for Mediabunny, based on FFmpeg.", "main": "./dist/bundles/mediabunny-ac3.mjs", "module": "./dist/bundles/mediabunny-ac3.mjs", diff --git a/packages/flac-encoder/package.json b/packages/flac-encoder/package.json index 0eeee46..c8a67d7 100644 --- a/packages/flac-encoder/package.json +++ b/packages/flac-encoder/package.json @@ -1,7 +1,7 @@ { "name": "@mediabunny/flac-encoder", "author": "Vanilagy", - "version": "1.48.0", + "version": "1.48.1", "description": "FLAC encoder extension for Mediabunny, based on libFLAC.", "main": "./dist/bundles/mediabunny-flac-encoder.mjs", "module": "./dist/bundles/mediabunny-flac-encoder.mjs", diff --git a/packages/mp3-encoder/package.json b/packages/mp3-encoder/package.json index 944a564..30f0b23 100644 --- a/packages/mp3-encoder/package.json +++ b/packages/mp3-encoder/package.json @@ -1,7 +1,7 @@ { "name": "@mediabunny/mp3-encoder", "author": "Vanilagy", - "version": "1.48.0", + "version": "1.48.1", "description": "MP3 encoder extension for Mediabunny, based on LAME.", "main": "./dist/bundles/mediabunny-mp3-encoder.mjs", "module": "./dist/bundles/mediabunny-mp3-encoder.mjs", diff --git a/packages/server/package.json b/packages/server/package.json index 9efdd53..9b5ac4e 100644 --- a/packages/server/package.json +++ b/packages/server/package.json @@ -1,7 +1,7 @@ { "name": "@mediabunny/server", "author": "Vanilagy", - "version": "1.48.0", + "version": "1.48.1", "description": "Adds full video and audio decoder and encoder support to Mediabunny for use in server-side environments (Node, Bun, Deno). Based on NodeAV.", "main": "./dist/bundles/mediabunny-server.cjs", "module": "./dist/bundles/mediabunny-server.mjs", diff --git a/src/mpeg-ts/mpeg-ts-demuxer.ts b/src/mpeg-ts/mpeg-ts-demuxer.ts index 0ac9e33..795c0fc 100644 --- a/src/mpeg-ts/mpeg-ts-demuxer.ts +++ b/src/mpeg-ts/mpeg-ts-demuxer.ts @@ -26,8 +26,6 @@ import { determineVideoPacketType, extractAvcDecoderConfigurationRecord, extractHevcDecoderConfigurationRecord, - extractNalUnitTypeForAvc, - extractNalUnitTypeForHevc, EAC3_NUMBLKS_TABLE, getEac3ChannelCount, getEac3SampleRate, @@ -38,6 +36,8 @@ import { parseEac3SyncFrame, parseHevcSps, AC3_FRAME_SIZES, + extractNalUnitTypeForAvc, + extractNalUnitTypeForHevc, } from '../codec-data'; import { Demuxer } from '../demuxer'; import { Input } from '../input'; @@ -57,6 +57,7 @@ import { floorToMultiple, last, MATRIX_COEFFICIENTS_MAP_INVERSE, + readExpGolomb, Rotation, roundIfAlmostInteger, toDataView, @@ -206,6 +207,12 @@ export class MpegTsDemuxer extends Demuxer { continue; } + if (hasProgramMap && !this.elementaryStreams.some(x => x.pid === packetHeader.pid)) { + // Don't care about this PID + currentPos += this.packetStride; + continue; + } + const section = await this.readSection( currentPos, true, @@ -1976,6 +1983,9 @@ class PacketReadingContext { const elementaryStream = this.elementaryStream; if (elementaryStream.info.type === 'video') { + // Our job here is to separate the video stream into access units. Sometimes this is easy (like when AUDs + // are present), sometimes it's a little harder. + const codec = elementaryStream.info.codec; const CHUNK_SIZE = 1024; @@ -1983,7 +1993,10 @@ class PacketReadingContext { throw new Error('Unhandled.'); } + const nalHeaderSize = codec === 'avc' ? 1 : 2; let packetStartPos: number | null = null; + let frameStartFound = false; + let lastFirstMacroblockInSlice = 0; while (true) { let remaining = this.ensureBuffered(CHUNK_SIZE); @@ -2005,11 +2018,10 @@ class PacketReadingContext { } i = zeroIndex; - // Check if we have enough bytes to identify a start code const posBeforeZero = chunkStartPos + i; - // Need at least 4 more bytes after the 0x00 to check for start code + NAL type - if (i + 4 >= length) { + // Need 3 more bytes after the 0x00 to recognize a start code prefix + if (i + 3 >= length) { // Not enough data in current chunk, seek back and let the next iteration handle it this.seekTo(posBeforeZero); break; @@ -2020,16 +2032,13 @@ class PacketReadingContext { const b3 = chunk[i + 3]!; let startCodeLength = 0; - let nalUnitTypeByte: number | null = null; // Check for 4-byte start code (0x00000001) if (b1 === 0x00 && b2 === 0x00 && b3 === 0x01) { startCodeLength = 4; - nalUnitTypeByte = chunk[i + 4]!; } else if (b1 === 0x00 && b2 === 0x01) { // 3-byte start code (0x000001) startCodeLength = 3; - nalUnitTypeByte = b3; } if (startCodeLength === 0) { @@ -2040,31 +2049,90 @@ class PacketReadingContext { const startCodePos = posBeforeZero; - if (packetStartPos === null) { - // This is our first start code, mark packet start - packetStartPos = startCodePos; - i += startCodeLength; - continue; + // The packet only really begins at the first NAL unit; anything before it isn't usable + packetStartPos ??= startCodePos; + + const nalHeaderStart = i + startCodeLength; + const payloadStart = nalHeaderStart + nalHeaderSize; + + // Bytes peeked from the start of a slice header to decode first_mb_in_slice. Six bytes (48 bits) + // comfortably covers the exp-Golomb code for any realistic macroblock count + const AVC_SLICE_HEADER_PEEK_SIZE = 6; + + // We read the NAL header plus, for slices, the start of the slice header to decode the + // first_mb_in_slice / first_slice_segment_in_pic_flag. Make sure all of it is buffered. + const bytesNeeded = payloadStart + (codec === 'avc' ? AVC_SLICE_HEADER_PEEK_SIZE : 1); + if (bytesNeeded > length) { + this.seekTo(posBeforeZero); + break; } - // We have a second start code. Check if it's an AUD. - if (nalUnitTypeByte !== null) { - const nalUnitType = codec === 'avc' - ? extractNalUnitTypeForAvc(nalUnitTypeByte) - : extractNalUnitTypeForHevc(nalUnitTypeByte); - const isAud = codec === 'avc' - ? nalUnitType === AvcNalUnitType.AUD - : nalUnitType === HevcNalUnitType.AUD_NUT; + const headerByte0 = chunk[nalHeaderStart]!; - if (isAud) { - // End the packet at this start code (before the AUD) - const packetLength = startCodePos - packetStartPos; - this.seekTo(packetStartPos); - return this.supplyPacket(packetLength, 0); + let nalUnitType: number; + let isSlice: boolean; + let isAccessUnitStart: boolean; + + if (codec === 'avc') { + nalUnitType = extractNalUnitTypeForAvc(headerByte0); + isSlice = nalUnitType === AvcNalUnitType.NON_IDR_SLICE + || nalUnitType === AvcNalUnitType.SLICE_DPA + || nalUnitType === AvcNalUnitType.IDR; + isAccessUnitStart = nalUnitType === AvcNalUnitType.SEI + || nalUnitType === AvcNalUnitType.SPS + || nalUnitType === AvcNalUnitType.PPS + || nalUnitType === AvcNalUnitType.AUD; + } else { + nalUnitType = extractNalUnitTypeForHevc(headerByte0); + const layerId = ((headerByte0 & 1) << 5) | (chunk[nalHeaderStart + 1]! >> 3); + if (layerId > 0) { + // Higher layers don't delimit the base-layer frames we care about + i += startCodeLength; + continue; } + + // VCL slices: 0..RASL_R, plus the IRAP range BLA_W_LP..CRA_NUT + isSlice = nalUnitType <= HevcNalUnitType.RASL_R + || (nalUnitType >= HevcNalUnitType.BLA_W_LP && nalUnitType <= 21); + // VPS..FD, prefix SEI, and the reserved/unspecified non-VCL ranges + isAccessUnitStart = (nalUnitType >= HevcNalUnitType.VPS_NUT && nalUnitType <= 37) + || nalUnitType === HevcNalUnitType.PREFIX_SEI_NUT + || (nalUnitType >= 41 && nalUnitType <= 44) + || (nalUnitType >= 48 && nalUnitType <= 55); + } + + let isFrameBoundary = false; + + if (isSlice) { + let startsNewPicture: boolean; + + if (codec === 'avc') { + const headerBytes = chunk.subarray(payloadStart, payloadStart + AVC_SLICE_HEADER_PEEK_SIZE); + const firstMacroblockInSlice = readExpGolomb(new Bitstream(headerBytes)); + startsNewPicture = !frameStartFound || firstMacroblockInSlice <= lastFirstMacroblockInSlice; + lastFirstMacroblockInSlice = firstMacroblockInSlice; + } else { + startsNewPicture = (chunk[payloadStart]! >> 7) === 1; + } + + if (startsNewPicture) { + if (frameStartFound) { + isFrameBoundary = true; + } else { + frameStartFound = true; + } + } + } else if (isAccessUnitStart && frameStartFound) { + isFrameBoundary = true; + } + + if (isFrameBoundary) { + // End the packet at this start code (the next frame begins here) + const packetLength = startCodePos - packetStartPos; + this.seekTo(packetStartPos); + return this.supplyPacket(packetLength, 0); } - // Not an AUD, continue searching i += startCodeLength; } @@ -2074,8 +2142,8 @@ class PacketReadingContext { } } - // End of stream - return remaining data if we have a packet start - if (packetStartPos !== null) { + // End of stream - emit whatever's left as the final packet + if (packetStartPos !== null && this.endPos > packetStartPos) { const packetLength = this.endPos - packetStartPos; this.seekTo(packetStartPos); return this.supplyPacket(packetLength, 0); diff --git a/test/node/mpeg-ts-demuxing.test.ts b/test/node/mpeg-ts-demuxing.test.ts index 4936dc3..25d2f94 100644 --- a/test/node/mpeg-ts-demuxing.test.ts +++ b/test/node/mpeg-ts-demuxing.test.ts @@ -875,3 +875,26 @@ test('MPEG-TS with "extension" PES packets without PTS', async () => { 4693, 223, 144, 174, 118, 9155, 1188, 379, 169, 213, ]); }); + +test('MPEG-TS with AUD-less video packets', async () => { + using input = new Input({ + source: new FilePathSource(path.join(__dirname, '../public/no-aud.ts')), + formats: ALL_FORMATS, + }); + + const videoTrack = await input.getPrimaryVideoTrack(); + assert(videoTrack); + + const sink = new EncodedPacketSink(videoTrack); + + const firstPacket = await sink.getFirstPacket(); + assert(firstPacket); + const secondPacket = await sink.getNextPacket(firstPacket); + assert(secondPacket); + const thirdPacket = await sink.getNextPacket(secondPacket); + assert(thirdPacket); + + expect(firstPacket.data.byteLength).toBe(331774); + expect(secondPacket.data.byteLength).toBe(1749); + expect(thirdPacket.data.byteLength).toBe(4273); +}); diff --git a/test/public/no-aud.ts b/test/public/no-aud.ts new file mode 100644 index 0000000..4bbd956 Binary files /dev/null and b/test/public/no-aud.ts differ