Source: lib/cea/mp4_cea_parser.js

  1. /*! @license
  2.  * Shaka Player
  3.  * Copyright 2016 Google LLC
  4.  * SPDX-License-Identifier: Apache-2.0
  5.  */
  7. goog.provide('shaka.cea.Mp4CeaParser');
  9. goog.require('goog.asserts');
  10. goog.require('shaka.cea.CeaUtils');
  11. goog.require('shaka.cea.SeiProcessor');
  12. goog.require('shaka.log');
  13. goog.require('shaka.media.ClosedCaptionParser');
  14. goog.require('shaka.util.DataViewReader');
  15. goog.require('shaka.util.Error');
  16. goog.require('shaka.util.Mp4Parser');
  17. goog.require('shaka.util.Mp4BoxParsers');
  19. /**
  20.  * MPEG4 stream parser used for extracting 708 closed captions data.
  21.  * @implements {shaka.extern.ICeaParser}
  22.  * @export
  23.  */
  24. shaka.cea.Mp4CeaParser = class {
  25.   constructor() {
  26.     /**
  27.      * SEI data processor.
  28.      * @private
  29.      * @const {!shaka.cea.SeiProcessor}
  30.      */
  31.     this.seiProcessor_ = new shaka.cea.SeiProcessor();
  33.     /**
  34.      * Map of track id to corresponding timescale.
  35.      * @private {!Map<number, number>}
  36.      */
  37.     this.trackIdToTimescale_ = new Map();
  39.     /**
  40.      * Default sample duration, as specified by the TREX box.
  41.      * @private {number}
  42.      */
  43.     this.defaultSampleDuration_ = 0;
  45.     /**
  46.      * Default sample size, as specified by the TREX box.
  47.      * @private {number}
  48.      */
  49.     this.defaultSampleSize_ = 0;
  51.     /**
  52.      * @private {shaka.cea.Mp4CeaParser.BitstreamFormat}
  53.      */
  54.     this.bitstreamFormat_ = shaka.cea.Mp4CeaParser.BitstreamFormat.UNKNOWN;
  55.   }
  57.   /**
  58.    * Parses the init segment. Gets Default Sample Duration and Size from the
  59.    * TREX box, and constructs a map of Track IDs to timescales. Each TRAK box
  60.    * contains a track header (TKHD) containing track ID, and a media header box
  61.    * (MDHD) containing the timescale for the track
  62.    * @override
  63.    */
  64.   init(initSegment) {
  65.     const Mp4Parser = shaka.util.Mp4Parser;
  66.     const BitstreamFormat = shaka.cea.Mp4CeaParser.BitstreamFormat;
  67.     const trackIds = [];
  68.     const timescales = [];
  70.     const codecBoxParser = (box) => this.setBitstreamFormat_(box.name);
  72.     new Mp4Parser()
  73.         .box('moov', Mp4Parser.children)
  74.         .box('mvex', Mp4Parser.children)
  75.         .fullBox('trex', (box) => {
  76.           const parsedTREXBox = shaka.util.Mp4BoxParsers.parseTREX(
  77.               box.reader);
  79.           this.defaultSampleDuration_ = parsedTREXBox.defaultSampleDuration;
  80.           this.defaultSampleSize_ = parsedTREXBox.defaultSampleSize;
  81.         })
  82.         .box('trak', Mp4Parser.children)
  83.         .fullBox('tkhd', (box) => {
  84.           goog.asserts.assert(
  85.               box.version != null,
  86.               'TKHD is a full box and should have a valid version.');
  87.           const parsedTKHDBox = shaka.util.Mp4BoxParsers.parseTKHD(
  88.               box.reader, box.version);
  89.           trackIds.push(parsedTKHDBox.trackId);
  90.         })
  91.         .box('mdia', Mp4Parser.children)
  92.         .fullBox('mdhd', (box) => {
  93.           goog.asserts.assert(
  94.               box.version != null,
  95.               'MDHD is a full box and should have a valid version.');
  96.           const parsedMDHDBox = shaka.util.Mp4BoxParsers.parseMDHD(
  97.               box.reader, box.version);
  98.           timescales.push(parsedMDHDBox.timescale);
  99.         })
  100.         .box('minf', Mp4Parser.children)
  101.         .box('stbl', Mp4Parser.children)
  102.         .fullBox('stsd', Mp4Parser.sampleDescription)
  104.         // These are the various boxes that signal a codec.
  105.         .box('avc1', codecBoxParser)
  106.         .box('avc3', codecBoxParser)
  107.         .box('dvav', codecBoxParser)
  108.         .box('dva1', codecBoxParser)
  109.         .box('hev1', codecBoxParser)
  110.         .box('hvc1', codecBoxParser)
  111.         .box('dvh1', codecBoxParser)
  112.         .box('dvhe', codecBoxParser)
  113.         .box('vvc1', codecBoxParser)
  114.         .box('vvi1', codecBoxParser)
  115.         .box('dvc1', codecBoxParser)
  116.         .box('dvi1', codecBoxParser)
  118.         // This signals an encrypted sample, which we can go inside of to find
  119.         // the codec used.
  120.         .box('encv', Mp4Parser.visualSampleEntry)
  121.         .box('sinf', Mp4Parser.children)
  122.         .box('frma', (box) => {
  123.           const {codec} = shaka.util.Mp4BoxParsers.parseFRMA(box.reader);
  124.           this.setBitstreamFormat_(codec);
  125.         })
  127.         .parse(initSegment, /* partialOkay= */ true, /* stopOnPartial= */ true);
  129.     // At least one track should exist, and each track should have a
  130.     // corresponding Id in TKHD box, and timescale in its MDHD box
  131.     if (!trackIds.length|| !timescales.length ||
  132.       trackIds.length != timescales.length) {
  133.       throw new shaka.util.Error(
  134.           shaka.util.Error.Severity.CRITICAL,
  135.           shaka.util.Error.Category.TEXT,
  136.           shaka.util.Error.Code.INVALID_MP4_CEA);
  137.     }
  139.     if (this.bitstreamFormat_ == BitstreamFormat.UNKNOWN) {
  140.       shaka.log.alwaysWarn(
  141.           'Unable to determine bitstream format for CEA parsing!');
  142.     }
  144.     // Populate the map from track Id to timescale
  145.     trackIds.forEach((trackId, idx) => {
  146.       this.trackIdToTimescale_.set(trackId, timescales[idx]);
  147.     });
  148.   }
  150.   /**
  151.    * Parses each video segment. In fragmented MP4s, MOOF and MDAT come in
  152.    * pairs. The following logic gets the necessary info from MOOFs to parse
  153.    * MDATs (base media decode time, sample sizes/offsets/durations, etc),
  154.    * and then parses the MDAT boxes for CEA-708 packets using this information.
  155.    * CEA-708 packets are returned in the callback.
  156.    * @override
  157.    */
  158.   parse(mediaSegment) {
  159.     const Mp4Parser = shaka.util.Mp4Parser;
  160.     const BitstreamFormat = shaka.cea.Mp4CeaParser.BitstreamFormat;
  162.     if (this.bitstreamFormat_ == BitstreamFormat.UNKNOWN) {
  163.       // We don't know how to extract SEI from this.
  164.       return [];
  165.     }
  167.     /** @type {!Array<!shaka.extern.ICeaParser.CaptionPacket>} **/
  168.     const captionPackets = [];
  170.     let moofOffset = 0;
  172.     /** @type {!Array<!shaka.cea.Mp4CeaParser.ParsedTRAF>} */
  173.     let parsedTRAFs = [];
  175.     new Mp4Parser()
  176.         .box('moof', (box) => {
  177.           moofOffset = box.start;
  178.           // traf box parsing is reset on each moof.
  179.           parsedTRAFs = [];
  180.           Mp4Parser.children(box);
  181.         })
  182.         .box('traf', (box) => {
  183.           parsedTRAFs.push({
  184.             baseMediaDecodeTime: null,
  185.             defaultSampleDuration: this.defaultSampleDuration_,
  186.             defaultSampleSize: this.defaultSampleSize_,
  187.             parsedTRUNs: [],
  188.             timescale: shaka.cea.CeaUtils.DEFAULT_TIMESCALE_VALUE,
  189.           });
  190.           Mp4Parser.children(box);
  191.         })
  192.         .fullBox('trun', (box) => {
  193.           goog.asserts.assert(
  194.               box.version != null && box.flags != null,
  195.               'TRUN is a full box and should have a valid version & flags.');
  197.           const lastTRAF = parsedTRAFs[parsedTRAFs.length - 1];
  199.           const parsedTRUN = shaka.util.Mp4BoxParsers.parseTRUN(
  200.               box.reader, box.version, box.flags);
  201.           lastTRAF.parsedTRUNs.push(parsedTRUN);
  202.         })
  203.         .fullBox('tfhd', (box) => {
  204.           goog.asserts.assert(
  205.               box.flags != null,
  206.               'TFHD is a full box and should have valid flags.');
  208.           const lastTRAF = parsedTRAFs[parsedTRAFs.length - 1];
  210.           const parsedTFHD = shaka.util.Mp4BoxParsers.parseTFHD(
  211.               box.reader, box.flags);
  213.           // If specified, defaultSampleDuration and defaultSampleSize
  214.           // override the ones specified in the TREX box
  215.           lastTRAF.defaultSampleDuration = parsedTFHD.defaultSampleDuration ||
  216.               this.defaultSampleDuration_;
  218.           lastTRAF.defaultSampleSize = parsedTFHD.defaultSampleSize ||
  219.               this.defaultSampleSize_;
  221.           const trackId = parsedTFHD.trackId;
  223.           // Get the timescale from the track Id
  224.           if (this.trackIdToTimescale_.has(trackId)) {
  225.             lastTRAF.timescale = this.trackIdToTimescale_.get(trackId);
  226.           }
  227.         })
  228.         .fullBox('tfdt', (box) => {
  229.           goog.asserts.assert(
  230.               box.version != null,
  231.               'TFDT is a full box and should have a valid version.');
  233.           const lastTRAF = parsedTRAFs[parsedTRAFs.length - 1];
  235.           const parsedTFDT = shaka.util.Mp4BoxParsers.parseTFDTInaccurate(
  236.               box.reader, box.version);
  238.           lastTRAF.baseMediaDecodeTime = parsedTFDT.baseMediaDecodeTime;
  239.         })
  240.         .box('mdat', (box) => {
  241.           const offset = moofOffset - box.start - 8;
  242.           const initialPosition = box.reader.getPosition();
  243.           for (const parsedTRAF of parsedTRAFs) {
  244.             if (parsedTRAF.baseMediaDecodeTime === null) {
  245.               // This field should have been populated by the Base Media Decode
  246.               // Time in the tfdt box.
  247.               shaka.log.alwaysWarn(
  248.                   'Unable to find base media decode time for CEA captions!');
  249.               continue;
  250.             }
  251.             box.reader.seek(initialPosition);
  252.             this.parseMdat_(box.reader,
  253.                 parsedTRAF.baseMediaDecodeTime,
  254.                 parsedTRAF.timescale,
  255.                 parsedTRAF.defaultSampleDuration,
  256.                 parsedTRAF.defaultSampleSize,
  257.                 offset,
  258.                 parsedTRAF.parsedTRUNs,
  259.                 captionPackets);
  260.           }
  261.         })
  262.         .parse(mediaSegment, /* partialOkay= */ false,
  263.             /* stopOnPartial= */ true);
  265.     return captionPackets;
  266.   }
  268.   /**
  269.    * Parse MDAT box.
  270.    * @param {!shaka.util.DataViewReader} reader
  271.    * @param {number} time
  272.    * @param {number} timescale
  273.    * @param {number} defaultSampleDuration
  274.    * @param {number} defaultSampleSize
  275.    * @param {number} offset
  276.    * @param {!Array<shaka.util.ParsedTRUNBox>} parsedTRUNs
  277.    * @param {!Array<!shaka.extern.ICeaParser.CaptionPacket>} captionPackets
  278.    * @private
  279.    */
  280.   parseMdat_(reader, time, timescale, defaultSampleDuration,
  281.       defaultSampleSize, offset, parsedTRUNs, captionPackets) {
  282.     const BitstreamFormat = shaka.cea.Mp4CeaParser.BitstreamFormat;
  283.     const CeaUtils = shaka.cea.CeaUtils;
  284.     let sampleIndex = 0;
  286.     // The fields in each ParsedTRUNSample contained in the sampleData
  287.     // array are nullable. In the case of sample data and sample duration,
  288.     // we use the defaults provided by the TREX/TFHD boxes. For sample
  289.     // composition time offset, we default to 0.
  290.     let sampleSize = defaultSampleSize;
  292.     // Combine all sample data.  This assumes that the samples described across
  293.     // multiple trun boxes are still continuous in the mdat box.
  294.     const sampleDatas = parsedTRUNs.map((t) => t.sampleData);
  295.     const sampleData = [].concat(...sampleDatas);
  297.     if (sampleData.length) {
  298.       sampleSize = sampleData[0].sampleSize || defaultSampleSize;
  299.     }
  301.     reader.skip(offset + parsedTRUNs[0].dataOffset);
  303.     while (reader.hasMoreData()) {
  304.       const naluSize = reader.readUint32();
  305.       const naluHeader = reader.readUint8();
  306.       let naluType = null;
  307.       let isSeiMessage = false;
  308.       let naluHeaderSize = 1;
  310.       goog.asserts.assert(this.bitstreamFormat_ != BitstreamFormat.UNKNOWN,
  311.           'Bitstream format should have been checked before now!');
  312.       switch (this.bitstreamFormat_) {
  313.         case BitstreamFormat.H264:
  314.           naluType = naluHeader & 0x1f;
  315.           isSeiMessage = naluType == CeaUtils.H264_NALU_TYPE_SEI;
  316.           break;
  318.         case BitstreamFormat.H265:
  319.           naluHeaderSize = 2;
  320.           reader.skip(1);
  321.           naluType = (naluHeader >> 1) & 0x3f;
  322.           isSeiMessage =
  323.               naluType == CeaUtils.H265_PREFIX_NALU_TYPE_SEI ||
  324.               naluType == CeaUtils.H265_SUFFIX_NALU_TYPE_SEI;
  325.           break;
  327.         case BitstreamFormat.H266:
  328.           naluHeaderSize = 2;
  329.           reader.skip(1);
  330.           naluType = (naluHeader >> 1) & 0x3f;
  331.           isSeiMessage =
  332.               naluType == CeaUtils.H266_PREFIX_NALU_TYPE_SEI ||
  333.               naluType == CeaUtils.H266_SUFFIX_NALU_TYPE_SEI;
  334.           break;
  336.         default:
  337.           return;
  338.       }
  340.       if (isSeiMessage) {
  341.         let timeOffset = 0;
  343.         if (sampleIndex < sampleData.length) {
  344.           timeOffset = sampleData[sampleIndex].sampleCompositionTimeOffset || 0;
  345.         }
  347.         const pts = (time + timeOffset) / timescale;
  348.         for (const packet of this.seiProcessor_
  349.             .process(reader.readBytes(naluSize - naluHeaderSize))) {
  350.           captionPackets.push({
  351.             packet,
  352.             pts,
  353.           });
  354.         }
  355.       } else {
  356.         try {
  357.           reader.skip(naluSize - naluHeaderSize);
  358.         } catch (e) {
  359.           // It is necessary to ignore this error because it can break the start
  360.           // of playback even if the user does not want to see the subtitles.
  361.           break;
  362.         }
  363.       }
  364.       sampleSize -= (naluSize + 4);
  365.       if (sampleSize == 0) {
  366.         if (sampleIndex < sampleData.length) {
  367.           time += sampleData[sampleIndex].sampleDuration ||
  368.               defaultSampleDuration;
  369.         } else {
  370.           time += defaultSampleDuration;
  371.         }
  373.         sampleIndex++;
  375.         if (sampleIndex < sampleData.length) {
  376.           sampleSize = sampleData[sampleIndex].sampleSize || defaultSampleSize;
  377.         } else {
  378.           sampleSize = defaultSampleSize;
  379.         }
  380.       }
  381.     }
  382.   }
  384.   /**
  385.    * @param {string} codec A fourcc for a codec.
  386.    * @private
  387.    */
  388.   setBitstreamFormat_(codec) {
  389.     if (shaka.cea.Mp4CeaParser.CodecBitstreamMap_.has(codec)) {
  390.       this.bitstreamFormat_ =
  391.           shaka.cea.Mp4CeaParser.CodecBitstreamMap_.get(codec);
  392.     }
  393.   }
  394. };
  396. /** @enum {number} */
  397. shaka.cea.Mp4CeaParser.BitstreamFormat = {
  398.   UNKNOWN: 0,
  399.   H264: 1,
  400.   H265: 2,
  401.   H266: 3,
  402. };
  404. /** @private {Map<string, shaka.cea.Mp4CeaParser.BitstreamFormat>} */
  405. shaka.cea.Mp4CeaParser.CodecBitstreamMap_ = new Map()
  406.     // AVC
  407.     .set('avc1', shaka.cea.Mp4CeaParser.BitstreamFormat.H264)
  408.     .set('avc3', shaka.cea.Mp4CeaParser.BitstreamFormat.H264)
  409.     // Dolby Vision based in AVC
  410.     .set('dvav', shaka.cea.Mp4CeaParser.BitstreamFormat.H264)
  411.     .set('dva1', shaka.cea.Mp4CeaParser.BitstreamFormat.H264)
  412.     // HEVC
  413.     .set('hev1', shaka.cea.Mp4CeaParser.BitstreamFormat.H265)
  414.     .set('hvc1', shaka.cea.Mp4CeaParser.BitstreamFormat.H265)
  415.     // Dolby Vision based in HEVC
  416.     .set('dvh1', shaka.cea.Mp4CeaParser.BitstreamFormat.H265)
  417.     .set('dvhe', shaka.cea.Mp4CeaParser.BitstreamFormat.H265)
  418.     // VVC
  419.     .set('vvc1', shaka.cea.Mp4CeaParser.BitstreamFormat.H266)
  420.     .set('vvi1', shaka.cea.Mp4CeaParser.BitstreamFormat.H266)
  421.     // Dolby Vision based in VVC
  422.     .set('dvc1', shaka.cea.Mp4CeaParser.BitstreamFormat.H266)
  423.     .set('dvi1', shaka.cea.Mp4CeaParser.BitstreamFormat.H266);
  425. /**
  426.  * @typedef {{
  427.  *    baseMediaDecodeTime: ?number,
  428.  *    defaultSampleDuration: number,
  429.  *    defaultSampleSize: number,
  430.  *    parsedTRUNs: !Array<shaka.util.ParsedTRUNBox>,
  431.  *    timescale: number,
  432.  * }}
  433.  *
  434.  * @property {?number} baseMediaDecodeTime
  435.  * @property {number} defaultSampleDuration
  436.  * @property {number} defaultSampleSize
  437.  * @property {!Array<shaka.util.ParsedTRUNBox>} parsedTRUNs
  438.  * @property {?number} timescale
  439.  */
  440. shaka.cea.Mp4CeaParser.ParsedTRAF;
  442. shaka.media.ClosedCaptionParser.registerParser('video/mp4',
  443.     () => new shaka.cea.Mp4CeaParser());