feat: Support switching codecs if supported by the browser

src/master-playlist-controller.js

@@ -545,9 +545,23 @@ export class MasterPlaylistController extends videojs.EventTarget {
       }, ABORT_EARLY_BLACKLIST_SECONDS);
     });
 
-    this.mainSegmentLoader_.on('trackinfo', () => {
-      this.tryToCreateSourceBuffers_();
-    });
+    const updateCodecs = () => {
+      if (!this.sourceUpdater_.ready()) {
+        return this.tryToCreateSourceBuffers_();
+      }
+
+      const codecs = this.getCodecsOrExclude_();
+
+      // no codecs means that the playlist was excluded
+      if (!codecs) {
+        return;
+      }
+
+      this.sourceUpdater_.addOrChangeSourceBuffers(codecs);
+    };
+
+    this.mainSegmentLoader_.on('trackinfo', updateCodecs);
+    this.audioSegmentLoader_.on('trackinfo', updateCodecs);
 
     this.mainSegmentLoader_.on('fmp4', () => {
       if (!this.triggeredFmp4Usage) {
@@ -569,10 +583,6 @@ export class MasterPlaylistController extends videojs.EventTarget {
       this.logger_('audioSegmentLoader ended');
       this.onEndOfStream();
     });
-
-    this.audioSegmentLoader_.on('trackinfo', () => {
-      this.tryToCreateSourceBuffers_();
-    });
   }
 
   mediaSecondsLoaded_() {
@@ -732,17 +742,7 @@ export class MasterPlaylistController extends videojs.EventTarget {
     // Only attempt to create the source buffer if none already exist.
     // handleSourceOpen is also called when we are "re-opening" a source buffer
     // after `endOfStream` has been called (in response to a seek for instance)
-    try {
-      this.tryToCreateSourceBuffers_();
-    } catch (e) {
-      videojs.log.warn('Failed to create Source Buffers', e);
-      if (this.mediaSource.readyState !== 'open') {
-        this.trigger('error');
-      } else {
-        this.sourceUpdater_.endOfStream('decode');
-      }
-      return;
-    }
+    this.tryToCreateSourceBuffers_();
 
     // if autoplay is enabled, begin playback. This is duplicative of
     // code in video.js but is required because play() must be invoked
@@ -1278,124 +1278,164 @@ export class MasterPlaylistController extends videojs.EventTarget {
     return this.masterPlaylistLoader_.media() || this.initialMedia_;
   }
 
-  /**
-   * Create source buffers and exlude any incompatible renditions.
-   *
-   * @private
-   */
-  tryToCreateSourceBuffers_() {
-    // media source is not ready yet
-    if (this.mediaSource.readyState !== 'open') {
-      return;
-    }
+  areMediaTypesKnown_() {
+    const usingAudioLoader = !!this.mediaTypes_.AUDIO.activePlaylistLoader;
 
-    // source buffers are already created
-    if (this.sourceUpdater_.ready()) {
-      return;
+    // one or both loaders has not loaded sufficently to get codecs
+    if (!this.mainSegmentLoader_.startingMedia_ || (usingAudioLoader && !this.audioSegmentLoader_.startingMedia_)) {
+      return false;
     }
 
-    const mainStartingMedia = this.mainSegmentLoader_.startingMedia_;
-    const hasAltAudio = !!this.mediaTypes_.AUDIO.activePlaylistLoader;
+    return true;
+  }
+
+  getCodecsOrExclude_() {
+    const media = {
+      main: this.mainSegmentLoader_.startingMedia_ || {},
+      audio: this.audioSegmentLoader_.startingMedia_ || {}
+    };
 
-    // Because a URI is required for EXT-X-STREAM-INF tags (therefore, there must always
-    // be a playlist, even for audio only playlists with alt audio), a segment will always
-    // be downloaded for the main segment loader, and the track info parsed from it.
-    // Therefore we must always wait for the segment loader's track info.
-    if (!mainStartingMedia || (hasAltAudio && !this.audioSegmentLoader_.startingMedia_)) {
-      return;
-    }
-    const audioStartingMedia = this.audioSegmentLoader_ && this.audioSegmentLoader_.startingMedia_ || {};
-    const media = this.masterPlaylistLoader_.media();
-    const playlistCodecs = codecsForPlaylist(this.masterPlaylistLoader_.master, media);
+    // set "main" media equal to video
+    media.video = media.main;
+    const playlistCodecs = codecsForPlaylist(this.master(), this.media());
     const codecs = {};
+    const usingAudioLoader = !!this.mediaTypes_.AUDIO.activePlaylistLoader;
 
-    // priority of codecs: playlist -> mux.js parsed codecs -> default
-    if (mainStartingMedia.isMuxed) {
-      codecs.video = playlistCodecs.video || mainStartingMedia.videoCodec || DEFAULT_VIDEO_CODEC;
-      codecs.video += ',' + (playlistCodecs.audio || mainStartingMedia.audioCodec || DEFAULT_AUDIO_CODEC);
-      if (hasAltAudio) {
-        codecs.audio = playlistCodecs.audio ||
-          audioStartingMedia.audioCodec ||
-          DEFAULT_AUDIO_CODEC;
-      }
-    } else {
-      if (mainStartingMedia.hasAudio || hasAltAudio) {
-        codecs.audio = playlistCodecs.audio ||
-          mainStartingMedia.audioCodec ||
-          audioStartingMedia.audioCodec ||
-          DEFAULT_AUDIO_CODEC;
-      }
+    if (media.main.hasVideo) {
+      codecs.video = playlistCodecs.video || media.main.videoCodec || DEFAULT_VIDEO_CODEC;
+    }
 
-      if (mainStartingMedia.hasVideo) {
-        codecs.video =
-          playlistCodecs.video ||
-          mainStartingMedia.videoCodec ||
-          DEFAULT_VIDEO_CODEC;
-      }
+    if (media.main.isMuxed) {
+      codecs.video += `,${playlistCodecs.audio || media.main.audioCodec || DEFAULT_AUDIO_CODEC}`;
+    }
+
+    if ((media.main.hasAudio && !media.main.isMuxed) || media.audio.hasAudio) {
+      codecs.audio = playlistCodecs.audio || media.main.audioCodec || media.audio.audioCodec || DEFAULT_AUDIO_CODEC;
+      // set audio isFmp4 so we use the correct "supports" function below
+      media.audio.isFmp4 = (media.main.hasAudio && !media.main.isMuxed) ? media.main.isFmp4 : media.audio.isFmp4;
+    }
+
+    // no codecs, no playback.
+    if (!codecs.audio && !codecs.video) {
+      this.blacklistCurrentPlaylist({
+        playlist: this.media(),
+        message: 'Could not determine codecs for playlist.',
+        blacklistDuration: Infinity
+      });
+      return;
     }
 
     // fmp4 relies on browser support, while ts relies on muxer support
-    const supportFunction = mainStartingMedia.isFmp4 ? browserSupportsCodec : muxerSupportsCodec;
-    const unsupportedCodecs = [];
+    const supportFunction = (isFmp4, codec) => (isFmp4 ? browserSupportsCodec(codec) : muxerSupportsCodec(codec));
+    const unsupportedCodecs = {};
+    let unsupportedAudio;
+
+    ['video', 'audio'].forEach(function(type) {
+      if (codecs.hasOwnProperty(type) && !supportFunction(media[type].isFmp4, codecs[type])) {
+        const supporter = media[type].isFmp4 ? 'browser' : 'muxer';
 
-    ['audio', 'video'].forEach(function(type) {
-      if (codecs.hasOwnProperty(type) && !supportFunction(codecs[type])) {
-        unsupportedCodecs.push(codecs[type]);
+        unsupportedCodecs[supporter] = unsupportedCodecs[supporter] || [];
+        unsupportedCodecs[supporter].push(codecs[type]);
+
+        if (type === 'audio') {
+          unsupportedAudio = supporter;
+        }
       }
     });
 
+    if (usingAudioLoader && unsupportedAudio && this.media().attributes.AUDIO) {
+      const audioGroup = this.media().attributes.AUDIO;
+
+      this.mediaTypes_.AUDIO.activePlaylistLoader.pause();
+      this.audioSegmentLoader_.pause();
+      this.audioSegmentLoader_.abort();
+      this.master().playlists.forEach(variant => {
+        const variantAudioGroup = variant.attributes && variant.attributes.AUDIO;
+
+        if (variantAudioGroup === audioGroup && variant !== this.media()) {
+          variant.excludeUntil = Infinity;
+        }
+      });
+      this.logger_(`excluding audio group ${audioGroup} as ${unsupportedAudio} does not support codec(s): "${codecs.audio}"`);
+    }
+
     // if we have any unsupported codecs blacklist this playlist.
-    if (unsupportedCodecs.length) {
-      const supporter = mainStartingMedia.isFmp4 ? 'browser' : 'muxer';
+    if (Object.keys(unsupportedCodecs).length) {
+      const message = Object.keys(unsupportedCodecs).reduce((acc, supporter) => {
+
+        if (acc) {
+          acc += ', ';
+        }
 
-      // reset startingMedia_ when the intial playlist is blacklisted.
-      this.mainSegmentLoader_.startingMedia_ = void 0;
+        acc += `${supporter} does not support codec(s): "${unsupportedCodecs[supporter].join(',')}"`;
+
+        return acc;
+      }, '') + '.';
 
       this.blacklistCurrentPlaylist({
-        playlist: media,
-        message: `${supporter} does not support codec(s): "${unsupportedCodecs.join(',')}".`,
-        internal: true
-      }, Infinity);
+        playlist: this.media(),
+        internal: true,
+        message,
+        blacklistDuration: Infinity
+      });
       return;
     }
+    // check if codec switching is happening
+    if (this.sourceUpdater_.ready() && !this.sourceUpdater_.canChangeType()) {
+      const switchMessages = [];
 
-    if (!codecs.video && !codecs.audio) {
-      const error = 'Failed to create SourceBuffers. No compatible SourceBuffer ' +
-        'configuration for the variant stream:' + media.resolvedUri;
+      ['video', 'audio'].forEach((type) => {
+        const newCodec = (parseCodecs(this.sourceUpdater_.codecs[type] || '')[type] || {}).type;
+        const oldCodec = (parseCodecs(codecs[type] || '')[type] || {}).type;
 
-      videojs.log.warn(error);
-      this.error = error;
+        if (newCodec && oldCodec && newCodec.toLowerCase() !== oldCodec.toLowerCase()) {
+          switchMessages.push(`"${this.sourceUpdater_.codecs[type]}" -> "${codecs[type]}"`);
+        }
+      });
 
-      if (this.mediaSource.readyState !== 'open') {
-        this.trigger('error');
-      } else {
-        this.sourceUpdater_.endOfStream('decode');
+      if (switchMessages.length) {
+        this.blacklistCurrentPlaylist({
+          playlist: this.media(),
+          message: `Codec switching not supported: ${switchMessages.join(', ')}.`,
+          blacklistDuration: Infinity,
+          internal: true
+        });
+        return;
       }
     }
 
-    try {
-      this.sourceUpdater_.createSourceBuffers(codecs);
-    } catch (e) {
-      const error = 'Failed to create SourceBuffers: ' + e;
+    // TODO: when using the muxer shouldn't we just return
+    // the codecs that the muxer outputs?
+    return codecs;
+  }
+
+  /**
+   * Create source buffers and exlude any incompatible renditions.
+   *
+   * @private
+   */
+  tryToCreateSourceBuffers_() {
+    // media source is not ready yet or sourceBuffers are already
+    // created.
+    if (this.mediaSource.readyState !== 'open' || this.sourceUpdater_.ready()) {
+      return;
+    }
+
+    if (!this.areMediaTypesKnown_()) {
+      return;
+    }
+
+    const codecs = this.getCodecsOrExclude_();
 
-      videojs.log.warn(error);
-      this.error = error;
-      if (this.mediaSource.readyState !== 'open') {
-        this.trigger('error');
-      } else {
-        this.sourceUpdater_.endOfStream('decode');
-      }
+    // no codecs means that the playlist was excluded
+    if (!codecs) {
       return;
     }
 
+    this.sourceUpdater_.createSourceBuffers(codecs);
+
     const codecString = [codecs.video, codecs.audio].filter(Boolean).join(',');
 
-    // TODO:
-    // blacklisting incompatible renditions will have to change
-    // once we add support for `changeType` on source buffers.
-    // We will have to not blacklist any rendition until we try to
-    // switch to it and learn that it is incompatible and if it is compatible
-    // we `changeType` on the sourceBuffer.
     this.excludeIncompatibleVariants_(codecString);
   }
 
@@ -1448,24 +1488,30 @@ export class MasterPlaylistController extends videojs.EventTarget {
         variantCodecCount = Object.keys(variantCodecs).length;
       }
 
+      // TODO: we can support this by removing the
+      // old media source and creating a new one, but it will take some work.
       // The number of streams cannot change
       if (variantCodecCount !== codecCount) {
         blacklistReasons.push(`codec count "${variantCodecCount}" !== "${codecCount}"`);
         variant.excludeUntil = Infinity;
       }
 
-      // the video codec cannot change
-      if (variantCodecs.video && codecs.video &&
-        variantCodecs.video.type.toLowerCase() !== codecs.video.type.toLowerCase()) {
-        blacklistReasons.push(`video codec "${variantCodecs.video.type}" !== "${codecs.video.type}"`);
-        variant.excludeUntil = Infinity;
-      }
+      // only exclude playlists by codec change, if codecs cannot switch
+      // during playback.
+      if (!this.sourceUpdater_.canChangeType()) {
+        // the video codec cannot change
+        if (variantCodecs.video && codecs.video &&
+          variantCodecs.video.type.toLowerCase() !== codecs.video.type.toLowerCase()) {
+          blacklistReasons.push(`video codec "${variantCodecs.video.type}" !== "${codecs.video.type}"`);
+          variant.excludeUntil = Infinity;
+        }
 
-      // the audio codec cannot change
-      if (variantCodecs.audio && codecs.audio &&
-        variantCodecs.audio.type.toLowerCase() !== codecs.audio.type.toLowerCase()) {
-        variant.excludeUntil = Infinity;
-        blacklistReasons.push(`audio codec "${variantCodecs.audio.type}" !== "${codecs.audio.type}"`);
+        // the audio codec cannot change
+        if (variantCodecs.audio && codecs.audio &&
+          variantCodecs.audio.type.toLowerCase() !== codecs.audio.type.toLowerCase()) {
+          variant.excludeUntil = Infinity;
+          blacklistReasons.push(`audio codec "${variantCodecs.audio.type}" !== "${codecs.audio.type}"`);
+        }
       }
 
       if (blacklistReasons.length) {

src/media-groups.js

@@ -736,10 +736,13 @@ export const setupMediaGroups = (settings) => {
   // DO NOT enable the default subtitle or caption track.
   // DO enable the default audio track
   const audioGroup = mediaTypes.AUDIO.activeGroup();
-  const groupId = (audioGroup.filter(group => group.default)[0] || audioGroup[0]).id;
 
-  mediaTypes.AUDIO.tracks[groupId].enabled = true;
-  mediaTypes.AUDIO.onTrackChanged();
+  if (audioGroup) {
+    const groupId = (audioGroup.filter(group => group.default)[0] || audioGroup[0]).id;
+
+    mediaTypes.AUDIO.tracks[groupId].enabled = true;
+    mediaTypes.AUDIO.onTrackChanged();
+  }
 
   masterPlaylistLoader.on('mediachange', () => {
     ['AUDIO', 'SUBTITLES'].forEach(type => mediaTypes[type].onGroupChanged());

src/media-segment-request.js

@@ -290,7 +290,8 @@ const transmuxAndNotify = ({
     if (probeResult) {
       trackInfoFn(segment, {
         hasAudio: probeResult.hasAudio,
-        hasVideo: probeResult.hasVideo
+        hasVideo: probeResult.hasVideo,
+        isMuxed
       });
       trackInfoFn = null;
 
@@ -318,6 +319,9 @@ const transmuxAndNotify = ({
     },
     onTrackInfo: (trackInfo) => {
       if (trackInfoFn) {
+        if (isMuxed) {
+          trackInfo.isMuxed = true;
+        }
         trackInfoFn(segment, trackInfo);
       }
     },