Add support for Dash.js rendered subtitles (#383)

Co-authored-by: Tom Coward <34864926+tom-coward@users.noreply.github.com>
Co-authored-by: Peter Clay Holden Morris-Hind <peter.morris-hind@bbc.co.uk>
Co-authored-by: Eirik Jo Björnerstedt <eirik.bjornerstedt@bbc.co.uk>

Author: 4 people

docs/tutorials/Subtitles.md

@@ -10,10 +10,12 @@ You provide subtitles to BigscreenPlayer by setting `media.captions` in the `.in
 
 ```js
 // 1️⃣ Add an array of caption blocks to your playback data.
-playbackData.media.captions = [/* caption blocks... */];
+playbackData.media.captions = [
+  /* caption blocks... */
+]
 
 // 2️⃣ Pass playback data that contains captions to the player.
-player.init(document.querySelector("video"), playbackData, /* other opts */);
+player.init(document.querySelector("video"), playbackData /* other opts */)
 ```
 
 1. `media.captions` MUST be an array containing at least one object.
@@ -34,7 +36,7 @@ const captions = [
   { url: "https://some.cdn/subtitles.xml" },
   { url: "https://other.cdn/subtitles.xml" },
   /* ... */
-];
+]
 ```
 
 Subtitles delivered as a whole do not require any additional metadata in the manifest to work.
@@ -49,13 +51,15 @@ const captions = [
   {
     url: "https://some.cdn/subtitles/$segment$.m4s",
     segmentLength: 3.84,
+    cdn: "default",
   },
   {
     url: "https://other.cdn/subtitles/$segment$.m4s",
     segmentLength: 3.84,
+    cdn: "default",
   },
   /* ... */
-];
+]
 ```
 
 The segment number is calculated from the presentation timeline. You MUST ensure your subtitle segments are enumerated to match your media segments and you account for offsets such as:
@@ -73,12 +77,24 @@ You can style the subtitles by setting `media.subtitleCustomisation` in the `.in
 
 ```js
 // 1️⃣ Create an object mapping out styles for your subtitles.
-playbackData.media.subtitleCustomisation = { lineHeight: 1.5, size: 1 };
+playbackData.media.subtitleCustomisation = { lineHeight: 1.5, size: 1 }
 
 // 2️⃣ Pass playback data that contains subtitle customisation (and captions) to the player.
-player.init(document.querySelector("video"), playbackData, /* other opts */);
+player.init(document.querySelector("video"), playbackData /* other opts */)
 ```
 
+### Low Latency Streams
+
+When using Dash.js with a low-latency MPD segments are delivered using Chunked Transfer Encoding (CTE) - the default side chain doesn't allow for delivery in this case.
+
+Whilst it is possible to collect chunks as they are delivered, wait until a full segment worth of subtitles have been delivered and pass these to the render function this breaks the low-latency workflow.
+
+An override has been added to allow subtitles to be rendered directly by Dash.js instead of the current side-chain.
+
+Subtitles can be enabled and disabled in the usual way using the `setSubtitlesEnabled()` function. However, they are signalled and delivered by the chosen MPD.
+
+Using Dash.js subtitles can be enabled using `window.bigscreenPlayer.overrides.embeddedSubtitles = true`.
+
 ##  Design
 
 ### Why not include subtitles in the manifest?

package-lock.json

@@ -30,7 +30,7 @@
     "@babel/plugin-transform-runtime": "^7.23.9",
     "@babel/preset-env": "^7.23.8",
     "@babel/preset-typescript": "^7.23.3",
-    "@rollup/plugin-alias": "^5.1.0",
+    "@rollup/plugin-alias": "^5.1.1",
     "@rollup/plugin-babel": "^6.0.4",
     "@rollup/plugin-commonjs": "^25.0.7",
     "@rollup/plugin-inject": "^5.0.5",
@@ -64,7 +64,7 @@
     "typescript-eslint": "^7.2.0"
   },
   "dependencies": {
-    "dashjs": "github:bbc/dash.js#smp-v4.7.3-6",
+    "dashjs": "github:bbc/dash.js#smp-v4.7.3-7",
     "smp-imsc": "github:bbc/imscJS#v1.0.3"
   },
   "repository": {

package.json

@@ -1,5 +1,6 @@
 import PackageJSON from "./package.json" assert { type: "json" }
 
+import alias from "@rollup/plugin-alias"
 import replace from "@rollup/plugin-replace"
 import typescript from "@rollup/plugin-typescript"
 import { dts } from "rollup-plugin-dts"
@@ -10,6 +11,9 @@ export default [
     external: [/^dashjs/, "smp-imsc", "tslib"],
     output: [{ dir: "dist/esm", format: "es" }],
     plugins: [
+      alias({
+        entries: [{ find: "imsc", replacement: "smp-imsc" }],
+      }),
       replace({
         preventAssignment: true,
         __VERSION__: () => PackageJSON.version,

rollup.config.js

@@ -1,5 +1,6 @@
 import PackageJSON from "./package.json" assert { type: "json" }
 
+import alias from "@rollup/plugin-alias"
 import babel from "@rollup/plugin-babel"
 import commonjs from "@rollup/plugin-commonjs"
 import resolve from "@rollup/plugin-node-resolve"
@@ -20,6 +21,9 @@ export default {
     format: "es",
   },
   plugins: [
+    alias({
+      entries: [{ find: "imsc", replacement: "smp-imsc" }],
+    }),
     replace({
       preventAssignment: true,
       __VERSION__: () => PackageJSON.version,

rollup.dev.config.js

@@ -114,13 +114,6 @@ function BigscreenPlayer() {
       !initialPresentationTime &&
       initialPresentationTime !== 0
 
-    readyHelper = ReadyHelper(
-      initialPresentationTime,
-      mediaSources.time().manifestType,
-      PlayerComponent.getLiveSupport(),
-      _callbacks.playerReady
-    )
-
     playerComponent = PlayerComponent(
       playbackElement,
       { media, enableAudioDescribed, initialPlaybackTime: initialPresentationTime },
@@ -130,13 +123,21 @@ function BigscreenPlayer() {
       callAudioDescribedCallbacks
     )
 
-    subtitles = Subtitles(
-      playerComponent,
-      enableSubtitles,
-      playbackElement,
-      media.subtitleCustomisation,
-      mediaSources,
-      callSubtitlesCallbacks
+    readyHelper = ReadyHelper(
+      initialPresentationTime,
+      mediaSources.time().manifestType,
+      PlayerComponent.getLiveSupport(),
+      () => {
+        _callbacks.playerReady()
+        subtitles = Subtitles(
+          playerComponent,
+          enableSubtitles,
+          playbackElement,
+          media.subtitleCustomisation,
+          mediaSources,
+          callSubtitlesCallbacks
+        )
+      }
     )
   }
 

src/bigscreenplayer.js

@@ -28,6 +28,8 @@ function MSEStrategy(
 
   let mediaPlayer
   let mediaElement
+  let subtitleElement
+  let subtitlesEnabled = false
   const manifestType = mediaSources.time().manifestType
 
   const playerSettings = Utils.merge(
@@ -37,12 +39,15 @@ function MSEStrategy(
       },
       streaming: {
         blacklistExpiryTime: mediaSources.failoverResetTime(),
+        lastMediaSettingsCachingInfo: { enabled: false },
         buffer: {
           bufferToKeep: 4,
           bufferTimeAtTopQuality: 12,
           bufferTimeAtTopQualityLongForm: 15,
         },
-        lastMediaSettingsCachingInfo: { enabled: false },
+        text: {
+          defaultEnabled: false,
+        },
       },
     },
     customPlayerSettings
@@ -108,6 +113,7 @@ function MSEStrategy(
     STREAM_INITIALIZED: "streamInitialized",
     FRAGMENT_CONTENT_LENGTH_MISMATCH: "fragmentContentLengthMismatch",
     QUOTA_EXCEEDED: "quotaExceeded",
+    TEXT_TRACKS_ADDED: "allTextTracksAdded",
     CURRENT_TRACK_CHANGED: "currentTrackChanged",
   }
 
@@ -555,6 +561,13 @@ function MSEStrategy(
     }
   }
 
+  function setUpSubtitleElement(playbackElement) {
+    subtitleElement = document.createElement("div")
+    subtitleElement.id = "bsp_subtitles"
+    subtitleElement.style.position = "absolute"
+    playbackElement.appendChild(subtitleElement, playbackElement.firstChild)
+  }
+
   function setUpMediaElement(playbackElement) {
     mediaElement = mediaKind === MediaKinds.AUDIO ? document.createElement("audio") : document.createElement("video")
 
@@ -578,6 +591,7 @@ function MSEStrategy(
 
   function setUpMediaPlayer(presentationTimeInSeconds) {
     const dashSettings = getDashSettings(playerSettings)
+    const embeddedSubs = window.bigscreenPlayer?.overrides?.embeddedSubtitles ?? false
     const protectionData = mediaSources.currentProtectionData()
 
     mediaPlayer = MediaPlayer().create()
@@ -589,6 +603,11 @@ function MSEStrategy(
 
     mediaPlayer.initialize(mediaElement, null)
 
+    if (embeddedSubs) {
+      setUpSubtitleElement(playbackElement)
+      mediaPlayer.attachTTMLRenderingDiv(subtitleElement)
+    }
+
     modifySource(presentationTimeInSeconds)
   }
 
@@ -670,10 +689,15 @@ function MSEStrategy(
     mediaPlayer.on(DashJSEvents.GAP_JUMP, onGapJump)
     mediaPlayer.on(DashJSEvents.GAP_JUMP_TO_END, onGapJump)
     mediaPlayer.on(DashJSEvents.QUOTA_EXCEEDED, onQuotaExceeded)
+    mediaPlayer.on(DashJSEvents.TEXT_TRACKS_ADDED, handleTextTracks)
     mediaPlayer.on(DashJSEvents.MANIFEST_LOADING_FINISHED, manifestLoadingFinished)
     mediaPlayer.on(DashJSEvents.CURRENT_TRACK_CHANGED, onCurrentTrackChanged)
   }
 
+  function handleTextTracks() {
+    mediaPlayer.enableText(subtitlesEnabled)
+  }
+
   function manifestLoadingFinished(event) {
     manifestLoadCount++
     manifestRequestTime = event.request.requestEndDate.getTime() - event.request.requestStartDate.getTime()
@@ -700,6 +724,10 @@ function MSEStrategy(
       : { start: 0, end: getDuration() }
   }
 
+  function customiseSubtitles(options) {
+    return mediaPlayer && mediaPlayer.updateSettings({ streaming: { text: { imsc: { options } } } })
+  }
+
   function getDuration() {
     const duration = mediaPlayer && mediaPlayer.isReady() && mediaPlayer.duration()
 
@@ -777,6 +805,11 @@ function MSEStrategy(
     )
   }
 
+  function isSubtitlesAvailable() {
+    const textTracks = mediaPlayer.getTracksFor("text")
+    return (textTracks && textTracks.length > 0) ?? false
+  }
+
   function isTrackAudioDescribed(track) {
     return (
       track.roles.includes("alternate") &&
@@ -855,9 +888,13 @@ function MSEStrategy(
       mediaElement.removeEventListener("ratechange", onRateChange)
 
       DOMHelpers.safeRemoveElement(mediaElement)
-
       mediaElement = undefined
     }
+
+    if (subtitleElement) {
+      DOMHelpers.safeRemoveElement(subtitleElement)
+      subtitleElement = undefined
+    }
   }
 
   function getSafelySeekableRange() {
@@ -952,9 +989,17 @@ function MSEStrategy(
     getCurrentTime,
     isAudioDescribedAvailable,
     isAudioDescribedEnabled,
+    isSubtitlesAvailable,
     setAudioDescribedOn,
     setAudioDescribedOff,
     getDuration,
+    setSubtitles: (state) => {
+      subtitlesEnabled = state ?? false
+
+      if (mediaPlayer) {
+        mediaPlayer.enableText(subtitlesEnabled)
+      }
+    },
     getPlayerElement: () => mediaElement,
     tearDown,
     reset: () => {
@@ -964,6 +1009,7 @@ function MSEStrategy(
     },
     isEnded: () => isEnded,
     isPaused,
+    customiseSubtitles,
     pause,
     play: () => mediaPlayer.play(),
     setCurrentTime,