Advance HTML5 multimedia

Creating tracks on the fly: example with sound sprites

INTRODUCTION

In this lesson we show:

• The addTextTrack method for adding a TextTrack to an html <track> element, 
• The VTTCue constructor, for creating cues: var cue = new VTTCue(startTime, endTime, id); 
• the addCue method for adding cues on the fly to a TextTrack etc.

These methods will allow us to create TextTrack objects and cues on the fly, programatically. The presented example shows how we can create "sound sprites": small sounds that are parts of a mp3 file, and that can be played separately. Each sound will be defined as a cue in a track associated with the <audio> element.

EXAMPLE: CREATE ON THE FLY A WEBVTT FILE WITH MANY CUES, IN ORDER TO CUT A BIG SOUND FILE INTO SEGMENTS AND PLAY THEM ON DEMAND

This JsBin demonstration, adapted from an original demo by Sam Dutton,  uses a single mp3 file that contains recorded animal sounds.

Below is the sound file. You can try to play it:

Explanations

The demo uses a JavaScript array for defining the different animal sounds in this audio file:

1.  var sounds = [
1.     {
2.        id: "purr",
3.        startTime: 0.200,
4.        endTime: 1.800
5.     },
6.     {
7.        id: "meow",
8.        startTime: 2.300,
9.        endTime: 3.300
10.     },
11.     {
12.        id: "bark",
13.        startTime: 3.900,
14.        endTime: 4.300
15.     },
16.     {
17.        id: "baa",
18.        startTime: 5.000,
19.        endTime: 5.800
20.     }
21.     ...
22. ];

The idea is to create a track on the fly, then add cues within this track. Each cue will be created with the id, the start and end time taken from the above JavaScript object. In the end, we will have a track with individual cues located at the time location where an animal sound is in the mp3 file.

Then we generate buttons in the HTML document, and when the user clicks on a button, the getCueById method is called, then the start and end time properties of the cue are accessed and the sound is played (using the currentTime property of the audio element).

HTML source code extract:

1. ...
2. <h1>Playing audio sprites with the track element</h1>
3.  <p>A demo by Sam Dutton, adapted for JsBin by M.Buffa</p>
4.  
5. <div id="soundButtons" class="isSupported"></div>
6. ...

JavaScript code extract and explanations are in the comments:

1.  window.onload = function() {
2.     // Create an audio element programmatically
3.     var audio = newAudio("http://mainline.i3s.unice.fr/mooc/animalSounds.mp3");
4.  
5.     audio.addEventListener("loadedmetadata", function() {
6.       // When the audio file has its metadata loaded, we can add
7.       // a new track to it, with mode = hidden. It will fire events
8.       // even if it is hidden
9.       var track = audio.addTextTrack("metadata", "sprite track", "en");
10.       track.mode = "hidden";
11.  
12.       // for browsers that do not implement the getCueById() method
13.       if (typeof track.getCueById !== "function") {
14.          track.getCueById = function(id) {
15.             var cues = track.cues;
16.             for (var i = 0; i != track.cues.length; ++i) {
17.                if (cues[i].id === id) {
18.                return cues[i];
19.             }
20.          }
21.       };
22.    }
23.  
24.    var sounds = [
25.       {
26.         id: "purr",
27.         startTime: 0.200,
28.         endTime: 1.800
29.       },
30.       {
31.         id: "meow",
32.         startTime: 2.300,
33.         endTime: 3.300
34.       },
35.       ...
36.    ];
37.  
38.    for (var i = 0; i !== sounds.length; ++i) {
39.       // for each animal sound, create a cue with id, start and end time
40.       var sound = sounds[i];
41.       var cue = new VTTCue(sound.startTime, sound.endTime, sound.id); 
42.       cue.id = sound.id;
43.       // add it to the track
44.       track.addCue(cue);
45.       // create a button and add it to the HTML document
46.       document.querySelector("#soundButtons").innerHTML += 
47.                        "<button class='playSound' id=" 
48.                        + sound.id + ">" +sound.id 
49.                        + "</button>";
       }
50.  
51.    var endTime;
52.    audio.addEventListener("timeupdate", function(event) {
53.       // When we play a sound, we set the endtime var.
54.       // We need to listen when the audio file is being played, 
55.       // in order to pause it when endTime is reached.
56.       if (event.target.currentTime > endTime)
57.          event.target.pause();
58.    });
59.  
60.    function playSound(id) {
61.      // Plays the sound corresponding to the cue with id equal
62.      // to the one passed as a parameter. We set the endTime var
63.      // and position the audio currentTime at the start time
64.      // of the sound
65.      var cue = track.getCueById(id);
66.      audio.currentTime = cue.startTime;
67.      endTime = cue.endTime;
68.      audio.play();
69.   };
70.   // create listeners for all buttons
71.   var buttons = document.querySelectorAll("button.playSound");
    
72.   for(var i=; i < buttons.length; i++) {    
73.      buttons[i].addEventListener("click", function(e) {
74.         playSound(this.id);
75.      });
76.   }
77.  });
78. };