r69006 MediaWiki - Code Review archive

Repository:MediaWiki
Revision:r69005‎ | r69006 | r69007 >
Date:15:52, 4 July 2010
Author:dale
Status:deferred
Tags:
Comment:
added audio track layout support to render
commented out some render time debug lines
Modified paths:
  • /branches/MwEmbedStandAlone/modules/EmbedPlayer/skins/kskin/mw.PlayerSkinKskin.js (modified) (history)
  • /branches/MwEmbedStandAlone/modules/EmbedPlayer/skins/mw.PlayerControlBuilder.js (modified) (history)
  • /branches/MwEmbedStandAlone/modules/Sequencer/mw.FirefoggRender.js (modified) (history)
  • /branches/MwEmbedStandAlone/modules/SmilPlayer/mw.EmbedPlayerSmil.js (modified) (history)
  • /branches/MwEmbedStandAlone/modules/SmilPlayer/mw.Smil.js (modified) (history)
  • /branches/MwEmbedStandAlone/modules/SmilPlayer/mw.SmilBody.js (modified) (history)
  • /branches/MwEmbedStandAlone/modules/SmilPlayer/mw.SmilLayout.js (modified) (history)
  • /branches/MwEmbedStandAlone/modules/SmilPlayer/tests/VideoCrossfadeSmil.xml (modified) (history)

Diff [purge]

Index: branches/MwEmbedStandAlone/modules/SmilPlayer/mw.SmilBody.js
@@ -21,6 +21,16 @@
2222 // This lets us cache a valid element list for a given amount of time
2323 cacheElementList: {},
2424
 25+ smilBlockTypeMap: {
 26+ 'body':'seq',
 27+ 'ref' : 'ref',
 28+ 'animation':'ref',
 29+ 'audio' : 'ref',
 30+ 'img' : 'ref',
 31+ 'textstream' : 'ref',
 32+ 'video' : 'ref'
 33+ },
 34+
2535 // Constructor:
2636 init: function( smilObject ){
2737 this.smil = smilObject;
@@ -57,7 +67,8 @@
5868 },
5969
6070 /**
61 - * Render the body elements for a given time, use layout engine to draw elements
 71+ * Render the body elements for a given time, use layout engine to draw elements
 72+ * @param time
6273 */
6374 renderTime: function( time, deltaTime ){
6475 var _this = this;
@@ -98,59 +109,172 @@
99110 },
100111
101112 /**
102 - * Check if we have a valid element cache:
103 - * XXX not yet working.
104 - */
105 - isValidElementCache: function( time ){
106 - if( time > this.cacheElementList.validStart && time > this.cacheElementList.validEnd ) {
107 - return this.cacheElementList.elements;
 113+ * Firefogg flattener can presently only sequence flat sequence of audio
 114+ * Also See: http://www.firefogg.org/dev/render.html
 115+ *
 116+ * Note if we could "blend" or play two audio files at the same time
 117+ * none of this would be needed
 118+ *
 119+ * ie this code should be replaced once we add improved audio support
 120+ *
 121+ * @return {Object} an array of audio with the following properties:
 122+ * start The start offset of the audio asset.
 123+ * duration Duration of the audio asset
 124+ * src The source url, if set to false, silence is inserted for duration
 125+ * type Used internally to let audio overlay video.
 126+ */
 127+ getFlatAudioTimeLine: function(){
 128+ var _this = this;
 129+
 130+ // Setup some flags:
 131+ var maxAudioTime =0;
 132+
 133+ var elementsWithAudio = [];
 134+ var audioTimeline = [];
 135+
 136+ // xxx could probably do this a bit cleaner
 137+ var getEarliest = function ( audioTimeline ){
 138+ var smallTime = null;
 139+ var smallIndex = null
 140+ for( var i =0; i < audioTimeline.length; i++ ){
 141+ if( smallTime === null ){
 142+ smallTime = audioTimeline[i]['startTime'];
 143+ smallIndex = i;
 144+ }
 145+
 146+ if( audioTimeline[i]['startTime'] < smallTime ){
 147+ smallTime = audioTimeline[i]['startTime'];
 148+ smallIndex = i;
 149+ }
 150+ }
 151+ return smallIndex = i;
108152 }
 153+ // Build an audio timeline starting from the top level node:
 154+ this.getRefElementsRecurse( this.$dom, 0, function( $node ){
 155+ var nodeType = _this.smil.getRefType( $node ) ;
 156+ // Check if the node is audio ( first in wins / "audio" wins over video)
 157+ if( nodeType == 'audio' || nodeType == 'video' ) {
 158+ var audioObj = {
 159+ 'type' : nodeType,
 160+ 'src' : _this.smil.getAssetUrl ( $node.attr('src') ),
 161+ 'duration' : _this.getNodeDuration( $node ),
 162+ 'startTime' : $node.data( 'startOffset' ),
 163+ 'offset' : 0 // have to add in media-offset support
 164+ };
 165+
 166+ // If audioTimeline is empty insert directly
 167+ if( audioTimeline.length == 0 ){
 168+ audioTimeline.push( audioObj )
 169+ return ;
 170+ }
 171+
 172+ // fill time
 173+ var addedAudioFlag = false;
 174+ for( var i = 0; i < audioTimeline.length; i++ ){
 175+ var currentAudioObj = audioTimeline[i];
 176+ var audioEndTime = audioObj['startTime'] + audioObj['duration'];
 177+ var currentAudioEndTime = currentAudioObj['startTime'] + currentAudioObj['duration'];
 178+ if( audioObj['startTime'] < currentAudioObj['startTime'] ){
 179+ addedAudioFlag = true;
 180+ var beforeAudioObj = $j.extend( audioObj, {
 181+ 'duration': ( currentAudioObj['startTime'] - audioObj['startTime'] )
 182+ });
 183+ // Add before
 184+ audioTimeline.splice( i, 0, beforeAudioObj );
 185+
 186+ // Update the audioObj if it extends past the currentAudioObject
 187+ if( audioEndTime > currentAudioEndTime){
 188+ audioObj['startTime'] = currentAudioEndTime;
 189+ audioObj['duration'] = audioEndTime - currentAudioEndTime;
 190+ } else {
 191+ // done adding audioObj
 192+ break;
 193+ }
 194+ }
 195+ }
 196+ // add audioObject to end ( currentAudioObj has latest startTime
 197+ if( ! addedAudioFlag && audioEndTime > currentAudioEndTime ){
 198+ var audioObjDuration = ( audioEndTime - currentAudioEndTime );
 199+ if( currentAudioEndTime + audioObjDuration > _this.getDuration() ){
 200+ audioObjDuration = _this.getDuration() - currentAudioEndTime;
 201+ }
 202+ audioTimeline.push( $j.extend( audioObj, {
 203+ 'duration': audioObjDuration,
 204+ 'startTime' : currentAudioEndTime
 205+ })
 206+ );
 207+ }
 208+
 209+ // Keep audioTimeline sorted via startTime
 210+ audioTimeline.sort( function( a, b){
 211+ return a.startTime - b.startTime;
 212+ });
 213+ }
 214+ });
 215+
 216+ return audioTimeline;
109217 },
110218
111219 /**
112220 * Gets all the elements for a given time.
 221+ *
 222+ * Note this gets called all the time we may need to build a more efficient structure to access this info
113223 */
114224 getElementsForTime: function ( time , inRangeCallback, outOfRangeCallback ) {
115 - var startOffset = 0;
 225+ var _this = this;
116226 if( !time ) {
117227 time =0;
118228 }
119 - // Empty out the requested element set:
120 - this.getElementsForTimeRecurse( this.$dom, time, startOffset, inRangeCallback, outOfRangeCallback);
 229+ // Recurse on every ref element and run relevant callbacks
 230+ this.getRefElementsRecurse( this.$dom, 0, function( $node ){
 231+ var startOffset = $node.data( 'startOffset' );
 232+ var nodeDuration = _this.getNodeDuration( $node );
 233+
 234+ // Check if element is in range:
 235+ if( time >= startOffset && time <= ( startOffset + nodeDuration) ){
 236+ if( typeof inRangeCallback == 'function' ){
 237+ inRangeCallback( $node );
 238+ }
 239+ } else {
 240+ if( typeof outOfRangeCallback == 'function'){
 241+ outOfRangeCallback( $node );
 242+ }
 243+ }
 244+ });
121245 },
122246
123247 /**
124248 * getElementsForTimeRecurse
125249 * @param {Object} $node Node to recursively search for elements in the given time range
126250 */
127 - getElementsForTimeRecurse: function( $node, time, startOffset, inRangeCallback, outOfRangeCallback){
 251+
 252+ /**
 253+ * Recurse over all body elements, issues a callback on all ref and smilText nodes
 254+ * adds startOffset info for easy timeline checks.
 255+ */
 256+ getRefElementsRecurse: function( $node, startOffset, callback ){
128257 var _this = this;
129 - // Setup local pointers:
130 - var nodeDuration = this.getNodeDuration( $node );
 258+ // Setup local pointers:
131259 var nodeType = this.getNodeSmilType( $node );
132 - var nodeParentType = this.getNodeSmilType( $node.parent() );
133 -
134 -
135 - // If 'par' or 'seq' recurse to get elements for layout
 260+
 261+ // If 'par' or 'seq' recurse on children
136262 if( nodeType == 'par' || nodeType == 'seq' ) {
137263 if( $node.children().length ) {
138264 $node.children().each( function( inx, childNode ){
139265 // mw.log(" recurse:: startOffset:" + nodeType + ' start offset:' + startOffset );
140 - var childDur = _this.getElementsForTimeRecurse(
141 - $j( childNode ),
142 - time,
143 - startOffset,
144 - inRangeCallback,
145 - outOfRangeCallback
 266+ var childDur = _this.getRefElementsRecurse(
 267+ $j( childNode ),
 268+ startOffset,
 269+ callback
146270 );
147271 // If element parent is a 'seq' increment startOffset as we recurse for each child
148272 if( nodeType == 'seq' ) {
149273 //mw.log(" Parent Seq:: add child dur: " + childDur );
150274 startOffset += childDur;
151 - }
 275+ }
152276 });
153277 }
154 - }
 278+ }
155279
156280 // If the nodeType is "ref" or smilText run the callback
157281 if( nodeType == 'ref' || nodeType == 'smilText' ) {
@@ -161,26 +285,17 @@
162286 }
163287
164288 // Add the parent startOffset
165 - $node.data( 'startOffset', startOffset );
 289+ $node.data( 'startOffset', startOffset );
166290
167 - // Check if element is in range:
168 - if( time >= startOffset && time <= ( startOffset + nodeDuration) ){
169 - if( typeof inRangeCallback == 'function' ){
170 - inRangeCallback( $node );
171 - }
172 - } else {
173 - if( typeof outOfRangeCallback == 'function'){
174 - outOfRangeCallback( $node );
175 - }
176 - }
177 - }
 291+ callback( $node )
 292+ }
178293 // Return the node Duration for tracking startOffset
179294 return this.getNodeDuration( $node );
180295 },
181296
182297 /**
183298 * Returns the smil body duration
184 - * ( wraps getDurationRecurse to get top level duration )
 299+ * ( wraps getDurationRecurse to get top level node duration )
185300 */
186301 getDuration: function(){
187302 this.duration = this.getNodeDuration( this.$dom );
@@ -215,7 +330,7 @@
216331 $node.data( 'implictDuration', $node.data('implictDuration') + childDuration );
217332 }
218333 // With par blocks ImplictDuration is longest duration child
219 - if( blockType == 'par' ){
 334+ if( blockType == 'par' ) {
220335 if( childDuration > $node.data( 'implictDuration' ) ){
221336 $node.data( 'implictDuration', childDuration);
222337 }
@@ -255,18 +370,10 @@
256371 */
257372 getNodeSmilType: function( $node ){
258373 var blockType = $j( $node ).get(0).nodeName;
259 - var blockMap = {
260 - 'body':'seq',
261 - 'ref' : 'ref',
262 - 'animation':'ref',
263 - 'audio' : 'ref',
264 - 'img' : 'ref',
265 - 'textstream' : 'ref',
266 - 'video' : 'ref'
 374+
 375+ if( this.smilBlockTypeMap[ blockType ] ){
 376+ blockType = this.smilBlockTypeMap[ blockType ];
267377 }
268 - if( blockMap[ blockType ] ){
269 - blockType = blockMap[ blockType ];
270 - }
271378 return blockType;
272379 }
273380 }
\ No newline at end of file
Index: branches/MwEmbedStandAlone/modules/SmilPlayer/mw.SmilLayout.js
@@ -158,6 +158,9 @@
159159 case 'video':
160160 return this.getSmilVideoHtml( smilElement );
161161 break;
 162+ case 'audio':
 163+ return this.getSmilAudioHtml( smilElement );
 164+ break;
162165 // Smil Text: http://www.w3.org/TR/SMIL/smil-text.html ( obviously we support a subset )
163166 case 'smiltext':
164167 return this.getSmilTextHtml( smilElement );
@@ -181,16 +184,28 @@
182185 /**
183186 * Return the video
184187 */
185 - getSmilVideoHtml: function( videoElement ){
 188+ getSmilVideoHtml: function( smilElement ){
186189 return $j('<video />')
187190 .attr( {
188 - 'id' : this.smil.getAssetId( videoElement ),
189 - 'src' : this.smil.getAssetUrl( $j( videoElement ).attr( 'src' ) )
 191+ 'id' : this.smil.getAssetId( smilElement ),
 192+ 'src' : this.smil.getAssetUrl( $j( smilElement ).attr( 'src' ) )
190193 } )
191194 .addClass( 'smilFillWindow' )
192195 },
193196
194197 /**
 198+ * Return audio element ( by default audio tracks are hidden )
 199+ */
 200+ getSmilAudioHtml: function ( smilElement ){
 201+ return $j('<audio />')
 202+ .attr( {
 203+ 'id' : this.smil.getAssetId( smilElement ),
 204+ 'src' : this.smil.getAssetUrl( $j( smilElement ).attr( 'src' ) )
 205+ } )
 206+ .css( 'display', 'none');
 207+ },
 208+
 209+ /**
195210 * Get Smil CDATA ( passed through jQuery .clean as part of fragment creation )
196211 * XXX Security XXX
197212 * Here we are parsing in SMIL -> HTML should be careful about XSS or script elevation
Index: branches/MwEmbedStandAlone/modules/SmilPlayer/mw.Smil.js
@@ -152,6 +152,13 @@
153153 },
154154
155155 /**
 156+ * Get the set of audio ranges for flattening.
 157+ */
 158+ getAudioTimeSet: function(){
 159+ return this.getBody().getFlatAudioTimeLine();
 160+ },
 161+
 162+ /**
156163 * Pass on the request to start buffering the entire sequence of clips
157164 */
158165 startBuffer: function(){
@@ -234,17 +241,10 @@
235242
236243 /**
237244 * Some Smil Utility functions
238 - */
 245+ */
239246
240247 /**
241 - * Returns a set of audio ranges for flattening.
242 - */
243 - getAudioTimeSet: function( ){
244 - return {};
245 - },
246 -
247 - /**
248 - * maps a smil element id to a html safe id
 248+ * maps a smil element id to a html 'safer' id
249249 * as a decedent subname of the embedPlayer parent
250250 *
251251 * @param {Object} smilElement Element to get id for
@@ -280,9 +280,15 @@
281281 return;
282282 }
283283 // Get the smil type
284 - var smilType = $j( smilElement ).get(0).nodeName.toLowerCase();
 284+ var smilType = $j( smilElement ).get(0).nodeName.toLowerCase();
 285+
 286+ if( this.getBody().smilBlockTypeMap[ smilType ] != 'ref' ){
 287+ mw.log("Error: trying to get ref type of node that is not a ref" + smilType);
 288+ return null;
 289+ }
 290+
 291+ // If the smilType is ref, check for a content type
285292 if( smilType == 'ref' ){
286 - // If the smilType is ref, check for a content type
287293 switch( $j( smilElement ).attr( 'type' ) ) {
288294 case 'text/html':
289295 smilType = 'cdata_html';
@@ -292,6 +298,9 @@
293299 case 'video/webm':
294300 smilType = 'video';
295301 break;
 302+ case 'audio/ogg':
 303+ smilType = 'audio';
 304+ break;
296305 }
297306 }
298307 return smilType;
Index: branches/MwEmbedStandAlone/modules/SmilPlayer/tests/VideoCrossfadeSmil.xml
@@ -17,7 +17,12 @@
1818 </head>
1919 <body>
2020 <par>
21 -
 21+
 22+ <audio src="http://upload.wikimedia.org/wikipedia/commons/5/5a/La_Donna_E_Mobile_Rigoletto.ogg"
 23+ begin="1s"
 24+ dur="10s"
 25+ />
 26+
2227 <video src="http://upload.wikimedia.org/wikipedia/commons/d/d3/Okapia_johnstoni5.ogg"
2328 transIn="fromGreen"
2429 type="video/ogg"
@@ -27,7 +32,7 @@
2833 />
2934
3035 <video src="http://upload.wikimedia.org/wikipedia/commons/0/0d/B-36_bomber.ogg"
31 - begin="5s"
 36+ begin="5s"
3237 transIn="xFade"
3338
3439 fill="transition"
Index: branches/MwEmbedStandAlone/modules/SmilPlayer/mw.EmbedPlayerSmil.js
@@ -79,7 +79,7 @@
8080 var _this = this;
8181 this.getSmil( function( smil ){
8282 smil.renderTime( time, function(){
83 - mw.log( "setCurrentTime:: renderTime callback" );
 83+ //mw.log( "setCurrentTime:: renderTime callback" );
8484 $j('#loadingSpinner_' + _this.id ).remove();
8585
8686 _this.monitor();
@@ -277,9 +277,9 @@
278278 * tracks from movie files.
279279 */
280280 getAudioTimeSet: function(){
281 - if(!this.smil)
 281+ if(!this.smil){
282282 return null;
283 -
 283+ }
284284 return this.smil.getAudioTimeSet();
285285 }
286286
Index: branches/MwEmbedStandAlone/modules/EmbedPlayer/skins/kskin/mw.PlayerSkinKskin.js
@@ -339,7 +339,7 @@
340340 .loadingSpinner()
341341 );
342342
343 - if( mw.getConfig( 'EmbedPlayer.kalturaAttribution' ) == true ){
 343+ if( mw.getConfig( 'EmbedPlayer.KalturaAttribution' ) == true ){
344344 $target.append(
345345 $j( '<div />' )
346346 .addClass( 'k-attribution' )
Index: branches/MwEmbedStandAlone/modules/EmbedPlayer/skins/mw.PlayerControlBuilder.js
@@ -162,7 +162,7 @@
163163 this.supportedComponets['timedText'] = true;
164164 }
165165 // Check for kalturaAttribution
166 - if( mw.getConfig( 'EmbedPlayer.KalturaAttribution' ) ){
 166+ if( mw.getConfig( 'EmbedPlayer.KalturaAttribution' ) ){
167167 this.supportedComponets[ 'kalturaAttribution' ] = true;
168168 }
169169
Index: branches/MwEmbedStandAlone/modules/Sequencer/mw.FirefoggRender.js
@@ -81,7 +81,9 @@
8282
8383
8484 },
85 -
 85+ getPlayer: function(){
 86+ return $j( this.playerTarget ).get( 0 );
 87+ },
8688 // Start rendering
8789 doRender: function() {
8890 var _this = this;
@@ -94,16 +96,30 @@
9597
9698 // Set the continue rendering flag to true:
9799 this.continueRendering = true;
98 -
99 - // Get the player:
100 - this.player = $j( this.playerTarget ).get( 0 );
101100
102101 // Set a target file:
103102 mw.log( "Firefogg Render Settings:" + JSON.stringify( _this.renderOptions ) );
104103 this.fogg.initRender( JSON.stringify( _this.renderOptions ), 'foggRender.ogv' );
105104
106105 // Add audio if we had any:
 106+ var audioSet = this.getPlayer().getAudioTimeSet();
 107+ var previusAudioTime = 0;
 108+ for( var i=0; i < audioSet.length ; i++) {
 109+ var currentAudio = audioSet[i];
 110+ // Check if we need to add silence
 111+ if( currentAudio.startTime > previusAudioTime ){
 112+ mw.log("FirefoggRender::addSilence " + ( currentAudio.startTime - previusAudioTime ));
 113+ this.fogg.addSilence( currentAudio.startTime - previusAudioTime );
 114+ }
 115+ // Add the block of audio from the url
 116+ mw.log("FirefoggRender::addAudioUrl " + currentAudio.src +
 117+ ', ' + currentAudio.offset + ', ' + currentAudio.duration );
 118+ this.fogg.addAudioUrl( currentAudio.src, currentAudio.offset, currentAudio.duration );
107119
 120+ // Update previusAudioTime
 121+ previusAudioTime = currentAudio.startTime + currentAudio.duration;
 122+ }
 123+
108124 // Now issue the save video as call
109125 _this.fogg.saveVideoAs();
110126
@@ -120,14 +136,14 @@
121137 ( Math.round( _this.player.getDuration() * 10 ) / 10 ) );
122138 */
123139
124 - _this.player.setCurrentTime( _this.renderTime, function() {
 140+ _this.getPlayer().setCurrentTime( _this.renderTime, function() {
125141
126142 _this.fogg.addFrame( $j( _this.playerTarget ).attr( 'id' ) );
127143 $j( _this.statusTarget ).text( "AddFrame::" + ( Math.round( _this.renderTime * 1000 ) / 1000 ) );
128144
129145 _this.renderTime += _this.interval;
130146
131 - if ( _this.renderTime >= _this.player.getDuration() || ! _this.continueRendering ) {
 147+ if ( _this.renderTime >= _this.getPlayer().getDuration() || ! _this.continueRendering ) {
132148 _this.doFinalRender();
133149 } else {
134150 // Don't block on render requests

Status & tagging log