r87953 MediaWiki - Code Review archive

Repository:	MediaWiki
Revision:	< r87952‎ \| r87953 \| r87954 >
Date:	09:13, 13 May 2011
Author:	dale
Status:	deferred
Tags:
Comment:	improved shared repo support for derivative sources fixed transcodeTable.sql tweeked output for job runner status
Modified paths:	/trunk/extensions/TimedMediaHandler/ApiQueryVideoInfo.php (modified) (history) /trunk/extensions/TimedMediaHandler/WebVideoTranscode/WebVideoTranscode.php (modified) (history) /trunk/extensions/TimedMediaHandler/WebVideoTranscode/transcodeTable.sql (modified) (history) /trunk/extensions/TimedMediaHandler/maintenance/WebVideoJobRunner.php (modified) (history)

Diff [purge]

Index: trunk/extensions/TimedMediaHandler/maintenance/WebVideoJobRunner.php
—	—	@@ -66,10 +66,10 @@
67	67	// Add one process:
68	68	$cmd = "php $wgMaintenancePath/runJobs.php --type webVideoTranscode --maxjobs 1 --maxtime {$this->transcodeTimeout}";
69	69	$status = $this->runBackgroundProc( $cmd );
70		~~- $this->output( "$runingJobsCount existing job runners, Starting new transcode job runner:" );~~
	70	+ $this->output( "$runingJobsCount existing job runners, Check for new transcode jobs: " );
71	71	} else {
72	72	// Just output a "tick"
73		~~- $this->output( "$runingJobsCount transcode jobs active" );~~
	73	+ $this->output( "$runingJobsCount transcode jobs active:\n" );
74	74	}
75	75	}
76	76
Index: trunk/extensions/TimedMediaHandler/ApiQueryVideoInfo.php
—	—	@@ -21,7 +21,7 @@
22	22	return 'Extends imageinfo to include video source information';
23	23	}
24	24
25		~~- static function getInfo( $file, $prop, $result, $thumbParams = null ) {~~
	25	+ static function getInfo( $file, $prop, $result, $thumbParams = null, $version = 'latest' ) {
26	26	$vals = parent::getInfo( $file, $prop, $result, $thumbParams = null );
27	27	if( isset( $prop['derivatives'] ) ){
28	28	$vals['derivatives'] = WebVideoTranscode::getSources( $file, array( 'nodata', 'fullurl') );
Index: trunk/extensions/TimedMediaHandler/WebVideoTranscode/transcodeTable.sql
—	—	@@ -2,14 +2,16 @@
3	3	-- 2011-04-05 <mdale@wikimedia.org>
4	4	--
5	5	CREATE TABLE /$wgDBprefix/transcode (
6		~~- transcode_id INT NOT NULL PRIMARY KEY,~~
7		~~- transcode_image_name varchar(255) binary NOT NULL,~~
8		~~- transcode_key varchar( 48 ) NOT NULL ,~~
9		~~- transcode_error BLOB NULL,~~
10		~~- transcode_time_addjob VARBINARY( 14 ) NULL,~~
11		~~- transcode_time_startwork VARBINARY( 14 ) NULL,~~
12		~~- transcode_time_success VARBINARY( 14 ) NULL,~~
13		~~- transcode_time_error VARBINARY( 14 ) NULL,~~
	6	+ transcode_id INT NOT NULL AUTO_INCREMENT,
	7	+ transcode_image_name VARCHAR(255) NOT NULL,
	8	+ transcode_key VARCHAR(48) NOT NULL,
	9	+ transcode_error BLOB NOT NULL,
	10	+ transcode_time_addjob DATETIME NULL,
	11	+ transcode_time_startwork DATETIME NULL,
	12	+ transcode_time_success DATETIME NULL,
	13	+ transcode_time_error DATETIME NULL,
	14	+ transcode_final_bitrate INT NOT NULL,
	15	+ PRIMARY KEY (`transcode_id`)
14	16	);
15		~~-CREATE INDEX ( transcode_image_name , transcode_state , transcode_key ),~~
16		~~-CREATE INDEX ( transcode_time_addjob ,transcode_time_startwork , transcode_time_success, transcode_time_error ),~~
	17	+CREATE INDEX /$wgDBprefix/transcode_name_inx ON /$wgDBprefix/transcode( transcode_image_name, transcode_key );
	18	+CREATE INDEX /$wgDBprefix/transcode_time_inx ON /$wgDBprefix/transcode( transcode_time_addjob ,transcode_time_startwork , transcode_time_success, transcode_time_error );
Index: trunk/extensions/TimedMediaHandler/WebVideoTranscode/WebVideoTranscode.php
—	—	@@ -98,7 +98,6 @@
99	99	'videoCodec' => 'theora',
100	100	),
101	101
102		-
103	102	// WebM transcode:
104	103	WebVideoTranscode::ENC_WEBM_5MBS =>
105	104	array(
—	—	@@ -120,7 +119,7 @@
121	120	'twopass' => 'true',
122	121	'keyframeInterval' => '128',
123	122	'bufDelay' => '256',
124		~~- 'videoCodec' => 'vp8',~~
	123	+ 'videoCodec' => 'vp8',
125	124	),
126	125	WebVideoTranscode::ENC_WEBM_HQ_VBR =>
127	126	array(
—	—	@@ -141,7 +140,14 @@
142	141	$file->getName() . '.' .
143	142	$transcodeKey ;
144	143	}
145		-
	144	+ /**
	145	+ * Get the target encode path for a video encode
	146	+ *
	147	+ * @param File $file
	148	+ * @param String $transcodeKey
	149	+ *
	150	+ * @returns the local target encode path
	151	+ */
146	152	static public function getTargetEncodePath( &$file, $transcodeKey ){
147	153	// TODO probably should use some other temporary non-web accessible location for
148	154	// in-progress encodes.
—	—	@@ -165,7 +171,74 @@
166	172
167	173	/**
168	174	* Static function to get the set of video assets
	175	+ * Checks if the file is local or remote and grabs respective sources
	176	+ */
	177	+ static public function getSources( &$file , $options=array() ){
	178	+ if( $file->isLocal() ){
	179	+ return self::getLocalSources( $file , $options );
	180	+ }else {
	181	+ return self::getRemoteSources( $file , $options );
	182	+ }
	183	+ }
	184	+ /**
	185	+ * Grabs sources from the remote repo via ApiQueryVideoInfo.php entry point.
169	186	*
	187	+ * Because this works on both TimedMediaHandler commons and no TimedMediaHandler commons
	188	+ */
	189	+ static public function getRemoteSources(&$file , $options=array() ){
	190	+ global $wgMemc;
	191	+ // Setup source attribute options
	192	+ $dataPrefix = in_array( 'nodata', $options )? '': 'data-';
	193	+
	194	+ // Use descriptionCacheExpiry as our expire for timed text tracks info
	195	+ if ( $file->repo->descriptionCacheExpiry > 0 ) {
	196	+ wfDebug("Attempting to get sources from cache...");
	197	+ $key = $this->file->repo->getLocalCacheKey( 'WebVideoSources', 'url', $file->getName() );
	198	+ $sources = $wgMemc->get($key);
	199	+ if ( $sources ) {
	200	+ wfDebug("Success found sources in local cache\n");
	201	+ return $sources;
	202	+ }
	203	+ wfDebug("source cache miss\n");
	204	+ }
	205	+ wfDebug("Get Video sources from remote api \n");
	206	+ $data = $file->repo->fetchImageQuery( array(
	207	+ 'action' => 'query',
	208	+ 'prop' => 'videoinfo',
	209	+ 'viprop' => 'derivatives',
	210	+ 'title' => $file->getTitle()->getDBKey()
	211	+ ) );
	212	+
	213	+ if( isset( $data['warnings'] ) && isset( $data['warnings']['query'] )
	214	+ && $data['warnings']['query']['*'] == "Unrecognized value for parameter 'prop': videoinfo" )
	215	+ {
	216	+ // Commons does not yet have TimedMediaHandler.
	217	+ // Use the normal file repo system single source:
	218	+ return array( self::getPrimarySourceAttributes( $file, $dataPrefix ) );
	219	+ }
	220	+ $sources = array();
	221	+ // Generate the source list from the data response:
	222	+ if( $data['query'] && $data['query']['pages'] ){
	223	+ $vidResult = first( $data['query']['pages'] );
	224	+ if( $vidResult['videoinfo'] ){
	225	+ $derResult = first( $vidResult['videoinfo'] );
	226	+ $derivatives = $derResult['derivatives'];
	227	+ foreach( $derivatives as $derivativeSource ){
	228	+ $sources[] = $derivativeSource;
	229	+ }
	230	+ }
	231	+ }
	232	+
	233	+ // Update the cache:
	234	+ if ( $sources && $this->file->repo->descriptionCacheExpiry > 0 ) {
	235	+ $wgMemc->set( $key, $sources, $this->file->repo->descriptionCacheExpiry );
	236	+ }
	237	+
	238	+ return $sources;
	239	+
	240	+ }
	241	+
	242	+ /*
170	243	* Based on the $wgEnabledTranscodeSet set of enabled derivatives we
171	244	* sync the database with $wgEnabledTranscodeSet
172	245	*
—	—	@@ -176,42 +249,18 @@
177	250	* 'nodata' Strips the data- attribute, useful when your output is not html
178	251	* @returns an associative array of sources suitable for <source> tag output
179	252	*/
180		~~- static public function getSources( &$file , $options=array() ){~~
	253	+ static public function getLocalSources( &$file , $options=array() ){
181	254	global $wgEnabledTranscodeSet, $wgLang;
182	255	$sources = array();
183	256
184		~~- // Setup options~~
185		~~- $dataPrefix = in_array( 'nodata', $options )? '': 'data-';~~
186		-
187	257	// Add the original file:
188		~~- $source = array(~~
189		~~- 'src' => $file->getUrl(),~~
190		~~- 'title' => wfMsg('timedmedia-source-file-desc',~~
191		~~- $file->getHandler()->getMetadataType(),~~
192		~~- $wgLang->formatNum( $file->getWidth() ),~~
193		~~- $wgLang->formatNum( $file->getHeight() ),~~
194		~~- $wgLang->formatBitrate( $file->getHandler()->getBitrate( $file ) )~~
195		~~- ),~~
196		~~- "{$dataPrefix}shorttitle" => wfMsg('timedmedia-source-file', wfMsg( 'timedmedia-' . $file->getHandler()->getMetadataType() ) ),~~
197		~~- "{$dataPrefix}width" => $file->getWidth(),~~
198		~~- "{$dataPrefix}height" => $file->getHeight(),~~
199		~~- // TODO add some title and data about the file~~
200		~~- );~~
	258	+ $source = self::getPrimarySourceAttributes( $file, $options );
201	259
202	260	// Just directly return audio sources ( No transcoding for audio for now )
203	261	if( $file->getHandler()->isAudio( $file ) ){
204	262	return array( $source );
205	263	}
206	264
207		~~- // For video include bitrate and framerate:~~
208		~~- $bitrate = $file->getHandler()->getBitrate( $file );~~
209		~~- if( $bitrate )~~
210		~~- $source["{$dataPrefix}bandwidth"] = round ( $bitrate );~~
211		-
212		~~- $framerate = $file->getHandler()->getFramerate( $file );~~
213		~~- if( $framerate )~~
214		~~- $source[ "{$dataPrefix}framerate" ] = $framerate;~~
215		-
216	265	// Add the source to the sources array
217	266	$sources[] = $source;
218	267
—	—	@@ -247,18 +296,18 @@
248	297	$addWebMFlag = true;
249	298	}
250	299	// Try and add the source
251		~~- self::tryAddSource( $file, $sources, $transcodeKey, $dataPrefix );~~
	300	+ self::addSourceIfReady( $file, $sources, $transcodeKey, $options );
252	301	}
253	302
254	303	// Make sure we have at least one ogg and webm encode
255	304	if( !$addOggFlag \|\| !$addWebMFlag){
256	305	foreach( $wgEnabledTranscodeSet as $transcodeKey ){
257	306	if( !$addOggFlag && self::$derivativeSettings[$transcodeKey]['videoCodec'] == 'theora' ){
258		~~- self::tryAddSource( $file, $sources, $transcodeKey, $dataPrefix );~~
	307	+ self::addSourceIfReady( $file, $sources, $transcodeKey, $options );
259	308	$addOggFlag = true;
260	309	}
261	310	if( !$addWebMFlag && self::$derivativeSettings[$transcodeKey]['videoCodec'] == 'vp8' ){
262		~~- self::tryAddSource( $file, $sources, $transcodeKey, $dataPrefix );~~
	311	+ self::addSourceIfReady( $file, $sources, $transcodeKey, $options );
263	312	$addWebMFlag = true;
264	313	}
265	314	}
—	—	@@ -266,7 +315,6 @@
267	316
268	317	return $sources;
269	318	}
270		-
271	319	/**
272	320	* Get the transcode state for a given filename and transcodeKey
273	321	*
—	—	@@ -330,31 +378,79 @@
331	379	}
332	380
333	381	/**
334		~~- * Try to add a source to the sources list~~
	382	+ * Add a source to the sources list if the transcode job is ready
335	383	* if the source is not found update the job queue
336	384	*/
337		~~- public static function tryAddSource( &$file, &$sources, $transcodeKey, $dataPrefix = '' ){~~
	385	+ public static function addSourceIfReady( &$file, &$sources, $transcodeKey, $dataPrefix = '' ){
338	386	global $wgLang;
339	387	$fileName = $file->getTitle()->getDbKey();
	388	+ // Check if the transcode is ready:
	389	+ if( self::isTranscodeReady( $fileName, $transcodeKey ) ){
	390	+ $sources[] = self::getDerivativeSourceAttributes( $file, $transcodeKey, $dataPrefix );
	391	+ } else {
	392	+ self::updateJobQueue( $file, $transcodeKey );
	393	+ }
	394	+ }
	395	+ /**
	396	+ * Get the primary "source" asset used for other derivatives
	397	+ */
	398	+ static public function getPrimarySourceAttributes($file, $options){
	399	+ global $wgLang;
	400	+ // Setup source attribute options
	401	+ $dataPrefix = in_array( 'nodata', $options )? '': 'data-';
	402	+ $src = in_array( 'fullurl', $options)? wfExpandUrl( $file->getUrl() ) : $file->getUrl();
340	403
	404	+ $source = array(
	405	+ 'src' => $src,
	406	+ 'title' => wfMsg('timedmedia-source-file-desc',
	407	+ $file->getHandler()->getMetadataType(),
	408	+ $wgLang->formatNum( $file->getWidth() ),
	409	+ $wgLang->formatNum( $file->getHeight() ),
	410	+ $wgLang->formatBitrate( $file->getHandler()->getBitrate( $file ) )
	411	+ ),
	412	+ "{$dataPrefix}shorttitle" => wfMsg('timedmedia-source-file', wfMsg( 'timedmedia-' . $file->getHandler()->getMetadataType() ) ),
	413	+ "{$dataPrefix}width" => $file->getWidth(),
	414	+ "{$dataPrefix}height" => $file->getHeight(),
	415	+ );
	416	+
	417	+ $bitrate = $file->getHandler()->getBitrate( $file );
	418	+ if( $bitrate )
	419	+ $source["{$dataPrefix}bandwidth"] = round ( $bitrate );
	420	+
	421	+ // For video include framerate:
	422	+ if( !$file->getHandler()->isAudio( $file ) ){
	423	+ $framerate = $file->getHandler()->getFramerate( $file );
	424	+ if( $framerate )
	425	+ $source[ "{$dataPrefix}framerate" ] = $framerate;
	426	+ }
	427	+ return $source;
	428	+ }
	429	+ /**
	430	+ * Get derivative "source" attributes
	431	+ */
	432	+ static public function getDerivativeSourceAttributes($file, $transcodeKey, $options){
	433	+ $dataPrefix = in_array( 'nodata', $options )? '': 'data-';
	434	+
	435	+
	436	+ $fileName = $file->getTitle()->getDbKey();
	437	+
341	438	$thumbName = $file->thumbName( array() );
342	439	$thumbUrl = $file->getThumbUrl( $thumbName );
343	440	$thumbUrlDir = dirname( $thumbUrl );
344		-
345		~~- // Check if the transcode is ready:~~
346		~~- if( self::isTranscodeReady( $fileName, $transcodeKey ) ){~~
347		-
348		~~- list( $width, $height ) = WebVideoTranscode::getMaxSizeTransform(~~
349		~~- $file,~~
350		~~- self::$derivativeSettings[$transcodeKey]['maxSize']~~
351		~~- );~~
352		-
353		~~- $framerate = ( isset( self::$derivativeSettings[$transcodeKey]['framerate'] ) )?~~
	441	+
	442	+ list( $width, $height ) = WebVideoTranscode::getMaxSizeTransform(
	443	+ $file,
	444	+ self::$derivativeSettings[$transcodeKey]['maxSize']
	445	+ );
	446	+
	447	+ $framerate = ( isset( self::$derivativeSettings[$transcodeKey]['framerate'] ) )?
354	448	self::$derivativeSettings[$transcodeKey]['framerate'] :
355	449	$file->getHandler()->getFramerate( $file );
356		-
357		~~- $sources[] = array(~~
358		~~- 'src' => $thumbUrlDir . '/' .$file->getName() . '.' . $transcodeKey,~~
	450	+ // Setup the url src:
	451	+ $src = $thumbUrlDir . '/' .$file->getName() . '.' . $transcodeKey;
	452	+ $src = in_array( 'fullurl', $options)? wfExpandUrl( $src ) : $src;
	453	+ return array(
	454	+ 'src' => $src,
359	455	'title' => wfMsg('timedmedia-derivative-desc-' . $transcodeKey ),
360	456	"{$dataPrefix}shorttitle" => wfMsg('timedmedia-derivative-' . $transcodeKey),
361	457
—	—	@@ -366,11 +462,7 @@
367	463	"{$dataPrefix}bandwidth" => self::$transcodeStateCache[$fileName][ $transcodeKey ]['bitrate'],
368	464	"{$dataPrefix}framerate" => $framerate,
369	465	);
370		~~- } else {~~
371		~~- self::updateJobQueue( $file, $transcodeKey );~~
372		~~- }~~
373	466	}
374		-
375	467	/**
376	468	* Update the job queue if the file is not already in the job queue:
377	469	* @param object File object

Status & tagging log

17:01, 14 May 2011 Reedy (talk | contribs) changed the status of r87953 [removed: new added: deferred]