1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/changeset/631c9dbca9bc/ changeset: 631c9dbca9bc user: inithello date: 2012-09-05 14:24:29 summary: Added API integration for genome downloader. Improved detection of existing liftover files. affected #: 4 files diff -r e4672c5b57b2234edc9f5e2c9d5abd6b7148fdca -r 631c9dbca9bc83223b886e01c8ae7f676ae8c599 lib/galaxy/jobs/deferred/liftover_transfer.py --- a/lib/galaxy/jobs/deferred/liftover_transfer.py +++ b/lib/galaxy/jobs/deferred/liftover_transfer.py @@ -77,7 +77,7 @@ self.sa_session.flush() return self.job_states.WAIT else: - log.error( "An error occurred while downloading from %s" % job.params[ 'url' ] ) + log.error( "An error occurred while downloading from %s" % job.transfer_job.params[ 'url' ] ) return self.job_states.INVALID elif job.params[ 'type' ] == 'extract_transfer': return self.job_states.READY diff -r e4672c5b57b2234edc9f5e2c9d5abd6b7148fdca -r 631c9dbca9bc83223b886e01c8ae7f676ae8c599 lib/galaxy/web/api/genomes.py --- a/lib/galaxy/web/api/genomes.py +++ b/lib/galaxy/web/api/genomes.py @@ -22,4 +22,73 @@ Returns information about build <id> """ - return self.app.genomes.chroms( trans, dbkey=id, num=num, chrom=chrom, low=low ) \ No newline at end of file + return self.app.genomes.chroms( trans, dbkey=id, num=num, chrom=chrom, low=low ) + + @web.expose_api + def create( self, trans, payload, **kwd ): + """ + POST /api/genomes + Download and/or index a genome. + + Parameters: + + dbkey DB key of the build to download, ignored unless 'UCSC' is specified as the source + ncbi_name NCBI's genome identifier, ignored unless NCBI is specified as the source + ensembl_dbkey Ensembl's genome identifier, ignored unless Ensembl is specified as the source + url_dbkey DB key to use for this build, ignored unless URL is specified as the source + source Data source for this build. Can be: UCSC, Ensembl, NCBI, URL + indexers POST array of indexers to run after downloading (indexers[] = first, indexers[] = second, ...) + func Allowed values: + 'download' Download and index + 'index' Index only + + Returns: + + If no error: + dict( status: 'ok', job: <job ID> ) + + If error: + dict( status: 'error', error: <error message> ) + + """ + params = util.Params( payload ) + from galaxy.web.controllers.data_admin import build_param_dict as massage + paramdict = massage( params, trans ) + func = params.get( 'func', 'download' ) + if func == 'download': + url = paramdict[ 'url' ] + liftover = paramdict[ 'liftover' ] + dbkey = paramdict[ 'dbkey' ] + indexers = paramdict[ 'indexers' ] + longname = paramdict[ 'longname' ] + jobid = trans.app.job_manager.deferred_job_queue.plugins['GenomeTransferPlugin'].create_job( trans, url, dbkey, longname, indexers ) + chainjob = [] + if liftover is not None: + for chain in liftover: + liftover_url = u'ftp://hgdownload.cse.ucsc.edu%s' % chain[0] + from_genome = chain[1] + to_genome = chain[2] + destfile = liftover_url.split('/')[-1].replace('.gz', '') + lochain = trans.app.job_manager.deferred_job_queue.plugins['LiftOverTransferPlugin'].create_job( trans, liftover_url, dbkey, from_genome, to_genome, destfile, jobid ) + chainjob.append( lochain ) + job = trans.app.job_manager.deferred_job_queue.plugins['GenomeTransferPlugin'].get_job_status( jobid ) + job.params['liftover'] = chainjob + trans.app.model.context.current.flush() + return dict( status='ok', job=jobid ) + elif func == 'index': + dbkey = paramdict[ 'dbkey' ] + indexer = [ params.get( 'indexer', None ) ] + longname = None + path = None + for build in trans.app.tool_data_tables.data_tables[ 'all_fasta' ].data: + if build[0] == dbkey: + longname = build[2] + path = build[3] + break + if longname is not None and indexer is not None and path is not None: + jobid = trans.app.job_manager.deferred_job_queue.plugins['GenomeIndexPlugin'].create_job( trans, path, indexer, dbkey, longname ) + return dict( status='ok', job=jobid ) + else: + return dict( status='error', error='Build not %s found in tool data table.' % dbkey ) + else: + return dict( status='error', error='Unkown function selected.' ) diff -r e4672c5b57b2234edc9f5e2c9d5abd6b7148fdca -r 631c9dbca9bc83223b886e01c8ae7f676ae8c599 lib/galaxy/web/controllers/data_admin.py --- a/lib/galaxy/web/controllers/data_admin.py +++ b/lib/galaxy/web/controllers/data_admin.py @@ -152,6 +152,8 @@ params = util.Params( kwd ) jobid = params.get( 'job', '' ) deferred = trans.app.model.context.current.query( model.DeferredJob ).filter_by( id=jobid ).first() + if deferred is None: + return trans.fill_template( '/admin/data_admin/generic_error.mako', message='Invalid genome downloader job specified.' ) gname = deferred.params[ 'intname' ] indexers = ', '.join( deferred.params[ 'indexes' ] ) jobs = self._get_jobs( deferred, trans ) @@ -244,18 +246,20 @@ checker = [] liftover = [] newlift = [] + ftp.retrlines('NLST /goldenPath/%s/liftOver/*.chain.gz' % dbkey, liftover.append) try: - ftp.retrlines('NLST /goldenPath/%s/liftOver/*.chain.gz' % dbkey, liftover.append) for chain in liftover: + lifts = [] fname = chain.split( '/' )[-1] - target = fname.replace( '.over.chain.gz', '' ).split( 'To' )[1] - target = target[0].lower() + target[1:] - if not os.path.exists( os.path.join( trans.app.config.get( 'genome_data_path', 'tool-data/genome' ), dbkey, 'liftOver', fname ) ): - newlift.append( [ chain, dbkey, target ] ) - current = dbkey[0].upper() + dbkey[1:] - targetfile = '%sTo%s.over.chain.gz' % ( target, current ) - if not os.path.exists( os.path.join( trans.app.config.get( 'genome_data_path', 'tool-data/genome' ), target, 'liftOver', targetfile ) ): - newlift.append( [ '/goldenPath/%s/liftOver/%s' % ( target, targetfile ), target, dbkey ] ) + organisms = fname.replace( '.over.chain.gz', '' ).split( 'To' ) + lifts.append( [ organisms[0], organisms[1][0].lower() + organisms[1][1:] ] ) + lifts.append( [ organisms[1][0].lower() + organisms[1][1:], organisms[0] ] ) + for organism in lifts: + remotepath = '/goldenPath/%s/liftOver/%sTo%s.over.chain.gz' % ( organism[0], organism[0], organism[1][0].upper() + organism[1][1:] ) + localfile = '%sTo%s.over.chain' % ( organism[0], organism[1][0].upper() + organism[1][1:] ) + localpath = os.path.join( trans.app.config.get( 'genome_data_path', 'tool-data/genome' ), organism[0], 'liftOver', localfile ) + if not os.path.exists( localpath ) or os.path.getsize( localpath ) == 0: + newlift.append( [ remotepath, organism[0], organism[1] ] ) except: newlift = None pass diff -r e4672c5b57b2234edc9f5e2c9d5abd6b7148fdca -r 631c9dbca9bc83223b886e01c8ae7f676ae8c599 templates/admin/data_admin/local_data.mako --- a/templates/admin/data_admin/local_data.mako +++ b/templates/admin/data_admin/local_data.mako @@ -74,7 +74,7 @@ %endfor </table><h2>Recent jobs:</h2> - <p>Click the job ID to see job details. Note that this list only shows jobs initiated by your account.</p> + <p>Click the job ID to see job details and the status of any individual sub-jobs. Note that this list only shows jobs initiated by your account.</p><div id="recentJobs"> %for job in jobgrid: <div id="job-${job['deferred']}" data-dbkey="${job['dbkey']}" data-name="${job['intname']}" data-indexes="${job['indexers']}" data-jobid="${job['deferred']}" data-state="${job['state']}" class="historyItem-${job['state']} historyItemWrapper historyItem"> Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.