- galaxy-commits - lists.galaxyproject.org

galaxy-dist commit 9415f380f2e7: Added validation checks in the data transfer script for sample tracking
by commits-noreply＠bitbucket.org 20 Nov '10

20 Nov '10

# HG changeset patch -- Bitbucket.org # Project galaxy-dist # URL http://bitbucket.org/galaxy/galaxy-dist/overview # User rc # Date 1288380270 14400 # Node ID 9415f380f2e7d4aca44f74923b6984ab3bcc1dfb # Parent 0518ce52d8d3ca15ec6070f2addb4bfe7608c23b Added validation checks in the data transfer script for sample tracking --- a/scripts/galaxy_messaging/server/data_transfer.py +++ b/scripts/galaxy_messaging/server/data_transfer.py @@ -3,12 +3,12 @@ Data Transfer Script: Sequencer to Galaxy -This script is called from Galaxy LIMS once the lab admin starts the data -transfer process using the user interface. +This script is called from Galaxy RabbitMQ listener ( amqp_consumer.py ) once +the lab admin starts the data transfer process using the user interface. Usage: -python data_transfer.py <data_transfer_xml><config_id_secret> +python data_transfer.py <config_file> """ @@ -19,13 +19,13 @@ import urllib,urllib2, cookielib, shutil import logging, time, datetime import xml.dom.minidom -log = logging.getLogger("datatx_"+str(os.getpid())) -log.setLevel(logging.DEBUG) -fh = logging.FileHandler("data_transfer.log") -fh.setLevel(logging.DEBUG) -formatter = logging.Formatter("%(asctime)s - %(name)s - %(message)s") -fh.setFormatter(formatter) -log.addHandler(fh) +log = logging.getLogger( "datatx_" + str( os.getpid() ) ) +log.setLevel( logging.DEBUG ) +fh = logging.FileHandler( "data_transfer.log" ) +fh.setLevel( logging.DEBUG ) +formatter = logging.Formatter( "%(asctime)s - %(name)s - %(message)s" ) +fh.setFormatter( formatter ) +log.addHandler( fh ) api_path = [ os.path.join( os.getcwd(), "scripts/api" ) ] sys.path.extend( api_path ) @@ -47,60 +47,76 @@ import pexpect pkg_resources.require( "simplejson" ) import simplejson -class DataTransfer(object): - - def __init__(self, msg, config_file): - log.info(msg) - self.dom = xml.dom.minidom.parseString(msg) - self.sequencer_host = self.get_value(self.dom, 'data_host') - self.sequencer_username = self.get_value(self.dom, 'data_user') - self.sequencer_password = self.get_value(self.dom, 'data_password') - self.request_id = self.get_value(self.dom, 'request_id') - self.sample_id = self.get_value(self.dom, 'sample_id') - self.library_id = self.get_value(self.dom, 'library_id') - self.folder_id = self.get_value(self.dom, 'folder_id') +class DataTransfer( object ): + + def __init__( self, msg, config_file ): + log.info( msg ) + self.dom = xml.dom.minidom.parseString( msg ) + self.sequencer_host = self.get_value( self.dom, 'data_host' ) + self.sequencer_username = self.get_value( self.dom, 'data_user' ) + self.sequencer_password = self.get_value( self.dom, 'data_password' ) + self.request_id = self.get_value( self.dom, 'request_id' ) + self.sample_id = self.get_value( self.dom, 'sample_id' ) + self.library_id = self.get_value( self.dom, 'library_id' ) + self.folder_id = self.get_value( self.dom, 'folder_id' ) self.dataset_files = [] count=0 while True: - dataset_id = self.get_value_index(self.dom, 'dataset_id', count) - file = self.get_value_index(self.dom, 'file', count) - name = self.get_value_index(self.dom, 'name', count) - if file: - self.dataset_files.append(dict(name=name, - dataset_id=int(dataset_id), - file=file)) - else: - break - count=count+1 + dataset_id = self.get_value_index( self.dom, 'dataset_id', count ) + file = self.get_value_index( self.dom, 'file', count ) + name = self.get_value_index( self.dom, 'name', count ) + if file: + self.dataset_files.append( dict( name=name, + dataset_id=int( dataset_id ), + file=file ) ) + else: + break + count=count+1 try: # Retrieve the upload user login information from the config file - transfer_datasets_config = ConfigParser.ConfigParser() - transfer_datasets_config.read('transfer_datasets.ini') - self.data_transfer_user_email = transfer_datasets_config.get("data_transfer_user_login_info", "email") - self.data_transfer_user_password = transfer_datasets_config.get("data_transfer_user_login_info", "password") - self.api_key = transfer_datasets_config.get("data_transfer_user_login_info", "api_key") - self.http_server_section = transfer_datasets_config.get("universe_wsgi_config", "http_server_section") - config = ConfigParser.ConfigParser() - config.read( config_file ) - self.server_host = config.get(self.http_server_section, "host") - self.server_port = config.get(self.http_server_section, "port") - self.database_connection = config.get("app:main", "database_connection") - self.import_dir = config.get("app:main", "library_import_dir") - self.config_id_secret = config.get("app:main", "id_secret") - - # create the destination directory within the import directory - self.server_dir = os.path.join( self.import_dir, 'datatx_'+str(os.getpid())+'_'+datetime.date.today().strftime("%d%b%Y") ) - os.mkdir(self.server_dir) - if not os.path.exists(self.server_dir): - raise Exception + transfer_datasets_config = ConfigParser.ConfigParser( ) + transfer_datasets_config.read( 'transfer_datasets.ini' ) + self.data_transfer_user_email = transfer_datasets_config.get( "data_transfer_user_login_info", "email" ) + self.data_transfer_user_password = transfer_datasets_config.get( "data_transfer_user_login_info", "password" ) + self.api_key = transfer_datasets_config.get( "data_transfer_user_login_info", "api_key" ) + self.http_server_section = transfer_datasets_config.get( "universe_wsgi_config", "http_server_section" ) except: - log.error(traceback.format_exc()) - log.error('FATAL ERROR') - if self.database_connection: - self.error_and_exit('Error') + log.error( traceback.format_exc() ) + log.error( 'ERROR reading config values from transfer_datasets.ini.' ) sys.exit(1) + # read config variables + config = ConfigParser.ConfigParser() + retval = config.read( config_file ) + if not retval: + error_msg = 'FATAL ERROR: Unable to open config file %s.' % config_file + log.error( error_msg ) + sys.exit(1) + try: + self.server_host = config.get( self.http_server_section, "host" ) + except ConfigParser.NoOptionError,e: + self.server_host = '127.0.0.1' + try: + self.server_port = config.get( self.http_server_section, "port" ) + except ConfigParser.NoOptionError,e: + self.server_port = '8080' + try: + self.config_id_secret = config.get( "app:main", "id_secret" ) + except ConfigParser.NoOptionError,e: + self.config_id_secret = "USING THE DEFAULT IS NOT SECURE!" + try: + self.import_dir = config.get( "app:main", "library_import_dir" ) + except ConfigParser.NoOptionError,e: + log.error( 'ERROR: library_import_dir config variable is not set in %s. ' % config_file ) + sys.exit( 1 ) + # create the destination directory within the import directory + self.server_dir = os.path.join( self.import_dir, + 'datatx_' + str( os.getpid() ) + '_' + datetime.date.today( ).strftime( "%d%b%Y" ) ) + try: + os.mkdir( self.server_dir ) + except Exception, e: + self.error_and_exit( str( e ) ) - def start(self): + def start( self ): ''' This method executes the file transfer from the sequencer, adds the dataset to the data library & finally updates the data transfer status in the db @@ -110,72 +126,73 @@ class DataTransfer(object): # add the dataset to the given library self.add_to_library() # update the data transfer status in the db - self.update_status(SampleDataset.transfer_status.COMPLETE) + self.update_status( SampleDataset.transfer_status.COMPLETE ) # cleanup #self.cleanup() - sys.exit(0) + sys.exit( 0 ) - def cleanup(self): + def cleanup( self ): ''' remove the directory created to store the dataset files temporarily before adding the same to the data library ''' try: - time.sleep(60) + time.sleep( 60 ) shutil.rmtree( self.server_dir ) except: self.error_and_exit() - def error_and_exit(self, msg=''): + def error_and_exit( self, msg='' ): ''' This method is called any exception is raised. This prints the traceback and terminates this script ''' - log.error(traceback.format_exc()) - log.error('FATAL ERROR.'+msg) - self.update_status('Error', 'All', msg+"\n"+traceback.format_exc()) - sys.exit(1) + log.error( traceback.format_exc() ) + log.error( 'FATAL ERROR.' + msg ) + self.update_status( 'Error', 'All', msg ) + sys.exit( 1 ) - def transfer_files(self): + def transfer_files( self ): ''' This method executes a scp process using pexpect library to transfer the dataset file from the remote sequencer to the Galaxy server ''' - def print_ticks(d): + def print_ticks( d ): pass - for i, df in enumerate(self.dataset_files): - self.update_status(SampleDataset.transfer_status.TRANSFERRING, df['dataset_id']) + for i, dataset_file in enumerate( self.dataset_files ): + self.update_status( SampleDataset.transfer_status.TRANSFERRING, dataset_file[ 'dataset_id' ] ) try: cmd = "scp %s@%s:'%s' '%s/%s'" % ( self.sequencer_username, self.sequencer_host, - df['file'].replace(' ', '\ '), - self.server_dir.replace(' ', '\ '), - df['name'].replace(' ', '\ ') ) - log.debug(cmd) - output = pexpect.run(cmd, events={'.ssword:*': self.sequencer_password+'\r\n', - pexpect.TIMEOUT:print_ticks}, - timeout=10) - log.debug(output) - path = os.path.join(self.server_dir, os.path.basename(df['name'])) - if not os.path.exists(path): - msg = 'Could not find the local file after transfer (%s)' % path - log.error(msg) - raise Exception(msg) + dataset_file[ 'file' ].replace( ' ', '\ ' ), + self.server_dir.replace( ' ', '\ ' ), + dataset_file[ 'name' ].replace( ' ', '\ ' ) ) + log.debug( cmd ) + output = pexpect.run( cmd, + events={ '.ssword:*': self.sequencer_password+'\r\n', + pexpect.TIMEOUT: print_ticks }, + timeout=10 ) + log.debug( output ) + path = os.path.join( self.server_dir, os.path.basename( dataset_file[ 'name' ] ) ) + if not os.path.exists( path ): + msg = 'Could not find the local file after transfer ( %s ).' % path + log.error( msg ) + raise Exception( msg ) except Exception, e: msg = traceback.format_exc() - self.update_status('Error', df['dataset_id'], msg) + self.update_status( 'Error', dataset_file['dataset_id'], msg) - def add_to_library(self): + def add_to_library( self ): ''' This method adds the dataset file to the target data library & folder by opening the corresponding url in Galaxy server running. ''' - self.update_status(SampleDataset.transfer_status.ADD_TO_LIBRARY) + self.update_status( SampleDataset.transfer_status.ADD_TO_LIBRARY ) try: data = {} - data[ 'folder_id' ] = api.encode_id( self.config_id_secret, '%s.%s' % ( 'folder', self.folder_id ) ) + data[ 'folder_id' ] = api.encode_id( self.config_id_secret, '%s.%s' % ( 'folder', self.folder_id ) ) data[ 'file_type' ] = 'auto' data[ 'server_dir' ] = self.server_dir data[ 'dbkey' ] = '' @@ -183,23 +200,23 @@ class DataTransfer(object): data[ 'create_type' ] = 'file' url = "http://%s:%s/api/libraries/%s/contents" % ( self.server_host, self.server_port, - api.encode_id( self.config_id_secret, self.library_id ) ) - log.debug( str( ( self.api_key, url, data ) ) ) + api.encode_id( self.config_id_secret, self.library_id ) ) + log.debug( str( ( self.api_key, url, data ) ) ) retval = api.submit( self.api_key, url, data, return_formatted=False ) - log.debug( str( retval ) ) + log.debug( str( retval ) ) except Exception, e: - self.error_and_exit( str( e ) ) + self.error_and_exit( str( e ) ) - def update_status(self, status, dataset_id='All', msg=''): + def update_status( self, status, dataset_id='All', msg='' ): ''' Update the data transfer status for this dataset in the database ''' try: - log.debug('Setting status "%s" for dataset "%s" of sample "%s"' % ( status, str(dataset_id), str(self.sample_id) ) ) + log.debug( 'Setting status "%s" for dataset "%s" of sample "%s"' % ( status, str( dataset_id ), str( self.sample_id) ) ) sample_dataset_ids = [] if dataset_id == 'All': for dataset in self.dataset_files: - sample_dataset_ids.append( api.encode_id( self.config_id_secret, dataset['dataset_id'] ) ) + sample_dataset_ids.append( api.encode_id( self.config_id_secret, dataset[ 'dataset_id' ] ) ) else: sample_dataset_ids.append( api.encode_id( self.config_id_secret, dataset_id ) ) # update the transfer status @@ -208,37 +225,37 @@ class DataTransfer(object): data[ 'sample_dataset_ids' ] = sample_dataset_ids data[ 'new_status' ] = status data[ 'error_msg' ] = msg - url = "http://%s:%s/api/requests/%s" % ( self.server_host, + url = "http://%s:%s/api/requests/%s" % ( self.server_host, self.server_port, - api.encode_id( self.config_id_secret, self.request_id ) ) - log.debug(str((self.api_key, url, data))) + api.encode_id( self.config_id_secret, self.request_id ) ) + log.debug( str( ( self.api_key, url, data))) retval = api.update( self.api_key, url, data, return_formatted=False ) log.debug( str( retval ) ) except urllib2.URLError, e: - log.debug( 'ERROR(sample_dataset_transfer_status (%s)): %s' % ( url, str(e) ) ) - log.error(traceback.format_exc()) + log.debug( 'ERROR( sample_dataset_transfer_status ( %s ) ): %s' % ( url, str( e ) ) ) + log.error( traceback.format_exc() ) except: - log.error(traceback.format_exc()) - log.error('FATAL ERROR') - sys.exit(1) + log.error( traceback.format_exc() ) + log.error( 'FATAL ERROR' ) + sys.exit( 1 ) - def get_value(self, dom, tag_name): + def get_value( self, dom, tag_name ): ''' This method extracts the tag value from the xml message ''' - nodelist = dom.getElementsByTagName(tag_name)[0].childNodes + nodelist = dom.getElementsByTagName( tag_name )[ 0 ].childNodes rc = "" for node in nodelist: if node.nodeType == node.TEXT_NODE: rc = rc + node.data return rc - def get_value_index(self, dom, tag_name, dataset_id): + def get_value_index( self, dom, tag_name, dataset_id ): ''' This method extracts the tag value from the xml message ''' try: - nodelist = dom.getElementsByTagName(tag_name)[dataset_id].childNodes + nodelist = dom.getElementsByTagName( tag_name )[ dataset_id ].childNodes except: return None rc = "" @@ -248,11 +265,11 @@ class DataTransfer(object): return rc if __name__ == '__main__': - log.info('STARTING %i %s' % (os.getpid(), str(sys.argv))) + log.info( 'STARTING %i %s' % ( os.getpid(), str( sys.argv ) ) ) # # Start the daemon # - dt = DataTransfer(sys.argv[1], sys.argv[2]) + dt = DataTransfer( sys.argv[1], sys.argv[2]) dt.start() - sys.exit(0) + sys.exit( 0 )

1 0

galaxy-dist commit a22917c97fbe: Merge.
by commits-noreply＠bitbucket.org 20 Nov '10

20 Nov '10

# HG changeset patch -- Bitbucket.org # Project galaxy-dist # URL http://bitbucket.org/galaxy/galaxy-dist/overview # User Dannon Baker <dannonbaker(a)me.com> # Date 1288377687 14400 # Node ID a22917c97fbe89aa2c0393d30f51d2a2183fd2f8 # Parent 9f74170b7781122577b8cf7ffe38f5c986617844 # Parent cbb895e2b2729d324c1693882de7939360e683a4 Merge. --- a/test-data/cuffdiff_out5.tracking +++ /dev/null @@ -1,2 +0,0 @@ -tracking_id class_code nearest_ref_id gene_short_name tss_id locus q0_FPKM q0_conf_lo q0_conf_hi q1_FPKM q1_conf_lo q1_conf_hi -TCONS_00000001 c Xkr4 - - chr1:3204754-3204833 0 0 0 67745 0 163551

1 0

galaxy-dist commit 2c934a168af9: More tweaking of lines_diff for cuffdiff test.
by commits-noreply＠bitbucket.org 20 Nov '10

20 Nov '10

# HG changeset patch -- Bitbucket.org # Project galaxy-dist # URL http://bitbucket.org/galaxy/galaxy-dist/overview # User jeremy goecks <jeremy.goecks(a)emory.edu> # Date 1288633591 14400 # Node ID 2c934a168af968dc84300412220c2f2cb74bf676 # Parent 9415f380f2e7d4aca44f74923b6984ab3bcc1dfb More tweaking of lines_diff for cuffdiff test. --- a/tools/ngs_rna/cuffdiff_wrapper.xml +++ b/tools/ngs_rna/cuffdiff_wrapper.xml @@ -78,11 +78,11 @@ TODO: can we find datasets that lead to deterministic behavior? --><output name="isoforms_exp" file="cuffdiff_out1.txt" lines_diff="14"/> - <output name="genes_exp" file="cuffdiff_out2.txt" lines_diff="4"/> + <output name="genes_exp" file="cuffdiff_out2.txt" lines_diff="10"/><output name="tss_groups_exp" file="cuffdiff_out3.txt"/><output name="cds_exp_fpkm_tracking" file="cuffdiff_out4.txt"/> - <output name="isoforms_fpkm_tracking" file="cuffdiff_out5.txt" lines_diff="8"/> - <output name="genes_fpkm_tracking" file="cuffdiff_out6.txt" lines_diff="4"/> + <output name="isoforms_fpkm_tracking" file="cuffdiff_out5.txt" lines_diff="16"/> + <output name="genes_fpkm_tracking" file="cuffdiff_out6.txt" lines_diff="8"/><output name="tss_groups_fpkm_tracking" file="cuffdiff_out7.txt"/><output name="cds_fpkm_tracking" file="cuffdiff_out8.txt"/><output name="splicing_diff" file="cuffdiff_out9.txt"/>

1 0

galaxy-dist commit 0518ce52d8d3: Allow jobs to be stopped when using track_jobs_in_database/enable_job_running (multiprocess config).
by commits-noreply＠bitbucket.org 20 Nov '10

20 Nov '10

# HG changeset patch -- Bitbucket.org # Project galaxy-dist # URL http://bitbucket.org/galaxy/galaxy-dist/overview # User Nate Coraor <nate(a)bx.psu.edu> # Date 1288379862 14400 # Node ID 0518ce52d8d3ca15ec6070f2addb4bfe7608c23b # Parent a22917c97fbe89aa2c0393d30f51d2a2183fd2f8 Allow jobs to be stopped when using track_jobs_in_database/enable_job_running (multiprocess config). --- a/lib/galaxy/jobs/__init__.py +++ b/lib/galaxy/jobs/__init__.py @@ -813,6 +813,8 @@ class JobStopQueue( object ): self.sa_session = app.model.context self.dispatcher = dispatcher + self.track_jobs_in_database = app.config.get_bool( 'track_jobs_in_database', False ) + # Keep track of the pid that started the job manager, only it # has valid threads self.parent_pid = os.getpid() @@ -848,21 +850,29 @@ class JobStopQueue( object ): Called repeatedly by `monitor` to stop jobs. """ # Pull all new jobs from the queue at once - jobs = [] - try: - while 1: - ( job_id, error_msg ) = self.queue.get_nowait() - if job_id is self.STOP_SIGNAL: - return - # Append to watch queue - jobs.append( ( job_id, error_msg ) ) - except Empty: - pass - - for job_id, error_msg in jobs: - job = self.sa_session.query( model.Job ).get( job_id ) - self.sa_session.refresh( job ) - # if desired, error the job so we can inform the user. + jobs_to_check = [] + if self.track_jobs_in_database: + # Clear the session so we get fresh states for job and all datasets + self.sa_session.expunge_all() + # Fetch all new jobs + newly_deleted_jobs = self.sa_session.query( model.Job ) \ + .options( lazyload( "external_output_metadata" ), lazyload( "parameters" ) ) \ + .filter( model.Job.state == model.Job.states.DELETED_NEW ).all() + for job in newly_deleted_jobs: + jobs_to_check.append( ( job, None ) ) + else: + try: + while 1: + message = self.queue.get_nowait() + if message is self.STOP_SIGNAL: + return + # Unpack the message + job_id, error_msg = message + # Get the job object and append to watch queue + jobs_to_check.append( ( self.sa_session.query( model.Job ).get( job_id ), error_msg ) ) + except Empty: + pass + for job, error_msg in jobs_to_check: if error_msg is not None: job.state = job.states.ERROR job.info = error_msg @@ -870,9 +880,6 @@ class JobStopQueue( object ): job.state = job.states.DELETED self.sa_session.add( job ) self.sa_session.flush() - # if job is in JobQueue or FooJobRunner's put method, - # job_runner_name will be unset and the job will be dequeued due to - # state change above if job.job_runner_name is not None: # tell the dispatcher to stop the job self.dispatcher.stop( job ) @@ -888,7 +895,8 @@ class JobStopQueue( object ): else: log.info( "sending stop signal to worker thread" ) self.running = False - self.queue.put( ( self.STOP_SIGNAL, None ) ) + if not self.track_jobs_in_database: + self.queue.put( self.STOP_SIGNAL ) self.sleeper.wake() log.info( "job stopper stopped" ) --- a/lib/galaxy/web/controllers/root.py +++ b/lib/galaxy/web/controllers/root.py @@ -432,7 +432,8 @@ class RootController( BaseController, Us if job.state in [ self.app.model.Job.states.QUEUED, self.app.model.Job.states.RUNNING, self.app.model.Job.states.NEW ]: # Are *all* of the job's other output datasets deleted? if job.check_if_output_datasets_deleted(): - job.mark_deleted() + job.mark_deleted( self.app.config.get_bool( 'enable_job_running', True ), + self.app.config.get_bool( 'track_jobs_in_database', False ) ) self.app.job_manager.job_stop_queue.put( job.id ) trans.sa_session.flush() --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -97,7 +97,8 @@ class Job( object ): RUNNING = 'running', OK = 'ok', ERROR = 'error', - DELETED = 'deleted' ) + DELETED = 'deleted', + DELETED_NEW = 'deleted_new' ) def __init__( self ): self.session_id = None self.user_id = None @@ -152,11 +153,17 @@ class Job( object ): if not dataset.deleted: return False return True - def mark_deleted( self ): + def mark_deleted( self, enable_job_running=True, track_jobs_in_database=False ): """ Mark this job as deleted, and mark any output datasets as discarded. """ - self.state = Job.states.DELETED + # This could be handled with *just* track_jobs_in_database, but I + # didn't want to make setting track_jobs_in_database required in + # non-runner configs. + if not enable_job_running or track_jobs_in_database: + self.state = Job.states.DELETED_NEW + else: + self.state = Job.states.DELETED self.info = "Job output deleted by user before job completed." for dataset_assoc in self.output_datasets: dataset = dataset_assoc.dataset

1 0

galaxy-dist commit 1efe19f6f3d1: Fixes for rerun action to recurse grouping options when checking unvalidated values and cloned HDAs. Better selection of corresponding HDAs from cloned histories, when multiple copies exist.
by commits-noreply＠bitbucket.org 20 Nov '10

20 Nov '10

# HG changeset patch -- Bitbucket.org # Project galaxy-dist # URL http://bitbucket.org/galaxy/galaxy-dist/overview # User Daniel Blankenberg <dan(a)bx.psu.edu> # Date 1288640846 14400 # Node ID 1efe19f6f3d1a75bda9c198633dd8102d3063410 # Parent 2c934a168af968dc84300412220c2f2cb74bf676 Fixes for rerun action to recurse grouping options when checking unvalidated values and cloned HDAs. Better selection of corresponding HDAs from cloned histories, when multiple copies exist. --- a/lib/galaxy/web/controllers/tool_runner.py +++ b/lib/galaxy/web/controllers/tool_runner.py @@ -123,16 +123,6 @@ class ToolRunner( BaseController ): params_objects = job.get_param_values( trans.app ) except: raise Exception( "Failed to get paramemeters for dataset id %d " % data.id ) - # Unpack unvalidated values to strings, they'll be validated when the - # form is submitted (this happens when re-running a job that was - # initially run by a workflow) - validated_params = {} - for name, value in params_objects.items(): - if isinstance( value, UnvalidatedValue ): - validated_params [ str(name) ] = str(value) - else: - validated_params [ str(name) ] = value - params_objects = validated_params # Need to remap dataset parameters. Job parameters point to original # dataset used; parameter should be the analygous dataset in the # current history. @@ -141,12 +131,22 @@ class ToolRunner( BaseController ): for hda in history.datasets: source_hda = hda.copied_from_history_dataset_association while source_hda:#should this check library datasets as well? - hda_source_dict[ source_hda ] = hda + #FIXME: could be multiple copies of a hda in a single history, this does a better job of matching on cloned histories, + #but is still less than perfect when eg individual datasets are copied between histories + if source_hda not in hda_source_dict or source_hda.hid == hda.hid: + hda_source_dict[ source_hda ] = hda source_hda = source_hda.copied_from_history_dataset_association - for name, value in validated_params.items(): - if isinstance( value, trans.app.model.HistoryDatasetAssociation ): - if value not in history.datasets: - validated_params[ name ] = hda_source_dict[ value ] + # Unpack unvalidated values to strings, they'll be validated when the + # form is submitted (this happens when re-running a job that was + # initially run by a workflow) + #This needs to be done recursively through grouping parameters + def rerun_callback( input, value, prefixed_name, prefixed_label ): + if isinstance( value, UnvalidatedValue ): + return str( value ) + if isinstance( input, DataToolParameter ): + if value not in history.datasets and value in hda_source_dict: + return hda_source_dict[ value ] + visit_input_values( tool.inputs, params_objects, rerun_callback ) # Create a fake tool_state for the tool, with the parameters values state = tool.new_state( trans ) state.inputs = params_objects

1 0

galaxy-dist commit 9f74170b7781: Fix for multiple hide dataset actions when using workflow outputs.
by commits-noreply＠bitbucket.org 20 Nov '10

20 Nov '10

# HG changeset patch -- Bitbucket.org # Project galaxy-dist # URL http://bitbucket.org/galaxy/galaxy-dist/overview # User Dannon Baker <dannonbaker(a)me.com> # Date 1288377653 14400 # Node ID 9f74170b7781122577b8cf7ffe38f5c986617844 # Parent a2e4d36c4318bc4ce73af9df869ae4071af33d55 Fix for multiple hide dataset actions when using workflow outputs. --- a/lib/galaxy/web/controllers/workflow.py +++ b/lib/galaxy/web/controllers/workflow.py @@ -1299,6 +1299,11 @@ class WorkflowController( BaseController step_outputs = [s.output_name for s in step.workflow_outputs] for output in tool.outputs.keys(): if output not in step_outputs: + # Necessary, unfortunately, to clean up workflows that might have more than one at this point. + for pja in step.post_job_actions: + if pja.action_type == "HideDatasetAction" and pja.output_name == output: + step.post_job_actions.remove(pja) + trans.sa_session.delete(pja) # Create a PJA for hiding this output. n_pja = PostJobAction('HideDatasetAction', step, output, {}) else:

1 0

galaxy-dist commit 0b278c24c19f: Tophat wrapper improvements: (a) from JJ and Ry4an Brase @ UMN: wire up parameters from wrapper to tool and (b) improve path joining.
by commits-noreply＠bitbucket.org 20 Nov '10

20 Nov '10

# HG changeset patch -- Bitbucket.org # Project galaxy-dist # URL http://bitbucket.org/galaxy/galaxy-dist/overview # User jeremy goecks <jeremy.goecks(a)emory.edu> # Date 1288217658 14400 # Node ID 0b278c24c19f0a0f7a57b8a33bd6d992461c04c3 # Parent d57ea326563cb4539e47aae49daa6c5dfc69efc5 Tophat wrapper improvements: (a) from JJ and Ry4an Brase @ UMN: wire up parameters from wrapper to tool and (b) improve path joining. --- a/tools/ngs_rna/tophat_wrapper.py +++ b/tools/ngs_rna/tophat_wrapper.py @@ -52,8 +52,6 @@ def __main__(): (options, args) = parser.parse_args() - #sys.stderr.write('*'*50+'\n'+str(options)+'\n'+'*'*50+'\n') - # Creat bowtie index if necessary. tmp_index_dir = tempfile.mkdtemp() if options.own_file != 'None': @@ -121,6 +119,14 @@ def __main__(): opts += ' --microexon-search' if options.single_paired == 'paired': opts += ' --mate-std-dev %s' % options.mate_std_dev + if options.seg_mismatches != None: + opts += ' --segment-mismatches %d' % int(options.seg_mismatches) + if options.seg_length != None: + opts += ' --segment-length %d' % int(options.seg_length) + if options.min_segment_intron != None: + opts += ' --min-segment-intron %d' % int(options.min_segment_intron) + if options.max_segment_intron != None: + opts += ' --max-segment-intron %d' % int(options.max_segment_intron) cmd = cmd % ( tmp_output_dir, opts, index_path, reads ) except Exception, e: # Clean up temp dirs @@ -166,12 +172,12 @@ def __main__(): # Postprocessing: copy output files from tmp directory to specified files. Also need to remove header lines from SAM file. try: try: - shutil.copyfile( tmp_output_dir + "/coverage.wig", options.coverage_output_file ) - shutil.copyfile( tmp_output_dir + "/junctions.bed", options.junctions_output_file ) + shutil.copyfile( os.path.join( tmp_output_dir, "coverage.wig" ), options.coverage_output_file ) + shutil.copyfile( os.path.join( tmp_output_dir, "junctions.bed" ), options.junctions_output_file ) # Remove headers from SAM file in place. in_header = True # Headers always at start of file. - for line in fileinput.input( tmp_output_dir + "/accepted_hits.sam", inplace=1 ): + for line in fileinput.input( os.path.join( tmp_output_dir, "accepted_hits.sam" ), inplace=1 ): if in_header and line.startswith("@"): continue else: @@ -179,7 +185,7 @@ def __main__(): sys.stdout.write( line ) # Copy SAM File. - shutil.copyfile( tmp_output_dir + "/accepted_hits.sam", options.accepted_hits_output_file ) + shutil.copyfile( os.path.join( tmp_output_dir, "accepted_hits.sam" ), options.accepted_hits_output_file ) except Exception, e: stop_err( 'Error in tophat:\n' + str( e ) ) finally:

1 0

galaxy-dist commit 247c19db7f89: fix for the Lab access bug caused by the last commit .
by commits-noreply＠bitbucket.org 20 Nov '10

20 Nov '10

# HG changeset patch -- Bitbucket.org # Project galaxy-dist # URL http://bitbucket.org/galaxy/galaxy-dist/overview # User rc # Date 1288299278 14400 # Node ID 247c19db7f8987d9da245bc750043188d4a7c373 # Parent 522c99a05163c1816e9fe3be4c0b4ad05bff3ea0 fix for the Lab access bug caused by the last commit . --- a/templates/webapps/galaxy/base_panels.mako +++ b/templates/webapps/galaxy/base_panels.mako @@ -84,7 +84,7 @@ [ 'Find Samples', h.url_for( controller='/requests', action='find_samples_index' ) ], [ 'Help', app.config.get( "lims_doc_url", "http://main.g2.bx.psu.edu/u/rkchak/p/sts" ), "galaxy_main" ] ] - tab( "lab", "Lab", None, menu_options=menu_options, visible=( trans.user and ( trans.user.requests or trans.user.accessible_request_types() ) ) ) + tab( "lab", "Lab", None, menu_options=menu_options, visible=( trans.user and ( trans.user.requests or trans.user.accessible_request_types( trans ) ) ) ) %> ## Visualization menu.

1 0

galaxy-dist commit d57ea326563c: More tweaking to try to get Cuffdiff to pass functional tests. The problem is that Cuffdiff's output is not deterministic, so lines_diff needs to be large enough (but no larger than needed) to accomodate small differences in output.
by commits-noreply＠bitbucket.org 20 Nov '10

20 Nov '10

# HG changeset patch -- Bitbucket.org # Project galaxy-dist # URL http://bitbucket.org/galaxy/galaxy-dist/overview # User jeremy goecks <jeremy.goecks(a)emory.edu> # Date 1288216747 14400 # Node ID d57ea326563cb4539e47aae49daa6c5dfc69efc5 # Parent 6a97d4fc4268493480a0a3e7462e91f48d926a72 More tweaking to try to get Cuffdiff to pass functional tests. The problem is that Cuffdiff's output is not deterministic, so lines_diff needs to be large enough (but no larger than needed) to accomodate small differences in output. --- a/tools/ngs_rna/cuffdiff_wrapper.xml +++ b/tools/ngs_rna/cuffdiff_wrapper.xml @@ -77,7 +77,7 @@ Line diffs are needed because cuffdiff does not produce deterministic output. TODO: can we find datasets that lead to deterministic behavior? --> - <output name="isoforms_exp" file="cuffdiff_out1.txt" lines_diff="8"/> + <output name="isoforms_exp" file="cuffdiff_out1.txt" lines_diff="14"/><output name="genes_exp" file="cuffdiff_out2.txt" lines_diff="4"/><output name="tss_groups_exp" file="cuffdiff_out3.txt"/><output name="cds_exp_fpkm_tracking" file="cuffdiff_out4.txt"/>

1 0

galaxy-dist commit a2e4d36c4318: Make fastx_barcode_splitter_galaxy_wrapper.sh invoke bash rather than sh.
by commits-noreply＠bitbucket.org 20 Nov '10

20 Nov '10

# HG changeset patch -- Bitbucket.org # Project galaxy-dist # URL http://bitbucket.org/galaxy/galaxy-dist/overview # User Nate Coraor <nate(a)bx.psu.edu> # Date 1288189891 14400 # Node ID a2e4d36c4318bc4ce73af9df869ae4071af33d55 # Parent 0185bb614c7721d6edf6822ad973e589620b91f5 Make fastx_barcode_splitter_galaxy_wrapper.sh invoke bash rather than sh. --- a/tools/fastx_toolkit/fastx_barcode_splitter_galaxy_wrapper.sh +++ b/tools/fastx_toolkit/fastx_barcode_splitter_galaxy_wrapper.sh @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/bash # FASTX-toolkit - FASTA/FASTQ preprocessing tools. # Copyright (C) 2009 A. Gordon (gordon(a)cshl.edu)

1 0