- galaxy-commits - lists.galaxyproject.org

galaxy-dist commit a22917c97fbe: Merge.
by commits-noreply＠bitbucket.org 20 Nov '10

20 Nov '10

# HG changeset patch -- Bitbucket.org # Project galaxy-dist # URL http://bitbucket.org/galaxy/galaxy-dist/overview # User Dannon Baker <dannonbaker(a)me.com> # Date 1288377687 14400 # Node ID a22917c97fbe89aa2c0393d30f51d2a2183fd2f8 # Parent 9f74170b7781122577b8cf7ffe38f5c986617844 # Parent cbb895e2b2729d324c1693882de7939360e683a4 Merge. --- a/test-data/cuffdiff_out5.tracking +++ /dev/null @@ -1,2 +0,0 @@ -tracking_id class_code nearest_ref_id gene_short_name tss_id locus q0_FPKM q0_conf_lo q0_conf_hi q1_FPKM q1_conf_lo q1_conf_hi -TCONS_00000001 c Xkr4 - - chr1:3204754-3204833 0 0 0 67745 0 163551

1 0

galaxy-dist commit 9f74170b7781: Fix for multiple hide dataset actions when using workflow outputs.
by commits-noreply＠bitbucket.org 20 Nov '10

20 Nov '10

# HG changeset patch -- Bitbucket.org # Project galaxy-dist # URL http://bitbucket.org/galaxy/galaxy-dist/overview # User Dannon Baker <dannonbaker(a)me.com> # Date 1288377653 14400 # Node ID 9f74170b7781122577b8cf7ffe38f5c986617844 # Parent a2e4d36c4318bc4ce73af9df869ae4071af33d55 Fix for multiple hide dataset actions when using workflow outputs. --- a/lib/galaxy/web/controllers/workflow.py +++ b/lib/galaxy/web/controllers/workflow.py @@ -1299,6 +1299,11 @@ class WorkflowController( BaseController step_outputs = [s.output_name for s in step.workflow_outputs] for output in tool.outputs.keys(): if output not in step_outputs: + # Necessary, unfortunately, to clean up workflows that might have more than one at this point. + for pja in step.post_job_actions: + if pja.action_type == "HideDatasetAction" and pja.output_name == output: + step.post_job_actions.remove(pja) + trans.sa_session.delete(pja) # Create a PJA for hiding this output. n_pja = PostJobAction('HideDatasetAction', step, output, {}) else:

1 0

galaxy-dist commit 0518ce52d8d3: Allow jobs to be stopped when using track_jobs_in_database/enable_job_running (multiprocess config).
by commits-noreply＠bitbucket.org 20 Nov '10

20 Nov '10

# HG changeset patch -- Bitbucket.org # Project galaxy-dist # URL http://bitbucket.org/galaxy/galaxy-dist/overview # User Nate Coraor <nate(a)bx.psu.edu> # Date 1288379862 14400 # Node ID 0518ce52d8d3ca15ec6070f2addb4bfe7608c23b # Parent a22917c97fbe89aa2c0393d30f51d2a2183fd2f8 Allow jobs to be stopped when using track_jobs_in_database/enable_job_running (multiprocess config). --- a/lib/galaxy/jobs/__init__.py +++ b/lib/galaxy/jobs/__init__.py @@ -813,6 +813,8 @@ class JobStopQueue( object ): self.sa_session = app.model.context self.dispatcher = dispatcher + self.track_jobs_in_database = app.config.get_bool( 'track_jobs_in_database', False ) + # Keep track of the pid that started the job manager, only it # has valid threads self.parent_pid = os.getpid() @@ -848,21 +850,29 @@ class JobStopQueue( object ): Called repeatedly by `monitor` to stop jobs. """ # Pull all new jobs from the queue at once - jobs = [] - try: - while 1: - ( job_id, error_msg ) = self.queue.get_nowait() - if job_id is self.STOP_SIGNAL: - return - # Append to watch queue - jobs.append( ( job_id, error_msg ) ) - except Empty: - pass - - for job_id, error_msg in jobs: - job = self.sa_session.query( model.Job ).get( job_id ) - self.sa_session.refresh( job ) - # if desired, error the job so we can inform the user. + jobs_to_check = [] + if self.track_jobs_in_database: + # Clear the session so we get fresh states for job and all datasets + self.sa_session.expunge_all() + # Fetch all new jobs + newly_deleted_jobs = self.sa_session.query( model.Job ) \ + .options( lazyload( "external_output_metadata" ), lazyload( "parameters" ) ) \ + .filter( model.Job.state == model.Job.states.DELETED_NEW ).all() + for job in newly_deleted_jobs: + jobs_to_check.append( ( job, None ) ) + else: + try: + while 1: + message = self.queue.get_nowait() + if message is self.STOP_SIGNAL: + return + # Unpack the message + job_id, error_msg = message + # Get the job object and append to watch queue + jobs_to_check.append( ( self.sa_session.query( model.Job ).get( job_id ), error_msg ) ) + except Empty: + pass + for job, error_msg in jobs_to_check: if error_msg is not None: job.state = job.states.ERROR job.info = error_msg @@ -870,9 +880,6 @@ class JobStopQueue( object ): job.state = job.states.DELETED self.sa_session.add( job ) self.sa_session.flush() - # if job is in JobQueue or FooJobRunner's put method, - # job_runner_name will be unset and the job will be dequeued due to - # state change above if job.job_runner_name is not None: # tell the dispatcher to stop the job self.dispatcher.stop( job ) @@ -888,7 +895,8 @@ class JobStopQueue( object ): else: log.info( "sending stop signal to worker thread" ) self.running = False - self.queue.put( ( self.STOP_SIGNAL, None ) ) + if not self.track_jobs_in_database: + self.queue.put( self.STOP_SIGNAL ) self.sleeper.wake() log.info( "job stopper stopped" ) --- a/lib/galaxy/web/controllers/root.py +++ b/lib/galaxy/web/controllers/root.py @@ -432,7 +432,8 @@ class RootController( BaseController, Us if job.state in [ self.app.model.Job.states.QUEUED, self.app.model.Job.states.RUNNING, self.app.model.Job.states.NEW ]: # Are *all* of the job's other output datasets deleted? if job.check_if_output_datasets_deleted(): - job.mark_deleted() + job.mark_deleted( self.app.config.get_bool( 'enable_job_running', True ), + self.app.config.get_bool( 'track_jobs_in_database', False ) ) self.app.job_manager.job_stop_queue.put( job.id ) trans.sa_session.flush() --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -97,7 +97,8 @@ class Job( object ): RUNNING = 'running', OK = 'ok', ERROR = 'error', - DELETED = 'deleted' ) + DELETED = 'deleted', + DELETED_NEW = 'deleted_new' ) def __init__( self ): self.session_id = None self.user_id = None @@ -152,11 +153,17 @@ class Job( object ): if not dataset.deleted: return False return True - def mark_deleted( self ): + def mark_deleted( self, enable_job_running=True, track_jobs_in_database=False ): """ Mark this job as deleted, and mark any output datasets as discarded. """ - self.state = Job.states.DELETED + # This could be handled with *just* track_jobs_in_database, but I + # didn't want to make setting track_jobs_in_database required in + # non-runner configs. + if not enable_job_running or track_jobs_in_database: + self.state = Job.states.DELETED_NEW + else: + self.state = Job.states.DELETED self.info = "Job output deleted by user before job completed." for dataset_assoc in self.output_datasets: dataset = dataset_assoc.dataset

1 0

galaxy-dist commit 1efe19f6f3d1: Fixes for rerun action to recurse grouping options when checking unvalidated values and cloned HDAs. Better selection of corresponding HDAs from cloned histories, when multiple copies exist.
by commits-noreply＠bitbucket.org 20 Nov '10

20 Nov '10

# HG changeset patch -- Bitbucket.org # Project galaxy-dist # URL http://bitbucket.org/galaxy/galaxy-dist/overview # User Daniel Blankenberg <dan(a)bx.psu.edu> # Date 1288640846 14400 # Node ID 1efe19f6f3d1a75bda9c198633dd8102d3063410 # Parent 2c934a168af968dc84300412220c2f2cb74bf676 Fixes for rerun action to recurse grouping options when checking unvalidated values and cloned HDAs. Better selection of corresponding HDAs from cloned histories, when multiple copies exist. --- a/lib/galaxy/web/controllers/tool_runner.py +++ b/lib/galaxy/web/controllers/tool_runner.py @@ -123,16 +123,6 @@ class ToolRunner( BaseController ): params_objects = job.get_param_values( trans.app ) except: raise Exception( "Failed to get paramemeters for dataset id %d " % data.id ) - # Unpack unvalidated values to strings, they'll be validated when the - # form is submitted (this happens when re-running a job that was - # initially run by a workflow) - validated_params = {} - for name, value in params_objects.items(): - if isinstance( value, UnvalidatedValue ): - validated_params [ str(name) ] = str(value) - else: - validated_params [ str(name) ] = value - params_objects = validated_params # Need to remap dataset parameters. Job parameters point to original # dataset used; parameter should be the analygous dataset in the # current history. @@ -141,12 +131,22 @@ class ToolRunner( BaseController ): for hda in history.datasets: source_hda = hda.copied_from_history_dataset_association while source_hda:#should this check library datasets as well? - hda_source_dict[ source_hda ] = hda + #FIXME: could be multiple copies of a hda in a single history, this does a better job of matching on cloned histories, + #but is still less than perfect when eg individual datasets are copied between histories + if source_hda not in hda_source_dict or source_hda.hid == hda.hid: + hda_source_dict[ source_hda ] = hda source_hda = source_hda.copied_from_history_dataset_association - for name, value in validated_params.items(): - if isinstance( value, trans.app.model.HistoryDatasetAssociation ): - if value not in history.datasets: - validated_params[ name ] = hda_source_dict[ value ] + # Unpack unvalidated values to strings, they'll be validated when the + # form is submitted (this happens when re-running a job that was + # initially run by a workflow) + #This needs to be done recursively through grouping parameters + def rerun_callback( input, value, prefixed_name, prefixed_label ): + if isinstance( value, UnvalidatedValue ): + return str( value ) + if isinstance( input, DataToolParameter ): + if value not in history.datasets and value in hda_source_dict: + return hda_source_dict[ value ] + visit_input_values( tool.inputs, params_objects, rerun_callback ) # Create a fake tool_state for the tool, with the parameters values state = tool.new_state( trans ) state.inputs = params_objects

1 0

galaxy-dist commit 9415f380f2e7: Added validation checks in the data transfer script for sample tracking
by commits-noreply＠bitbucket.org 20 Nov '10

20 Nov '10

# HG changeset patch -- Bitbucket.org # Project galaxy-dist # URL http://bitbucket.org/galaxy/galaxy-dist/overview # User rc # Date 1288380270 14400 # Node ID 9415f380f2e7d4aca44f74923b6984ab3bcc1dfb # Parent 0518ce52d8d3ca15ec6070f2addb4bfe7608c23b Added validation checks in the data transfer script for sample tracking --- a/scripts/galaxy_messaging/server/data_transfer.py +++ b/scripts/galaxy_messaging/server/data_transfer.py @@ -3,12 +3,12 @@ Data Transfer Script: Sequencer to Galaxy -This script is called from Galaxy LIMS once the lab admin starts the data -transfer process using the user interface. +This script is called from Galaxy RabbitMQ listener ( amqp_consumer.py ) once +the lab admin starts the data transfer process using the user interface. Usage: -python data_transfer.py <data_transfer_xml><config_id_secret> +python data_transfer.py <config_file> """ @@ -19,13 +19,13 @@ import urllib,urllib2, cookielib, shutil import logging, time, datetime import xml.dom.minidom -log = logging.getLogger("datatx_"+str(os.getpid())) -log.setLevel(logging.DEBUG) -fh = logging.FileHandler("data_transfer.log") -fh.setLevel(logging.DEBUG) -formatter = logging.Formatter("%(asctime)s - %(name)s - %(message)s") -fh.setFormatter(formatter) -log.addHandler(fh) +log = logging.getLogger( "datatx_" + str( os.getpid() ) ) +log.setLevel( logging.DEBUG ) +fh = logging.FileHandler( "data_transfer.log" ) +fh.setLevel( logging.DEBUG ) +formatter = logging.Formatter( "%(asctime)s - %(name)s - %(message)s" ) +fh.setFormatter( formatter ) +log.addHandler( fh ) api_path = [ os.path.join( os.getcwd(), "scripts/api" ) ] sys.path.extend( api_path ) @@ -47,60 +47,76 @@ import pexpect pkg_resources.require( "simplejson" ) import simplejson -class DataTransfer(object): - - def __init__(self, msg, config_file): - log.info(msg) - self.dom = xml.dom.minidom.parseString(msg) - self.sequencer_host = self.get_value(self.dom, 'data_host') - self.sequencer_username = self.get_value(self.dom, 'data_user') - self.sequencer_password = self.get_value(self.dom, 'data_password') - self.request_id = self.get_value(self.dom, 'request_id') - self.sample_id = self.get_value(self.dom, 'sample_id') - self.library_id = self.get_value(self.dom, 'library_id') - self.folder_id = self.get_value(self.dom, 'folder_id') +class DataTransfer( object ): + + def __init__( self, msg, config_file ): + log.info( msg ) + self.dom = xml.dom.minidom.parseString( msg ) + self.sequencer_host = self.get_value( self.dom, 'data_host' ) + self.sequencer_username = self.get_value( self.dom, 'data_user' ) + self.sequencer_password = self.get_value( self.dom, 'data_password' ) + self.request_id = self.get_value( self.dom, 'request_id' ) + self.sample_id = self.get_value( self.dom, 'sample_id' ) + self.library_id = self.get_value( self.dom, 'library_id' ) + self.folder_id = self.get_value( self.dom, 'folder_id' ) self.dataset_files = [] count=0 while True: - dataset_id = self.get_value_index(self.dom, 'dataset_id', count) - file = self.get_value_index(self.dom, 'file', count) - name = self.get_value_index(self.dom, 'name', count) - if file: - self.dataset_files.append(dict(name=name, - dataset_id=int(dataset_id), - file=file)) - else: - break - count=count+1 + dataset_id = self.get_value_index( self.dom, 'dataset_id', count ) + file = self.get_value_index( self.dom, 'file', count ) + name = self.get_value_index( self.dom, 'name', count ) + if file: + self.dataset_files.append( dict( name=name, + dataset_id=int( dataset_id ), + file=file ) ) + else: + break + count=count+1 try: # Retrieve the upload user login information from the config file - transfer_datasets_config = ConfigParser.ConfigParser() - transfer_datasets_config.read('transfer_datasets.ini') - self.data_transfer_user_email = transfer_datasets_config.get("data_transfer_user_login_info", "email") - self.data_transfer_user_password = transfer_datasets_config.get("data_transfer_user_login_info", "password") - self.api_key = transfer_datasets_config.get("data_transfer_user_login_info", "api_key") - self.http_server_section = transfer_datasets_config.get("universe_wsgi_config", "http_server_section") - config = ConfigParser.ConfigParser() - config.read( config_file ) - self.server_host = config.get(self.http_server_section, "host") - self.server_port = config.get(self.http_server_section, "port") - self.database_connection = config.get("app:main", "database_connection") - self.import_dir = config.get("app:main", "library_import_dir") - self.config_id_secret = config.get("app:main", "id_secret") - - # create the destination directory within the import directory - self.server_dir = os.path.join( self.import_dir, 'datatx_'+str(os.getpid())+'_'+datetime.date.today().strftime("%d%b%Y") ) - os.mkdir(self.server_dir) - if not os.path.exists(self.server_dir): - raise Exception + transfer_datasets_config = ConfigParser.ConfigParser( ) + transfer_datasets_config.read( 'transfer_datasets.ini' ) + self.data_transfer_user_email = transfer_datasets_config.get( "data_transfer_user_login_info", "email" ) + self.data_transfer_user_password = transfer_datasets_config.get( "data_transfer_user_login_info", "password" ) + self.api_key = transfer_datasets_config.get( "data_transfer_user_login_info", "api_key" ) + self.http_server_section = transfer_datasets_config.get( "universe_wsgi_config", "http_server_section" ) except: - log.error(traceback.format_exc()) - log.error('FATAL ERROR') - if self.database_connection: - self.error_and_exit('Error') + log.error( traceback.format_exc() ) + log.error( 'ERROR reading config values from transfer_datasets.ini.' ) sys.exit(1) + # read config variables + config = ConfigParser.ConfigParser() + retval = config.read( config_file ) + if not retval: + error_msg = 'FATAL ERROR: Unable to open config file %s.' % config_file + log.error( error_msg ) + sys.exit(1) + try: + self.server_host = config.get( self.http_server_section, "host" ) + except ConfigParser.NoOptionError,e: + self.server_host = '127.0.0.1' + try: + self.server_port = config.get( self.http_server_section, "port" ) + except ConfigParser.NoOptionError,e: + self.server_port = '8080' + try: + self.config_id_secret = config.get( "app:main", "id_secret" ) + except ConfigParser.NoOptionError,e: + self.config_id_secret = "USING THE DEFAULT IS NOT SECURE!" + try: + self.import_dir = config.get( "app:main", "library_import_dir" ) + except ConfigParser.NoOptionError,e: + log.error( 'ERROR: library_import_dir config variable is not set in %s. ' % config_file ) + sys.exit( 1 ) + # create the destination directory within the import directory + self.server_dir = os.path.join( self.import_dir, + 'datatx_' + str( os.getpid() ) + '_' + datetime.date.today( ).strftime( "%d%b%Y" ) ) + try: + os.mkdir( self.server_dir ) + except Exception, e: + self.error_and_exit( str( e ) ) - def start(self): + def start( self ): ''' This method executes the file transfer from the sequencer, adds the dataset to the data library & finally updates the data transfer status in the db @@ -110,72 +126,73 @@ class DataTransfer(object): # add the dataset to the given library self.add_to_library() # update the data transfer status in the db - self.update_status(SampleDataset.transfer_status.COMPLETE) + self.update_status( SampleDataset.transfer_status.COMPLETE ) # cleanup #self.cleanup() - sys.exit(0) + sys.exit( 0 ) - def cleanup(self): + def cleanup( self ): ''' remove the directory created to store the dataset files temporarily before adding the same to the data library ''' try: - time.sleep(60) + time.sleep( 60 ) shutil.rmtree( self.server_dir ) except: self.error_and_exit() - def error_and_exit(self, msg=''): + def error_and_exit( self, msg='' ): ''' This method is called any exception is raised. This prints the traceback and terminates this script ''' - log.error(traceback.format_exc()) - log.error('FATAL ERROR.'+msg) - self.update_status('Error', 'All', msg+"\n"+traceback.format_exc()) - sys.exit(1) + log.error( traceback.format_exc() ) + log.error( 'FATAL ERROR.' + msg ) + self.update_status( 'Error', 'All', msg ) + sys.exit( 1 ) - def transfer_files(self): + def transfer_files( self ): ''' This method executes a scp process using pexpect library to transfer the dataset file from the remote sequencer to the Galaxy server ''' - def print_ticks(d): + def print_ticks( d ): pass - for i, df in enumerate(self.dataset_files): - self.update_status(SampleDataset.transfer_status.TRANSFERRING, df['dataset_id']) + for i, dataset_file in enumerate( self.dataset_files ): + self.update_status( SampleDataset.transfer_status.TRANSFERRING, dataset_file[ 'dataset_id' ] ) try: cmd = "scp %s@%s:'%s' '%s/%s'" % ( self.sequencer_username, self.sequencer_host, - df['file'].replace(' ', '\ '), - self.server_dir.replace(' ', '\ '), - df['name'].replace(' ', '\ ') ) - log.debug(cmd) - output = pexpect.run(cmd, events={'.ssword:*': self.sequencer_password+'\r\n', - pexpect.TIMEOUT:print_ticks}, - timeout=10) - log.debug(output) - path = os.path.join(self.server_dir, os.path.basename(df['name'])) - if not os.path.exists(path): - msg = 'Could not find the local file after transfer (%s)' % path - log.error(msg) - raise Exception(msg) + dataset_file[ 'file' ].replace( ' ', '\ ' ), + self.server_dir.replace( ' ', '\ ' ), + dataset_file[ 'name' ].replace( ' ', '\ ' ) ) + log.debug( cmd ) + output = pexpect.run( cmd, + events={ '.ssword:*': self.sequencer_password+'\r\n', + pexpect.TIMEOUT: print_ticks }, + timeout=10 ) + log.debug( output ) + path = os.path.join( self.server_dir, os.path.basename( dataset_file[ 'name' ] ) ) + if not os.path.exists( path ): + msg = 'Could not find the local file after transfer ( %s ).' % path + log.error( msg ) + raise Exception( msg ) except Exception, e: msg = traceback.format_exc() - self.update_status('Error', df['dataset_id'], msg) + self.update_status( 'Error', dataset_file['dataset_id'], msg) - def add_to_library(self): + def add_to_library( self ): ''' This method adds the dataset file to the target data library & folder by opening the corresponding url in Galaxy server running. ''' - self.update_status(SampleDataset.transfer_status.ADD_TO_LIBRARY) + self.update_status( SampleDataset.transfer_status.ADD_TO_LIBRARY ) try: data = {} - data[ 'folder_id' ] = api.encode_id( self.config_id_secret, '%s.%s' % ( 'folder', self.folder_id ) ) + data[ 'folder_id' ] = api.encode_id( self.config_id_secret, '%s.%s' % ( 'folder', self.folder_id ) ) data[ 'file_type' ] = 'auto' data[ 'server_dir' ] = self.server_dir data[ 'dbkey' ] = '' @@ -183,23 +200,23 @@ class DataTransfer(object): data[ 'create_type' ] = 'file' url = "http://%s:%s/api/libraries/%s/contents" % ( self.server_host, self.server_port, - api.encode_id( self.config_id_secret, self.library_id ) ) - log.debug( str( ( self.api_key, url, data ) ) ) + api.encode_id( self.config_id_secret, self.library_id ) ) + log.debug( str( ( self.api_key, url, data ) ) ) retval = api.submit( self.api_key, url, data, return_formatted=False ) - log.debug( str( retval ) ) + log.debug( str( retval ) ) except Exception, e: - self.error_and_exit( str( e ) ) + self.error_and_exit( str( e ) ) - def update_status(self, status, dataset_id='All', msg=''): + def update_status( self, status, dataset_id='All', msg='' ): ''' Update the data transfer status for this dataset in the database ''' try: - log.debug('Setting status "%s" for dataset "%s" of sample "%s"' % ( status, str(dataset_id), str(self.sample_id) ) ) + log.debug( 'Setting status "%s" for dataset "%s" of sample "%s"' % ( status, str( dataset_id ), str( self.sample_id) ) ) sample_dataset_ids = [] if dataset_id == 'All': for dataset in self.dataset_files: - sample_dataset_ids.append( api.encode_id( self.config_id_secret, dataset['dataset_id'] ) ) + sample_dataset_ids.append( api.encode_id( self.config_id_secret, dataset[ 'dataset_id' ] ) ) else: sample_dataset_ids.append( api.encode_id( self.config_id_secret, dataset_id ) ) # update the transfer status @@ -208,37 +225,37 @@ class DataTransfer(object): data[ 'sample_dataset_ids' ] = sample_dataset_ids data[ 'new_status' ] = status data[ 'error_msg' ] = msg - url = "http://%s:%s/api/requests/%s" % ( self.server_host, + url = "http://%s:%s/api/requests/%s" % ( self.server_host, self.server_port, - api.encode_id( self.config_id_secret, self.request_id ) ) - log.debug(str((self.api_key, url, data))) + api.encode_id( self.config_id_secret, self.request_id ) ) + log.debug( str( ( self.api_key, url, data))) retval = api.update( self.api_key, url, data, return_formatted=False ) log.debug( str( retval ) ) except urllib2.URLError, e: - log.debug( 'ERROR(sample_dataset_transfer_status (%s)): %s' % ( url, str(e) ) ) - log.error(traceback.format_exc()) + log.debug( 'ERROR( sample_dataset_transfer_status ( %s ) ): %s' % ( url, str( e ) ) ) + log.error( traceback.format_exc() ) except: - log.error(traceback.format_exc()) - log.error('FATAL ERROR') - sys.exit(1) + log.error( traceback.format_exc() ) + log.error( 'FATAL ERROR' ) + sys.exit( 1 ) - def get_value(self, dom, tag_name): + def get_value( self, dom, tag_name ): ''' This method extracts the tag value from the xml message ''' - nodelist = dom.getElementsByTagName(tag_name)[0].childNodes + nodelist = dom.getElementsByTagName( tag_name )[ 0 ].childNodes rc = "" for node in nodelist: if node.nodeType == node.TEXT_NODE: rc = rc + node.data return rc - def get_value_index(self, dom, tag_name, dataset_id): + def get_value_index( self, dom, tag_name, dataset_id ): ''' This method extracts the tag value from the xml message ''' try: - nodelist = dom.getElementsByTagName(tag_name)[dataset_id].childNodes + nodelist = dom.getElementsByTagName( tag_name )[ dataset_id ].childNodes except: return None rc = "" @@ -248,11 +265,11 @@ class DataTransfer(object): return rc if __name__ == '__main__': - log.info('STARTING %i %s' % (os.getpid(), str(sys.argv))) + log.info( 'STARTING %i %s' % ( os.getpid(), str( sys.argv ) ) ) # # Start the daemon # - dt = DataTransfer(sys.argv[1], sys.argv[2]) + dt = DataTransfer( sys.argv[1], sys.argv[2]) dt.start() - sys.exit(0) + sys.exit( 0 )

1 0

galaxy-dist commit cf7ec71c5613: Shift management of the interaction between workflow outputs and HideDatasetActions to the front end editor.
by commits-noreply＠bitbucket.org 20 Nov '10

20 Nov '10

# HG changeset patch -- Bitbucket.org # Project galaxy-dist # URL http://bitbucket.org/galaxy/galaxy-dist/overview # User Dannon Baker <dannonbaker(a)me.com> # Date 1288645730 14400 # Node ID cf7ec71c561323940ce95c8148d2008508920092 # Parent 1efe19f6f3d1a75bda9c198633dd8102d3063410 Shift management of the interaction between workflow outputs and HideDatasetActions to the front end editor. This resolves the issue with multiple HideDatasetActions being created. Existing workflows displaying multiple HideDatasetActions per step on the Run Workflow screen will persist. These extra HideDatasetActions are harmless, but a simple edit workflow -> save will remove them. --- a/lib/galaxy/web/controllers/workflow.py +++ b/lib/galaxy/web/controllers/workflow.py @@ -1270,13 +1270,6 @@ class WorkflowController( BaseController workflow_invocation = model.WorkflowInvocation() workflow_invocation.workflow = workflow outputs = odict() - # Find out if there are any workflow outputs defined, as that influences our actions. - use_workflow_outputs = False - for step in workflow.steps: - if step.type == 'tool' or step.type is None: - if step.workflow_outputs: - use_workflow_outputs = True - break for i, step in enumerate( workflow.steps ): # Execute module job = None @@ -1294,24 +1287,6 @@ class WorkflowController( BaseController job, out_data = tool.execute( trans, step.state.inputs ) outputs[ step.id ] = out_data # Create new PJA associations with the created job, to be run on completion. - if use_workflow_outputs: - # We're using outputs. Check the step for outputs to be displayed. Create PJAs to hide the rest upon completion. - step_outputs = [s.output_name for s in step.workflow_outputs] - for output in tool.outputs.keys(): - if output not in step_outputs: - # Necessary, unfortunately, to clean up workflows that might have more than one at this point. - for pja in step.post_job_actions: - if pja.action_type == "HideDatasetAction" and pja.output_name == output: - step.post_job_actions.remove(pja) - trans.sa_session.delete(pja) - # Create a PJA for hiding this output. - n_pja = PostJobAction('HideDatasetAction', step, output, {}) - else: - # Remove any HideDatasetActions, step is flagged for output. - for pja in step.post_job_actions: - if pja.action_type == "HideDatasetAction" and pja.output_name == output: - step.post_job_actions.remove(pja) - trans.sa_session.delete(pja) for pja in step.post_job_actions: if pja.action_type in ActionBox.immediate_actions: ActionBox.execute(trans.app, trans.sa_session, pja, job) --- a/static/scripts/galaxy.workflow_editor.canvas.js +++ b/static/scripts/galaxy.workflow_editor.canvas.js @@ -475,6 +475,63 @@ function Workflow( canvas_container ) { wf.remove_node( v ); }); }, + rectify_workflow_outputs : function() { + console.log("RECTIFICATION!"); + // Find out if we're using workflow_outputs or not. + var using_workflow_outputs = false; + $.each( this.nodes, function ( k, node ) { + if (node.workflow_outputs && node.workflow_outputs.length > 0){ + using_workflow_outputs = true; + } + }); + if (using_workflow_outputs == false){ + //We're done, leave PJAs alone. + return true; + } + wf = this; + $.each(this.nodes, function (k, node ){ + if (node.type == 'tool'){ + var node_changed = false; + if (node.post_job_actions == null){ + console.log("CREATED FOR NEW NODE"); + node.post_job_actions = {}; + } + var pjas_to_rem = []; + $.each(node.post_job_actions, function(pja_id, pja){ + if (pja.action_type == "HideDatasetAction"){ + pjas_to_rem.push(pja_id); + } + }); + if (pjas_to_rem.length > 0 && node == workflow.active_node) + $.each(pjas_to_rem, function(i, pja_name){ + node_changed = true; + delete node.post_job_actions[pja_name]; + }) + $.each(node.output_terminals, function(ot_id, ot){ + var create_pja = true; + $.each(node.workflow_outputs, function(i, wo_name){ + if (ot.name == wo_name){ + create_pja = false; + } + }); + if (create_pja == true){ + node_changed = true; + var pja = { + action_type : "HideDatasetAction", + output_name : ot.name, + action_arguments : {} + } + node.post_job_actions['HideDatasetAction'+ot.name] = null; + node.post_job_actions['HideDatasetAction'+ot.name] = pja; + } + }); + // lastly, if this is the active node, and we made changes, reload the display at right. + if (wf.active_node == node && node_changed == true) { + wf.reload_active_node(); + } + } + }); + }, to_simple : function () { var nodes = {}; $.each( this.nodes, function ( i, node ) { @@ -491,7 +548,6 @@ function Workflow( canvas_container ) { if (node.post_job_actions){ $.each( node.post_job_actions, function ( i, act ) { var pja = { - job_id : act.id, action_type : act.action_type, output_name : act.output_name, action_arguments : act.action_arguments @@ -559,6 +615,10 @@ function Workflow( canvas_container ) { this.active_form_has_changes = false; } }, + reload_active_node : function() { + this.clear_active_node(); + this.activate_node(node); + }, clear_active_node : function() { if ( this.active_node ) { this.active_node.make_inactive(); --- a/templates/workflow/editor.mako +++ b/templates/workflow/editor.mako @@ -627,6 +627,7 @@ } return; } + workflow.rectify_workflow_outputs(); var savefn = function(callback) { $.ajax( { url: "${h.url_for( action='save_workflow' )}",

1 0

galaxy-dist commit 243d80d9905a: adding setup_rabbitmq.sh script.
by commits-noreply＠bitbucket.org 20 Nov '10

20 Nov '10

# HG changeset patch -- Bitbucket.org # Project galaxy-dist # URL http://bitbucket.org/galaxy/galaxy-dist/overview # User rc # Date 1288197416 14400 # Node ID 243d80d9905af9121452f1a0955b2b49c076d66e # Parent 9c6040ab75a8397a5708cfec4edfe216927b8220 adding setup_rabbitmq.sh script. web api minor code fixes --- a/scripts/api/common.py +++ b/scripts/api/common.py @@ -91,7 +91,10 @@ def submit( api_key, url, data, return_f except urllib2.HTTPError, e: print e print e.read( 1024 ) - sys.exit( 1 ) + if return_formatted: + sys.exit( 1 ) + else: + return 'Error. '+ str( e ) if not return_formatted: return r print 'Response' --- /dev/null +++ b/setup_rabbitmq.sh @@ -0,0 +1,4 @@ +#!/bin/sh + +cd `dirname $0` +python scripts/galaxy_messaging/server/setup_rabbitmq.py universe_wsgi.ini --- a/scripts/galaxy_messaging/server/data_transfer.py +++ b/scripts/galaxy_messaging/server/data_transfer.py @@ -184,10 +184,11 @@ class DataTransfer(object): url = "http://%s:%s/api/libraries/%s/contents" % ( self.server_host, self.server_port, api.encode_id( self.config_id_secret, self.library_id ) ) - log.debug(str((self.api_key, url, data))) + log.debug( str( ( self.api_key, url, data ) ) ) retval = api.submit( self.api_key, url, data, return_formatted=False ) - except: - self.error_and_exit(str(e)) + log.debug( str( retval ) ) + except Exception, e: + self.error_and_exit( str( e ) ) def update_status(self, status, dataset_id='All', msg=''): ''' @@ -212,7 +213,7 @@ class DataTransfer(object): api.encode_id( self.config_id_secret, self.request_id ) ) log.debug(str((self.api_key, url, data))) retval = api.update( self.api_key, url, data, return_formatted=False ) - log.debug('done.') + log.debug( str( retval ) ) except urllib2.URLError, e: log.debug( 'ERROR(sample_dataset_transfer_status (%s)): %s' % ( url, str(e) ) ) log.error(traceback.format_exc()) --- a/scripts/galaxy_messaging/server/setup_rabbitmq.py +++ b/scripts/galaxy_messaging/server/setup_rabbitmq.py @@ -11,13 +11,17 @@ import os, sys, csv, ConfigParser def main( config_file ): - config = ConfigParser.ConfigParser() - config.read( config_file ) - rabbitmqctl_path = config.get( 'galaxy_amqp', 'rabbitmqctl_path' ) - username = config.get( 'galaxy_amqp', 'userid' ) - password = config.get( 'galaxy_amqp', 'password' ) - virtual_host = config.get( 'galaxy_amqp', 'virtual_host' ) - + try: + config = ConfigParser.ConfigParser() + config.read( config_file ) + rabbitmqctl_path = config.get( 'galaxy_amqp', 'rabbitmqctl_path' ) + username = config.get( 'galaxy_amqp', 'userid' ) + password = config.get( 'galaxy_amqp', 'password' ) + virtual_host = config.get( 'galaxy_amqp', 'virtual_host' ) + except Exception, e: + print 'Fatal error:', str(e) + sys.exit(1) + cmd_list = [ 'add_user %s %s' % ( username, password ), 'add_vhost %s' % config.get( 'galaxy_amqp', 'virtual_host' ),

1 0

galaxy-dist commit 6a97d4fc4268: Update test data for GFF-to-BED to accomodate change made in 4475:aaacde471f03.
by commits-noreply＠bitbucket.org 20 Nov '10

20 Nov '10

# HG changeset patch -- Bitbucket.org # Project galaxy-dist # URL http://bitbucket.org/galaxy/galaxy-dist/overview # User jeremy goecks <jeremy.goecks(a)emory.edu> # Date 1288198736 14400 # Node ID 6a97d4fc4268493480a0a3e7462e91f48d926a72 # Parent 121be69ca30675593dc9fc247a4b10a118fed227 Update test data for GFF-to-BED to accomodate change made in 4475:aaacde471f03. --- a/test-data/gff2bed_out2.bed +++ b/test-data/gff2bed_out2.bed @@ -1,3 +1,3 @@ -chr1 3204562 3661579 Xkr4 0 - 3204562 3204562 0 3 2487,200,947 0,207220,456070 -chr1 4481008 4486494 Sox17 0 - 4481008 4481008 0 5 1741,367,92,807,123 0,2172,2844,4208,5363 -chr1 4763278 4775807 Mrpl15 0 - 4763278 4763278 0 5 3604,124,166,155,154 0,4327,9370,10753,12375 +chr1 3204562 3661579 Xkr4 0 - 3204562 3661579 0 3 2487,200,947 0,207220,456070 +chr1 4481008 4486494 Sox17 0 - 4481008 4486494 0 5 1741,367,92,807,123 0,2172,2844,4208,5363 +chr1 4763278 4775807 Mrpl15 0 - 4763278 4775807 0 5 3604,124,166,155,154 0,4327,9370,10753,12375

1 0

galaxy-dist commit a2e4d36c4318: Make fastx_barcode_splitter_galaxy_wrapper.sh invoke bash rather than sh.
by commits-noreply＠bitbucket.org 20 Nov '10

20 Nov '10

# HG changeset patch -- Bitbucket.org # Project galaxy-dist # URL http://bitbucket.org/galaxy/galaxy-dist/overview # User Nate Coraor <nate(a)bx.psu.edu> # Date 1288189891 14400 # Node ID a2e4d36c4318bc4ce73af9df869ae4071af33d55 # Parent 0185bb614c7721d6edf6822ad973e589620b91f5 Make fastx_barcode_splitter_galaxy_wrapper.sh invoke bash rather than sh. --- a/tools/fastx_toolkit/fastx_barcode_splitter_galaxy_wrapper.sh +++ b/tools/fastx_toolkit/fastx_barcode_splitter_galaxy_wrapper.sh @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/bash # FASTX-toolkit - FASTA/FASTQ preprocessing tools. # Copyright (C) 2009 A. Gordon (gordon(a)cshl.edu)

1 0

galaxy-dist commit 522c99a05163: fixed Lab menu access bug
by commits-noreply＠bitbucket.org 20 Nov '10

20 Nov '10

# HG changeset patch -- Bitbucket.org # Project galaxy-dist # URL http://bitbucket.org/galaxy/galaxy-dist/overview # User rc # Date 1288298593 14400 # Node ID 522c99a05163c1816e9fe3be4c0b4ad05bff3ea0 # Parent 7327a60e33968cb067edf6541cc84643f970180d fixed Lab menu access bug fixed rabbitmq setup script bug --- a/templates/webapps/galaxy/base_panels.mako +++ b/templates/webapps/galaxy/base_panels.mako @@ -84,7 +84,7 @@ [ 'Find Samples', h.url_for( controller='/requests', action='find_samples_index' ) ], [ 'Help', app.config.get( "lims_doc_url", "http://main.g2.bx.psu.edu/u/rkchak/p/sts" ), "galaxy_main" ] ] - tab( "lab", "Lab", None, menu_options=menu_options, visible=( trans.user and trans.user.requests ) ) + tab( "lab", "Lab", None, menu_options=menu_options, visible=( trans.user and ( trans.user.requests or trans.user.accessible_request_types() ) ) ) %> ## Visualization menu. --- a/scripts/galaxy_messaging/server/setup_rabbitmq.py +++ b/scripts/galaxy_messaging/server/setup_rabbitmq.py @@ -25,7 +25,7 @@ def main( config_file ): cmd_list = [ 'add_user %s %s' % ( username, password ), 'add_vhost %s' % config.get( 'galaxy_amqp', 'virtual_host' ), - 'set_permissions -p %s %s "%s.*" ".*" ".*"' % ( virtual_host, username, username ) + 'set_permissions -p %s %s ".*" ".*" ".*"' % ( virtual_host, username ) ] for cmd in cmd_list:

1 0