November 2010 - galaxy-commits - lists.galaxyproject.org

galaxy-dist commit 328b57b1e2e5: sample tracking data transfer mechanism streamlined.
by commits-noreply＠bitbucket.org 20 Nov '10

20 Nov '10

# HG changeset patch -- Bitbucket.org # Project galaxy-dist # URL http://bitbucket.org/galaxy/galaxy-dist/overview # User rc # Date 1289177219 18000 # Node ID 328b57b1e2e53cbc9ab208379cdd3dc0864f3cc4 # Parent 4b78bad81e777cf503ac66acb5dbb01da841400f sample tracking data transfer mechanism streamlined. - eliminated the transfer_datasets.ini config file - eliminated the data transfer user - now the admin user initiating the data transfer is used as the data transfer user. - the admin user is provided the add_library_item permission before data transfer if they dont have it - sample update & data transfer amqp messages now includes api_key --- a/scripts/galaxy_messaging/client/galaxy_amq.ini.sample +++ b/scripts/galaxy_messaging/client/galaxy_amq.ini.sample @@ -14,6 +14,7 @@ #queue = galaxy_queue #exchange = galaxy_exchange #routing_key = bar_code_scanner +#api_key = # The following section(s) 'scanner#' is for specifying the state of the # sample this scanner represents. This state name should be one of the --- a/scripts/galaxy_messaging/server/amqp_consumer.py +++ b/scripts/galaxy_messaging/server/amqp_consumer.py @@ -15,6 +15,9 @@ import optparse import xml.dom.minidom import subprocess import urllib2 + +from xml_helper import get_value, get_value_index + from galaxydb_interface import GalaxyDbInterface api_path = [ os.path.join( os.getcwd(), "scripts/api" ) ] @@ -44,77 +47,58 @@ log.addHandler(fh) # data transfer script data_transfer_script = os.path.join( os.getcwd(), "scripts/galaxy_messaging/server/data_transfer.py" ) +global config +global config_file_name +global http_server_section -global dbconnstr -global webconfig -global config -def get_value(dom, tag_name): - ''' - This method extracts the tag value from the xml message - ''' - nodelist = dom.getElementsByTagName(tag_name)[0].childNodes - rc = "" - for node in nodelist: - if node.nodeType == node.TEXT_NODE: - rc = rc + node.data - return rc - -def get_value_index(dom, tag_name, index): - ''' - This method extracts the tag value from the xml message - ''' - try: - nodelist = dom.getElementsByTagName(tag_name)[index].childNodes - except: - return None - rc = "" - for node in nodelist: - if node.nodeType == node.TEXT_NODE: - rc = rc + node.data - return rc - -def start_data_transfer(message): +def start_data_transfer( message ): # fork a new process to transfer datasets cmd = '%s "%s" "%s" "%s"' % ( "python", data_transfer_script, message.body, - sys.argv[1] ) # Galaxy config file name + config_file_name ) # Galaxy config file name pid = subprocess.Popen(cmd, shell=True).pid log.debug('Started process (%i): %s' % (pid, str(cmd))) -def update_sample_state(message): +def update_sample_state( message ): dom = xml.dom.minidom.parseString(message.body) barcode = get_value(dom, 'barcode') state = get_value(dom, 'state') + api_key = get_value(dom, 'api_key') log.debug('Barcode: ' + barcode) log.debug('State: ' + state) + log.debug('api_key: ' + api_key) + # validate + if not barcode or not state or not api_key: + log.debug( 'Incomplete sample_state_update message received. Sample barcode, desired state and user API key is required.' ) + return # update the sample state in Galaxy db - galaxydb = GalaxyDbInterface(dbconnstr) - sample_id = galaxydb.get_sample_id(field_name='bar_code', value=barcode) + dbconnstr = config.get("app:main", "database_connection") + galaxydb = GalaxyDbInterface( dbconnstr ) + sample_id = galaxydb.get_sample_id( field_name='bar_code', value=barcode ) if sample_id == -1: - log.debug('Invalid barcode.') + log.debug( 'Invalid barcode.' ) return galaxydb.change_state(sample_id, state) # after updating the sample state, update request status request_id = galaxydb.get_request_id(sample_id) - update_request( request_id ) + update_request( api_key, request_id ) -def update_request( request_id ): - http_server_section = webconfig.get( "universe_wsgi_config", "http_server_section" ) +def update_request( api_key, request_id ): encoded_request_id = api.encode_id( config.get( "app:main", "id_secret" ), request_id ) - api_key = webconfig.get( "data_transfer_user_login_info", "api_key" ) data = dict( update_type=RequestsController.update_types.REQUEST ) url = "http://%s:%s/api/requests/%s" % ( config.get(http_server_section, "host"), config.get(http_server_section, "port"), encoded_request_id ) log.debug( 'Updating request %i' % request_id ) try: - api.update( api_key, url, data, return_formatted=False ) + retval = api.update( api_key, url, data, return_formatted=False ) + log.debug( str( retval ) ) except urllib2.URLError, e: log.debug( 'ERROR(update_request (%s)): %s' % ( str((self.api_key, url, data)), str(e) ) ) -def recv_callback(message): +def recv_callback( message ): # check the meesage type. msg_type = message.properties['application_headers'].get('msg_type') log.debug( 'MESSAGE RECVD: ' + str( msg_type ) ) @@ -126,22 +110,25 @@ def recv_callback(message): update_sample_state( message ) def main(): - if len(sys.argv) < 2: - print 'Usage: python amqp_consumer.py <Galaxy configuration file>' - return + parser = optparse.OptionParser() + parser.add_option('-c', '--config-file', help='Galaxy configuration file', + dest='config_file', action='store') + parser.add_option('-s', '--http-server-section', help='Name of the HTTP server section in the Galaxy configuration file', + dest='http_server_section', action='store') + (opts, args) = parser.parse_args() + log.debug( "GALAXY LISTENER PID: " + str(os.getpid()) + " - " + str( opts ) ) + # read the Galaxy config file + global config_file_name + config_file_name = opts.config_file global config config = ConfigParser.ConfigParser() - config.read( sys.argv[1] ) - global dbconnstr - dbconnstr = config.get("app:main", "database_connection") + config.read( opts.config_file ) + global http_server_section + http_server_section = opts.http_server_section amqp_config = {} for option in config.options("galaxy_amqp"): amqp_config[option] = config.get("galaxy_amqp", option) - log.debug("PID: " + str(os.getpid()) + ", " + str(amqp_config)) - # web server config - global webconfig - webconfig = ConfigParser.ConfigParser() - webconfig.read('transfer_datasets.ini') + log.debug( str( amqp_config ) ) # connect conn = amqp.Connection(host=amqp_config['host']+":"+amqp_config['port'], userid=amqp_config['userid'], --- /dev/null +++ b/scripts/galaxy_messaging/server/xml_helper.py @@ -0,0 +1,28 @@ +#======= XML helper methods ==================================================== + +import xml.dom.minidom + +def get_value( dom, tag_name ): + ''' + This method extracts the tag value from the xml message + ''' + nodelist = dom.getElementsByTagName( tag_name )[ 0 ].childNodes + rc = "" + for node in nodelist: + if node.nodeType == node.TEXT_NODE: + rc = rc + node.data + return rc + +def get_value_index( dom, tag_name, dataset_id ): + ''' + This method extracts the tag value from the xml message + ''' + try: + nodelist = dom.getElementsByTagName( tag_name )[ dataset_id ].childNodes + except: + return None + rc = "" + for node in nodelist: + if node.nodeType == node.TEXT_NODE: + rc = rc + node.data + return rc --- a/lib/galaxy/web/controllers/requests_common.py +++ b/lib/galaxy/web/controllers/requests_common.py @@ -655,8 +655,8 @@ class RequestsCommon( BaseController, Us event = trans.model.RequestEvent( request, request.states.SUBMITTED, message ) trans.sa_session.add( event ) trans.sa_session.flush() - if cntrller == 'api': - return 200, message + if cntrller == 'api': + return 200, message return trans.response.send_redirect( web.url_for( controller='requests_common', action='edit_samples', cntrller=cntrller, --- a/scripts/galaxy_messaging/server/data_transfer.py +++ b/scripts/galaxy_messaging/server/data_transfer.py @@ -19,6 +19,8 @@ import urllib,urllib2, cookielib, shutil import logging, time, datetime import xml.dom.minidom +from xml_helper import get_value, get_value_index + log = logging.getLogger( "datatx_" + str( os.getpid() ) ) log.setLevel( logging.DEBUG ) fh = logging.FileHandler( "data_transfer.log" ) @@ -36,35 +38,38 @@ new_path = [ os.path.join( os.getcwd(), new_path.extend( sys.path[1:] ) # remove scripts/ from the path sys.path = new_path +from galaxy import eggs from galaxy.util.json import from_json_string, to_json_string from galaxy.model import SampleDataset from galaxy.web.api.requests import RequestsController -from galaxy import eggs import pkg_resources pkg_resources.require( "pexpect" ) import pexpect - pkg_resources.require( "simplejson" ) import simplejson +log.debug(str(dir(api))) + class DataTransfer( object ): def __init__( self, msg, config_file ): log.info( msg ) self.dom = xml.dom.minidom.parseString( msg ) - self.sequencer_host = self.get_value( self.dom, 'data_host' ) - self.sequencer_username = self.get_value( self.dom, 'data_user' ) - self.sequencer_password = self.get_value( self.dom, 'data_password' ) - self.request_id = self.get_value( self.dom, 'request_id' ) - self.sample_id = self.get_value( self.dom, 'sample_id' ) - self.library_id = self.get_value( self.dom, 'library_id' ) - self.folder_id = self.get_value( self.dom, 'folder_id' ) + self.galaxy_host = get_value( self.dom, 'galaxy_host' ) + self.api_key = get_value( self.dom, 'api_key' ) + self.sequencer_host = get_value( self.dom, 'data_host' ) + self.sequencer_username = get_value( self.dom, 'data_user' ) + self.sequencer_password = get_value( self.dom, 'data_password' ) + self.request_id = get_value( self.dom, 'request_id' ) + self.sample_id = get_value( self.dom, 'sample_id' ) + self.library_id = get_value( self.dom, 'library_id' ) + self.folder_id = get_value( self.dom, 'folder_id' ) self.dataset_files = [] count=0 while True: - dataset_id = self.get_value_index( self.dom, 'dataset_id', count ) - file = self.get_value_index( self.dom, 'file', count ) - name = self.get_value_index( self.dom, 'name', count ) + dataset_id = get_value_index( self.dom, 'dataset_id', count ) + file = get_value_index( self.dom, 'file', count ) + name = get_value_index( self.dom, 'name', count ) if file: self.dataset_files.append( dict( name=name, dataset_id=int( dataset_id ), @@ -72,18 +77,6 @@ class DataTransfer( object ): else: break count=count+1 - try: - # Retrieve the upload user login information from the config file - transfer_datasets_config = ConfigParser.ConfigParser( ) - transfer_datasets_config.read( 'transfer_datasets.ini' ) - self.data_transfer_user_email = transfer_datasets_config.get( "data_transfer_user_login_info", "email" ) - self.data_transfer_user_password = transfer_datasets_config.get( "data_transfer_user_login_info", "password" ) - self.api_key = transfer_datasets_config.get( "data_transfer_user_login_info", "api_key" ) - self.http_server_section = transfer_datasets_config.get( "universe_wsgi_config", "http_server_section" ) - except: - log.error( traceback.format_exc() ) - log.error( 'ERROR reading config values from transfer_datasets.ini.' ) - sys.exit(1) # read config variables config = ConfigParser.ConfigParser() retval = config.read( config_file ) @@ -92,14 +85,6 @@ class DataTransfer( object ): log.error( error_msg ) sys.exit(1) try: - self.server_host = config.get( self.http_server_section, "host" ) - except ConfigParser.NoOptionError,e: - self.server_host = '127.0.0.1' - try: - self.server_port = config.get( self.http_server_section, "port" ) - except ConfigParser.NoOptionError,e: - self.server_port = '8080' - try: self.config_id_secret = config.get( "app:main", "id_secret" ) except ConfigParser.NoOptionError,e: self.config_id_secret = "USING THE DEFAULT IS NOT SECURE!" @@ -198,9 +183,8 @@ class DataTransfer( object ): data[ 'dbkey' ] = '' data[ 'upload_option' ] = 'upload_directory' data[ 'create_type' ] = 'file' - url = "http://%s:%s/api/libraries/%s/contents" % ( self.server_host, - self.server_port, - api.encode_id( self.config_id_secret, self.library_id ) ) + url = "http://%s/api/libraries/%s/contents" % ( self.galaxy_host, + api.encode_id( self.config_id_secret, self.library_id ) ) log.debug( str( ( self.api_key, url, data ) ) ) retval = api.submit( self.api_key, url, data, return_formatted=False ) log.debug( str( retval ) ) @@ -225,9 +209,8 @@ class DataTransfer( object ): data[ 'sample_dataset_ids' ] = sample_dataset_ids data[ 'new_status' ] = status data[ 'error_msg' ] = msg - url = "http://%s:%s/api/requests/%s" % ( self.server_host, - self.server_port, - api.encode_id( self.config_id_secret, self.request_id ) ) + url = "http://%s/api/requests/%s" % ( self.galaxy_host, + api.encode_id( self.config_id_secret, self.request_id ) ) log.debug( str( ( self.api_key, url, data))) retval = api.update( self.api_key, url, data, return_formatted=False ) log.debug( str( retval ) ) @@ -239,31 +222,6 @@ class DataTransfer( object ): log.error( 'FATAL ERROR' ) sys.exit( 1 ) - def get_value( self, dom, tag_name ): - ''' - This method extracts the tag value from the xml message - ''' - nodelist = dom.getElementsByTagName( tag_name )[ 0 ].childNodes - rc = "" - for node in nodelist: - if node.nodeType == node.TEXT_NODE: - rc = rc + node.data - return rc - - def get_value_index( self, dom, tag_name, dataset_id ): - ''' - This method extracts the tag value from the xml message - ''' - try: - nodelist = dom.getElementsByTagName( tag_name )[ dataset_id ].childNodes - except: - return None - rc = "" - for node in nodelist: - if node.nodeType == node.TEXT_NODE: - rc = rc + node.data - return rc - if __name__ == '__main__': log.info( 'STARTING %i %s' % ( os.getpid(), str( sys.argv ) ) ) # --- a/lib/galaxy/web/controllers/requests_admin.py +++ b/lib/galaxy/web/controllers/requests_admin.py @@ -109,7 +109,7 @@ class DataTransferGrid( grids.Grid ): default_sort_key = "-create_time" num_rows_per_page = 50 preserve_state = True - use_paging = True + use_paging = False columns = [ NameColumn( "Name", link=( lambda item: dict( operation="view", id=item.id ) ), @@ -299,7 +299,7 @@ class RequestsAdmin( BaseController, Use sample=selected_sample_datasets[0].sample, id_list=id_list ) elif operation == "transfer": - self.__start_datatx( trans, selected_sample_datasets[0].sample, selected_sample_datasets ) + self.initiate_data_transfer( trans, selected_sample_datasets[0].sample, selected_sample_datasets ) # Render the grid view sample_id = params.get( 'sample_id', None ) try: @@ -538,68 +538,36 @@ class RequestsAdmin( BaseController, Use return sample.request.name + '_' + sample.name + '_' + name if opt == options.EXPERIMENT_NAME: return sample.request.name + '_' + name - def __setup_datatx_user( self, trans, sample ): + def __check_library_add_permission( self, trans, target_library, target_folder ): """ - Sets up the datatx user: - - Checks if the user exists, if not creates them. - - Checks if the user had ADD_LIBRARY permission on the target library - and the target folder, if not sets up the permissions. + Checks if the current admin user had ADD_LIBRARY permission on the target library + and the target folder, if not provide the permissions. """ - # Retrieve the upload user login information from the config file - config = ConfigParser.ConfigParser() - ok = True - try: - config.read( 'transfer_datasets.ini' ) - except Exception, e: - message = "Error attempting to read config file named 'transfer_datasets.ini'. Make sure this file is correct." - ok = False - try: - email = config.get( "data_transfer_user_login_info", "email" ) - password = config.get( "data_transfer_user_login_info", "password" ) - except Exception, e: - message = "The 'data_transfer_user_login_info' section is missing from the 'transfer_datasets.ini'. Make sure this file is correct." - ok = False - if not ok: - status = 'error' - return trans.response.send_redirect( web.url_for( controller='requests_admin', - action='manage_datasets', - sample_id=trans.security.encode_id( sample.id ), - status=status, - message=message ) ) - # check if the user already exists - datatx_user = trans.sa_session.query( trans.model.User ) \ - .filter( trans.model.User.table.c.email==email ) \ - .first() - if not datatx_user: - # if not create the user - datatx_user = trans.model.User( email=email, password=passsword ) - if trans.app.config.use_remote_user: - datatx_user.external = True - trans.sa_session.add( datatx_user ) - trans.sa_session.flush() - trans.app.security_agent.create_private_user_role( datatx_user ) - trans.app.security_agent.user_set_default_permissions( datatx_user, history=False, dataset=False ) - datatx_user_roles = datatx_user.all_roles() - datatx_user_private_role = trans.app.security_agent.get_private_user_role( datatx_user ) + current_user_roles = trans.user.all_roles() + current_user_private_role = trans.app.security_agent.get_private_user_role( trans.user ) # Make sure this user has LIBRARY_ADD permissions on the target library and folder. # If not, give them permission. - if not trans.app.security_agent.can_add_library_item( datatx_user_roles, sample.library ): + if not trans.app.security_agent.can_add_library_item( current_user_roles, target_library ): lp = trans.model.LibraryPermissions( trans.app.security_agent.permitted_actions.LIBRARY_ADD.action, - sample.library, - datatx_user_private_role ) + target_library, + current_user_private_role ) trans.sa_session.add( lp ) - if not trans.app.security_agent.can_add_library_item( datatx_user_roles, sample.folder ): + if not trans.app.security_agent.can_add_library_item( current_user_roles, target_folder ): lfp = trans.model.LibraryFolderPermissions( trans.app.security_agent.permitted_actions.LIBRARY_ADD.action, - sample.folder, - datatx_user_private_role ) + target_folder, + current_user_private_role ) trans.sa_session.add( lfp ) trans.sa_session.flush() - return datatx_user - def __send_message( self, trans, datatx_info, sample, selected_sample_datasets ): - """Ceates an xml message and sends it to the rabbitmq server""" + def __create_data_transfer_message( self, trans, sample, selected_sample_datasets ): + """ + Creates an xml message to send to the rabbitmq server + """ + datatx_info = sample.request.type.datatx_info # Create the xml message based on the following template xml = \ ''' <data_transfer> + <galaxy_host>%(GALAXY_HOST)s</galaxy_host> + <api_key>%(API_KEY)s</api_key><data_host>%(DATA_HOST)s</data_host><data_user>%(DATA_USER)s</data_user><data_password>%(DATA_PASSWORD)s</data_password> @@ -624,50 +592,77 @@ class RequestsAdmin( BaseController, Use sample_dataset.status = trans.app.model.SampleDataset.transfer_status.IN_QUEUE trans.sa_session.add( sample_dataset ) trans.sa_session.flush() - data = xml % dict( DATA_HOST=datatx_info['host'], - DATA_USER=datatx_info['username'], - DATA_PASSWORD=datatx_info['password'], - REQUEST_ID=str(sample.request.id), - SAMPLE_ID=str(sample.id), - LIBRARY_ID=str(sample.library.id), - FOLDER_ID=str(sample.folder.id), - DATASETS=datasets ) - # Send the message - try: - conn = amqp.Connection( host=trans.app.config.amqp['host'] + ":" + trans.app.config.amqp['port'], - userid=trans.app.config.amqp['userid'], - password=trans.app.config.amqp['password'], - virtual_host=trans.app.config.amqp['virtual_host'], - insist=False ) - chan = conn.channel() - msg = amqp.Message( data.replace( '\n', '' ).replace( '\r', '' ), - content_type='text/plain', - application_headers={'msg_type': 'data_transfer'} ) - msg.properties["delivery_mode"] = 2 - chan.basic_publish( msg, - exchange=trans.app.config.amqp['exchange'], - routing_key=trans.app.config.amqp['routing_key'] ) - chan.close() - conn.close() - except Exception, e: - message = "Error in sending the data transfer message to the Galaxy AMQP message queue:<br/>%s" % str(e) - status = "error" - return trans.response.send_redirect( web.url_for( controller='requests_admin', - action='manage_datasets', - sample_id=trans.security.encode_id( sample.id ), - status=status, - message=message) ) - - def __start_datatx( self, trans, sample, selected_sample_datasets ): - datatx_user = self.__setup_datatx_user( trans, sample ) - # Validate sequencer information + message = xml % dict( GALAXY_HOST=trans.request.host, + API_KEY=trans.user.api_keys[0].key, + DATA_HOST=datatx_info[ 'host' ], + DATA_USER=datatx_info[ 'username' ], + DATA_PASSWORD=datatx_info[ 'password' ], + REQUEST_ID=str( sample.request.id ), + SAMPLE_ID=str( sample.id ), + LIBRARY_ID=str( sample.library.id ), + FOLDER_ID=str( sample.folder.id ), + DATASETS=datasets ) + return message + def __validate_data_transfer_settings( self, trans, sample ): + err_msg = '' + # check the sequencer login info datatx_info = sample.request.type.datatx_info - if not datatx_info['host'] or not datatx_info['username'] or not datatx_info['password']: - message = "Error in sequencer login information." - status = "error" - else: - self.__send_message( trans, datatx_info, sample, selected_sample_datasets ) - message = "%i datasets have been queued for transfer from the sequencer. Click the Refresh button above to see the latest transfer status." % len( selected_sample_datasets ) + if not datatx_info[ 'host' ] \ + or not datatx_info[ 'username' ] \ + or not datatx_info[ 'password' ]: + err_msg = "Error in sequencer login information." + # check if web API is enabled and API key exists + if not trans.user.api_keys or not trans.app.config.enable_api: + err_msg = "Could not start data transfer as Galaxy Web API is not enabled. Enable Galaxy Web API in the Galaxy config file and create an API key." + # check if library_import_dir is set + if not trans.app.config.library_import_dir: + err_msg = "'library_import_dir' config variable is not set in the Galaxy config file." + # check the RabbitMQ server settings in the config file + for k, v in trans.app.config.amqp.items(): + if not v: + err_msg = 'Set RabbitMQ server settings in the "galaxy_amqp" section of the Galaxy config file. %s is not set.' % k + break + return err_msg + def initiate_data_transfer( self, trans, sample, selected_sample_datasets ): + ''' + This method initiates the transfer of the datasets from the sequencer. It + happens in the following steps: + - The current admin user needs to have ADD_LIBRARY_ITEM permission for the + target library and folder + - Create an XML message encapsulating all the data transfer info and send it + to the message queue (RabbitMQ broker) + ''' + # check data transfer settings + err_msg = self.__validate_data_transfer_settings( trans, sample ) + if not err_msg: + # check if the current user has add_library_item permission to the sample + # target library & folder + self.__check_library_add_permission( trans, sample.library, sample.folder ) + # create the message + message = self.__create_data_transfer_message( trans, + sample, + selected_sample_datasets ) + # Send the message + try: + conn = amqp.Connection( host=trans.app.config.amqp[ 'host' ] + ":" + trans.app.config.amqp[ 'port' ], + userid=trans.app.config.amqp[ 'userid' ], + password=trans.app.config.amqp[ 'password' ], + virtual_host=trans.app.config.amqp[ 'virtual_host' ], + insist=False ) + chan = conn.channel() + msg = amqp.Message( message.replace( '\n', '' ).replace( '\r', '' ), + content_type='text/plain', + application_headers={ 'msg_type': 'data_transfer' } ) + msg.properties[ "delivery_mode" ] = 2 + chan.basic_publish( msg, + exchange=trans.app.config.amqp[ 'exchange' ], + routing_key=trans.app.config.amqp[ 'routing_key' ] ) + chan.close() + conn.close() + except Exception, e: + err_msg = "Error in sending the data transfer message to the Galaxy AMQP message queue:<br/>%s" % str(e) + if not err_msg: + message = "%i datasets have been queued for transfer from the sequencer. Click the Refresh button above to monitor the transfer status." % len( selected_sample_datasets ) status = "done" return trans.response.send_redirect( web.url_for( controller='requests_admin', action='manage_datasets', --- a/transfer_datasets.ini.sample +++ /dev/null @@ -1,10 +0,0 @@ -# Galaxy LIMS Transfer Datasets Configuration File - -[data_transfer_user_login_info] -#email = datatx(a)bx.psu.edu -#password = datatx -#api_key = Generate it from the User menu in Galaxy - -[universe_wsgi_config] -# The http server section name in the Galaxy config file (universe_wsgi.ini) -#http_server_section = server:main --- a/lib/galaxy/web/api/requests.py +++ b/lib/galaxy/web/api/requests.py @@ -88,7 +88,7 @@ class RequestsController( BaseController request_id = trans.security.decode_id( id ) except TypeError: trans.response.status = 400 - return "Malformed %s id ( %s ) specified, unable to decode." % ( update_type, str( id ) ) + return "Malformed request id ( %s ) specified, unable to decode." % str( id ) try: request = trans.sa_session.query( trans.app.model.Request ).get( request_id ) except: --- a/run_galaxy_listener.sh +++ b/run_galaxy_listener.sh @@ -1,4 +1,4 @@ #!/bin/sh cd `dirname $0` -python scripts/galaxy_messaging/server/amqp_consumer.py universe_wsgi.ini 2>&1 +python scripts/galaxy_messaging/server/amqp_consumer.py --config-file=universe_wsgi.ini --http-server-section=server:main 2>&1 --- a/scripts/galaxy_messaging/client/amqp_publisher.py +++ b/scripts/galaxy_messaging/client/amqp_publisher.py @@ -19,13 +19,15 @@ xml = \ ''' <sample><barcode>%(BARCODE)s</barcode><state>%(STATE)s</state> + <api_key>%(API_KEY)s</api_key></sample>''' def handle_scan(states, amqp_config, barcode): if states.get(barcode[:2], None): values = dict( BARCODE=barcode[2:], - STATE=states.get(barcode[:2]) ) + STATE=states.get(barcode[:2]), + API_KEY=amqp_config['api_key'] ) print values data = xml % values print data @@ -68,6 +70,10 @@ def main(): states = {} for option in config.options("galaxy:amqp"): amqp_config[option] = config.get("galaxy:amqp", option) + # abort if api_key is not set in the config file + if not amqp_config['api_key']: + print 'Error: Set the api_key config variable in the config file before starting the amqp_publisher script.' + sys.exit( 1 ) count = 1 while True: section = 'scanner%i' % count

1 0

galaxy-dist commit dcc68da403f0: Fix for failing rgManQQ functional test - changed R outputs. cut down size of default png - much faster...
by commits-noreply＠bitbucket.org 20 Nov '10

20 Nov '10

# HG changeset patch -- Bitbucket.org # Project galaxy-dist # URL http://bitbucket.org/galaxy/galaxy-dist/overview # User Ross Lazarus <ross.lazarus(a)gmail.com> # Date 1289178885 18000 # Node ID dcc68da403f0e8d3ae0da07529ff96327ca05489 # Parent 74620fac33c74b60c0a2faa69364dea0115e8815 Fix for failing rgManQQ functional test - changed R outputs. cut down size of default png - much faster... Binary file test-data/rgtestouts/rgManQQ/Allelep_manhattan.png has changed --- a/tools/rgenetics/rgManQQ.py +++ b/tools/rgenetics/rgManQQ.py @@ -6,6 +6,7 @@ from rgutils import timenow, RRun, galht progname = os.path.split(sys.argv[0])[1] myversion = 'V000.1 March 2010' verbose = False +debug = False rcode=""" # license not stated so I'm assuming LGPL is ok for my derived work? @@ -156,7 +157,7 @@ if (plen > 0) { mytitle = paste('p=',cname,', ',title,sep='') myfname = chartr(' ','_',cname) myqqplot = qq(rawd[,pvalscolumn],title=mytitle) - ggsave(filename=paste(myfname,"qqplot.png",sep='_'),myqqplot,width=11,height=8,dpi=100) + ggsave(filename=paste(myfname,"qqplot.png",sep='_'),myqqplot,width=6,height=4,dpi=100) print(paste('## qqplot on',cname,'done')) if ((chromcolumn > 0) & (offsetcolumn > 0)) { if (doreorder) { @@ -168,7 +169,7 @@ if (plen > 0) { print(paste('## manhattan on',cname,'starting',chromcolumn,offsetcolumn,pvalscolumn)) mymanplot= manhattan(chrom=rawd[,chromcolumn],offset=rawd[,offsetcolumn],pvals=rawd[,pvalscolumn],title=mytitle,grey=grey) print(paste('## manhattan plot on',cname,'done')) - ggsave(filename=paste(myfname,"manhattan.png",sep='_'),mymanplot,width=11,height=8,dpi=100) + ggsave(filename=paste(myfname,"manhattan.png",sep='_'),mymanplot,width=6,height=4,dpi=100) } else { print(paste('chrom column =',chromcolumn,'offset column = ',offsetcolumn, @@ -196,6 +197,8 @@ def doManQQ(input_fname,chrom_col,offset to make them do our bidding - and save the resulting code for posterity this can be called externally, I guess...for QC eg? """ + if debug: + print 'doManQQ',input_fname,chrom_col,offset_col,pval_cols,title,grey,ctitle,outdir ffd,filtered_fname = tempfile.mkstemp(prefix='rgManQQtemp') f = open(filtered_fname,'w') inf = open(input_fname,'r') @@ -232,6 +235,8 @@ def doManQQ(input_fname,chrom_col,offset pvc = [x+3 for x in range(len(pval_cols))] # 2 for offset and chrom, 1 for r offset start pvc = 'c(%s)' % (','.join(map(str,pvc))) rcmd = '%s%s' % (rcode,rcode2 % (filtered_fname,'1','2',pvc,title,grey)) + if debug: + print 'running\n%s\n' % rcmd rlog,flist = RRun(rcmd=rcmd,title=ctitle,outdir=outdir) rlog.append('## R script=') rlog.append(rcmd) --- a/test-data/rgtestouts/rgManQQ/rgManQQtest1.R +++ b/test-data/rgtestouts/rgManQQ/rgManQQtest1.R @@ -124,7 +124,7 @@ qq = function(pvector, title=NULL, spart if (spartan) plot=plot+opts(panel.background=theme_rect(col="grey50"), panel.grid.minor=theme_blank()) qq } -rgqqMan = function(infile="/tmp/rgManQQtempcWfFkc",chromcolumn=1, offsetcolumn=2, pvalscolumns=c(3), +rgqqMan = function(infile="/tmp/rgManQQtemplYC5wa",chromcolumn=1, offsetcolumn=2, pvalscolumns=c(3), title="rgManQQtest1",grey=0) { rawd = read.table(infile,head=T,sep='\t') dn = names(rawd) @@ -142,7 +142,7 @@ if (plen > 0) { mytitle = paste('p=',cname,', ',title,sep='') myfname = chartr(' ','_',cname) myqqplot = qq(rawd[,pvalscolumn],title=mytitle) - ggsave(filename=paste(myfname,"qqplot.png",sep='_'),myqqplot,width=11,height=8,dpi=100) + ggsave(filename=paste(myfname,"qqplot.png",sep='_'),myqqplot,width=6,height=4,dpi=100) print(paste('## qqplot on',cname,'done')) if ((chromcolumn > 0) & (offsetcolumn > 0)) { if (doreorder) { @@ -154,7 +154,7 @@ if (plen > 0) { print(paste('## manhattan on',cname,'starting',chromcolumn,offsetcolumn,pvalscolumn)) mymanplot= manhattan(chrom=rawd[,chromcolumn],offset=rawd[,offsetcolumn],pvals=rawd[,pvalscolumn],title=mytitle,grey=grey) print(paste('## manhattan plot on',cname,'done')) - ggsave(filename=paste(myfname,"manhattan.png",sep='_'),mymanplot,width=11,height=8,dpi=100) + ggsave(filename=paste(myfname,"manhattan.png",sep='_'),mymanplot,width=6,height=4,dpi=100) } else { print(paste('chrom column =',chromcolumn,'offset column = ',offsetcolumn, @@ -171,4 +171,4 @@ if (plen > 0) { rgqqMan() # execute with defaults as substituted -#R script autogenerated by rgenetics/rgutils.py on 18/09/2010 20:46:49 +#R script autogenerated by rgenetics/rgutils.py on 07/11/2010 20:03:37 --- a/test-data/rgtestouts/rgManQQ/rgManQQtest1.html +++ b/test-data/rgtestouts/rgManQQ/rgManQQtest1.html @@ -43,7 +43,7 @@ Loading required package: grid Loading required package: proto -[1] "### 101 values read from /tmp/rgManQQtempcWfFkc read - now running plots" +[1] "### 101 values read from /tmp/rgManQQtemplYC5wa read - now running plots" [1] "## qqplot on Allelep done" @@ -178,7 +178,7 @@ qq = function(pvector, title=NULL, spart if (spartan) plot=plot+opts(panel.background=theme_rect(col="grey50"), panel.grid.minor=theme_blank()) qq } -rgqqMan = function(infile="/tmp/rgManQQtempcWfFkc",chromcolumn=1, offsetcolumn=2, pvalscolumns=c(3), +rgqqMan = function(infile="/tmp/rgManQQtemplYC5wa",chromcolumn=1, offsetcolumn=2, pvalscolumns=c(3), title="rgManQQtest1",grey=0) { rawd = read.table(infile,head=T,sep='\t') dn = names(rawd) @@ -196,7 +196,7 @@ if (plen > 0) { mytitle = paste('p=',cname,', ',title,sep='') myfname = chartr(' ','_',cname) myqqplot = qq(rawd[,pvalscolumn],title=mytitle) - ggsave(filename=paste(myfname,"qqplot.png",sep='_'),myqqplot,width=11,height=8,dpi=100) + ggsave(filename=paste(myfname,"qqplot.png",sep='_'),myqqplot,width=6,height=4,dpi=100) print(paste('## qqplot on',cname,'done')) if ((chromcolumn > 0) & (offsetcolumn > 0)) { if (doreorder) { @@ -208,7 +208,7 @@ if (plen > 0) { print(paste('## manhattan on',cname,'starting',chromcolumn,offsetcolumn,pvalscolumn)) mymanplot= manhattan(chrom=rawd[,chromcolumn],offset=rawd[,offsetcolumn],pvals=rawd[,pvalscolumn],title=mytitle,grey=grey) print(paste('## manhattan plot on',cname,'done')) - ggsave(filename=paste(myfname,"manhattan.png",sep='_'),mymanplot,width=11,height=8,dpi=100) + ggsave(filename=paste(myfname,"manhattan.png",sep='_'),mymanplot,width=6,height=4,dpi=100) } else { print(paste('chrom column =',chromcolumn,'offset column = ',offsetcolumn, @@ -227,6 +227,6 @@ rgqqMan() </pre> -<h3><a href="http://rgenetics.org">Rgenetics</a> tool rgManQQ.py run at 18/09/2010 20:48:26</h3> +<h3><a href="http://rgenetics.org">Rgenetics</a> tool rgManQQ.py run at 07/11/2010 20:04:20</h3></div></body></html> --- a/tools/rgenetics/rgManQQ.xml +++ b/tools/rgenetics/rgManQQ.xml @@ -42,7 +42,7 @@ <param name='chrom_col' value='1' /><param name='offset_col' value='2' /><param name='grey' value='0' /> - <output name='out_html' file='rgtestouts/rgManQQ/rgManQQtest1.html' ftype='html' lines_diff='40'> + <output name='out_html' file='rgtestouts/rgManQQ/rgManQQtest1.html' ftype='html' lines_diff='60'><extra_files type="file" name='Allelep_manhattan.png' value='rgtestouts/rgManQQ/Allelep_manhattan.png' compare="sim_size" delta = "10000"/><extra_files type="file" name='Allelep_qqplot.png' value='rgtestouts/rgManQQ/Allelep_qqplot.png' compare="sim_size" Binary file test-data/rgtestouts/rgManQQ/Allelep_qqplot.png has changed

1 0

galaxy-dist commit 4b78bad81e77: Trackster: save current viewport when saving viz
by commits-noreply＠bitbucket.org 20 Nov '10

20 Nov '10

# HG changeset patch -- Bitbucket.org # Project galaxy-dist # URL http://bitbucket.org/galaxy/galaxy-dist/overview # User James Taylor <james(a)jamestaylor.org> # Date 1289169067 18000 # Node ID 4b78bad81e777cf503ac66acb5dbb01da841400f # Parent fb55bfc792b1d087783055a736d9fb439e5bb41c Trackster: save current viewport when saving viz --- a/static/scripts/trackster.js +++ b/static/scripts/trackster.js @@ -81,7 +81,7 @@ var Cache = function( num_elements ) { } }); -var View = function( container, title, vis_id, dbkey ) { +var View = function( container, title, vis_id, dbkey, callback ) { this.container = container; this.vis_id = vis_id; this.dbkey = dbkey; @@ -95,11 +95,11 @@ var View = function( container, title, v this.zoom_factor = 3; this.min_separation = 30; this.has_changes = false; - this.init(); + this.init( callback ); this.reset(); }; $.extend( View.prototype, { - init: function() { + init: function( callback ) { // Create DOM elements var parent_element = this.container, view = this; @@ -177,6 +177,9 @@ var View = function( container, title, v view.chrom_select.bind("change", function() { view.change_chrom(view.chrom_select.val()); }); + if ( callback ) { + callback(); + } }, error: function() { alert( "Could not load chroms for this dbkey:", view.dbkey ); @@ -244,8 +247,8 @@ var View = function( container, title, v this.drag_origin_x = e.clientX; this.drag_origin_pos = e.clientX / view.viewport_container.width() * (view.high - view.low) + view.low; this.drag_div = $("<div />").css( { - "height": view.content_div.height()+30, "top": "0px", "position": "absolute", - "background-color": "#cfc", "border": "1px solid #6a6", "opacity": 0.5, "z-index": 1000 + "height": view.content_div.height() + view.top_labeltrack.height() + view.nav_labeltrack.height(), "top": "0px", "position": "absolute", + "background-color": "#ccf", "opacity": 0.5, "z-index": 1000 } ).appendTo( $(this) ); }).bind( "drag", function(e) { var min = Math.min(e.clientX, this.drag_origin_x) - view.container.offset().left, @@ -1216,7 +1219,7 @@ var FeatureTrack = function ( name, view this.show_labels_scale = 0.001; this.showing_details = false; this.vertical_detail_px = 10; - this.vertical_nodetail_px = 2; + this.vertical_nodetail_px = 3; this.summary_draw_height = 30; this.default_font = "9px Monaco, Lucida Console, monospace"; this.inc_slots = {}; --- a/lib/galaxy/web/controllers/tracks.py +++ b/lib/galaxy/web/controllers/tracks.py @@ -1,12 +1,5 @@ """ Support for constructing and viewing custom "track" browsers within Galaxy. - -Track browsers are currently transient -- nothing is stored to the database -when a browser is created. Building a browser consists of selecting a set -of datasets associated with the same dbkey to display. Once selected, jobs -are started to create any necessary indexes in the background, and the user -is redirected to the browser interface, which loads the appropriate datasets. - """ import re, pkg_resources @@ -317,7 +310,7 @@ class TracksController( BaseController, if 'vis_id' in kwargs: vis_id = kwargs['vis_id'].strip('"') dbkey = kwargs['dbkey'] - + # Lookup or create Visualization object if vis_id == "undefined": # new vis vis = model.Visualization() vis.user = trans.user @@ -328,20 +321,30 @@ class TracksController( BaseController, else: decoded_id = trans.security.decode_id( vis_id ) vis = session.query( model.Visualization ).get( decoded_id ) - + # Decode the payload decoded_payload = simplejson.loads( kwargs['payload'] ) + # Create new VisualizationRevision that will be attached to the viz vis_rev = model.VisualizationRevision() vis_rev.visualization = vis vis_rev.title = vis.title vis_rev.dbkey = dbkey + # Tracks from payload tracks = [] - for track in decoded_payload: + for track in decoded_payload['tracks']: tracks.append( { "dataset_id": str(track['dataset_id']), "name": track['name'], "track_type": track['track_type'], "prefs": track['prefs'] } ) - vis_rev.config = { "tracks": tracks } + # Viewport from payload + if 'viewport' in decoded_payload: + chrom = decoded_payload['viewport']['chrom'] + start = decoded_payload['viewport']['start'] + end = decoded_payload['viewport']['end'] + vis_rev.config = { "tracks": tracks, "viewport": { 'chrom': chrom, 'start': start, 'end': end } } + else: + vis_rev.config = { "tracks": tracks } + print vis_rev.config vis.latest_revision = vis_rev session.add( vis_rev ) session.flush() --- a/templates/tracks/browser.mako +++ b/templates/tracks/browser.mako @@ -77,7 +77,11 @@ $(function() { %if config: - view = new View( $("#browser-container"), "${config.get('title') | h}", "${config.get('vis_id')}", "${config.get('dbkey')}" ); + var callback; + %if 'viewport' in config: + var callback = function() { view.change_chrom( '${config['viewport']['chrom']}', ${config['viewport']['start']}, ${config['viewport']['end']} ); } + %endif + view = new View( $("#browser-container"), "${config.get('title') | h}", "${config.get('vis_id')}", "${config.get('dbkey')}", callback ); view.editor = true; %for track in config.get('tracks'): view.add_track( @@ -180,21 +184,25 @@ $("#save-button").bind("click", function(e) { var sorted = $(".viewport-container").sortable('toArray'), - payload = []; + tracks = []; + + // Show saving dialog box + show_modal("Saving...", "<img src='${h.url_for('/static/images/yui/rel_interstitial_loading.gif')}'/>"); for (var i in sorted) { var track_id = parseInt(sorted[i].split("track_")[1]), track = view.tracks[track_id]; - payload.push( { + tracks.push( { "track_type": track.track_type, "name": track.name, "dataset_id": track.dataset_id, "prefs": track.prefs }); } - // Show saving dialog box - show_modal("Saving...", "<img src='${h.url_for('/static/images/yui/rel_interstitial_loading.gif')}'/>"); + + var payload = { 'tracks': tracks, 'viewport': { 'chrom': view.chrom, 'start': view.low , 'end': view.high } } + console.log( payload ); $.ajax({ url: "${h.url_for( action='save' )}", --- a/lib/galaxy/web/base/controller.py +++ b/lib/galaxy/web/base/controller.py @@ -185,6 +185,9 @@ class UsesVisualization( SharableItemSec config = { "title": visualization.title, "vis_id": trans.security.encode_id( visualization.id ), "tracks": tracks, "chrom": "", "dbkey": visualization.dbkey } + + if 'viewport' in latest_revision.config: + config['viewport'] = latest_revision.config['viewport'] return config

1 0

galaxy-dist commit 74620fac33c7: bug fix caused by the prev commit.
by commits-noreply＠bitbucket.org 20 Nov '10

20 Nov '10

# HG changeset patch -- Bitbucket.org # Project galaxy-dist # URL http://bitbucket.org/galaxy/galaxy-dist/overview # User rc # Date 1289178573 18000 # Node ID 74620fac33c74b60c0a2faa69364dea0115e8815 # Parent 328b57b1e2e53cbc9ab208379cdd3dc0864f3cc4 bug fix caused by the prev commit. --- a/lib/galaxy/web/controllers/requests_admin.py +++ b/lib/galaxy/web/controllers/requests_admin.py @@ -664,6 +664,8 @@ class RequestsAdmin( BaseController, Use if not err_msg: message = "%i datasets have been queued for transfer from the sequencer. Click the Refresh button above to monitor the transfer status." % len( selected_sample_datasets ) status = "done" + else: + status = 'error' return trans.response.send_redirect( web.url_for( controller='requests_admin', action='manage_datasets', sample_id=trans.security.encode_id( sample.id ),

1 0

galaxy-dist commit 4187ae2fdb9e: Improved error handling in api methods.
by commits-noreply＠bitbucket.org 20 Nov '10

20 Nov '10

# HG changeset patch -- Bitbucket.org # Project galaxy-dist # URL http://bitbucket.org/galaxy/galaxy-dist/overview # User rc # Date 1289186302 18000 # Node ID 4187ae2fdb9e3f8b7ff5878e7517a24868b8ffca # Parent dcc68da403f0e8d3ae0da07529ff96327ca05489 Improved error handling in api methods. --- a/scripts/api/common.py +++ b/scripts/api/common.py @@ -48,6 +48,13 @@ def put( api_key, url, data ): req.get_method = lambda: 'PUT' return simplejson.loads( urllib2.urlopen( req ).read() ) +def __del( api_key, url, data ): + # Do the actual DELETE + url = make_url( api_key, url ) + req = urllib2.Request( url, headers = { 'Content-Type': 'application/json' }, data = simplejson.dumps( data )) + req.get_method = lambda: 'DELETE' + return simplejson.loads( urllib2.urlopen( req ).read() ) + def display( api_key, url, return_formatted=True ): # Sends an API GET request and acts as a generic formatter for the JSON response. @@ -89,12 +96,12 @@ def submit( api_key, url, data, return_f try: r = post( api_key, url, data ) except urllib2.HTTPError, e: - print e - print e.read( 1024 ) if return_formatted: + print e + print e.read( 1024 ) sys.exit( 1 ) else: - return 'Error. '+ str( e ) + return 'Error. '+ str( e.read( 1024 ) ) if not return_formatted: return r print 'Response' @@ -123,30 +130,35 @@ def update( api_key, url, data, return_f try: r = put( api_key, url, data ) except urllib2.HTTPError, e: - print e - print e.read( 1024 ) - sys.exit( 1 ) + if return_formatted: + print e + print e.read( 1024 ) + sys.exit( 1 ) + else: + return 'Error. '+ str( e.read( 1024 ) ) if not return_formatted: return r print 'Response' print '--------' - if type( r ) == list: - # Currently the only implemented responses are lists of dicts, because - # submission creates some number of collection elements. - for i in r: - if type( i ) == dict: - if 'url' in i: - print i.pop( 'url' ) - else: - print '----' - if 'name' in i: - print ' name: %s' % i.pop( 'name' ) - for k, v in i.items(): - print ' %s: %s' % ( k, v ) - else: - print i - else: - print r + print r + +def delete( api_key, url, data, return_formatted=True ): + # Sends an API DELETE request and acts as a generic formatter for the JSON response. + # 'data' will become the JSON payload read by Galaxy. + try: + r = __del( api_key, url, data ) + except urllib2.HTTPError, e: + if return_formatted: + print e + print e.read( 1024 ) + sys.exit( 1 ) + else: + return 'Error. '+ str( e.read( 1024 ) ) + if not return_formatted: + return r + print 'Response' + print '--------' + print r # utility method to encode ID's def encode_id( config_id_secret, obj_id ): --- a/scripts/galaxy_messaging/server/amqp_consumer.py +++ b/scripts/galaxy_messaging/server/amqp_consumer.py @@ -95,7 +95,7 @@ def update_request( api_key, request_id try: retval = api.update( api_key, url, data, return_formatted=False ) log.debug( str( retval ) ) - except urllib2.URLError, e: + except Exception, e: log.debug( 'ERROR(update_request (%s)): %s' % ( str((self.api_key, url, data)), str(e) ) ) def recv_callback( message ): --- a/lib/galaxy/web/controllers/requests_admin.py +++ b/lib/galaxy/web/controllers/requests_admin.py @@ -662,7 +662,7 @@ class RequestsAdmin( BaseController, Use except Exception, e: err_msg = "Error in sending the data transfer message to the Galaxy AMQP message queue:<br/>%s" % str(e) if not err_msg: - message = "%i datasets have been queued for transfer from the sequencer. Click the Refresh button above to monitor the transfer status." % len( selected_sample_datasets ) + err_msg = "%i datasets have been queued for transfer from the sequencer. Click the Refresh button above to monitor the transfer status." % len( selected_sample_datasets ) status = "done" else: status = 'error' @@ -670,7 +670,7 @@ class RequestsAdmin( BaseController, Use action='manage_datasets', sample_id=trans.security.encode_id( sample.id ), status=status, - message=message ) ) + message=err_msg ) ) @web.expose def update_sample_dataset_status(self, trans, cntrller, sample_dataset_ids, new_status, error_msg=None ): # check if the new status is a valid transfer status

1 0

galaxy-dist commit cdd6ce1f38b0: Enable filter tool to operate correctly on files with a variable number of columns (such as SAM files). Tool prepares and processes only columns up to and including the right-most column (i.e. column with the largest index) in the filtering condition, and subsequent columns are ignored. New functional test added for this functionality as well.
by commits-noreply＠bitbucket.org 20 Nov '10

20 Nov '10

# HG changeset patch -- Bitbucket.org # Project galaxy-dist # URL http://bitbucket.org/galaxy/galaxy-dist/overview # User jeremy goecks <jeremy.goecks(a)emory.edu> # Date 1288902634 14400 # Node ID cdd6ce1f38b0e22802ed63e3e0cb4f6a58cb93b2 # Parent f372ea5a601443a9b9cd1d3f7e04ddc24db5b57a Enable filter tool to operate correctly on files with a variable number of columns (such as SAM files). Tool prepares and processes only columns up to and including the right-most column (i.e. column with the largest index) in the filtering condition, and subsequent columns are ignored. New functional test added for this functionality as well. --- a/tools/stats/filtering.py +++ b/tools/stats/filtering.py @@ -60,10 +60,18 @@ for operand in operands: except: if operand in secured: stop_err( "Illegal value '%s' in condition '%s'" % ( operand, cond_text ) ) + +# Find the largest column used in the filter. +largest_col_index = -1 +for match in re.finditer( 'c(\d)+', cond_text ): + col_index = int( match.group()[1:] ) + if col_index > largest_col_index: + largest_col_index = col_index -# Prepare the column variable names and wrappers for column data types +# Prepare the column variable names and wrappers for column data types. Only +# prepare columns up to largest column in condition. cols, type_casts = [], [] -for col in range( 1, in_columns + 1 ): +for col in range( 1, largest_col_index + 1 ): col_name = "c%d" % col cols.append( col_name ) col_type = in_column_types[ col - 1 ] @@ -72,7 +80,7 @@ for col in range( 1, in_columns + 1 ): col_str = ', '.join( cols ) # 'c1, c2, c3, c4' type_cast_str = ', '.join( type_casts ) # 'str(c1), int(c2), int(c3), str(c4)' -assign = "%s = line.split( '\\t' )" % col_str +assign = "%s, = line.split( '\\t' )[:%i]" % ( col_str, largest_col_index ) wrap = "%s = %s" % ( col_str, type_cast_str ) skipped_lines = 0 first_invalid_line = 0 --- a/tools/stats/filtering.xml +++ b/tools/stats/filtering.xml @@ -23,6 +23,13 @@ <param name="cond" value="c1=='chr1' and c3-c2>=2000 and c6=='+'"/><output name="out_file1" file="filter1_test2.bed"/></test> +  + <test> + <param name="input" value="filter1_in3.sam"/> + <param name="cond" value="c3=='chr1' and c5>5"/> + <output name="out_file1" file="filter1_test3.sam"/> + </test> + </tests><help> --- /dev/null +++ b/test-data/filter1_in3.sam @@ -0,0 +1,100 @@ +HWI-EAS269B:8:34:797:623 145 chr1 16632182 255 27M = 16631977 0 CCATTTCCTGTATGCTGTAAAGTACAA IIIIIIIIIIIIIIIIIIIIIIIIIII NM:i:0 NH:i:1 +HWI-EAS269B:8:40:39:1184 145 chr1 24620331 3 27M = 24620115 0 ATTTATGTGGTTTCGTTTACCTTCTAT IIIIIIIIIIIIIIIIIIIIIIIIIII NM:i:0 NH:i:2 CC:Z:chrM CP:i:9060 +HWI-EAS269B:8:58:533:1198 81 chr1 88426079 255 27M = 88423429 0 GAAGAGGAAGAAGGTGGGGAGGAAGAG IIIG?IIIIIIIIIIIIIIIIIIIIII NM:i:0 NH:i:1 +HWI-EAS269:3:89:1776:1815 97 chr1 134085638 255 27M = 134085824 0 GAATGATTCTCTGGGTGTTACTTTGCA IIIIIIIIIIIIIIIDIII:IIII>F5 NM:i:0 NH:i:1 +HWI-EAS269B:8:74:1134:1670 161 chr1 138166886 255 27M = 138167084 0 TTACTAGTGTCTCTCTTACCATCATAT .IIIIIIIIIIIIIIIIIIIIIIIIII NM:i:1 NH:i:1 +HWI-EAS269:3:59:1321:1427 147 chr1 173149715 255 27M = 173149555 0 AAGGGCTAGGGTGACAGGCAGGGGACG -C<CID?IIIIIIIDIIIIIIIIIIII NM:i:0 NH:i:1 +HWI-EAS269B:8:8:164:1678 145 chr1 178660716 255 27M = 178660493 0 CAATTGGTGTTTTTCTTAAGAGACTCA IIIIIIIIIIIIIIIIIIIIIIIIIII NM:i:0 NH:i:1 +HWI-EAS269:3:1:1048:638 137 chr10 12456478 1 27M * 0 0 TAAAANAAATAAAACAAAACAATAAAA $'&(+")'%$(&*&#&$$#%%$$%%$$ NM:i:2 NH:i:4 CC:Z:chr16 CP:i:21639623 +HWI-EAS269:3:62:1211:798 147 chr10 28135294 255 27M = 28135117 0 ACATGGTGTGGGGACAGAGATGTGAAG I;IIIIIIAIIIIIIIIIIIIIIIIII NM:i:1 NH:i:1 +HWI-EAS269:3:27:410:424 145 chr10 76950070 255 27M = 76949871 0 GAGTCTGTGTCCAGGCCAATTCACTAT 25-8I+.6B.IIIIIIIIIIIIIIIII NM:i:0 NH:i:1 +HWI-EAS269:3:48:1180:1122 73 chr10 83772157 0 27M * 0 0 GATCATCTTTTCTAAAACAATAAAGAC /IIIIIIIIIIIIIIIIIIIIIIIIII NM:i:0 NH:i:6 CC:Z:chr11 CP:i:93811140 +HWI-EAS269:3:77:654:131 99 chr10 93419810 255 27M = 93419993 0 TTGCATCCCTAGGAACTGGAGTTATAG IIIIIIGIHIIIED@CIIH5I3D9G6: NM:i:0 NH:i:1 +HWI-EAS269B:8:98:895:1810 177 chr11 3371472 255 27M = 37952212 0 CTACATAGTGGGAACCGGCGACCGCTG IIIIIIIIIIIIIIIIIIIIIIIIIII NM:i:2 NH:i:1 +HWI-EAS269B:8:58:1126:883 97 chr11 30080486 255 27M = 30080702 0 TACTTCCACCAGGCTCCAGTTTTGTGA IIIIIIIIIIIIIIIIIIIIIIIIII5 NM:i:0 NH:i:1 +HWI-EAS269B:8:89:1763:1446 81 chr11 59589605 255 27M = 59589400 0 GTACCTGGCCACTGATGCAGCTTAGAA II<IIIIIIIIIIIIIIIIIIIIIIII NM:i:0 NH:i:1 +HWI-EAS269B:8:6:739:1329 161 chr11 68629628 3 27M = 68629835 0 GTTTTTGGTTGATAGTTGAGCAAACTG IIIIIIIIIIIIIIIIIIIIIIIIIII NM:i:0 NH:i:2 CC:Z:chr4 CP:i:126661877 +HWI-EAS269:3:19:1683:1350 81 chr11 70846915 255 27M = 70846725 0 CACGGGCAGATAAGCTGCTGAGACTAA H50BB:IF=IIGIIIIIIIIIIIIIII NM:i:0 NH:i:1 +HWI-EAS269B:8:9:1691:1087 97 chr11 84640783 255 27M = 84641010 0 ACATTATCCATCTCTCTGTCATTGTCC IIIIIIIIIIIIIIIIIIIIIIIIIII NM:i:0 NH:i:1 +HWI-EAS269B:8:48:1350:424 81 chr11 94150331 255 27M = 94150113 0 TTGCAAGCCAACCCTGAGTGAAGTGTC IIDHIIIIIIIIIIIIIIIIIIIIIII NM:i:0 NH:i:1 +HWI-EAS269:3:22:1176:1405 97 chr11 98629843 255 27M = 98630030 0 CCCTCGGAGGGCAACGGGAAAGGAGAA IIIIIIIIIIIDIIIII?III/79E14 NM:i:0 NH:i:1 +HWI-EAS269:3:67:1311:1512 97 chr11 100488190 255 27M = 100488374 0 CATGCCCTGAGACTTAGCAAGACTCTT IIIIIIIIIIIIIIIEIA3,I57GBI@ NM:i:0 NH:i:1 +HWI-EAS269B:8:41:1142:1307 161 chr11 101320424 255 27M = 101320625 0 GCCTCCCTACATAGCAAAGGAAAGAAA IIIIIIIIIIIIIIIIIIIIIIIIIII NM:i:0 NH:i:1 +HWI-EAS269B:8:31:1101:21 97 chr11 116711287 255 27M = 116711485 0 CATAAGCAAAAGATACACCATGTTTTA IIIIIIIIIIIIIIIIIIIIIIIIIII NM:i:0 NH:i:1 +HWI-EAS269:3:84:637:1203 145 chr12 35829424 255 27M = 35829229 0 CTCCCTCAAGGATCTTGTGGGGCATCA </'>I,I5B26$II;EB=%IIII1III NM:i:0 NH:i:1 +HWI-EAS269:3:65:1158:1081 163 chr12 52234332 255 27M = 52234505 0 TTTTTTTTTTAAGACAGGGAGTTTTTT IIIIIIIIIIIC4,,*>II%-,%III> NM:i:1 NH:i:1 +HWI-EAS269B:8:65:1325:1287 137 chr12 52234513 3 27M * 0 0 GCAGCACCTTGTGACATTTAATTTAGT IIIII+IIIIIIIIIIIIIIIIIIIII NM:i:0 NH:i:2 CC:Z:chr15 CP:i:98780443 +HWI-EAS269:3:38:488:1093 97 chr12 80471642 255 27M = 80471839 0 TGGAGAGATGACTCCTTGTTTGAGACA IIIIIIIIIIIIIIIIIIIBAE0B<7D NM:i:0 NH:i:1 +HWI-EAS269:3:76:290:1451 99 chr12 117745566 0 27M = 117745739 0 CAATCTGAAGATCCACAATCTTTTATA IIIIIIIIIIIIIIFII5I9IIII+I7 NM:i:0 NH:i:5 CC:Z:chr4 CP:i:133520935 +HWI-EAS269B:8:39:353:1526 65 chr13 18122558 255 27M = 18122761 0 ACTCTACTCAAAACCACACTAAGCCTC @IIIIII6IIIIIDII9IIIIII<III NM:i:0 NH:i:1 +HWI-EAS269:3:63:1260:365 99 chr13 41278147 255 27M = 41278319 0 AAGACAAGAACTTATCCACCAATATGT IIIIIIIIIIIIIII<IIA<CIIDII> NM:i:0 NH:i:1 +HWI-EAS269:3:70:152:1609 161 chr13 55421426 255 27M = 55421687 0 CGCTTGACCATTCCAGCCCAGACAAGA IIIIIIIII8IIII3IIII-C(3-%B' NM:i:0 NH:i:1 +HWI-EAS269:3:9:1343:1414 99 chr13 83721797 255 27M = 83721967 0 AACTACAGCTGAGGCAGCCTCCTGCCT IIIIIIIIIIIIIIIII;E+?7&819& NM:i:0 NH:i:1 +HWI-EAS269:3:4:1480:1956 137 chr14 46703849 0 27M * 0 0 GTGGAGCCCAGTGACACCATCGAGAAC IIIIIIIIIIIII?C3IH?1@I@@=27 NM:i:0 NH:i:8 CC:Z:= CP:i:46704077 +HWI-EAS269:3:86:616:1519 137 chr14 56244763 255 27M * 0 0 CATGGCCTGAAGTTCCTGAGCTTTATC IIIIIIIII3DIBI9II3)73BIG'G+ NM:i:0 NH:i:1 +HWI-EAS269:3:42:656:1216 97 chr14 100271567 255 27M = 100271751 0 ATAAGTCTCAGTTCTTGGGCCAGATCA IIIIIIIIICIIIIII3IA854I1C+) NM:i:0 NH:i:1 +HWI-EAS269:3:87:95:767 73 chr16 7368880 255 27M * 0 0 AATATCTGAAACATTCAAATGAGCAAT F>II/4I@9H=II?IIIIIIIIIIII- NM:i:0 NH:i:1 +HWI-EAS269B:8:50:1572:711 145 chr16 32230400 255 27M = 32230196 0 TACAGCAACAACAAATTCAACGACACG IIIIIIIIIIIIIIIIIIIIIIIIIII NM:i:0 NH:i:1 +HWI-EAS269:3:56:1436:121 147 chr16 49887529 255 27M = 49887356 0 TTAAGGCCCAGCTCTACATAAAACACT IIIIAHIIIIIIIIIIIIIIIIIIIII NM:i:0 NH:i:1 +HWI-EAS269:3:53:584:1696 99 chr17 45705815 255 27M = 45705987 0 CCTTCTTGTCCAAGATCTCCTTCATGA IIIIIIIIIIIIBI6IIII=II:GI@< NM:i:0 NH:i:1 +HWI-EAS269B:8:16:643:1950 177 chr18 38000738 255 27M = 38000570 0 ATTCTGGTTCAGCCTGGGCAGCTTGGG III@==I0<IIGIII;IIIIIIIIIII NM:i:0 NH:i:1 +HWI-EAS269:3:66:1760:1539 137 chr18 38624681 3 27M * 0 0 TTTGGAACTTTTGAGAGGATCCCTAGC IIIIIIIIIIIIIII1II.IIIII1II NM:i:0 NH:i:2 +HWI-EAS269:3:2:1723:1277 163 chr19 16341705 255 27M = 16341870 0 ATGGGCTCGTCGCAGCTCAGCGGCTGG IIIIIIIIIIIIICI=8<&I7:F(+;2 NM:i:1 NH:i:1 +HWI-EAS269B:8:66:317:1676 97 chr19 47099316 255 27M = 47099535 0 ATCTGGTCAATCAACACCACCAGCAGA IIIIIIIIIIIIIIIIIIIIIIIIIII NM:i:0 NH:i:1 +HWI-EAS269:3:10:1428:1315 81 chr2 22444224 3 27M = 22444039 0 GGAATTGCGATAATTATAGTGGCTGAT ?6>CI>GIIIIIIIIIIIIIIIIIIII NM:i:0 NH:i:2 CC:Z:chrM CP:i:6245 +HWI-EAS269:3:72:152:785 97 chr2 22796881 255 27M = 22797074 0 TGAGTACTCCAGATAATCGTTACACAA IIIIIIII6IIIIIIII0IA@?,7752 NM:i:0 NH:i:1 +HWI-EAS269:3:79:321:1095 99 chr2 62083760 255 27M = 62083938 0 CAAAAATTGAGATATCAAAAAGCTCTT IIIIIIIII2IGIBIIC8:4IB1H4I; NM:i:0 NH:i:1 +HWI-EAS269:3:93:1529:881 97 chr2 129998362 255 27M = 129998827 0 CGCATGCCAGGAGGGGGCATGCCCATT IIIFIIIIFIIIIIFII?72I.,020* NM:i:0 NH:i:1 +HWI-EAS269B:8:20:813:376 113 chr2 130534278 255 27M = 130534454 0 TGTATTAGCAGGAGGTGGGGAGGCTGA IIIIIIIIIIIIIIIIIIIIIIIIIII NM:i:0 NH:i:1 +HWI-EAS269:3:89:1221:1082 83 chr2 151285071 0 27M = 151284944 0 ACCAGTGCACAGGTCTCCAGGGCTTCT -IC=B+8IIIEIIFI5IIIIIIIIIII NM:i:0 NH:i:5 CC:Z:chr9 CP:i:31743980 +HWI-EAS269:3:66:46:336 163 chr2 156679837 255 27M = 156680016 0 TATTTTCCTTTTGCTGTGGTTTGTGTT IIIIIIIIIIIIIEI@E-I?GI&B%3I NM:i:0 NH:i:1 +HWI-EAS269B:8:52:1139:1381 73 chr2 174953004 0 27M * 0 0 AATGCTCAACTCTTAGTTTCTTATTCA IIIIIIIIIIIIIIIIB@?IIIII?+0 NM:i:0 NH:i:19 CC:Z:= CP:i:175018018 +HWI-EAS269:3:98:585:19 161 chr3 26005087 255 27M = 26005271 0 ACCTAACACATGTAAACTTAAATTCAT I:IIIIIIII<IIE==DDI):,D1--8 NM:i:0 NH:i:1 +HWI-EAS269:3:74:447:1309 83 chr3 80492396 255 27M = 80492220 0 TCGGATGCCTCTCACCACTTTGACAAT )H():F=85IIIIEIIIIIIIIIIIII NM:i:0 NH:i:1 +HWI-EAS269B:8:74:425:1427 161 chr3 96447603 255 27M = 96447820 0 TGGAGTAGGAGTCTCAGGAGGAGTAGA IIIIIIIIIIIIIIIIIIIII:I?II< NM:i:0 NH:i:1 +HWI-EAS269:3:10:739:1474 163 chr3 124089371 255 27M = 124089535 0 AAAAAAACAATCTTATTCCGAGCATTC IIIIIIIIIIIIIIIIIIIIIIIGIII NM:i:0 NH:i:1 +HWI-EAS269:3:20:357:1709 65 chr4 128895987 255 27M = 128895822 0 CCTACCTTCTTCCCTTGGCAGCTGACT IIII>IIIBIIA:I18):,*3&,/*'+ NM:i:0 NH:i:1 +HWI-EAS269B:8:22:623:129 113 chr4 135492867 255 27M = 135490613 0 CCACTTTCCTGTACTGGCCAGAAAATG IIIIIIIIIIIIIIIIIIIIIIIIIII NM:i:0 NH:i:1 +HWI-EAS269B:8:48:824:702 65 chr4 139584010 255 27M = 139582899 0 CTGGGCAGTGCAGCGGTACATGGAGCC IIIIIIIIIIIIIIIIII<AIIB;I>: NM:i:0 NH:i:1 +HWI-EAS269B:8:33:527:1211 73 chr5 37627410 255 27M * 0 0 GGGAAGGTGTGAGTACAACAGCCAAAG IIIIIIIIIIIIIIIIIIIIIIII@II NM:i:0 NH:i:1 +HWI-EAS269:3:2:1518:599 163 chr5 136908928 255 27M = 136909101 0 CTATTGCCAAAAAACTATGTTCACAAA IIIIIIIIIIIIIIIIIIIIIIIII=I NM:i:0 NH:i:1 +HWI-EAS269B:8:62:1493:1455 161 chr5 138715332 255 27M = 138715534 0 CCCAAATGAAAAAATAAATATTATGAA IIIIIIIIIIIIIIIIIIIIIIIIIII NM:i:0 NH:i:1 +HWI-EAS269:3:69:384:1894 147 chr5 143665243 255 27M = 143665064 0 TGTTTGCTCCAACCAACTGCTGTCGCC &++*,2H-IGI+IFIGI?IIIIIIIII NM:i:0 NH:i:1 +HWI-EAS269:3:56:1358:762 113 chr6 15024174 255 7M12235N20M = 51485453 0 TGGGTACTTTCTCTAGCTCCTCCATTG /@)3I?IIIIIIIIIIIIIIIIIIIII NM:i:2 XS:A:- NH:i:1 +HWI-EAS269:3:100:228:1799 163 chr6 37756378 3 27M = 37756558 0 GTGTGGGGCTGCGTGGCCTGGCTGGTG DII4A+>5EI.)F634820&1(0%&&& NM:i:2 NH:i:2 +HWI-EAS269B:8:50:188:1253 161 chr6 52296684 1 27M = 52296901 0 ACTTTTCAGGGTTTTCAATAGTCACAC IIIIIIIIIIIIIIIIIIIIIIIIIII NM:i:1 NH:i:3 CC:Z:= CP:i:140695441 +HWI-EAS269B:8:19:440:1687 161 chr6 52850377 255 27M = 52850580 0 TACAAAGATGGACTTTTAAAATTCATT IIIIIIIIIIIIIIIIIIIIIIIIIII NM:i:0 NH:i:1 +HWI-EAS269:3:47:1555:1520 161 chr6 67231655 255 27M = 67233919 0 ATTCAGTTATGGACCATCATTTCCGGA IIIIIIIIIIIIIIIIIIIIII<AII1 NM:i:0 NH:i:1 +HWI-EAS269B:8:79:1563:1318 161 chr6 89267414 255 27M = 89267614 0 GTGCGTGTTCCAGGCAGAGCTGGAAGA IIIIIIIIIIIIIIIIIIIIIII=;IA NM:i:0 NH:i:1 +HWI-EAS269B:8:54:954:565 161 chr6 120882513 255 27M = 120882716 0 ACGTCATGGCTGACCAGGACAGAGGTG IIIIIIIIIIIIIII?IEG;II<II*8 NM:i:0 NH:i:1 +HWI-EAS269B:8:16:570:1775 145 chr7 26081268 255 27M = 26080313 0 GAAAGAGTGACACAAATCAATAGTAAA IIIIIIIIIIIIIIIIIIIIIIIIIII NM:i:0 NH:i:1 +HWI-EAS269B:8:13:280:189 161 chr7 29184615 255 27M = 29184814 0 CTTTTCCAACAGCGAGAAAAATGTACA IIIIIIIIIIIIIIIIIIIIIIIIDII NM:i:0 NH:i:1 +HWI-EAS269B:8:19:72:273 137 chr7 52258521 255 27M * 0 0 ATCAGAGGCACAGGGACAGGGTAAGGA &&2IIIII;E(IIIIIIIIIIIIIIII NM:i:1 NH:i:1 +HWI-EAS269B:8:40:235:869 161 chr7 52631974 255 27M = 52632179 0 GGCGCTGACTCCATCAGATATCCATTC IIIIIII7IIIIIII6IIIIIIIBIII NM:i:1 NH:i:1 +HWI-EAS269B:8:18:304:346 129 chr7 107789557 255 27M = 33158390 0 CACGTACTGTCACCTTGTAACATTTGG IIIIIIIIIIIIIIIIIIIIIIIIIIF NM:i:0 NH:i:1 +HWI-EAS269B:8:3:698:297 161 chr7 110976065 255 27M = 110976385 0 GCAGTTATCACTTTCTTGCCATGGGCC IIIIIIIIIIIIIIIIIIIIIIIIIII NM:i:0 NH:i:1 +HWI-EAS269B:8:50:1269:965 145 chr7 118217605 255 27M = 118216239 0 ACCTGTAGATCCACATGATCATGAAGA IIIIIIIIIIIIIIIIIIIIIIIIIII NM:i:0 NH:i:1 +HWI-EAS269:3:38:1381:1429 83 chr7 136141344 255 27M = 136141280 0 ATCTGAAGTATCCCACATGTTGAGCTC <III@IIIIIIIII>IIIIIIIIIIII NM:i:1 NH:i:1 +HWI-EAS269:3:25:781:386 83 chr8 29844005 255 27M = 29843841 0 TAAGGGAGGAAAGTGTTTCAGAGTGTA ;83<159<<III@IIIIIIIIIIIIII NM:i:0 NH:i:1 +HWI-EAS269:3:25:979:679 163 chr8 72664237 255 27M = 72664413 0 CTGATGGGAGCCCTGCGTGGTAAGAGG IIIIIIIIIII=III@I(II$27I>I4 NM:i:0 NH:i:1 +HWI-EAS269B:8:23:1069:816 145 chr8 74704433 255 27M = 74704222 0 TGTTCTCAGTTGTGGACAAGTGACAGC I<GII@IIIIIIII;IIIIIIIIII0I NM:i:0 NH:i:1 +HWI-EAS269B:8:34:435:1679 73 chr8 87366211 3 27M * 0 0 AAGCCTAGGGCTTCTCCTCTACACCCC I556I;FCAIIIFI<IIIIIIIIIIII NM:i:0 NH:i:2 +HWI-EAS269B:8:32:1486:294 97 chr8 124121215 255 27M = 124121411 0 TGTTACCATACGCCCTTCTGCTGAGGC IIIIIIIIIIIIIIIIIIDIIIIGIII NM:i:0 NH:i:1 +HWI-EAS269B:8:32:142:1471 81 chr8 124496752 3 27M = 124496536 0 GGATCTGCTTCATGAGTTGCCACATTG IIIIIIIIIIIIIIIIIIIIIIIIIII NM:i:1 NH:i:2 +HWI-EAS269B:8:60:82:1806 97 chr8 125945215 255 27M = 125945423 0 AGATGCTGGCCATCCAGAGTAAGAACA IIIIIIIIIIIIIIIIIIIHIIIIIII NM:i:0 NH:i:1 +HWI-EAS269B:8:16:1162:495 81 chr8 125945584 255 27M = 125945381 0 GTTGCTCGCAGCTGGGGTGTGGGGCCA <CIIIIIIIIIIIIIIIIIIIIIIIII NM:i:0 NH:i:1 +HWI-EAS269B:8:1:165:1889 81 chr8 126016885 255 27M = 126016469 0 TGAGCAGGAAAACACTTTAAACCAGAT IIIIIIIIIIIIIIIIIIIIIIIIIII NM:i:0 NH:i:1 +HWI-EAS269B:8:66:939:1154 145 chr9 17537149 255 27M = 17536959 0 CTCCTTCAAGTACGCCTGGGTCTTAGA IDIIIIIIIIIIIIIIIIIIIIIIIII NM:i:0 NH:i:1 +HWI-EAS269:3:68:1189:285 81 chr9 61249807 255 27M = 61242460 0 AGTCAAGCTCACTTGGCGGTGAAGGAT @.0;;A,?-F5I;7IIIIGIIIIIIII NM:i:0 NH:i:1 +HWI-EAS269:3:15:784:1754 163 chr9 74846937 255 27M = 74847108 0 GCACAGCACTGAGGAAAGGATCATCTC IIIIIIIIIIIIIIIIIIGII1CI7II NM:i:0 NH:i:1 +HWI-EAS269B:8:26:126:1382 145 chrM 3413 255 27M = 3190 0 GTTATTCTTTATAGCAGAGTACACTAA IIII8IB328I9IIIIIIIIIIIIIII NM:i:0 NH:i:1 +HWI-EAS269:3:90:320:1336 73 chrM 6326 255 27M * 0 0 ATGATCTCCAGCTATCCTATGAGCCTT IIIIIIIIA.ICI.4'(=,C>7-*&@8 NM:i:1 NH:i:1 +HWI-EAS269B:8:23:469:1215 161 chrM 7472 3 27M = 7676 0 ATTTCATCTGAAGACGTCCTCCACTCA IIIIIIIIIIIIIIIIIIIIIIIIIII NM:i:0 NH:i:2 +HWI-EAS269:3:73:36:783 161 chrM 7958 3 27M = 8152 0 CCCCAACAATAATAGGATTCCCAATCG IIIIIIIIIIIIIIIIIIIII09I>>I NM:i:0 NH:i:2 +HWI-EAS269:3:33:1528:954 99 chrM 8012 3 27M = 8186 0 TCCTATTCCCATCCTCAAAACGCCTAA IIIIIIIIIIIIIIIIIIIIIIIDII: NM:i:0 NH:i:2 +HWI-EAS269:3:2:192:1456 161 chrM 9071 3 27M = 9255 0 ACGAAACCACATAAATCAAGCCCTACT III;IEIIDI7III+III*?I@CH+5I NM:i:0 NH:i:2 +HWI-EAS269:3:63:192:470 97 chrM 14787 255 27M = 14982 0 GCAGATAAAATTCCATTTCACCCCTAC IIIIIIIIIIIIIIFIII=B5042:4E NM:i:0 NH:i:1 +HWI-EAS269B:8:49:528:63 83 chrM 14963 255 27M = 14829 0 TTTCCTATTTGCATACGCCATTCTACG IIIIIIIIIIIIIIIIIIIIIIIIIII NM:i:0 NH:i:1 +HWI-EAS269B:8:98:856:1516 145 chrX 20257960 255 27M = 20256746 0 TACCCGGATTTAAGATGTACCCCATTG IIIIIIIIIIIIIIIIIIIIIIIIIII NM:i:0 NH:i:1 +HWI-EAS269B:8:28:1174:110 145 chrX 53916348 255 27M = 53916153 0 TGAATGTCAGCATCATTGACCCACAAA IIIIFIIIIIIIIIIIIIIIIIIIIII NM:i:0 NH:i:1 +HWI-EAS269B:8:58:663:410 147 chrX 75006902 255 27M = 75006834 0 TCAGGTGGTTTACAGTGTTCTGACAAA IIIIIIIIIIIIIIIIIIIIIIIIIII NM:i:0 NH:i:1 --- /dev/null +++ b/test-data/filter1_test3.sam @@ -0,0 +1,6 @@ +HWI-EAS269B:8:34:797:623 145 chr1 16632182 255 27M = 16631977 0 CCATTTCCTGTATGCTGTAAAGTACAA IIIIIIIIIIIIIIIIIIIIIIIIIII NM:i:0 NH:i:1 +HWI-EAS269B:8:58:533:1198 81 chr1 88426079 255 27M = 88423429 0 GAAGAGGAAGAAGGTGGGGAGGAAGAG IIIG?IIIIIIIIIIIIIIIIIIIIII NM:i:0 NH:i:1 +HWI-EAS269:3:89:1776:1815 97 chr1 134085638 255 27M = 134085824 0 GAATGATTCTCTGGGTGTTACTTTGCA IIIIIIIIIIIIIIIDIII:IIII>F5 NM:i:0 NH:i:1 +HWI-EAS269B:8:74:1134:1670 161 chr1 138166886 255 27M = 138167084 0 TTACTAGTGTCTCTCTTACCATCATAT .IIIIIIIIIIIIIIIIIIIIIIIIII NM:i:1 NH:i:1 +HWI-EAS269:3:59:1321:1427 147 chr1 173149715 255 27M = 173149555 0 AAGGGCTAGGGTGACAGGCAGGGGACG -C<CID?IIIIIIIDIIIIIIIIIIII NM:i:0 NH:i:1 +HWI-EAS269B:8:8:164:1678 145 chr1 178660716 255 27M = 178660493 0 CAATTGGTGTTTTTCTTAAGAGACTCA IIIIIIIIIIIIIIIIIIIIIIIIIII NM:i:0 NH:i:1

1 0

galaxy-dist commit 9545e8675a5d: Fix uploads when ftp_file is not defined.
by commits-noreply＠bitbucket.org 20 Nov '10

20 Nov '10

# HG changeset patch -- Bitbucket.org # Project galaxy-dist # URL http://bitbucket.org/galaxy/galaxy-dist/overview # User Nate Coraor <nate(a)bx.psu.edu> # Date 1288978480 14400 # Node ID 9545e8675a5d43408d75c66fcf3115cbdaf4af8a # Parent 7e9ae46051057d822aff82a9b1f3aec0200086f2 Fix uploads when ftp_file is not defined. --- a/lib/galaxy/tools/parameters/grouping.py +++ b/lib/galaxy/tools/parameters/grouping.py @@ -287,7 +287,7 @@ class UploadDataset( Group ): rval.append( file_bunch ) # look for files uploaded via FTP valid_files = [] - if ftp_files: + if ftp_files is not None: if trans.user is None: log.warning( 'Anonymous user passed values in ftp_files: %s' % ftp_files ) ftp_files = [] @@ -299,6 +299,8 @@ class UploadDataset( Group ): path = relpath( os.path.join( dirpath, filename ), user_ftp_dir ) if not os.path.islink( os.path.join( dirpath, filename ) ): valid_files.append( path ) + else: + ftp_files = [] for ftp_file in ftp_files: if ftp_file not in valid_files: log.warning( 'User passed an invalid file path in ftp_files: %s' % ftp_file )

1 0

galaxy-dist commit eb79ab327351: Remove missing hapmapmart tool from sample tool conf
by commits-noreply＠bitbucket.org 20 Nov '10

20 Nov '10

# HG changeset patch -- Bitbucket.org # Project galaxy-dist # URL http://bitbucket.org/galaxy/galaxy-dist/overview # User Nate Coraor <nate(a)bx.psu.edu> # Date 1288983276 14400 # Node ID eb79ab327351e4789a89748269fc34428f791463 # Parent 11abfc8eed60c0bdd231d06992f72001a2c785fa Remove missing hapmapmart tool from sample tool conf --- a/tool_conf.xml.sample +++ b/tool_conf.xml.sample @@ -319,7 +319,6 @@ <label text="Data: Import and upload" id="rgimport" /><tool file="data_source/upload.xml"/><tool file="data_source/access_libraries.xml" /> - <tool file="data_source/hapmapmart.xml" /><label text="Data: Filter and Clean" id="rgfilter" /><tool file="rgenetics/rgClean.xml"/><tool file="rgenetics/rgPedSub.xml"/>

1 0

galaxy-dist commit 6d94416b2ccf: Automatically find new .sample files in tool-data/.
by commits-noreply＠bitbucket.org 20 Nov '10

20 Nov '10

# HG changeset patch -- Bitbucket.org # Project galaxy-dist # URL http://bitbucket.org/galaxy/galaxy-dist/overview # User Nate Coraor <nate(a)bx.psu.edu> # Date 1288984386 14400 # Node ID 6d94416b2ccf9205283ec197154d99522615cbf8 # Parent 1e8d7c6ad88b074b33b2029f3baa15c2eb929e70 Automatically find new .sample files in tool-data/. --- a/run.sh +++ b/run.sh @@ -5,47 +5,22 @@ cd `dirname $0` python ./scripts/check_python.py [ $? -ne 0 ] && exit 1 -FROM_SAMPLE=" - datatypes_conf.xml - reports_wsgi.ini - tool_conf.xml - tool_data_table_conf.xml - universe_wsgi.ini - tool-data/add_scores.loc - tool-data/alignseq.loc - tool-data/annotation_profiler_options.xml - tool-data/annotation_profiler_valid_builds.txt - tool-data/bfast_indexes.loc - tool-data/binned_scores.loc - tool-data/blastdb.loc - tool-data/blastdb_p.loc - tool-data/bowtie_indices.loc - tool-data/bowtie_indices_color.loc - tool-data/codingSnps.loc - tool-data/encode_datasets.loc - tool-data/funDo.loc - tool-data/lastz_seqs.loc - tool-data/liftOver.loc - tool-data/maf_index.loc - tool-data/maf_pairwise.loc - tool-data/microbial_data.loc - tool-data/phastOdds.loc - tool-data/perm_base_index.loc - tool-data/perm_color_index.loc - tool-data/quality_scores.loc - tool-data/regions.loc - tool-data/sam_fa_indices.loc - tool-data/sift_db.loc - tool-data/srma_index.loc - tool-data/twobit.loc - tool-data/shared/ucsc/builds.txt +SAMPLES=" + datatypes_conf.xml.sample + reports_wsgi.ini.sample + tool_conf.xml.sample + tool_data_table_conf.xml.sample + universe_wsgi.ini.sample + tool-data/shared/ucsc/builds.txt.sample + tool-data/*.sample " # Create any missing config/location files -for file in $FROM_SAMPLE; do - if [ ! -f "$file" -a -f "$file.sample" ]; then - echo "Initializing $file from `basename $file`.sample" - cp $file.sample $file +for sample in $SAMPLES; do + file=`echo $sample | sed -e 's/\.sample$//'` + if [ ! -f "$file" -a -f "$sample" ]; then + echo "Initializing $file from `basename $sample`" + cp $sample $file fi done

1 0

galaxy-dist commit 32e5efb7a7d5: Remove the need for setup.sh - All configuration is done in run.sh/galaxy.config.
by commits-noreply＠bitbucket.org 20 Nov '10

20 Nov '10

# HG changeset patch -- Bitbucket.org # Project galaxy-dist # URL http://bitbucket.org/galaxy/galaxy-dist/overview # User Nate Coraor <nate(a)bx.psu.edu> # Date 1288983302 14400 # Node ID 32e5efb7a7d5606a4dc1c8808fe4888ade2892e8 # Parent eb79ab327351e4789a89748269fc34428f791463 Remove the need for setup.sh - All configuration is done in run.sh/galaxy.config. --- a/run.sh +++ b/run.sh @@ -2,6 +2,53 @@ cd `dirname $0` +python ./scripts/check_python.py +[ $? -ne 0 ] && exit 1 + +FROM_SAMPLE=" + datatypes_conf.xml + reports_wsgi.ini + tool_conf.xml + tool_data_table_conf.xml + universe_wsgi.ini + tool-data/add_scores.loc + tool-data/alignseq.loc + tool-data/annotation_profiler_options.xml + tool-data/annotation_profiler_valid_builds.txt + tool-data/bfast_indexes.loc + tool-data/binned_scores.loc + tool-data/blastdb.loc + tool-data/blastdb_p.loc + tool-data/bowtie_indices.loc + tool-data/bowtie_indices_color.loc + tool-data/codingSnps.loc + tool-data/encode_datasets.loc + tool-data/funDo.loc + tool-data/lastz_seqs.loc + tool-data/liftOver.loc + tool-data/maf_index.loc + tool-data/maf_pairwise.loc + tool-data/microbial_data.loc + tool-data/phastOdds.loc + tool-data/perm_base_index.loc + tool-data/perm_color_index.loc + tool-data/quality_scores.loc + tool-data/regions.loc + tool-data/sam_fa_indices.loc + tool-data/sift_db.loc + tool-data/srma_index.loc + tool-data/twobit.loc + tool-data/shared/ucsc/builds.txt +" + +# Create any missing config/location files +for file in $FROM_SAMPLE; do + if [ ! -f "$file" -a -f "$file.sample" ]; then + echo "Initializing $file from `basename $file`.sample" + cp $file.sample $file + fi +done + # explicitly attempt to fetch eggs before running FETCH_EGGS=1 for arg in "$@"; do @@ -21,10 +68,4 @@ if [ $FETCH_EGGS -eq 1 ]; then fi fi -# Temporary: since builds.txt is now removed from source control, create it -# from the sample if necessary -if [ ! -f "tool-data/shared/ucsc/builds.txt" ]; then - cp tool-data/shared/ucsc/builds.txt.sample tool-data/shared/ucsc/builds.txt -fi - python ./scripts/paster.py serve universe_wsgi.ini $@ --- a/lib/galaxy/config.py +++ b/lib/galaxy/config.py @@ -94,14 +94,8 @@ class Configuration( object ): self.blog_url = kwargs.get( 'blog_url', None ) self.screencasts_url = kwargs.get( 'screencasts_url', None ) self.library_import_dir = kwargs.get( 'library_import_dir', None ) - if self.library_import_dir is not None and not os.path.exists( self.library_import_dir ): - raise ConfigurationError( "library_import_dir specified in config (%s) does not exist" % self.library_import_dir ) self.user_library_import_dir = kwargs.get( 'user_library_import_dir', None ) - if self.user_library_import_dir is not None and not os.path.exists( self.user_library_import_dir ): - raise ConfigurationError( "user_library_import_dir specified in config (%s) does not exist" % self.user_library_import_dir ) self.ftp_upload_dir = kwargs.get( 'ftp_upload_dir', None ) - if self.ftp_upload_dir is not None and not os.path.exists( self.ftp_upload_dir ): - os.makedirs( self.ftp_upload_dir ) self.ftp_upload_site = kwargs.get( 'ftp_upload_site', None ) self.allow_library_path_paste = kwargs.get( 'allow_library_path_paste', False ) self.disable_library_comptypes = kwargs.get( 'disable_library_comptypes', '' ).lower().split( ',' ) @@ -159,9 +153,16 @@ class Configuration( object ): return default def check( self ): # Check that required directories exist - for path in self.root, self.file_path, self.tool_path, self.tool_data_path, self.template_path, self.job_working_directory, self.cluster_files_directory: + for path in self.root, self.tool_path, self.tool_data_path, self.template_path: if not os.path.isdir( path ): raise ConfigurationError("Directory does not exist: %s" % path ) + # Create the directories that it makes sense to create + for path in self.file_path, self.new_file_path, self.job_working_directory, self.cluster_files_directory, self.template_cache, self.ftp_upload_dir, self.library_import_dir, self.user_library_import_dir, self.nginx_upload_store, './static/genetrack/plots', os.path.join( self.tool_data_path, 'shared', 'jars' ): + if path not in [ None, False ] and not os.path.isdir( path ): + try: + os.makedirs( path ) + except Exception, e: + raise ConfigurationError( "Unable to create missing directory: %s\n%s" % ( path, e ) ) # Check that required files exist for path in self.tool_config, self.datatypes_config: if not os.path.isfile(path):

1 0