- galaxy-dev - lists.galaxyproject.org

[hg] galaxy 3131: Added ability for a user to add additional sto...
by Greg Von Kuster 02 Dec '09

02 Dec '09

details: http://www.bx.psu.edu/hg/galaxy/rev/1d93eb88c93f changeset: 3131:1d93eb88c93f user: Enis Afgan <afgane(a)gmail.com> date: Mon Nov 30 16:36:38 2009 -0500 description: Added ability for a user to add additional storage volume(s) to a running EC2 instance. In this context, a new EBS volume is created, attached to a running instance, and then added to remote zpool. When an instance is shut down and then started back up, all of the associated storage volumes are automatically attached to it and the zpool is imported. diffstat: lib/galaxy/cloud/__init__.py | 100 +++++++++- lib/galaxy/cloud/providers/ec2.py | 300 +++++++++++++++++++++++++++----- lib/galaxy/cloud/providers/eucalyptus.py | 20 +- lib/galaxy/web/controllers/cloud.py | 93 +++++++++- templates/cloud/configure_cloud.mako | 19 +- 5 files changed, 445 insertions(+), 87 deletions(-) diffs (922 lines): diff -r f4654abcec1e -r 1d93eb88c93f lib/galaxy/cloud/__init__.py --- a/lib/galaxy/cloud/__init__.py Fri Nov 20 11:03:41 2009 -0500 +++ b/lib/galaxy/cloud/__init__.py Mon Nov 30 16:36:38 2009 -0500 @@ -29,6 +29,8 @@ SUBMITTED = "submitted", SHUTTING_DOWN_UCI = "shutting-downUCI", SHUTTING_DOWN = "shutting-down", + ADD_STORAGE_UCI = "add-storageUCI", + ADD_STORAGE = "add-storage", AVAILABLE = "available", RUNNING = "running", PENDING = "pending", @@ -40,6 +42,7 @@ TERMINATED = "terminated", SUBMITTED = "submitted", RUNNING = "running", + ADDING = "adding-storage", PENDING = "pending", SHUTTING_DOWN = "shutting-down", ERROR = "error" @@ -48,6 +51,7 @@ store_status = Bunch( WAITING = "waiting", IN_USE = "in-use", + ADDING = "adding", CREATING = "creating", DELETED = 'deleted', ERROR = "error" @@ -161,7 +165,8 @@ model.UCI.table.c.state==uci_states.SUBMITTED_UCI, model.UCI.table.c.state==uci_states.SHUTTING_DOWN_UCI, model.UCI.table.c.state==uci_states.DELETING_UCI, - model.UCI.table.c.state==uci_states.SNAPSHOT_UCI ) ) \ + model.UCI.table.c.state==uci_states.SNAPSHOT_UCI, + model.UCI.table.c.state==uci_states.ADD_STORAGE_UCI ) ) \ .all(): uci_wrapper = UCIwrapper( r, self.app ) new_requests.append( uci_wrapper ) @@ -195,6 +200,12 @@ self.uci_id = uci.id self.app = app self.sa_session = self.app.model.context + base_directory = os.path.join( self.app.config.job_working_directory, "cloud" ) + self.working_directory = os.path.join( base_directory, str( self.uci_id ) ) +# log.debug( "Cloud controller working directory for UCI DB ID '%s': '%s'" % ( self.uci_id, self.working_directory ) ) + if not os.path.exists( base_directory ): + os.mkdir( base_directory ) + # --------- Setter methods ----------------- @@ -364,11 +375,18 @@ self.sa_session.add( uci ) self.sa_session.flush() - def set_store_device( self, store_id, device ): +# def set_store_device( self, store_id, device ): +# uci = self.sa_session.query( model.UCI ).get( self.uci_id ) +# self.sa_session.refresh( uci ) +# uci.store[store_id].device = device +# uci.store[store_id].flush() + + def set_uci_total_size( self, total_size ): uci = self.sa_session.query( model.UCI ).get( self.uci_id ) self.sa_session.refresh( uci ) - uci.store[store_id].device = device - uci.store[store_id].flush() + uci.total_size = total_size + self.sa_session.add( uci ) + self.sa_session.flush() def set_store_error( self, error, store_index=None, store_id=None ): if store_index != None: @@ -405,15 +423,24 @@ def set_store_volume_id( self, store_index, volume_id ): """ - Given store index associated with this UCI in local database, set volume ID as it is registered + Given store index as it is stored in local database, set volume ID as it is registered on the cloud provider (e.g., vol-39890501) """ - uci = self.sa_session.query( model.UCI ).get( self.uci_id ) - self.sa_session.refresh( uci ) - uci.store[store_index].volume_id = volume_id - #uci.store[store_index].flush() - self.sa_session.add( uci ) - self.sa_session.flush() + + if store_index != None: + store = self.sa_session.query( model.CloudStore ).get( store_index ) + store.volume_id = volume_id + self.sa_session.add( store ) + self.sa_session.flush() + else: + return None + +# uci = self.sa_session.query( model.UCI ).get( self.uci_id ) +# self.sa_session.refresh( uci ) +# uci.store[store_index].volume_id = volume_id +# #uci.store[store_index].flush() +# self.sa_session.add( uci ) +# self.sa_session.flush() def set_store_instance( self, vol_id, instance_id ): """ @@ -421,7 +448,29 @@ be given in following format: 'vol-78943248' """ vol = self.sa_session.query( model.CloudStore ).filter( model.CloudStore.table.c.volume_id == vol_id ).first() - vol.inst.instance_id = instance_id + inst = self.sa_session.query( model.CloudInstance ).filter_by( instance_id=instance_id ).first() + vol.inst = inst + self.sa_session.add( vol ) + self.sa_session.flush() + + def set_store_device( self, vol_id, device ): + """ + Stores instance ID that given store volume is attached to. Store volume ID should + be given in following format: 'vol-78943248' + """ + vol = self.sa_session.query( model.CloudStore ).filter( model.CloudStore.table.c.volume_id == vol_id ).first() + vol.device = str( device ) + self.sa_session.add( vol ) + self.sa_session.flush() + + def set_store_deleted( self, vol_id, status=None ): + """ + Set storage volume as deleted in local database. Optionally, set the volume status too. + """ + vol = self.sa_session.query( model.CloudStore ).filter( model.CloudStore.table.c.volume_id == vol_id ).first() + vol.deleted = True + if status != None: + vol.status = status self.sa_session.add( vol ) self.sa_session.flush() @@ -503,6 +552,12 @@ self.sa_session.refresh( uci ) return uci.instance[instance_id].state + def get_instaces_in_state( self, state ): + """ Get database objects of all instances associated with this UCI in given state. """ + return self.sa_session.query( model.CloudInstance ) \ + .filter_by( uci_id=self.uci_id, state = state ) \ + .all() + def get_instances_ids( self ): """ Returns list IDs of all instances' associated with this UCI that are not in 'terminated' or @@ -590,6 +645,11 @@ self.sa_session.refresh( uci ) return uci.store[0].availability_zone + def get_uci_total_size( self ): + uci = self.sa_session.query( model.UCI ).get( self.uci_id ) + self.sa_session.refresh( uci ) + return uci.total_size + def get_store_size( self, store_id=0 ): uci = self.sa_session.query( model.UCI ).get( self.uci_id ) self.sa_session.refresh( uci ) @@ -604,9 +664,18 @@ self.sa_session.refresh( uci ) return uci.store[store_id].volume_id + def get_all_stores_in_status( self, status ): + """ + Return database objects of all stores associated with this UCI that have their + status set to value passed as parameter. + """ + return self.sa_session.query( model.CloudStore ).filter_by( deleted=False, uci_id=self.uci_id, status=status ).all() + def get_all_stores( self ): - """ Returns all storage volumes' database objects associated with this UCI. """ - return self.sa_session.query( model.CloudStore ).filter( model.CloudStore.table.c.uci_id == self.uci_id ).all() + """ Returns all storage volumes' database objects associated with this UCI that have not been marked as 'deleted'. """ + return self.sa_session.query( model.CloudStore ) \ + .filter_by( deleted=False, uci_id=self.uci_id ) \ + .all() def get_snapshots( self, status=None ): """ Returns database objects for all snapshots associated with this UCI and in given status.""" @@ -618,6 +687,9 @@ self.sa_session.refresh( uci ) return uci + def get_uci_working_directory( self ): + return self.working_directory + def uci_launch_time_set( self ): uci = self.sa_session.query( model.UCI ).get( self.uci_id ) self.sa_session.refresh( uci ) diff -r f4654abcec1e -r 1d93eb88c93f lib/galaxy/cloud/providers/ec2.py --- a/lib/galaxy/cloud/providers/ec2.py Fri Nov 20 11:03:41 2009 -0500 +++ b/lib/galaxy/cloud/providers/ec2.py Mon Nov 30 16:36:38 2009 -0500 @@ -1,4 +1,4 @@ -import subprocess, threading, os, errno, time, datetime +import subprocess, threading, os, errno, time, datetime, stat from Queue import Queue, Empty from datetime import datetime @@ -30,6 +30,8 @@ SUBMITTED = "submitted", SHUTTING_DOWN_UCI = "shutting-downUCI", SHUTTING_DOWN = "shutting-down", + ADD_STORAGE_UCI = "add-storageUCI", + ADD_STORAGE = "add-storage", AVAILABLE = "available", RUNNING = "running", PENDING = "pending", @@ -43,6 +45,7 @@ TERMINATED = "terminated", SUBMITTED = "submitted", RUNNING = "running", + ADDING = "adding-storage", PENDING = "pending", SHUTTING_DOWN = "shutting-down", ERROR = "error" @@ -51,6 +54,7 @@ store_status = Bunch( WAITING = "waiting", IN_USE = "in-use", + ADDING = "adding", CREATING = "creating", DELETED = 'deleted', ERROR = "error" @@ -122,6 +126,9 @@ self.stop_uci( uci_wrapper ) elif uci_state==uci_states.SNAPSHOT: self.snapshot_uci( uci_wrapper ) + elif uci_state==uci_states.ADD_STORAGE: + self.add_storage_to_uci( uci_wrapper ) + #self.dummy_start_uci( uci_wrapper ) except: log.exception( "Uncaught exception executing cloud request." ) cnt += 1 @@ -241,25 +248,14 @@ log.info( "Availability zone for UCI (i.e., storage volume) was not selected, using default zone: %s" % self.zone ) uci_wrapper.set_store_availability_zone( self.zone ) - log.info( "Creating volume in zone '%s'..." % uci_wrapper.get_uci_availability_zone() ) + store = uci_wrapper.get_all_stores_in_status( store_status.ADDING )[0] # Because at UCI creation time only 1 storage volume can be created, reference it directly + + log.info( "Creating storage volume in zone '%s' of size '%s'..." % ( uci_wrapper.get_uci_availability_zone(), store.size ) ) # Because only 1 storage volume may be created at UCI config time, index of this storage volume in local Galaxy DB w.r.t # current UCI is 0, so reference it in following methods - vol = conn.create_volume( uci_wrapper.get_store_size( 0 ), uci_wrapper.get_uci_availability_zone(), snapshot=None ) - uci_wrapper.set_store_volume_id( 0, vol.id ) - - # Wait for a while to ensure volume was created -# vol_status = vol.status -# for i in range( 30 ): -# if vol_status is not "available": -# log.debug( 'Updating volume status; current status: %s' % vol_status ) -# vol_status = vol.status -# time.sleep(3) -# if i is 29: -# log.debug( "Error while creating volume '%s'; stuck in state '%s'; deleting volume." % ( vol.id, vol_status ) ) -# conn.delete_volume( vol.id ) -# uci_wrapper.change_state( uci_state='error' ) -# return - + vol = conn.create_volume( store.size, uci_wrapper.get_uci_availability_zone(), snapshot=None ) + uci_wrapper.set_store_volume_id( store.id, vol.id ) + # Retrieve created volume again to get updated status try: vl = conn.get_all_volumes( [vol.id] ) @@ -306,6 +302,7 @@ if conn.delete_volume( v.volume_id ): deletedList.append( v.volume_id ) v.deleted = True + v.status = store_status.DELETED self.sa_session.add( v ) self.sa_session.flush() count += 1 @@ -358,15 +355,180 @@ uci_wrapper.change_state( uci_state=uci_states.AVAILABLE ) - def add_storage_to_uci( self, name ): - """ Adds more storage to specified UCI - TODO""" - + def add_storage_to_uci( self, uci_wrapper ): + """ + Add an additional storage volume to specified UCI by creating the storage volume + on cloud provider, attaching it to currently running instance and adding it to + 'galaxyData' zpool on remote instance. + """ + conn = self.get_connection( uci_wrapper ) + + stores = uci_wrapper.get_all_stores_in_status( store_status.ADDING ) + for store in stores: + vol_size = store.size + availability_zone = uci_wrapper.get_uci_availability_zone() + log.info( "Adding storage volume to UCI '%s' in zone '%s' of size '%s'..." % ( uci_wrapper.get_name(), availability_zone, vol_size ) ) + + try: + vol = conn.create_volume( vol_size, availability_zone, snapshot=None ) + uci_wrapper.set_store_volume_id( store.id, vol.id ) + uci_wrapper.set_store_availability_zone( availability_zone, vol.id ) + log.debug( "New storage volume created: '%s'" % vol.id ) + except boto.exception.EC2ResponseError, e: + err = "EC2 response error while creating storage volume: " + str( e ) + log.error( err ) + uci_wrapper.set_store_error( err, store_id=vol.id ) + uci_wrapper.set_error( err, True ) + return + except Exception, ex: + err = "Error while creating storage volume: " + str( ex ) + log.error( err ) + uci_wrapper.set_error( err, True ) + return + + # Retrieve created volume again to get updated status + try: + vl = conn.get_all_volumes( [vol.id] ) + except boto.exception.EC2ResponseError, e: + err = "EC2 response error while retrieving (i.e., updating status) of just created storage volume '" + vol.id + "': " + str( e ) + log.error( err ) + uci_wrapper.set_store_error( err, store_id=vol.id ) + uci_wrapper.set_error( err, True ) + return + except Exception, ex: + err = "Error while retrieving (i.e., updating status) of just created storage volume '" + vol.id + "': " + str( ex ) + log.error( err ) + uci_wrapper.set_error( err, True ) + return + + # Wait for a while to ensure volume was created + if len( vl ) > 0: + vol_status = vl[0].status # Bc. only single vol is queried, reference it as 0th list element + for i in range( 30 ): + if vol_status != "available": + log.debug( "(%s) Updating volume status; current status: '%s'" % (i, vol_status ) ) + uci_wrapper.change_state( uci_state=vol_status ) + time.sleep(5) + vol_status = vl[0].status + if vol_status == "available": + log.debug( "(%s) New volume status '%s', continuing with file system adjustment." % (i, vol_status ) ) + uci_wrapper.set_store_status( vl[0].id, vol_status ) + break + if i is 29: + err = "Error while creating volume '"+vl[0].id+"'; stuck in state '"+vol_status+"'; deleting volume." + conn.delete_volume( vl[0].id ) + log.error( err ) + uci_wrapper.set_error( err, True ) + uci_wrapper.set_store_error( err, store_id=vol.id ) + conn.delete_volume( vl[0].id ) + uci_wrapper.set_store_deleted( vl[0].id ) + return + else: + err = "Volume '" + vol.id +"' not found by EC2 after being created." + log.error( err ) + uci_wrapper.set_store_error( err, store_id=vol.id ) + uci_wrapper.set_error( err, True ) + return + + # Get private key for given instance + pk = uci_wrapper.get_key_pair_material() + if pk == None: #If pk does not exist, create it + self.check_key_pair( uci_wrapper, conn ) + pk = uci_wrapper.get_key_pair_material() + + # Get working directory for this UCI and store pk into a file + wd = uci_wrapper.get_uci_working_directory() + if not os.path.exists( wd ): + os.mkdir( wd ) + pk_file_path = os.path.join( wd, "pk" ) + + if pk != None: + # Save private key to a file + pk_file = open( pk_file_path, "w" ) + pk_file.write( pk ) + pk_file.close() + else: + err = "ERROR: Private key not available for this UCI." + log.error( err ) + uci_wrapper.set_store_error( err, store_id=vol.id ) + uci_wrapper.set_error( err, True ) + return + + if os.path.exists( pk_file_path ): + # Change permissions of the file - this is required by later used ssh + os.chmod( pk_file_path, stat.S_IRUSR | stat.S_IWUSR ) + + # Get # of storage volumes associated with this UCI to know as which device to connect new volume to the instance + device_num = len( uci_wrapper.get_all_stores_in_status( store_status.IN_USE ) ) + 5 # First device num is 5, so all subsequent ones should follow + + # Get instance that the new storage volume is to be attached to. Although a list is returned, + # only 1 instance can be in 'adding-storage' state (because, for now, only 1 instance is assoc. with + # each UCI) and volume can be attached to only to it + il = uci_wrapper.get_instaces_in_state( instance_states.ADDING ) + if len( il ) > 0: + # Attach new volume to the instance + log.debug( "Attaching new storage volume '%s' to UCI '%s' as device '%s'" % + ( vol.id, uci_wrapper.get_name(), device_num ) ) + try: + vol_status = conn.attach_volume( vol.id, il[0].instance_id, device_num ) + except boto.exception.EC2ResponseError, e: + err = "Attaching just created storage volume '" + vol.id + "'to instance '" + \ + il[0].instance_id + "' as device '" + str( device_num ) + "' failed: " + str( e ) + log.error( err ) + uci_wrapper.set_store_error( err, store_id=vol.id ) + uci_wrapper.set_error( err, True ) + return + # For a while, keep checking attachment status of the new volume + for i in range(30): + log.debug( "Checking attachment status of new volume '%s': '%s'" % ( vol.id, vol_status ) ) + if vol_status == 'attached': + uci_wrapper.set_store_status( vol.id, vol_status ) + uci_wrapper.set_store_device( vol.id, device_num ) + break + if i == 29: + err = "Storage volume '" + vol.id + "' failed to attach to instance '" + il[0].instance_id + \ + "'. Manual check needed." + log.error( err ) + uci_wrapper.set_store_error( err, store_id=vol.id ) + uci_wrapper.set_error( err, False ) + return + + time.sleep(4) + vol_list = conn.get_all_volumes( [vol.id] ) + for v in vol_list: + vol_status = v.attachment_state() + + # Once storage volume is attached, add it to the zpool by issuing system level command + cmd = 'ssh -o StrictHostKeyChecking=no -i '+ pk_file_path +' root@'+il[0].public_dns+' "zpool add galaxyData c7d' + str( device_num )+'"' + log.debug( "Adding new storage volume to zpool cmd: %s" % cmd ) + stdout = os.system( cmd ) + if stdout != 0: + err = "Adding newly created storage volume to zpool on instance '" + il[0].instance_id + \ + "' failed. Error code: " + str( stdout ) + log.error( err ) + uci_wrapper.set_store_error( err, store_id=vol.id ) + uci_wrapper.set_error( err, False ) + return + else: + err = "No instance(s) found in 'adding-storage' state. New disk not added to UCI's zpool." + log.error( err ) + uci_wrapper.set_store_error( err, store_id=vol.id ) + uci_wrapper.set_error( err, True ) + return + + # Update UCI's total storage size + uci_wrapper.set_uci_total_size( uci_wrapper.get_uci_total_size() + vol.size ) + # Reset UCI's and instance's state + uci_wrapper.change_state( uci_state=uci_states.RUNNING, instance_id=il[0].instance_id, i_state=instance_states.RUNNING ) + log.debug( "Successfully added storage volume '%s' to UCI '%s'." % ( vol.id, uci_wrapper.get_name() ) ) + def dummy_start_uci( self, uci_wrapper ): uci = uci_wrapper.get_uci() - log.debug( "Would be starting instance '%s'" % uci.name ) - uci_wrapper.change_state( uci_state.PENDING ) + log.debug( "Dummy start UCI '%s'" % uci.name ) + + +# uci_wrapper.change_state( uci_state.PENDING ) # log.debug( "Sleeping a bit... (%s)" % uci.name ) # time.sleep(20) # log.debug( "Woke up! (%s)" % uci.name ) @@ -418,10 +580,25 @@ if uci_wrapper.get_uci_state() != uci_states.ERROR: # Start an instance log.debug( "Starting instance for UCI '%s'" % uci_wrapper.get_name() ) - #TODO: Once multiple volumes can be attached to a single instance, update 'userdata' composition - userdata = uci_wrapper.get_store_volume_id()+"|"+uci_wrapper.get_access_key()+"|"+uci_wrapper.get_secret_key() + #TODO: Once multiple volumes can be attached to a single instance, update 'userdata' composition + # Compose user data; for storage volumes, separate multiple volumes with a colon (:) ensuring that + # the last volume in the list is not followed by a colon. + stores = uci_wrapper.get_all_stores() + volume_ids = "" + if len( stores ) > 0: + for i, store in enumerate( stores ): + volume_ids += store.volume_id + if i < len( stores )-1: + volume_ids += ":" + else: + err = "No storage volumes found that are associated with UCI '%s'" + uci_wrapper.get_name() + log.error( err ) + uci_wrapper.set_error( err, True ) + return + userdata = volume_ids+"|"+uci_wrapper.get_access_key()+"|"+uci_wrapper.get_secret_key() log.debug( "Using following command: conn.run_instances( image_id='%s', key_name='%s', security_groups=['%s'], user_data=[OMITTED], instance_type='%s', placement='%s' )" % ( mi_id, uci_wrapper.get_key_pair_name(), self.security_group, uci_wrapper.get_instance_type( i_index ), uci_wrapper.get_uci_availability_zone() ) ) + # Start an instance reservation = None try: reservation = conn.run_instances( image_id=mi_id, @@ -454,7 +631,10 @@ uci_wrapper.change_state( s, i_id, s ) uci_wrapper.set_security_group_name( self.security_group, i_id=i_id ) vol_id = uci_wrapper.get_store_volume_id( store_id=0 ) # TODO: Once more that one vol/UCI is allowed, update this! - uci_wrapper.set_store_status( vol_id, store_status.WAITING ) + # Following line is pointless bc. general update updates status of volume to 'available' + # before it actually connects to starting instance... This has been dealt w/ in general update method + #uci_wrapper.set_store_status( vol_id, store_status.WAITING ) + uci_wrapper.set_store_instance( vol_id, i_id ) log.debug( "Instance of UCI '%s' started, current state: '%s'" % ( uci_wrapper.get_name(), uci_wrapper.get_uci_state() ) ) except boto.exception.EC2ResponseError, e: err = "EC2 response error when retrieving instance information for UCI '" + uci_wrapper.get_name() + "': " + str( e ) @@ -571,19 +751,30 @@ .all() for inst in instances: if self.type == inst.uci.credentials.provider.type: - log.debug( "[%s] Running general status update on instance '%s'" % ( inst.uci.credentials.provider.type, inst.instance_id ) ) + log.debug( "[%s] Running general status update on instance '%s'" + % ( inst.uci.credentials.provider.type, inst.instance_id ) ) self.update_instance( inst ) + # Update storage volume(s) associated with current instance + stores = self.sa_session.query( model.CloudStore ) \ + .filter_by( uci_id=inst.uci_id, deleted=False ) \ + .all() + for store in stores: + if self.type == store.uci.credentials.provider.type: # and store.volume_id != None: + log.debug( "[%s] Running general status update on store with local database ID: '%s'" + % ( store.uci.credentials.provider.type, store.id ) ) + self.update_store( store ) # Update storage volume(s) stores = self.sa_session.query( model.CloudStore ) \ - .filter( or_( model.CloudStore.table.c.status==store_status.IN_USE, - model.CloudStore.table.c.status==store_status.CREATING, - model.CloudStore.table.c.status==store_status.WAITING, + .filter( or_( model.CloudStore.table.c.status==store_status.CREATING, +# model.CloudStore.table.c.status==store_status.IN_USE, +# model.CloudStore.table.c.status==store_status.WAITING, model.CloudStore.table.c.status==None ) ) \ .all() for store in stores: if self.type == store.uci.credentials.provider.type: # and store.volume_id != None: - log.debug( "[%s] Running general status update on store with local database ID: '%s'" % ( store.uci.credentials.provider.type, store.id ) ) + log.debug( "[%s] Running general status update on store with local database ID: '%s'" + % ( store.uci.credentials.provider.type, store.id ) ) self.update_store( store ) # else: # log.error( "[%s] There exists an entry for UCI (%s) storage volume without an ID. Storage volume might have been created with " @@ -668,7 +859,8 @@ for i, cInst in enumerate( r.instances ): try: s = cInst.update() - log.debug( "Checking state of cloud instance '%s' associated with UCI '%s' and reservation '%s'. State='%s'" % ( cInst, uci.name, r, s ) ) + log.debug( "Checking state of cloud instance '%s' associated with UCI '%s' " \ + "and reservation '%s'. State='%s'" % ( cInst, uci.name, r, s ) ) if s != inst.state: inst.state = s self.sa_session.add( inst ) @@ -693,7 +885,8 @@ self.sa_session.add( inst ) self.sa_session.flush() except boto.exception.EC2ResponseError, e: - err = "Updating instance status from cloud failed for UCI '"+ uci.name + "' during general status update: " + str( e ) + err = "Updating instance status from cloud failed for UCI '"+ uci.name + \ + "' during general status update: " + str( e ) log.error( err ) uci.error = err uci.state = uci_states.ERROR @@ -715,7 +908,7 @@ # Get reservations handle for given store try: - log.debug( "Updating storage volume command: vl = conn.get_all_volumes( [%s] )" % store.volume_id ) + log.debug( "Retrieving reference to storage volume '%s' during update..." % store.volume_id ) vl = conn.get_all_volumes( [store.volume_id] ) except boto.exception.EC2ResponseError, e: err = "Retrieving volume(s) from cloud failed for UCI '"+ uci.name + "' during general status update: " + str( e ) @@ -729,7 +922,7 @@ # Update store status in local DB with info from cloud provider if len(vl) > 0: try: - log.debug( "Storage volume '%s' current status: '%s'" % (store.volume_id, vl[0].status ) ) + log.debug( "General status update for storage volume '%s'; current status: '%s'" % (store.volume_id, vl[0].status ) ) if store.status != vl[0].status: # In case something failed during creation of UCI but actual storage volume was created and yet # UCI state remained as 'new', try to remedy this by updating UCI state here @@ -746,21 +939,26 @@ store.status = vl[0].status self.sa_session.add( store ) self.sa_session.flush() - if store.inst != None: - if store.inst.instance_id != vl[0].instance_id: - store.inst.instance_id = vl[0].instance_id - self.sa_session.add( store ) - self.sa_session.flush() - if store.attach_time != vl[0].attach_time: - store.attach_time = vl[0].attach_time - self.sa_session.add( store ) - self.sa_session.flush() - if store.device != vl[0].device: - store.device = vl[0].device - self.sa_session.add( store ) - self.sa_session.flush() + # Boto does not seem to be reporting these values although fields exist so comment them out... +# log.debug( "vl[0].instance_id: '%s'" % vl[0].instance_id ) +# if store.inst != None: +# if store.inst.instance_id != vl[0].instance_id: +# store.inst.instance_id = vl[0].instance_id +# self.sa_session.add( store ) +# self.sa_session.flush() +# log.debug( "vl[0].attach_time: '%s'" % vl[0].attach_time ) +# if store.attach_time != vl[0].attach_time: +# store.attach_time = vl[0].attach_time +# self.sa_session.add( store ) +# self.sa_session.flush() +## log.debug( "vl[0].device: '%s'" % vl[0].device ) +# if store.device != vl[0].device: +# store.device = vl[0].device +# self.sa_session.add( store ) +# self.sa_session.flush() except boto.exception.EC2ResponseError, e: - err = "Updating status of volume(s) from cloud failed for UCI '"+ uci.name + "' during general status update: " + str( e ) + err = "Updating status of volume(s) from cloud failed for UCI '"+ uci.name + \ + "' during general status update: " + str( e ) log.error( err ) uci.error = err uci.state = uci_states.ERROR @@ -768,7 +966,7 @@ self.sa_session.flush() return None else: - err = "No storage volumes returned by cloud provider on general update" + err = "No storage volumes returned by cloud provider on general update for volume with id: " + store.volume_id log.error( "%s for UCI '%s'" % ( err, uci.name ) ) store.status = store_status.ERROR store.error = err diff -r f4654abcec1e -r 1d93eb88c93f lib/galaxy/cloud/providers/eucalyptus.py --- a/lib/galaxy/cloud/providers/eucalyptus.py Fri Nov 20 11:03:41 2009 -0500 +++ b/lib/galaxy/cloud/providers/eucalyptus.py Mon Nov 30 16:36:38 2009 -0500 @@ -30,6 +30,8 @@ SUBMITTED = "submitted", SHUTTING_DOWN_UCI = "shutting-downUCI", SHUTTING_DOWN = "shutting-down", + ADD_STORAGE_UCI = "add-storageUCI", + ADD_STORAGE = "add-storage", AVAILABLE = "available", RUNNING = "running", PENDING = "pending", @@ -43,6 +45,7 @@ TERMINATED = "terminated", SUBMITTED = "submitted", RUNNING = "running", + ADDING = "adding-storage", PENDING = "pending", SHUTTING_DOWN = "shutting-down", ERROR = "error" @@ -51,6 +54,7 @@ store_status = Bunch( WAITING = "waiting", IN_USE = "in-use", + ADDING = "adding", CREATING = "creating", DELETED = 'deleted', ERROR = "error" @@ -121,6 +125,8 @@ self.stop_uci( uci_wrapper ) elif uci_state==uci_states.SNAPSHOT: self.snapshot_uci( uci_wrapper ) + elif uci_state==uci_states.ADD_STORAGE: + self.add_storage_to_uci( uci_wrapper ) except: log.exception( "Uncaught exception executing cloud request." ) cnt += 1 @@ -245,9 +251,16 @@ log.info( "Availability zone for UCI (i.e., storage volume) was not selected, using default zone: %s" % self.zone ) uci_wrapper.set_store_availability_zone( self.zone ) - log.debug( "Creating volume; using command: conn.create_volume( %s, '%s', snapshot=None )" % ( uci_wrapper.get_store_size( 0 ), uci_wrapper.get_uci_availability_zone() )) - vol = conn.create_volume( uci_wrapper.get_store_size( 0 ), uci_wrapper.get_uci_availability_zone(), snapshot=None ) - uci_wrapper.set_store_volume_id( 0, vol.id ) +# log.debug( "Creating volume; using command: conn.create_volume( %s, '%s', snapshot=None )" % ( uci_wrapper.get_store_size( 0 ), uci_wrapper.get_uci_availability_zone() )) +# vol = conn.create_volume( uci_wrapper.get_store_size( 0 ), uci_wrapper.get_uci_availability_zone(), snapshot=None ) +# uci_wrapper.set_store_volume_id( 0, vol.id ) + store = uci_wrapper.get_all_stores_in_status( store_status.ADDING )[0] # Because at UCI creation time only 1 storage volume can be created, reference it directly + + log.info( "Creating storage volume in zone '%s' of size '%s'..." % ( uci_wrapper.get_uci_availability_zone(), store.size ) ) + # Because only 1 storage volume may be created at UCI config time, index of this storage volume in local Galaxy DB w.r.t + # current UCI is 0, so reference it in following methods + vol = conn.create_volume( store.size, uci_wrapper.get_uci_availability_zone(), snapshot=None ) + uci_wrapper.set_store_volume_id( store.id, vol.id ) # Retrieve created volume again to get updated status try: @@ -364,6 +377,7 @@ def add_storage_to_uci( self, uci_wrapper ): """ Adds more storage to specified UCI """ + uci_wrapper.set_error( "Adding storage to eucalyptus-based clouds is not yet supported.", True ) def dummy_start_uci( self, uci_wrapper ): diff -r f4654abcec1e -r 1d93eb88c93f lib/galaxy/web/controllers/cloud.py --- a/lib/galaxy/web/controllers/cloud.py Fri Nov 20 11:03:41 2009 -0500 +++ b/lib/galaxy/web/controllers/cloud.py Mon Nov 30 16:36:38 2009 -0500 @@ -41,6 +41,8 @@ SUBMITTED = "submitted", SHUTTING_DOWN_UCI = "shutting-downUCI", SHUTTING_DOWN = "shutting-down", + ADD_STORAGE_UCI = "add-storageUCI", + ADD_STORAGE = "add-storage", AVAILABLE = "available", RUNNING = "running", PENDING = "pending", @@ -54,13 +56,16 @@ TERMINATED = "terminated", SUBMITTED = "submitted", RUNNING = "running", + ADDING = "adding-storage", PENDING = "pending", SHUTTING_DOWN = "shutting-down", ERROR = "error" ) store_status = Bunch( + WAITING = "waiting", IN_USE = "in-use", + ADDING = "adding", CREATING = "creating", DELETED = 'deleted', ERROR = "error" @@ -108,7 +113,9 @@ model.UCI.table.c.state==uci_states.SUBMITTED, model.UCI.table.c.state==uci_states.SUBMITTED_UCI, model.UCI.table.c.state==uci_states.SHUTTING_DOWN, - model.UCI.table.c.state==uci_states.SHUTTING_DOWN_UCI ) ) \ + model.UCI.table.c.state==uci_states.SHUTTING_DOWN_UCI, + model.UCI.table.c.state==uci_states.ADD_STORAGE, + model.UCI.table.c.state==uci_states.ADD_STORAGE_UCI ) ) \ .order_by( desc( model.UCI.table.c.update_time ) ) \ .all() @@ -201,7 +208,8 @@ storage.user = user storage.uci = uci storage.size = volSize - storage.availability_zone = zone + storage.availability_zone = zone + storage.status = store_status.ADDING # Persist session = trans.sa_session session.add( uci ) @@ -306,6 +314,8 @@ ( uci.state != uci_states.ERROR ) and \ ( uci.state != uci_states.SHUTTING_DOWN_UCI ) and \ ( uci.state != uci_states.SHUTTING_DOWN ) and \ + ( uci.state != uci_states.ADD_STORAGE_UCI ) and \ + ( uci.state != uci_states.ADD_STORAGE ) and \ ( uci.state != uci_states.AVAILABLE ): uci.state = uci_states.SHUTTING_DOWN_UCI session = trans.sa_session @@ -496,13 +506,53 @@ @web.expose @web.require_login( "add instance storage" ) - def add_storage( self, trans, id ): - instance = get_uci( trans, id ) + def add_storage( self, trans, id, vol_size=None ): + error = None + uci = get_uci( trans, id ) + stores = get_stores_in_status( trans, uci, store_status.IN_USE ) + # Start adding of storage making sure given UCI is running and that at least one + # storage volume is attached to it (this is needed to by cloud controller to know + # as which device to attach the new storage volume) + if uci.state == uci_states.RUNNING and len( stores ) > 0: + if vol_size is not None: + try: + vol_size = int( vol_size ) + except ValueError: + error = "Volume size must be integer value between 1 and 1000." + + if not error: + user = trans.get_user() + + storage = model.CloudStore() + storage.user = user + storage.uci = uci + storage.size = vol_size + storage.status = store_status.ADDING + + # Set state of instance - NOTE that this code will only work (with code in cloud controller) + # for scenario where a UCI is associated with *1* compute instance!!! + instances = get_instances( trans, uci ) + instances.state = instance_states.ADDING + + uci.state = uci_states.ADD_STORAGE_UCI + # Persist + session = trans.sa_session + session.add( instances ) + session.add( storage ) + session.add( uci ) + session.flush() + # Log and display the management page + trans.log_event( "User added storage volume to UCI: '%s'" % uci.name ) + trans.set_message( "Adding of storage to instance '%s' initiated." % uci.name ) + return self.list( trans ) + else: + error( "Storage can only be added to instances that are in state 'RUNNING' with existing " \ + "storage volume(s) already attached." ) - error( "Adding storage to instance '%s' is not supported yet." % instance.name ) - - return self.list( trans ) + return trans.show_form( + web.FormBuilder( url_for( id=trans.security.encode_id(uci.id) ), "Add storage to an instance", submit_text="Add" ) + .add_text( "vol_size", "Storage size (1-1000 GB)", value='', error=error ) ) # ----- Image methods ----- @web.expose @@ -1034,6 +1084,7 @@ dict = {} dict['id'] = uci.id dict['state'] = uci.state + dict['total_size'] = uci.total_size if uci.error != None: dict['error'] = str( uci.error ) else: @@ -1136,19 +1187,30 @@ def get_stores( trans, uci ): """ - Get stores objects that are connected to uci object + Get stores objects that are associated with given uci and are not in 'error' status """ user = trans.get_user() stores = trans.sa_session.query( model.CloudStore ) \ - .filter_by( user=user, uci_id=uci.id ) \ + .filter_by( user=user, uci_id=uci.id, deleted=False ) \ .filter( model.CloudStore.table.c.status != store_status.ERROR ) \ .all() return stores +def get_stores_in_status( trans, uci, status ): + """ + Get stores objects that are associated with given uci and are not have given status + """ + user = trans.get_user() + stores = trans.sa_session.query( model.CloudStore ) \ + .filter_by( user=user, uci_id=uci.id, status=status ) \ + .all() + + return stores + def get_instances( trans, uci ): """ - Get objects of instances that are pending or running and are connected to uci object + Get objects of instances that are pending or running and are connected to the given uci object """ user = trans.get_user() instances = trans.sa_session.query( model.CloudInstance ) \ @@ -1159,6 +1221,17 @@ return instances +def get_instances_in_state( trans, uci, state ): + """ + Get objects of instances that are in specified state and are connected to the given uci object + """ + user = trans.get_user() + instances = trans.sa_session.query( model.CloudInstance ) \ + .filter_by( user=user, uci_id=uci.id, state=state ) \ + .all() + + return instances + def get_connection( trans, creds ): """ Establishes cloud connection using user's credentials diff -r f4654abcec1e -r 1d93eb88c93f templates/cloud/configure_cloud.mako --- a/templates/cloud/configure_cloud.mako Fri Nov 20 11:03:41 2009 -0500 +++ b/templates/cloud/configure_cloud.mako Mon Nov 30 16:36:38 2009 -0500 @@ -79,7 +79,8 @@ }); } - // Update 'state' and 'time alive' fields + // Update 'size', 'state' and 'time alive' fields + $(elem + "-size").text( data[i].total_size ); $(elem + "-state").text( data[i].state ); if ( new_state != 'error' ) { // Because 'error' state is handled as a JS link, don't include it in update $(elem + "-state-p").text( data[i].state ); @@ -219,7 +220,7 @@ ${liveInstance.name} (${liveInstance.credentials.name}) <a id="li-${i}-popup" class="popup-arrow" style="display: none;">▼</a> </td> - <td>${str(liveInstance.total_size)}</td> + <td id="${ liveInstance.id }-size">${str(liveInstance.total_size)}</td> <td id="${ liveInstance.id }-state">${str(liveInstance.state)}</td> <td id="${ liveInstance.id }-launch_time"> ##${str(liveInstance.launch_time)[:16]} @@ -251,7 +252,8 @@ <a class="action-button" confirm="Are you sure you want to stop instance '${liveInstance.name}'?" href="${h.url_for( action='stop', id=trans.security.encode_id(liveInstance.id) )}">Stop</a> <a class="action-button" href="${h.url_for( action='rename_uci', id=trans.security.encode_id(liveInstance.id) )}">Rename</a> <a class="action-button" href="${h.url_for( action='view_uci_details', id=trans.security.encode_id(liveInstance.id) )}">View details</a> - <a class="action-button" href="${h.url_for( action='uci_usage_report', id=trans.security.encode_id(liveInstance.id) )}">Usage report</a> + <a class="action-button" href="${h.url_for( action='add_storage', id=trans.security.encode_id(liveInstance.id) )}">Add storage</a> + <a class="action-button" href="${h.url_for( action='uci_usage_report', id=trans.security.encode_id(liveInstance.id) )}">Usage report</a> </div> </td> </tr> @@ -315,14 +317,13 @@ </td> <td> <div popupmenu="pi-${i}-popup"> - <a class="action-button" href="${h.url_for( action='start', id=trans.security.encode_id(prevInstance.id), type='m1.small' )}"> Start m1.small</a> + <a class="action-button" href="${h.url_for( action='uci_usage_report', id=trans.security.encode_id(prevInstance.id) )}">Usage report</a> + <a class="action-button" href="${h.url_for( action='rename_uci', id=trans.security.encode_id(prevInstance.id) )}">Rename</a> + <a class="action-button" href="${h.url_for( action='start', id=trans.security.encode_id(prevInstance.id), type='m1.small' )}"> Start m1.small</a> <a class="action-button" href="${h.url_for( action='start', id=trans.security.encode_id(prevInstance.id), type='c1.medium' )}"> Start c1.medium</a> - <a class="action-button" href="${h.url_for( action='rename_uci', id=trans.security.encode_id(prevInstance.id) )}">Rename</a> - <a class="action-button" href="${h.url_for( action='uci_usage_report', id=trans.security.encode_id(prevInstance.id) )}">Usage report</a> - <a class="action-button" href="${h.url_for( action='create_snapshot', id=trans.security.encode_id(prevInstance.id) )}">Create snapshot</a> + <a class="action-button" href="${h.url_for( action='create_snapshot', id=trans.security.encode_id(prevInstance.id) )}">Create snapshot</a> <a class="action-button" href="${h.url_for( action='view_snapshots', id=trans.security.encode_id(prevInstance.id) )}">View snapshots</a> - <a class="action-button" href="${h.url_for( action='add_storage', id=trans.security.encode_id(prevInstance.id) )}" target="_parent">Add storage</a> - <a class="action-button" confirm="Are you sure you want to delete instance '${prevInstance.name}'? This will delete all of your data assocaiated with this instance!" href="${h.url_for( action='delete_uci', id=trans.security.encode_id(prevInstance.id) )}">Delete</a> + <a class="action-button" confirm="Are you sure you want to delete instance '${prevInstance.name}'? This will delete all of your data assocaiated with this instance!" href="${h.url_for( action='delete_uci', id=trans.security.encode_id(prevInstance.id) )}">Delete</a> </div> </td> </tr>

1 0

[hg] galaxy 3135: Adding back 'wiggle to interval' converter
by Greg Von Kuster 02 Dec '09

02 Dec '09

details: http://www.bx.psu.edu/hg/galaxy/rev/eb317734e391 changeset: 3135:eb317734e391 user: guru date: Tue Dec 01 11:49:07 2009 -0500 description: Adding back 'wiggle to interval' converter diffstat: datatypes_conf.xml.sample | 1 + lib/galaxy/datatypes/converters/wiggle_to_simple_converter.py | 44 +++++++ lib/galaxy/datatypes/converters/wiggle_to_simple_converter.xml | 11 + test-data/3.wig | 37 ++++++ test-data/3_wig.bed | 26 ++++ tool_conf.xml.main | 1 + tool_conf.xml.sample | 1 + tools/filters/wiggle_to_simple.py | 43 +++++++ tools/filters/wiggle_to_simple.xml | 88 ++++++++++++++ 9 files changed, 252 insertions(+), 0 deletions(-) diffs (306 lines): diff -r 6e742ffb16a5 -r eb317734e391 datatypes_conf.xml.sample --- a/datatypes_conf.xml.sample Tue Dec 01 11:07:20 2009 -0500 +++ b/datatypes_conf.xml.sample Tue Dec 01 11:49:07 2009 -0500 @@ -58,6 +58,7 @@ <datatype extension="txtseq.zip" type="galaxy.datatypes.data:Txtseq" mimetype="application/zip" display_in_upload="true"/> <datatype extension="wig" type="galaxy.datatypes.interval:Wiggle" display_in_upload="true"> <converter file="wiggle_to_array_tree_converter.xml" target_datatype="array_tree"/> + <converter file="wiggle_to_simple_converter.xml" target_datatype="interval"/> </datatype> <datatype extension="array_tree" type="galaxy.datatypes.data:Data" /> <datatype extension="interval_index" type="galaxy.datatypes.data:Data" /> diff -r 6e742ffb16a5 -r eb317734e391 lib/galaxy/datatypes/converters/wiggle_to_simple_converter.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/lib/galaxy/datatypes/converters/wiggle_to_simple_converter.py Tue Dec 01 11:49:07 2009 -0500 @@ -0,0 +1,44 @@ +#!/usr/bin/env python +#code is same as ~/tools/stats/wiggle_to_simple.py + +""" +Read a wiggle track and print out a series of lines containing +"chrom position score". Ignores track lines, handles bed, variableStep +and fixedStep wiggle lines. +""" +import sys +from galaxy import eggs +import pkg_resources; pkg_resources.require( "bx-python" ) +import bx.wiggle +from galaxy.tools.exception_handling import * + +def stop_err( msg ): + sys.stderr.write( msg ) + sys.exit() + +def main(): + if len( sys.argv ) > 1: + in_file = open( sys.argv[1] ) + else: + in_file = open( sys.stdin ) + + if len( sys.argv ) > 2: + out_file = open( sys.argv[2], "w" ) + else: + out_file = sys.stdout + + try: + for fields in bx.wiggle.IntervalReader( UCSCOutWrapper( in_file ) ): + out_file.write( "%s\n" % "\t".join( map( str, fields ) ) ) + except UCSCLimitException: + # Wiggle data was truncated, at the very least need to warn the user. + print 'Encountered message from UCSC: "Reached output limit of 100000 data values", so be aware your data was truncated.' + except ValueError, e: + in_file.close() + out_file.close() + stop_err( str( e ) ) + + in_file.close() + out_file.close() + +if __name__ == "__main__": main() diff -r 6e742ffb16a5 -r eb317734e391 lib/galaxy/datatypes/converters/wiggle_to_simple_converter.xml --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/lib/galaxy/datatypes/converters/wiggle_to_simple_converter.xml Tue Dec 01 11:49:07 2009 -0500 @@ -0,0 +1,11 @@ +<tool id="CONVERTER_wiggle_to_interval_0" name="Wiggle to Interval"> +  +  + <command interpreter="python">wiggle_to_simple_converter.py $input $out_file1 </command> + <inputs> + <param format="wig" name="input" type="data" label="Convert"/> + </inputs> + <outputs> + <data format="interval" name="out_file1" /> + </outputs> +</tool> diff -r 6e742ffb16a5 -r eb317734e391 test-data/3.wig --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/test-data/3.wig Tue Dec 01 11:49:07 2009 -0500 @@ -0,0 +1,37 @@ +track type=wiggle_0 name="LaminB1 (Tig3)" description="NKI LaminB1 DamID Map (log2-ratio scores, Tig3 cells)" +# output date: 2009-07-22 15:55:44 UTC +# chrom specified: chr20 +# position specified: 1-62435964 +# data values >= 4 +# This data has been compressed with a minor loss in resolution. +# (Worst case: 0.0448672) The original source data +# (before querying and compression) is available at +# http://hgdownload.cse.ucsc.edu/downloads.html +variableStep chrom=chr20 span=60 +1953668 4.029 +9749454 4.033 +12219969 4.053 +12571904 4.548 +12663417 4.207 +15198014 4.041 +16613945 4.526 +16994767 4.08 +19456653 4.014 +37706076 4.293 +38010695 4.14 +38418792 4.133 +38595669 4.032 +40038834 4.144 +50693133 4.206 +51294426 4 +52890600 4.12 +57414708 4.026 +57623989 4.031 +58977191 5.678 +59059925 4.325 +variableStep chrom=chr12 span=60 +1953668 -4.029 +9749454 -4.033 +12219969 -4.053 +12571904 -4.548 +12663417 -4.207 diff -r 6e742ffb16a5 -r eb317734e391 test-data/3_wig.bed --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/test-data/3_wig.bed Tue Dec 01 11:49:07 2009 -0500 @@ -0,0 +1,26 @@ +chr20 1953667 1953727 + 4.029 +chr20 9749453 9749513 + 4.033 +chr20 12219968 12220028 + 4.053 +chr20 12571903 12571963 + 4.548 +chr20 12663416 12663476 + 4.207 +chr20 15198013 15198073 + 4.041 +chr20 16613944 16614004 + 4.526 +chr20 16994766 16994826 + 4.08 +chr20 19456652 19456712 + 4.014 +chr20 37706075 37706135 + 4.293 +chr20 38010694 38010754 + 4.14 +chr20 38418791 38418851 + 4.133 +chr20 38595668 38595728 + 4.032 +chr20 40038833 40038893 + 4.144 +chr20 50693132 50693192 + 4.206 +chr20 51294425 51294485 + 4.0 +chr20 52890599 52890659 + 4.12 +chr20 57414707 57414767 + 4.026 +chr20 57623988 57624048 + 4.031 +chr20 58977190 58977250 + 5.678 +chr20 59059924 59059984 + 4.325 +chr12 1953667 1953727 + -4.029 +chr12 9749453 9749513 + -4.033 +chr12 12219968 12220028 + -4.053 +chr12 12571903 12571963 + -4.548 +chr12 12663416 12663476 + -4.207 diff -r 6e742ffb16a5 -r eb317734e391 tool_conf.xml.main --- a/tool_conf.xml.main Tue Dec 01 11:07:20 2009 -0500 +++ b/tool_conf.xml.main Tue Dec 01 11:49:07 2009 -0500 @@ -47,6 +47,7 @@ <tool file="maf/maf_to_fasta.xml" /> <tool file="fasta_tools/tabular_to_fasta.xml" /> <tool file="fastx_toolkit/fastq_to_fasta.xml" /> + <tool file="filters/wiggle_to_simple.xml" /> </section> <section name="FASTA manipulation" id="fasta_manipulation"> <tool file="fasta_tools/fasta_compute_length.xml" /> diff -r 6e742ffb16a5 -r eb317734e391 tool_conf.xml.sample --- a/tool_conf.xml.sample Tue Dec 01 11:07:20 2009 -0500 +++ b/tool_conf.xml.sample Tue Dec 01 11:49:07 2009 -0500 @@ -71,6 +71,7 @@ <tool file="maf/maf_to_fasta.xml" /> <tool file="fasta_tools/tabular_to_fasta.xml" /> <tool file="fastx_toolkit/fastq_to_fasta.xml" /> + <tool file="filters/wiggle_to_simple.xml" /> </section> <section name="Extract Features" id="features"> <tool file="filters/ucsc_gene_bed_to_exon_bed.xml" /> diff -r 6e742ffb16a5 -r eb317734e391 tools/filters/wiggle_to_simple.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/tools/filters/wiggle_to_simple.py Tue Dec 01 11:49:07 2009 -0500 @@ -0,0 +1,43 @@ +#!/usr/bin/env python + +""" +Read a wiggle track and print out a series of lines containing +"chrom position score". Ignores track lines, handles bed, variableStep +and fixedStep wiggle lines. +""" +import sys +from galaxy import eggs +import pkg_resources; pkg_resources.require( "bx-python" ) +import bx.wiggle +from galaxy.tools.exception_handling import * + +def stop_err( msg ): + sys.stderr.write( msg ) + sys.exit() + +def main(): + if len( sys.argv ) > 1: + in_file = open( sys.argv[1] ) + else: + in_file = open( sys.stdin ) + + if len( sys.argv ) > 2: + out_file = open( sys.argv[2], "w" ) + else: + out_file = sys.stdout + + try: + for fields in bx.wiggle.IntervalReader( UCSCOutWrapper( in_file ) ): + out_file.write( "%s\n" % "\t".join( map( str, fields ) ) ) + except UCSCLimitException: + # Wiggle data was truncated, at the very least need to warn the user. + print 'Encountered message from UCSC: "Reached output limit of 100000 data values", so be aware your data was truncated.' + except ValueError, e: + in_file.close() + out_file.close() + stop_err( str( e ) ) + + in_file.close() + out_file.close() + +if __name__ == "__main__": main() diff -r 6e742ffb16a5 -r eb317734e391 tools/filters/wiggle_to_simple.xml --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/tools/filters/wiggle_to_simple.xml Tue Dec 01 11:49:07 2009 -0500 @@ -0,0 +1,88 @@ +<tool id="wiggle2simple1" name="Wiggle-to-Interval"> + <description>converter</description> + <command interpreter="python">wiggle_to_simple.py $input $out_file1 </command> + <inputs> + <param format="wig" name="input" type="data" label="Convert"/> + </inputs> + <outputs> + <data format="interval" name="out_file1" /> + </outputs> + <tests> + <test> + <param name="input" value="2.wig" /> + <output name="out_file1" file="2.interval"/> + </test> + <test> + <param name="input" value="3.wig" /> + <output name="out_file1" file="3_wig.bed"/> + </test> + </tests> + <help> +**Syntax** + +This tool converts wiggle data into interval type. + +- **Wiggle format**: The .wig format is line-oriented. Wiggle data is preceded by a UCSC track definition line. Following the track definition line is the track data, which can be entered in three different formats described below. + + - **BED format** with no declaration line and four columns of data:: + + chromA chromStartA chromEndA dataValueA + chromB chromStartB chromEndB dataValueB + + - **variableStep** two column data; started by a declaration line and followed with chromosome positions and data values:: + + variableStep chrom=chrN [span=windowSize] + chromStartA dataValueA + chromStartB dataValueB + + - **fixedStep** single column data; started by a declaration line and followed with data values:: + + fixedStep chrom=chrN start=position step=stepInterval [span=windowSize] + dataValue1 + dataValue2 + +----- + +**Example** + +- input wiggle format file:: + + #track type=wiggle_0 name="Bed Format" description="BED format" + chr19 59302000 59302300 -1.0 + chr19 59302300 59302600 -0.75 + chr19 59302600 59302900 -0.50 + chr19 59302900 59303200 -0.25 + chr19 59303200 59303500 0.0 + #track type=wiggle_0 name="variableStep" description="variableStep format" + variableStep chrom=chr19 span=150 + 59304701 10.0 + 59304901 12.5 + 59305401 15.0 + 59305601 17.5 + #track type=wiggle_0 name="fixedStep" description="fixed step" visibility=full + fixedStep chrom=chr19 start=59307401 step=300 span=200 + 1000 + 900 + 800 + 700 + 600 + +- convert the above file to interval file:: + + chr19 59302000 59302300 + -1.0 + chr19 59302300 59302600 + -0.75 + chr19 59302600 59302900 + -0.5 + chr19 59302900 59303200 + -0.25 + chr19 59303200 59303500 + 0.0 + chr19 59304701 59304851 + 10.0 + chr19 59304901 59305051 + 12.5 + chr19 59305401 59305551 + 15.0 + chr19 59305601 59305751 + 17.5 + chr19 59307701 59307901 + 1000.0 + chr19 59308001 59308201 + 900.0 + chr19 59308301 59308501 + 800.0 + chr19 59308601 59308801 + 700.0 + chr19 59308901 59309101 + 600.0 + +</help> +</tool>

1 0

[hg] galaxy 3130: Framework for recording user actions for repor...
by Greg Von Kuster 02 Dec '09

02 Dec '09

details: http://www.bx.psu.edu/hg/galaxy/rev/f98643c26eb7 changeset: 3130:f98643c26eb7 user: jeremy goecks <jeremy.goecks(a)emory.edu> date: Mon Nov 30 17:53:25 2009 -0500 description: Framework for recording user actions for reports; framework is modelled after event recording. Currently two types of actions are recorded: (a) tagging/untagging and (b) filtering/searching in grids. diffstat: lib/galaxy/config.py | 1 + lib/galaxy/model/__init__.py | 10 +++++ lib/galaxy/model/mapping.py | 13 ++++++ lib/galaxy/model/migrate/versions/0029_user_actions.py | 47 +++++++++++++++++++++++ lib/galaxy/web/controllers/dataset.py | 2 +- lib/galaxy/web/controllers/history.py | 2 +- lib/galaxy/web/controllers/tag.py | 24 ++++++++--- lib/galaxy/web/framework/__init__.py | 17 ++++++++ lib/galaxy/web/framework/helpers/grids.py | 13 +++++- templates/dataset/edit_attributes.mako | 2 +- templates/history/view.mako | 2 +- templates/root/history.mako | 2 +- templates/tagging_common.mako | 8 ++-- universe_wsgi.ini.sample | 3 + 14 files changed, 128 insertions(+), 18 deletions(-) diffs (335 lines): diff -r 9d4945bbdcf5 -r f98643c26eb7 lib/galaxy/config.py --- a/lib/galaxy/config.py Mon Nov 30 15:33:45 2009 -0500 +++ b/lib/galaxy/config.py Mon Nov 30 17:53:25 2009 -0500 @@ -79,6 +79,7 @@ self.use_heartbeat = string_as_bool( kwargs.get( 'use_heartbeat', 'False' ) ) self.use_memdump = string_as_bool( kwargs.get( 'use_memdump', 'False' ) ) self.log_memory_usage = string_as_bool( kwargs.get( 'log_memory_usage', 'False' ) ) + self.log_actions = string_as_bool( kwargs.get( 'log_actions', 'False' ) ) self.log_events = string_as_bool( kwargs.get( 'log_events', 'False' ) ) self.ucsc_display_sites = kwargs.get( 'ucsc_display_sites', "main,test,archaea,ucla" ).lower().split(",") self.gbrowse_display_sites = kwargs.get( 'gbrowse_display_sites', "main,test,tair" ).lower().split(",") diff -r 9d4945bbdcf5 -r f98643c26eb7 lib/galaxy/model/__init__.py --- a/lib/galaxy/model/__init__.py Mon Nov 30 15:33:45 2009 -0500 +++ b/lib/galaxy/model/__init__.py Mon Nov 30 17:53:25 2009 -0500 @@ -1441,6 +1441,16 @@ def __init__( self, name=None, value=None ): self.name = name self.value = value + +class UserAction( object ): + def __init__( self, id=None, create_time=None, user_id=None, session_id=None, action=None, params=None, context=None): + self.id = id + self.create_time = create_time + self.user_id = user_id + self.session_id = session_id + self.action = action + self.params = params + self.context = context ## ---- Utility methods ------------------------------------------------------- diff -r 9d4945bbdcf5 -r f98643c26eb7 lib/galaxy/model/mapping.py --- a/lib/galaxy/model/mapping.py Mon Nov 30 15:33:45 2009 -0500 +++ b/lib/galaxy/model/mapping.py Mon Nov 30 17:53:25 2009 -0500 @@ -743,6 +743,15 @@ Column( "name", Unicode( 255 ), index=True), Column( "value", Unicode( 1024 ) ) ) +UserAction.table = Table( "user_action", metadata, + Column( "id", Integer, primary_key=True ), + Column( "create_time", DateTime, default=now ), + Column( "user_id", Integer, ForeignKey( "galaxy_user.id" ), index=True ), + Column( "session_id", Integer, ForeignKey( "galaxy_session.id" ), index=True ), + Column( "action", Unicode( 255 ) ), + Column( "context", Unicode( 512 ) ), + Column( "params", Unicode( 1024 ) ) ) + # With the tables defined we can define the mappers and setup the # relationships between the model objects. @@ -1237,6 +1246,10 @@ assign_mapper( context, UserPreference, UserPreference.table, properties = {} ) + +assign_mapper( context, UserAction, UserAction.table, + properties = dict( user=relation( User.mapper ) ) + ) def db_next_hid( self ): """ diff -r 9d4945bbdcf5 -r f98643c26eb7 lib/galaxy/model/migrate/versions/0029_user_actions.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/lib/galaxy/model/migrate/versions/0029_user_actions.py Mon Nov 30 17:53:25 2009 -0500 @@ -0,0 +1,47 @@ +""" +This migration script adds a user actions table to Galaxy. +""" + +from sqlalchemy import * +from migrate import * + +import datetime +now = datetime.datetime.utcnow + +import logging +log = logging.getLogger( __name__ ) + +metadata = MetaData( migrate_engine ) + +def display_migration_details(): + print "" + print "This migration script adds a user actions table to Galaxy." + print "" + + +# New table to store user actions. +UserAction_table = Table( "user_action", metadata, + Column( "id", Integer, primary_key=True ), + Column( "create_time", DateTime, default=now ), + Column( "user_id", Integer, ForeignKey( "galaxy_user.id" ), index=True ), + Column( "session_id", Integer, ForeignKey( "galaxy_session.id" ), index=True ), + Column( "action", Unicode( 255 ) ), + Column( "context", Unicode( 512 ) ), + Column( "params", Unicode( 1024 ) ) ) + +def upgrade(): + display_migration_details() + metadata.reflect() + try: + UserAction_table.create() + except Exception, e: + print str(e) + log.debug( "Creating user_action table failed: %s" % str( e ) ) + +def downgrade(): + metadata.reflect() + try: + UserAction_table.drop() + except Exception, e: + print str(e) + log.debug( "Dropping user_action table failed: %s" % str( e ) ) \ No newline at end of file diff -r 9d4945bbdcf5 -r f98643c26eb7 lib/galaxy/web/controllers/dataset.py --- a/lib/galaxy/web/controllers/dataset.py Mon Nov 30 15:33:45 2009 -0500 +++ b/lib/galaxy/web/controllers/dataset.py Mon Nov 30 17:53:25 2009 -0500 @@ -87,7 +87,7 @@ link=( lambda item: iff( item.history.deleted, None, dict( operation="switch", id=item.id ) ) ), filterable="advanced" ), HistoryColumn( "History", key="history", link=( lambda item: iff( item.history.deleted, None, dict( operation="switch_history", id=item.id ) ) ) ), - grids.TagsColumn( "Tags", "tags", model.HistoryDatasetAssociation, model.HistoryDatasetAssociationTagAssociation, filterable="advanced" ), + grids.TagsColumn( "Tags", "tags", model.HistoryDatasetAssociation, model.HistoryDatasetAssociationTagAssociation, filterable="advanced", grid_name="HistoryDatasetAssocationListGrid" ), StatusColumn( "Status", key="deleted", attach_popup=False ), grids.GridColumn( "Created", key="create_time", format=time_ago ), grids.GridColumn( "Last Updated", key="update_time", format=time_ago ), diff -r 9d4945bbdcf5 -r f98643c26eb7 lib/galaxy/web/controllers/history.py --- a/lib/galaxy/web/controllers/history.py Mon Nov 30 15:33:45 2009 -0500 +++ b/lib/galaxy/web/controllers/history.py Mon Nov 30 17:53:25 2009 -0500 @@ -94,7 +94,7 @@ link=( lambda history: iff( history.deleted, None, dict( operation="Switch", id=history.id ) ) ), attach_popup=True, filterable="advanced" ), DatasetsByStateColumn( "Datasets (by state)", ncells=4 ), - grids.TagsColumn( "Tags", "tags", model.History, model.HistoryTagAssociation, filterable="advanced"), + grids.TagsColumn( "Tags", "tags", model.History, model.HistoryTagAssociation, filterable="advanced", grid_name="HistoryListGrid" ), StatusColumn( "Status", attach_popup=False ), grids.GridColumn( "Created", key="create_time", format=time_ago ), grids.GridColumn( "Last Updated", key="update_time", format=time_ago ), diff -r 9d4945bbdcf5 -r f98643c26eb7 lib/galaxy/web/controllers/tag.py --- a/lib/galaxy/web/controllers/tag.py Mon Nov 30 15:33:45 2009 -0500 +++ b/lib/galaxy/web/controllers/tag.py Mon Nov 30 17:53:25 2009 -0500 @@ -28,28 +28,38 @@ @web.expose @web.require_login( "Add tag to an item." ) - def add_tag_async( self, trans, id=None, item_class=None, new_tag=None ): + def add_tag_async( self, trans, id=None, item_class=None, new_tag=None, context=None ): """ Add tag to an item. """ - item = self._get_item(trans, item_class, trans.security.decode_id(id)) - self._do_security_check(trans, item) + # Check that user owns item. + item = self._get_item(trans, item_class, trans.security.decode_id( id ) ) + self._do_security_check( trans, item ) + # Apply tag. self.tag_handler.apply_item_tags( trans.sa_session, item, new_tag.encode('utf-8') ) trans.sa_session.flush() + # Log. + params = dict( item_id=item.id, item_class=item_class, tag=new_tag) + trans.log_action( unicode( "tag"), context, params ) + @web.expose @web.require_login( "Remove tag from an item." ) - def remove_tag_async( self, trans, id=None, item_class=None, tag_name=None ): + def remove_tag_async( self, trans, id=None, item_class=None, tag_name=None, context=None ): """ Remove tag from an item. """ + + # Check that user owns item. item = self._get_item(trans, item_class, trans.security.decode_id(id)) - self._do_security_check(trans, item) + # Remove tag. self.tag_handler.remove_item_tag( trans, item, tag_name.encode('utf-8') ) - #print tag_name - #print unicode(tag_name) trans.sa_session.flush() + # Log. + params = dict( item_id=item.id, item_class=item_class, tag=tag_name) + trans.log_action( unicode( "untag"), context, params ) + # Retag an item. All previous tags are deleted and new tags are applied. @web.expose @web.require_login( "Apply a new set of tags to an item; previous tags are deleted." ) diff -r 9d4945bbdcf5 -r f98643c26eb7 lib/galaxy/web/framework/__init__.py --- a/lib/galaxy/web/framework/__init__.py Mon Nov 30 15:33:45 2009 -0500 +++ b/lib/galaxy/web/framework/__init__.py Mon Nov 30 17:53:25 2009 -0500 @@ -10,6 +10,7 @@ import base import pickle from galaxy import util +from galaxy.util.json import to_json_string pkg_resources.require( "simplejson" ) import simplejson @@ -170,6 +171,22 @@ to allow migration toward a more SQLAlchemy 0.4 style of use. """ return self.app.model.context.current + def log_action( self, action, context, params): + """ + Application-level logging of user actions. + """ + if self.app.config.log_actions: + action = self.app.model.UserAction(action=action, context=context, params=unicode( to_json_string( params ) ) ) + try: + action.user = self.user + except: + action.user = None + try: + action.session_id = self.galaxy_session.id + except: + action.session_id = None + self.sa_session.add( action ) + self.sa_session.flush() def log_event( self, message, tool_id=None, **kwargs ): """ Application level logging. Still needs fleshing out (log levels and such) diff -r 9d4945bbdcf5 -r f98643c26eb7 lib/galaxy/web/framework/helpers/grids.py --- a/lib/galaxy/web/framework/helpers/grids.py Mon Nov 30 15:33:45 2009 -0500 +++ b/lib/galaxy/web/framework/helpers/grids.py Mon Nov 30 17:53:25 2009 -0500 @@ -200,6 +200,13 @@ trans.get_user().preferences[pref_name] = unicode( to_json_string( sort_key ) ) trans.sa_session.flush() + # Log grid view. + context = unicode( self.__class__.__name__ ) + params = cur_filter_dict.copy() + params['sort'] = sort_key + params['async'] = ( 'async' in kwargs ) + trans.log_action( unicode( "grid.view"), context, params ) + # Render grid. def url( *args, **kwargs ): # Only include sort/filter arguments if not linking to another @@ -340,17 +347,19 @@ # Generic column that supports tagging. class TagsColumn( TextColumn ): - def __init__( self, col_name, key, model_class, model_tag_association_class, filterable ): + def __init__( self, col_name, key, model_class, model_tag_association_class, filterable, grid_name=None ): GridColumn.__init__(self, col_name, key=key, model_class=model_class, filterable=filterable) self.model_tag_association_class = model_tag_association_class # Tags cannot be sorted. self.sortable = False + # Column-specific attributes. self.tag_elt_id_gen = 0 + self.grid_name = grid_name def get_value( self, trans, grid, item ): self.tag_elt_id_gen += 1 elt_id="tagging-elt" + str( self.tag_elt_id_gen ) div_elt = "<div id=%s></div>" % elt_id - return div_elt + trans.fill_template( "/tagging_common.mako", trans=trans, tagged_item=item, + return div_elt + trans.fill_template( "/tagging_common.mako", trans=trans, tagged_item=item, elt_context=self.grid_name, elt_id = elt_id, in_form="true", input_size="20", tag_click_fn="add_tag_to_grid_filter" ) def filter( self, db_session, query, column_filter ): """ Modify query to filter model_class by tag. Multiple filters are ANDed. """ diff -r 9d4945bbdcf5 -r f98643c26eb7 templates/dataset/edit_attributes.mako --- a/templates/dataset/edit_attributes.mako Mon Nov 30 15:33:45 2009 -0500 +++ b/templates/dataset/edit_attributes.mako Mon Nov 30 17:53:25 2009 -0500 @@ -59,7 +59,7 @@ </div> <div style="clear: both"></div> </div> - ${render_tagging_element(data, "dataset-tag-area", use_toggle_link="false", in_form="true", input_size="30")} + ${render_tagging_element(data, "dataset-tag-area", "edit_attributes.mako", use_toggle_link="false", in_form="true", input_size="30")} %endif %for name, spec in data.metadata.spec.items(): %if spec.visible: diff -r 9d4945bbdcf5 -r f98643c26eb7 templates/history/view.mako --- a/templates/history/view.mako Mon Nov 30 15:33:45 2009 -0500 +++ b/templates/history/view.mako Mon Nov 30 17:53:25 2009 -0500 @@ -327,7 +327,7 @@ %if trans.get_user() is not None: <div id='history-tag-area' class="tag-element"></div> - ${render_tagging_element(history, "history-tag-area", use_toggle_link='false', get_toggle_link_text_fn='get_toggle_link_text', editable=user_owns_history)} + ${render_tagging_element(history, "history-tag-area", "history/view.mako", use_toggle_link='false', get_toggle_link_text_fn='get_toggle_link_text', editable=user_owns_history)} %endif %if not datasets: diff -r 9d4945bbdcf5 -r f98643c26eb7 templates/root/history.mako --- a/templates/root/history.mako Mon Nov 30 15:33:45 2009 -0500 +++ b/templates/root/history.mako Mon Nov 30 17:53:25 2009 -0500 @@ -325,7 +325,7 @@ %if trans.get_user() is not None: <div id='history-tag-area' class="tag-element"></div> - ${render_tagging_element(history, "history-tag-area", get_toggle_link_text_fn='get_toggle_link_text')} + ${render_tagging_element( tagged_item=history, elt_id="history-tag-area", elt_context="history.mako", get_toggle_link_text_fn='get_toggle_link_text' )} %endif %if not datasets: diff -r 9d4945bbdcf5 -r f98643c26eb7 templates/tagging_common.mako --- a/templates/tagging_common.mako Mon Nov 30 15:33:45 2009 -0500 +++ b/templates/tagging_common.mako Mon Nov 30 17:53:25 2009 -0500 @@ -1,10 +1,10 @@ ## Render a tagging element if there is a tagged_item. %if tagged_item is not None and elt_id is not None: - ${render_tagging_element(tagged_item, elt_id=elt_id, in_form=in_form, input_size=input_size, tag_click_fn=tag_click_fn)} + ${render_tagging_element(tagged_item=tagged_item, elt_id=elt_id, elt_context=elt_context, in_form=in_form, input_size=input_size, tag_click_fn=tag_click_fn)} %endif ## Render the tags 'tags' as an autocomplete element. -<%def name="render_tagging_element(tagged_item, elt_id, use_toggle_link='true', in_form='false', input_size='15', tag_click_fn='default_tag_click_fn', get_toggle_link_text_fn='default_get_toggle_link_text_fn', editable='true')"> +<%def name="render_tagging_element(tagged_item, elt_id, elt_context, use_toggle_link='true', in_form='false', input_size='15', tag_click_fn='default_tag_click_fn', get_toggle_link_text_fn='default_get_toggle_link_text_fn', editable='true')"> <script type="text/javascript"> // // Set up autocomplete tagger. @@ -75,8 +75,8 @@ tag_click_fn: ${tag_click_fn}, <% tagged_item_id = trans.security.encode_id(tagged_item.id) %> ajax_autocomplete_tag_url: "${h.url_for( controller='tag', action='tag_autocomplete_data', id=tagged_item_id, item_class=tagged_item.__class__.__name__ )}", - ajax_add_tag_url: "${h.url_for( controller='tag', action='add_tag_async', id=tagged_item_id, item_class=tagged_item.__class__.__name__ )}", - ajax_delete_tag_url: "${h.url_for( controller='tag', action='remove_tag_async', id=tagged_item_id, item_class=tagged_item.__class__.__name__ )}", + ajax_add_tag_url: "${h.url_for( controller='tag', action='add_tag_async', id=tagged_item_id, item_class=tagged_item.__class__.__name__, context=elt_context )}", + ajax_delete_tag_url: "${h.url_for( controller='tag', action='remove_tag_async', id=tagged_item_id, item_class=tagged_item.__class__.__name__, context=elt_context )}", delete_tag_img: "${h.url_for('/static/images/delete_tag_icon_gray.png')}", delete_tag_img_rollover: "${h.url_for('/static/images/delete_tag_icon_white.png')}", add_tag_img: "${h.url_for('/static/images/add_icon.png')}", diff -r 9d4945bbdcf5 -r f98643c26eb7 universe_wsgi.ini.sample --- a/universe_wsgi.ini.sample Mon Nov 30 15:33:45 2009 -0500 +++ b/universe_wsgi.ini.sample Mon Nov 30 17:53:25 2009 -0500 @@ -144,6 +144,9 @@ # Log events log_events = True +# Log user actions +log_actions = True + # Configuration for debugging middleware debug = True use_lint = False

1 0

[hg] galaxy 3134: Improved loading state for AJAXed grids: when ...
by Greg Von Kuster 02 Dec '09

02 Dec '09

details: http://www.bx.psu.edu/hg/galaxy/rev/6e742ffb16a5 changeset: 3134:6e742ffb16a5 user: jeremy goecks <jeremy.goecks at emory.edu> date: Tue Dec 01 11:07:20 2009 -0500 description: Improved loading state for AJAXed grids: when grid is loading, gray out table and prevent further user actions. diffstat: templates/grid_base.mako | 16 +++++++++++++++- 1 files changed, 15 insertions(+), 1 deletions(-) diffs (47 lines): diff -r e75b55cc9e08 -r 6e742ffb16a5 templates/grid_base.mako --- a/templates/grid_base.mako Tue Dec 01 09:53:36 2009 -0500 +++ b/templates/grid_base.mako Tue Dec 01 11:07:20 2009 -0500 @@ -601,6 +601,7 @@ // If there's an operation in the args, do POST; otherwise, do GET. var operation = url_args['operation']; var method = (operation != null && operation != undefined ? "POST" : "GET" ); + $('.loading-elt-overlay').show(); // Show overlay to indicate loading and prevent user actions. $.ajax({ type: method, url: "${h.url_for()}", @@ -660,6 +661,9 @@ } } + // Hide loading overlay. + $('.loading-elt-overlay').hide(); + // Show message if there is one. var message = $.trim( parsed_response_text[2] ); if (message != "") @@ -711,6 +715,15 @@ .gray-background { background-color: #DDDDDD; } + .loading-elt-overlay { + background-color : white; + opacity : 0.5; + width : 100%; + height : 85%; + z-index : 14000; + position : absolute; + display: none; + } </style> </%def> @@ -750,7 +763,8 @@ ## Render grid. <%def name="render_grid_table()"> <form action="${url()}" method="post" onsubmit="return false;"> - <table class="grid"> + <div class='loading-elt-overlay'></div> + <table id='grid-table' class="grid"> <thead id="grid-table-header"> <tr> <th></th>

1 0

[hg] galaxy 3136: Added some error checking to pileup_parser too...
by Greg Von Kuster 02 Dec '09

02 Dec '09

details: http://www.bx.psu.edu/hg/galaxy/rev/fe853d8e062d changeset: 3136:fe853d8e062d user: Kelly Vincent <kpvincent(a)bx.psu.edu> date: Tue Dec 01 16:18:25 2009 -0500 description: Added some error checking to pileup_parser tool so that it exits with a meaningful message if the user tries to do some 6-column work on a 10-column file, or vice versa. diffstat: tools/samtools/pileup_interval.py | 16 ++++++++++++++-- 1 files changed, 14 insertions(+), 2 deletions(-) diffs (40 lines): diff -r eb317734e391 -r fe853d8e062d tools/samtools/pileup_interval.py --- a/tools/samtools/pileup_interval.py Tue Dec 01 11:49:07 2009 -0500 +++ b/tools/samtools/pileup_interval.py Tue Dec 01 16:18:25 2009 -0500 @@ -1,7 +1,7 @@ #! /usr/bin/python """ -Creates a pileup file from a bam file and a reference. +Condenses pileup format into ranges of bases. usage: %prog [options] -i, --input=i: Input pileup file @@ -18,6 +18,7 @@ from galaxy import eggs import pkg_resources; pkg_resources.require( "bx-python" ) from bx.cookbook import doc_optparse +import sys def stop_err( msg ): sys.stderr.write( msg ) @@ -56,7 +57,18 @@ bases = [] while inLine.strip() != '': lineParts = inLine.split('\t') - seq, loc, base, cov = lineParts[seqIndex], int(lineParts[locIndex]), lineParts[baseIndex], int(lineParts[covIndex]) + try: + seq, loc, base, cov = lineParts[seqIndex], int(lineParts[locIndex]), lineParts[baseIndex], int(lineParts[covIndex]) + except IndexError, ei: + if options.format == 'ten': + stop_err( 'It appears that you have selected 10 columns while your file has 6. Make sure that the number of columns you specify matches the number in your file.\n' + str( ei ) ) + else: + stop_err( 'There appears to be something wrong with your column index values.\n' + str( ei ) ) + except ValueError, ev: + if options.format == 'six': + stop_err( 'It appears that you have selected 6 columns while your file has 10. Make sure that the number of columns you specify matches the number in your file.\n' + str( ev ) ) + else: + stop_err( 'There appears to be something wrong with your column index values.\n' + str( ev ) ) # strout += str(startLoc) + '\n' # strout += str(bases) + '\n' # strout += '%s\t%s\t%s\t%s\n' % (seq, loc, base, cov)

1 0

[hg] galaxy 3129: Fix MAF to BED tool. This tool should be rewri...
by Greg Von Kuster 02 Dec '09

02 Dec '09

details: http://www.bx.psu.edu/hg/galaxy/rev/9d4945bbdcf5 changeset: 3129:9d4945bbdcf5 user: Dan Blankenberg <dan(a)bx.psu.edu> date: Mon Nov 30 15:33:45 2009 -0500 description: Fix MAF to BED tool. This tool should be rewritten to not use a code file, but instead use one of the standard ways to create additional outputs, similar to how maf_to_interval functions. diffstat: tools/maf/maf_to_bed.py | 3 +-- tools/maf/maf_to_bed.xml | 4 ++-- tools/maf/maf_to_bed_code.py | 1 + 3 files changed, 4 insertions(+), 4 deletions(-) diffs (36 lines): diff -r 92d6a9796b24 -r 9d4945bbdcf5 tools/maf/maf_to_bed.py --- a/tools/maf/maf_to_bed.py Mon Nov 30 14:55:30 2009 -0500 +++ b/tools/maf/maf_to_bed.py Mon Nov 30 15:33:45 2009 -0500 @@ -15,8 +15,7 @@ input_filename = sys.argv[1] output_filename = sys.argv[2] #where to store files that become additional output - database_tmp_dir = './database/tmp' #os.path.join(os.path.split(os.path.split(os.path.realpath(output_filename))[0])[0],'tmp') #database/tmp - + database_tmp_dir = sys.argv[5] species = sys.argv[3].split(',') partial = sys.argv[4] diff -r 92d6a9796b24 -r 9d4945bbdcf5 tools/maf/maf_to_bed.xml --- a/tools/maf/maf_to_bed.xml Mon Nov 30 14:55:30 2009 -0500 +++ b/tools/maf/maf_to_bed.xml Mon Nov 30 15:33:45 2009 -0500 @@ -1,6 +1,6 @@ -<tool id="MAF_To_BED1" name="Maf to BED"> +<tool id="MAF_To_BED1" name="Maf to BED" force_history_refresh="True"> <description>Converts a MAF formated file to the BED format</description> - <command interpreter="python">maf_to_bed.py $input1 $out_file1 $species $complete_blocks</command> + <command interpreter="python">maf_to_bed.py $input1 $out_file1 $species $complete_blocks $__new_file_path__</command> <inputs> <param format="maf" name="input1" type="data" label="MAF file to convert"/> <param name="species" type="select" label="Select species" display="checkboxes" multiple="true" help="a separate history item will be created for each checked species"> diff -r 92d6a9796b24 -r 9d4945bbdcf5 tools/maf/maf_to_bed_code.py --- a/tools/maf/maf_to_bed_code.py Mon Nov 30 14:55:30 2009 -0500 +++ b/tools/maf/maf_to_bed_code.py Mon Nov 30 15:33:45 2009 -0500 @@ -46,6 +46,7 @@ newdata.state = newdata.states.ERROR newdata.dbkey = dbkey newdata.init_meta() + newdata.set_meta() newdata.set_peek() app.model.context.flush() output_data_list.append(newdata)

1 0

[hg] galaxy 3128: When setting metadata externally, substitute a...
by Greg Von Kuster 02 Dec '09

02 Dec '09

details: http://www.bx.psu.edu/hg/galaxy/rev/92d6a9796b24 changeset: 3128:92d6a9796b24 user: Dan Blankenberg <dan(a)bx.psu.edu> date: Mon Nov 30 14:55:30 2009 -0500 description: When setting metadata externally, substitute a MetadataTempFile object for all populated metadata FileParameters. Fixes the problem with writting to metadata FileParameter files on cluster nodes with read-only database/files directory. diffstat: lib/galaxy/datatypes/metadata.py | 21 +++- lib/galaxy/model/mapping.py | 1 + lib/galaxy/model/migrate/versions/0028_external_metadata_file_override.py | 47 +++++++++ scripts/set_metadata.py | 23 ++++- tools/regVariation/quality_filter.xml | 2 +- 5 files changed, 87 insertions(+), 7 deletions(-) diffs (172 lines): diff -r 9f7a2a3be0e7 -r 92d6a9796b24 lib/galaxy/datatypes/metadata.py --- a/lib/galaxy/datatypes/metadata.py Mon Nov 30 11:59:54 2009 -0500 +++ b/lib/galaxy/datatypes/metadata.py Mon Nov 30 14:55:30 2009 -0500 @@ -429,7 +429,9 @@ if MetadataTempFile.is_JSONified_value( value ): value = MetadataTempFile.from_JSON( value ) if isinstance( value, MetadataTempFile ): - mf = self.new_file( dataset = parent, **value.kwds ) + mf = parent.metadata.get( self.spec.name, None) + if mf is None: + mf = self.new_file( dataset = parent, **value.kwds ) shutil.move( value.file_name, mf.file_name ) value = mf.id return value @@ -521,7 +523,7 @@ if dataset_path.false_path and dataset_path.real_path == metadata_files.dataset.file_name: return dataset_path.false_path return "" - return "%s,%s,%s,%s,%s" % ( metadata_files.filename_in, metadata_files.filename_kwds, metadata_files.filename_out, metadata_files.filename_results_code, __get_filename_override() ) + return "%s,%s,%s,%s,%s,%s" % ( metadata_files.filename_in, metadata_files.filename_kwds, metadata_files.filename_out, metadata_files.filename_results_code, __get_filename_override(), metadata_files.filename_override_metadata ) if not isinstance( datasets, list ): datasets = [ datasets ] if exec_dir is None: @@ -558,11 +560,22 @@ open( metadata_files.filename_out, 'wb+' ) # create the file on disk, so it cannot be reused by tempfile (unlikely, but possible) #file to store a 'return code' indicating the results of the set_meta() call #results code is like (True/False - if setting metadata was successful/failed , exception or string of reason of success/failure ) - metadata_files.filename_results_code = relpath( tempfile.NamedTemporaryFile( dir = tmp_dir, prefix = "metadata_out_%s_" % key ).name ) + metadata_files.filename_results_code = relpath( tempfile.NamedTemporaryFile( dir = tmp_dir, prefix = "metadata_results_%s_" % key ).name ) simplejson.dump( ( False, 'External set_meta() not called' ), open( metadata_files.filename_results_code, 'wb+' ) ) # create the file on disk, so it cannot be reused by tempfile (unlikely, but possible) #file to store kwds passed to set_meta() metadata_files.filename_kwds = relpath( tempfile.NamedTemporaryFile( dir = tmp_dir, prefix = "metadata_kwds_%s_" % key ).name ) simplejson.dump( kwds, open( metadata_files.filename_kwds, 'wb+' ), ensure_ascii=True ) + #existing metadata file parameters need to be overridden with cluster-writable file locations + metadata_files.filename_override_metadata = relpath( tempfile.NamedTemporaryFile( dir = tmp_dir, prefix = "metadata_override_%s_" % key ).name ) + open( metadata_files.filename_override_metadata, 'wb+' ) # create the file on disk, so it cannot be reused by tempfile (unlikely, but possible) + override_metadata = [] + for meta_key, spec_value in dataset.metadata.spec.iteritems(): + if isinstance( spec_value.param, FileParameter ) and dataset.metadata.get( meta_key, None ) is not None: + metadata_temp = MetadataTempFile() + shutil.copy( dataset.metadata.get( meta_key, None ).file_name, metadata_temp.file_name ) + override_metadata.append( ( meta_key, metadata_temp.to_JSON() ) ) + simplejson.dump( override_metadata, open( metadata_files.filename_override_metadata, 'wb+' ) ) + #add to session and flush sa_session.add( metadata_files ) sa_session.flush() metadata_files_list.append( metadata_files ) @@ -585,7 +598,7 @@ #can occur if the job was stopped before completion, but a MetadataTempFile is used in the set_meta MetadataTempFile.cleanup_from_JSON_dict_filename( metadata_files.filename_out ) dataset_key = self.get_dataset_metadata_key( metadata_files.dataset ) - for key, fname in [ ( 'filename_in', metadata_files.filename_in ), ( 'filename_out', metadata_files.filename_out ), ( 'filename_results_code', metadata_files.filename_results_code ), ( 'filename_kwds', metadata_files.filename_kwds ) ]: + for key, fname in [ ( 'filename_in', metadata_files.filename_in ), ( 'filename_out', metadata_files.filename_out ), ( 'filename_results_code', metadata_files.filename_results_code ), ( 'filename_kwds', metadata_files.filename_kwds ), ( 'filename_override_metadata', metadata_files.filename_override_metadata ) ]: try: os.remove( fname ) except Exception, e: diff -r 9f7a2a3be0e7 -r 92d6a9796b24 lib/galaxy/model/mapping.py --- a/lib/galaxy/model/mapping.py Mon Nov 30 11:59:54 2009 -0500 +++ b/lib/galaxy/model/mapping.py Mon Nov 30 14:55:30 2009 -0500 @@ -358,6 +358,7 @@ Column( "filename_out", String( 255 ) ), Column( "filename_results_code", String( 255 ) ), Column( "filename_kwds", String( 255 ) ), + Column( "filename_override_metadata", String( 255 ) ), Column( "job_runner_external_pid", String( 255 ) ) ) Event.table = Table( "event", metadata, diff -r 9f7a2a3be0e7 -r 92d6a9796b24 lib/galaxy/model/migrate/versions/0028_external_metadata_file_override.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/lib/galaxy/model/migrate/versions/0028_external_metadata_file_override.py Mon Nov 30 14:55:30 2009 -0500 @@ -0,0 +1,47 @@ +""" +This script adds the filename_override_metadata column to the JobExternalOutputMetadata table, +allowing existing metadata files to be written when using external metadata and a cluster +set up with read-only access to database/files +""" +from sqlalchemy import * +from sqlalchemy.orm import * +from sqlalchemy.exceptions import * +from migrate import * +from migrate.changeset import * +import datetime +now = datetime.datetime.utcnow +import sys, logging +# Need our custom types, but don't import anything else from model +from galaxy.model.custom_types import * + +log = logging.getLogger( __name__ ) +log.setLevel(logging.DEBUG) +handler = logging.StreamHandler( sys.stdout ) +format = "%(name)s %(levelname)s %(asctime)s %(message)s" +formatter = logging.Formatter( format ) +handler.setFormatter( formatter ) +log.addHandler( handler ) + +metadata = MetaData( migrate_engine ) +db_session = scoped_session( sessionmaker( bind=migrate_engine, autoflush=False, autocommit=True ) ) + +def display_migration_details(): + print "========================================" + print "This script adds the filename_override_metadata column to the JobExternalOutputMetadata table," + print" allowing existing metadata files to be written when using external metadata and a cluster" + print "set up with read-only access to database/files" + print "========================================" +def upgrade(): + display_migration_details() + # Load existing tables + metadata.reflect() + try: + job_external_output_metadata = Table( "job_external_output_metadata", metadata, autoload=True ) + col = Column( "filename_override_metadata", String( 255 ) ) + col.create( job_external_output_metadata ) + assert col is job_external_output_metadata.c.filename_override_metadata + except Exception, e: + log.debug( "Adding column 'filename_override_metadata' to job_external_output_metadata table failed: %s" % ( str( e ) ) ) + +def downgrade(): + pass diff -r 9f7a2a3be0e7 -r 92d6a9796b24 scripts/set_metadata.py --- a/scripts/set_metadata.py Mon Nov 30 11:59:54 2009 -0500 +++ b/scripts/set_metadata.py Mon Nov 30 14:55:30 2009 -0500 @@ -50,17 +50,36 @@ except: continue for filenames in sys.argv[1:]: - filename_in, filename_kwds, filename_out, filename_results_code, dataset_filename_override = filenames.split( ',' ) + fields = filenames.split( ',' ) + filename_in = fields.pop( 0 ) + filename_kwds = fields.pop( 0 ) + filename_out = fields.pop( 0 ) + filename_results_code = fields.pop( 0 ) + dataset_filename_override = fields.pop( 0 ) + #Need to be careful with the way that these parameters are populated from the filename splitting, + #because if a job is running when the server is updated, any existing external metadata command-lines + #will not have info about the newly added override_metadata file + if fields: + override_metadata = fields.pop( 0 ) + else: + override_metadata = None try: dataset = cPickle.load( open( filename_in ) ) #load DatasetInstance if dataset_filename_override: dataset.dataset.external_filename = dataset_filename_override if ext_override.get( dataset.dataset.id, None ): dataset.extension = ext_override[ dataset.dataset.id ] + #Metadata FileParameter types may not be writable on a cluster node, and are therefore temporarily substituted with MetadataTempFiles + if override_metadata: + override_metadata = simplejson.load( open( override_metadata ) ) + for metadata_name, metadata_file_override in override_metadata: + if galaxy.datatypes.metadata.MetadataTempFile.is_JSONified_value( metadata_file_override ): + metadata_file_override = galaxy.datatypes.metadata.MetadataTempFile.from_JSON( metadata_file_override ) + setattr( dataset.metadata, metadata_name, metadata_file_override ) kwds = stringify_dictionary_keys( simplejson.load( open( filename_kwds ) ) )#load kwds; need to ensure our keywords are not unicode dataset.datatype.set_meta( dataset, **kwds ) dataset.metadata.to_JSON_dict( filename_out ) # write out results of set_meta - simplejson.dump( ( True, 'Metadata has been set successfully' ), open( filename_results_code, 'wb+' ) ) #setting metadata has suceeded + simplejson.dump( ( True, 'Metadata has been set successfully' ), open( filename_results_code, 'wb+' ) ) #setting metadata has succeeded except Exception, e: simplejson.dump( ( False, str( e ) ), open( filename_results_code, 'wb+' ) ) #setting metadata has failed somehow clear_mappers() diff -r 9f7a2a3be0e7 -r 92d6a9796b24 tools/regVariation/quality_filter.xml --- a/tools/regVariation/quality_filter.xml Mon Nov 30 11:59:54 2009 -0500 +++ b/tools/regVariation/quality_filter.xml Mon Nov 30 14:55:30 2009 -0500 @@ -61,7 +61,7 @@ </conditional> </inputs> <outputs> - <data format="maf" name="out_file1"/> + <data format="maf" name="out_file1" metadata_source="input"/> </outputs> <requirements> <requirement type="python-module">numpy</requirement>

1 0

[hg] galaxy 3125: Fix for request_events table migration script ...
by Greg Von Kuster 02 Dec '09

02 Dec '09

details: http://www.bx.psu.edu/hg/galaxy/rev/a8234f4d7012 changeset: 3125:a8234f4d7012 user: rc date: Mon Nov 30 09:35:21 2009 -0500 description: Fix for request_events table migration script for MySQL & sqlite diffstat: lib/galaxy/model/migrate/versions/0027_request_events.py | 5 +++-- 1 files changed, 3 insertions(+), 2 deletions(-) diffs (16 lines): diff -r 341d7c1c5c9a -r a8234f4d7012 lib/galaxy/model/migrate/versions/0027_request_events.py --- a/lib/galaxy/model/migrate/versions/0027_request_events.py Mon Nov 30 09:05:28 2009 -0500 +++ b/lib/galaxy/model/migrate/versions/0027_request_events.py Mon Nov 30 09:35:21 2009 -0500 @@ -73,9 +73,10 @@ "%s AS create_time," + \ "%s AS update_time," + \ "request.id AS request_id," + \ - "request.state AS state " + \ + "request.state AS state," + \ + "'%s' AS comment " + \ "FROM request;" - cmd = cmd % ( nextval('request_event'), localtimestamp(), localtimestamp() ) + cmd = cmd % ( nextval('request_event'), localtimestamp(), localtimestamp(), 'Imported from request table') db_session.execute( cmd ) # Delete the state column

1 0

[hg] galaxy 3127: Fixed a bug in 'fetch closest features' tool.
by Greg Von Kuster 02 Dec '09

02 Dec '09

details: http://www.bx.psu.edu/hg/galaxy/rev/9f7a2a3be0e7 changeset: 3127:9f7a2a3be0e7 user: guru date: Mon Nov 30 11:59:54 2009 -0500 description: Fixed a bug in 'fetch closest features' tool. diffstat: tools/new_operations/flanking_features.py | 5 ++--- 1 files changed, 2 insertions(+), 3 deletions(-) diffs (22 lines): diff -r 71b779fde315 -r 9f7a2a3be0e7 tools/new_operations/flanking_features.py --- a/tools/new_operations/flanking_features.py Mon Nov 30 11:56:58 2009 -0500 +++ b/tools/new_operations/flanking_features.py Mon Nov 30 11:59:54 2009 -0500 @@ -121,7 +121,7 @@ map(outfields.append, result_down[-1].other) #The last element of result_down will be the closest element to the given interval yield outfields - if either: + if either and (result_up or result_down): if result_up and result_down: if abs(start - int(result_up[res_ind].end)) <= abs(end - int(result_down[-1].start)): map(outfields.append, result_up[res_ind].other) @@ -131,8 +131,7 @@ map(outfields.append, result_up[res_ind].other) elif result_down: map(outfields.append, result_down[-1].other) #The last element of result_down will be the closest element to the given interval - yield outfields - + yield outfields def main(): options, args = doc_optparse.parse( __doc__ )

1 0

[hg] galaxy 3122: Request events & request rejection (resolves i...
by Greg Von Kuster 02 Dec '09

02 Dec '09

details: http://www.bx.psu.edu/hg/galaxy/rev/8d149264b777 changeset: 3122:8d149264b777 user: rc date: Wed Nov 25 14:06:28 2009 -0500 description: Request events & request rejection (resolves issue #221) - added request_events table - removed 'state' field in the request table - added 'new' & 'rejected' request states - color coded request states in the request grid Grids framework - added a confirm flag in grid operations which require confirmation before going through with operation diffstat: lib/galaxy/model/__init__.py | 35 +- lib/galaxy/model/mapping.py | 16 +- lib/galaxy/model/migrate/versions/0027_request_events.py | 94 ++++++ lib/galaxy/web/controllers/requests.py | 157 +++++++-- lib/galaxy/web/controllers/requests_admin.py | 299 +++++++++++++------ lib/galaxy/web/framework/helpers/grids.py | 4 +- templates/admin/requests/events.mako | 36 ++ templates/admin/requests/reject.mako | 42 ++ templates/admin/requests/show_request.mako | 13 +- templates/grid_base.mako | 6 +- templates/requests/events.mako | 39 ++ templates/requests/show_request.mako | 17 +- templates/sample/sample_events.mako | 43 +- test/base/twilltestcase.py | 9 +- test/functional/test_forms_and_requests.py | 30 +- 15 files changed, 651 insertions(+), 189 deletions(-) diffs (1296 lines): diff -r 5761948422a8 -r 8d149264b777 lib/galaxy/model/__init__.py --- a/lib/galaxy/model/__init__.py Tue Nov 24 14:53:19 2009 -0500 +++ b/lib/galaxy/model/__init__.py Wed Nov 25 14:06:28 2009 -0500 @@ -1260,11 +1260,12 @@ self.content = content class Request( object ): - states = Bunch( UNSUBMITTED = 'Unsubmitted', + states = Bunch( NEW = 'New', SUBMITTED = 'Submitted', + REJECTED = 'Rejected', COMPLETE = 'Complete') def __init__(self, name=None, desc=None, request_type=None, user=None, - form_values=None, library=None, folder=None, state=False): + form_values=None, library=None, folder=None): self.name = name self.desc = desc self.type = request_type @@ -1272,19 +1273,39 @@ self.user = user self.library = library self.folder = folder - self.state = state self.samples_list = [] + def state(self): + if self.events: + return self.events[0].state + return None + def last_comment(self): + if self.events: + if self.events[0].comment: + return self.events[0].comment + else: + return '' + return 'No comment' def has_sample(self, sample_name): for s in self.samples: if s.name == sample_name: return s return False + def unsubmitted(self): + return self.state() in [ self.states.REJECTED, self.states.NEW ] + def rejected(self): + return self.state() == self.states.REJECTED def submitted(self): - return self.state == self.states.SUBMITTED - def unsubmitted(self): - return self.state == self.states.UNSUBMITTED + return self.state() == self.states.SUBMITTED + def new(self): + return self.state() == self.states.NEW def complete(self): - return self.state == self.states.COMPLETE + return self.state() == self.states.COMPLETE + +class RequestEvent( object ): + def __init__(self, request=None, request_state=None, comment=''): + self.request = request + self.state = request_state + self.comment = comment class RequestType( object ): def __init__(self, name=None, desc=None, request_form=None, sample_form=None): diff -r 5761948422a8 -r 8d149264b777 lib/galaxy/model/mapping.py --- a/lib/galaxy/model/mapping.py Tue Nov 24 14:53:19 2009 -0500 +++ b/lib/galaxy/model/mapping.py Wed Nov 25 14:06:28 2009 -0500 @@ -618,8 +618,15 @@ Column( "user_id", Integer, ForeignKey( "galaxy_user.id" ), index=True ), Column( "library_id", Integer, ForeignKey( "library.id" ), index=True ), Column( "folder_id", Integer, ForeignKey( "library_folder.id" ), index=True ), + Column( "deleted", Boolean, index=True, default=False ) ) + +RequestEvent.table = Table('request_event', metadata, + Column( "id", Integer, primary_key=True), + Column( "create_time", DateTime, default=now ), + Column( "update_time", DateTime, default=now, onupdate=now ), + Column( "request_id", Integer, ForeignKey( "request.id" ), index=True ), Column( "state", TrimmedString( 255 ), index=True ), - Column( "deleted", Boolean, index=True, default=False ) ) + Column( "comment", TEXT ) ) Sample.table = Table('sample', metadata, Column( "id", Integer, primary_key=True ), @@ -768,9 +775,14 @@ folder=relation( LibraryFolder, primaryjoin=( Request.table.c.folder_id == LibraryFolder.table.c.id ) ), library=relation( Library, - primaryjoin=( Request.table.c.library_id == Library.table.c.id ) ) + primaryjoin=( Request.table.c.library_id == Library.table.c.id ) ), + events=relation( RequestEvent, backref="request", + order_by=desc(RequestEvent.table.c.update_time) ) ) ) +assign_mapper( context, RequestEvent, RequestEvent.table, + properties=None ) + assign_mapper( context, RequestType, RequestType.table, properties=dict( states=relation( SampleState, backref="request_type", diff -r 5761948422a8 -r 8d149264b777 lib/galaxy/model/migrate/versions/0027_request_events.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/lib/galaxy/model/migrate/versions/0027_request_events.py Wed Nov 25 14:06:28 2009 -0500 @@ -0,0 +1,94 @@ +""" +This migration script adds the request_event table and +removes the state field in the request table +""" +from sqlalchemy import * +from sqlalchemy.orm import * +from sqlalchemy.exc import * +from migrate import * +from migrate.changeset import * + +import datetime +now = datetime.datetime.utcnow + +import sys, logging +log = logging.getLogger( __name__ ) +log.setLevel(logging.DEBUG) +handler = logging.StreamHandler( sys.stdout ) +format = "%(name)s %(levelname)s %(asctime)s %(message)s" +formatter = logging.Formatter( format ) +handler.setFormatter( formatter ) +log.addHandler( handler ) + +# Need our custom types, but don't import anything else from model +from galaxy.model.custom_types import * + +metadata = MetaData( migrate_engine ) +db_session = scoped_session( sessionmaker( bind=migrate_engine, autoflush=False, autocommit=True ) ) + +def display_migration_details(): + print "========================================" + print "This migration script adds the request_event table and" + print "removes the state field in the request table" + print "========================================" + +def localtimestamp(): + if migrate_engine.name == 'postgres' or migrate_engine.name == 'mysql': + return "LOCALTIMESTAMP" + elif migrate_engine.name == 'sqlite': + return "current_date || ' ' || current_time" + else: + raise Exception( 'Unable to convert data for unknown database type: %s' % db ) + +def nextval( table, col='id' ): + if migrate_engine.name == 'postgres': + return "nextval('%s_%s_seq')" % ( table, col ) + elif migrate_engine.name == 'mysql' or migrate_engine.name == 'sqlite': + return "null" + else: + raise Exception( 'Unable to convert data for unknown database type: %s' % migrate_engine.name ) + + +RequestEvent_table = Table('request_event', metadata, + Column( "id", Integer, primary_key=True), + Column( "create_time", DateTime, default=now ), + Column( "update_time", DateTime, default=now, onupdate=now ), + Column( "request_id", Integer, ForeignKey( "request.id" ), index=True ), + Column( "state", TrimmedString( 255 ), index=True ), + Column( "comment", TEXT ) ) + +def upgrade(): + display_migration_details() + # Load existing tables + metadata.reflect() + # Add new request_event table + try: + RequestEvent_table.create() + except Exception, e: + log.debug( "Creating request_event table failed: %s" % str( e ) ) + # move the current state of all existing requests to the request_event table + cmd = \ + "INSERT INTO request_event " + \ + "SELECT %s AS id," + \ + "%s AS create_time," + \ + "%s AS update_time," + \ + "request.id AS request_id," + \ + "request.state AS state " + \ + "FROM request;" + cmd = cmd % ( nextval('request_event'), localtimestamp(), localtimestamp() ) + db_session.execute( cmd ) + + # Delete the state column + try: + Request_table = Table( "request", metadata, autoload=True ) + except NoSuchTableError: + Request_table = None + log.debug( "Failed loading table request" ) + if Request_table: + try: + Request_table.c.state.drop() + except Exception, e: + log.debug( "Deleting column 'state' to request table failed: %s" % ( str( e ) ) ) + +def downgrade(): + pass \ No newline at end of file diff -r 5761948422a8 -r 8d149264b777 lib/galaxy/web/controllers/requests.py --- a/lib/galaxy/web/controllers/requests.py Tue Nov 24 14:53:19 2009 -0500 +++ b/lib/galaxy/web/controllers/requests.py Wed Nov 25 14:06:28 2009 -0500 @@ -28,27 +28,57 @@ return request.type.name class LastUpdateColumn( grids.TextColumn ): def get_value(self, trans, grid, request): - return request.update_time + delta = datetime.utcnow() - request.update_time + if delta > timedelta( minutes=60 ): + last_update = '%s hours' % int( delta.seconds / 60 / 60 ) + else: + last_update = '%s minutes' % int( delta.seconds / 60 ) + return last_update class StateColumn( grids.GridColumn ): + def __init__( self, col_name, key, model_class, event_class, filterable, link ): + grids.GridColumn.__init__(self, col_name, key=key, model_class=model_class, filterable=filterable, link=link) + self.event_class = event_class + def get_value(self, trans, grid, request): + if request.state() == request.states.REJECTED: + return '<div class="count-box state-color-error">%s</div>' % request.state() + elif request.state() == request.states.NEW: + return '<div class="count-box state-color-queued">%s</div>' % request.state() + elif request.state() == request.states.SUBMITTED: + return '<div class="count-box state-color-running">%s</div>' % request.state() + elif request.state() == request.states.COMPLETE: + return '<div class="count-box state-color-ok">%s</div>' % request.state() + return request.state() def filter( self, db_session, query, column_filter ): """ Modify query to filter request by state. """ if column_filter == "All": return query if column_filter: - query = query.filter( model.Request.state == column_filter ) - return query + # select r.id, r.name, re.id, re.state + # from request as r, request_event as re + # where re.request_id=r.id and re.state='Complete' and re.create_time in + # (select MAX( create_time) + # from request_event + # group by request_id) + q = query.join(self.event_class.table)\ + .filter( self.model_class.table.c.id==self.event_class.table.c.request_id )\ + .filter( self.event_class.table.c.state==column_filter )\ + .filter( self.event_class.table.c.id.in_(select(columns=[func.max(self.event_class.table.c.id)], + from_obj=self.event_class.table, + group_by=self.event_class.table.c.request_id))) + return q def get_accepted_filters( self ): - """ Returns a list of accepted filters for this column. """ - accepted_filter_labels_and_vals = [ model.Request.states.UNSUBMITTED, - model.Request.states.SUBMITTED, - model.Request.states.COMPLETE, - "All"] - accepted_filters = [] - for val in accepted_filter_labels_and_vals: - label = val.lower() - args = { self.key: val } - accepted_filters.append( grids.GridColumnFilter( label, args) ) - return accepted_filters + """ Returns a list of accepted filters for this column. """ + accepted_filter_labels_and_vals = [ model.Request.states.NEW, + model.Request.states.REJECTED, + model.Request.states.SUBMITTED, + model.Request.states.COMPLETE, + "All"] + accepted_filters = [] + for val in accepted_filter_labels_and_vals: + label = val.lower() + args = { self.key: val } + accepted_filters.append( grids.GridColumnFilter( label, args) ) + return accepted_filters class DeletedColumn( grids.GridColumn ): def get_accepted_filters( self ): """ Returns a list of accepted filters for this column. """ @@ -66,7 +96,7 @@ num_rows_per_page = 50 preserve_state = True use_paging = True - default_filter = dict( deleted="False", state=model.Request.states.UNSUBMITTED) + default_filter = dict( deleted="False", state=model.Request.states.NEW) columns = [ NameColumn( "Name", key="name", @@ -83,23 +113,27 @@ TypeColumn( "Type" ), LastUpdateColumn( "Last update", format=time_ago ), - StateColumn( "State", - key='state', - filterable="advanced"), DeletedColumn( "Deleted", key="deleted", - visible=True, - filterable="advanced" ) + visible=False, + filterable="advanced" ), + StateColumn( "State", + model_class=model.Request, + event_class=model.RequestEvent, + key='state', + filterable="advanced", + link=( lambda item: iff( item.deleted, None, dict( operation="events", id=item.id ) ) ) ) ] columns.append( grids.MulticolFilterColumn( "Search", - cols_to_filter=[ columns[0], columns[1] ], + cols_to_filter=[ columns[0], columns[1], columns[6] ], key="free-text-search", visible=False, filterable="standard" ) ) operations = [ - grids.GridOperation( "Submit", allow_multiple=False, condition=( lambda item: not item.deleted and item.unsubmitted() and item.samples ) ), + grids.GridOperation( "Submit", allow_multiple=False, condition=( lambda item: not item.deleted and item.unsubmitted() and item.samples ), + confirm="More samples cannot be added to this request once it is submitted. Click OK to submit." ), grids.GridOperation( "Edit", allow_multiple=False, condition=( lambda item: not item.deleted and item.unsubmitted() ) ), - grids.GridOperation( "Delete", allow_multiple=True, condition=( lambda item: not item.deleted and item.unsubmitted() ) ), + grids.GridOperation( "Delete", allow_multiple=True, condition=( lambda item: not item.deleted and item.new() ) ), grids.GridOperation( "Undelete", allow_multiple=True, condition=( lambda item: item.deleted ) ) ] @@ -145,9 +179,23 @@ return self.__undelete_request( trans, **kwd ) elif operation == "edit": return self.__edit_request( trans, **kwd ) + elif operation == "events": + return self.__request_events( trans, **kwd ) + # if there are one or more requests that has been rejected by the admin + # recently, then show a msg as a reminder to the user + rlist = trans.sa_session.query( trans.app.model.Request ) \ + .filter( trans.app.model.Request.table.c.deleted==False ) \ + .filter( trans.app.model.Request.table.c.user_id==trans.user.id ) + rejected = 0 + for r in rlist: + if r.rejected(): + rejected = rejected + 1 + if rejected: + kwd['status'] = 'warning' + kwd['message'] = "%d requests (highlighted in red) were rejected, click on the request name for details." \ + % rejected # Render the list view return self.request_grid( trans, **kwd ) - def __show_request(self, trans, **kwd): params = util.Params( kwd ) msg = util.restore_text( params.get( 'msg', '' ) ) @@ -172,6 +220,28 @@ sample_copy=self.__copy_sample(current_samples), details='hide', edit_mode='False', msg=msg, messagetype=messagetype ) + def __request_events(self, trans, **kwd): + try: + request = trans.sa_session.query( trans.app.model.Request ).get( trans.security.decode_id(kwd['id']) ) + except: + msg = "Invalid request ID" + log.warn( msg ) + return trans.response.send_redirect( web.url_for( controller='requests', + action='list', + status='error', + message=msg, + **kwd) ) + events_list = [] + all_events = request.events + for event in all_events: + delta = datetime.utcnow() - event.update_time + if delta > timedelta( minutes=60 ): + last_update = '%s hours' % int( delta.seconds / 60 / 60 ) + else: + last_update = '%s minutes' % int( delta.seconds / 60 ) + events_list.append((event.state, last_update, event.comment)) + return trans.fill_template( '/requests/events.mako', + events_list=events_list, request=request) def request_details(self, trans, id): ''' Shows the request details @@ -187,7 +257,7 @@ value=request.type.name, helptext='')) request_details.append(dict(label='State', - value=request.state, + value=request.state(), helptext='')) request_details.append(dict(label='Date created', value=request.create_time, @@ -366,9 +436,9 @@ trans.sa_session.add( sample ) trans.sa_session.flush() return trans.response.send_redirect( web.url_for( controller='requests', - action='list', - operation='show_request', - id=trans.security.encode_id(request.id)) ) + action='list', + operation='show_request', + id=trans.security.encode_id(request.id)) ) elif params.get('edit_samples_button', False) == 'Edit samples': edit_mode = 'True' return trans.fill_template( '/requests/show_request.mako', @@ -700,10 +770,15 @@ if not request: request = trans.app.model.Request(name, desc, request_type, trans.user, form_values, - library=library, folder=folder, - state=trans.app.model.Request.states.UNSUBMITTED) + library=library, folder=folder) trans.sa_session.add( request ) trans.sa_session.flush() + trans.sa_session.refresh( request ) + # create an event with state 'New' for this new request + comments = "Request created." + event = trans.app.model.RequestEvent(request, request.states.NEW, comments) + trans.sa_session.add( event ) + trans.sa_session.flush() else: request.name = name request.desc = desc @@ -712,7 +787,6 @@ request.values = form_values request.library = library request.folder = folder - request.state = trans.app.model.Request.states.UNSUBMITTED trans.sa_session.add( request ) trans.sa_session.flush() return request @@ -731,7 +805,7 @@ message="Invalid request ID", **kwd) ) if params.get('show', False) == 'True': - return self.__edit_request(trans, **kwd) + return self.__edit_request(trans, id=trans.security.encode_id(request.id), **kwd) elif params.get('save_changes_request_button', False) == 'Save changes' \ or params.get('edit_samples_button', False) == 'Edit samples': request_type = trans.sa_session.query( trans.app.model.RequestType ).get( int( params.select_request_type ) ) @@ -760,7 +834,7 @@ messagetype='done', **new_kwd) ) elif params.get('refresh', False) == 'true': - return self.__edit_request(trans, **kwd) + return self.__edit_request(trans, id=trans.security.encode_id(request.id), **kwd) def __edit_request(self, trans, **kwd): try: @@ -819,7 +893,7 @@ message=msg, **kwd) ) # a request cannot be deleted once its submitted - if not request.unsubmitted(): + if not request.new(): delete_failed.append(request.name) else: request.deleted = True @@ -870,21 +944,22 @@ msg = self.__validate(trans, request) if msg: return trans.response.send_redirect( web.url_for( controller='requests', - action='edit', + action='list', + operation='edit', messagetype = 'error', msg=msg, - request_id=request.id, - show='True') ) + id=trans.security.encode_id(request.id) )) + # change the request state to 'Submitted' + comments = "Request moved to 'Submitted' state." + event = trans.app.model.RequestEvent(request, request.states.SUBMITTED, comments) + trans.sa_session.add( event ) + trans.sa_session.flush() # get the new state new_state = request.type.states[0] for s in request.samples: event = trans.app.model.SampleEvent(s, new_state, 'Samples submitted to the system') trans.sa_session.add( event ) trans.sa_session.flush() - # change request's submitted field - request.state = request.states.SUBMITTED - trans.sa_session.add( request ) - trans.sa_session.flush() return trans.response.send_redirect( web.url_for( controller='requests', action='list', id=trans.security.encode_id(request.id), diff -r 5761948422a8 -r 8d149264b777 lib/galaxy/web/controllers/requests_admin.py --- a/lib/galaxy/web/controllers/requests_admin.py Tue Nov 24 14:53:19 2009 -0500 +++ b/lib/galaxy/web/controllers/requests_admin.py Wed Nov 25 14:06:28 2009 -0500 @@ -8,6 +8,8 @@ from galaxy.web.form_builder import * from datetime import datetime, timedelta from galaxy.web.controllers.forms import get_all_forms +from sqlalchemy.sql.expression import func, and_ +from sqlalchemy.sql import select log = logging.getLogger( __name__ ) @@ -31,27 +33,58 @@ return request.type.name class LastUpdateColumn( grids.TextColumn ): def get_value(self, trans, grid, request): - return request.update_time + delta = datetime.utcnow() - request.update_time + if delta > timedelta( minutes=60 ): + last_update = '%s hours' % int( delta.seconds / 60 / 60 ) + else: + last_update = '%s minutes' % int( delta.seconds / 60 ) + return last_update class StateColumn( grids.GridColumn ): + def __init__( self, col_name, key, model_class, event_class, filterable, link ): + grids.GridColumn.__init__(self, col_name, key=key, model_class=model_class, filterable=filterable, link=link) + self.event_class = event_class + def get_value(self, trans, grid, request): + if request.state() == request.states.REJECTED: + return '<div class="count-box state-color-error">%s</div>' % request.state() + elif request.state() == request.states.NEW: + return '<div class="count-box state-color-queued">%s</div>' % request.state() + elif request.state() == request.states.SUBMITTED: + return '<div class="count-box state-color-running">%s</div>' % request.state() + elif request.state() == request.states.COMPLETE: + return '<div class="count-box state-color-ok">%s</div>' % request.state() + return request.state() def filter( self, db_session, query, column_filter ): """ Modify query to filter request by state. """ if column_filter == "All": return query if column_filter: - query = query.filter( model.Request.state == column_filter ) - return query + # select r.id, r.name, re.id, re.state + # from request as r, request_event as re + # where re.request_id=r.id and re.state='Complete' and re.create_time in + # (select MAX( create_time) + # from request_event + # group by request_id) + q = query.join(self.event_class.table)\ + .filter( self.model_class.table.c.id==self.event_class.table.c.request_id )\ + .filter( self.event_class.table.c.state==column_filter )\ + .filter( self.event_class.table.c.id.in_(select(columns=[func.max(self.event_class.table.c.id)], + from_obj=self.event_class.table, + group_by=self.event_class.table.c.request_id))) + #print column_filter, q + return q def get_accepted_filters( self ): - """ Returns a list of accepted filters for this column. """ - accepted_filter_labels_and_vals = [ model.Request.states.UNSUBMITTED, - model.Request.states.SUBMITTED, - model.Request.states.COMPLETE, - "All"] - accepted_filters = [] - for val in accepted_filter_labels_and_vals: - label = val.lower() - args = { self.key: val } - accepted_filters.append( grids.GridColumnFilter( label, args) ) - return accepted_filters + """ Returns a list of accepted filters for this column. """ + accepted_filter_labels_and_vals = [ model.Request.states.NEW, + model.Request.states.REJECTED, + model.Request.states.SUBMITTED, + model.Request.states.COMPLETE, + "All"] + accepted_filters = [] + for val in accepted_filter_labels_and_vals: + label = val.lower() + args = { self.key: val } + accepted_filters.append( grids.GridColumnFilter( label, args) ) + return accepted_filters class UserColumn( grids.TextColumn ): def get_value(self, trans, grid, request): return request.user.email @@ -86,20 +119,25 @@ filterable="advanced" ), SamplesColumn( "Sample(s)", link=( lambda item: iff( item.deleted, None, dict( operation="show_request", id=item.id ) ) ), ), - TypeColumn( "Type" ), + TypeColumn( "Type", + link=( lambda item: iff( item.deleted, None, dict( operation="view_type", id=item.type.id ) ) ), ), LastUpdateColumn( "Last update", format=time_ago ), - StateColumn( "State", - key='state', - filterable="advanced"), - UserColumn( "User", - key='user.email', - model_class=model.Request, - filterable="advanced" ), DeletedColumn( "Deleted", key="deleted", - visible=True, - filterable="advanced" ) + visible=False, + filterable="advanced" ), + StateColumn( "State", + model_class=model.Request, + event_class=model.RequestEvent, + key='state', + filterable="advanced", + link=( lambda item: iff( item.deleted, None, dict( operation="events", id=item.id ) ) ), + ), + UserColumn( "User", + #key='user.email', + model_class=model.Request) + ] columns.append( grids.MulticolFilterColumn( "Search", cols_to_filter=[ columns[0], columns[1], columns[6] ], @@ -107,10 +145,11 @@ visible=False, filterable="standard" ) ) operations = [ - grids.GridOperation( "Submit", allow_multiple=False, condition=( lambda item: not item.deleted and item.unsubmitted() and item.samples ) ), + grids.GridOperation( "Submit", allow_multiple=False, condition=( lambda item: not item.deleted and item.unsubmitted() and item.samples ), + confirm="More samples cannot be added to this request once it is submitted. Click OK to submit." ), grids.GridOperation( "Edit", allow_multiple=False, condition=( lambda item: not item.deleted ) ), grids.GridOperation( "Reject", allow_multiple=False, condition=( lambda item: not item.deleted and item.submitted() ) ), - grids.GridOperation( "Delete", allow_multiple=True, condition=( lambda item: not item.deleted and item.unsubmitted() ) ), + grids.GridOperation( "Delete", allow_multiple=True, condition=( lambda item: not item.deleted and item.new() ) ), grids.GridOperation( "Undelete", condition=( lambda item: item.deleted ) ), ] global_actions = [ @@ -227,6 +266,11 @@ return self.__edit_request( trans, **kwd ) elif operation == "reject": return self.__reject_request( trans, **kwd ) + elif operation == "events": + return self.__request_events( trans, **kwd ) + elif operation == "view_type": + return self.__view_request_type( trans, **kwd ) + # Render the grid view return self.request_grid( trans, **kwd ) def __show_request(self, trans, **kwd): @@ -253,7 +297,52 @@ sample_copy=self.__copy_sample(current_samples), details='hide', edit_mode='False', msg=msg, messagetype=messagetype ) - + @web.expose + @web.require_admin + def edit(self, trans, **kwd): + params = util.Params( kwd ) + msg = util.restore_text( params.get( 'msg', '' ) ) + messagetype = params.get( 'messagetype', 'done' ) + try: + request = trans.sa_session.query( trans.app.model.Request ).get( int( params.get( 'request_id', None ) ) ) + except: + return trans.response.send_redirect( web.url_for( controller='requests_admin', + action='list', + status='error', + message="Invalid request ID", + **kwd) ) + if params.get('show', False) == 'True': + return self.__edit_request(trans, id=trans.security.encode_id(request.id), **kwd) + elif params.get('save_changes_request_button', False) == 'Save changes' \ + or params.get('edit_samples_button', False) == 'Edit samples': + request_type = trans.sa_session.query( trans.app.model.RequestType ).get( int( params.select_request_type ) ) + if not util.restore_text(params.get('name', '')): + msg = 'Please enter the <b>Name</b> of the request' + kwd['messagetype'] = 'error' + kwd['msg'] = msg + kwd['show'] = 'True' + return trans.response.send_redirect( web.url_for( controller='requests_admin', + action='edit', + **kwd) ) + request = self.__save_request(trans, request, **kwd) + msg = 'The changes made to the request named %s has been saved' % request.name + if params.get('save_changes_request_button', False) == 'Save changes': + return trans.response.send_redirect( web.url_for( controller='requests_admin', + action='list', + message=msg , + status='done') ) + elif params.get('edit_samples_button', False) == 'Edit samples': + new_kwd = {} + new_kwd['request_id'] = request.id + new_kwd['edit_samples_button'] = 'Edit samples' + return trans.response.send_redirect( web.url_for( controller='requests_admin', + action='show_request', + msg=msg , + messagetype='done', + **new_kwd) ) + elif params.get('refresh', False) == 'true': + return self.__edit_request(trans, id=trans.security.encode_id(request.id), **kwd) + def __edit_request(self, trans, **kwd): try: request = trans.sa_session.query( trans.app.model.Request ).get( trans.security.decode_id(kwd['id']) ) @@ -311,7 +400,7 @@ message=msg, **kwd) ) # a request cannot be deleted once its submitted - if not request.unsubmitted(): + if not request.new(): delete_failed.append(request.name) else: request.deleted = True @@ -362,18 +451,24 @@ msg = self.__validate(trans, request) if msg: return trans.response.send_redirect( web.url_for( controller='requests_admin', - action='edit', + action='list', + operation='edit', messagetype = 'error', msg=msg, - request_id=request.id, - show='True') ) - # get the new state + id=trans.security.encode_id(request.id) ) ) + # change the request state to 'Submitted' + if request.user.email is not trans.user: + comments = "Request moved to 'Submitted' state by admin (%s) on behalf of %s." % (trans.user.email, request.user.email) + else: + comments = "" + event = trans.app.model.RequestEvent(request, request.states.SUBMITTED, comments) + trans.sa_session.add( event ) + trans.sa_session.flush() + # change the state of each of the samples of thus request new_state = request.type.states[0] for s in request.samples: event = trans.app.model.SampleEvent(s, new_state, 'Samples submitted to the system') trans.sa_session.add( event ) - # change request's submitted field - request.state = request.states.SUBMITTED trans.sa_session.add( request ) trans.sa_session.flush() return trans.response.send_redirect( web.url_for( controller='requests_admin', @@ -393,15 +488,64 @@ status='error', message=msg, **kwd) ) - # change request's submitted field - request.state = request.states.UNSUBMITTED - trans.sa_session.add( request ) + return trans.fill_template( '/admin/requests/reject.mako', + request=request) + @web.expose + @web.require_admin + def reject(self, trans, **kwd): + params = util.Params( kwd ) + if params.get('cancel_reject_button', False): + return trans.response.send_redirect( web.url_for( controller='requests_admin', + action='list', + operation='show_request', + id=kwd['id'])) + try: + request = trans.sa_session.query( trans.app.model.Request ).get( trans.security.decode_id(kwd['id']) ) + except: + msg = "Invalid request ID" + log.warn( msg ) + return trans.response.send_redirect( web.url_for( controller='requests_admin', + action='list', + status='error', + message=msg, + **kwd) ) + # validate + if not params.get('comment', ''): + return trans.fill_template( '/admin/requests/reject.mako', + request=request, messagetype='error', + msg='A comment is required for rejecting a request.') + # create an event with state 'Rejected' for this request + comments = util.restore_text( params.comment ) + event = trans.app.model.RequestEvent(request, request.states.REJECTED, comments) + trans.sa_session.add( event ) trans.sa_session.flush() return trans.response.send_redirect( web.url_for( controller='requests_admin', action='list', status='done', - message='The request <b>%s</b> is now unsubmitted.' % request.name - ) ) + message='Request <b>%s</b> has been rejected.' % request.name) ) + + def __request_events(self, trans, **kwd): + try: + request = trans.sa_session.query( trans.app.model.Request ).get( trans.security.decode_id(kwd['id']) ) + except: + msg = "Invalid request ID" + log.warn( msg ) + return trans.response.send_redirect( web.url_for( controller='requests_admin', + action='list', + status='error', + message=msg, + **kwd) ) + events_list = [] + all_events = request.events + for event in all_events: + delta = datetime.utcnow() - event.update_time + if delta > timedelta( minutes=60 ): + last_update = '%s hours' % int( delta.seconds / 60 / 60 ) + else: + last_update = '%s minutes' % int( delta.seconds / 60 ) + events_list.append((event.state, last_update, event.comment)) + return trans.fill_template( '/admin/requests/events.mako', + events_list=events_list, request=request) # #---- Request Creation ---------------------------------------------------------- # @@ -732,9 +876,18 @@ if not request: request = trans.app.model.Request(name, desc, request_type, user, form_values, - library=library, folder=folder, - state=trans.app.model.Request.states.UNSUBMITTED) + library=library, folder=folder) trans.sa_session.add( request ) + trans.sa_session.flush() + trans.sa_session.refresh( request ) + # create an event with state 'New' for this new request + if request.user.email is not trans.user: + comments = "Request created by admin (%s) on behalf of %s." % (trans.user.email, request.user.email) + else: + comments = "Request created." + event = trans.app.model.RequestEvent(request, request.states.NEW, comments) + trans.sa_session.add( event ) + trans.sa_session.flush() else: request.name = name request.desc = desc @@ -744,58 +897,14 @@ request.library = library request.folder = folder trans.sa_session.add( request ) - trans.sa_session.flush() + trans.sa_session.flush() + return request # -#---- Request Editing ---------------------------------------------------------- +#---- Request Page ---------------------------------------------------------- # - @web.expose - @web.require_admin - def edit(self, trans, **kwd): - params = util.Params( kwd ) - msg = util.restore_text( params.get( 'msg', '' ) ) - messagetype = params.get( 'messagetype', 'done' ) - try: - request = trans.sa_session.query( trans.app.model.Request ).get( int( params.get( 'request_id', None ) ) ) - except: - return trans.response.send_redirect( web.url_for( controller='requests_admin', - action='list', - status='error', - message="Invalid request ID", - **kwd) ) - if params.get('show', False) == 'True': - return self.__edit_request(trans, request.id, **kwd) - elif params.get('save_changes_request_button', False) == 'Save changes' \ - or params.get('edit_samples_button', False) == 'Edit samples': - request_type = trans.sa_session.query( trans.app.model.RequestType ).get( int( params.select_request_type ) ) - if not util.restore_text(params.get('name', '')): - msg = 'Please enter the <b>Name</b> of the request' - kwd['messagetype'] = 'error' - kwd['msg'] = msg - kwd['show'] = 'True' - return trans.response.send_redirect( web.url_for( controller='requests_admin', - action='edit', - **kwd) ) - request = self.__save_request(trans, request, **kwd) - msg = 'The changes made to the request named %s has been saved' % request.name - if params.get('save_changes_request_button', False) == 'Save changes': - return trans.response.send_redirect( web.url_for( controller='requests_admin', - action='list', - message=msg , - status='done') ) - elif params.get('edit_samples_button', False) == 'Edit samples': - new_kwd = {} - new_kwd['request_id'] = request.id - new_kwd['edit_samples_button'] = 'Edit samples' - return trans.response.send_redirect( web.url_for( controller='requests_admin', - action='show_request', - msg=msg , - messagetype='done', - **new_kwd) ) - elif params.get('refresh', False) == 'true': - return self.__edit_request(trans, request.id, **kwd) def __update_samples(self, request, **kwd): ''' This method retrieves all the user entered sample information and @@ -1013,7 +1122,7 @@ value=request.type.name, helptext='')) request_details.append(dict(label='State', - value=request.state, + value=request.state(), helptext='')) request_details.append(dict(label='Date created', value=request.create_time, @@ -1167,11 +1276,13 @@ if s.current_state().id != request.type.states[-1].id: complete = False if complete: - request.state = request.states.COMPLETE - else: - request.state = request.states.SUBMITTED - trans.sa_session.add( request ) - trans.sa_session.flush() + # change the request state to 'Complete' + comments = "All samples of this request are in the last sample state (%s)." % request.type.states[-1].name + event = trans.app.model.RequestEvent(request, request.states.COMPLETE, comments) + trans.sa_session.add( event ) + trans.sa_session.flush() +# trans.sa_session.add( request ) +# trans.sa_session.flush() def change_state(self, trans, sample): possible_states = sample.request.type.states curr_state = sample.current_state() diff -r 5761948422a8 -r 8d149264b777 lib/galaxy/web/framework/helpers/grids.py --- a/lib/galaxy/web/framework/helpers/grids.py Tue Nov 24 14:53:19 2009 -0500 +++ b/lib/galaxy/web/framework/helpers/grids.py Wed Nov 25 14:06:28 2009 -0500 @@ -402,7 +402,7 @@ return query.filter( complete_filter ) class GridOperation( object ): - def __init__( self, label, key=None, condition=None, allow_multiple=True, allow_popup=True, target=None, url_args=None, async_compatible=False ): + def __init__( self, label, key=None, condition=None, allow_multiple=True, allow_popup=True, target=None, url_args=None, async_compatible=False, confirm=None ): self.label = label self.key = key self.allow_multiple = allow_multiple @@ -411,6 +411,8 @@ self.target = target self.url_args = url_args self.async_compatible = async_compatible + # if 'confirm' is set, then ask before completing the operation + self.confirm = confirm def get_url_args( self, item ): if self.url_args: temp = dict( self.url_args ) diff -r 5761948422a8 -r 8d149264b777 templates/admin/requests/events.mako --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/templates/admin/requests/events.mako Wed Nov 25 14:06:28 2009 -0500 @@ -0,0 +1,36 @@ +<%inherit file="/base.mako"/> +<%namespace file="/message.mako" import="render_msg" /> + +<h2>History of Sequencing Request "${request.name}"</h2> +<ul class="manage-table-actions"> + <li> + <a class="action-button" href="${h.url_for( controller='requests_admin', action='list', operation='show_request', id=trans.security.encode_id(request.id) )}"> + <span>Browse this request</span></a> + </li> +</ul> +<h3>User: ${request.user.email}</h3> + +%if msg: + ${render_msg( msg, messagetype )} +%endif + +<div class="toolForm"> + <table class="grid"> + <thead> + <tr> + <th>State</th> + <th>Last Update</th> + <th>Comments</th> + </tr> + </thead> + <tbody> + %for state, updated, comments in events_list: + <tr class="libraryRow libraryOrFolderRow" id="libraryRow"> + <td><b><a>${state}</a></b></td> + <td><a>${updated}</a></td> + <td><a>${comments}</a></td> + </tr> + %endfor + </tbody> + </table> +</div> diff -r 5761948422a8 -r 8d149264b777 templates/admin/requests/reject.mako --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/templates/admin/requests/reject.mako Wed Nov 25 14:06:28 2009 -0500 @@ -0,0 +1,42 @@ +<%inherit file="/base.mako"/> +<%namespace file="/message.mako" import="render_msg" /> + +%if msg: + ${render_msg( msg, messagetype )} +%endif + +<h2>Reject Sequencing Request "${request.name}"</h2> +<ul class="manage-table-actions"> + <li> + <a class="action-button" href="${h.url_for( controller='requests_admin', action='list', operation='show_request', id=trans.security.encode_id(request.id) )}"> + <span>Browse this request</span></a> + </li> + <li> + <a class="action-button" href="${h.url_for( controller='requests_admin', action='list', operation='events', id=trans.security.encode_id(request.id) )}"> + <span>Events</span></a> + </li> +</ul> +<h3>User: ${request.user.email}</h3> + + + +<div class="toolForm"> + <div class="toolFormTitle">Reject request</div> + <form name="event" action="${h.url_for( controller='requests_admin', action='reject', id=trans.security.encode_id(request.id))}" method="post" > + <div class="form-row"> + Rejecting this request will move the request state to <b>Rejected</b>. + </div> + <div class="form-row"> + <label>Comments</label> + <textarea name="comment" rows="5" cols="40"></textarea> + <div class="toolParamHelp" style="clear: both;"> + Required + </div> + </div> + <div class="form-row"> + <input type="submit" name="reject_button" value="Reject"/> + <input type="submit" name="cancel_reject_button" value="Cancel"/> + </div> + </form> + </div> +</div> \ No newline at end of file diff -r 5761948422a8 -r 8d149264b777 templates/admin/requests/show_request.mako --- a/templates/admin/requests/show_request.mako Tue Nov 24 14:53:19 2009 -0500 +++ b/templates/admin/requests/show_request.mako Wed Nov 25 14:06:28 2009 -0500 @@ -5,11 +5,16 @@ ${render_msg( msg, messagetype )} %endif +%if request.rejected(): + ${render_msg( "Reason for rejection: "+request.last_comment(), "warning" )} +%endif + <div class="grid-header"> <h2>Sequencing Request "${request.name}"</h2> </div> <ul class="manage-table-actions"> + %if request.unsubmitted() and request.samples: <li> <a class="action-button" confirm="More samples cannot be added to this request once it is submitted. Click OK to submit." href="${h.url_for( controller='requests_admin', action='list', operation='Submit', id=trans.security.encode_id(request.id) )}"> @@ -28,6 +33,10 @@ <span>Bar codes</span></a> </li> %endif + <li> + <a class="action-button" href="${h.url_for( controller='requests_admin', action='list', operation='events', id=trans.security.encode_id(request.id) )}"> + <span>History</span></a> + </li> </ul> @@ -82,7 +91,7 @@ %if grid_index == 0: <td>${sample.bar_code}</td> <td> - %if sample.request.unsubmitted(): + %if sample.request.new(): Unsubmitted %else: <a href="${h.url_for( controller='requests_admin', action='show_events', sample_id=sample.id)}">${sample.current_state().name}</a> @@ -117,6 +126,8 @@ %else: <i>None</i> %endif + %elif rd['label'] == 'State': + <a href="${h.url_for( controller='requests_admin', action='list', operation='events', id=trans.security.encode_id(request.id) )}">${rd['value']}</a> %else: ${rd['value']} %endif diff -r 5761948422a8 -r 8d149264b777 templates/grid_base.mako --- a/templates/grid_base.mako Tue Nov 24 14:53:19 2009 -0500 +++ b/templates/grid_base.mako Wed Nov 25 14:06:28 2009 -0500 @@ -860,7 +860,11 @@ if operation.target: target = "target='" + operation.target + "'" %> - <a class="action-button" ${target} href="${ url( **operation.get_url_args( item ) ) }">${operation.label}</a> + %if operation.confirm: + <a class="action-button" ${target} confirm="${operation.confirm}" href="${ url( **operation.get_url_args( item ) ) }">${operation.label}</a> + %else: + <a class="action-button" ${target} href="${ url( **operation.get_url_args( item ) ) }">${operation.label}</a> + %endif %endif %endfor </div> diff -r 5761948422a8 -r 8d149264b777 templates/requests/events.mako --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/templates/requests/events.mako Wed Nov 25 14:06:28 2009 -0500 @@ -0,0 +1,39 @@ +<%inherit file="/base.mako"/> +<%namespace file="/message.mako" import="render_msg" /> + +<h2>History of Sequencing Request "${request.name}"</h2> +<ul class="manage-table-actions"> + <li> + <a class="action-button" href="${h.url_for( controller='requests', action='list', operation='show_request', id=trans.security.encode_id(request.id) )}"> + <span>Browse this request</span></a> + </li> + <li> + <a class="action-button" href="${h.url_for( controller='requests', action='list')}"> + <span>Browse all requests</span></a> + </li> +</ul> + +%if msg: + ${render_msg( msg, messagetype )} +%endif + +<div class="toolForm"> + <table class="grid"> + <thead> + <tr> + <th>State</th> + <th>Last Update</th> + <th>Comments</th> + </tr> + </thead> + <tbody> + %for state, updated, comments in events_list: + <tr class="libraryRow libraryOrFolderRow" id="libraryRow"> + <td><b><a>${state}</a></b></td> + <td><a>${updated}</a></td> + <td><a>${comments}</a></td> + </tr> + %endfor + </tbody> + </table> +</div> diff -r 5761948422a8 -r 8d149264b777 templates/requests/show_request.mako --- a/templates/requests/show_request.mako Tue Nov 24 14:53:19 2009 -0500 +++ b/templates/requests/show_request.mako Wed Nov 25 14:06:28 2009 -0500 @@ -6,6 +6,9 @@ ${render_msg( msg, messagetype )} %endif +%if request.rejected(): + ${render_msg( "Reason for rejection: "+request.last_comment(), "warning" )} +%endif <div class="grid-header"> <h2>Sequencing Request "${request.name}"</h2> @@ -19,6 +22,10 @@ </li> %endif <li> + <a class="action-button" href="${h.url_for( controller='requests', action='list', operation='events', id=trans.security.encode_id(request.id) )}"> + <span>History</span></a> + </li> + <li> <a class="action-button" href="${h.url_for( controller='requests', action='list')}"> <span>Browse requests</span></a> </li> @@ -73,7 +80,7 @@ </td> %if grid_index == 0: <td> - %if sample.request.unsubmitted(): + %if sample.request.new(): Unsubmitted %else: <a href="${h.url_for( controller='requests', action='show_events', sample_id=sample.id)}">${sample.current_state().name}</a> @@ -105,9 +112,11 @@ %if rd['label'] == 'Data library': %if rd['value']: <a href="${h.url_for( controller='library', action='browse_library', obj_id=request.library.id )}">${rd['value']}</a> - %else: - <i>None</i> - %endif + %elif rd['label'] == 'State': + <a href="${h.url_for( controller='requests', action='list', operation='events', id=trans.security.encode_id(request.id) )}">${rd['value']}</a> + %else: + <i>None</i> + %endif %else: ${rd['value']} %endif diff -r 5761948422a8 -r 8d149264b777 templates/sample/sample_events.mako --- a/templates/sample/sample_events.mako Tue Nov 24 14:53:19 2009 -0500 +++ b/templates/sample/sample_events.mako Wed Nov 25 14:06:28 2009 -0500 @@ -21,24 +21,25 @@ ${render_msg( msg, messagetype )} %endif - -<table class="grid"> - <thead> - <tr> - <th>State</th> - <th>Description</th> - <th>Updated</th> - <th>Comments</th> - </tr> - </thead> - <tbody> - %for state, desc, updated, comments in events_list: - <tr class="libraryRow libraryOrFolderRow" id="libraryRow"> - <td><b><a>${state}</a></b></td> - <td><a>${desc}</a></td> - <td><a>${updated}</a></td> - <td><a>${comments}</a></td> - </tr> - %endfor - </tbody> -</table> \ No newline at end of file +<div class="toolForm"> + <table class="grid"> + <thead> + <tr> + <th>State</th> + <th>Description</th> + <th>Updated</th> + <th>Comments</th> + </tr> + </thead> + <tbody> + %for state, desc, updated, comments in events_list: + <tr class="libraryRow libraryOrFolderRow" id="libraryRow"> + <td><b><a>${state}</a></b></td> + <td><a>${desc}</a></td> + <td><a>${updated}</a></td> + <td><a>${comments}</a></td> + </tr> + %endfor + </tbody> + </table> +</div> \ No newline at end of file diff -r 5761948422a8 -r 8d149264b777 test/base/twilltestcase.py --- a/test/base/twilltestcase.py Tue Nov 24 14:53:19 2009 -0500 +++ b/test/base/twilltestcase.py Wed Nov 25 14:06:28 2009 -0500 @@ -1298,10 +1298,15 @@ self.home() self.visit_url( "%s/requests_admin/list?operation=Submit&id=%s" % ( self.url, self.security.encode_id( request_id ) )) self.check_page_for_string( 'The request <b>%s</b> has been submitted.' % request_name ) - def reject_request( self, request_id, request_name ): + def reject_request( self, request_id, request_name, comment ): self.home() self.visit_url( "%s/requests_admin/list?operation=Reject&id=%s" % ( self.url, self.security.encode_id( request_id ) )) - self.check_page_for_string( 'The request <b>%s</b> is now unsubmitted.' % request_name ) + self.check_page_for_string( 'Reject Sequencing Request "%s"' % request_name ) + tc.fv( "1", "comment", comment ) + tc.submit( "reject_button" ) + self.check_page_for_string( 'Request <b>%s</b> has been rejected.' % request_name ) + self.visit_url( "%s/requests/list?sort=-create_time&operation=show_request&id=%s" % ( self.url, self.security.encode_id( request_id ) )) + self.check_page_for_string( comment ) def add_bar_codes( self, request_id, request_name, bar_codes ): self.home() self.visit_url( "%s/requests_admin/bar_codes?request_id=%i" % (self.url, request_id) ) diff -r 5761948422a8 -r 8d149264b777 test/functional/test_forms_and_requests.py --- a/test/functional/test_forms_and_requests.py Tue Nov 24 14:53:19 2009 -0500 +++ b/test/functional/test_forms_and_requests.py Wed Nov 25 14:06:28 2009 -0500 @@ -194,9 +194,9 @@ .filter( and_( galaxy.model.Request.table.c.name==request_name, galaxy.model.Request.table.c.deleted==False ) ) \ .first() - # check if the request's state is now set to 'unsubmitted' - assert request_one.state is not request_one.states.UNSUBMITTED, "The state of the request '%s' should be set to '%s'" \ - % ( request_one.name, request_one.states.UNSUBMITTED ) + # check if the request's state is now set to 'new' + assert request_one.state is not request_one.states.NEW, "The state of the request '%s' should be set to '%s'" \ + % ( request_one.name, request_one.states.NEW ) # sample fields samples = [ ( 'Sample One', [ 'S1 Field 0 Value' ] ), ( 'Sample Two', [ 'S2 Field 0 Value' ] ) ] @@ -207,8 +207,8 @@ self.edit_request(request_one.id, request_one.name, request_one.name+' (Renamed)', request_one.desc+' (Re-described)', library_one.id, folder_one.id, fields) sa_session.refresh( request_one ) - # check if the request is showing in the 'unsubmitted' filter - self.check_request_grid(state='Unsubmitted', request_name=request_one.name) + # check if the request is showing in the 'new' filter + self.check_request_grid(state='New', request_name=request_one.name) # submit the request self.submit_request( request_one.id, request_one.name ) sa_session.refresh( request_one ) @@ -257,11 +257,11 @@ .filter( and_( galaxy.model.Request.table.c.name==request_name, galaxy.model.Request.table.c.deleted==False ) ) \ .first() - # check if the request is showing in the 'unsubmitted' filter - self.check_request_admin_grid(state='Unsubmitted', request_name=request_two.name) - # check if the request's state is now set to 'unsubmitted' - assert request_two.state is not request_two.states.UNSUBMITTED, "The state of the request '%s' should be set to '%s'" \ - % ( request_two.name, request_two.states.UNSUBMITTED ) + # check if the request is showing in the 'new' filter + self.check_request_admin_grid(state='New', request_name=request_two.name) + # check if the request's state is now set to 'new' + assert request_two.state is not request_two.states.NEW, "The state of the request '%s' should be set to '%s'" \ + % ( request_two.name, request_two.states.NEW ) # sample fields samples = [ ( 'Sample One', [ 'S1 Field 0 Value' ] ), ( 'Sample Two', [ 'S2 Field 0 Value' ] ) ] @@ -282,10 +282,10 @@ '''Testing rejecting a request''' self.logout() self.login( email='test(a)bx.psu.edu' ) - self.reject_request( request_two.id, request_two.name ) + self.reject_request( request_two.id, request_two.name, "Rejection test comment" ) sa_session.refresh( request_two ) - # check if the request is showing in the 'unsubmitted' filter - self.check_request_admin_grid(state='Unsubmitted', request_name=request_two.name) + # check if the request is showing in the 'rejected' filter + self.check_request_admin_grid(state='Rejected', request_name=request_two.name) # check if the request's state is now set to 'submitted' - assert request_two.state is not request_two.states.UNSUBMITTED, "The state of the request '%s' should be set to '%s'" \ - % ( request_two.name, request_two.states.UNSUBMITTED ) + assert request_two.state is not request_two.states.REJECTED, "The state of the request '%s' should be set to '%s'" \ + % ( request_two.name, request_two.states.REJECTED )

1 0