details: http://www.bx.psu.edu/hg/galaxy/rev/e21f605d2766 changeset: 3091:e21f605d2766 user: Enis Afgan <afgane@gmail.com> date: Thu Nov 12 12:42:17 2009 -0500 description: Cleaned up code dealing with error reporting in cloud providers (EC2 and Eucalyptus). diffstat: lib/galaxy/cloud/__init__.py | 26 +- lib/galaxy/cloud/providers/ec2.py | 264 +++++++++++++-------- lib/galaxy/cloud/providers/eucalyptus.py | 267 ++++++++++++++------- lib/galaxy/model/mapping.py | 14 +- lib/galaxy/model/migrate/versions/0014_cloud_tables.py | 25 +- lib/galaxy/web/controllers/cloud.py | 6 +- 6 files changed, 376 insertions(+), 226 deletions(-) diffs (1173 lines): diff -r b90db227df6d -r e21f605d2766 lib/galaxy/cloud/__init__.py --- a/lib/galaxy/cloud/__init__.py Thu Nov 12 10:28:54 2009 -0500 +++ b/lib/galaxy/cloud/__init__.py Thu Nov 12 12:42:17 2009 -0500 @@ -21,6 +21,7 @@ uci_states = Bunch( NEW_UCI = "newUCI", NEW = "new", + CREATING = "creating", DELETING_UCI = "deletingUCI", DELETING = "deleting", DELETED = "deleted", @@ -317,6 +318,17 @@ uci.store[store_id].device = device uci.store[store_id].flush() + def set_store_error( self, error, store_index=None, store_id=None ): + if store_index != None: + store = model.CloudStore.get( store_index ) + elif store_id != None: + store = model.CloudStore.filter_by( volume_id = store_id ).first() + else: + return None + + store.error = error + store.flush() + def set_store_status( self, vol_id, status ): vol = model.CloudStore.filter( model.CloudStore.c.volume_id == vol_id ).first() vol.status = status @@ -404,6 +416,13 @@ i.state = instance_states.ERROR i.flush() uci.flush() + + def set_deleted( self ): + uci = model.UCI.get( self.uci_id ) + uci.refresh() + uci.state = uci_states.DELETED # for bookkeeping reasons, mark as deleted but don't actually delete. + uci.deleted = True + uci.flush() # --------- Getter methods ----------------- @@ -563,13 +582,6 @@ uci.refresh() return uci.launch_time - def delete( self ): - uci = model.UCI.get( self.uci_id ) - uci.refresh() - uci.state = uci_states.DELETED # for bookkeeping reasons, mark as deleted but don't actually delete. - uci.deleted = True - uci.flush() - class CloudProvider( object ): def __init__( self, app ): import providers.eucalyptus diff -r b90db227df6d -r e21f605d2766 lib/galaxy/cloud/providers/ec2.py --- a/lib/galaxy/cloud/providers/ec2.py Thu Nov 12 10:28:54 2009 -0500 +++ b/lib/galaxy/cloud/providers/ec2.py Thu Nov 12 12:42:17 2009 -0500 @@ -6,8 +6,9 @@ from galaxy.model import mapping from galaxy.datatypes.data import nice_size from galaxy.util.bunch import Bunch +from galaxy.cloud import UCIwrapper from Queue import Queue -from sqlalchemy import or_ +from sqlalchemy import or_, and_ import galaxy.eggs galaxy.eggs.require("boto") @@ -22,6 +23,7 @@ uci_states = Bunch( NEW_UCI = "newUCI", NEW = "new", + CREATING = "creating", DELETING_UCI = "deletingUCI", DELETING = "deleting", SUBMITTED_UCI = "submittedUCI", @@ -49,6 +51,7 @@ store_states = Bunch( IN_USE = "in-use", CREATING = "creating", + DELETED = 'deleted', ERROR = "error" ) @@ -69,7 +72,6 @@ def __init__( self, app ): self.type = "ec2" # cloud provider type (e.g., ec2, eucalyptus, opennebula) self.zone = "us-east-1a" - self.key_pair = "galaxy-keypair" self.security_group = "galaxyWeb" self.queue = Queue() @@ -114,9 +116,10 @@ provider = uci_wrapper.get_provider() try: region = RegionInfo( None, provider.region_name, provider.region_endpoint ) - except Exception, e: - log.error( "Selecting region with cloud provider failed: %s" % str(e) ) - uci_wrapper.set_error( "Selecting region with cloud provider failed: " + str(e), True ) + except Exception, ex: + err = "Selecting region with cloud provider failed: " + str( ex ) + log.error( err ) + uci_wrapper.set_error( err, True ) return None try: conn = EC2Connection( aws_access_key_id=uci_wrapper.get_access_key(), @@ -124,9 +127,10 @@ is_secure=provider.is_secure, region=region, path=provider.path ) - except Exception, e: - log.error( "Establishing connection with cloud failed: %s" % str(e) ) - uci_wrapper.set_error( "Establishing connection with cloud failed: " + str(e), True ) + except Exception, ex: + err = "Establishing connection with cloud failed: " + str( ex ) + log.error( err ) + uci_wrapper.set_error( err, True ) return None return conn @@ -150,27 +154,30 @@ kp = self.create_key_pair( conn, kp_name ) uci_wrapper.set_key_pair( kp.name, kp.material ) except boto.exception.EC2ResponseError, e: - log.error( "EC2 response error when deleting key pair: '%s'" % e ) - uci_wrapper.set_error( "EC2 response error while deleting key pair: " + str( e ), True ) + err = "EC2 response error while deleting key pair: " + str( e ) + log.error( err ) + uci_wrapper.set_error( err, True ) else: try: kp = self.create_key_pair( conn, kp_name ) uci_wrapper.set_key_pair( kp.name, kp.material ) except boto.exception.EC2ResponseError, e: - log.error( "EC2 response error when creating key pair: '%s'" % e ) - uci_wrapper.set_error( "EC2 response error while creating key pair: " + str( e ), True ) + err = "EC2 response error while creating key pair: " + str( e ) + log.error( err ) + uci_wrapper.set_error( err, True ) except Exception, ex: - log.error( "Exception when creating key pair: '%s'" % e ) - uci_wrapper.set_error( "Error while creating key pair: " + str( e ), True ) - + err = "Exception while creating key pair: " + str( ex ) + log.error( err ) + uci_wrapper.set_error( err, True ) except boto.exception.EC2ResponseError, e: # No keypair under this name exists so create it if e.code == 'InvalidKeyPair.NotFound': log.info( "No keypair found, creating keypair '%s'" % kp_name ) kp = self.create_key_pair( conn, kp_name ) uci_wrapper.set_key_pair( kp.name, kp.material ) else: - log.error( "EC2 response error while retrieving key pair: '%s'" % e ) - uci_wrapper.set_error( "Cloud provider response error while retrieving key pair: " + str( e ), True ) + err = "EC2 response error while retrieving key pair: " + str( e ) + log.error( err ) + uci_wrapper.set_error( err, True ) if kp != None: return kp.name @@ -197,8 +204,9 @@ if mi: return mi.image_id else: - log.error( "Machine image could not be retrieved for UCI '%s'." % uci_wrapper.get_name() ) - uci_wrapper.set_error( "Machine image could not be retrieved. Contact site administrator to ensure needed machine image is registered.", True ) + err = "Machine image could not be retrieved" + log.error( "%s for UCI '%s'." % (err, uci_wrapper.get_name() ) ) + uci_wrapper.set_error( err+". Contact site administrator to ensure needed machine image is registered.", True ) return None def shutdown( self ): @@ -242,14 +250,30 @@ # conn.delete_volume( vol.id ) # uci_wrapper.change_state( uci_state='error' ) # return - vl = conn.get_all_volumes( [vol.id] ) + + # Retrieve created volume again to get updated status + try: + vl = conn.get_all_volumes( [vol.id] ) + except boto.exception.EC2ResponseError, e: + err = "EC2 response error while retrieving (i.e., updating status) of just created storage volume '" + vol.id + "': " + str( e ) + log.error( err ) + uci_wrapper.set_store_status( vol.id, uci_states.ERROR ) + uci_wrapper.set_error( err, True ) + return + except Exception, ex: + err = "Error while retrieving (i.e., updating status) of just created storage volume '" + vol.id + "': " + str( ex ) + log.error( err ) + uci_wrapper.set_error( err, True ) + return + if len( vl ) > 0: uci_wrapper.change_state( uci_state=vl[0].status ) uci_wrapper.set_store_status( vol.id, vl[0].status ) else: - uci_wrapper.change_state( uci_state=uci_states.ERROR ) + err = "Volume '" + vol.id +"' not found by EC2 after being created." + log.error( err ) uci_wrapper.set_store_status( vol.id, uci_states.ERROR ) - uci_wrapper.set_error( "Volume '%s' not found by EC2 after being created" % vol.id ) + uci_wrapper.set_error( err, True ) def deleteUCI( self, uci_wrapper ): """ @@ -266,23 +290,28 @@ failedList = [] for v in vl: log.debug( "Deleting volume with id='%s'" % v.volume_id ) - if conn.delete_volume( v.volume_id ): - deletedList.append( v.volume_id ) - v.deleted = True - v.flush() - count += 1 - else: - failedList.append( v.volume_id ) + try: + if conn.delete_volume( v.volume_id ): + deletedList.append( v.volume_id ) + v.deleted = True + v.flush() + count += 1 + else: + failedList.append( v.volume_id ) + except boto.exception.EC2ResponseError, e: + err = "EC2 response error while deleting storage volume '" + v.volume_id + "': " + str( e ) + log.error( err ) + uci_wrapper.set_store_error( err, store_id = v.volume_id ) + uci_wrapper.set_error( err, True ) # Delete UCI if all of associated if count == len( vl ): - uci_wrapper.delete() + uci_wrapper.set_deleted() else: - log.error( "Deleting following volume(s) failed: %s. However, these volumes were successfully deleted: %s. \ - MANUAL intervention and processing needed." % ( failedList, deletedList ) ) - uci_wrapper.change_state( uci_state=uci_state.ERROR ) - uci_wrapper.set_error( "Deleting following volume(s) failed: "+failedList+". However, these volumes were successfully deleted: "+deletedList+". \ - MANUAL intervention and processing needed." ) + err = "Deleting following volume(s) failed: "+failedList+". However, these volumes were successfully deleted: "+deletedList+". \ + MANUAL intervention and processing needed." + log.error( err ) + uci_wrapper.set_error( err, True ) def snapshotUCI( self, uci_wrapper ): """ @@ -300,17 +329,17 @@ uci_wrapper.set_snapshot_id( snapshot.id, snap_id ) sh = conn.get_all_snapshots( snap_id ) # get updated status uci_wrapper.set_snapshot_status( status=sh[0].status, snap_id=snap_id ) - except boto.exception.EC2ResponseError, ex: - err = "Cloud provider response error while creating snapshot: " + str( e ) + except boto.exception.EC2ResponseError, e: + err = "EC2 response error while creating snapshot: " + str( e ) log.error( err ) uci_wrapper.set_snapshot_error( error=err, snap_index=snapshot.id, set_status=True ) - uci_wrapper.set_error( error=err, True ) + uci_wrapper.set_error( err, True ) return except Exception, ex: err = "Error while creating snapshot: " + str( ex ) log.error( err ) uci_wrapper.set_snapshot_error( error=err, snap_index=snapshot.id, set_status=True ) - uci_wrapper.set_error( error=err, True ) + uci_wrapper.set_error( err, True ) return uci_wrapper.change_state( uci_state=uci_states.AVAILABLE ) @@ -335,6 +364,11 @@ if uci_wrapper.get_state() != uci_states.ERROR: conn = self.get_connection( uci_wrapper ) self.check_key_pair( uci_wrapper, conn ) + if uci_wrapper.get_key_pair_name() == None: + err = "Key pair not found" + log.error( "%s for UCI '%s'." % ( err, uci_wrapper.get_name() ) ) + uci_wrapper.set_error( err + ". Try resetting the state and starting the instance again.", True ) + return i_indexes = uci_wrapper.get_instances_indexes( state=instance_states.SUBMITTED ) # Get indexes of i_indexes associated with this UCI that are in 'submitted' state log.debug( "Starting instances with IDs: '%s' associated with UCI '%s' " % ( i_indexes, uci_wrapper.get_name(), ) ) @@ -342,12 +376,9 @@ for i_index in i_indexes: # Get machine image for current instance mi_id = self.get_mi_id( uci_wrapper, i_index ) + log.debug( "mi_id: %s, uci_wrapper.get_key_pair_name(): %s" % ( mi_id, uci_wrapper.get_key_pair_name() ) ) uci_wrapper.set_mi( i_index, mi_id ) - if uci_wrapper.get_key_pair_name() == None: - log.error( "Key pair for UCI '%s' is None." % uci_wrapper.get_name() ) - uci_wrapper.set_error( "Key pair not found. Try resetting the state and starting the instance again.", True ) - return - + # Check if galaxy security group exists (and create it if it does not) log.debug( "Setting up '%s' security group." % self.security_group ) try: @@ -359,16 +390,18 @@ gSecurityGroup = conn.create_security_group(self.security_group, 'Security group for Galaxy.') gSecurityGroup.authorize( 'tcp', 80, 80, '0.0.0.0/0' ) # Open HTTP port gSecurityGroup.authorize( 'tcp', 22, 22, '0.0.0.0/0' ) # Open SSH port - except boto.exception.EC2ResponseError, ex: - log.error( "EC2 response error while creating security group: '%s'" % e ) - uci_wrapper.set_error( "EC2 response error while creating security group: " + str( e ), True ) + except boto.exception.EC2ResponseError, ee: + err = "EC2 response error while creating security group: " + str( ee ) + log.error( err ) + uci_wrapper.set_error( err, True ) else: - log.error( "EC2 response error while retrieving security group: '%s'" % e ) - uci_wrapper.set_error( "EC2 response error while retrieving security group: " + str( e ), True ) + err = "EC2 response error while retrieving security group: " + str( e ) + log.error( err ) + uci_wrapper.set_error( err, True ) if uci_wrapper.get_state() != uci_states.ERROR: - # Start an instance + # Start an instance log.debug( "Starting instance for UCI '%s'" % uci_wrapper.get_name() ) #TODO: Once multiple volumes can be attached to a single instance, update 'userdata' composition userdata = uci_wrapper.get_store_volume_id()+"|"+uci_wrapper.get_access_key()+"|"+uci_wrapper.get_secret_key() @@ -383,11 +416,13 @@ instance_type=uci_wrapper.get_type( i_index ), placement=uci_wrapper.get_uci_availability_zone() ) except boto.exception.EC2ResponseError, e: - log.error( "EC2 response error when starting UCI '%s': '%s'" % ( uci_wrapper.get_name(), str(e) ) ) - uci_wrapper.set_error( "EC2 response error when starting: " + str(e), True ) + err = "EC2 response error when starting UCI '"+ uci_wrapper.get_name() +"': " + str( e ) + log.error( err ) + uci_wrapper.set_error( err, True ) except Exception, ex: - log.error( "Error when starting UCI '%s': '%s'" % ( uci_wrapper.get_name(), str( ex ) ) ) - uci_wrapper.set_error( "Cloud provider error when starting: " + str( ex ), True ) + err = "Error when starting UCI '" + uci_wrapper.get_name() + "': " + str( ex ) + log.error( err ) + uci_wrapper.set_error( err, True ) # Record newly available instance data into local Galaxy database if reservation: uci_wrapper.set_launch_time( self.format_time( reservation.instances[0].launch_time ), i_index=i_index ) @@ -403,13 +438,16 @@ uci_wrapper.set_security_group_name( self.security_group, i_id=i_id ) log.debug( "Instance of UCI '%s' started, current state: '%s'" % ( uci_wrapper.get_name(), uci_wrapper.get_state() ) ) except boto.exception.EC2ResponseError, e: - log.error( "EC2 response error when retrieving instance information for UCI '%s': '%s'" % ( uci_wrapper.get_name(), str(e) ) ) - uci_wrapper.set_error( "EC2 response error when retrieving instance information: " + str(e), True ) + err = "EC2 response error when retrieving instance information for UCI '" + uci_wrapper.get_name() + "': " + str( e ) + log.error( err ) + uci_wrapper.set_error( err, True ) else: log.error( "UCI '%s' is in 'error' state, starting instance was aborted." % uci_wrapper.get_name() ) else: - log.error( "No instances were found for UCI '%s'" % uci_wrapper.get_name() ) - uci_wrapper.set_error( "EC2 response error when retrieving instance information: " + str(e), True ) + err = "No instances in state '"+ instance_states.SUBMITTED +"' found for UCI '" + uci_wrapper.get_name() + \ + "'. Nothing to start." + log.error( err ) + uci_wrapper.set_error( err, True ) else: log.error( "UCI '%s' is in 'error' state, starting instance was aborted." % uci_wrapper.get_name() ) @@ -426,14 +464,20 @@ # Initiate shutdown of all instances under given UCI cnt = 0 stopped = [] - notStopped = [] + not_stopped = [] for r in rl: for inst in r.instances: log.debug( "Sending stop signal to instance '%s' associated with reservation '%s'." % ( inst, r ) ) - inst.stop() - uci_wrapper.set_stop_time( datetime.utcnow(), i_id=inst.id ) - uci_wrapper.change_state( instance_id=inst.id, i_state=inst.update() ) - stopped.append( inst ) + try: + inst.stop() + uci_wrapper.set_stop_time( datetime.utcnow(), i_id=inst.id ) + uci_wrapper.change_state( instance_id=inst.id, i_state=inst.update() ) + stopped.append( inst ) + except boto.exception.EC2ResponseError, e: + not_stopped.append( inst ) + err = "EC2 response error when stopping instance '" + inst.instance_id + "': " + str(e) + log.error( err ) + uci_wrapper.set_error( err, True ) uci_wrapper.reset_uci_launch_time() log.debug( "Termination was initiated for all instances of UCI '%s'." % uci_wrapper.get_name() ) @@ -556,19 +600,21 @@ try: rl= conn.get_all_instances( [inst.instance_id] ) except boto.exception.EC2ResponseError, e: - log.error( "Retrieving instance(s) from cloud for UCI '%s' failed: " % ( uci.name, str(e) ) ) - uci.error( "Retrieving instance(s) from cloud failed: " + str(e) ) - uci.state( uci_states.ERROR ) + err = "Retrieving instance(s) from cloud failed for UCI '"+ uci.name +"' during general status update: " + str( e ) + log.error( err ) + uci.error = err + uci.state = uci_states.ERROR return None - # Because EPC deletes references to reservations after a short while after instances have terminated, getting an empty list as a response to a query - # typically means the instance has successfully shut down but the check was not performed in short enough amount of time. Until alternative solution + # Because references to reservations are deleted shortly after instances have been terminated, getting an empty list as a response to a query + # typically means the instance has successfully shut down but the check was not performed in short enough amount of time. Until an alternative solution # is found, below code sets state of given UCI to 'error' to indicate to the user something out of ordinary happened. if len( rl ) == 0: - log.info( "Instance ID '%s' was not found by the cloud provider. Instance might have crashed or otherwise been terminated." % inst.instance_id ) - inst.error = "Instance ID was not found by the cloud provider. Instance might have crashed or otherwise been terminated. State set to 'terminated'." - uci.error = "Instance ID '"+inst.instance_id+"' was not found by the cloud provider. Instance might have crashed or otherwise been terminated."+ \ + err = "Instance ID '"+inst.instance_id+"' was not found by the cloud provider. Instance might have crashed or otherwise been terminated."+ \ "Manual check is recommended." + log.error( err ) + inst.error = err + uci.error = err inst.state = instance_states.TERMINATED uci.state = uci_states.ERROR uci.launch_time = None @@ -599,9 +645,10 @@ inst.private_dns = cInst.private_dns_name inst.flush() except boto.exception.EC2ResponseError, e: - log.error( "Updating status of instance(s) from cloud for UCI '%s' failed: " % ( uci.name, str(e) ) ) - uci.error( "Updating instance status from cloud failed: " + str(e) ) - uci.state( uci_states.ERROR ) + err = "Updating instance status from cloud failed for UCI '"+ uci.name + "' during general status update: " + str( e ) + log.error( err ) + uci.error = err + uci.state = uci_states.ERROR return None def updateStore( self, store ): @@ -614,11 +661,11 @@ # Get reservations handle for given store try: vl = conn.get_all_volumes( [store.volume_id] ) -# log.debug( "Store '%s' vl: '%s'" % ( store.volume_id, vl ) ) except boto.exception.EC2ResponseError, e: - log.error( "Retrieving volume(s) from cloud for UCI '%s' failed: " % ( uci.name, str(e) ) ) - uci.error( "Retrieving volume(s) from cloud failed: " + str(e) ) - uci.state( uci_states.ERROR ) + err = "Retrieving volume(s) from cloud failed for UCI '"+ uci.name + "' during general status update: " + str( e ) + log.error( err ) + uci.error = err + uci.state = uci_states.ERROR uci.flush() return None @@ -631,7 +678,11 @@ if ( store.status == None ) and ( store.volume_id != None ): uci.state = vl[0].status uci.flush() - + # If UCI was marked in state 'CREATING', update its status to reflect new status + elif ( uci.state == uci_states.CREATING ): + uci.state = vl[0].status + uci.flush() + store.status = vl[0].status store.flush() if store.i_id != vl[0].instance_id: @@ -644,10 +695,21 @@ store.device = vl[0].device store.flush() except boto.exception.EC2ResponseError, e: - log.error( "Updating status of volume(s) from cloud for UCI '%s' failed: " % ( uci.name, str(e) ) ) - uci.error( "Updating volume status from cloud failed: " + str(e) ) - uci.state( uci_states.ERROR ) + err = "Updating status of volume(s) from cloud failed for UCI '"+ uci.name + "' during general status update: " + str( e ) + log.error( err ) + uci.error = err + uci.state = uci_states.ERROR + uci.flush() return None + else: + err = "No storage volumes returned by cloud provider on general update" + log.error( "%s for UCI '%s'" % ( err, uci.name ) ) + store.status = store_status.ERROR + store.error = err + uci.error = err + uci.state = uci_states.ERROR + uci.flush() + store.flush() def updateSnapshot( self, snapshot ): # Get credentials associated wit this store @@ -664,7 +726,7 @@ snapshot.status = snap[0].status snapshot.flush() else: - err = "No snapshots returned by EC2" + err = "No snapshots returned by EC2 on general update" log.error( "%s for UCI '%s'" % ( err, uci.name ) ) snapshot.status = snapshot_status.ERROR snapshot.error = err @@ -775,30 +837,25 @@ except: # something failed, so skip pass else: - inst.error = "Starting a machine instance associated with UCI '" + str(inst.uci.name) + "' seems to have failed. " \ - "Because it appears that cloud instance might have gotten started, manual check is recommended." + err = "Starting a machine instance (DB id: '"+str(inst.id)+"') associated with this UCI '" + str(inst.uci.name) + \ + "' seems to have failed. Because it appears that cloud instance might have gotten started, manual check is recommended." + inst.error = err inst.state = instance_states.ERROR - inst.uci.error = "Starting a machine instance (DB id: '"+str(inst.id)+"') associated with this UCI seems to have failed. " \ - "Because it appears that cloud instance might have gotten started, manual check is recommended." + inst.uci.error = err inst.uci.state = uci_states.ERROR - log.error( "Starting a machine instance (DB id: '%s') associated with UCI '%s' seems to have failed. " \ - "Because it appears that cloud instance might have gotten started, manual check is recommended." - % ( inst.id, inst.uci.name ) ) + log.error( err ) inst.flush() - inst.uci.flush() + inst.uci.flush() else: #Instance most likely never got processed, so set error message suggesting user to try starting instance again. - inst.error = "Starting a machine instance associated with UCI '" + str(inst.uci.name) + "' seems to have failed. " \ - "Because it appears that cloud instance never got started, it should be safe to reset state and try " \ - "starting the instance again." + err = "Starting a machine instance (DB id: '"+str(inst.id)+"') associated with this UCI '" + str(inst.uci.name) + \ + "' seems to have failed. Because it appears that cloud instance never got started, it should be safe to reset state and try " \ + "starting the instance again." + inst.error = err inst.state = instance_states.ERROR - inst.uci.error = "Starting a machine instance (DB id: '"+str(inst.id)+"') associated with this UCI seems to have failed. " \ - "Because it appears that cloud instance never got started, it should be safe to reset state and try " \ - "starting the instance again." + inst.uci.error = err inst.uci.state = uci_states.ERROR - log.error( "Starting a machine instance (DB id: '%s') associated with UCI '%s' seems to have failed. " \ - "Because it appears that cloud instance never got started, it should be safe to reset state and try " \ - "starting the instance again." % ( inst.id, inst.uci.name ) ) + log.error( err ) inst.flush() inst.uci.flush() # uw = UCIwrapper( inst.uci ) @@ -821,9 +878,10 @@ region=region, path=uci.credentials.provider.path ) except boto.exception.EC2ResponseError, e: - log.error( "Establishing connection with cloud failed: %s" % str(e) ) - uci.error( "Establishing connection with cloud failed: " + str(e) ) - uci.state( uci_states.ERROR ) + err = "Establishing connection with cloud failed: " + str( e ) + log.error( err ) + uci.error = err + uci.state = uci_states.ERROR uci.flush() return None diff -r b90db227df6d -r e21f605d2766 lib/galaxy/cloud/providers/eucalyptus.py --- a/lib/galaxy/cloud/providers/eucalyptus.py Thu Nov 12 10:28:54 2009 -0500 +++ b/lib/galaxy/cloud/providers/eucalyptus.py Thu Nov 12 12:42:17 2009 -0500 @@ -23,6 +23,7 @@ uci_states = Bunch( NEW_UCI = "newUCI", NEW = "new", + CREATING = "creating", DELETING_UCI = "deletingUCI", DELETING = "deleting", SUBMITTED_UCI = "submittedUCI", @@ -47,9 +48,19 @@ ERROR = "error" ) -store_states = Bunch( +store_status = Bunch( IN_USE = "in-use", CREATING = "creating", + DELETED = 'deleted', + ERROR = "error" +) + +snapshot_status = Bunch( + SUBMITTED = 'submitted', + PENDING = 'pending', + COMPLETED = 'completed', + DELETE = 'delete', + DELETED= 'deleted', ERROR = "error" ) @@ -105,10 +116,10 @@ try: euca_region = RegionInfo( None, provider.region_name, provider.region_endpoint ) except Exception, e: - log.error( "Selecting region with cloud provider failed: %2" % str(e) ) - uci_wrapper.set_error( "Selecting region with cloud provider failed: " + str(e), True ) - return None - + err = "Selecting region with cloud provider failed: " + str( e ) + log.error( err ) + uci_wrapper.set_error( err, True ) + return None try: conn = EC2Connection( aws_access_key_id=uci_wrapper.get_access_key(), aws_secret_access_key=uci_wrapper.get_secret_key(), @@ -117,8 +128,9 @@ region=euca_region, path=provider.path ) except boto.exception.EC2ResponseError, e: - log.error( "Establishing connection with cloud failed: %s" % str(e) ) - uci_wrapper.set_error( "Establishing connection with cloud failed: " + str(e), True ) + err = "Establishing connection with cloud failed: " + str( e ) + log.error( err ) + uci_wrapper.set_error( err, True ) return None return conn @@ -142,27 +154,30 @@ kp = self.create_key_pair( conn, kp_name ) uci_wrapper.set_key_pair( kp.name, kp.material ) except boto.exception.EC2ResponseError, e: - log.error( "EC2 response error when deleting key pair: '%s'" % e ) - uci_wrapper.set_error( "Cloud provider response error while deleting key pair: " + str( e ), True ) + err = "EC2 response error while deleting key pair: " + str( e ) + log.error( err ) + uci_wrapper.set_error( err, True ) else: try: kp = self.create_key_pair( conn, kp_name ) uci_wrapper.set_key_pair( kp.name, kp.material ) except boto.exception.EC2ResponseError, e: - log.error( "EC2 response error when creating key pair: '%s'" % e ) - uci_wrapper.set_error( "Cloud provider response error while creating key pair: " + str( e ), True ) + err = "EC2 response error while creating key pair: " + str( e ) + log.error( err ) + uci_wrapper.set_error( err, True ) except Exception, ex: - log.error( "Exception when creating key pair: '%s'" % e ) - uci_wrapper.set_error( "Error while creating key pair: " + str( e ), True ) - + err = "Exception while creating key pair: " + str( ex ) + log.error( err ) + uci_wrapper.set_error( err, True ) except boto.exception.EC2ResponseError, e: # No keypair under this name exists so create it if e.code == 'InvalidKeyPair.NotFound': log.info( "No keypair found, creating keypair '%s'" % kp_name ) kp = self.create_key_pair( conn, kp_name ) uci_wrapper.set_key_pair( kp.name, kp.material ) else: - log.error( "EC2 response error while retrieving key pair: '%s'" % e ) - uci_wrapper.set_error( "Cloud provider response error while retrieving key pair: " + str( e ), True ) + err = "EC2 response error while retrieving key pair: " + str( e ) + log.error( err ) + uci_wrapper.set_error( err, True ) if kp != None: return kp.name @@ -189,8 +204,9 @@ if mi: return mi.image_id else: - log.error( "Machine image could not be retrieved for UCI '%s'." % uci_wrapper.get_name() ) - uci_wrapper.set_error( "Machine image could not be retrieved. Contact site administrator to ensure needed machine image is registered.", True ) + err = "Machine image could not be retrieved" + log.error( "%s for UCI '%s'." % (err, uci_wrapper.get_name() ) ) + uci_wrapper.set_error( err+". Contact site administrator to ensure needed machine image is registered.", True ) return None def shutdown( self ): @@ -220,12 +236,37 @@ log.info( "Availability zone for UCI (i.e., storage volume) was not selected, using default zone: %s" % self.zone ) uci_wrapper.set_store_availability_zone( self.zone ) + log.debug( "Creating volume; using command: conn.create_volume( %s, '%s', snapshot=None )" % ( uci_wrapper.get_store_size( 0 ), uci_wrapper.get_uci_availability_zone() )) vol = conn.create_volume( uci_wrapper.get_store_size( 0 ), uci_wrapper.get_uci_availability_zone(), snapshot=None ) uci_wrapper.set_store_volume_id( 0, vol.id ) - # EPC does not allow creation of storage volumes (it deletes one as soon as it is created, so manually set uci_state here) - uci_wrapper.change_state( uci_state=uci_states.AVAILABLE ) - uci_wrapper.set_store_status( vol.id, vol.status ) + # Retrieve created volume again to get updated status + try: + vl = conn.get_all_volumes( [vol.id] ) + except boto.exception.EC2ResponseError, e: + err = "EC2 response error while retrieving (i.e., updating status) of just created storage volume '" + vol.id + "': " + str( e ) + log.error( err ) + uci_wrapper.set_store_status( vol.id, uci_states.ERROR ) + uci_wrapper.set_error( err, True ) + return + except Exception, ex: + err = "Error while retrieving (i.e., updating status) of just created storage volume '" + vol.id + "': " + str( ex ) + log.error( err ) + uci_wrapper.set_error( err, True ) + return + + if len( vl ) > 0: + # EPC does not allow creation of storage volumes (it deletes one as soon as it is created, so manually set uci_state here) + if vl[0].status == store_status.DELETED: + uci_wrapper.change_state( uci_state=uci_states.AVAILABLE ) + else: + uci_wrapper.change_state( uci_state=vl[0].status ) + uci_wrapper.set_store_status( vol.id, vl[0].status ) + else: + err = "Volume '" + vol.id +"' not found by EC2 after being created." + log.error( err ) + uci_wrapper.set_store_status( vol.id, uci_states.ERROR ) + uci_wrapper.set_error( err, True ) def deleteUCI( self, uci_wrapper ): """ @@ -242,24 +283,29 @@ failedList = [] for v in vl: log.debug( "Deleting volume with id='%s'" % v.volume_id ) - if conn.delete_volume( v.volume_id ): - deletedList.append( v.volume_id ) - v.deleted = True - v.flush() - count += 1 - else: - failedList.append( v.volume_id ) - + try: + if conn.delete_volume( v.volume_id ): + deletedList.append( v.volume_id ) + v.deleted = True + v.flush() + count += 1 + else: + failedList.append( v.volume_id ) + except boto.exception.EC2ResponseError, e: + err = "EC2 response error while deleting storage volume '" + v.volume_id + "': " + str( e ) + log.error( err ) + uci_wrapper.set_store_error( err, store_id = v.volume_id ) + uci_wrapper.set_error( err, True ) + # Delete UCI if all of associated if count == len( vl ): - uci_wrapper.delete() + uci_wrapper.set_deleted() else: - log.error( "Deleting following volume(s) failed: %s. However, these volumes were successfully deleted: %s. \ - Manual intervention and processing needed." % ( str( failedList ), str( deletedList ) ) ) - uci_wrapper.change_state( uci_state=uci_states.ERROR ) - uci_wrapper.set_error( "Deleting following volume(s) failed: "+str(failedList)+". However, these volumes were \ - successfully deleted: "+str(deletedList)+". Manual intervention and processing needed." ) - + err = "Deleting following volume(s) failed: "+failedList+". However, these volumes were successfully deleted: "+deletedList+". \ + MANUAL intervention and processing needed." + log.error( err ) + uci_wrapper.set_error( err, True ) + def snapshotUCI( self, uci_wrapper ): """ Creates snapshot of all storage volumes associated with this UCI. @@ -276,17 +322,17 @@ uci_wrapper.set_snapshot_id( snapshot.id, snap_id ) sh = conn.get_all_snapshots( snap_id ) # get updated status uci_wrapper.set_snapshot_status( status=sh[0].status, snap_id=snap_id ) - except boto.exception.EC2ResponseError, ex: + except boto.exception.EC2ResponseError, e: err = "Cloud provider response error while creating snapshot: " + str( e ) log.error( err ) uci_wrapper.set_snapshot_error( error=err, snap_index=snapshot.id, set_status=True ) - uci_wrapper.set_error( error=err, True ) + uci_wrapper.set_error( err, True ) return except Exception, ex: err = "Error while creating snapshot: " + str( ex ) log.error( err ) uci_wrapper.set_snapshot_error( error=err, snap_index=snapshot.id, set_status=True ) - uci_wrapper.set_error( error=err, True ) + uci_wrapper.set_error( err, True ) return uci_wrapper.change_state( uci_state=uci_states.AVAILABLE ) @@ -323,18 +369,22 @@ conn = self.get_connection( uci_wrapper ) self.check_key_pair( uci_wrapper, conn ) if uci_wrapper.get_key_pair_name() == None: - log.error( "Key pair for UCI '%s' is NULL." % uci_wrapper.get_name() ) - uci_wrapper.set_error( "Key pair not found. Try resetting the state and starting the instance again.", True ) + err = "Key pair not found" + log.error( "%s for UCI '%s'." % ( err, uci_wrapper.get_name() ) ) + uci_wrapper.set_error( err + ". Try resetting the state and starting the instance again.", True ) return i_indexes = uci_wrapper.get_instances_indexes( state=instance_states.SUBMITTED ) # Get indexes of i_indexes associated with this UCI that are in 'submitted' state + log.debug( "Starting instances with IDs: '%s' associated with UCI '%s' " % ( i_indexes, uci_wrapper.get_name(), ) ) if len( i_indexes ) > 0: for i_index in i_indexes: + # Get machine image for current instance mi_id = self.get_mi_id( uci_wrapper, i_index ) log.debug( "mi_id: %s, uci_wrapper.get_key_pair_name(): %s" % ( mi_id, uci_wrapper.get_key_pair_name() ) ) uci_wrapper.set_mi( i_index, mi_id ) - if uci_wrapper.get_state() != uci_states.ERROR and uci_wrapper.get_key_pair_name() != None: + if uci_wrapper.get_state() != uci_states.ERROR: + # Start an instance log.debug( "Starting UCI instance '%s'" % uci_wrapper.get_name() ) log.debug( "Using following command: conn.run_instances( image_id='%s', key_name='%s', instance_type='%s' )" % ( mi_id, uci_wrapper.get_key_pair_name(), uci_wrapper.get_type( i_index ) ) ) @@ -344,11 +394,13 @@ key_name=uci_wrapper.get_key_pair_name(), instance_type=uci_wrapper.get_type( i_index ) ) except boto.exception.EC2ResponseError, e: - log.error( "EC2 response error when starting UCI '%s': '%s'" % ( uci_wrapper.get_name(), str( e ) ) ) - uci_wrapper.set_error( "Cloud provider response error when starting: " + str( e ), True ) + err = "EC2 response error when starting UCI '"+ uci_wrapper.get_name() +"': " + str( e ) + log.error( err ) + uci_wrapper.set_error( err, True ) except Exception, ex: - log.error( "Error when starting UCI '%s': '%s'" % ( uci_wrapper.get_name(), str( ex ) ) ) - uci_wrapper.set_error( "Cloud provider error when starting: " + str( ex ), True ) + err = "Error when starting UCI '" + uci_wrapper.get_name() + "': " + str( ex ) + log.error( err ) + uci_wrapper.set_error( err, True ) # Record newly available instance data into local Galaxy database if reservation: uci_wrapper.set_launch_time( self.format_time( reservation.instances[0].launch_time ), i_index=i_index ) @@ -363,13 +415,16 @@ uci_wrapper.change_state( s, i_id, s ) log.debug( "Instance of UCI '%s' started, current state: '%s'" % ( uci_wrapper.get_name(), uci_wrapper.get_state() ) ) except boto.exception.EC2ResponseError, e: - log.error( "EC2 response error when retrieving instance information for UCI '%s': '%s'" % ( uci_wrapper.get_name(), str(e) ) ) - uci_wrapper.set_error( "Cloud provider response error when retrieving instance information: " + str(e), True ) + err = "EC2 response error when retrieving instance information for UCI '" + uci_wrapper.get_name() + "': " + str( e ) + log.error( err ) + uci_wrapper.set_error( err, True ) else: log.error( "UCI '%s' is in 'error' state, starting instance was aborted." % uci_wrapper.get_name() ) else: - log.error( "No instances were found for UCI '%s'" % uci_wrapper.get_name() ) - uci_wrapper.set_error( "EC2 response error when retrieving instance information: " + str(e), True ) + err = "No instances in state '"+ instance_states.SUBMITTED +"' found for UCI '" + uci_wrapper.get_name() + \ + "'. Nothing to start." + log.error( err ) + uci_wrapper.set_error( err, True ) else: log.error( "UCI '%s' is in 'error' state, starting instance was aborted." % uci_wrapper.get_name() ) @@ -387,14 +442,20 @@ # Initiate shutdown of all instances under given UCI cnt = 0 stopped = [] - notStopped = [] + not_stopped = [] for r in rl: for inst in r.instances: log.debug( "Sending stop signal to instance '%s' associated with reservation '%s' (UCI: %s)." % ( inst, r, uci_wrapper.get_name() ) ) - inst.stop() - uci_wrapper.set_stop_time( datetime.utcnow(), i_id=inst.id ) - uci_wrapper.change_state( instance_id=inst.id, i_state=inst.update() ) - stopped.append( inst ) + try: + inst.stop() + uci_wrapper.set_stop_time( datetime.utcnow(), i_id=inst.id ) + uci_wrapper.change_state( instance_id=inst.id, i_state=inst.update() ) + stopped.append( inst ) + except boto.exception.EC2ResponseError, e: + not_stopped.append( inst ) + err = "EC2 response error when stopping instance '" + inst.instance_id + "': " + str( e ) + log.error( err ) + uci_wrapper.set_error( err, True ) uci_wrapper.reset_uci_launch_time() log.debug( "Termination was initiated for all instances of UCI '%s'." % uci_wrapper.get_name() ) @@ -460,8 +521,8 @@ self.updateInstance( inst ) # Update storage volume(s) - stores = model.CloudStore.filter( or_( model.CloudStore.c.status==store_states.IN_USE, - model.CloudStore.c.status==store_states.CREATING, + stores = model.CloudStore.filter( or_( model.CloudStore.c.status==store_status.IN_USE, + model.CloudStore.c.status==store_status.CREATING, model.CloudStore.c.status==None ) ).all() for store in stores: if self.type == store.uci.credentials.provider.type: # and store.volume_id != None: @@ -473,7 +534,7 @@ # store.uci.error = "There exists an entry in local database for a storage volume without an ID. Storage volume might have been created " \ # "with cloud provider though. Manual check is recommended. After understanding what happened, local database entry for given " \ # "storage volume should be updated." -# store.status = store_states.ERROR +# store.status = store_status.ERROR # store.uci.state = uci_states.ERROR # store.uci.flush() # store.flush() @@ -516,19 +577,21 @@ try: rl= conn.get_all_instances( [inst.instance_id] ) except boto.exception.EC2ResponseError, e: - log.error( "Retrieving instance(s) from cloud for UCI '%s' failed: " % ( uci.name, str(e) ) ) - uci.error( "Retrieving instance(s) from cloud failed: " + str(e) ) - uci.state( uci_states.ERROR ) + err = "Retrieving instance(s) from cloud failed for UCI '"+ uci.name +"' during general status update: " + str( e ) + log.error( err ) + uci.error = err + uci.state = uci_states.ERROR return None # Because EPC deletes references to reservations after a short while after instances have terminated, getting an empty list as a response to a query # typically means the instance has successfully shut down but the check was not performed in short enough amount of time. Until alternative solution # is found, below code sets state of given UCI to 'error' to indicate to the user something out of ordinary happened. if len( rl ) == 0: - log.info( "Instance ID '%s' was not found by the cloud provider. Instance might have crashed or otherwise been terminated." % inst.instance_id ) - inst.error = "Instance ID was not found by the cloud provider. Instance might have crashed or otherwise been terminated. State set to 'terminated'." - uci.error = "Instance ID '"+inst.instance_id+"' was not found by the cloud provider. Instance might have crashed or otherwise been terminated."+ \ + err = "Instance ID '"+inst.instance_id+"' was not found by the cloud provider. Instance might have crashed or otherwise been terminated."+ \ "Manual check is recommended." + log.error( err ) + inst.error = err + uci.error = err inst.state = instance_states.TERMINATED uci.state = uci_states.ERROR uci.launch_time = None @@ -559,9 +622,10 @@ inst.private_dns = cInst.private_dns_name inst.flush() except boto.exception.EC2ResponseError, e: - log.error( "Updating status of instance(s) from cloud for UCI '%s' failed: " % ( uci.name, str(e) ) ) - uci.error( "Updating volume status from cloud failed: " + str(e) ) - uci.state( uci_states.ERROR ) + err = "Updating instance status from cloud failed for UCI '"+ uci.name + "' during general status update: " + str( e ) + log.error( err ) + uci.error = err + uci.state = uci_states.ERROR return None def updateStore( self, store ): @@ -574,9 +638,10 @@ try: vl = conn.get_all_volumes( [store.volume_id] ) except boto.exception.EC2ResponseError, e: - log.error( "Retrieving volume(s) from cloud for UCI '%s' failed: " % ( uci.name, str(e) ) ) - uci.error( "Retrieving volume(s) from cloud failed: " + str(e) ) - uci.state( uci_states.ERROR ) + err = "Retrieving volume(s) from cloud failed for UCI '"+ uci.name + "' during general status update: " + str( e ) + log.error( err ) + uci.error = err + uci.state = uci_states.ERROR uci.flush() return None @@ -589,6 +654,16 @@ if ( store.status == None ) and ( store.volume_id != None ): uci.state = vl[0].status uci.flush() + # If UCI was marked in state 'CREATING', update its status to reflect new status + elif ( uci.state == uci_states.CREATING ): + # Because Eucalyptus Public Cloud (EPC) deletes volumes immediately after they are created, artificially + # set status of given UCI to 'available' based on storage volume's availability zone (i.e., it's residing + # in EPC as opposed to some other Eucalyptus based cloud that allows creation of storage volumes. + if store.availability_zone == 'epc': + uci.state = uci_states.AVAILABLE + else: + uci.state = vl[0].status + uci.flush() store.status = vl[0].status store.flush() @@ -602,11 +677,21 @@ store.device = vl[0].device store.flush() except boto.exception.EC2ResponseError, e: - log.error( "Updating status of volume(s) from cloud for UCI '%s' failed: " % ( uci.name, str(e) ) ) - uci.error( "Updating volume status from cloud failed: " + str(e) ) + err = "Updating status of volume(s) from cloud failed for UCI '"+ uci.name + "' during general status update: " + str( e ) + log.error( err ) + uci.error = err uci.state = uci_states.ERROR uci.flush() return None + else: + err = "No storage volumes returned by cloud provider on general update" + log.error( "%s for UCI '%s'" % ( err, uci.name ) ) + store.status = store_status.ERROR + store.error = err + uci.error = err + uci.state = uci_states.ERROR + uci.flush() + store.flush() def updateSnapshot( self, snapshot ): # Get credentials associated wit this store @@ -623,7 +708,7 @@ snapshot.status = snap[0].status snapshot.flush() else: - err = "No snapshots returned by cloud provider" + err = "No snapshots returned by cloud provider on general update" log.error( "%s for UCI '%s'" % ( err, uci.name ) ) snapshot.status = snapshot_status.ERROR snapshot.error = err @@ -734,30 +819,25 @@ except: # something failed, so skip pass else: - inst.error = "Starting a machine instance associated with UCI '" + str(inst.uci.name) + "' seems to have failed. " \ - "Because it appears that cloud instance might have gotten started, manual check is recommended." + err = "Starting a machine instance (DB id: '"+str(inst.id)+"') associated with this UCI '" + str(inst.uci.name) + \ + "' seems to have failed. Because it appears that cloud instance might have gotten started, manual check is recommended." + inst.error = err inst.state = instance_states.ERROR - inst.uci.error = "Starting a machine instance (DB id: '"+str(inst.id)+"') associated with this UCI seems to have failed. " \ - "Because it appears that cloud instance might have gotten started, manual check is recommended." + inst.uci.error = err inst.uci.state = uci_states.ERROR - log.error( "Starting a machine instance (DB id: '%s') associated with UCI '%s' seems to have failed. " \ - "Because it appears that cloud instance might have gotten started, manual check is recommended." - % ( inst.id, inst.uci.name ) ) + log.error( err ) inst.flush() - inst.uci.flush() + inst.uci.flush() else: #Instance most likely never got processed, so set error message suggesting user to try starting instance again. - inst.error = "Starting a machine instance associated with UCI '" + str(inst.uci.name) + "' seems to have failed. " \ - "Because it appears that cloud instance never got started, it should be safe to reset state and try " \ - "starting the instance again." + err = "Starting a machine instance (DB id: '"+str(inst.id)+"') associated with this UCI '" + str(inst.uci.name) + \ + "' seems to have failed. Because it appears that cloud instance never got started, it should be safe to reset state and try " \ + "starting the instance again." + inst.error = err inst.state = instance_states.ERROR - inst.uci.error = "Starting a machine instance (DB id: '"+str(inst.id)+"') associated with this UCI seems to have failed. " \ - "Because it appears that cloud instance never got started, it should be safe to reset state and try " \ - "starting the instance again." + inst.uci.error = err inst.uci.state = uci_states.ERROR - log.error( "Starting a machine instance (DB id: '%s') associated with UCI '%s' seems to have failed. " \ - "Because it appears that cloud instance never got started, it should be safe to reset state and try " \ - "starting the instance again." % ( inst.id, inst.uci.name ) ) + log.error( err ) inst.flush() inst.uci.flush() # uw = UCIwrapper( inst.uci ) @@ -781,9 +861,10 @@ region=euca_region, path=uci.credentials.provider.path ) except boto.exception.EC2ResponseError, e: - log.error( "Establishing connection with cloud failed: %s" % str(e) ) - uci.error( "Establishing connection with cloud failed: " + str(e) ) - uci.state( uci_states.ERROR ) + err = "Establishing connection with cloud failed: " + str( e ) + log.error( err ) + uci.error = err + uci.state = uci_states.ERROR uci.flush() return None diff -r b90db227df6d -r e21f605d2766 lib/galaxy/model/mapping.py --- a/lib/galaxy/model/mapping.py Thu Nov 12 10:28:54 2009 -0500 +++ b/lib/galaxy/model/mapping.py Thu Nov 12 12:42:17 2009 -0500 @@ -379,13 +379,7 @@ Column( "session_id", Integer, ForeignKey( "galaxy_session.id" ), index=True ), Column( "history_id", Integer, ForeignKey( "history.id" ), index=True ) ) - - - - -# *************************************************************************** -# *************************** Cloud tables*********************************** -# *************************************************************************** +# *************************** Start cloud tables*********************************** CloudImage.table = Table( "cloud_image", metadata, Column( "id", Integer, primary_key=True ), Column( "create_time", DateTime, default=now ), @@ -446,6 +440,7 @@ Column( "status", TEXT ), Column( "device", TEXT ), Column( "space_consumed", Integer ), + Column( "error", TEXT ), Column( "deleted", Boolean, default=False ) ) CloudSnapshot.table = Table( "cloud_snapshot", metadata, @@ -493,7 +488,7 @@ Column( "https_connection_factory", TEXT ), Column( "path", TEXT ), Column( "deleted", Boolean, default=False ) ) -# *************************************************************************** +# *************************** End cloud tables*********************************** StoredWorkflow.table = Table( "stored_workflow", metadata, Column( "id", Integer, primary_key=True ), @@ -999,9 +994,8 @@ primaryjoin=( WorkflowStepConnection.table.c.input_step_id == WorkflowStep.table.c.id ) ), output_step=relation( WorkflowStep, backref="output_connections", cascade="all", primaryjoin=( WorkflowStepConnection.table.c.output_step_id == WorkflowStep.table.c.id ) ) ) ) -# ************************************************************ + # vvvvvvvvvvvvvvvv Start cloud table mappings vvvvvvvvvvvvvvvv -# ************************************************************ assign_mapper( context, CloudImage, CloudImage.table ) assign_mapper( context, UCI, UCI.table, diff -r b90db227df6d -r e21f605d2766 lib/galaxy/model/migrate/versions/0014_cloud_tables.py --- a/lib/galaxy/model/migrate/versions/0014_cloud_tables.py Thu Nov 12 10:28:54 2009 -0500 +++ b/lib/galaxy/model/migrate/versions/0014_cloud_tables.py Thu Nov 12 12:42:17 2009 -0500 @@ -78,6 +78,7 @@ Column( "status", TEXT ), Column( "device", TEXT ), Column( "space_consumed", Integer ), + Column( "error", TEXT ), Column( "deleted", Boolean, default=False ) ) CloudSnapshot_table = Table( "cloud_snapshot", metadata, @@ -131,21 +132,21 @@ # Load existing tables metadata.reflect() -# CloudImage_table.create() -# UCI_table.create() -# CloudUserCredentials_table.create() -# CloudStore_table.create() + CloudImage_table.create() + UCI_table.create() + CloudUserCredentials_table.create() + CloudStore_table.create() CloudSnapshot_table.create() -# CloudInstance_table.create() -# CloudProvider_table.create() + CloudInstance_table.create() + CloudProvider_table.create() def downgrade(): metadata.reflect() -# CloudImage_table.drop() -# CloudInstance_table.drop() -# CloudStore_table.drop() + CloudImage_table.drop() + CloudInstance_table.drop() + CloudStore_table.drop() CloudSnapshot_table.drop() -# CloudUserCredentials_table.drop() -# UCI_table.drop() -# CloudProvider_table.drop() \ No newline at end of file + CloudUserCredentials_table.drop() + UCI_table.drop() + CloudProvider_table.drop() \ No newline at end of file diff -r b90db227df6d -r e21f605d2766 lib/galaxy/web/controllers/cloud.py --- a/lib/galaxy/web/controllers/cloud.py Thu Nov 12 10:28:54 2009 -0500 +++ b/lib/galaxy/web/controllers/cloud.py Thu Nov 12 12:42:17 2009 -0500 @@ -34,6 +34,7 @@ uci_states = Bunch( NEW_UCI = "newUCI", NEW = "new", + CREATING = "creating", DELETING_UCI = "deletingUCI", DELETING = "deleting", SUBMITTED_UCI = "submittedUCI", @@ -61,6 +62,7 @@ store_states = Bunch( IN_USE = "in-use", CREATING = "creating", + DELETED = 'deleted', ERROR = "error" ) @@ -111,11 +113,13 @@ .all() prevInstances = trans.sa_session.query( model.UCI ) \ - .filter_by( user=user ) \ + .filter_by( user=user, deleted=False ) \ .filter( or_( model.UCI.c.state==uci_states.AVAILABLE, model.UCI.c.state==uci_states.NEW, model.UCI.c.state==uci_states.NEW_UCI, + model.UCI.c.state==uci_states.CREATING, model.UCI.c.state==uci_states.ERROR, + model.UCI.c.state==uci_states.DELETED, model.UCI.c.state==uci_states.DELETING, model.UCI.c.state==uci_states.DELETING_UCI, model.UCI.c.state==uci_states.SNAPSHOT,