details: http://www.bx.psu.edu/hg/galaxy/rev/b90db227df6d changeset: 3090:b90db227df6d user: Enis Afgan <afgane@gmail.com> date: Thu Nov 12 10:28:54 2009 -0500 description: Fixed a bug in EC2 controller regarding error reporting during snapshot update. Cleaned up code dealing with snapshots error reporting a bit. diffstat: lib/galaxy/cloud/__init__.py | 11 ++- lib/galaxy/cloud/providers/ec2.py | 131 ++++++++++++++++++-------------- lib/galaxy/cloud/providers/eucalyptus.py | 83 +++++++++++--------- lib/galaxy/web/controllers/cloud.py | 15 +++- 4 files changed, 140 insertions(+), 100 deletions(-) diffs (426 lines): diff -r c1dc30106721 -r b90db227df6d lib/galaxy/cloud/__init__.py --- a/lib/galaxy/cloud/__init__.py Wed Nov 11 20:11:58 2009 -0500 +++ b/lib/galaxy/cloud/__init__.py Thu Nov 12 10:28:54 2009 -0500 @@ -44,6 +44,15 @@ ERROR = "error" ) +snapshot_status = Bunch( + SUBMITTED = 'submitted', + PENDING = 'pending', + COMPLETED = 'completed', + DELETE = 'delete', + DELETED= 'deleted', + ERROR = "error" +) + class CloudManager( object ): """ Highest level interface to cloud management. @@ -338,7 +347,7 @@ snap.error = error if set_status: - snap.status = 'error' + snap.status = snapshot_status.ERROR snap.flush() diff -r c1dc30106721 -r b90db227df6d lib/galaxy/cloud/providers/ec2.py --- a/lib/galaxy/cloud/providers/ec2.py Wed Nov 11 20:11:58 2009 -0500 +++ b/lib/galaxy/cloud/providers/ec2.py Thu Nov 12 10:28:54 2009 -0500 @@ -52,6 +52,15 @@ ERROR = "error" ) +snapshot_status = Bunch( + SUBMITTED = 'submitted', + PENDING = 'pending', + COMPLETED = 'completed', + DELETE = 'delete', + DELETED= 'deleted', + ERROR = "error" +) + class EC2CloudProvider( object ): """ Amazon EC2-based cloud provider implementation for managing instances. @@ -282,7 +291,7 @@ if uci_wrapper.get_state() != uci_states.ERROR: conn = self.get_connection( uci_wrapper ) - snapshots = uci_wrapper.get_snapshots( status = 'submitted' ) + snapshots = uci_wrapper.get_snapshots( status = snapshot_status.SUBMITTED ) for snapshot in snapshots: log.debug( "Snapshot DB id: '%s', volume id: '%s'" % ( snapshot.id, snapshot.store.volume_id ) ) try: @@ -292,14 +301,16 @@ sh = conn.get_all_snapshots( snap_id ) # get updated status uci_wrapper.set_snapshot_status( status=sh[0].status, snap_id=snap_id ) except boto.exception.EC2ResponseError, ex: - log.error( "EC2 response error while creating snapshot: '%s'" % e ) - uci_wrapper.set_snapshot_error( error="EC2 response error while creating snapshot: " + str( e ), snap_index=snapshot.id, set_status=True ) - uci_wrapper.set_error( "EC2 response error while creating snapshot: " + str( e ), True ) + err = "Cloud provider response error while creating snapshot: " + str( e ) + log.error( err ) + uci_wrapper.set_snapshot_error( error=err, snap_index=snapshot.id, set_status=True ) + uci_wrapper.set_error( error=err, True ) return except Exception, ex: - log.error( "Error while creating snapshot: '%s'" % ex ) - uci_wrapper.set_snapshot_error( error="Error while creating snapshot: "+str( ex ), snap_index=snapshot.id, set_status=True ) - uci_wrapper.set_error( "Error while creating snapshot: " + str( ex ), True ) + err = "Error while creating snapshot: " + str( ex ) + log.error( err ) + uci_wrapper.set_snapshot_error( error=err, snap_index=snapshot.id, set_status=True ) + uci_wrapper.set_error( error=err, True ) return uci_wrapper.change_state( uci_state=uci_states.AVAILABLE ) @@ -509,12 +520,12 @@ # store.flush() # Update pending snapshots or delete ones marked for deletion - snapshots = model.CloudSnapshot.filter_by( status='pending', status='delete' ).all() + snapshots = model.CloudSnapshot.filter_by( status=snapshot_status.PENDING, status=snapshot_status.DELETE ).all() for snapshot in snapshots: - if self.type == snapshot.uci.credentials.provider.type and snapshot.status == 'pending': + if self.type == snapshot.uci.credentials.provider.type and snapshot.status == snapshot_status.PENDING: log.debug( "[%s] Running general status update on snapshot '%s'" % ( snapshot.uci.credentials.provider.type, snapshot.snapshot_id ) ) self.update_snapshot( snapshot ) - elif self.type == snapshot.uci.credentials.provider.type and snapshot.status == 'delete': + elif self.type == snapshot.uci.credentials.provider.type and snapshot.status == snapshot_status.DELETE: log.debug( "[%s] Initiating deletion of snapshot '%s'" % ( snapshot.uci.credentials.provider.type, snapshot.snapshot_id ) ) self.delete_snapshot( snapshot ) @@ -640,49 +651,48 @@ def updateSnapshot( self, snapshot ): # Get credentials associated wit this store - if snapshot.status == 'completed': - uci_id = snapshot.uci_id - uci = model.UCI.get( uci_id ) - uci.refresh() - conn = self.get_connection_from_uci( uci ) - - try: - log.debug( "Updating status of snapshot '%s'" % snapshot.snapshot_id ) - snap = conn.get_all_snapshots( [snapshot.snapshot_id] ) - if len( snap ) > 0: - log.debug( "Snapshot '%s' status: %s" % ( snapshot.snapshot_id, snap[0].status ) ) - snapshot.status = snap[0].status - snapshot.flush() - else: - log.error( "No snapshots returned by EC2 for UCI '%s'" % uci.name ) - snapshot.status = 'No snapshots returned by EC2.' - uci.error = "No snapshots returned by EC2." - uci.state = uci_states.ERROR - uci.flush() - snapshot.flush() - except boto.exception.EC2ResponseError, e: - log.error( "EC2 response error while updating snapshot: '%s'" % e ) - snapshot.status = 'error' - snapshot.error = "EC2 response error while updating snapshot status: " + str( e ) - uci.error = "EC2 response error while updating snapshot status: " + str( e ) + uci_id = snapshot.uci_id + uci = model.UCI.get( uci_id ) + uci.refresh() + conn = self.get_connection_from_uci( uci ) + + try: + log.debug( "Updating status of snapshot '%s'" % snapshot.snapshot_id ) + snap = conn.get_all_snapshots( [snapshot.snapshot_id] ) + if len( snap ) > 0: + log.debug( "Snapshot '%s' status: %s" % ( snapshot.snapshot_id, snap[0].status ) ) + snapshot.status = snap[0].status + snapshot.flush() + else: + err = "No snapshots returned by EC2" + log.error( "%s for UCI '%s'" % ( err, uci.name ) ) + snapshot.status = snapshot_status.ERROR + snapshot.error = err + uci.error = err uci.state = uci_states.ERROR uci.flush() snapshot.flush() - except Exception, ex: - log.error( "Error while updating snapshot: '%s'" % ex ) - snapshot.status = 'error' - snapshot.error = "Error while updating snapshot status: " + str( e ) - uci.error = "Error while updating snapshot status: " + str( ex ) - uci.state = uci_states.ERROR - uci.flush() - snapshot.flush() - else: - log.error( "Cannot delete snapshot '%s' because its status is '%s'. Only snapshots with 'completed' status can be deleted." % ( snapshot.snapshot_id, snapshot.status ) ) - snapshot.error = "Cannot delete snapshot because its status is '"+snapshot.status+"'. Only snapshots with 'completed' status can be deleted." + except boto.exception.EC2ResponseError, e: + err = "EC2 response error while updating snapshot status: " + str( e ) + log.error( err ) + snapshot.status = snapshot_status.ERROR + snapshot.error = err + uci.error = err + uci.state = uci_states.ERROR + uci.flush() + snapshot.flush() + except Exception, ex: + err = "Error while updating snapshot status: " + str( ex ) + log.error( err ) + snapshot.status = snapshot_status.ERROR + snapshot.error = err + uci.error = err + uci.state = uci_states.ERROR + uci.flush() snapshot.flush() def delete_snapshot( self, snapshot ): - if snapshot.status == 'delete': + if snapshot.status == snapshot_status.DELETE: # Get credentials associated wit this store uci_id = snapshot.uci_id uci = model.UCI.get( uci_id ) @@ -694,29 +704,32 @@ snap = conn.delete_snapshot( snapshot.snapshot_id ) if snap == True: snapshot.deleted = True - snapshot.status = 'deleted' + snapshot.status = snapshot_status.DELETED snapshot.flush() return snap except boto.exception.EC2ResponseError, e: - log.error( "EC2 response error while deleting snapshot: '%s'" % e ) - snapshot.status = 'error' - snapshot.error = "EC2 response error while deleting snapshot: " + str( e ) - uci.error = "EC2 response error while deleting snapshot: " + str( e ) + err = "EC2 response error while deleting snapshot: " + str( e ) + log.error( err ) + snapshot.status = snapshot_status.ERROR + snapshot.error = err + uci.error = err uci.state = uci_states.ERROR uci.flush() snapshot.flush() except Exception, ex: - log.error( "Error while deleting snapshot: '%s'" % ex ) - snapshot.status = 'error' - snapshot.error = "Cloud provider error while deleting snapshot: " + str( ex ) - uci.error = "Cloud provider error while deleting snapshot: " + str( ex ) + err = "Error while deleting snapshot: " + str( ex ) + log.error( err ) + snapshot.status = snapshot_status.ERROR + snapshot.error = err + uci.error = err uci.state = uci_states.ERROR uci.flush() snapshot.flush() else: - log.error( "Cannot delete snapshot '%s' because its status is '%s'. Only snapshots with 'completed' status can be deleted." % ( snapshot.snapshot_id, snapshot.status ) ) - snapshot.error = "Cannot delete snapshot because its status is '"+snapshot.status+"'. Only snapshots with 'completed' status can be deleted." - snapshot.status = 'error' + err = "Cannot delete snapshot '"+snapshot.snapshot_id+"' because its status is '"+snapshot.status+"'. Only snapshots with '" + \ + snapshot_status.COMPLETED+"' status can be deleted." + log.error( err ) + snapshot.error = err snapshot.flush() def processZombie( self, inst ): diff -r c1dc30106721 -r b90db227df6d lib/galaxy/cloud/providers/eucalyptus.py --- a/lib/galaxy/cloud/providers/eucalyptus.py Wed Nov 11 20:11:58 2009 -0500 +++ b/lib/galaxy/cloud/providers/eucalyptus.py Thu Nov 12 10:28:54 2009 -0500 @@ -267,7 +267,7 @@ if uci_wrapper.get_state() != uci_states.ERROR: conn = self.get_connection( uci_wrapper ) - snapshots = uci_wrapper.get_snapshots( status = 'submitted' ) + snapshots = uci_wrapper.get_snapshots( status = snapshot_status.SUBMITTED ) for snapshot in snapshots: log.debug( "Snapshot DB id: '%s', volume id: '%s'" % ( snapshot.id, snapshot.store.volume_id ) ) try: @@ -277,16 +277,18 @@ sh = conn.get_all_snapshots( snap_id ) # get updated status uci_wrapper.set_snapshot_status( status=sh[0].status, snap_id=snap_id ) except boto.exception.EC2ResponseError, ex: - log.error( "EC2 response error while creating snapshot: '%s'" % e ) - uci_wrapper.set_snapshot_error( error="EC2 response error while creating snapshot: " + str( e ), snap_index=snapshot.id, set_status=True ) - uci_wrapper.set_error( "Cloud provider response error while creating snapshot: " + str( e ), True ) + err = "Cloud provider response error while creating snapshot: " + str( e ) + log.error( err ) + uci_wrapper.set_snapshot_error( error=err, snap_index=snapshot.id, set_status=True ) + uci_wrapper.set_error( error=err, True ) return except Exception, ex: - log.error( "Error while creating snapshot: '%s'" % ex ) - uci_wrapper.set_snapshot_error( error="Error while creating snapshot: "+str( ex ), snap_index=snapshot.id, set_status=True ) - uci_wrapper.set_error( "Error while creating snapshot: " + str( ex ), True ) + err = "Error while creating snapshot: " + str( ex ) + log.error( err ) + uci_wrapper.set_snapshot_error( error=err, snap_index=snapshot.id, set_status=True ) + uci_wrapper.set_error( error=err, True ) return - + uci_wrapper.change_state( uci_state=uci_states.AVAILABLE ) # if uci_wrapper.get_state() != uci_states.ERROR: @@ -477,12 +479,12 @@ # store.flush() # Update pending snapshots or delete ones marked for deletion - snapshots = model.CloudSnapshot.filter_by( status='pending', status='delete' ).all() + snapshots = model.CloudSnapshot.filter_by( status=snapshot_status.PENDING, status=snapshot_status.DELETE ).all() for snapshot in snapshots: - if self.type == snapshot.uci.credentials.provider.type and snapshot.status == 'pending': + if self.type == snapshot.uci.credentials.provider.type and snapshot.status == snapshot_status.PENDING: log.debug( "[%s] Running general status update on snapshot '%s'" % ( snapshot.uci.credentials.provider.type, snapshot.snapshot_id ) ) self.update_snapshot( snapshot ) - elif self.type == snapshot.uci.credentials.provider.type and snapshot.status == 'delete': + elif self.type == snapshot.uci.credentials.provider.type and snapshot.status == snapshot_status.DELETE: log.debug( "[%s] Initiating deletion of snapshot '%s'" % ( snapshot.uci.credentials.provider.type, snapshot.snapshot_id ) ) self.delete_snapshot( snapshot ) @@ -606,7 +608,7 @@ uci.flush() return None - def update_snapshot( self, snapshot ): + def updateSnapshot( self, snapshot ): # Get credentials associated wit this store uci_id = snapshot.uci_id uci = model.UCI.get( uci_id ) @@ -617,35 +619,39 @@ log.debug( "Updating status of snapshot '%s'" % snapshot.snapshot_id ) snap = conn.get_all_snapshots( [snapshot.snapshot_id] ) if len( snap ) > 0: + log.debug( "Snapshot '%s' status: %s" % ( snapshot.snapshot_id, snap[0].status ) ) snapshot.status = snap[0].status - log.debug( "Snapshot '%s' status: %s" % ( snapshot.snapshot_id, snapshot.status ) ) snapshot.flush() else: - log.error( "No snapshots returned by cloud provider for UCI '%s'" % uci.name ) - snapshot.status = 'No snapshots returned by cloud provider.' - uci.error = "No snapshots returned by cloud provider." + err = "No snapshots returned by cloud provider" + log.error( "%s for UCI '%s'" % ( err, uci.name ) ) + snapshot.status = snapshot_status.ERROR + snapshot.error = err + uci.error = err uci.state = uci_states.ERROR uci.flush() snapshot.flush() except boto.exception.EC2ResponseError, e: - log.error( "Cloud provider response error while updating snapshot: '%s'" % e ) - snapshot.status = 'error' - snapshot.error = "Cloud provider response error while updating snapshot status: " + str( e ) - uci.error = "Cloud provider response error while updating snapshot status: " + str( e ) + err = "Cloud provider response error while updating snapshot status: " + str( e ) + log.error( err ) + snapshot.status = snapshot_status.ERROR + snapshot.error = err + uci.error = err uci.state = uci_states.ERROR uci.flush() snapshot.flush() except Exception, ex: - log.error( "Error while updating snapshot: '%s'" % ex ) - snapshot.status = 'error' - snapshot.error = "Error while updating snapshot status: " + str( e ) - uci.error = "Error while updating snapshot status: " + str( ex ) + err = "Error while updating snapshot status: " + str( ex ) + log.error( err ) + snapshot.status = snapshot_status.ERROR + snapshot.error = err + uci.error = err uci.state = uci_states.ERROR uci.flush() snapshot.flush() def delete_snapshot( self, snapshot ): - if snapshot.status == 'delete': + if snapshot.status == snapshot_status.DELETE: # Get credentials associated wit this store uci_id = snapshot.uci_id uci = model.UCI.get( uci_id ) @@ -657,29 +663,32 @@ snap = conn.delete_snapshot( snapshot.snapshot_id ) if snap == True: snapshot.deleted = True - snapshot.status = 'deleted' + snapshot.status = snapshot_status.DELETED snapshot.flush() return snap except boto.exception.EC2ResponseError, e: - log.error( "EC2 response error while deleting snapshot: '%s'" % e ) - snapshot.status = 'error' - snapshot.error = "Cloud provider response error while deleting snapshot: " + str( e ) - uci.error = "Cloud provider response error while deleting snapshot: " + str( e ) + err = "EC2 response error while deleting snapshot: " + str( e ) + log.error( err ) + snapshot.status = snapshot_status.ERROR + snapshot.error = err + uci.error = err uci.state = uci_states.ERROR uci.flush() snapshot.flush() except Exception, ex: - log.error( "Error while deleting snapshot: '%s'" % ex ) - snapshot.status = 'error' - snapshot.error = "Cloud provider error while deleting snapshot: " + str( ex ) - uci.error = "Cloud provider error while deleting snapshot: " + str( ex ) + err = "Error while deleting snapshot: " + str( ex ) + log.error( err ) + snapshot.status = snapshot_status.ERROR + snapshot.error = err + uci.error = err uci.state = uci_states.ERROR uci.flush() snapshot.flush() else: - log.error( "Cannot delete snapshot '%s' because its status is '%s'. Only snapshots with 'completed' status can be deleted." % ( snapshot.snapshot_id, snapshot.status ) ) - snapshot.error = "Cannot delete snapshot because its status is '"+snapshot.status+"'. Only snapshots with 'completed' status can be deleted." - snapshot.status = 'error' + err = "Cannot delete snapshot '"+snapshot.snapshot_id+"' because its status is '"+snapshot.status+"'. Only snapshots with '" + \ + snapshot_status.COMPLETED+"' status can be deleted." + log.error( err ) + snapshot.error = err snapshot.flush() def processZombie( self, inst ): diff -r c1dc30106721 -r b90db227df6d lib/galaxy/web/controllers/cloud.py --- a/lib/galaxy/web/controllers/cloud.py Wed Nov 11 20:11:58 2009 -0500 +++ b/lib/galaxy/web/controllers/cloud.py Thu Nov 12 10:28:54 2009 -0500 @@ -64,6 +64,15 @@ ERROR = "error" ) +snapshot_status = Bunch( + SUBMITTED = 'submitted', + PENDING = 'pending', + COMPLETED = 'completed', + DELETE = 'delete', + DELETED= 'deleted', + ERROR = "error" +) + class CloudController( BaseController ): @web.expose @@ -243,7 +252,7 @@ snapshot.user = user snapshot.uci = uci snapshot.store = store - snapshot.status = 'submitted' + snapshot.status = snapshot_status.SUBMITTED uci.state = uci_states.SNAPSHOT_UCI # Persist session = trans.sa_session @@ -288,8 +297,8 @@ # Set snapshot as 'ready for deletion' to be picked up by general updater snap = trans.sa_session.query( model.CloudSnapshot ).get( snap_id ) - if snap.status == 'completed': - snap.status = 'delete' + if snap.status == snapshot_status.COMPLETED: + snap.status = snapshot_status.DELETE snap.flush() trans.set_message( "Snapshot '%s' is marked for deletion. Once the deletion is complete, it will no longer be visible in this list. " "Please note that this process may take up to a minute." % snap.snapshot_id )