details: http://www.bx.psu.edu/hg/galaxy/rev/c1dc30106721 changeset: 3089:c1dc30106721 user: Enis Afgan <afgane@gmail.com> date: Wed Nov 11 20:11:58 2009 -0500 description: Added ability for users to take snapshots of EBS storage volumes for configured UCIs. Note that, currently, this is supported only for UCIs that are not running at the time of snapshot creation. diffstat: lib/galaxy/cloud/__init__.py | 39 ++++++- lib/galaxy/cloud/providers/ec2.py | 136 +++++++++++++++++++++- lib/galaxy/cloud/providers/eucalyptus.py | 138 ++++++++++++++++++++++- lib/galaxy/model/__init__.py | 7 + lib/galaxy/model/mapping.py | 23 +++- lib/galaxy/model/migrate/versions/0014_cloud_tables.py | 39 ++++-- lib/galaxy/web/controllers/cloud.py | 89 ++++++++++++++- templates/cloud/configure_cloud.mako | 18 ++- templates/cloud/view_snapshots.mako | 90 +++++++++++++++ 9 files changed, 548 insertions(+), 31 deletions(-) diffs (841 lines): diff -r c78e8e1514e4 -r c1dc30106721 lib/galaxy/cloud/__init__.py --- a/lib/galaxy/cloud/__init__.py Tue Nov 10 18:36:21 2009 -0500 +++ b/lib/galaxy/cloud/__init__.py Wed Nov 11 20:11:58 2009 -0500 @@ -32,7 +32,8 @@ RUNNING = "running", PENDING = "pending", ERROR = "error", - CREATING = "creating" + SNAPSHOT_UCI = "snapshotUCI", + SNAPSHOT = "snapshot" ) instance_states = Bunch( TERMINATED = "terminated", @@ -141,7 +142,8 @@ .filter( or_( model.UCI.c.state==uci_states.NEW_UCI, model.UCI.c.state==uci_states.SUBMITTED_UCI, model.UCI.c.state==uci_states.SHUTTING_DOWN_UCI, - model.UCI.c.state==uci_states.DELETING_UCI ) ) \ + model.UCI.c.state==uci_states.DELETING_UCI, + model.UCI.c.state==uci_states.SNAPSHOT_UCI ) ) \ .all(): uci_wrapper = UCIwrapper( r ) new_requests.append( uci_wrapper ) @@ -310,6 +312,35 @@ vol = model.CloudStore.filter( model.CloudStore.c.volume_id == vol_id ).first() vol.status = status vol.flush() + + def set_snapshot_id( self, snap_index, id ): + snap = model.CloudSnapshot.get( snap_index ) + snap.snapshot_id = id + snap.flush() + + def set_snapshot_status( self, status, snap_index=None, snap_id=None ): + if snap_index != None: + snap = model.CloudSnapshot.get( snap_index ) + elif snap_id != None: + snap = model.CloudSnapshot.filter_by( snapshot_id = snap_id).first() + else: + return + snap.status = status + snap.flush() + + def set_snapshot_error( self, error, snap_index=None, snap_id=None, set_status=False ): + if snap_index != None: + snap = model.CloudSnapshot.get( snap_index ) + elif snap_id != None: + snap = model.CloudSnapshot.filter_by( snapshot_id = snap_id).first() + else: + return + snap.error = error + + if set_status: + snap.status = 'error' + + snap.flush() def set_store_availability_zone( self, availability_zone, vol_id=None ): """ @@ -501,6 +532,10 @@ def get_all_stores( self ): """ Returns all storage volumes' database objects associated with this UCI. """ return model.CloudStore.filter( model.CloudStore.c.uci_id == self.uci_id ).all() + + def get_snapshots( self, status=None ): + """ Returns database objects for all snapshots associated with this UCI and in given status.""" + return model.CloudSnapshot.filter_by( uci_id=self.uci_id, status=status ).all() def get_uci( self ): """ Returns database object for given UCI. """ diff -r c78e8e1514e4 -r c1dc30106721 lib/galaxy/cloud/providers/ec2.py --- a/lib/galaxy/cloud/providers/ec2.py Tue Nov 10 18:36:21 2009 -0500 +++ b/lib/galaxy/cloud/providers/ec2.py Wed Nov 11 20:11:58 2009 -0500 @@ -32,7 +32,9 @@ RUNNING = "running", PENDING = "pending", ERROR = "error", - DELETED = "deleted" + DELETED = "deleted", + SNAPSHOT_UCI = "snapshotUCI", + SNAPSHOT = "snapshot" ) instance_states = Bunch( @@ -89,6 +91,8 @@ self.startUCI( uci_wrapper ) elif uci_state==uci_states.SHUTTING_DOWN: self.stopUCI( uci_wrapper ) + elif uci_state==uci_states.SNAPSHOT: + self.snapshotUCI( uci_wrapper ) except: log.exception( "Uncaught exception executing request." ) cnt += 1 @@ -271,6 +275,35 @@ uci_wrapper.set_error( "Deleting following volume(s) failed: "+failedList+". However, these volumes were successfully deleted: "+deletedList+". \ MANUAL intervention and processing needed." ) + def snapshotUCI( self, uci_wrapper ): + """ + Creates snapshot of all storage volumes associated with this UCI. + """ + if uci_wrapper.get_state() != uci_states.ERROR: + conn = self.get_connection( uci_wrapper ) + + snapshots = uci_wrapper.get_snapshots( status = 'submitted' ) + for snapshot in snapshots: + log.debug( "Snapshot DB id: '%s', volume id: '%s'" % ( snapshot.id, snapshot.store.volume_id ) ) + try: + snap = conn.create_snapshot( volume_id=snapshot.store.volume_id ) + snap_id = str( snap ).split(':')[1] + uci_wrapper.set_snapshot_id( snapshot.id, snap_id ) + sh = conn.get_all_snapshots( snap_id ) # get updated status + uci_wrapper.set_snapshot_status( status=sh[0].status, snap_id=snap_id ) + except boto.exception.EC2ResponseError, ex: + log.error( "EC2 response error while creating snapshot: '%s'" % e ) + uci_wrapper.set_snapshot_error( error="EC2 response error while creating snapshot: " + str( e ), snap_index=snapshot.id, set_status=True ) + uci_wrapper.set_error( "EC2 response error while creating snapshot: " + str( e ), True ) + return + except Exception, ex: + log.error( "Error while creating snapshot: '%s'" % ex ) + uci_wrapper.set_snapshot_error( error="Error while creating snapshot: "+str( ex ), snap_index=snapshot.id, set_status=True ) + uci_wrapper.set_error( "Error while creating snapshot: " + str( ex ), True ) + return + + uci_wrapper.change_state( uci_state=uci_states.AVAILABLE ) + def addStorageToUCI( self, name ): """ Adds more storage to specified UCI TODO""" @@ -328,7 +361,7 @@ log.debug( "Starting instance for UCI '%s'" % uci_wrapper.get_name() ) #TODO: Once multiple volumes can be attached to a single instance, update 'userdata' composition userdata = uci_wrapper.get_store_volume_id()+"|"+uci_wrapper.get_access_key()+"|"+uci_wrapper.get_secret_key() - log.debug( "Using following command: conn.run_instances( image_id='%s', key_name='%s', security_groups='%s', user_data=[OMITTED], instance_type='%s', placement='%s' )" + log.debug( "Using following command: conn.run_instances( image_id='%s', key_name='%s', security_groups=['%s'], user_data=[OMITTED], instance_type='%s', placement='%s' )" % ( mi_id, uci_wrapper.get_key_pair_name(), self.security_group, uci_wrapper.get_type( i_index ), uci_wrapper.get_uci_availability_zone() ) ) reservation = None try: @@ -474,7 +507,17 @@ # store.uci.state = uci_states.ERROR # store.uci.flush() # store.flush() - + + # Update pending snapshots or delete ones marked for deletion + snapshots = model.CloudSnapshot.filter_by( status='pending', status='delete' ).all() + for snapshot in snapshots: + if self.type == snapshot.uci.credentials.provider.type and snapshot.status == 'pending': + log.debug( "[%s] Running general status update on snapshot '%s'" % ( snapshot.uci.credentials.provider.type, snapshot.snapshot_id ) ) + self.update_snapshot( snapshot ) + elif self.type == snapshot.uci.credentials.provider.type and snapshot.status == 'delete': + log.debug( "[%s] Initiating deletion of snapshot '%s'" % ( snapshot.uci.credentials.provider.type, snapshot.snapshot_id ) ) + self.delete_snapshot( snapshot ) + # Attempt at updating any zombie UCIs (i.e., instances that have been in SUBMITTED state for longer than expected - see below for exact time) zombies = model.UCI.filter_by( state=uci_states.SUBMITTED ).all() for zombie in zombies: @@ -496,7 +539,7 @@ uci_id = inst.uci_id uci = model.UCI.get( uci_id ) uci.refresh() - conn = self.get_connection_from_uci( inst.uci ) + conn = self.get_connection_from_uci( uci ) # Get reservations handle for given instance try: @@ -555,7 +598,7 @@ uci_id = store.uci_id uci = model.UCI.get( uci_id ) uci.refresh() - conn = self.get_connection_from_uci( inst.uci ) + conn = self.get_connection_from_uci( uci ) # Get reservations handle for given store try: @@ -595,6 +638,87 @@ uci.state( uci_states.ERROR ) return None + def updateSnapshot( self, snapshot ): + # Get credentials associated wit this store + if snapshot.status == 'completed': + uci_id = snapshot.uci_id + uci = model.UCI.get( uci_id ) + uci.refresh() + conn = self.get_connection_from_uci( uci ) + + try: + log.debug( "Updating status of snapshot '%s'" % snapshot.snapshot_id ) + snap = conn.get_all_snapshots( [snapshot.snapshot_id] ) + if len( snap ) > 0: + log.debug( "Snapshot '%s' status: %s" % ( snapshot.snapshot_id, snap[0].status ) ) + snapshot.status = snap[0].status + snapshot.flush() + else: + log.error( "No snapshots returned by EC2 for UCI '%s'" % uci.name ) + snapshot.status = 'No snapshots returned by EC2.' + uci.error = "No snapshots returned by EC2." + uci.state = uci_states.ERROR + uci.flush() + snapshot.flush() + except boto.exception.EC2ResponseError, e: + log.error( "EC2 response error while updating snapshot: '%s'" % e ) + snapshot.status = 'error' + snapshot.error = "EC2 response error while updating snapshot status: " + str( e ) + uci.error = "EC2 response error while updating snapshot status: " + str( e ) + uci.state = uci_states.ERROR + uci.flush() + snapshot.flush() + except Exception, ex: + log.error( "Error while updating snapshot: '%s'" % ex ) + snapshot.status = 'error' + snapshot.error = "Error while updating snapshot status: " + str( e ) + uci.error = "Error while updating snapshot status: " + str( ex ) + uci.state = uci_states.ERROR + uci.flush() + snapshot.flush() + else: + log.error( "Cannot delete snapshot '%s' because its status is '%s'. Only snapshots with 'completed' status can be deleted." % ( snapshot.snapshot_id, snapshot.status ) ) + snapshot.error = "Cannot delete snapshot because its status is '"+snapshot.status+"'. Only snapshots with 'completed' status can be deleted." + snapshot.flush() + + def delete_snapshot( self, snapshot ): + if snapshot.status == 'delete': + # Get credentials associated wit this store + uci_id = snapshot.uci_id + uci = model.UCI.get( uci_id ) + uci.refresh() + conn = self.get_connection_from_uci( uci ) + + try: + log.debug( "Deleting snapshot '%s'" % snapshot.snapshot_id ) + snap = conn.delete_snapshot( snapshot.snapshot_id ) + if snap == True: + snapshot.deleted = True + snapshot.status = 'deleted' + snapshot.flush() + return snap + except boto.exception.EC2ResponseError, e: + log.error( "EC2 response error while deleting snapshot: '%s'" % e ) + snapshot.status = 'error' + snapshot.error = "EC2 response error while deleting snapshot: " + str( e ) + uci.error = "EC2 response error while deleting snapshot: " + str( e ) + uci.state = uci_states.ERROR + uci.flush() + snapshot.flush() + except Exception, ex: + log.error( "Error while deleting snapshot: '%s'" % ex ) + snapshot.status = 'error' + snapshot.error = "Cloud provider error while deleting snapshot: " + str( ex ) + uci.error = "Cloud provider error while deleting snapshot: " + str( ex ) + uci.state = uci_states.ERROR + uci.flush() + snapshot.flush() + else: + log.error( "Cannot delete snapshot '%s' because its status is '%s'. Only snapshots with 'completed' status can be deleted." % ( snapshot.snapshot_id, snapshot.status ) ) + snapshot.error = "Cannot delete snapshot because its status is '"+snapshot.status+"'. Only snapshots with 'completed' status can be deleted." + snapshot.status = 'error' + snapshot.flush() + def processZombie( self, inst ): """ Attempt at discovering if starting an instance was successful but local database was not updated @@ -609,7 +733,7 @@ # report as error. # Fields attempting to be recovered are: reservation_id, instance status, and launch_time if inst.instance_id != None: - conn = self.get_connection_from_uci( inst.uci ) + conn = self.get_connection_from_uci( uci ) rl = conn.get_all_instances( [inst.instance_id] ) # reservation list # Update local DB with relevant data from instance if inst.reservation_id == None: diff -r c78e8e1514e4 -r c1dc30106721 lib/galaxy/cloud/providers/eucalyptus.py --- a/lib/galaxy/cloud/providers/eucalyptus.py Tue Nov 10 18:36:21 2009 -0500 +++ b/lib/galaxy/cloud/providers/eucalyptus.py Wed Nov 11 20:11:58 2009 -0500 @@ -33,7 +33,9 @@ RUNNING = "running", PENDING = "pending", ERROR = "error", - DELETED = "deleted" + DELETED = "deleted", + SNAPSHOT_UCI = "snapshotUCI", + SNAPSHOT = "snapshot" ) instance_states = Bunch( @@ -88,6 +90,8 @@ #self.dummyStartUCI( uci_wrapper ) elif uci_state==uci_states.SHUTTING_DOWN: self.stopUCI( uci_wrapper ) + elif uci_state==uci_states.SNAPSHOT: + self.snapshotUCI( uci_wrapper ) except: log.exception( "Uncaught exception executing cloud request." ) cnt += 1 @@ -255,8 +259,48 @@ uci_wrapper.change_state( uci_state=uci_states.ERROR ) uci_wrapper.set_error( "Deleting following volume(s) failed: "+str(failedList)+". However, these volumes were \ successfully deleted: "+str(deletedList)+". Manual intervention and processing needed." ) + + def snapshotUCI( self, uci_wrapper ): + """ + Creates snapshot of all storage volumes associated with this UCI. + """ + if uci_wrapper.get_state() != uci_states.ERROR: + conn = self.get_connection( uci_wrapper ) - def addStorageToUCI( self, name ): + snapshots = uci_wrapper.get_snapshots( status = 'submitted' ) + for snapshot in snapshots: + log.debug( "Snapshot DB id: '%s', volume id: '%s'" % ( snapshot.id, snapshot.store.volume_id ) ) + try: + snap = conn.create_snapshot( volume_id=snapshot.store.volume_id ) + snap_id = str( snap ).split(':')[1] + uci_wrapper.set_snapshot_id( snapshot.id, snap_id ) + sh = conn.get_all_snapshots( snap_id ) # get updated status + uci_wrapper.set_snapshot_status( status=sh[0].status, snap_id=snap_id ) + except boto.exception.EC2ResponseError, ex: + log.error( "EC2 response error while creating snapshot: '%s'" % e ) + uci_wrapper.set_snapshot_error( error="EC2 response error while creating snapshot: " + str( e ), snap_index=snapshot.id, set_status=True ) + uci_wrapper.set_error( "Cloud provider response error while creating snapshot: " + str( e ), True ) + return + except Exception, ex: + log.error( "Error while creating snapshot: '%s'" % ex ) + uci_wrapper.set_snapshot_error( error="Error while creating snapshot: "+str( ex ), snap_index=snapshot.id, set_status=True ) + uci_wrapper.set_error( "Error while creating snapshot: " + str( ex ), True ) + return + + uci_wrapper.change_state( uci_state=uci_states.AVAILABLE ) + +# if uci_wrapper.get_state() != uci_states.ERROR: +# +# snapshots = uci_wrapper.get_snapshots( status = 'submitted' ) +# for snapshot in snapshots: +# uci_wrapper.set_snapshot_id( snapshot.id, None, 'euca_error' ) +# +# log.debug( "Eucalyptus snapshot attempted by user for UCI '%s'" % uci_wrapper.get_name() ) +# uci_wrapper.set_error( "Eucalyptus does not support creation of snapshots at this moment. No snapshot or other changes were performed. \ +# Feel free to resent state of this instance and use it normally.", True ) + + + def addStorageToUCI( self, uci_wrapper ): """ Adds more storage to specified UCI """ def dummyStartUCI( self, uci_wrapper ): @@ -432,6 +476,16 @@ # store.uci.flush() # store.flush() + # Update pending snapshots or delete ones marked for deletion + snapshots = model.CloudSnapshot.filter_by( status='pending', status='delete' ).all() + for snapshot in snapshots: + if self.type == snapshot.uci.credentials.provider.type and snapshot.status == 'pending': + log.debug( "[%s] Running general status update on snapshot '%s'" % ( snapshot.uci.credentials.provider.type, snapshot.snapshot_id ) ) + self.update_snapshot( snapshot ) + elif self.type == snapshot.uci.credentials.provider.type and snapshot.status == 'delete': + log.debug( "[%s] Initiating deletion of snapshot '%s'" % ( snapshot.uci.credentials.provider.type, snapshot.snapshot_id ) ) + self.delete_snapshot( snapshot ) + # Attempt at updating any zombie UCIs (i.e., instances that have been in SUBMITTED state for longer than expected - see below for exact time) zombies = model.UCI.filter_by( state=uci_states.SUBMITTED ).all() for zombie in zombies: @@ -548,10 +602,86 @@ except boto.exception.EC2ResponseError, e: log.error( "Updating status of volume(s) from cloud for UCI '%s' failed: " % ( uci.name, str(e) ) ) uci.error( "Updating volume status from cloud failed: " + str(e) ) - uci.state( uci_states.ERROR ) + uci.state = uci_states.ERROR uci.flush() return None - + + def update_snapshot( self, snapshot ): + # Get credentials associated wit this store + uci_id = snapshot.uci_id + uci = model.UCI.get( uci_id ) + uci.refresh() + conn = self.get_connection_from_uci( uci ) + + try: + log.debug( "Updating status of snapshot '%s'" % snapshot.snapshot_id ) + snap = conn.get_all_snapshots( [snapshot.snapshot_id] ) + if len( snap ) > 0: + snapshot.status = snap[0].status + log.debug( "Snapshot '%s' status: %s" % ( snapshot.snapshot_id, snapshot.status ) ) + snapshot.flush() + else: + log.error( "No snapshots returned by cloud provider for UCI '%s'" % uci.name ) + snapshot.status = 'No snapshots returned by cloud provider.' + uci.error = "No snapshots returned by cloud provider." + uci.state = uci_states.ERROR + uci.flush() + snapshot.flush() + except boto.exception.EC2ResponseError, e: + log.error( "Cloud provider response error while updating snapshot: '%s'" % e ) + snapshot.status = 'error' + snapshot.error = "Cloud provider response error while updating snapshot status: " + str( e ) + uci.error = "Cloud provider response error while updating snapshot status: " + str( e ) + uci.state = uci_states.ERROR + uci.flush() + snapshot.flush() + except Exception, ex: + log.error( "Error while updating snapshot: '%s'" % ex ) + snapshot.status = 'error' + snapshot.error = "Error while updating snapshot status: " + str( e ) + uci.error = "Error while updating snapshot status: " + str( ex ) + uci.state = uci_states.ERROR + uci.flush() + snapshot.flush() + + def delete_snapshot( self, snapshot ): + if snapshot.status == 'delete': + # Get credentials associated wit this store + uci_id = snapshot.uci_id + uci = model.UCI.get( uci_id ) + uci.refresh() + conn = self.get_connection_from_uci( uci ) + + try: + log.debug( "Deleting snapshot '%s'" % snapshot.snapshot_id ) + snap = conn.delete_snapshot( snapshot.snapshot_id ) + if snap == True: + snapshot.deleted = True + snapshot.status = 'deleted' + snapshot.flush() + return snap + except boto.exception.EC2ResponseError, e: + log.error( "EC2 response error while deleting snapshot: '%s'" % e ) + snapshot.status = 'error' + snapshot.error = "Cloud provider response error while deleting snapshot: " + str( e ) + uci.error = "Cloud provider response error while deleting snapshot: " + str( e ) + uci.state = uci_states.ERROR + uci.flush() + snapshot.flush() + except Exception, ex: + log.error( "Error while deleting snapshot: '%s'" % ex ) + snapshot.status = 'error' + snapshot.error = "Cloud provider error while deleting snapshot: " + str( ex ) + uci.error = "Cloud provider error while deleting snapshot: " + str( ex ) + uci.state = uci_states.ERROR + uci.flush() + snapshot.flush() + else: + log.error( "Cannot delete snapshot '%s' because its status is '%s'. Only snapshots with 'completed' status can be deleted." % ( snapshot.snapshot_id, snapshot.status ) ) + snapshot.error = "Cannot delete snapshot because its status is '"+snapshot.status+"'. Only snapshots with 'completed' status can be deleted." + snapshot.status = 'error' + snapshot.flush() + def processZombie( self, inst ): """ Attempt at discovering if starting an instance was successful but local database was not updated diff -r c78e8e1514e4 -r c1dc30106721 lib/galaxy/model/__init__.py --- a/lib/galaxy/model/__init__.py Tue Nov 10 18:36:21 2009 -0500 +++ b/lib/galaxy/model/__init__.py Wed Nov 11 20:11:58 2009 -0500 @@ -963,6 +963,13 @@ self.size = None self.availability_zone = None +class CloudSnapshot( object ): + def __init__( self ): + self.id = None + self.user = None + self.store_id = None + self.snapshot_id = None + class CloudProvider( object ): def __init__( self ): self.id = None diff -r c78e8e1514e4 -r c1dc30106721 lib/galaxy/model/mapping.py --- a/lib/galaxy/model/mapping.py Tue Nov 10 18:36:21 2009 -0500 +++ b/lib/galaxy/model/mapping.py Wed Nov 11 20:11:58 2009 -0500 @@ -448,6 +448,19 @@ Column( "space_consumed", Integer ), Column( "deleted", Boolean, default=False ) ) +CloudSnapshot.table = Table( "cloud_snapshot", metadata, + Column( "id", Integer, primary_key=True ), + Column( "create_time", DateTime, default=now ), + Column( "update_time", DateTime, default=now, onupdate=now ), + Column( "user_id", Integer, ForeignKey( "galaxy_user.id" ), index=True, nullable=False ), + Column( "uci_id", Integer, ForeignKey( "cloud_uci.id" ), index=True ), + Column( "store_id", Integer, ForeignKey( "cloud_store.id" ), index=True, nullable=False ), + Column( "snapshot_id", TEXT ), + Column( "status", TEXT ), + Column( "description", TEXT ), + Column( "error", TEXT ), + Column( "deleted", Boolean, default=False ) ) + CloudUserCredentials.table = Table( "cloud_user_credentials", metadata, Column( "id", Integer, primary_key=True ), Column( "create_time", DateTime, default=now ), @@ -995,7 +1008,8 @@ properties=dict( user=relation( User ), credentials=relation( CloudUserCredentials ), instance=relation( CloudInstance, backref='uci' ), - store=relation( CloudStore, backref='uci', cascade='all, delete-orphan' ) + store=relation( CloudStore, backref='uci', cascade='all, delete-orphan' ), + snapshot=relation( CloudSnapshot, backref='uci' ) ) ) assign_mapper( context, CloudInstance, CloudInstance.table, @@ -1005,7 +1019,12 @@ assign_mapper( context, CloudStore, CloudStore.table, properties=dict( user=relation( User ), - i=relation( CloudInstance ) + i=relation( CloudInstance ), + snapshot=relation( CloudSnapshot, backref="store" ) + ) ) + +assign_mapper( context, CloudSnapshot, CloudSnapshot.table, + properties=dict( user=relation( User ) ) ) assign_mapper( context, CloudProvider, CloudProvider.table, diff -r c78e8e1514e4 -r c1dc30106721 lib/galaxy/model/migrate/versions/0014_cloud_tables.py --- a/lib/galaxy/model/migrate/versions/0014_cloud_tables.py Tue Nov 10 18:36:21 2009 -0500 +++ b/lib/galaxy/model/migrate/versions/0014_cloud_tables.py Wed Nov 11 20:11:58 2009 -0500 @@ -80,6 +80,19 @@ Column( "space_consumed", Integer ), Column( "deleted", Boolean, default=False ) ) +CloudSnapshot_table = Table( "cloud_snapshot", metadata, + Column( "id", Integer, primary_key=True ), + Column( "create_time", DateTime, default=now ), + Column( "update_time", DateTime, default=now, onupdate=now ), + Column( "user_id", Integer, ForeignKey( "galaxy_user.id" ), index=True, nullable=False ), + Column( "uci_id", Integer, ForeignKey( "cloud_uci.id" ), index=True ), + Column( "store_id", Integer, ForeignKey( "cloud_store.id" ), index=True, nullable=False ), + Column( "snapshot_id", TEXT ), + Column( "status", TEXT ), + Column( "description", TEXT ), + Column( "error", TEXT ), + Column( "deleted", Boolean, default=False ) ) + CloudUserCredentials_table = Table( "cloud_user_credentials", metadata, Column( "id", Integer, primary_key=True ), Column( "create_time", DateTime, default=now ), @@ -118,19 +131,21 @@ # Load existing tables metadata.reflect() - CloudImage_table.create() - UCI_table.create() - CloudUserCredentials_table.create() - CloudStore_table.create() - CloudInstance_table.create() - CloudProvider_table.create() +# CloudImage_table.create() +# UCI_table.create() +# CloudUserCredentials_table.create() +# CloudStore_table.create() + CloudSnapshot_table.create() +# CloudInstance_table.create() +# CloudProvider_table.create() def downgrade(): metadata.reflect() - CloudImage_table.drop() - CloudInstance_table.drop() - CloudStore_table.drop() - CloudUserCredentials_table.drop() - UCI_table.drop() - CloudProvider_table.drop() \ No newline at end of file +# CloudImage_table.drop() +# CloudInstance_table.drop() +# CloudStore_table.drop() + CloudSnapshot_table.drop() +# CloudUserCredentials_table.drop() +# UCI_table.drop() +# CloudProvider_table.drop() \ No newline at end of file diff -r c78e8e1514e4 -r c1dc30106721 lib/galaxy/web/controllers/cloud.py --- a/lib/galaxy/web/controllers/cloud.py Tue Nov 10 18:36:21 2009 -0500 +++ b/lib/galaxy/web/controllers/cloud.py Wed Nov 11 20:11:58 2009 -0500 @@ -44,7 +44,9 @@ RUNNING = "running", PENDING = "pending", ERROR = "error", - DELETED = "deleted" + DELETED = "deleted", + SNAPSHOT_UCI = "snapshotUCI", + SNAPSHOT = "snapshot" ) instance_states = Bunch( @@ -106,7 +108,9 @@ model.UCI.c.state==uci_states.NEW_UCI, model.UCI.c.state==uci_states.ERROR, model.UCI.c.state==uci_states.DELETING, - model.UCI.c.state==uci_states.DELETING_UCI ) ) \ + model.UCI.c.state==uci_states.DELETING_UCI, + model.UCI.c.state==uci_states.SNAPSHOT, + model.UCI.c.state==uci_states.SNAPSHOT_UCI ) ) \ .order_by( desc( model.UCI.c.update_time ) ) \ .all() @@ -223,6 +227,87 @@ return self.list( trans ) @web.expose + @web.require_login( "use Galaxy cloud" ) + def create_snapshot( self, trans, id ): + user = trans.get_user() + id = trans.security.decode_id( id ) + uci = get_uci( trans, id ) + + stores = trans.sa_session.query( model.CloudStore ) \ + .filter_by( user=user, deleted=False, uci_id=id ) \ + .all() + + if ( len( stores ) > 0 ) and ( uci.state == uci_states.AVAILABLE ): + for store in stores: + snapshot = model.CloudSnapshot() + snapshot.user = user + snapshot.uci = uci + snapshot.store = store + snapshot.status = 'submitted' + uci.state = uci_states.SNAPSHOT_UCI + # Persist + session = trans.sa_session + session.save_or_update( snapshot ) + session.save_or_update( uci ) + session.flush() + elif len( stores ) == 0: + error( "No storage volumes found that are associated with this instance." ) + else: + error( "Snapshot can be created only for an instance that is in 'available' state." ) + + # Log and display the management page + trans.log_event( "User initiated creation of new snapshot." ) + trans.set_message( "Creation of new snapshot initiated. " ) + return self.list( trans ) + + @web.expose + @web.require_login( "use Galaxy cloud" ) + def view_snapshots( self, trans, id=None ): + """ + View details about any snapshots associated with given UCI + """ + user = trans.get_user() + id = trans.security.decode_id( id ) + + snaps = trans.sa_session.query( model.CloudSnapshot ) \ + .filter_by( user=user, uci_id=id, deleted=False ) \ + .order_by( desc( model.CloudSnapshot.c.update_time ) ) \ + .all() + + return trans.fill_template( "cloud/view_snapshots.mako", + snaps = snaps ) + + @web.expose + @web.require_login( "use Galaxy cloud" ) + def delete_snapshot( self, trans, uci_id=None, snap_id=None ): + """ + Initiates deletion of a snapshot + """ + user = trans.get_user() + snap_id = trans.security.decode_id( snap_id ) + # Set snapshot as 'ready for deletion' to be picked up by general updater + snap = trans.sa_session.query( model.CloudSnapshot ).get( snap_id ) + + if snap.status == 'completed': + snap.status = 'delete' + snap.flush() + trans.set_message( "Snapshot '%s' is marked for deletion. Once the deletion is complete, it will no longer be visible in this list. " + "Please note that this process may take up to a minute." % snap.snapshot_id ) + else: + error( "Only snapshots in state 'completed' can be deleted. See the cloud provider directly " + "if you believe the snapshot is available and can be deleted." ) + + # Display new list of snapshots + uci_id = trans.security.decode_id( uci_id ) + snaps = trans.sa_session.query( model.CloudSnapshot ) \ + .filter_by( user=user, uci_id=uci_id, deleted=False ) \ + .order_by( desc( model.CloudSnapshot.c.update_time ) ) \ + .all() + + return trans.fill_template( "cloud/view_snapshots.mako", + snaps = snaps ) + + @web.expose @web.require_login( "add instance storage" ) def addStorage( self, trans, id ): instance = get_uci( trans, id ) diff -r c78e8e1514e4 -r c1dc30106721 templates/cloud/configure_cloud.mako --- a/templates/cloud/configure_cloud.mako Tue Nov 10 18:36:21 2009 -0500 +++ b/templates/cloud/configure_cloud.mako Wed Nov 11 20:11:58 2009 -0500 @@ -20,17 +20,26 @@ ${h.js( "jquery" )} <script type="text/javascript"> +function trim19(str){ + var str = str.replace(/^\s\s*/, ''), + ws = /\s/, + i = str.length; + while (ws.test(str.charAt(--i))); + return str.slice(0, i + 1); +} + function update_state() { $.getJSON( "${h.url_for( action='json_update' )}", {}, function ( data ) { for (var i in data) { var elem = '#' + data[i].id; // Because of different list managing 'live' vs. 'available' instances, reload url on various state changes old_state = $(elem + "-state").text(); - prev_old_state = $(elem + "-state-p").text(); + prev_old_state = trim19( $(elem + "-state-p").text() ); new_state = data[i].state; console.log( "old_state[%d] = %s", i, old_state ); console.log( "prev_old_state[%d] = %s", i, prev_old_state ); console.log( "new_state[%d] = %s", i, new_state ); + //console.log( trim19(prev_old_state) ); if ( ( old_state=='pending' && new_state=='running' ) || ( old_state=='shutting-down' && new_state=='available' ) || \ ( old_state=='running' && new_state=='available' ) || ( old_state=='running' && new_state=='error' ) || \ ( old_state=='pending' && new_state=='error' ) || ( old_state=='pending' && new_state=='available' ) || \ @@ -44,9 +53,9 @@ ( old_state=='submitted' && new_state=='error' ) || ( old_state=='submittedUCI' && new_state=='error' ) || \ ( old_state=='shutting-down' && new_state=='error' ) || ( prev_old_state.match('newUCI') && new_state=='error' ) || \ ( prev_old_state.match('new') && new_state=='error' ) || \ - ( prev_old_state.match('deleting') && new_state=='error' ) || ( prev_old_state.match('deletingUCI') && new_state=='error' ) ) { + ( prev_old_state.match('deletingUCI') && new_state=='error' ) ) { // TODO: Following clause causes constant page refresh for an exception thrown as a result of instance not starting correctly - need alternative method! - //( prev_old_state.match('available') && new_state=='error' ) || \ + //( prev_old_state.match('available') && new_state=='error' ) || ( prev_old_state.match('deleting') && new_state=='error' ) \ var url = "${h.url_for( controller='cloud', action='list')}"; location.replace( url ); @@ -80,6 +89,7 @@ // Update 'state' and 'time alive' fields $(elem + "-state").text( data[i].state ); + //$(elem + "-state-p").text( data[i].state ); if (data[i].launch_time) { $(elem + "-launch_time").text( data[i].launch_time.substring(0, 16 ) + " UTC (" + data[i].time_ago + ")" ); } @@ -313,6 +323,8 @@ <a class="action-button" href="${h.url_for( action='start', id=trans.security.encode_id(prevInstance.id), type='m1.small' )}"> Start m1.small</a> <a class="action-button" href="${h.url_for( action='start', id=trans.security.encode_id(prevInstance.id), type='c1.medium' )}"> Start c1.medium</a> <a class="action-button" href="${h.url_for( action='renameInstance', id=trans.security.encode_id(prevInstance.id) )}">Rename</a> + <a class="action-button" href="${h.url_for( action='create_snapshot', id=trans.security.encode_id(prevInstance.id) )}">Create snapshot</a> + <a class="action-button" href="${h.url_for( action='view_snapshots', id=trans.security.encode_id(prevInstance.id) )}">View snapshots</a> <a class="action-button" href="${h.url_for( action='addStorage', id=trans.security.encode_id(prevInstance.id) )}" target="_parent">Add storage</a> <a class="action-button" href="${h.url_for( action='usageReport', id=trans.security.encode_id(prevInstance.id) )}">Usage report</a> <a class="action-button" confirm="Are you sure you want to delete instance '${prevInstance.name}'? This will delete all of your data assocaiated with this instance!" href="${h.url_for( action='deleteInstance', id=trans.security.encode_id(prevInstance.id) )}">Delete</a> diff -r c78e8e1514e4 -r c1dc30106721 templates/cloud/view_snapshots.mako --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/templates/cloud/view_snapshots.mako Wed Nov 11 20:11:58 2009 -0500 @@ -0,0 +1,90 @@ +<%inherit file="/base.mako"/> + +<%def name="title()">Snapshots</%def> + +%if message: +<% + try: + messagetype + except: + messagetype = "done" +%> + + + +<p /> +<div class="${messagetype}message"> + ${message} +</div> +%endif + +%if snaps: + <h2>Snapshots for instance ${snaps[0].uci.name}</h2> +%else: + <h2>Selected instance has no recorded or associated snapshots.</h2> +%endif + +<ul class="manage-table-actions"> + <li> + <a class="action-button" href="${h.url_for( action='list' )}"> + <img src="${h.url_for('/static/images/silk/resultset_previous.png')}" /> + <span>Return to cloud management console</span> + </a> + </li> +</ul> + +%if snaps: + <table class="mange-table colored" border="0" cellspacing="0" cellpadding="0" width="100%"> + <colgroup width="2%"></colgroup> + <colgroup width="16%"></colgroup> + <colgroup width="16%"></colgroup> + <colgroup width="10%"></colgroup> + <colgroup width="5%"></colgroup> + <tr class="header"> + <th>#</th> + <th>Create time</th> + <th>Snapshot ID</th> + <th>Status</th> + <th>Delete?</th> + <th></th> + </tr> + <% + total_hours = 0 + %> + %for i, snap in enumerate( snaps ): + <tr> + <td>${i+1}</td> + <td> + %if snap.create_time: + ${str(snap.create_time)[:16]} UCT + %else: + N/A + %endif + </td> + <td> + %if snap.snapshot_id: + ${snap.snapshot_id} + %else: + N/A + %endif + </td> + <td> + %if snap.status: + ${snap.status} + %else: + N/A + %endif + </td> + <td> + <a confirm="Are you sure you want to delete snapshot '${snap.snapshot_id}'?" + href="${h.url_for( controller='cloud', action='delete_snapshot', uci_id=trans.security.encode_id(snap.uci.id), snap_id=trans.security.encode_id(snap.id) )}">x</a> + </td> + </tr> + %endfor + </table> +%endif + + + + +