November 2009 - galaxy-dev - lists.galaxyproject.org

[hg] galaxy 3089: Added ability for users to take snapshots of E...
by Greg Von Kuster 23 Nov '09

23 Nov '09

details: http://www.bx.psu.edu/hg/galaxy/rev/c1dc30106721 changeset: 3089:c1dc30106721 user: Enis Afgan <afgane(a)gmail.com> date: Wed Nov 11 20:11:58 2009 -0500 description: Added ability for users to take snapshots of EBS storage volumes for configured UCIs. Note that, currently, this is supported only for UCIs that are not running at the time of snapshot creation. diffstat: lib/galaxy/cloud/__init__.py | 39 ++++++- lib/galaxy/cloud/providers/ec2.py | 136 +++++++++++++++++++++- lib/galaxy/cloud/providers/eucalyptus.py | 138 ++++++++++++++++++++++- lib/galaxy/model/__init__.py | 7 + lib/galaxy/model/mapping.py | 23 +++- lib/galaxy/model/migrate/versions/0014_cloud_tables.py | 39 ++++-- lib/galaxy/web/controllers/cloud.py | 89 ++++++++++++++- templates/cloud/configure_cloud.mako | 18 ++- templates/cloud/view_snapshots.mako | 90 +++++++++++++++ 9 files changed, 548 insertions(+), 31 deletions(-) diffs (841 lines): diff -r c78e8e1514e4 -r c1dc30106721 lib/galaxy/cloud/__init__.py --- a/lib/galaxy/cloud/__init__.py Tue Nov 10 18:36:21 2009 -0500 +++ b/lib/galaxy/cloud/__init__.py Wed Nov 11 20:11:58 2009 -0500 @@ -32,7 +32,8 @@ RUNNING = "running", PENDING = "pending", ERROR = "error", - CREATING = "creating" + SNAPSHOT_UCI = "snapshotUCI", + SNAPSHOT = "snapshot" ) instance_states = Bunch( TERMINATED = "terminated", @@ -141,7 +142,8 @@ .filter( or_( model.UCI.c.state==uci_states.NEW_UCI, model.UCI.c.state==uci_states.SUBMITTED_UCI, model.UCI.c.state==uci_states.SHUTTING_DOWN_UCI, - model.UCI.c.state==uci_states.DELETING_UCI ) ) \ + model.UCI.c.state==uci_states.DELETING_UCI, + model.UCI.c.state==uci_states.SNAPSHOT_UCI ) ) \ .all(): uci_wrapper = UCIwrapper( r ) new_requests.append( uci_wrapper ) @@ -310,6 +312,35 @@ vol = model.CloudStore.filter( model.CloudStore.c.volume_id == vol_id ).first() vol.status = status vol.flush() + + def set_snapshot_id( self, snap_index, id ): + snap = model.CloudSnapshot.get( snap_index ) + snap.snapshot_id = id + snap.flush() + + def set_snapshot_status( self, status, snap_index=None, snap_id=None ): + if snap_index != None: + snap = model.CloudSnapshot.get( snap_index ) + elif snap_id != None: + snap = model.CloudSnapshot.filter_by( snapshot_id = snap_id).first() + else: + return + snap.status = status + snap.flush() + + def set_snapshot_error( self, error, snap_index=None, snap_id=None, set_status=False ): + if snap_index != None: + snap = model.CloudSnapshot.get( snap_index ) + elif snap_id != None: + snap = model.CloudSnapshot.filter_by( snapshot_id = snap_id).first() + else: + return + snap.error = error + + if set_status: + snap.status = 'error' + + snap.flush() def set_store_availability_zone( self, availability_zone, vol_id=None ): """ @@ -501,6 +532,10 @@ def get_all_stores( self ): """ Returns all storage volumes' database objects associated with this UCI. """ return model.CloudStore.filter( model.CloudStore.c.uci_id == self.uci_id ).all() + + def get_snapshots( self, status=None ): + """ Returns database objects for all snapshots associated with this UCI and in given status.""" + return model.CloudSnapshot.filter_by( uci_id=self.uci_id, status=status ).all() def get_uci( self ): """ Returns database object for given UCI. """ diff -r c78e8e1514e4 -r c1dc30106721 lib/galaxy/cloud/providers/ec2.py --- a/lib/galaxy/cloud/providers/ec2.py Tue Nov 10 18:36:21 2009 -0500 +++ b/lib/galaxy/cloud/providers/ec2.py Wed Nov 11 20:11:58 2009 -0500 @@ -32,7 +32,9 @@ RUNNING = "running", PENDING = "pending", ERROR = "error", - DELETED = "deleted" + DELETED = "deleted", + SNAPSHOT_UCI = "snapshotUCI", + SNAPSHOT = "snapshot" ) instance_states = Bunch( @@ -89,6 +91,8 @@ self.startUCI( uci_wrapper ) elif uci_state==uci_states.SHUTTING_DOWN: self.stopUCI( uci_wrapper ) + elif uci_state==uci_states.SNAPSHOT: + self.snapshotUCI( uci_wrapper ) except: log.exception( "Uncaught exception executing request." ) cnt += 1 @@ -271,6 +275,35 @@ uci_wrapper.set_error( "Deleting following volume(s) failed: "+failedList+". However, these volumes were successfully deleted: "+deletedList+". \ MANUAL intervention and processing needed." ) + def snapshotUCI( self, uci_wrapper ): + """ + Creates snapshot of all storage volumes associated with this UCI. + """ + if uci_wrapper.get_state() != uci_states.ERROR: + conn = self.get_connection( uci_wrapper ) + + snapshots = uci_wrapper.get_snapshots( status = 'submitted' ) + for snapshot in snapshots: + log.debug( "Snapshot DB id: '%s', volume id: '%s'" % ( snapshot.id, snapshot.store.volume_id ) ) + try: + snap = conn.create_snapshot( volume_id=snapshot.store.volume_id ) + snap_id = str( snap ).split(':')[1] + uci_wrapper.set_snapshot_id( snapshot.id, snap_id ) + sh = conn.get_all_snapshots( snap_id ) # get updated status + uci_wrapper.set_snapshot_status( status=sh[0].status, snap_id=snap_id ) + except boto.exception.EC2ResponseError, ex: + log.error( "EC2 response error while creating snapshot: '%s'" % e ) + uci_wrapper.set_snapshot_error( error="EC2 response error while creating snapshot: " + str( e ), snap_index=snapshot.id, set_status=True ) + uci_wrapper.set_error( "EC2 response error while creating snapshot: " + str( e ), True ) + return + except Exception, ex: + log.error( "Error while creating snapshot: '%s'" % ex ) + uci_wrapper.set_snapshot_error( error="Error while creating snapshot: "+str( ex ), snap_index=snapshot.id, set_status=True ) + uci_wrapper.set_error( "Error while creating snapshot: " + str( ex ), True ) + return + + uci_wrapper.change_state( uci_state=uci_states.AVAILABLE ) + def addStorageToUCI( self, name ): """ Adds more storage to specified UCI TODO""" @@ -328,7 +361,7 @@ log.debug( "Starting instance for UCI '%s'" % uci_wrapper.get_name() ) #TODO: Once multiple volumes can be attached to a single instance, update 'userdata' composition userdata = uci_wrapper.get_store_volume_id()+"|"+uci_wrapper.get_access_key()+"|"+uci_wrapper.get_secret_key() - log.debug( "Using following command: conn.run_instances( image_id='%s', key_name='%s', security_groups='%s', user_data=[OMITTED], instance_type='%s', placement='%s' )" + log.debug( "Using following command: conn.run_instances( image_id='%s', key_name='%s', security_groups=['%s'], user_data=[OMITTED], instance_type='%s', placement='%s' )" % ( mi_id, uci_wrapper.get_key_pair_name(), self.security_group, uci_wrapper.get_type( i_index ), uci_wrapper.get_uci_availability_zone() ) ) reservation = None try: @@ -474,7 +507,17 @@ # store.uci.state = uci_states.ERROR # store.uci.flush() # store.flush() - + + # Update pending snapshots or delete ones marked for deletion + snapshots = model.CloudSnapshot.filter_by( status='pending', status='delete' ).all() + for snapshot in snapshots: + if self.type == snapshot.uci.credentials.provider.type and snapshot.status == 'pending': + log.debug( "[%s] Running general status update on snapshot '%s'" % ( snapshot.uci.credentials.provider.type, snapshot.snapshot_id ) ) + self.update_snapshot( snapshot ) + elif self.type == snapshot.uci.credentials.provider.type and snapshot.status == 'delete': + log.debug( "[%s] Initiating deletion of snapshot '%s'" % ( snapshot.uci.credentials.provider.type, snapshot.snapshot_id ) ) + self.delete_snapshot( snapshot ) + # Attempt at updating any zombie UCIs (i.e., instances that have been in SUBMITTED state for longer than expected - see below for exact time) zombies = model.UCI.filter_by( state=uci_states.SUBMITTED ).all() for zombie in zombies: @@ -496,7 +539,7 @@ uci_id = inst.uci_id uci = model.UCI.get( uci_id ) uci.refresh() - conn = self.get_connection_from_uci( inst.uci ) + conn = self.get_connection_from_uci( uci ) # Get reservations handle for given instance try: @@ -555,7 +598,7 @@ uci_id = store.uci_id uci = model.UCI.get( uci_id ) uci.refresh() - conn = self.get_connection_from_uci( inst.uci ) + conn = self.get_connection_from_uci( uci ) # Get reservations handle for given store try: @@ -595,6 +638,87 @@ uci.state( uci_states.ERROR ) return None + def updateSnapshot( self, snapshot ): + # Get credentials associated wit this store + if snapshot.status == 'completed': + uci_id = snapshot.uci_id + uci = model.UCI.get( uci_id ) + uci.refresh() + conn = self.get_connection_from_uci( uci ) + + try: + log.debug( "Updating status of snapshot '%s'" % snapshot.snapshot_id ) + snap = conn.get_all_snapshots( [snapshot.snapshot_id] ) + if len( snap ) > 0: + log.debug( "Snapshot '%s' status: %s" % ( snapshot.snapshot_id, snap[0].status ) ) + snapshot.status = snap[0].status + snapshot.flush() + else: + log.error( "No snapshots returned by EC2 for UCI '%s'" % uci.name ) + snapshot.status = 'No snapshots returned by EC2.' + uci.error = "No snapshots returned by EC2." + uci.state = uci_states.ERROR + uci.flush() + snapshot.flush() + except boto.exception.EC2ResponseError, e: + log.error( "EC2 response error while updating snapshot: '%s'" % e ) + snapshot.status = 'error' + snapshot.error = "EC2 response error while updating snapshot status: " + str( e ) + uci.error = "EC2 response error while updating snapshot status: " + str( e ) + uci.state = uci_states.ERROR + uci.flush() + snapshot.flush() + except Exception, ex: + log.error( "Error while updating snapshot: '%s'" % ex ) + snapshot.status = 'error' + snapshot.error = "Error while updating snapshot status: " + str( e ) + uci.error = "Error while updating snapshot status: " + str( ex ) + uci.state = uci_states.ERROR + uci.flush() + snapshot.flush() + else: + log.error( "Cannot delete snapshot '%s' because its status is '%s'. Only snapshots with 'completed' status can be deleted." % ( snapshot.snapshot_id, snapshot.status ) ) + snapshot.error = "Cannot delete snapshot because its status is '"+snapshot.status+"'. Only snapshots with 'completed' status can be deleted." + snapshot.flush() + + def delete_snapshot( self, snapshot ): + if snapshot.status == 'delete': + # Get credentials associated wit this store + uci_id = snapshot.uci_id + uci = model.UCI.get( uci_id ) + uci.refresh() + conn = self.get_connection_from_uci( uci ) + + try: + log.debug( "Deleting snapshot '%s'" % snapshot.snapshot_id ) + snap = conn.delete_snapshot( snapshot.snapshot_id ) + if snap == True: + snapshot.deleted = True + snapshot.status = 'deleted' + snapshot.flush() + return snap + except boto.exception.EC2ResponseError, e: + log.error( "EC2 response error while deleting snapshot: '%s'" % e ) + snapshot.status = 'error' + snapshot.error = "EC2 response error while deleting snapshot: " + str( e ) + uci.error = "EC2 response error while deleting snapshot: " + str( e ) + uci.state = uci_states.ERROR + uci.flush() + snapshot.flush() + except Exception, ex: + log.error( "Error while deleting snapshot: '%s'" % ex ) + snapshot.status = 'error' + snapshot.error = "Cloud provider error while deleting snapshot: " + str( ex ) + uci.error = "Cloud provider error while deleting snapshot: " + str( ex ) + uci.state = uci_states.ERROR + uci.flush() + snapshot.flush() + else: + log.error( "Cannot delete snapshot '%s' because its status is '%s'. Only snapshots with 'completed' status can be deleted." % ( snapshot.snapshot_id, snapshot.status ) ) + snapshot.error = "Cannot delete snapshot because its status is '"+snapshot.status+"'. Only snapshots with 'completed' status can be deleted." + snapshot.status = 'error' + snapshot.flush() + def processZombie( self, inst ): """ Attempt at discovering if starting an instance was successful but local database was not updated @@ -609,7 +733,7 @@ # report as error. # Fields attempting to be recovered are: reservation_id, instance status, and launch_time if inst.instance_id != None: - conn = self.get_connection_from_uci( inst.uci ) + conn = self.get_connection_from_uci( uci ) rl = conn.get_all_instances( [inst.instance_id] ) # reservation list # Update local DB with relevant data from instance if inst.reservation_id == None: diff -r c78e8e1514e4 -r c1dc30106721 lib/galaxy/cloud/providers/eucalyptus.py --- a/lib/galaxy/cloud/providers/eucalyptus.py Tue Nov 10 18:36:21 2009 -0500 +++ b/lib/galaxy/cloud/providers/eucalyptus.py Wed Nov 11 20:11:58 2009 -0500 @@ -33,7 +33,9 @@ RUNNING = "running", PENDING = "pending", ERROR = "error", - DELETED = "deleted" + DELETED = "deleted", + SNAPSHOT_UCI = "snapshotUCI", + SNAPSHOT = "snapshot" ) instance_states = Bunch( @@ -88,6 +90,8 @@ #self.dummyStartUCI( uci_wrapper ) elif uci_state==uci_states.SHUTTING_DOWN: self.stopUCI( uci_wrapper ) + elif uci_state==uci_states.SNAPSHOT: + self.snapshotUCI( uci_wrapper ) except: log.exception( "Uncaught exception executing cloud request." ) cnt += 1 @@ -255,8 +259,48 @@ uci_wrapper.change_state( uci_state=uci_states.ERROR ) uci_wrapper.set_error( "Deleting following volume(s) failed: "+str(failedList)+". However, these volumes were \ successfully deleted: "+str(deletedList)+". Manual intervention and processing needed." ) + + def snapshotUCI( self, uci_wrapper ): + """ + Creates snapshot of all storage volumes associated with this UCI. + """ + if uci_wrapper.get_state() != uci_states.ERROR: + conn = self.get_connection( uci_wrapper ) - def addStorageToUCI( self, name ): + snapshots = uci_wrapper.get_snapshots( status = 'submitted' ) + for snapshot in snapshots: + log.debug( "Snapshot DB id: '%s', volume id: '%s'" % ( snapshot.id, snapshot.store.volume_id ) ) + try: + snap = conn.create_snapshot( volume_id=snapshot.store.volume_id ) + snap_id = str( snap ).split(':')[1] + uci_wrapper.set_snapshot_id( snapshot.id, snap_id ) + sh = conn.get_all_snapshots( snap_id ) # get updated status + uci_wrapper.set_snapshot_status( status=sh[0].status, snap_id=snap_id ) + except boto.exception.EC2ResponseError, ex: + log.error( "EC2 response error while creating snapshot: '%s'" % e ) + uci_wrapper.set_snapshot_error( error="EC2 response error while creating snapshot: " + str( e ), snap_index=snapshot.id, set_status=True ) + uci_wrapper.set_error( "Cloud provider response error while creating snapshot: " + str( e ), True ) + return + except Exception, ex: + log.error( "Error while creating snapshot: '%s'" % ex ) + uci_wrapper.set_snapshot_error( error="Error while creating snapshot: "+str( ex ), snap_index=snapshot.id, set_status=True ) + uci_wrapper.set_error( "Error while creating snapshot: " + str( ex ), True ) + return + + uci_wrapper.change_state( uci_state=uci_states.AVAILABLE ) + +# if uci_wrapper.get_state() != uci_states.ERROR: +# +# snapshots = uci_wrapper.get_snapshots( status = 'submitted' ) +# for snapshot in snapshots: +# uci_wrapper.set_snapshot_id( snapshot.id, None, 'euca_error' ) +# +# log.debug( "Eucalyptus snapshot attempted by user for UCI '%s'" % uci_wrapper.get_name() ) +# uci_wrapper.set_error( "Eucalyptus does not support creation of snapshots at this moment. No snapshot or other changes were performed. \ +# Feel free to resent state of this instance and use it normally.", True ) + + + def addStorageToUCI( self, uci_wrapper ): """ Adds more storage to specified UCI """ def dummyStartUCI( self, uci_wrapper ): @@ -432,6 +476,16 @@ # store.uci.flush() # store.flush() + # Update pending snapshots or delete ones marked for deletion + snapshots = model.CloudSnapshot.filter_by( status='pending', status='delete' ).all() + for snapshot in snapshots: + if self.type == snapshot.uci.credentials.provider.type and snapshot.status == 'pending': + log.debug( "[%s] Running general status update on snapshot '%s'" % ( snapshot.uci.credentials.provider.type, snapshot.snapshot_id ) ) + self.update_snapshot( snapshot ) + elif self.type == snapshot.uci.credentials.provider.type and snapshot.status == 'delete': + log.debug( "[%s] Initiating deletion of snapshot '%s'" % ( snapshot.uci.credentials.provider.type, snapshot.snapshot_id ) ) + self.delete_snapshot( snapshot ) + # Attempt at updating any zombie UCIs (i.e., instances that have been in SUBMITTED state for longer than expected - see below for exact time) zombies = model.UCI.filter_by( state=uci_states.SUBMITTED ).all() for zombie in zombies: @@ -548,10 +602,86 @@ except boto.exception.EC2ResponseError, e: log.error( "Updating status of volume(s) from cloud for UCI '%s' failed: " % ( uci.name, str(e) ) ) uci.error( "Updating volume status from cloud failed: " + str(e) ) - uci.state( uci_states.ERROR ) + uci.state = uci_states.ERROR uci.flush() return None - + + def update_snapshot( self, snapshot ): + # Get credentials associated wit this store + uci_id = snapshot.uci_id + uci = model.UCI.get( uci_id ) + uci.refresh() + conn = self.get_connection_from_uci( uci ) + + try: + log.debug( "Updating status of snapshot '%s'" % snapshot.snapshot_id ) + snap = conn.get_all_snapshots( [snapshot.snapshot_id] ) + if len( snap ) > 0: + snapshot.status = snap[0].status + log.debug( "Snapshot '%s' status: %s" % ( snapshot.snapshot_id, snapshot.status ) ) + snapshot.flush() + else: + log.error( "No snapshots returned by cloud provider for UCI '%s'" % uci.name ) + snapshot.status = 'No snapshots returned by cloud provider.' + uci.error = "No snapshots returned by cloud provider." + uci.state = uci_states.ERROR + uci.flush() + snapshot.flush() + except boto.exception.EC2ResponseError, e: + log.error( "Cloud provider response error while updating snapshot: '%s'" % e ) + snapshot.status = 'error' + snapshot.error = "Cloud provider response error while updating snapshot status: " + str( e ) + uci.error = "Cloud provider response error while updating snapshot status: " + str( e ) + uci.state = uci_states.ERROR + uci.flush() + snapshot.flush() + except Exception, ex: + log.error( "Error while updating snapshot: '%s'" % ex ) + snapshot.status = 'error' + snapshot.error = "Error while updating snapshot status: " + str( e ) + uci.error = "Error while updating snapshot status: " + str( ex ) + uci.state = uci_states.ERROR + uci.flush() + snapshot.flush() + + def delete_snapshot( self, snapshot ): + if snapshot.status == 'delete': + # Get credentials associated wit this store + uci_id = snapshot.uci_id + uci = model.UCI.get( uci_id ) + uci.refresh() + conn = self.get_connection_from_uci( uci ) + + try: + log.debug( "Deleting snapshot '%s'" % snapshot.snapshot_id ) + snap = conn.delete_snapshot( snapshot.snapshot_id ) + if snap == True: + snapshot.deleted = True + snapshot.status = 'deleted' + snapshot.flush() + return snap + except boto.exception.EC2ResponseError, e: + log.error( "EC2 response error while deleting snapshot: '%s'" % e ) + snapshot.status = 'error' + snapshot.error = "Cloud provider response error while deleting snapshot: " + str( e ) + uci.error = "Cloud provider response error while deleting snapshot: " + str( e ) + uci.state = uci_states.ERROR + uci.flush() + snapshot.flush() + except Exception, ex: + log.error( "Error while deleting snapshot: '%s'" % ex ) + snapshot.status = 'error' + snapshot.error = "Cloud provider error while deleting snapshot: " + str( ex ) + uci.error = "Cloud provider error while deleting snapshot: " + str( ex ) + uci.state = uci_states.ERROR + uci.flush() + snapshot.flush() + else: + log.error( "Cannot delete snapshot '%s' because its status is '%s'. Only snapshots with 'completed' status can be deleted." % ( snapshot.snapshot_id, snapshot.status ) ) + snapshot.error = "Cannot delete snapshot because its status is '"+snapshot.status+"'. Only snapshots with 'completed' status can be deleted." + snapshot.status = 'error' + snapshot.flush() + def processZombie( self, inst ): """ Attempt at discovering if starting an instance was successful but local database was not updated diff -r c78e8e1514e4 -r c1dc30106721 lib/galaxy/model/__init__.py --- a/lib/galaxy/model/__init__.py Tue Nov 10 18:36:21 2009 -0500 +++ b/lib/galaxy/model/__init__.py Wed Nov 11 20:11:58 2009 -0500 @@ -963,6 +963,13 @@ self.size = None self.availability_zone = None +class CloudSnapshot( object ): + def __init__( self ): + self.id = None + self.user = None + self.store_id = None + self.snapshot_id = None + class CloudProvider( object ): def __init__( self ): self.id = None diff -r c78e8e1514e4 -r c1dc30106721 lib/galaxy/model/mapping.py --- a/lib/galaxy/model/mapping.py Tue Nov 10 18:36:21 2009 -0500 +++ b/lib/galaxy/model/mapping.py Wed Nov 11 20:11:58 2009 -0500 @@ -448,6 +448,19 @@ Column( "space_consumed", Integer ), Column( "deleted", Boolean, default=False ) ) +CloudSnapshot.table = Table( "cloud_snapshot", metadata, + Column( "id", Integer, primary_key=True ), + Column( "create_time", DateTime, default=now ), + Column( "update_time", DateTime, default=now, onupdate=now ), + Column( "user_id", Integer, ForeignKey( "galaxy_user.id" ), index=True, nullable=False ), + Column( "uci_id", Integer, ForeignKey( "cloud_uci.id" ), index=True ), + Column( "store_id", Integer, ForeignKey( "cloud_store.id" ), index=True, nullable=False ), + Column( "snapshot_id", TEXT ), + Column( "status", TEXT ), + Column( "description", TEXT ), + Column( "error", TEXT ), + Column( "deleted", Boolean, default=False ) ) + CloudUserCredentials.table = Table( "cloud_user_credentials", metadata, Column( "id", Integer, primary_key=True ), Column( "create_time", DateTime, default=now ), @@ -995,7 +1008,8 @@ properties=dict( user=relation( User ), credentials=relation( CloudUserCredentials ), instance=relation( CloudInstance, backref='uci' ), - store=relation( CloudStore, backref='uci', cascade='all, delete-orphan' ) + store=relation( CloudStore, backref='uci', cascade='all, delete-orphan' ), + snapshot=relation( CloudSnapshot, backref='uci' ) ) ) assign_mapper( context, CloudInstance, CloudInstance.table, @@ -1005,7 +1019,12 @@ assign_mapper( context, CloudStore, CloudStore.table, properties=dict( user=relation( User ), - i=relation( CloudInstance ) + i=relation( CloudInstance ), + snapshot=relation( CloudSnapshot, backref="store" ) + ) ) + +assign_mapper( context, CloudSnapshot, CloudSnapshot.table, + properties=dict( user=relation( User ) ) ) assign_mapper( context, CloudProvider, CloudProvider.table, diff -r c78e8e1514e4 -r c1dc30106721 lib/galaxy/model/migrate/versions/0014_cloud_tables.py --- a/lib/galaxy/model/migrate/versions/0014_cloud_tables.py Tue Nov 10 18:36:21 2009 -0500 +++ b/lib/galaxy/model/migrate/versions/0014_cloud_tables.py Wed Nov 11 20:11:58 2009 -0500 @@ -80,6 +80,19 @@ Column( "space_consumed", Integer ), Column( "deleted", Boolean, default=False ) ) +CloudSnapshot_table = Table( "cloud_snapshot", metadata, + Column( "id", Integer, primary_key=True ), + Column( "create_time", DateTime, default=now ), + Column( "update_time", DateTime, default=now, onupdate=now ), + Column( "user_id", Integer, ForeignKey( "galaxy_user.id" ), index=True, nullable=False ), + Column( "uci_id", Integer, ForeignKey( "cloud_uci.id" ), index=True ), + Column( "store_id", Integer, ForeignKey( "cloud_store.id" ), index=True, nullable=False ), + Column( "snapshot_id", TEXT ), + Column( "status", TEXT ), + Column( "description", TEXT ), + Column( "error", TEXT ), + Column( "deleted", Boolean, default=False ) ) + CloudUserCredentials_table = Table( "cloud_user_credentials", metadata, Column( "id", Integer, primary_key=True ), Column( "create_time", DateTime, default=now ), @@ -118,19 +131,21 @@ # Load existing tables metadata.reflect() - CloudImage_table.create() - UCI_table.create() - CloudUserCredentials_table.create() - CloudStore_table.create() - CloudInstance_table.create() - CloudProvider_table.create() +# CloudImage_table.create() +# UCI_table.create() +# CloudUserCredentials_table.create() +# CloudStore_table.create() + CloudSnapshot_table.create() +# CloudInstance_table.create() +# CloudProvider_table.create() def downgrade(): metadata.reflect() - CloudImage_table.drop() - CloudInstance_table.drop() - CloudStore_table.drop() - CloudUserCredentials_table.drop() - UCI_table.drop() - CloudProvider_table.drop() \ No newline at end of file +# CloudImage_table.drop() +# CloudInstance_table.drop() +# CloudStore_table.drop() + CloudSnapshot_table.drop() +# CloudUserCredentials_table.drop() +# UCI_table.drop() +# CloudProvider_table.drop() \ No newline at end of file diff -r c78e8e1514e4 -r c1dc30106721 lib/galaxy/web/controllers/cloud.py --- a/lib/galaxy/web/controllers/cloud.py Tue Nov 10 18:36:21 2009 -0500 +++ b/lib/galaxy/web/controllers/cloud.py Wed Nov 11 20:11:58 2009 -0500 @@ -44,7 +44,9 @@ RUNNING = "running", PENDING = "pending", ERROR = "error", - DELETED = "deleted" + DELETED = "deleted", + SNAPSHOT_UCI = "snapshotUCI", + SNAPSHOT = "snapshot" ) instance_states = Bunch( @@ -106,7 +108,9 @@ model.UCI.c.state==uci_states.NEW_UCI, model.UCI.c.state==uci_states.ERROR, model.UCI.c.state==uci_states.DELETING, - model.UCI.c.state==uci_states.DELETING_UCI ) ) \ + model.UCI.c.state==uci_states.DELETING_UCI, + model.UCI.c.state==uci_states.SNAPSHOT, + model.UCI.c.state==uci_states.SNAPSHOT_UCI ) ) \ .order_by( desc( model.UCI.c.update_time ) ) \ .all() @@ -223,6 +227,87 @@ return self.list( trans ) @web.expose + @web.require_login( "use Galaxy cloud" ) + def create_snapshot( self, trans, id ): + user = trans.get_user() + id = trans.security.decode_id( id ) + uci = get_uci( trans, id ) + + stores = trans.sa_session.query( model.CloudStore ) \ + .filter_by( user=user, deleted=False, uci_id=id ) \ + .all() + + if ( len( stores ) > 0 ) and ( uci.state == uci_states.AVAILABLE ): + for store in stores: + snapshot = model.CloudSnapshot() + snapshot.user = user + snapshot.uci = uci + snapshot.store = store + snapshot.status = 'submitted' + uci.state = uci_states.SNAPSHOT_UCI + # Persist + session = trans.sa_session + session.save_or_update( snapshot ) + session.save_or_update( uci ) + session.flush() + elif len( stores ) == 0: + error( "No storage volumes found that are associated with this instance." ) + else: + error( "Snapshot can be created only for an instance that is in 'available' state." ) + + # Log and display the management page + trans.log_event( "User initiated creation of new snapshot." ) + trans.set_message( "Creation of new snapshot initiated. " ) + return self.list( trans ) + + @web.expose + @web.require_login( "use Galaxy cloud" ) + def view_snapshots( self, trans, id=None ): + """ + View details about any snapshots associated with given UCI + """ + user = trans.get_user() + id = trans.security.decode_id( id ) + + snaps = trans.sa_session.query( model.CloudSnapshot ) \ + .filter_by( user=user, uci_id=id, deleted=False ) \ + .order_by( desc( model.CloudSnapshot.c.update_time ) ) \ + .all() + + return trans.fill_template( "cloud/view_snapshots.mako", + snaps = snaps ) + + @web.expose + @web.require_login( "use Galaxy cloud" ) + def delete_snapshot( self, trans, uci_id=None, snap_id=None ): + """ + Initiates deletion of a snapshot + """ + user = trans.get_user() + snap_id = trans.security.decode_id( snap_id ) + # Set snapshot as 'ready for deletion' to be picked up by general updater + snap = trans.sa_session.query( model.CloudSnapshot ).get( snap_id ) + + if snap.status == 'completed': + snap.status = 'delete' + snap.flush() + trans.set_message( "Snapshot '%s' is marked for deletion. Once the deletion is complete, it will no longer be visible in this list. " + "Please note that this process may take up to a minute." % snap.snapshot_id ) + else: + error( "Only snapshots in state 'completed' can be deleted. See the cloud provider directly " + "if you believe the snapshot is available and can be deleted." ) + + # Display new list of snapshots + uci_id = trans.security.decode_id( uci_id ) + snaps = trans.sa_session.query( model.CloudSnapshot ) \ + .filter_by( user=user, uci_id=uci_id, deleted=False ) \ + .order_by( desc( model.CloudSnapshot.c.update_time ) ) \ + .all() + + return trans.fill_template( "cloud/view_snapshots.mako", + snaps = snaps ) + + @web.expose @web.require_login( "add instance storage" ) def addStorage( self, trans, id ): instance = get_uci( trans, id ) diff -r c78e8e1514e4 -r c1dc30106721 templates/cloud/configure_cloud.mako --- a/templates/cloud/configure_cloud.mako Tue Nov 10 18:36:21 2009 -0500 +++ b/templates/cloud/configure_cloud.mako Wed Nov 11 20:11:58 2009 -0500 @@ -20,17 +20,26 @@ ${h.js( "jquery" )} <script type="text/javascript"> +function trim19(str){ + var str = str.replace(/^\s\s*/, ''), + ws = /\s/, + i = str.length; + while (ws.test(str.charAt(--i))); + return str.slice(0, i + 1); +} + function update_state() { $.getJSON( "${h.url_for( action='json_update' )}", {}, function ( data ) { for (var i in data) { var elem = '#' + data[i].id; // Because of different list managing 'live' vs. 'available' instances, reload url on various state changes old_state = $(elem + "-state").text(); - prev_old_state = $(elem + "-state-p").text(); + prev_old_state = trim19( $(elem + "-state-p").text() ); new_state = data[i].state; console.log( "old_state[%d] = %s", i, old_state ); console.log( "prev_old_state[%d] = %s", i, prev_old_state ); console.log( "new_state[%d] = %s", i, new_state ); + //console.log( trim19(prev_old_state) ); if ( ( old_state=='pending' && new_state=='running' ) || ( old_state=='shutting-down' && new_state=='available' ) || \ ( old_state=='running' && new_state=='available' ) || ( old_state=='running' && new_state=='error' ) || \ ( old_state=='pending' && new_state=='error' ) || ( old_state=='pending' && new_state=='available' ) || \ @@ -44,9 +53,9 @@ ( old_state=='submitted' && new_state=='error' ) || ( old_state=='submittedUCI' && new_state=='error' ) || \ ( old_state=='shutting-down' && new_state=='error' ) || ( prev_old_state.match('newUCI') && new_state=='error' ) || \ ( prev_old_state.match('new') && new_state=='error' ) || \ - ( prev_old_state.match('deleting') && new_state=='error' ) || ( prev_old_state.match('deletingUCI') && new_state=='error' ) ) { + ( prev_old_state.match('deletingUCI') && new_state=='error' ) ) { // TODO: Following clause causes constant page refresh for an exception thrown as a result of instance not starting correctly - need alternative method! - //( prev_old_state.match('available') && new_state=='error' ) || \ + //( prev_old_state.match('available') && new_state=='error' ) || ( prev_old_state.match('deleting') && new_state=='error' ) \ var url = "${h.url_for( controller='cloud', action='list')}"; location.replace( url ); @@ -80,6 +89,7 @@ // Update 'state' and 'time alive' fields $(elem + "-state").text( data[i].state ); + //$(elem + "-state-p").text( data[i].state ); if (data[i].launch_time) { $(elem + "-launch_time").text( data[i].launch_time.substring(0, 16 ) + " UTC (" + data[i].time_ago + ")" ); } @@ -313,6 +323,8 @@ <a class="action-button" href="${h.url_for( action='start', id=trans.security.encode_id(prevInstance.id), type='m1.small' )}"> Start m1.small</a> <a class="action-button" href="${h.url_for( action='start', id=trans.security.encode_id(prevInstance.id), type='c1.medium' )}"> Start c1.medium</a> <a class="action-button" href="${h.url_for( action='renameInstance', id=trans.security.encode_id(prevInstance.id) )}">Rename</a> + <a class="action-button" href="${h.url_for( action='create_snapshot', id=trans.security.encode_id(prevInstance.id) )}">Create snapshot</a> + <a class="action-button" href="${h.url_for( action='view_snapshots', id=trans.security.encode_id(prevInstance.id) )}">View snapshots</a> <a class="action-button" href="${h.url_for( action='addStorage', id=trans.security.encode_id(prevInstance.id) )}" target="_parent">Add storage</a> <a class="action-button" href="${h.url_for( action='usageReport', id=trans.security.encode_id(prevInstance.id) )}">Usage report</a> <a class="action-button" confirm="Are you sure you want to delete instance '${prevInstance.name}'? This will delete all of your data assocaiated with this instance!" href="${h.url_for( action='deleteInstance', id=trans.security.encode_id(prevInstance.id) )}">Delete</a> diff -r c78e8e1514e4 -r c1dc30106721 templates/cloud/view_snapshots.mako --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/templates/cloud/view_snapshots.mako Wed Nov 11 20:11:58 2009 -0500 @@ -0,0 +1,90 @@ +<%inherit file="/base.mako"/> + +<%def name="title()">Snapshots</%def> + +%if message: +<% + try: + messagetype + except: + messagetype = "done" +%> + + + +<p /> +<div class="${messagetype}message"> + ${message} +</div> +%endif + +%if snaps: + <h2>Snapshots for instance ${snaps[0].uci.name}</h2> +%else: + <h2>Selected instance has no recorded or associated snapshots.</h2> +%endif + +<ul class="manage-table-actions"> + <li> + <a class="action-button" href="${h.url_for( action='list' )}"> + <img src="${h.url_for('/static/images/silk/resultset_previous.png')}" /> + <span>Return to cloud management console</span> + </a> + </li> +</ul> + +%if snaps: + <table class="mange-table colored" border="0" cellspacing="0" cellpadding="0" width="100%"> + <colgroup width="2%"></colgroup> + <colgroup width="16%"></colgroup> + <colgroup width="16%"></colgroup> + <colgroup width="10%"></colgroup> + <colgroup width="5%"></colgroup> + <tr class="header"> + <th>#</th> + <th>Create time</th> + <th>Snapshot ID</th> + <th>Status</th> + <th>Delete?</th> + <th></th> + </tr> + <% + total_hours = 0 + %> + %for i, snap in enumerate( snaps ): + <tr> + <td>${i+1}</td> + <td> + %if snap.create_time: + ${str(snap.create_time)[:16]} UCT + %else: + N/A + %endif + </td> + <td> + %if snap.snapshot_id: + ${snap.snapshot_id} + %else: + N/A + %endif + </td> + <td> + %if snap.status: + ${snap.status} + %else: + N/A + %endif + </td> + <td> + <a confirm="Are you sure you want to delete snapshot '${snap.snapshot_id}'?" + href="${h.url_for( controller='cloud', action='delete_snapshot', uci_id=trans.security.encode_id(snap.uci.id), snap_id=trans.security.encode_id(snap.id) )}">x</a> + </td> + </tr> + %endfor + </table> +%endif + + + + +

1 0

[hg] galaxy 3086: Implemented on-line checking of zone availabil...
by Greg Von Kuster 23 Nov '09

23 Nov '09

details: http://www.bx.psu.edu/hg/galaxy/rev/5963dd169715 changeset: 3086:5963dd169715 user: Enis Afgan <afgane(a)gmail.com> date: Fri Nov 06 13:45:57 2009 -0500 description: Implemented on-line checking of zone availability when registering a UCI. This introduces provider-specific ties in cloud.py controller that will need to be modified if non-EC2 cloud providers are added. diffstat: lib/galaxy/cloud/providers/ec2.py | 1 + lib/galaxy/cloud/providers/eucalyptus.py | 1 + lib/galaxy/web/controllers/cloud.py | 72 ++++++++++++++++++++++++----------- 3 files changed, 51 insertions(+), 23 deletions(-) diffs (127 lines): diff -r dc1a6f3a2c08 -r 5963dd169715 lib/galaxy/cloud/providers/ec2.py --- a/lib/galaxy/cloud/providers/ec2.py Fri Nov 06 12:27:52 2009 -0500 +++ b/lib/galaxy/cloud/providers/ec2.py Fri Nov 06 13:45:57 2009 -0500 @@ -633,6 +633,7 @@ log.error( "Establishing connection with cloud failed: %s" % str(e) ) uci.error( "Establishing connection with cloud failed: " + str(e) ) uci.state( uci_states.ERROR ) + uci.flush() return None return conn diff -r dc1a6f3a2c08 -r 5963dd169715 lib/galaxy/cloud/providers/eucalyptus.py --- a/lib/galaxy/cloud/providers/eucalyptus.py Fri Nov 06 12:27:52 2009 -0500 +++ b/lib/galaxy/cloud/providers/eucalyptus.py Fri Nov 06 13:45:57 2009 -0500 @@ -599,6 +599,7 @@ log.error( "Establishing connection with cloud failed: %s" % str(e) ) uci.error( "Establishing connection with cloud failed: " + str(e) ) uci.state( uci_states.ERROR ) + uci.flush() return None return conn diff -r dc1a6f3a2c08 -r 5963dd169715 lib/galaxy/web/controllers/cloud.py --- a/lib/galaxy/web/controllers/cloud.py Fri Nov 06 12:27:52 2009 -0500 +++ b/lib/galaxy/web/controllers/cloud.py Fri Nov 06 13:45:57 2009 -0500 @@ -25,6 +25,8 @@ from boto.ec2.connection import EC2Connection from boto.ec2.regioninfo import RegionInfo from galaxy.cloud import CloudManager +import boto.exception +import boto import logging log = logging.getLogger( __name__ ) @@ -270,22 +272,38 @@ user = trans.get_user() storedCreds = trans.sa_session.query( model.CloudUserCredentials ).filter_by( user=user ).all() if len( storedCreds ) == 0: - return trans.show_error_message( "You must register credentials before configuring a Galaxy instance." ) - # TODO: This should be filled automatically but ties to implementation for diff provider is a problem... + return trans.show_error_message( "You must register credentials before configuring a Galaxy cloud instance." ) # Create dict mapping of cloud providers to zones available by those providers providersToZones = {} for storedCred in storedCreds: - if storedCred.provider.region_name == 'us-east-1': - ec2_zones = ['us-east-1a', 'us-east-1b', 'us-east-1c', 'us-east-1d'] - providersToZones[storedCred.name] = ec2_zones - elif storedCred.provider.region_name == 'eu-west-1': - ec2_zones = ['eu-west-1a', 'eu-west-1b'] - providersToZones[storedCred.name] = ec2_zones - elif storedCred.provider.type == 'eucalyptus': - providersToZones[storedCred.name] = ['epc'] + zones = None + conn = get_connection( trans, storedCred ) + if conn != None: + avail_zones = [] + try: + zones = conn.get_all_zones() + for zone in zones: + zone = str( zone ).split(':')[1] + avail_zones.append( zone ) + providersToZones[storedCred.name] = avail_zones + except boto.exception.EC2ResponseError, e: + log.error( "Retrieving zones for credentials '%s' failed." % storedCred.name ) + providersToZones[storedCred.name] = ['Unknown provider zone'] else: providersToZones[storedCred.name] = ['Unknown provider zone'] + # Hard-coded solution +# if storedCred.provider.storedCred.provider.region_name == 'us-east-1': +# ec2_zones = ['us-east-1a', 'us-east-1b', 'us-east-1c', 'us-east-1d'] +# providersToZones[storedCred.name] = ec2_zones +# elif storedCred.provider.region_name == 'eu-west-1': +# ec2_zones = ['eu-west-1a', 'eu-west-1b'] +# providersToZones[storedCred.name] = ec2_zones +# elif storedCred.provider.type == 'eucalyptus': +# providersToZones[storedCred.name] = ['epc'] +# else: +# providersToZones[storedCred.name] = ['Unknown provider zone'] + if instanceName: # Check if volume size is entered as an integer try: @@ -1030,22 +1048,30 @@ return instances -def get_connection( trans, credName ): +def get_connection( trans, creds ): """ - Establishes EC2 connection using user's default credentials + Establishes cloud connection using user's credentials """ - log.debug( '##### Establishing cloud connection.' ) - user = trans.get_user() - creds = trans.sa_session.query( model.CloudUserCredentials ).filter_by( user=user, name=credName ).first() + log.debug( 'Establishing cloud connection.' ) +# user = trans.get_user() +# creds = trans.sa_session.query( model.CloudUserCredentials ) \ +# .filter_by( user=user, name=credName ) \ +# .first() + #.filter( model.CloudUserCredentials.c.deleted != True ) \ MOVE TO LINE ABOVE ONCE DELETE COLUMS ARE IMPLEMENTED + if creds: a_key = creds.access_key s_key = creds.secret_key - # Amazon EC2 - #conn = EC2Connection( a_key, s_key ) - # Eucalyptus Public Cloud - euca_region = RegionInfo( None, "eucalyptus", "mayhem9.cs.ucsb.edu" ) - conn = EC2Connection( aws_access_key_id=a_key, aws_secret_access_key=s_key, is_secure=False, port=8773, region=euca_region, path="/services/Eucalyptus" ) + try: + euca_region = RegionInfo( None, creds.provider.region_name, creds.provider.region_endpoint ) + conn = EC2Connection( aws_access_key_id=a_key, + aws_secret_access_key=s_key, + is_secure=creds.provider.is_secure, + port=creds.provider.port, + region=euca_region, + path=creds.provider.path ) + except boto.exception.EC2ResponseError, e: + log.error( "Establishing connection with cloud failed: %s" % str(e) ) + return None + return conn - else: - error( "You must specify default credentials before starting an instance." ) - return 0

1 0

[hg] galaxy 3087: Changed DB tables to support ghost deletion. C...
by Greg Von Kuster 23 Nov '09

23 Nov '09

details: http://www.bx.psu.edu/hg/galaxy/rev/7c4cfc243cc3 changeset: 3087:7c4cfc243cc3 user: Enis Afgan <afgane(a)gmail.com> date: Mon Nov 09 18:55:14 2009 -0500 description: Changed DB tables to support ghost deletion. Changed keypair saving location to UCI table. Adjusted code accordingly. *Have not tested with cloud providers yet thoughtadd templates/cloud/add_image.mako diffstat: lib/galaxy/cloud/__init__.py | 86 ++++++++--- lib/galaxy/cloud/providers/ec2.py | 146 +++++++++++--------- lib/galaxy/cloud/providers/eucalyptus.py | 130 ++++++++++-------- lib/galaxy/model/__init__.py | 1 - lib/galaxy/model/mapping.py | 46 +++-- lib/galaxy/model/migrate/versions/0014_cloud_tables.py | 44 ++++-- lib/galaxy/web/controllers/cloud.py | 126 +++++++++-------- templates/cloud/add_image.mako | 98 ++++++++++++++ templates/cloud/configure_cloud.mako | 5 +- templates/cloud/edit_image.mako | 34 ++++- templates/cloud/list_images.mako | 22 ++- templates/cloud/view.mako | 6 +- templates/cloud/viewInstance.mako | 14 +- 13 files changed, 501 insertions(+), 257 deletions(-) diffs (1468 lines): diff -r 5963dd169715 -r 7c4cfc243cc3 lib/galaxy/cloud/__init__.py --- a/lib/galaxy/cloud/__init__.py Fri Nov 06 13:45:57 2009 -0500 +++ b/lib/galaxy/cloud/__init__.py Mon Nov 09 18:55:14 2009 -0500 @@ -23,6 +23,7 @@ NEW = "new", DELETING_UCI = "deletingUCI", DELETING = "deleting", + DELETED = "deleted", SUBMITTED_UCI = "submittedUCI", SUBMITTED = "submitted", SHUTTING_DOWN_UCI = "shutting-downUCI", @@ -139,16 +140,15 @@ for uci_wrapper in new_requests: session.clear() - self.provider.put( uci_wrapper ) + self.put( uci_wrapper ) # Done with the session mapping.Session.remove() - def put( self, job_id, tool ): - """Add a job to the queue (by job identifier)""" - if not self.track_jobs_in_database: - self.queue.put( ( job_id, tool.id ) ) - self.sleeper.wake() + def put( self, uci_wrapper ): + """Add a request to the queue.""" + self.provider.put( uci_wrapper ) + self.sleeper.wake() def shutdown( self ): """Attempts to gracefully shut down the worker thread""" @@ -156,7 +156,7 @@ # We're not the real queue, do nothing return else: - log.info( "sending stop signal to worker thread" ) + log.info( "Sending stop signal to worker thread" ) self.running = False self.sleeper.wake() log.info( "cloud manager stopped" ) @@ -175,7 +175,7 @@ """ Sets state for UCI and/or UCI's instance with instance_id as provided by cloud provider and stored in local Galaxy database. - Need to provide either state for the UCI or instance_id and it's state or all arguments. + Need to provide either: (1) state for the UCI, or (2) instance_id and it's state, or (3) all arguments. """ # log.debug( "Changing state - new uci_state: %s, instance_id: %s, i_state: %s" % ( uci_state, instance_id, i_state ) ) if uci_state is not None: @@ -198,22 +198,22 @@ instance.image = mi instance.flush() - def set_key_pair( self, i_index, key_name, key_material=None ): + def set_key_pair( self, key_name, key_material=None ): """ - Single UCI may instantiate many instances, i_index refers to the numeric index - of instance controlled by this UCI as it is stored in local DB (see get_instances_ids()). + Sets key pair value for current UCI. """ - instance = model.CloudInstance.get( i_index ) - instance.keypair_name = key_name + uci = model.UCI.get( self.uci_id ) + uci.refresh() + uci.key_pair_name = key_name if key_material is not None: - instance.keypair_material = key_material - instance.flush() + uci.key_pair_material = key_material + uci.flush() def set_launch_time( self, launch_time, i_index=None, i_id=None ): """ - Stores launch time in local database for instance with specified index (as it is stored in local - Galaxy database) or with specified instance ID (as obtained from the cloud provider AND stored - in local Galaxy Database). Only one of i_index or i_id needs to be provided. + Stores launch time in local database for instance with specified index - i_index (as it is stored in local + Galaxy database) or with specified instance ID - i_id (as obtained from the cloud provider AND stored + in local Galaxy Database). Either 'i_index' or 'i_id' needs to be provided. """ if i_index != None: instance = model.CloudInstance.get( i_index ) @@ -231,6 +231,11 @@ uci.flush() def set_stop_time( self, stop_time, i_index=None, i_id=None ): + """ + Stores stop time in local database for instance with specified index - i_index (as it is stored in local + Galaxy database) or with specified instance ID - i_id (as obtained from the cloud provider AND stored + in local Galaxy Database). Either 'i_index' or 'i_id' needs to be provided. + """ if i_index != None: instance = model.CloudInstance.get( i_index ) instance.stop_time = stop_time @@ -245,6 +250,21 @@ uci.refresh() uci.launch_time = None uci.flush() + + def set_security_group_name( self, security_group_name, i_index=None, i_id=None ): + """ + Stores security group name in local database for instance with specified index - i_index (as it is stored in local + Galaxy database) or with specified instance ID - i_id (as obtained from the cloud provider AND stored + in local Galaxy Database). Either 'i_index' or 'i_id' needs to be provided. + """ + if i_index != None: + instance = model.CloudInstance.get( i_index ) + instance.security_group = security_group_name + instance.flush() + elif i_id != None: + instance = model.CloudInstance.filter_by( uci_id=self.uci_id, instance_id=i_id).first() + instance.security_group = security_group_name + instance.flush() def set_reservation_id( self, i_index, reservation_id ): instance = model.CloudInstance.get( i_index ) @@ -382,19 +402,35 @@ uci.refresh() return uci.name - def get_key_pair_name( self, i_index=None, i_id=None ): + def get_key_pair_name( self ): + """ + Returns keypair name associated with given UCI. + """ + uci = model.UCI.get( self.uci_id ) + uci.refresh() + return uci.key_pair_name + + def get_key_pair_material( self ): + """ + Returns keypair material (i.e., private key) associated with given UCI. + """ + uci = model.UCI.get( self.uci_id ) + uci.refresh() + return uci.key_pair_material + + def get_security_group_name( self, i_index=None, i_id=None ): """ Given EITHER instance index as it is stored in local Galaxy database OR instance ID as it is - obtained from cloud provider and stored in local Galaxy database, return keypair name assocaited + obtained from cloud provider and stored in local Galaxy database, return security group name associated with given instance. """ if i_index != None: instance = model.CloudInstance.get( i_index ) - return instance.keypair_name + return instance.security_group elif i_id != None: instance = model.CloudInstance.filter_by( uci_id=self.uci_id, instance_id=i_id).first() - return instance.keypair_name - + return instance.security_group + def get_access_key( self ): uci = model.UCI.get( self.uci_id ) uci.refresh() @@ -469,8 +505,8 @@ def delete( self ): uci = model.UCI.get( self.uci_id ) uci.refresh() -# uci.delete() - uci.state = 'deleted' # for bookkeeping reasons, mark as deleted but don't actually delete. + uci.state = uci_states.DELETED # for bookkeeping reasons, mark as deleted but don't actually delete. + uci.deleted = True uci.flush() class CloudProvider( object ): diff -r 5963dd169715 -r 7c4cfc243cc3 lib/galaxy/cloud/providers/ec2.py --- a/lib/galaxy/cloud/providers/ec2.py Fri Nov 06 13:45:57 2009 -0500 +++ b/lib/galaxy/cloud/providers/ec2.py Mon Nov 09 18:55:14 2009 -0500 @@ -58,11 +58,12 @@ self.type = "ec2" # cloud provider type (e.g., ec2, eucalyptus, opennebula) self.zone = "us-east-1a" self.key_pair = "galaxy-keypair" + self.security_group = "galaxyWeb" self.queue = Queue() self.threads = [] nworkers = 5 - log.info( "Starting EC2 cloud controller workers" ) + log.info( "Starting EC2 cloud controller workers..." ) for i in range( nworkers ): worker = threading.Thread( target=self.run_next ) worker.start() @@ -116,28 +117,43 @@ return conn - def set_keypair( self, uci_wrapper, conn ): + def check_key_pair( self, uci_wrapper, conn ): """ Generate keypair using user's default credentials """ - log.debug( "Getting user's keypair" ) - instances = uci_wrapper.get_instances_indexes() + log.debug( "Getting user's key pair: '%s'" % self.key_pair ) try: kp = conn.get_key_pair( self.key_pair ) - for inst in instances: - uci_wrapper.set_key_pair( inst, kp.name ) - return kp.name + uci_kp = uci_wrapper.get_key_pair_name() + uci_material = uci_wrapper.get_key_pair_material() + if kp.name != uci_kp or uci_material == None: + try: # key pair exists on the cloud but not in local database, so re-generate it (i.e., delete and then create) + conn.delete_key_pair( self.key_pair ) + except boto.exception.EC2ResponseError: + pass + kp = self.create_key_pair( conn ) + uci_wrapper.set_key_pair( kp.name, kp.material ) + else: + return kp.name except boto.exception.EC2ResponseError, e: # No keypair under this name exists so create it if e.code == 'InvalidKeyPair.NotFound': log.info( "No keypair found, creating keypair '%s'" % self.key_pair ) - kp = conn.create_key_pair( self.key_pair ) - for inst in instances: - uci_wrapper.set_key_pair( inst, kp.name, kp.material ) + kp = self.create_key_pair( conn ) + uci_wrapper.set_key_pair( kp.name, kp.material ) else: log.error( "EC2 response error: '%s'" % e ) - uci_wrapper.set_error( "EC2 response error while creating key pair: " + str(e), True ) - - return kp.name + uci_wrapper.set_error( "EC2 response error while creating key pair: " + str( e ), True ) + + if kp != None: + return kp.name + else: + return None + + def create_key_pair( self, conn ): + try: + return conn.create_key_pair( self.key_pair ) + except boto.exception.EC2ResponseError, e: + return None def get_mi_id( self, type ): """ @@ -170,7 +186,6 @@ log.info( "Availability zone for UCI (i.e., storage volume) was not selected, using default zone: %s" % self.zone ) uci_wrapper.set_store_availability_zone( self.zone ) - #TODO: check if volume associated with UCI already exists (if server crashed for example) and don't recreate it log.info( "Creating volume in zone '%s'..." % uci_wrapper.get_uci_availability_zone() ) # Because only 1 storage volume may be created at UCI config time, index of this storage volume in local Galaxy DB w.r.t # current UCI is 0, so reference it in following methods @@ -196,7 +211,7 @@ else: uci_wrapper.change_state( uci_state=uci_states.ERROR ) uci_wrapper.set_store_status( vol.id, uci_states.ERROR ) - uci_wrapper.set_error( "Volume '%s' not found by cloud provider after being created" % vol.id ) + uci_wrapper.set_error( "Volume '%s' not found by EC2 after being created" % vol.id ) def deleteUCI( self, uci_wrapper ): """ @@ -215,14 +230,13 @@ log.debug( "Deleting volume with id='%s'" % v.volume_id ) if conn.delete_volume( v.volume_id ): deletedList.append( v.volume_id ) - v.delete() + v.deleted = True v.flush() count += 1 else: failedList.append( v.volume_id ) # Delete UCI if all of associated - log.debug( "count=%s, len(vl)=%s" % (count, len( vl ) ) ) if count == len( vl ): uci_wrapper.delete() else: @@ -249,31 +263,34 @@ """ Starts instance(s) of given UCI on the cloud. """ - conn = self.get_connection( uci_wrapper ) - if uci_wrapper.get_state() != uci_states.ERROR: - self.set_keypair( uci_wrapper, conn ) - i_indexes = uci_wrapper.get_instances_indexes() # Get indexes of i_indexes associated with this UCI whose state is 'None' - log.debug( "Starting instances with IDs: '%s' associated with UCI '%s' " % ( uci_wrapper.get_name(), i_indexes ) ) - - if uci_wrapper.get_state() != uci_states.ERROR: + conn = self.get_connection( uci_wrapper ) + self.check_key_pair( uci_wrapper, conn ) + + i_indexes = uci_wrapper.get_instances_indexes( state=None ) # Get indexes of i_indexes associated with this UCI whose state is 'None' + log.debug( "Starting instances with IDs: '%s' associated with UCI '%s' " % ( uci_wrapper.get_name(), i_indexes ) ) for i_index in i_indexes: mi_id = self.get_mi_id( uci_wrapper.get_type( i_index ) ) uci_wrapper.set_mi( i_index, mi_id ) # Check if galaxy security group exists (and create it if it does not) - security_group = 'galaxyWeb' - log.debug( "Setting up '%s' security group." % security_group ) - sgs = conn.get_all_security_groups() # security groups - gsgt = False # galaxy security group test - for sg in sgs: - if sg.name == security_group: - gsgt = True - # If security group does not exist, create it - if not gsgt: - gSecurityGroup = conn.create_security_group(security_group, 'Security group for Galaxy.') - gSecurityGroup.authorize( 'tcp', 80, 80, '0.0.0.0/0' ) # Open HTTP port - gSecurityGroup.authorize( 'tcp', 22, 22, '0.0.0.0/0' ) # Open SSH port + log.debug( "Setting up '%s' security group." % self.security_group ) + try: + conn.get_all_security_groups( [self.security_group] ) # security groups + except boto.exception.EC2ResponseError, e: + if e.code == 'InvalidGroup.NotFound': + log.info( "No security group found, creating security group '%s'" % self.security_group ) + try: + gSecurityGroup = conn.create_security_group(self.security_group, 'Security group for Galaxy.') + gSecurityGroup.authorize( 'tcp', 80, 80, '0.0.0.0/0' ) # Open HTTP port + gSecurityGroup.authorize( 'tcp', 22, 22, '0.0.0.0/0' ) # Open SSH port + except boto.exception.EC2ResponseError, ex: + log.error( "EC2 response error while creating security group: '%s'" % e ) + uci_wrapper.set_error( "EC2 response error while creating security group: " + str( e ), True ) + else: + log.error( "EC2 response error while retrieving security group: '%s'" % e ) + uci_wrapper.set_error( "EC2 response error while retrieving security group: " + str( e ), True ) + if uci_wrapper.get_state() != uci_states.ERROR: # Start an instance @@ -281,34 +298,40 @@ #TODO: Once multiple volumes can be attached to a single instance, update 'userdata' composition userdata = uci_wrapper.get_store_volume_id()+"|"+uci_wrapper.get_access_key()+"|"+uci_wrapper.get_secret_key() log.debug( 'Using following command: conn.run_instances( image_id=%s, key_name=%s, security_groups=[%s], user_data=[OMITTED], instance_type=%s, placement=%s )' - % ( mi_id, uci_wrapper.get_key_pair_name( i_index ), [security_group], uci_wrapper.get_type( i_index ), uci_wrapper.get_uci_availability_zone() ) ) + % ( mi_id, uci_wrapper.get_key_pair_name( i_index ), self.security_group, uci_wrapper.get_type( i_index ), uci_wrapper.get_uci_availability_zone() ) ) + reservation = None try: reservation = conn.run_instances( image_id=mi_id, - key_name=uci_wrapper.get_key_pair_name( i_index ), - security_groups=[security_group], - user_data=userdata, - instance_type=uci_wrapper.get_type( i_index ), - placement=uci_wrapper.get_uci_availability_zone() ) + key_name=uci_wrapper.get_key_pair_name( i_index ), + security_groups=[self.security_group], + user_data=userdata, + instance_type=uci_wrapper.get_type( i_index ), + placement=uci_wrapper.get_uci_availability_zone() ) except boto.exception.EC2ResponseError, e: log.error( "EC2 response error when starting UCI '%s': '%s'" % ( uci_wrapper.get_name(), str(e) ) ) uci_wrapper.set_error( "EC2 response error when starting: " + str(e), True ) + except Exception, ex: + log.error( "Error when starting UCI '%s': '%s'" % ( uci_wrapper.get_name(), str( ex ) ) ) + uci_wrapper.set_error( "Cloud provider error when starting: " + str( ex ), True ) # Record newly available instance data into local Galaxy database - l_time = datetime.utcnow() +# l_time = datetime.utcnow() # uci_wrapper.set_launch_time( l_time, i_index=i_index ) # format_time( reservation.i_indexes[0].launch_time ) ) - uci_wrapper.set_launch_time( self.format_time( reservation.instances[0].launch_time ), i_index=i_index ) - if not uci_wrapper.uci_launch_time_set(): - uci_wrapper.set_uci_launch_time( l_time ) - try: - uci_wrapper.set_reservation_id( i_index, str( reservation ).split(":")[1] ) - # TODO: if more than a single instance will be started through single reservation, change this reference to element [0] - i_id = str( reservation.instances[0]).split(":")[1] - uci_wrapper.set_instance_id( i_index, i_id ) - s = reservation.instances[0].state - uci_wrapper.change_state( s, i_id, s ) - log.debug( "Instance of UCI '%s' started, current state: '%s'" % ( uci_wrapper.get_name(), uci_wrapper.get_state() ) ) - except boto.exception.EC2ResponseError, e: - log.error( "EC2 response error when retrieving instance information for UCI '%s': '%s'" % ( uci_wrapper.get_name(), str(e) ) ) - uci_wrapper.set_error( "EC2 response error when retrieving instance information: " + str(e), True ) + if reservation: + uci_wrapper.set_launch_time( self.format_time( reservation.instances[0].launch_time ), i_index=i_index ) + if not uci_wrapper.uci_launch_time_set(): + uci_wrapper.set_uci_launch_time( self.format_time( reservation.instances[0].launch_time ) ) + try: + uci_wrapper.set_reservation_id( i_index, str( reservation ).split(":")[1] ) + # TODO: if more than a single instance will be started through single reservation, change this reference to element [0] + i_id = str( reservation.instances[0]).split(":")[1] + uci_wrapper.set_instance_id( i_index, i_id ) + s = reservation.instances[0].state + uci_wrapper.change_state( s, i_id, s ) + uci_wrapper.set_security_group_name( self.security_group, i_id=i_id ) + log.debug( "Instance of UCI '%s' started, current state: '%s'" % ( uci_wrapper.get_name(), uci_wrapper.get_state() ) ) + except boto.exception.EC2ResponseError, e: + log.error( "EC2 response error when retrieving instance information for UCI '%s': '%s'" % ( uci_wrapper.get_name(), str(e) ) ) + uci_wrapper.set_error( "EC2 response error when retrieving instance information: " + str(e), True ) def stopUCI( self, uci_wrapper): """ @@ -544,11 +567,11 @@ """ # Check if any instance-specific information was written to local DB; if 'yes', set instance and UCI's error message # suggesting manual check. - if inst.launch_time != None or inst.reservation_id != None or inst.instance_id != None or inst.keypair_name != None: + if inst.launch_time != None or inst.reservation_id != None or inst.instance_id != None: # Try to recover state - this is best-case effort, so if something does not work immediately, not # recovery steps are attempted. Recovery is based on hope that instance_id is available in local DB; if not, # report as error. - # Fields attempting to be recovered are: reservation_id, keypair_name, instance status, and launch_time + # Fields attempting to be recovered are: reservation_id, instance status, and launch_time if inst.instance_id != None: conn = self.get_connection_from_uci( inst.uci ) rl = conn.get_all_instances( [inst.instance_id] ) # reservation list @@ -559,11 +582,6 @@ except: # something failed, so skip pass - if inst.keypair_name == None: - try: - inst.keypair_name = rl[0].instances[0].key_name - except: # something failed, so skip - pass try: state = rl[0].instances[0].update() inst.state = state diff -r 5963dd169715 -r 7c4cfc243cc3 lib/galaxy/cloud/providers/eucalyptus.py --- a/lib/galaxy/cloud/providers/eucalyptus.py Fri Nov 06 13:45:57 2009 -0500 +++ b/lib/galaxy/cloud/providers/eucalyptus.py Mon Nov 09 18:55:14 2009 -0500 @@ -63,7 +63,7 @@ self.threads = [] nworkers = 5 - log.info( "Starting eucalyptus cloud controller workers" ) + log.info( "Starting eucalyptus cloud controller workers..." ) for i in range( nworkers ): worker = threading.Thread( target=self.run_next ) worker.start() @@ -74,7 +74,6 @@ """Run the next job, waiting until one is available if necessary""" cnt = 0 while 1: - #log.debug( '[%d] run_next->queue.qsize(): %s' % ( cnt, self.queue.qsize() ) ) uci_wrapper = self.queue.get() uci_state = uci_wrapper.get_state() if uci_state is self.STOP_SIGNAL: @@ -90,7 +89,7 @@ elif uci_state==uci_states.SHUTTING_DOWN: self.stopUCI( uci_wrapper ) except: - log.exception( "Uncaught exception executing request." ) + log.exception( "Uncaught exception executing cloud request." ) cnt += 1 def get_connection( self, uci_wrapper ): @@ -120,28 +119,43 @@ return conn - def set_keypair( self, uci_wrapper, conn ): + def check_key_pair( self, uci_wrapper, conn ): """ - Generate keypair using user's default credentials + Generate key pair using user's credentials """ - log.debug( "Getting user's keypair: '%s'" % self.key_pair ) - instances = uci_wrapper.get_instances_indexes() + log.debug( "Getting user's key pair: '%s'" % self.key_pair ) try: kp = conn.get_key_pair( self.key_pair ) - for inst in instances: - uci_wrapper.set_key_pair( inst, kp.name ) - return kp.name + uci_kp = uci_wrapper.get_key_pair_name() + uci_material = uci_wrapper.get_key_pair_material() + if kp.name != uci_kp or uci_material == None: + try: # key pair exists on the cloud but not in local database, so re-generate it (i.e., delete and then create) + conn.delete_key_pair( self.key_pair ) + except boto.exception.EC2ResponseError: + pass + kp = self.create_key_pair( conn ) + uci_wrapper.set_key_pair( kp.name, kp.material ) + else: + return kp.name except boto.exception.EC2ResponseError, e: # No keypair under this name exists so create it if e.code == 'InvalidKeyPair.NotFound': log.info( "No keypair found, creating keypair '%s'" % self.key_pair ) - kp = conn.create_key_pair( self.key_pair ) - for inst in instances: - uci_wrapper.set_key_pair( inst, kp.name, kp.material ) + kp = self.create_key_pair( conn ) + uci_wrapper.set_key_pair( kp.name, kp.material ) else: log.error( "EC2 response error: '%s'" % e ) - uci_wrapper.set_error( "EC2 response error while creating key pair: " + str(e), True ) + uci_wrapper.set_error( "Cloud provider response error while creating key pair: " + str( e ), True ) - return kp.name + if kp != None: + return kp.name + else: + return None + + def create_key_pair( self, conn ): + try: + return conn.create_key_pair( self.key_pair ) + except boto.exception.EC2ResponseError, e: + return None def get_mi_id( self, type ): """ @@ -203,7 +217,7 @@ log.debug( "Deleting volume with id='%s'" % v.volume_id ) if conn.delete_volume( v.volume_id ): deletedList.append( v.volume_id ) - v.delete() + v.deleted = True v.flush() count += 1 else: @@ -236,46 +250,50 @@ """ Starts instance(s) of given UCI on the cloud. """ - conn = self.get_connection( uci_wrapper ) -# if uci_wrapper.get_state() != uci_states.ERROR: - self.set_keypair( uci_wrapper, conn ) - - i_indexes = uci_wrapper.get_instances_indexes() # Get indexes of i_indexes associated with this UCI - - if uci_wrapper.get_state() != uci_states.ERROR: + conn = self.get_connection( uci_wrapper ) + self.check_key_pair( uci_wrapper, conn ) + + i_indexes = uci_wrapper.get_instances_indexes( state=None ) # Get indexes of i_indexes associated with this UCI for i_index in i_indexes: mi_id = self.get_mi_id( uci_wrapper.get_type( i_index ) ) - log.debug( "mi_id: %s, uci_wrapper.get_key_pair_name( i_index ): %s" % ( mi_id, uci_wrapper.get_key_pair_name( i_index ) ) ) + log.debug( "mi_id: %s, uci_wrapper.get_key_pair_name(): %s" % ( mi_id, uci_wrapper.get_key_pair_name() ) ) uci_wrapper.set_mi( i_index, mi_id ) - if uci_wrapper.get_state() != uci_states.ERROR: + if uci_wrapper.get_state() != uci_states.ERROR and uci_wrapper.get_key_pair_name() != None: log.debug( "Starting UCI instance '%s'" % uci_wrapper.get_name() ) - log.debug( 'Using following command: conn.run_instances( image_id=%s, key_name=%s )' % ( mi_id, uci_wrapper.get_key_pair_name( i_index ) ) ) + log.debug( 'Using following command: conn.run_instances( image_id=%s, key_name=%s )' % ( mi_id, uci_wrapper.get_key_pair_name() ) ) + reservation = None try: - reservation = conn.run_instances( image_id=mi_id, key_name=uci_wrapper.get_key_pair_name( i_index ) ) + reservation = conn.run_instances( image_id=mi_id, key_name=uci_wrapper.get_key_pair_name() ) #reservation = conn.run_instances( image_id=instance.image, key_name=instance.keypair_name, security_groups=['galaxy'], instance_type=instance.type, placement=instance.availability_zone ) except boto.exception.EC2ResponseError, e: - log.error( "EC2 response error when starting UCI '%s': '%s'" % ( uci_wrapper.get_name(), str(e) ) ) - uci_wrapper.set_error( "EC2 response error when starting: " + str(e), True ) - - l_time = datetime.utcnow() + log.error( "EC2 response error when starting UCI '%s': '%s'" % ( uci_wrapper.get_name(), str( e ) ) ) + uci_wrapper.set_error( "Cloud provider response error when starting: " + str( e ), True ) + except Exception, ex: + log.error( "Error when starting UCI '%s': '%s'" % ( uci_wrapper.get_name(), str( ex ) ) ) + uci_wrapper.set_error( "Cloud provider error when starting: " + str( ex ), True ) +# l_time = datetime.utcnow() # uci_wrapper.set_launch_time( l_time, i_index=i_index ) - uci_wrapper.set_launch_time( self.format_time( reservation.instances[0].launch_time ), i_index=i_index ) - if not uci_wrapper.uci_launch_time_set(): - uci_wrapper.set_uci_launch_time( l_time ) - try: - uci_wrapper.set_reservation_id( i_index, str( reservation ).split(":")[1] ) - # TODO: if more than a single instance will be started through single reservation, change this reference from element [0] - i_id = str( reservation.instances[0]).split(":")[1] - uci_wrapper.set_instance_id( i_index, i_id ) - s = reservation.instances[0].state - uci_wrapper.change_state( s, i_id, s ) - log.debug( "Instance of UCI '%s' started, current state: '%s'" % ( uci_wrapper.get_name(), uci_wrapper.get_state() ) ) - except boto.exception.EC2ResponseError, e: - log.error( "EC2 response error when retrieving instance information for UCI '%s': '%s'" % ( uci_wrapper.get_name(), str(e) ) ) - uci_wrapper.set_error( "EC2 response error when retrieving instance information: " + str(e), True ) - + if reservation: + uci_wrapper.set_launch_time( self.format_time( reservation.instances[0].launch_time ), i_index=i_index ) + if not uci_wrapper.uci_launch_time_set(): + uci_wrapper.set_uci_launch_time( self.format_time( reservation.instances[0].launch_time ) ) + try: + uci_wrapper.set_reservation_id( i_index, str( reservation ).split(":")[1] ) + # TODO: if more than a single instance will be started through single reservation, change this reference from element [0] + i_id = str( reservation.instances[0]).split(":")[1] + uci_wrapper.set_instance_id( i_index, i_id ) + s = reservation.instances[0].state + uci_wrapper.change_state( s, i_id, s ) + log.debug( "Instance of UCI '%s' started, current state: '%s'" % ( uci_wrapper.get_name(), uci_wrapper.get_state() ) ) + except boto.exception.EC2ResponseError, e: + log.error( "EC2 response error when retrieving instance information for UCI '%s': '%s'" % ( uci_wrapper.get_name(), str(e) ) ) + uci_wrapper.set_error( "Cloud provider response error when retrieving instance information: " + str(e), True ) + + if uci_wrapper.get_key_pair_name() == None: + log.debug( "Key pair for UCI '%s' is NULL." % uci_wrapper.get_name() ) + uci_wrapper.set_error( "Key pair not found. Try resetting the state and starting the instance again.", True ) def stopUCI( self, uci_wrapper): """ @@ -294,7 +312,7 @@ notStopped = [] for r in rl: for inst in r.instances: - log.debug( "Sending stop signal to instance '%s' associated with reservation '%s'." % ( inst, r ) ) + log.debug( "Sending stop signal to instance '%s' associated with reservation '%s' (UCI: %s)." % ( inst, r, uci_wrapper.get_name() ) ) inst.stop() uci_wrapper.set_stop_time( datetime.utcnow(), i_id=inst.id ) uci_wrapper.change_state( instance_id=inst.id, i_state=inst.update() ) @@ -385,16 +403,17 @@ # Attempt at updating any zombie UCIs (i.e., instances that have been in SUBMITTED state for longer than expected - see below for exact time) zombies = model.UCI.filter_by( state=uci_states.SUBMITTED ).all() for zombie in zombies: - z_instances = model.CloudInstance.filter_by( uci_id=zombie.id) \ - .filter( or_( model.CloudInstance.c.state != instance_states.TERMINATED, - model.CloudInstance.c.state == None ) ) \ + log.debug( "zombie UCI: %s" % zombie.name ) + z_instances = model.CloudInstance \ + .filter_by( uci_id=zombie.id, state=None ) \ .all() for z_inst in z_instances: if self.type == z_inst.uci.credentials.provider.type: # log.debug( "z_inst.id: '%s', state: '%s'" % ( z_inst.id, z_inst.state ) ) td = datetime.utcnow() - z_inst.update_time + log.debug( "z_inst.id: %s, time delta is %s sec" % ( z_inst.id, td.seconds ) ) if td.seconds > 180: # if instance has been in SUBMITTED state for more than 3 minutes - log.debug( "[%s] Running zombie repair update on instance with DB id '%s'" % ( z_inst.uci.credentials.provider.type, z_inst.id ) ) + log.debug( "[%s](td=%s) Running zombie repair update on instance with DB id '%s'" % ( z_inst.uci.credentials.provider.type, td.seconds, z_inst.id ) ) self.processZombie( z_inst ) def updateInstance( self, inst ): @@ -509,11 +528,11 @@ """ # Check if any instance-specific information was written to local DB; if 'yes', set instance and UCI's error message # suggesting manual check. - if inst.launch_time != None or inst.reservation_id != None or inst.instance_id != None or inst.keypair_name != None: + if inst.launch_time != None or inst.reservation_id != None or inst.instance_id != None: # Try to recover state - this is best-case effort, so if something does not work immediately, not # recovery steps are attempted. Recovery is based on hope that instance_id is available in local DB; if not, # report as error. - # Fields attempting to be recovered are: reservation_id, keypair_name, instance status, and launch_time + # Fields attempting to be recovered are: reservation_id, instance status, and launch_time if inst.instance_id != None: conn = self.get_connection_from_uci( inst.uci ) rl = conn.get_all_instances( [inst.instance_id] ) # reservation list @@ -524,11 +543,6 @@ except: # something failed, so skip pass - if inst.keypair_name == None: - try: - inst.keypair_name = rl[0].instances[0].key_name - except: # something failed, so skip - pass try: state = rl[0].instances[0].update() inst.state = state diff -r 5963dd169715 -r 7c4cfc243cc3 lib/galaxy/model/__init__.py --- a/lib/galaxy/model/__init__.py Fri Nov 06 13:45:57 2009 -0500 +++ b/lib/galaxy/model/__init__.py Mon Nov 09 18:55:14 2009 -0500 @@ -951,7 +951,6 @@ self.instance_id = None self.mi = None self.state = None - self.keypair_name = None self.public_dns = None self.availability_zone = None diff -r 5963dd169715 -r 7c4cfc243cc3 lib/galaxy/model/mapping.py --- a/lib/galaxy/model/mapping.py Fri Nov 06 13:45:57 2009 -0500 +++ b/lib/galaxy/model/mapping.py Mon Nov 09 18:55:14 2009 -0500 @@ -390,9 +390,12 @@ Column( "id", Integer, primary_key=True ), Column( "create_time", DateTime, default=now ), Column( "update_time", DateTime, default=now, onupdate=now ), + Column( "provider_type", TEXT ), Column( "image_id", TEXT, nullable=False ), Column( "manifest", TEXT ), - Column( "state", TEXT ) ) + Column( "state", TEXT ), + Column( "architecture", TEXT ), + Column( "deleted", Boolean, default=False ) ) """ UserConfiguredInstance (UCI) table """ UCI.table = Table( "cloud_uci", metadata, @@ -401,11 +404,14 @@ Column( "update_time", DateTime, default=now, onupdate=now ), Column( "user_id", Integer, ForeignKey( "galaxy_user.id" ), index=True, nullable=False ), Column( "credentials_id", Integer, ForeignKey( "cloud_user_credentials.id" ), index=True ), + Column( "key_pair_name", TEXT ), + Column( "key_pair_material", TEXT ), Column( "name", TEXT ), Column( "state", TEXT ), Column( "error", TEXT ), Column( "total_size", Integer ), - Column( "launch_time", DateTime ) ) + Column( "launch_time", DateTime ), + Column( "deleted", Boolean, default=False ) ) CloudInstance.table = Table( "cloud_instance", metadata, Column( "id", Integer, primary_key=True ), @@ -418,13 +424,12 @@ Column( "type", TEXT ), Column( "reservation_id", TEXT ), Column( "instance_id", TEXT ), - Column( "mi_id", TEXT, ForeignKey( "cloud_image.image_id" ), index=True, nullable=False ), + Column( "mi_id", TEXT, ForeignKey( "cloud_image.image_id" ), index=True ), Column( "state", TEXT ), Column( "error", TEXT ), Column( "public_dns", TEXT ), Column( "private_dns", TEXT ), - Column( "keypair_name", TEXT ), - Column( "keypair_material", TEXT ), + Column( "security_group", TEXT ), Column( "availability_zone", TEXT ) ) CloudStore.table = Table( "cloud_store", metadata, @@ -437,10 +442,22 @@ Column( "volume_id", TEXT ), Column( "size", Integer, nullable=False ), Column( "availability_zone", TEXT ), - Column( "i_id", TEXT, ForeignKey( "cloud_instance.instance_id" ), index=True ), + Column( "i_id", TEXT, ForeignKey( "cloud_instance.instance_id" ) ), Column( "status", TEXT ), Column( "device", TEXT ), - Column( "space_consumed", Integer ) ) + Column( "space_consumed", Integer ), + Column( "deleted", Boolean, default=False ) ) + +CloudUserCredentials.table = Table( "cloud_user_credentials", metadata, + Column( "id", Integer, primary_key=True ), + Column( "create_time", DateTime, default=now ), + Column( "update_time", DateTime, default=now, onupdate=now ), + Column( "user_id", Integer, ForeignKey( "galaxy_user.id" ), index=True, nullable=False ), + Column( "provider_id", Integer, ForeignKey( "cloud_provider.id" ), index=True, nullable=False ), + Column( "name", TEXT ), + Column( "access_key", TEXT ), + Column( "secret_key", TEXT ), + Column( "deleted", Boolean, default=False ) ) CloudProvider.table = Table( "cloud_provider", metadata, Column( "id", Integer, primary_key=True ), @@ -461,19 +478,8 @@ Column( "proxy_pass", TEXT ), Column( "debug", Integer ), Column( "https_connection_factory", TEXT ), - Column( "path", TEXT ) ) - -CloudUserCredentials.table = Table( "cloud_user_credentials", metadata, - Column( "id", Integer, primary_key=True ), - Column( "create_time", DateTime, default=now ), - Column( "update_time", DateTime, default=now, onupdate=now ), - Column( "user_id", Integer, ForeignKey( "galaxy_user.id" ), index=True, nullable=False ), - Column( "name", TEXT ), - Column( "access_key", TEXT ), - Column( "secret_key", TEXT ), - Column( "defaultCred", Boolean, default=False ), - Column( "provider_id", Integer, ForeignKey( "cloud_provider.id" ), index=True, nullable=False ) ) - + Column( "path", TEXT ), + Column( "deleted", Boolean, default=False ) ) # *************************************************************************** StoredWorkflow.table = Table( "stored_workflow", metadata, diff -r 5963dd169715 -r 7c4cfc243cc3 lib/galaxy/model/migrate/versions/0014_cloud_tables.py --- a/lib/galaxy/model/migrate/versions/0014_cloud_tables.py Fri Nov 06 13:45:57 2009 -0500 +++ b/lib/galaxy/model/migrate/versions/0014_cloud_tables.py Mon Nov 09 18:55:14 2009 -0500 @@ -12,25 +12,38 @@ metadata = MetaData( migrate_engine ) +def display_migration_details(): + print + print "========================================" + print "This script adds tables needed for Galaxy cloud functionality." + print "========================================" + CloudImage_table = Table( "cloud_image", metadata, Column( "id", Integer, primary_key=True ), Column( "create_time", DateTime, default=now ), Column( "update_time", DateTime, default=now, onupdate=now ), + Column( "provider_type", TEXT ), Column( "image_id", TEXT, nullable=False ), Column( "manifest", TEXT ), - Column( "state", TEXT ) ) + Column( "state", TEXT ), + Column( "architecture", TEXT ), + Column( "deleted", Boolean, default=False ) ) +""" UserConfiguredInstance (UCI) table """ UCI_table = Table( "cloud_uci", metadata, Column( "id", Integer, primary_key=True ), Column( "create_time", DateTime, default=now ), Column( "update_time", DateTime, default=now, onupdate=now ), Column( "user_id", Integer, ForeignKey( "galaxy_user.id" ), index=True, nullable=False ), Column( "credentials_id", Integer, ForeignKey( "cloud_user_credentials.id" ), index=True ), + Column( "key_pair_name", TEXT ), + Column( "key_pair_material", TEXT ), Column( "name", TEXT ), Column( "state", TEXT ), Column( "error", TEXT ), Column( "total_size", Integer ), - Column( "launch_time", DateTime ) ) + Column( "launch_time", DateTime ), + Column( "deleted", Boolean, default=False ) ) CloudInstance_table = Table( "cloud_instance", metadata, Column( "id", Integer, primary_key=True ), @@ -39,17 +52,16 @@ Column( "launch_time", DateTime ), Column( "stop_time", DateTime ), Column( "user_id", Integer, ForeignKey( "galaxy_user.id" ), index=True, nullable=False ), - Column( "uci_id", Integer, ForeignKey( "uci.id" ), index=True ), + Column( "uci_id", Integer, ForeignKey( "cloud_uci.id" ), index=True ), Column( "type", TEXT ), Column( "reservation_id", TEXT ), Column( "instance_id", TEXT ), - Column( "mi_id", TEXT, ForeignKey( "cloud_image.image_id" ), index=True, nullable=False ), + Column( "mi_id", TEXT, ForeignKey( "cloud_image.image_id" ), index=True ), Column( "state", TEXT ), Column( "error", TEXT ), Column( "public_dns", TEXT ), Column( "private_dns", TEXT ), - Column( "keypair_name", TEXT ), - Column( "keypair_material", TEXT ), + Column( "security_group", TEXT ), Column( "availability_zone", TEXT ) ) CloudStore_table = Table( "cloud_store", metadata, @@ -58,25 +70,26 @@ Column( "update_time", DateTime, default=now, onupdate=now ), Column( "attach_time", DateTime ), Column( "user_id", Integer, ForeignKey( "galaxy_user.id" ), index=True, nullable=False ), - Column( "uci_id", Integer, ForeignKey( "uci.id" ), index=True, nullable=False ), + Column( "uci_id", Integer, ForeignKey( "cloud_uci.id" ), index=True, nullable=False ), Column( "volume_id", TEXT ), Column( "size", Integer, nullable=False ), Column( "availability_zone", TEXT ), - Column( "i_id", TEXT, ForeignKey( "cloud_instance.instance_id" ), index=True ), + Column( "i_id", TEXT, ForeignKey( "cloud_instance.instance_id" ) ), Column( "status", TEXT ), Column( "device", TEXT ), - Column( "space_consumed", Integer ) ) + Column( "space_consumed", Integer ), + Column( "deleted", Boolean, default=False ) ) CloudUserCredentials_table = Table( "cloud_user_credentials", metadata, Column( "id", Integer, primary_key=True ), Column( "create_time", DateTime, default=now ), Column( "update_time", DateTime, default=now, onupdate=now ), Column( "user_id", Integer, ForeignKey( "galaxy_user.id" ), index=True, nullable=False ), + Column( "provider_id", Integer, ForeignKey( "cloud_provider.id" ), index=True, nullable=False ), Column( "name", TEXT ), Column( "access_key", TEXT ), Column( "secret_key", TEXT ), - Column( "defaultCred", Boolean, default=False ), - Column( "provider_id", Integer, ForeignKey( "cloud_provider.id" ), index=True, nullable=False ) ) + Column( "deleted", Boolean, default=False ) ) CloudProvider_table = Table( "cloud_provider", metadata, Column( "id", Integer, primary_key=True ), @@ -97,17 +110,20 @@ Column( "proxy_pass", TEXT ), Column( "debug", Integer ), Column( "https_connection_factory", TEXT ), - Column( "path", TEXT ) ) + Column( "path", TEXT ), + Column( "deleted", Boolean, default=False ) ) def upgrade(): + display_migration_details() + # Load existing tables metadata.reflect() CloudImage_table.create() UCI_table.create() CloudUserCredentials_table.create() + CloudStore_table.create() + CloudInstance_table.create() CloudProvider_table.create() - CloudInstance_table.create() - CloudStore_table.create() def downgrade(): metadata.reflect() diff -r 5963dd169715 -r 7c4cfc243cc3 lib/galaxy/web/controllers/cloud.py --- a/lib/galaxy/web/controllers/cloud.py Fri Nov 06 13:45:57 2009 -0500 +++ b/lib/galaxy/web/controllers/cloud.py Mon Nov 09 18:55:14 2009 -0500 @@ -76,11 +76,13 @@ cloudCredentials = trans.sa_session.query( model.CloudUserCredentials ) \ .filter_by( user=user ) \ + .filter( model.CloudUserCredentials.c.deleted != True ) \ .order_by( model.CloudUserCredentials.c.name ) \ .all() cloudProviders = trans.sa_session.query( model.CloudProvider ) \ .filter_by( user=user ) \ + .filter( model.CloudProvider.c.deleted != True ) \ .order_by( model.CloudProvider.c.name ) \ .all() @@ -131,25 +133,6 @@ prevInstances = prevInstances, cloudProviders = cloudProviders ) - @web.require_login( "use Galaxy cloud" ) - def makeDefault( self, trans, id=None ): - """ - Set current credentials as default. - *NOT USED* - """ - currentDefault = get_default_credentials (trans) - if currentDefault: - currentDefault.defaultCred = False - - newDefault = get_stored_credentials( trans, id ) - newDefault.defaultCred = True - trans.sa_session.flush() - trans.set_message( "Credentials '%s' set as default." % newDefault.name ) - - # TODO: Fix bug that when this function returns, top Galaxy tab bar is missing from the webpage - return self.list( trans ) #trans.fill_template( "cloud/configure_cloud.mako", - #awsCredentials = awsCredentials ) - @web.expose @web.require_login( "start Galaxy cloud instance" ) def start( self, trans, id, type='m1.small' ): @@ -158,14 +141,14 @@ """ user = trans.get_user() uci = get_uci( trans, id ) - mi = get_mi( trans, uci, type ) +# mi = get_mi( trans, uci, type ) stores = get_stores( trans, uci ) # Ensure instance is available and then store relevant data # into DB to initiate instance startup by cloud manager if ( len(stores) is not 0 ) and ( uci.state == uci_states.AVAILABLE ): instance = model.CloudInstance() instance.user = user - instance.image = mi +# instance.image = mi instance.uci = uci instance.availability_zone = stores[0].availability_zone # Bc. all EBS volumes need to be in the same avail. zone, just check 1st instance.type = type @@ -177,9 +160,9 @@ session.flush() # Log trans.log_event ("User initiated starting of UCI '%s'." % uci.name ) - trans.set_message( "Galaxy instance started. NOTE: Please wait about 3-5 minutes for the instance to " - "start up and then refresh this page. A button to connect to the instance will then appear alongside " - "instance description." ) + trans.set_message( "Galaxy instance started. NOTE: Please wait about 5 minutes for the instance to " + "start up. A button to connect to the instance will appear alongside " + "instance description once cloud instance of Galaxy is ready." ) return self.list( trans ) if len(stores) == 0: @@ -270,10 +253,10 @@ inst_error = vol_error = cred_error = None error = {} user = trans.get_user() - storedCreds = trans.sa_session.query( model.CloudUserCredentials ).filter_by( user=user ).all() + storedCreds = trans.sa_session.query( model.CloudUserCredentials ).filter_by( user=user, deleted=False ).all() if len( storedCreds ) == 0: return trans.show_error_message( "You must register credentials before configuring a Galaxy cloud instance." ) - # Create dict mapping of cloud providers to zones available by those providers + # Create dict mapping of cloud-providers-to-zones available by those providers providersToZones = {} for storedCred in storedCreds: zones = None @@ -314,8 +297,7 @@ # Create new user configured instance try: if trans.app.model.UCI \ - .filter_by (user=user) \ - .filter( and_( trans.app.model.UCI.table.c.name==instanceName, trans.app.model.UCI.table.c.state!=uci_states.DELETED ) ) \ + .filter_by (user=user, deleted=False, name=instanceName ) \ .first(): error['inst_error'] = "An instance with that name already exist." elif instanceName=='' or len( instanceName ) > 255: @@ -327,7 +309,7 @@ elif ( int( volSize ) < 1 ) or ( int( volSize ) > 1000 ): error['vol_error'] = "Volume size must be integer value between 1 and 1000." elif zone=='': - error['zone_error'] = "You must select zone where this UCI will be registered." + error['zone_error'] = "You must select a zone where this UCI will be registered." else: # Capture user configured instance information uci = model.UCI() @@ -349,14 +331,12 @@ session.save_or_update( storage ) session.flush() # Log and display the management page - trans.log_event( "User configured new cloud instance" ) + trans.log_event( "User configured new cloud instance: '%s'" % instanceName ) trans.set_message( "New Galaxy instance '%s' configured. Once instance status shows 'available' you will be able to start the instance." % instanceName ) return self.list( trans ) - except ValueError: - vol_error = "Volume size must be specified as an integer value only, between 1 and 1000." except AttributeError, ae: inst_error = "No registered cloud images. You must contact administrator to add some before proceeding." - log.debug("AttributeError: %s " % str( ae ) ) + log.debug("AttributeError when registering new UCI '%s': %s " % ( instanceName, str( ae ) ) ) return trans.fill_template( "cloud/configure_uci.mako", instanceName = instanceName, @@ -368,21 +348,29 @@ @web.expose @web.require_admin - def addNewImage( self, trans, image_id='', manifest='', state=None ): - id_error = None - manifest_error = None - if image_id: - if image_id=='' or len( image_id ) > 255: - id_error = "Image ID must be between 1 and 255 characters long." - elif trans.app.model.CloudUserCredentials.filter( - trans.app.model.CloudImage.table.c.image_id==image_id ).first(): - id_error = "Image with ID '" + image_id + "' is already registered. \ - Please choose another ID.ga" + def addNewImage( self, trans, provider_type='', image_id='', manifest='', architecture='', state=None ): + #id_error = arch_error = provider_error = manifest_error = None + error = {} + if provider_type or image_id or manifest or architecture: + if provider_type=='': + error['provider_error'] = "You must select cloud provider type for this machine image." + elif image_id=='' or len( image_id ) > 255: + error['id_error'] = "Image ID must be between 1 and 255 characters long." + elif trans.app.model.CloudUserCredentials \ + .filter_by( deleted=False ) \ + .filter( trans.app.model.CloudImage.table.c.image_id == image_id ) \ + .first(): + error['id_error'] = "Image with ID '" + image_id + "' is already registered. \ + Please choose another ID." + elif architecture=='': + error['arch_error'] = "You must select architecture type for this machine image." else: # Create new image image = model.CloudImage() + image.provider_type = provider_type image.image_id = image_id image.manifest = manifest + image.architecture = architecture # Persist session = trans.sa_session session.save_or_update( image ) @@ -394,16 +382,24 @@ image.state = state images = trans.sa_session.query( model.CloudImage ).all() return trans.fill_template( '/cloud/list_images.mako', images=images ) - - return trans.show_form( - web.FormBuilder( web.url_for(), "Add new cloud image", submit_text="Add" ) - .add_text( "image_id", "Machine Image ID (AMI or EMI)", value='', error=id_error ) - .add_text( "manifest", "Manifest", value='', error=manifest_error ) ) + + return trans.fill_template( "cloud/add_image.mako", + provider_type = provider_type, + image_id = image_id, + manifest = manifest, + architecture = architecture, + error = error ) +# return trans.show_form( +# web.FormBuilder( web.url_for(), "Add new cloud image", submit_text="Add" ) +# .add_text( "provider_type", "Provider type", value='ec2 or eucalyptus', error=provider_error ) +# .add_text( "image_id", "Machine Image ID (AMI or EMI)", value='', error=id_error ) +# .add_text( "manifest", "Manifest", value='', error=manifest_error ) +# .add_text( "architecture", "Architecture", value='i386 or x86_64', error=arch_error ) ) @web.expose @web.require_login( "use Galaxy cloud" ) def listMachineImages( self, trans ): - images = trans.sa_session.query( model.CloudImage ).all() + images = trans.sa_session.query( model.CloudImage ).filter( trans.app.model.CloudImage.table.c.deleted != True ).all() return trans.fill_template( '/cloud/list_images.mako', images=images ) @web.expose @@ -413,13 +409,13 @@ id = trans.security.decode_id( id ) image = trans.sa_session.query( model.CloudImage ).get( id ) - image.delete() + image.deleted = True image.flush() return self.listMachineImages( trans ) @web.expose @web.require_admin - def editImage( self, trans, image_id, manifest, id=None, edited=False ): + def editImage( self, trans, provider_type='', image_id='', manifest='', architecture='', id='', edited=False ): error = {} if not isinstance( id, int ): id = trans.security.decode_id( id ) @@ -435,9 +431,12 @@ if image_id=='' or len( image_id ) > 255: error['id_error'] = "Image ID must be between 1 and 255 characters in length." elif trans.app.model.CloudImage \ + .filter_by( deleted=False ) \ .filter( and_( trans.app.model.CloudImage.table.c.id != image.id, trans.app.model.CloudImage.table.c.image_id==image_id ) ) \ .first(): error['id_error'] = "Image with ID '" + image_id + "' already exist. Please choose an alternative name." + elif architecture=='' or len( architecture ) > 255: + error['arch_error'] = "Architecture type must be between 1 and 255 characters long." if error: return trans.fill_template( "cloud/edit_image.mako", image = image, @@ -446,12 +445,13 @@ else: image.image_id = image_id image.manifest = manifest + image.architecture = architecture # Persist session = trans.sa_session session.save_or_update( image ) session.flush() # Log and display the management page - trans.set_message( "Image '%s' edited." % image.image_id ) + trans.set_message( "Machine image '%s' edited." % image.image_id ) return self.listMachineImages( trans ) @web.expose @@ -548,8 +548,10 @@ if credName or providerName or accessKey or secretKey: if credName=='' or len( credName ) > 255: error['cred_error'] = "Credentials name must be between 1 and 255 characters in length." - elif trans.app.model.CloudUserCredentials.filter_by( user=user ).filter( - trans.app.model.CloudUserCredentials.table.c.name==credName ).first(): + elif trans.app.model.CloudUserCredentials \ + .filter_by( user=user, deleted=False ) \ + .filter( trans.app.model.CloudUserCredentials.table.c.name == credName ) \ + .first(): error['cred_error'] = "Credentials with that name already exist." elif providerName=='': error['provider_error'] = "You must select cloud provider associated with these credentials." @@ -627,13 +629,12 @@ stored = get_stored_credentials( trans, id ) UCIs = trans.sa_session.query( model.UCI ) \ .filter_by( user=user, credentials_id=stored.id ) \ - .filter( model.UCI.c.state!=uci_states.DELETED ) \ + .filter( model.UCI.c.deleted != True ) \ .all() if len(UCIs) == 0: # Delete and save - sess = trans.sa_session - sess.delete( stored ) + stored.deleted = True stored.flush() # Display the management page trans.set_message( "Credentials '%s' deleted." % stored.name ) @@ -658,6 +659,7 @@ if trans.app.model.CloudProvider \ .filter_by (user=user, name=name) \ + .filter( model.CloudProvider.c.deleted != True ) \ .first(): error['name_error'] = "A provider with that name already exist." elif name=='' or len( name ) > 255: @@ -889,13 +891,13 @@ provider = get_provider_by_id( trans, id ) creds = trans.sa_session.query( model.CloudUserCredentials ) \ .filter_by( user=user, provider_id=provider.id ) \ - .filter( model.UCI.c.state!=uci_states.DELETED ) \ + .filter( model.CloudUserCredentials.c.deleted != True ) \ .all() - + if len( creds ) == 0: # Delete and save - sess = trans.sa_session - sess.delete( provider ) + #sess = trans.sa_session + provider.deleted = True provider.flush() # Display the management page trans.set_message( "Cloud provider '%s' deleted." % provider.name ) @@ -908,7 +910,7 @@ @web.json def json_update( self, trans ): user = trans.get_user() - UCIs = trans.sa_session.query( model.UCI ).filter_by( user=user ).filter( model.UCI.c.state != uci_states.DELETED ).all() + UCIs = trans.sa_session.query( model.UCI ).filter_by( user=user ).filter( model.UCI.c.deleted != True ).all() insd = {} # instance name-state dict for uci in UCIs: dict = {} diff -r 5963dd169715 -r 7c4cfc243cc3 templates/cloud/add_image.mako --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/templates/cloud/add_image.mako Mon Nov 09 18:55:14 2009 -0500 @@ -0,0 +1,98 @@ +<% _=n_ %> +<%inherit file="/base.mako"/> +<%def name="title()">Add machine image</%def> + +<%def name="javascripts()"> +${parent.javascripts()} +<script type="text/javascript"> +$(function(){ + //$("input:text:first").focus(); +}) +</script> +</%def> + +%if header: + ${header} +%endif + +<div class="form"> + <div class="form-title">Add machine image</div> + <div class="form-body"> + <form name="add_image" action="${h.url_for( action='addNewImage' )}" method="post" > + <% + cls = "form-row" + if error.has_key('provider_error'): + cls += " form-row-error" + %> + <div class="${cls}"> + <label>Cloud provider type:</label> + <div class="form-row-input"> + <select name="provider_type" style="width:40em"> + <option value="">Select Provider Type...</option> + <option value="eucalyptus">Eucalyptus</option> + <option value="ec2">Amazon EC2</option> + </select> + </div> + %if error.has_key('provider_error'): + <div class="form-row-error-message">${error['provider_error']}</div> + %endif + <div style="clear: both"></div> + </div> + + <% + cls = "form-row" + if error.has_key('id_error'): + cls += " form-row-error" + %> + <div class="${cls}"> + <label>Machine Image ID (AMI or EMI):</label> + <div class="form-row-input"> + <input type="text" name="image_id" value="${image_id}" size="40"> + </div> + %if error.has_key('id_error'): + <div class="form-row-error-message">${error['id_error']}</div> + %endif + <div style="clear: both"></div> + </div> + + <% + cls = "form-row" + if error.has_key('manifest_error'): + cls += " form-row-error" + %> + <div class="${cls}"> + <label>Manifest:</label> + <div class="form-row-input"> + <input type="text" name="manifest" value="${manifest}" size="40"> + </div> + %if error.has_key('manifest_error'): + <div class="form-row-error-message">${error['manifest_error']}</div> + %endif + <div style="clear: both"></div> + </div> + + + <% + cls = "form-row" + if error.has_key('arch_error'): + cls += " form-row-error" + %> + <div class="${cls}"> + <label>Image architecture:</label> + <div class="form-row-input"> + <select name="architecture" style="width:40em"> + <option value="">Select Architecture Type...</option> + <option value="i386">i386 (32 bit)</option> + <option value="x86_64">x86_64 (64 bit)</option> + </select> + </div> + %if error.has_key('arch_error'): + <div class="form-row-error-message">${error['arch_error']}</div> + %endif + <div style="clear: both"></div> + </div> + + <div class="form-row"><input type="submit" value="Add"></div> + </form> + </div> +</div> \ No newline at end of file diff -r 5963dd169715 -r 7c4cfc243cc3 templates/cloud/configure_cloud.mako --- a/templates/cloud/configure_cloud.mako Fri Nov 06 13:45:57 2009 -0500 +++ b/templates/cloud/configure_cloud.mako Mon Nov 09 18:55:14 2009 -0500 @@ -42,8 +42,11 @@ else if ( ( old_state=='running' && new_state=='error' ) || ( old_state=='pending' && new_state=='error' ) || \ ( old_state=='submitted' && new_state=='error' ) || ( old_state=='submittedUCI' && new_state=='error' ) || \ ( old_state=='shutting-down' && new_state=='error' ) || ( prev_old_state.match('newUCI') && new_state=='error' ) || \ - ( prev_old_state.match('new') && new_state=='error' ) || ( prev_old_state.match('available') && new_state=='error' ) || \ + ( prev_old_state.match('new') && new_state=='error' ) || \ ( prev_old_state.match('deleting') && new_state=='error' ) || ( prev_old_state.match('deletingUCI') && new_state=='error' ) ) { + // TODO: Following clause causes constant page refresh for an exception thrown as a result of instance not starting correctly - need alternative method! + //( prev_old_state.match('available') && new_state=='error' ) || \ + var url = "${h.url_for( controller='cloud', action='list')}"; location.replace( url ); } diff -r 5963dd169715 -r 7c4cfc243cc3 templates/cloud/edit_image.mako --- a/templates/cloud/edit_image.mako Fri Nov 06 13:45:57 2009 -0500 +++ b/templates/cloud/edit_image.mako Mon Nov 09 18:55:14 2009 -0500 @@ -21,8 +21,22 @@ <div class="form-title">Edit image</div> <div class="form-body"> <form name="edit_image" action="${h.url_for( action='editImage', id=trans.security.encode_id(image.id), edited="true" )}" method="post" > - - <% + <% + cls = "form-row" + if error.has_key('provider_error'): + cls += " form-row-error" + %> + <div class="${cls}"> + <label>Provider type:</label> + <div class="form-row-input"> + ${image.provider_type} + </div> + %if error.has_key('provider_error'): + <div class="form-row-error-message">${error['provider_error']}</div> + %endif + <div style="clear: both"></div> + </div> + <% cls = "form-row" if error.has_key('id_error'): cls += " form-row-error" @@ -52,7 +66,21 @@ %endif <div style="clear: both"></div> </div> - + <% + cls = "form-row" + if error.has_key('arch_error'): + cls += " form-row-error" + %> + <div class="${cls}"> + <label>Architecture:</label> + <div class="form-row-input"> + <input type="text" name="architecture" value="${image.architecture}" size="40"> + </div> + %if error.has_key('arch_error'): + <div class="form-row-error-message">${error['arch_error']}</div> + %endif + <div style="clear: both"></div> + </div> <div class="form-row"><input type="submit" value="Save"></div> </form> diff -r 5963dd169715 -r 7c4cfc243cc3 templates/cloud/list_images.mako --- a/templates/cloud/list_images.mako Fri Nov 06 13:45:57 2009 -0500 +++ b/templates/cloud/list_images.mako Mon Nov 09 18:55:14 2009 -0500 @@ -21,14 +21,18 @@ <table class="mange-table colored" border="0" cellspacing="0" cellpadding="0" width="100%"> <colgroup width="2%"></colgroup> + <colgroup width="10%"></colgroup> <colgroup width="13%"></colgroup> - <colgroup width="70%"></colgroup> + <colgroup width="55%"></colgroup> + <colgroup width="10%"></colgroup> <colgroup width="5%"></colgroup> <colgroup width="5%"></colgroup> <tr class="header"> <th>#</th> - <th>Machime image ID</th> + <th>Provider type</th> + <th>Machime image ID</th> <th>Manifest</th> + <th>Architecture</th> <th>Edit</th> <th>Delete</th> <th></th> @@ -37,6 +41,13 @@ <tr> <td>${i+1}</td> <td> + %if image.provider_type: + ${image.provider_type} + %else: + N/A + %endif + </td> + <td> %if image.image_id: ${image.image_id} %else: @@ -51,6 +62,13 @@ %endif </td> <td> + %if image.architecture: + ${image.architecture} + %else: + N/A + %endif + </td> + <td> <a href="${h.url_for( controller='cloud', action='editImage', image_id=image.image_id, manifest=image.manifest, id=trans.security.encode_id(image.id) )}">e</a> </td> <td> diff -r 5963dd169715 -r 7c4cfc243cc3 templates/cloud/view.mako --- a/templates/cloud/view.mako Fri Nov 06 13:45:57 2009 -0500 +++ b/templates/cloud/view.mako Mon Nov 09 18:55:14 2009 -0500 @@ -48,11 +48,11 @@ </tr> <tr> <td> Cloud provider type: </td> - <td> ${str(credDetails.provider.type)[:16]}</td> + <td> ${str(credDetails.provider.type)}</td> </tr> <tr> <td> Cloud provider name: </td> - <td> ${str(credDetails.provider.name)[:16]}</td> + <td> ${str(credDetails.provider.name)}</td> </tr> <tr> <td> Access key: </td> @@ -76,7 +76,7 @@ href="javascript:void(0)"> - Hide </a><br /> - <nobr><b>${credDetails.secret_key}</b></nobr><br/> + <nobr>${credDetails.secret_key}</nobr><br/> </div> </td> </tr> diff -r 5963dd169715 -r 7c4cfc243cc3 templates/cloud/viewInstance.mako --- a/templates/cloud/viewInstance.mako Fri Nov 06 13:45:57 2009 -0500 +++ b/templates/cloud/viewInstance.mako Mon Nov 09 18:55:14 2009 -0500 @@ -94,19 +94,25 @@ <td> ${liveInstance.private_dns}</td> </tr> %endif + %if liveInstance.security_group != None: + <tr> + <td> Security group zone:</td> + <td> ${liveInstance.security_group} </td> + </tr> + %endif %if liveInstance.availability_zone != None: <tr> <td> Availabilty zone:</td> <td> ${liveInstance.availability_zone} </td> </tr> %endif - %if liveInstance.keypair_name != None: + %if liveInstance.uci.key_pair_name != None: <tr> <td> Keypair file name:</td> - <td> ${liveInstance.keypair_name} </td> + <td> ${liveInstance.uci.key_pair_name} </td> </tr> %endif - %if liveInstance.keypair_material != None: + %if liveInstance.uci.key_pair_material != None: <tr> <td> Keypair material:</td> <div id="shortComment2"> @@ -117,7 +123,7 @@ </a> </div> <div id="fullComment2" style="DISPLAY: none"> - <nobr><b>${liveInstance.keypair_material}</b></nobr><br/> + <nobr><b>${liveInstance.uci.key_pair_material}</b></nobr><br/> <a onclick="document.getElementById('shortComment2').style.display = 'block'; document.getElementById('fullComment2').style.display = 'none'; return 0;" href="javascript:void(0)">

1 0

[hg] galaxy 3088: Fixed bugs leading to cloud provder code for E...
by Greg Von Kuster 23 Nov '09

23 Nov '09

details: http://www.bx.psu.edu/hg/galaxy/rev/c78e8e1514e4 changeset: 3088:c78e8e1514e4 user: Enis Afgan <afgane(a)gmail.com> date: Tue Nov 10 18:36:21 2009 -0500 description: Fixed bugs leading to cloud provder code for EC2 and EPC to be very functional. diffstat: lib/galaxy/cloud/__init__.py | 39 ++++- lib/galaxy/cloud/providers/ec2.py | 210 ++++++++++++++++++++-------------- lib/galaxy/cloud/providers/eucalyptus.py | 162 ++++++++++++++++---------- lib/galaxy/web/controllers/cloud.py | 88 +++++++------- templates/cloud/configure_cloud.mako | 42 +++--- templates/cloud/viewInstance.mako | 21 +- 6 files changed, 325 insertions(+), 237 deletions(-) diffs (798 lines): diff -r 7c4cfc243cc3 -r c78e8e1514e4 lib/galaxy/cloud/__init__.py --- a/lib/galaxy/cloud/__init__.py Mon Nov 09 18:55:14 2009 -0500 +++ b/lib/galaxy/cloud/__init__.py Tue Nov 10 18:36:21 2009 -0500 @@ -34,6 +34,14 @@ ERROR = "error", CREATING = "creating" ) +instance_states = Bunch( + TERMINATED = "terminated", + SUBMITTED = "submitted", + RUNNING = "running", + PENDING = "pending", + SHUTTING_DOWN = "shutting-down", + ERROR = "error" +) class CloudManager( object ): """ @@ -338,14 +346,23 @@ def set_error( self, error, set_state=False ): """ - Sets error field of given UCI in local Galaxy database. If set_state is set to 'true', - method also sets state of give UCI to 'error' + Sets error field of given UCI in local Galaxy database as well as any instances associated with + this UCI whose state is 'None' or 'SUBMITTED'. If set_state is set to 'true', + method also sets state of give UCI and corresponding instances to 'error' """ uci = model.UCI.get( self.uci_id ) uci.refresh() uci.error = error if set_state: uci.state = uci_states.ERROR + instances = model.CloudInstance \ + .filter_by( uci=uci ) \ + .filter( or_( model.CloudInstance.c.state==None, model.CloudInstance.c.state==instance_states.SUBMITTED ) ) \ + .all() + for i in instances: + i.error = error + i.state = instance_states.ERROR + i.flush() uci.flush() # --------- Getter methods ----------------- @@ -358,6 +375,15 @@ # cred = model.CloudUserCredentials.get( cred_id ) return uci.credentials.provider.type + def get_type( self, i_index ): + instance = model.CloudInstance.get( i_index ) + return instance.type + + def get_state( self ): + uci = model.UCI.get( self.uci_id ) + uci.refresh() + return uci.state + def get_instances_indexes( self, state=None ): """ Returns indexes of instances associated with given UCI as they are stored in local Galaxy database and @@ -372,15 +398,6 @@ return il - def get_type( self, i_index ): - instance = model.CloudInstance.get( i_index ) - return instance.type - - def get_state( self ): - uci = model.UCI.get( self.uci_id ) - uci.refresh() - return uci.state - def get_instance_state( self, instance_id ): uci = model.UCI.get( self.uci_id ) uci.refresh() diff -r 7c4cfc243cc3 -r c78e8e1514e4 lib/galaxy/cloud/providers/ec2.py --- a/lib/galaxy/cloud/providers/ec2.py Mon Nov 09 18:55:14 2009 -0500 +++ b/lib/galaxy/cloud/providers/ec2.py Tue Nov 10 18:36:21 2009 -0500 @@ -37,6 +37,7 @@ instance_states = Bunch( TERMINATED = "terminated", + SUBMITTED = "submitted", RUNNING = "running", PENDING = "pending", SHUTTING_DOWN = "shutting-down", @@ -119,49 +120,73 @@ def check_key_pair( self, uci_wrapper, conn ): """ - Generate keypair using user's default credentials + Generate key pair using user's credentials """ - log.debug( "Getting user's key pair: '%s'" % self.key_pair ) + kp = None + kp_name = uci_wrapper.get_name().replace(' ','_') + "_kp" + log.debug( "Checking user's key pair: '%s'" % kp_name ) try: - kp = conn.get_key_pair( self.key_pair ) - uci_kp = uci_wrapper.get_key_pair_name() + kp = conn.get_key_pair( kp_name ) + uci_kp_name = uci_wrapper.get_key_pair_name() uci_material = uci_wrapper.get_key_pair_material() - if kp.name != uci_kp or uci_material == None: - try: # key pair exists on the cloud but not in local database, so re-generate it (i.e., delete and then create) - conn.delete_key_pair( self.key_pair ) - except boto.exception.EC2ResponseError: - pass - kp = self.create_key_pair( conn ) + if kp != None: + if kp.name != uci_kp_name or uci_material == None: + # key pair exists on the cloud but not in local database, so re-generate it (i.e., delete and then create) + try: + conn.delete_key_pair( kp_name ) + kp = self.create_key_pair( conn, kp_name ) + uci_wrapper.set_key_pair( kp.name, kp.material ) + except boto.exception.EC2ResponseError, e: + log.error( "EC2 response error when deleting key pair: '%s'" % e ) + uci_wrapper.set_error( "EC2 response error while deleting key pair: " + str( e ), True ) + else: + try: + kp = self.create_key_pair( conn, kp_name ) + uci_wrapper.set_key_pair( kp.name, kp.material ) + except boto.exception.EC2ResponseError, e: + log.error( "EC2 response error when creating key pair: '%s'" % e ) + uci_wrapper.set_error( "EC2 response error while creating key pair: " + str( e ), True ) + except Exception, ex: + log.error( "Exception when creating key pair: '%s'" % e ) + uci_wrapper.set_error( "Error while creating key pair: " + str( e ), True ) + + except boto.exception.EC2ResponseError, e: # No keypair under this name exists so create it + if e.code == 'InvalidKeyPair.NotFound': + log.info( "No keypair found, creating keypair '%s'" % kp_name ) + kp = self.create_key_pair( conn, kp_name ) uci_wrapper.set_key_pair( kp.name, kp.material ) else: - return kp.name - except boto.exception.EC2ResponseError, e: # No keypair under this name exists so create it - if e.code == 'InvalidKeyPair.NotFound': - log.info( "No keypair found, creating keypair '%s'" % self.key_pair ) - kp = self.create_key_pair( conn ) - uci_wrapper.set_key_pair( kp.name, kp.material ) - else: - log.error( "EC2 response error: '%s'" % e ) - uci_wrapper.set_error( "EC2 response error while creating key pair: " + str( e ), True ) + log.error( "EC2 response error while retrieving key pair: '%s'" % e ) + uci_wrapper.set_error( "Cloud provider response error while retrieving key pair: " + str( e ), True ) if kp != None: return kp.name else: return None - def create_key_pair( self, conn ): + def create_key_pair( self, conn, kp_name ): try: - return conn.create_key_pair( self.key_pair ) + return conn.create_key_pair( kp_name ) except boto.exception.EC2ResponseError, e: return None - def get_mi_id( self, type ): + def get_mi_id( self, uci_wrapper, i_index ): """ Get appropriate machine image (mi) based on instance size. - TODO: Dummy method - need to implement logic - For valid sizes, see http://aws.amazon.com/ec2/instance-types/ """ - return model.CloudImage.filter( model.CloudImage.table.c.id==2 ).first().image_id + i_type = uci_wrapper.get_type( i_index ) + if i_type=='m1.small' or i_type=='c1.medium': + arch = 'i386' + else: + arch = 'x86_64' + + mi = model.CloudImage.filter_by( deleted=False, provider_type=self.type, architecture=arch ).first() + if mi: + return mi.image_id + else: + log.error( "Machine image could not be retrieved for UCI '%s'." % uci_wrapper.get_name() ) + uci_wrapper.set_error( "Machine image could not be retrieved. Contact site administrator to ensure needed machine image is registered.", True ) + return None def shutdown( self ): """Attempts to gracefully shut down the monitor thread""" @@ -267,71 +292,82 @@ conn = self.get_connection( uci_wrapper ) self.check_key_pair( uci_wrapper, conn ) - i_indexes = uci_wrapper.get_instances_indexes( state=None ) # Get indexes of i_indexes associated with this UCI whose state is 'None' - log.debug( "Starting instances with IDs: '%s' associated with UCI '%s' " % ( uci_wrapper.get_name(), i_indexes ) ) - for i_index in i_indexes: - mi_id = self.get_mi_id( uci_wrapper.get_type( i_index ) ) - uci_wrapper.set_mi( i_index, mi_id ) + i_indexes = uci_wrapper.get_instances_indexes( state=instance_states.SUBMITTED ) # Get indexes of i_indexes associated with this UCI that are in 'submitted' state + log.debug( "Starting instances with IDs: '%s' associated with UCI '%s' " % ( i_indexes, uci_wrapper.get_name(), ) ) + if len( i_indexes ) > 0: + for i_index in i_indexes: + # Get machine image for current instance + mi_id = self.get_mi_id( uci_wrapper, i_index ) + uci_wrapper.set_mi( i_index, mi_id ) + if uci_wrapper.get_key_pair_name() == None: + log.error( "Key pair for UCI '%s' is None." % uci_wrapper.get_name() ) + uci_wrapper.set_error( "Key pair not found. Try resetting the state and starting the instance again.", True ) + return - # Check if galaxy security group exists (and create it if it does not) - log.debug( "Setting up '%s' security group." % self.security_group ) - try: - conn.get_all_security_groups( [self.security_group] ) # security groups - except boto.exception.EC2ResponseError, e: - if e.code == 'InvalidGroup.NotFound': - log.info( "No security group found, creating security group '%s'" % self.security_group ) + # Check if galaxy security group exists (and create it if it does not) + log.debug( "Setting up '%s' security group." % self.security_group ) + try: + conn.get_all_security_groups( [self.security_group] ) # security groups + except boto.exception.EC2ResponseError, e: + if e.code == 'InvalidGroup.NotFound': + log.info( "No security group found, creating security group '%s'" % self.security_group ) + try: + gSecurityGroup = conn.create_security_group(self.security_group, 'Security group for Galaxy.') + gSecurityGroup.authorize( 'tcp', 80, 80, '0.0.0.0/0' ) # Open HTTP port + gSecurityGroup.authorize( 'tcp', 22, 22, '0.0.0.0/0' ) # Open SSH port + except boto.exception.EC2ResponseError, ex: + log.error( "EC2 response error while creating security group: '%s'" % e ) + uci_wrapper.set_error( "EC2 response error while creating security group: " + str( e ), True ) + else: + log.error( "EC2 response error while retrieving security group: '%s'" % e ) + uci_wrapper.set_error( "EC2 response error while retrieving security group: " + str( e ), True ) + + + if uci_wrapper.get_state() != uci_states.ERROR: + # Start an instance + log.debug( "Starting instance for UCI '%s'" % uci_wrapper.get_name() ) + #TODO: Once multiple volumes can be attached to a single instance, update 'userdata' composition + userdata = uci_wrapper.get_store_volume_id()+"|"+uci_wrapper.get_access_key()+"|"+uci_wrapper.get_secret_key() + log.debug( "Using following command: conn.run_instances( image_id='%s', key_name='%s', security_groups='%s', user_data=[OMITTED], instance_type='%s', placement='%s' )" + % ( mi_id, uci_wrapper.get_key_pair_name(), self.security_group, uci_wrapper.get_type( i_index ), uci_wrapper.get_uci_availability_zone() ) ) + reservation = None try: - gSecurityGroup = conn.create_security_group(self.security_group, 'Security group for Galaxy.') - gSecurityGroup.authorize( 'tcp', 80, 80, '0.0.0.0/0' ) # Open HTTP port - gSecurityGroup.authorize( 'tcp', 22, 22, '0.0.0.0/0' ) # Open SSH port - except boto.exception.EC2ResponseError, ex: - log.error( "EC2 response error while creating security group: '%s'" % e ) - uci_wrapper.set_error( "EC2 response error while creating security group: " + str( e ), True ) + reservation = conn.run_instances( image_id=mi_id, + key_name=uci_wrapper.get_key_pair_name(), + security_groups=[self.security_group], + user_data=userdata, + instance_type=uci_wrapper.get_type( i_index ), + placement=uci_wrapper.get_uci_availability_zone() ) + except boto.exception.EC2ResponseError, e: + log.error( "EC2 response error when starting UCI '%s': '%s'" % ( uci_wrapper.get_name(), str(e) ) ) + uci_wrapper.set_error( "EC2 response error when starting: " + str(e), True ) + except Exception, ex: + log.error( "Error when starting UCI '%s': '%s'" % ( uci_wrapper.get_name(), str( ex ) ) ) + uci_wrapper.set_error( "Cloud provider error when starting: " + str( ex ), True ) + # Record newly available instance data into local Galaxy database + if reservation: + uci_wrapper.set_launch_time( self.format_time( reservation.instances[0].launch_time ), i_index=i_index ) + if not uci_wrapper.uci_launch_time_set(): + uci_wrapper.set_uci_launch_time( self.format_time( reservation.instances[0].launch_time ) ) + try: + uci_wrapper.set_reservation_id( i_index, str( reservation ).split(":")[1] ) + # TODO: if more than a single instance will be started through single reservation, change this reference to element [0] + i_id = str( reservation.instances[0]).split(":")[1] + uci_wrapper.set_instance_id( i_index, i_id ) + s = reservation.instances[0].state + uci_wrapper.change_state( s, i_id, s ) + uci_wrapper.set_security_group_name( self.security_group, i_id=i_id ) + log.debug( "Instance of UCI '%s' started, current state: '%s'" % ( uci_wrapper.get_name(), uci_wrapper.get_state() ) ) + except boto.exception.EC2ResponseError, e: + log.error( "EC2 response error when retrieving instance information for UCI '%s': '%s'" % ( uci_wrapper.get_name(), str(e) ) ) + uci_wrapper.set_error( "EC2 response error when retrieving instance information: " + str(e), True ) else: - log.error( "EC2 response error while retrieving security group: '%s'" % e ) - uci_wrapper.set_error( "EC2 response error while retrieving security group: " + str( e ), True ) - - - if uci_wrapper.get_state() != uci_states.ERROR: - # Start an instance - log.debug( "Starting instance for UCI '%s'" % uci_wrapper.get_name() ) - #TODO: Once multiple volumes can be attached to a single instance, update 'userdata' composition - userdata = uci_wrapper.get_store_volume_id()+"|"+uci_wrapper.get_access_key()+"|"+uci_wrapper.get_secret_key() - log.debug( 'Using following command: conn.run_instances( image_id=%s, key_name=%s, security_groups=[%s], user_data=[OMITTED], instance_type=%s, placement=%s )' - % ( mi_id, uci_wrapper.get_key_pair_name( i_index ), self.security_group, uci_wrapper.get_type( i_index ), uci_wrapper.get_uci_availability_zone() ) ) - reservation = None - try: - reservation = conn.run_instances( image_id=mi_id, - key_name=uci_wrapper.get_key_pair_name( i_index ), - security_groups=[self.security_group], - user_data=userdata, - instance_type=uci_wrapper.get_type( i_index ), - placement=uci_wrapper.get_uci_availability_zone() ) - except boto.exception.EC2ResponseError, e: - log.error( "EC2 response error when starting UCI '%s': '%s'" % ( uci_wrapper.get_name(), str(e) ) ) - uci_wrapper.set_error( "EC2 response error when starting: " + str(e), True ) - except Exception, ex: - log.error( "Error when starting UCI '%s': '%s'" % ( uci_wrapper.get_name(), str( ex ) ) ) - uci_wrapper.set_error( "Cloud provider error when starting: " + str( ex ), True ) - # Record newly available instance data into local Galaxy database -# l_time = datetime.utcnow() -# uci_wrapper.set_launch_time( l_time, i_index=i_index ) # format_time( reservation.i_indexes[0].launch_time ) ) - if reservation: - uci_wrapper.set_launch_time( self.format_time( reservation.instances[0].launch_time ), i_index=i_index ) - if not uci_wrapper.uci_launch_time_set(): - uci_wrapper.set_uci_launch_time( self.format_time( reservation.instances[0].launch_time ) ) - try: - uci_wrapper.set_reservation_id( i_index, str( reservation ).split(":")[1] ) - # TODO: if more than a single instance will be started through single reservation, change this reference to element [0] - i_id = str( reservation.instances[0]).split(":")[1] - uci_wrapper.set_instance_id( i_index, i_id ) - s = reservation.instances[0].state - uci_wrapper.change_state( s, i_id, s ) - uci_wrapper.set_security_group_name( self.security_group, i_id=i_id ) - log.debug( "Instance of UCI '%s' started, current state: '%s'" % ( uci_wrapper.get_name(), uci_wrapper.get_state() ) ) - except boto.exception.EC2ResponseError, e: - log.error( "EC2 response error when retrieving instance information for UCI '%s': '%s'" % ( uci_wrapper.get_name(), str(e) ) ) - uci_wrapper.set_error( "EC2 response error when retrieving instance information: " + str(e), True ) + log.error( "UCI '%s' is in 'error' state, starting instance was aborted." % uci_wrapper.get_name() ) + else: + log.error( "No instances were found for UCI '%s'" % uci_wrapper.get_name() ) + uci_wrapper.set_error( "EC2 response error when retrieving instance information: " + str(e), True ) + else: + log.error( "UCI '%s' is in 'error' state, starting instance was aborted." % uci_wrapper.get_name() ) def stopUCI( self, uci_wrapper): """ diff -r 7c4cfc243cc3 -r c78e8e1514e4 lib/galaxy/cloud/providers/eucalyptus.py --- a/lib/galaxy/cloud/providers/eucalyptus.py Mon Nov 09 18:55:14 2009 -0500 +++ b/lib/galaxy/cloud/providers/eucalyptus.py Tue Nov 10 18:36:21 2009 -0500 @@ -38,6 +38,7 @@ instance_states = Bunch( TERMINATED = "terminated", + SUBMITTED = "submitted", RUNNING = "running", PENDING = "pending", SHUTTING_DOWN = "shutting-down", @@ -58,7 +59,6 @@ def __init__( self, app ): self.type = "eucalyptus" # cloud provider type (e.g., ec2, eucalyptus, opennebula) self.zone = "epc" - self.key_pair = "galaxy-keypair" self.queue = Queue() self.threads = [] @@ -123,49 +123,72 @@ """ Generate key pair using user's credentials """ - log.debug( "Getting user's key pair: '%s'" % self.key_pair ) + kp = None + kp_name = uci_wrapper.get_name().replace(' ','_') + "_kp" + log.debug( "Checking user's key pair: '%s'" % kp_name ) try: - kp = conn.get_key_pair( self.key_pair ) - uci_kp = uci_wrapper.get_key_pair_name() + kp = conn.get_key_pair( kp_name ) + uci_kp_name = uci_wrapper.get_key_pair_name() uci_material = uci_wrapper.get_key_pair_material() - if kp.name != uci_kp or uci_material == None: - try: # key pair exists on the cloud but not in local database, so re-generate it (i.e., delete and then create) - conn.delete_key_pair( self.key_pair ) - except boto.exception.EC2ResponseError: - pass - kp = self.create_key_pair( conn ) + if kp != None: + if kp.name != uci_kp_name or uci_material == None: + # key pair exists on the cloud but not in local database, so re-generate it (i.e., delete and then create) + try: + conn.delete_key_pair( kp_name ) + kp = self.create_key_pair( conn, kp_name ) + uci_wrapper.set_key_pair( kp.name, kp.material ) + except boto.exception.EC2ResponseError, e: + log.error( "EC2 response error when deleting key pair: '%s'" % e ) + uci_wrapper.set_error( "Cloud provider response error while deleting key pair: " + str( e ), True ) + else: + try: + kp = self.create_key_pair( conn, kp_name ) + uci_wrapper.set_key_pair( kp.name, kp.material ) + except boto.exception.EC2ResponseError, e: + log.error( "EC2 response error when creating key pair: '%s'" % e ) + uci_wrapper.set_error( "Cloud provider response error while creating key pair: " + str( e ), True ) + except Exception, ex: + log.error( "Exception when creating key pair: '%s'" % e ) + uci_wrapper.set_error( "Error while creating key pair: " + str( e ), True ) + + except boto.exception.EC2ResponseError, e: # No keypair under this name exists so create it + if e.code == 'InvalidKeyPair.NotFound': + log.info( "No keypair found, creating keypair '%s'" % kp_name ) + kp = self.create_key_pair( conn, kp_name ) uci_wrapper.set_key_pair( kp.name, kp.material ) else: - return kp.name - except boto.exception.EC2ResponseError, e: # No keypair under this name exists so create it - if e.code == 'InvalidKeyPair.NotFound': - log.info( "No keypair found, creating keypair '%s'" % self.key_pair ) - kp = self.create_key_pair( conn ) - uci_wrapper.set_key_pair( kp.name, kp.material ) - else: - log.error( "EC2 response error: '%s'" % e ) - uci_wrapper.set_error( "Cloud provider response error while creating key pair: " + str( e ), True ) + log.error( "EC2 response error while retrieving key pair: '%s'" % e ) + uci_wrapper.set_error( "Cloud provider response error while retrieving key pair: " + str( e ), True ) if kp != None: return kp.name else: return None - def create_key_pair( self, conn ): + def create_key_pair( self, conn, kp_name ): try: - return conn.create_key_pair( self.key_pair ) + return conn.create_key_pair( kp_name ) except boto.exception.EC2ResponseError, e: return None - def get_mi_id( self, type ): + def get_mi_id( self, uci_wrapper, i_index ): """ Get appropriate machine image (mi) based on instance size. - TODO: Dummy method - need to implement logic - For valid sizes, see http://aws.amazon.com/ec2/instance-types/ """ - log.debug( "image id: '%s'" % model.CloudImage.get( 1 ).image_id ) - return model.CloudImage.get( 1 ).image_id - + i_type = uci_wrapper.get_type( i_index ) + if i_type=='m1.small' or i_type=='c1.medium': + arch = 'i386' + else: + arch = 'x86_64' + + mi = model.CloudImage.filter_by( deleted=False, provider_type=self.type, architecture=arch ).first() + if mi: + return mi.image_id + else: + log.error( "Machine image could not be retrieved for UCI '%s'." % uci_wrapper.get_name() ) + uci_wrapper.set_error( "Machine image could not be retrieved. Contact site administrator to ensure needed machine image is registered.", True ) + return None + def shutdown( self ): """Attempts to gracefully shut down the monitor thread""" log.info( "sending stop signal to worker threads in eucalyptus cloud manager" ) @@ -253,47 +276,56 @@ if uci_wrapper.get_state() != uci_states.ERROR: conn = self.get_connection( uci_wrapper ) self.check_key_pair( uci_wrapper, conn ) + if uci_wrapper.get_key_pair_name() == None: + log.error( "Key pair for UCI '%s' is NULL." % uci_wrapper.get_name() ) + uci_wrapper.set_error( "Key pair not found. Try resetting the state and starting the instance again.", True ) + return - i_indexes = uci_wrapper.get_instances_indexes( state=None ) # Get indexes of i_indexes associated with this UCI - for i_index in i_indexes: - mi_id = self.get_mi_id( uci_wrapper.get_type( i_index ) ) - log.debug( "mi_id: %s, uci_wrapper.get_key_pair_name(): %s" % ( mi_id, uci_wrapper.get_key_pair_name() ) ) - uci_wrapper.set_mi( i_index, mi_id ) - - if uci_wrapper.get_state() != uci_states.ERROR and uci_wrapper.get_key_pair_name() != None: - log.debug( "Starting UCI instance '%s'" % uci_wrapper.get_name() ) - log.debug( 'Using following command: conn.run_instances( image_id=%s, key_name=%s )' % ( mi_id, uci_wrapper.get_key_pair_name() ) ) - reservation = None - try: - reservation = conn.run_instances( image_id=mi_id, key_name=uci_wrapper.get_key_pair_name() ) - #reservation = conn.run_instances( image_id=instance.image, key_name=instance.keypair_name, security_groups=['galaxy'], instance_type=instance.type, placement=instance.availability_zone ) - except boto.exception.EC2ResponseError, e: - log.error( "EC2 response error when starting UCI '%s': '%s'" % ( uci_wrapper.get_name(), str( e ) ) ) - uci_wrapper.set_error( "Cloud provider response error when starting: " + str( e ), True ) - except Exception, ex: - log.error( "Error when starting UCI '%s': '%s'" % ( uci_wrapper.get_name(), str( ex ) ) ) - uci_wrapper.set_error( "Cloud provider error when starting: " + str( ex ), True ) -# l_time = datetime.utcnow() -# uci_wrapper.set_launch_time( l_time, i_index=i_index ) - if reservation: - uci_wrapper.set_launch_time( self.format_time( reservation.instances[0].launch_time ), i_index=i_index ) - if not uci_wrapper.uci_launch_time_set(): - uci_wrapper.set_uci_launch_time( self.format_time( reservation.instances[0].launch_time ) ) + i_indexes = uci_wrapper.get_instances_indexes( state=instance_states.SUBMITTED ) # Get indexes of i_indexes associated with this UCI that are in 'submitted' state + if len( i_indexes ) > 0: + for i_index in i_indexes: + mi_id = self.get_mi_id( uci_wrapper, i_index ) + log.debug( "mi_id: %s, uci_wrapper.get_key_pair_name(): %s" % ( mi_id, uci_wrapper.get_key_pair_name() ) ) + uci_wrapper.set_mi( i_index, mi_id ) + + if uci_wrapper.get_state() != uci_states.ERROR and uci_wrapper.get_key_pair_name() != None: + log.debug( "Starting UCI instance '%s'" % uci_wrapper.get_name() ) + log.debug( "Using following command: conn.run_instances( image_id='%s', key_name='%s', instance_type='%s' )" + % ( mi_id, uci_wrapper.get_key_pair_name(), uci_wrapper.get_type( i_index ) ) ) + reservation = None try: - uci_wrapper.set_reservation_id( i_index, str( reservation ).split(":")[1] ) - # TODO: if more than a single instance will be started through single reservation, change this reference from element [0] - i_id = str( reservation.instances[0]).split(":")[1] - uci_wrapper.set_instance_id( i_index, i_id ) - s = reservation.instances[0].state - uci_wrapper.change_state( s, i_id, s ) - log.debug( "Instance of UCI '%s' started, current state: '%s'" % ( uci_wrapper.get_name(), uci_wrapper.get_state() ) ) + reservation = conn.run_instances( image_id=mi_id, + key_name=uci_wrapper.get_key_pair_name(), + instance_type=uci_wrapper.get_type( i_index ) ) except boto.exception.EC2ResponseError, e: - log.error( "EC2 response error when retrieving instance information for UCI '%s': '%s'" % ( uci_wrapper.get_name(), str(e) ) ) - uci_wrapper.set_error( "Cloud provider response error when retrieving instance information: " + str(e), True ) - - if uci_wrapper.get_key_pair_name() == None: - log.debug( "Key pair for UCI '%s' is NULL." % uci_wrapper.get_name() ) - uci_wrapper.set_error( "Key pair not found. Try resetting the state and starting the instance again.", True ) + log.error( "EC2 response error when starting UCI '%s': '%s'" % ( uci_wrapper.get_name(), str( e ) ) ) + uci_wrapper.set_error( "Cloud provider response error when starting: " + str( e ), True ) + except Exception, ex: + log.error( "Error when starting UCI '%s': '%s'" % ( uci_wrapper.get_name(), str( ex ) ) ) + uci_wrapper.set_error( "Cloud provider error when starting: " + str( ex ), True ) + # Record newly available instance data into local Galaxy database + if reservation: + uci_wrapper.set_launch_time( self.format_time( reservation.instances[0].launch_time ), i_index=i_index ) + if not uci_wrapper.uci_launch_time_set(): + uci_wrapper.set_uci_launch_time( self.format_time( reservation.instances[0].launch_time ) ) + try: + uci_wrapper.set_reservation_id( i_index, str( reservation ).split(":")[1] ) + # TODO: if more than a single instance will be started through single reservation, change this reference from element [0] + i_id = str( reservation.instances[0]).split(":")[1] + uci_wrapper.set_instance_id( i_index, i_id ) + s = reservation.instances[0].state + uci_wrapper.change_state( s, i_id, s ) + log.debug( "Instance of UCI '%s' started, current state: '%s'" % ( uci_wrapper.get_name(), uci_wrapper.get_state() ) ) + except boto.exception.EC2ResponseError, e: + log.error( "EC2 response error when retrieving instance information for UCI '%s': '%s'" % ( uci_wrapper.get_name(), str(e) ) ) + uci_wrapper.set_error( "Cloud provider response error when retrieving instance information: " + str(e), True ) + else: + log.error( "UCI '%s' is in 'error' state, starting instance was aborted." % uci_wrapper.get_name() ) + else: + log.error( "No instances were found for UCI '%s'" % uci_wrapper.get_name() ) + uci_wrapper.set_error( "EC2 response error when retrieving instance information: " + str(e), True ) + else: + log.error( "UCI '%s' is in 'error' state, starting instance was aborted." % uci_wrapper.get_name() ) def stopUCI( self, uci_wrapper): """ diff -r 7c4cfc243cc3 -r c78e8e1514e4 lib/galaxy/web/controllers/cloud.py --- a/lib/galaxy/web/controllers/cloud.py Mon Nov 09 18:55:14 2009 -0500 +++ b/lib/galaxy/web/controllers/cloud.py Tue Nov 10 18:36:21 2009 -0500 @@ -49,9 +49,11 @@ instance_states = Bunch( TERMINATED = "terminated", + SUBMITTED = "submitted", RUNNING = "running", PENDING = "pending", - SHUTTING_DOWN = "shutting-down" + SHUTTING_DOWN = "shutting-down", + ERROR = "error" ) store_states = Bunch( @@ -141,15 +143,14 @@ """ user = trans.get_user() uci = get_uci( trans, id ) -# mi = get_mi( trans, uci, type ) stores = get_stores( trans, uci ) # Ensure instance is available and then store relevant data # into DB to initiate instance startup by cloud manager if ( len(stores) is not 0 ) and ( uci.state == uci_states.AVAILABLE ): instance = model.CloudInstance() instance.user = user -# instance.image = mi instance.uci = uci + instance.state = instance_states.SUBMITTED instance.availability_zone = stores[0].availability_zone # Bc. all EBS volumes need to be in the same avail. zone, just check 1st instance.type = type uci.state = uci_states.SUBMITTED_UCI @@ -253,39 +254,6 @@ inst_error = vol_error = cred_error = None error = {} user = trans.get_user() - storedCreds = trans.sa_session.query( model.CloudUserCredentials ).filter_by( user=user, deleted=False ).all() - if len( storedCreds ) == 0: - return trans.show_error_message( "You must register credentials before configuring a Galaxy cloud instance." ) - # Create dict mapping of cloud-providers-to-zones available by those providers - providersToZones = {} - for storedCred in storedCreds: - zones = None - conn = get_connection( trans, storedCred ) - if conn != None: - avail_zones = [] - try: - zones = conn.get_all_zones() - for zone in zones: - zone = str( zone ).split(':')[1] - avail_zones.append( zone ) - providersToZones[storedCred.name] = avail_zones - except boto.exception.EC2ResponseError, e: - log.error( "Retrieving zones for credentials '%s' failed." % storedCred.name ) - providersToZones[storedCred.name] = ['Unknown provider zone'] - else: - providersToZones[storedCred.name] = ['Unknown provider zone'] - - # Hard-coded solution -# if storedCred.provider.storedCred.provider.region_name == 'us-east-1': -# ec2_zones = ['us-east-1a', 'us-east-1b', 'us-east-1c', 'us-east-1d'] -# providersToZones[storedCred.name] = ec2_zones -# elif storedCred.provider.region_name == 'eu-west-1': -# ec2_zones = ['eu-west-1a', 'eu-west-1b'] -# providersToZones[storedCred.name] = ec2_zones -# elif storedCred.provider.type == 'eucalyptus': -# providersToZones[storedCred.name] = ['epc'] -# else: -# providersToZones[storedCred.name] = ['Unknown provider zone'] if instanceName: # Check if volume size is entered as an integer @@ -337,14 +305,48 @@ except AttributeError, ae: inst_error = "No registered cloud images. You must contact administrator to add some before proceeding." log.debug("AttributeError when registering new UCI '%s': %s " % ( instanceName, str( ae ) ) ) + else: + storedCreds = trans.sa_session.query( model.CloudUserCredentials ).filter_by( user=user, deleted=False ).all() + if len( storedCreds ) == 0: + return trans.show_error_message( "You must register credentials before configuring a Galaxy cloud instance." ) + # Create dict mapping of cloud-providers-to-zones available by those providers + providersToZones = {} + for storedCred in storedCreds: + zones = None + conn = get_connection( trans, storedCred ) + if conn != None: + avail_zones = [] + try: + zones = conn.get_all_zones() + for z in zones: + z = str( z ).split(':')[1] + avail_zones.append( z ) + providersToZones[storedCred.name] = avail_zones + except boto.exception.EC2ResponseError, e: + log.error( "Retrieving zones for credentials '%s' failed: %s" % ( storedCred.name, e ) ) + providersToZones[storedCred.name] = [ "Retrieving zones failed: " + str( e ) ] + else: + providersToZones[storedCred.name] = ['Connection with cloud provider could not be established.'] - return trans.fill_template( "cloud/configure_uci.mako", - instanceName = instanceName, - credName = storedCreds, - volSize = volSize, - zone = zone, - error = error, - providersToZones = providersToZones ) + # Hard-coded solution + # if storedCred.provider.storedCred.provider.region_name == 'us-east-1': + # ec2_zones = ['us-east-1a', 'us-east-1b', 'us-east-1c', 'us-east-1d'] + # providersToZones[storedCred.name] = ec2_zones + # elif storedCred.provider.region_name == 'eu-west-1': + # ec2_zones = ['eu-west-1a', 'eu-west-1b'] + # providersToZones[storedCred.name] = ec2_zones + # elif storedCred.provider.type == 'eucalyptus': + # providersToZones[storedCred.name] = ['epc'] + # else: + # providersToZones[storedCred.name] = ['Unknown provider zone'] + + return trans.fill_template( "cloud/configure_uci.mako", + instanceName = instanceName, + credName = storedCreds, + volSize = volSize, + zone = zone, + error = error, + providersToZones = providersToZones ) @web.expose @web.require_admin diff -r 7c4cfc243cc3 -r c78e8e1514e4 templates/cloud/configure_cloud.mako --- a/templates/cloud/configure_cloud.mako Mon Nov 09 18:55:14 2009 -0500 +++ b/templates/cloud/configure_cloud.mako Tue Nov 10 18:36:21 2009 -0500 @@ -28,14 +28,15 @@ old_state = $(elem + "-state").text(); prev_old_state = $(elem + "-state-p").text(); new_state = data[i].state; - //console.log( "old_state[%d] = %s", i, old_state ); - //console.log( "prev_old_state[%d] = %s", i, prev_old_state ); - //console.log( "new_state[%d] = %s", i, new_state ); + console.log( "old_state[%d] = %s", i, old_state ); + console.log( "prev_old_state[%d] = %s", i, prev_old_state ); + console.log( "new_state[%d] = %s", i, new_state ); if ( ( old_state=='pending' && new_state=='running' ) || ( old_state=='shutting-down' && new_state=='available' ) || \ ( old_state=='running' && new_state=='available' ) || ( old_state=='running' && new_state=='error' ) || \ ( old_state=='pending' && new_state=='error' ) || ( old_state=='pending' && new_state=='available' ) || \ ( old_state=='submitted' && new_state=='available' ) || ( prev_old_state.match('newUCI') && new_state=='available' ) || \ - ( prev_old_state.match('new') && new_state=='available' ) ) { + ( prev_old_state.match('new') && new_state=='available' ) || ( prev_old_state.match('deletingUCI') && new_state=='deleted' ) || \ + ( prev_old_state.match('deleting') && new_state=='deleted' ) ) { var url = "${h.url_for( controller='cloud', action='list')}"; location.replace( url ); } @@ -50,7 +51,8 @@ var url = "${h.url_for( controller='cloud', action='list')}"; location.replace( url ); } - else if ( new_state=='shutting-down' || new_state=='shutting-downUCI' ) { + + if ( new_state=='shutting-down' || new_state=='shutting-downUCI' ) { $(elem + "-link").text( "" ); } @@ -198,7 +200,7 @@ <colgroup width="25%"></colgroup> <colgroup width="10%"></colgroup> <tr class="header"> - <th>Instance name (credentials)</th> + <th>Live instance name (credentials)</th> <th>Storage size (GB)</th> <th>State</th> <th>Alive since</th> @@ -255,24 +257,23 @@ </tr> %endif </table> - + <p /> ## ***************************************************** ## Manage previously configured instances - <table class="mange-table noHR" border="0" cellspacing="0" cellpadding="0" width="100%"> + ## <table class="mange-table noHR" border="0" cellspacing="0" cellpadding="0" width="100%"> + <table class="mange-table colored" border="0" cellspacing="0" cellpadding="0" width="100%"> <colgroup width="40%"></colgroup> <colgroup width="15%"></colgroup> <colgroup width="10%"></colgroup> - <colgroup width="35%"></colgroup> - ##<tr class="header"> - ##<th>Previously configured instances</th> - ##<th>Storage size (GB)</th> - ##<th>State</th> - ##<th>Alive since</th> - ##<th></th> - ##<th></th> - ##<th></th> - ##<th></th> - ##</tr> + <colgroup width="25%"></colgroup> + <colgroup width="10%"></colgroup> + <tr class="header"> + <th>Configured instance name (credentials)</th> + <th>Storage size (GB)</th> + <th>State</th> + <th></th> + <th></th> + </tr> %if prevInstances: %for i, prevInstance in enumerate( prevInstances ): @@ -307,8 +308,7 @@ ${str(prevInstance.state)} %endif </td> - <td>N/A</td> - <td> + <td> <div popupmenu="pi-${i}-popup"> <a class="action-button" href="${h.url_for( action='start', id=trans.security.encode_id(prevInstance.id), type='m1.small' )}"> Start m1.small</a> <a class="action-button" href="${h.url_for( action='start', id=trans.security.encode_id(prevInstance.id), type='c1.medium' )}"> Start c1.medium</a> diff -r 7c4cfc243cc3 -r c78e8e1514e4 templates/cloud/viewInstance.mako --- a/templates/cloud/viewInstance.mako Mon Nov 09 18:55:14 2009 -0500 +++ b/templates/cloud/viewInstance.mako Tue Nov 10 18:36:21 2009 -0500 @@ -115,21 +115,22 @@ %if liveInstance.uci.key_pair_material != None: <tr> <td> Keypair material:</td> - <div id="shortComment2"> - <a onclick="document.getElementById('fullComment2').style.display = 'block'; - document.getElementById('shortComment2').style.display = 'none'; return 0" + <td> + <div id="short"> + <a onclick="document.getElementById('full').style.display = 'block'; + document.getElementById('short').style.display = 'none'; return 0" href="javascript:void(0)"> + Show </a> </div> - <div id="fullComment2" style="DISPLAY: none"> - <nobr><b>${liveInstance.uci.key_pair_material}</b></nobr><br/> - <a onclick="document.getElementById('shortComment2').style.display = 'block'; - document.getElementById('fullComment2').style.display = 'none'; return 0;" + <div id="full" style="DISPLAY: none"> + <a onclick="document.getElementById('short').style.display = 'block'; + document.getElementById('full').style.display = 'none'; return 0;" href="javascript:void(0)"> - - Hide - </a> - </div> + - Hide</a> + ${liveInstance.uci.key_pair_material}<br/> + </div> + </td> </tr> %endif

1 0

[hg] galaxy 3084: Minor style modifications/clarifications.
by Greg Von Kuster 23 Nov '09

23 Nov '09

details: http://www.bx.psu.edu/hg/galaxy/rev/b35215d0913e changeset: 3084:b35215d0913e user: Enis Afgan <afgane(a)gmail.com> date: Thu Nov 05 23:30:50 2009 -0500 description: Minor style modifications/clarifications. diffstat: lib/galaxy/cloud/providers/ec2.py | 1 + lib/galaxy/model/migrate/versions/0014_cloud_tables.py | 73 ++---------------- lib/galaxy/web/controllers/cloud.py | 5 + 3 files changed, 15 insertions(+), 64 deletions(-) diffs (115 lines): diff -r 28252033d66a -r b35215d0913e lib/galaxy/cloud/providers/ec2.py --- a/lib/galaxy/cloud/providers/ec2.py Thu Nov 05 11:27:00 2009 -0500 +++ b/lib/galaxy/cloud/providers/ec2.py Thu Nov 05 23:30:50 2009 -0500 @@ -506,6 +506,7 @@ log.error( "Retrieving volume(s) from cloud for UCI '%s' failed: " % ( uci.name, str(e) ) ) uci.error( "Retrieving volume(s) from cloud failed: " + str(e) ) uci.state( uci_states.ERROR ) + uci.flush() return None # Update store status in local DB with info from cloud provider diff -r 28252033d66a -r b35215d0913e lib/galaxy/model/migrate/versions/0014_cloud_tables.py --- a/lib/galaxy/model/migrate/versions/0014_cloud_tables.py Thu Nov 05 11:27:00 2009 -0500 +++ b/lib/galaxy/model/migrate/versions/0014_cloud_tables.py Thu Nov 05 23:30:50 2009 -0500 @@ -101,75 +101,20 @@ def upgrade(): metadata.reflect() - log.debug( "Creating cloud_image table." ) - try: - CloudImage_table.create() - except Exception, e: - log.debug( "Creating cloud_image table failed. Table probably exists already." ) - log.debug( "Creating cloud_uci table." ) - try: - UCI_table.create() - except Exception, e: - log.debug( "Creating UCI table failed. Table probably exists already." ) - try: - CloudUserCredentials_table.create() - except Exception, e: - log.debug( "Creating cloud_image table failed. Table probably exists already." ) - log.debug( "Creating cloud_provider table." ) - try: - CloudProvider_table.create() - except Exception, e: - log.debug( "Creating cloud_provider table failed. Table probably exists already." ) - log.debug( "Creating cloud_instance table." ) - #try: + + CloudImage_table.create() + UCI_table.create() + CloudUserCredentials_table.create() + CloudProvider_table.create() CloudInstance_table.create() - #except Exception, e: - # log.debug( "Creating cloud_instance table failed. Table probably exists already." ) - #log.debug( "Creating cloud_store table." ) - #try: CloudStore_table.create() - #except Exception: - # log.debug( "Creating cloud_store table failed. Table probably exists already." ) def downgrade(): metadata.reflect() - try: - #log.debug( "Would drop cloud_image table." ) - CloudImage_table.drop() #Enable before release - except Exception, e: - log.debug( "Dropping cloud_image table failed: %s" % str( e ) ) -# try: -# #log.debug( "Would drop cloud_instance table." ) -# print "inst" + CloudImage_table.drop() CloudInstance_table.drop() -# except Exception, e: -# log.debug( "Dropping cloud_instance table failed: %s" % str( e ) ) - -# try: -# #log.debug( "Would drop cloud_store table." ) -# print "store" CloudStore_table.drop() -# except Exception, e: -# log.debug( "Dropping cloud_store table failed: %s" % str( e ) ) - - try: - log.debug( "Would drop cloud_user_credentials table." ) - CloudUserCredentials_table.drop() #Enable before putting final version - except Exception, e: - log.debug( "Dropping cloud_user_credentials table failed: %s" % str( e ) ) - - try: - log.debug( "Would drop UCI table." ) - UCI_table.drop() - except Exception, e: - log.debug( "Dropping UCI table failed: %s" % str( e ) ) - - try: - log.debug( "Would drop cloud_provider table." ) - CloudProvider_table.drop() - except Exception, e: - log.debug( "Dropping cloud_provider table failed: %s" % str( e ) ) - - - + CloudUserCredentials_table.drop() + UCI_table.drop() + CloudProvider_table.drop() \ No newline at end of file diff -r 28252033d66a -r b35215d0913e lib/galaxy/web/controllers/cloud.py --- a/lib/galaxy/web/controllers/cloud.py Thu Nov 05 11:27:00 2009 -0500 +++ b/lib/galaxy/web/controllers/cloud.py Thu Nov 05 23:30:50 2009 -0500 @@ -76,6 +76,11 @@ .filter_by( user=user ) \ .order_by( model.CloudUserCredentials.c.name ) \ .all() + + cloudProviders = trans.sa_session.query( model.CloudProvider ) \ + .filter_by( user=user ) \ + .order_by( model.CloudUserCredentials.c.name ) \ + .all() liveInstances = trans.sa_session.query( model.UCI ) \ .filter_by( user=user ) \

1 0

[hg] galaxy 3085: Added table for seeing user-registerd cloud pr...
by Greg Von Kuster 23 Nov '09

23 Nov '09

details: http://www.bx.psu.edu/hg/galaxy/rev/dc1a6f3a2c08 changeset: 3085:dc1a6f3a2c08 user: Enis Afgan <afgane(a)gmail.com> date: Fri Nov 06 12:27:52 2009 -0500 description: Added table for seeing user-registerd cloud providers to the main UI and ability to manipulate the same. diffstat: lib/galaxy/web/controllers/cloud.py | 174 ++++++++++++++++- templates/cloud/add_provider.mako | 4 +- templates/cloud/configure_cloud.mako | 373 +++++++++++++++++++++---------------- templates/cloud/edit_provider.mako | 261 ++++++++++++++++++++++++++ templates/cloud/view_provider.mako | 126 ++++++++++++ 5 files changed, 766 insertions(+), 172 deletions(-) diffs (1068 lines): diff -r b35215d0913e -r dc1a6f3a2c08 lib/galaxy/web/controllers/cloud.py --- a/lib/galaxy/web/controllers/cloud.py Thu Nov 05 23:30:50 2009 -0500 +++ b/lib/galaxy/web/controllers/cloud.py Fri Nov 06 12:27:52 2009 -0500 @@ -79,7 +79,7 @@ cloudProviders = trans.sa_session.query( model.CloudProvider ) \ .filter_by( user=user ) \ - .order_by( model.CloudUserCredentials.c.name ) \ + .order_by( model.CloudProvider.c.name ) \ .all() liveInstances = trans.sa_session.query( model.UCI ) \ @@ -126,7 +126,8 @@ return trans.fill_template( "cloud/configure_cloud.mako", cloudCredentials = cloudCredentials, liveInstances = liveInstances, - prevInstances = prevInstances ) + prevInstances = prevInstances, + cloudProviders = cloudProviders ) @web.require_login( "use Galaxy cloud" ) def makeDefault( self, trans, id=None ): @@ -597,7 +598,6 @@ return trans.fill_template( "cloud/viewInstance.mako", liveInstance = instances ) - @web.expose @web.require_login( "delete credentials" ) def delete( self, trans, id=None ): @@ -636,7 +636,7 @@ try: is_secure = int(is_secure) except ValueError: - error['is_secure_error'] = "Field 'is secure' can only take on a value '0' or '1'" + error['is_secure_error'] = "Field 'is secure' can only take on an integer value '0' or '1'" if trans.app.model.CloudProvider \ .filter_by (user=user, name=name) \ @@ -647,7 +647,7 @@ elif type=='': error['type_error'] = "Provider type must be selected." elif not (is_secure == 0 or is_secure == 1): - error['is_secure_error'] = "Field 'is secure' can only take on a value '0' or '1'" + error['is_secure_error'] = "Field 'is secure' can only take on an integer value '0' or '1'" else: provider = model.CloudProvider() provider.user = user @@ -664,10 +664,8 @@ provider.region_endpoint = None if is_secure==0: - log.debug("is_secure is false") provider.is_secure = False else: - log.debug("is_secure is true") provider.is_secure = True if host: @@ -745,6 +743,150 @@ self.add_provider( trans, name='Amazon EC2', type='ec2', region_name='us-east-1', region_endpoint='us-east-1.ec2.amazonaws.com', is_secure=1, path='/' ) return self.add( trans ) + @web.expose + @web.require_login( "use Galaxy cloud" ) + def view_provider( self, trans, id=None ): + """ + View details about given cloud provider + """ + # Load credentials from database + provider = get_provider_by_id( trans, id ) + + return trans.fill_template( "cloud/view_provider.mako", + provider = provider ) + + @web.expose + @web.require_login( "use Galaxy cloud" ) + def edit_provider( self, trans, id, name='', type='', region_name='', region_endpoint='', is_secure='', host='', port='', proxy='', proxy_port='', + proxy_user='', proxy_pass='', debug='', https_connection_factory='', path='', edited=False ): + error = {} + if edited == False: + provider = get_provider_by_id( trans, id ) + return trans.fill_template( "cloud/edit_provider.mako", + provider = provider, + error = error + ) + else: + user = trans.get_user() + provider = get_provider_by_id( trans, id ) + + try: + is_secure = int(is_secure) + except ValueError: + error['is_secure_error'] = "Field 'is secure' can only take on an integer value '0' or '1'" + + if name=='' or len( name ) > 255: + error['name_error'] = "Cloud provider name must be between 1 and 255 characters in length." + elif trans.app.model.CloudProvider \ + .filter_by( user=user ) \ + .filter( and_( trans.app.model.CloudProvider.table.c.id != provider.id, trans.app.model.CloudProvider.table.c.name == name ) ) \ + .first(): + error['name_error'] = "Cloud provider with name '" + name + "' already exist. Please choose an alternative name." + elif not ( is_secure == 0 or is_secure == 1): + error['is_secure_error'] = "Field 'is secure' can only take on an integer value '0' or '1'" + + if error: + return trans.fill_template( "cloud/edit_provider.mako", + provider = provider, + error = error + ) + else: + provider.name = name + if region_name and region_name != 'None': + provider.region_name = region_name + else: + provider.region_name = None + + if region_endpoint and region_endpoint != 'None': + provider.region_endpoint = region_endpoint + else: + provider.region_endpoint = None + + if is_secure==0: + provider.is_secure = False + else: + provider.is_secure = True + + if host and host != 'None': + provider.host = host + else: + provider.host = None + + if port and port != 'None': + provider.port = port + else: + provider.port = None + + if proxy and proxy != 'None': + provider.proxy = proxy + else: + provider.proxy = None + + if proxy_port and proxy_port != 'None': + provider.proxy_port = proxy_port + else: + provider.proxy_port = None + + if proxy_user and proxy_user != 'None': + provider.proxy_user = proxy_user + else: + provider.proxy_user = None + + if proxy_pass and proxy_pass != 'None': + provider.proxy_pass = proxy_pass + else: + provider.proxy_pass = None + + if debug and debug != 'None': + provider.debug = debug + else: + provider.debug = None + + if https_connection_factory and https_connection_factory != 'None': + provider.https_connection_factory = https_connection_factory + else: + provider.https_connection_factory = None + + if path and path != 'None': + provider.path = path + else: + provider.path = None + # Persist + session = trans.sa_session + session.save_or_update( provider ) + session.flush() + # Log and display the management page + trans.log_event( "User edited cloud provider: '%s'" % name ) + trans.set_message( "Cloud provider '%s' edited." % name ) + return self.list( trans ) + + @web.expose + @web.require_login( "delete credentials" ) + def delete_provider( self, trans, id=None ): + """ + Delete use-registered cloud provider checking that no registered credentials are tied to given provider. + """ + # Load provider from database + user = trans.get_user() + provider = get_provider_by_id( trans, id ) + creds = trans.sa_session.query( model.CloudUserCredentials ) \ + .filter_by( user=user, provider_id=provider.id ) \ + .filter( model.UCI.c.state!=uci_states.DELETED ) \ + .all() + + if len( creds ) == 0: + # Delete and save + sess = trans.sa_session + sess.delete( provider ) + provider.flush() + # Display the management page + trans.set_message( "Cloud provider '%s' deleted." % provider.name ) + return self.list( trans ) + + error( "Existing credentails depend on cloud provider '%s'. You must delete those credentials before being able \ + to delete this cloud provider." % provider.name ) + return self.list( trans ) + @web.json def json_update( self, trans ): user = trans.get_user() @@ -789,6 +931,24 @@ return trans.app.model.CloudProvider \ .filter_by (user=user, name=name) \ .first() + +def get_provider_by_id( trans, id, check_ownership=True ): + # Check if 'id' is in int (i.e., it was called from this program) or + # it was passed from the web (in which case decode it) + if not isinstance( id, int ): + id = trans.security.decode_id( id ) + + stored = trans.sa_session.query( model.CloudProvider ).get( id ) + if not stored: + error( "Cloud provider not found" ) + # Verify ownership + user = trans.get_user() + if not user: + error( "Must be logged in to use the cloud." ) + if check_ownership and not( stored.user == user ): + error( "Cloud provider '%s' is not registered by current user." % stored.name ) + # Looks good + return stored def get_stored_credentials( trans, id, check_ownership=True ): """ diff -r b35215d0913e -r dc1a6f3a2c08 templates/cloud/add_provider.mako --- a/templates/cloud/add_provider.mako Thu Nov 05 23:30:50 2009 -0500 +++ b/templates/cloud/add_provider.mako Fri Nov 06 12:27:52 2009 -0500 @@ -32,15 +32,13 @@ function af(){ if ( $("#autofill").attr('checked') ) { + $("#name").val("Eucalyptus Public Cloud"); $("#region_name").val("eucalyptus"); $("#region_endpoint").val("mayhem9.cs.ucsb.edu"); $("#is_secure").val("0"); $("#port").val("8773"); $("#path").val("/services/Eucalyptus"); } - else { - clear(); - } } function clear() { diff -r b35215d0913e -r dc1a6f3a2c08 templates/cloud/configure_cloud.mako --- a/templates/cloud/configure_cloud.mako Thu Nov 05 23:30:50 2009 -0500 +++ b/templates/cloud/configure_cloud.mako Fri Nov 06 12:27:52 2009 -0500 @@ -94,202 +94,251 @@ </%def> <h2>Galaxy in the clouds</h2> - -%if cloudCredentials: - ## Manage user credentials - <h3>Your registered credentials</h3> + +%if cloudProviders: + ## Manage user-registered cloud providers + <h3>Your registered cloud providers</h3> <ul class="manage-table-actions"> <li> - <a class="action-button" href="${h.url_for( action='add' )}"> + <a class="action-button" href="${h.url_for( action='add_provider' )}"> <img src="${h.url_for('/static/images/silk/add.png')}" /> - <span>Add credentials</span> + <span>Add provider</span> </a> </li> </ul> <table class="mange-table colored" border="0" cellspacing="0" cellpadding="0" width="100%"> <tr class="header"> - <th>Credentials name</th> - <th>Provider name (type)</th> + <th>Provider name</th> + <th>Provider type</th> <th></th> </tr> - %for i, cloudCredential in enumerate( cloudCredentials ): + %for i, cloudProvder in enumerate( cloudProviders ): <tr> <td> - ${cloudCredential.name} - <a id="cr-${i}-popup" class="popup-arrow" style="display: none;">▼</a> + ${cloudProvder.name} + <a id="cp-${i}-popup" class="popup-arrow" style="display: none;">▼</a> </td> <td> - ${cloudCredential.provider.name} - (${cloudCredential.provider.type}) + ${cloudProvder.type} </td> <td> - <div popupmenu="cr-${i}-popup"> - <a class="action-button" href="${h.url_for( action='view', id=trans.security.encode_id(cloudCredential.id) )}">View</a> - <a class="action-button" href="${h.url_for( action='edit', id=trans.security.encode_id(cloudCredential.id) )}">Edit</a> - <a class="action-button" confirm="Are you sure you want to delete credentials '${cloudCredential.name}'?" href="${h.url_for( action='delete', id=trans.security.encode_id(cloudCredential.id) )}">Delete</a> + <div popupmenu="cp-${i}-popup"> + <a class="action-button" href="${h.url_for( action='view_provider', id=trans.security.encode_id(cloudProvder.id) )}">View</a> + <a class="action-button" href="${h.url_for( action='edit_provider', id=trans.security.encode_id(cloudProvder.id) )}">Edit</a> + <a class="action-button" confirm="Are you sure you want to delete cloud provider '${cloudProvder.name}'?" href="${h.url_for( action='delete_provider', id=trans.security.encode_id(cloudProvder.id) )}">Delete</a> </div> </td> </tr> %endfor </table> + ## ***************************************************** - ## Manage live instances - <p /> - <h3>Manage your cloud instances</h3> - <ul class="manage-table-actions"> - <li> - <a class="action-button" href="${h.url_for( action='configureNew' )}"> - <img src="${h.url_for('/static/images/silk/add.png')}" /> - <span>Configure new instance</span> - </a> - </li> - </ul> - - <table class="mange-table colored" border="0" cellspacing="0" cellpadding="0" width="100%"> - <colgroup width="40%"></colgroup> - <colgroup width="15%"></colgroup> - <colgroup width="10%"></colgroup> - <colgroup width="25%"></colgroup> - <colgroup width="10%"></colgroup> - <tr class="header"> - <th>Instance name (credentials)</th> - <th>Storage size (GB)</th> - <th>State</th> - <th>Alive since</th> - <th></th> - <th></th> - </tr> - %if liveInstances: - %for i, liveInstance in enumerate( liveInstances ): + ## Manage user credentials + <h3>Your registered cloud credentials</h3> + + %if cloudCredentials: + <ul class="manage-table-actions"> + <li> + <a class="action-button" href="${h.url_for( action='add' )}"> + <img src="${h.url_for('/static/images/silk/add.png')}" /> + <span>Add credentials</span> + </a> + </li> + </ul> + <table class="mange-table colored" border="0" cellspacing="0" cellpadding="0" width="100%"> + <tr class="header"> + <th>Credentials name</th> + <th>Provider name (type)</th> + <th></th> + </tr> + + %for i, cloudCredential in enumerate( cloudCredentials ): <tr> <td> - ${liveInstance.name} (${liveInstance.credentials.name}) - <a id="li-${i}-popup" class="popup-arrow" style="display: none;">▼</a> + ${cloudCredential.name} + <a id="cr-${i}-popup" class="popup-arrow" style="display: none;">▼</a> </td> - <td>${str(liveInstance.total_size)}</td> - <td id="${ liveInstance.id }-state">${str(liveInstance.state)}</td> - <td id="${ liveInstance.id }-launch_time"> - ##${str(liveInstance.launch_time)[:16]} - <% - #from datetime import datetime - #from datetime import timedelta - - # DB stores all times in GMT, so adjust for difference (4 hours) - #adjustedStarttime = liveInstance.update_time - timedelta(hours=4) - - #delta = datetime.now() - adjustedStarttime - #context.write( str(datetime.utcnow() ) ) - #context.write( str(delta) ) - - # This is where current time and since duration is calculated - if liveInstance.launch_time is None: - context.write( 'N/A' ) - else: - context.write( str( liveInstance.launch_time )[:16] ) - context.write( ' UTC (' ) - context.write( str(h.date.distance_of_time_in_words (liveInstance.launch_time, h.date.datetime.utcnow() ) ) ) - context.write( ')' ) - %> + <td> + ${cloudCredential.provider.name} + (${cloudCredential.provider.type}) </td> - ## Handled by JavaScript function - <td id="${ liveInstance.id }-link"></td> - <td> - <div popupmenu="li-${i}-popup"> - <a class="action-button" confirm="Are you sure you want to stop instance '${liveInstance.name}'? Please note that this may take up to 1 minute during which time the page will not refresh." href="${h.url_for( action='stop', id=trans.security.encode_id(liveInstance.id) )}">Stop</a> - <a class="action-button" href="${h.url_for( action='renameInstance', id=trans.security.encode_id(liveInstance.id) )}">Rename</a> - <a class="action-button" href="${h.url_for( action='viewInstance', id=trans.security.encode_id(liveInstance.id) )}">View details</a> - <a class="action-button" href="${h.url_for( action='usageReport', id=trans.security.encode_id(liveInstance.id) )}">Usage report</a> + <td> + <div popupmenu="cr-${i}-popup"> + <a class="action-button" href="${h.url_for( action='view', id=trans.security.encode_id(cloudCredential.id) )}">View</a> + <a class="action-button" href="${h.url_for( action='edit', id=trans.security.encode_id(cloudCredential.id) )}">Edit</a> + <a class="action-button" confirm="Are you sure you want to delete credentials '${cloudCredential.name}'?" href="${h.url_for( action='delete', id=trans.security.encode_id(cloudCredential.id) )}">Delete</a> </div> </td> </tr> %endfor - %else: - <tr> - <td>Currently, you have no live instances.</td> - </tr> - %endif - </table> + </table> + + ## ***************************************************** + ## Manage live instances + <p /> + <h3>Manage your cloud instances</h3> + <ul class="manage-table-actions"> + <li> + <a class="action-button" href="${h.url_for( action='configureNew' )}"> + <img src="${h.url_for('/static/images/silk/add.png')}" /> + <span>Configure new instance</span> + </a> + </li> + </ul> - ## ***************************************************** - ## Manage previously configured instances - <table class="mange-table noHR" border="0" cellspacing="0" cellpadding="0" width="100%"> - <colgroup width="40%"></colgroup> - <colgroup width="15%"></colgroup> - <colgroup width="10%"></colgroup> - <colgroup width="35%"></colgroup> - ##<tr class="header"> - ##<th>Previously configured instances</th> - ##<th>Storage size (GB)</th> - ##<th>State</th> - ##<th>Alive since</th> - ##<th></th> - ##<th></th> - ##<th></th> - ##<th></th> - ##</tr> - - %if prevInstances: - %for i, prevInstance in enumerate( prevInstances ): - <tr> - <td> - ${prevInstance.name} (${prevInstance.credentials.name}) - <a id="pi-${i}-popup" class="popup-arrow" style="display: none;">▼</a> - </td> - <td>${str(prevInstance.total_size)}</td> - <td id="${ prevInstance.id }-state-p"> - <%state = str(prevInstance.state)%> - %if state =='error': - <div id="${prevInstance.name}-short"> - <a onclick="document.getElementById('${prevInstance.name}-full').style.display = 'block'; - document.getElementById('${prevInstance.name}-short').style.display = 'none'; return 0" - href="javascript:void(0)"> - error - </a> - </div> - <div id="${prevInstance.name}-full" style="DISPLAY: none"> - <a onclick="document.getElementById('${prevInstance.name}-short').style.display = 'block'; - document.getElementById('${prevInstance.name}-full').style.display = 'none'; return 0;" - href="javascript:void(0)"> - error:</a><br /> - ${str(prevInstance.error)} - <p /> - <div style="font-size:10px;"> - <a href="${h.url_for( action='set_uci_state', id=trans.security.encode_id(prevInstance.id), state='available' )}">reset state</a> - </div> - </div> - %else: - ${str(prevInstance.state)} - %endif - </td> - <td>N/A</td> - <td> - <div popupmenu="pi-${i}-popup"> - <a class="action-button" href="${h.url_for( action='start', id=trans.security.encode_id(prevInstance.id), type='m1.small' )}"> Start m1.small</a> - <a class="action-button" href="${h.url_for( action='start', id=trans.security.encode_id(prevInstance.id), type='c1.medium' )}"> Start c1.medium</a> - <a class="action-button" href="${h.url_for( action='renameInstance', id=trans.security.encode_id(prevInstance.id) )}">Rename</a> - <a class="action-button" href="${h.url_for( action='addStorage', id=trans.security.encode_id(prevInstance.id) )}" target="_parent">Add storage</a> - <a class="action-button" href="${h.url_for( action='usageReport', id=trans.security.encode_id(prevInstance.id) )}">Usage report</a> - <a class="action-button" confirm="Are you sure you want to delete instance '${prevInstance.name}'? This will delete all of your data assocaiated with this instance!" href="${h.url_for( action='deleteInstance', id=trans.security.encode_id(prevInstance.id) )}">Delete</a> - </div> - </td> - </tr> - %endfor - %else: - <tr> - <td>You have no previously configured instances (or they are all currently alive).</td> - </tr> - %endif - </table> - + <table class="mange-table colored" border="0" cellspacing="0" cellpadding="0" width="100%"> + <colgroup width="40%"></colgroup> + <colgroup width="15%"></colgroup> + <colgroup width="10%"></colgroup> + <colgroup width="25%"></colgroup> + <colgroup width="10%"></colgroup> + <tr class="header"> + <th>Instance name (credentials)</th> + <th>Storage size (GB)</th> + <th>State</th> + <th>Alive since</th> + <th></th> + <th></th> + </tr> + %if liveInstances: + %for i, liveInstance in enumerate( liveInstances ): + <tr> + <td> + ${liveInstance.name} (${liveInstance.credentials.name}) + <a id="li-${i}-popup" class="popup-arrow" style="display: none;">▼</a> + </td> + <td>${str(liveInstance.total_size)}</td> + <td id="${ liveInstance.id }-state">${str(liveInstance.state)}</td> + <td id="${ liveInstance.id }-launch_time"> + ##${str(liveInstance.launch_time)[:16]} + <% + #from datetime import datetime + #from datetime import timedelta + + # DB stores all times in GMT, so adjust for difference (4 hours) + #adjustedStarttime = liveInstance.update_time - timedelta(hours=4) + + #delta = datetime.now() - adjustedStarttime + #context.write( str(datetime.utcnow() ) ) + #context.write( str(delta) ) + + # This is where current time and since duration is calculated + if liveInstance.launch_time is None: + context.write( 'N/A' ) + else: + context.write( str( liveInstance.launch_time )[:16] ) + context.write( ' UTC (' ) + context.write( str(h.date.distance_of_time_in_words (liveInstance.launch_time, h.date.datetime.utcnow() ) ) ) + context.write( ')' ) + %> + </td> + ## Handled by JavaScript function + <td id="${ liveInstance.id }-link"></td> + <td> + <div popupmenu="li-${i}-popup"> + <a class="action-button" confirm="Are you sure you want to stop instance '${liveInstance.name}'? Please note that this may take up to 1 minute during which time the page will not refresh." href="${h.url_for( action='stop', id=trans.security.encode_id(liveInstance.id) )}">Stop</a> + <a class="action-button" href="${h.url_for( action='renameInstance', id=trans.security.encode_id(liveInstance.id) )}">Rename</a> + <a class="action-button" href="${h.url_for( action='viewInstance', id=trans.security.encode_id(liveInstance.id) )}">View details</a> + <a class="action-button" href="${h.url_for( action='usageReport', id=trans.security.encode_id(liveInstance.id) )}">Usage report</a> + </div> + </td> + </tr> + %endfor + %else: + <tr> + <td>Currently, you have no live instances.</td> + </tr> + %endif + </table> + + ## ***************************************************** + ## Manage previously configured instances + <table class="mange-table noHR" border="0" cellspacing="0" cellpadding="0" width="100%"> + <colgroup width="40%"></colgroup> + <colgroup width="15%"></colgroup> + <colgroup width="10%"></colgroup> + <colgroup width="35%"></colgroup> + ##<tr class="header"> + ##<th>Previously configured instances</th> + ##<th>Storage size (GB)</th> + ##<th>State</th> + ##<th>Alive since</th> + ##<th></th> + ##<th></th> + ##<th></th> + ##<th></th> + ##</tr> + + %if prevInstances: + %for i, prevInstance in enumerate( prevInstances ): + <tr> + <td> + ${prevInstance.name} (${prevInstance.credentials.name}) + <a id="pi-${i}-popup" class="popup-arrow" style="display: none;">▼</a> + </td> + <td>${str(prevInstance.total_size)}</td> + <td id="${ prevInstance.id }-state-p"> + <%state = str(prevInstance.state)%> + %if state =='error': + <div id="${prevInstance.name}-short"> + <a onclick="document.getElementById('${prevInstance.name}-full').style.display = 'block'; + document.getElementById('${prevInstance.name}-short').style.display = 'none'; return 0" + href="javascript:void(0)"> + error + </a> + </div> + <div id="${prevInstance.name}-full" style="DISPLAY: none"> + <a onclick="document.getElementById('${prevInstance.name}-short').style.display = 'block'; + document.getElementById('${prevInstance.name}-full').style.display = 'none'; return 0;" + href="javascript:void(0)"> + error:</a><br /> + ${str(prevInstance.error)} + <p /> + <div style="font-size:10px;"> + <a href="${h.url_for( action='set_uci_state', id=trans.security.encode_id(prevInstance.id), state='available' )}">reset state</a> + </div> + </div> + %else: + ${str(prevInstance.state)} + %endif + </td> + <td>N/A</td> + <td> + <div popupmenu="pi-${i}-popup"> + <a class="action-button" href="${h.url_for( action='start', id=trans.security.encode_id(prevInstance.id), type='m1.small' )}"> Start m1.small</a> + <a class="action-button" href="${h.url_for( action='start', id=trans.security.encode_id(prevInstance.id), type='c1.medium' )}"> Start c1.medium</a> + <a class="action-button" href="${h.url_for( action='renameInstance', id=trans.security.encode_id(prevInstance.id) )}">Rename</a> + <a class="action-button" href="${h.url_for( action='addStorage', id=trans.security.encode_id(prevInstance.id) )}" target="_parent">Add storage</a> + <a class="action-button" href="${h.url_for( action='usageReport', id=trans.security.encode_id(prevInstance.id) )}">Usage report</a> + <a class="action-button" confirm="Are you sure you want to delete instance '${prevInstance.name}'? This will delete all of your data assocaiated with this instance!" href="${h.url_for( action='deleteInstance', id=trans.security.encode_id(prevInstance.id) )}">Delete</a> + </div> + </td> + </tr> + %endfor + %else: + <tr> + <td>You have no previously configured instances (or they are all currently alive).</td> + </tr> + %endif + </table> + + %else: + You have no credentials associated with your Galaxy account: + <a class="action-button" href="${h.url_for( action='add' )}"> + <img src="${h.url_for('/static/images/silk/add.png')}" /> + <span>add credentials</span> + </a> + or + <a href="http://aws.amazon.com/" target="_blank"> + open AWS account with Amazon</a>. + %endif + %else: - You have no AWS credentials associated with your Galaxy account: - <a class="action-button" href="${h.url_for( action='add' )}"> + You have no cloud providers registered with your Galaxy account: + <a class="action-button" href="${h.url_for( action='add_provider' )}"> <img src="${h.url_for('/static/images/silk/add.png')}" /> - <span>add credentials</span> + <span>add provider now</span> </a> - or - <a href="http://aws.amazon.com/" target="_blank"> - open AWS account with Amazon</a>. %endif \ No newline at end of file diff -r b35215d0913e -r dc1a6f3a2c08 templates/cloud/edit_provider.mako --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/templates/cloud/edit_provider.mako Fri Nov 06 12:27:52 2009 -0500 @@ -0,0 +1,261 @@ +<% _=n_ %> +<%inherit file="/base.mako"/> +<%def name="title()">Edit provider</%def> + +<%def name="javascripts()"> +${parent.javascripts()} +<script type="text/javascript"> +$(function(){ + + $("input:text:first").focus(); + + $("#type").change(function() { + if ($(this).val() == 'ec2') { + clear(); + $("#autofill").attr( 'disabled', true ); + $("#autofill").attr( 'checked', false ); + $("#name").val( "EC2" ); + $("#region_name").val( "us-east-1" ); + $("#region_endpoint").val( "us-east-1.ec2.amazonaws.com" ); + $("#is_secure").val("1"); + $("#debug").val(""); + $("#path").val("/"); + } + else if ($(this).val() == 'eucalyptus') { + clear(); + $("#autofill").attr( 'disabled', false ); + } + }); +}) + + +function af(){ + + if ( $("#autofill_epc").attr('checked') ) { + $("#region_name").val("eucalyptus"); + $("#region_endpoint").val("mayhem9.cs.ucsb.edu"); + $("#is_secure").val("0"); + $("#port").val("8773"); + $("#path").val("/services/Eucalyptus"); + } + else if ( $("#autofill_ec2").attr('checked') ) { + $("#region_name").val( "us-east-1" ); + $("#region_endpoint").val( "us-east-1.ec2.amazonaws.com" ); + $("#is_secure").val("1"); + $("#debug").val(""); + $("#path").val("/"); + } +} + +function clear() { + //$("#name").val(""); + $("#region_name").val(""); + $("#region_endpoint").val(""); + $("#is_secure").val(""); + $("#port").val(""); + $("#proxy").val(""); + $("#proxy_port").val(""); + $("#proxy_user").val(""); + $("#proxy_pass").val(""); + $("#debug").val(""); + $("#https_connection_factory").val(""); + $("#path").val(""); + +} + +</script> +</%def> + +%if header: + ${header} +%endif + +%if provider: + <div class="form"> + <div class="form-title">Edit cloud provider</div> + <div class="form-body"> + <form name="edit_provider_form" action="${h.url_for( action='edit_provider', id=trans.security.encode_id(provider.id), edited="true" )}" method="post" > + <% + cls = "form-row" + if error.has_key('type_error'): + cls += " form-row-error" + %> + <div class="${cls}"> + <label>Provider type:</label> + <div class="form-row-input">${provider.type} + %if provider.type == 'eucalyptus': + <p><input type="checkbox" id="autofill_epc" onclick="javascript:af()"> + auto fill using Eucalyptus Public Cloud values + </p></div> + %elif provider.type == 'ec2': + <p><input type="checkbox" id="autofill_ec2" onclick="javascript:af()"> + auto fill for Amazon EC2 (us-east-1 region) + </p></div> + %endif + <div style="clear: both"></div> + </div> + + <% + cls = "form-row" + if error.has_key('name_error'): + cls += " form-row-error" + %> + <div class="${cls}"> + <label>Provider name:</label> + <div class="form-row-input"> + <input type="text" id="name" name="name" value="${provider.name}" size="40"> + </div> + %if error.has_key('name_error'): + <div class="form-row-error-message">${error['name_error']}</div> + %endif + <div style="clear: both"></div> + </div> + + <% + cls = "form-row" + %> + <div class="${cls}"> + <label>Region name:</label> + <div id="region_selection" class="form-row-input"> + <input type="text" name="region_name" id="region_name" value="${provider.region_name}" size="40"> + </div> + <div style="clear: both"></div> + </div> + + <% + cls = "form-row" + %> + <div class="${cls}"> + <label>Region endpoint:</label> + <div class="form-row-input"> + <input type="text" name="region_endpoint" id="region_endpoint" value="${provider.region_endpoint}" size="40"> + </div> + <div style="clear: both"></div> + </div> + + <% + cls = "form-row" + if error.has_key('is_secure_error'): + cls += " form-row-error" + %> + <div class="${cls}"> + <label>Is secure ('O' for False or '1' for True):</label> + <div class="form-row-input"> + %if provider.is_secure == True: + <input type="text" name="is_secure" id="is_secure" value="1" size="40"> + %else: + <input type="text" name="is_secure" id="is_secure" value="0" size="40"> + %endif + </div> + %if error.has_key('is_secure_error'): + <div class="form-row-error-message">${error['is_secure_error']}; you entered: '${provider.is_secure}'</div> + %endif + <div style="clear: both"></div> + </div> + + <% + cls = "form-row" + %> + <div class="${cls}"> + <label>Host:</label> + <div class="form-row-input"> + <input type="text" name="host" value="${provider.host}" size="40"> + </div> + <div style="clear: both"></div> + </div> + + <% + cls = "form-row" + %> + <div class="${cls}"> + <label>Port:</label> + <div class="form-row-input"> + <input type="text" name="port" id="port" value="${provider.port}" size="40"> + </div> + <div style="clear: both"></div> + </div> + + <% + cls = "form-row" + %> + <div class="${cls}"> + <label>Proxy:</label> + <div class="form-row-input"> + <input type="text" name="proxy" value="${provider.proxy}" size="40"> + </div> + <div style="clear: both"></div> + </div> + + <% + cls = "form-row" + %> + <div class="${cls}"> + <label>Proxy port:</label> + <div class="form-row-input"> + <input type="text" name="proxy_port" value="${provider.proxy_port}" size="40"> + </div> + <div style="clear: both"></div> + </div> + + <% + cls = "form-row" + %> + <div class="${cls}"> + <label>Proxy user:</label> + <div class="form-row-input"> + <input type="text" name="proxy_user" value="${provider.proxy_user}" size="40"> + </div> + <div style="clear: both"></div> + </div> + + <% + cls = "form-row" + %> + <div class="${cls}"> + <label>Proxy pass:</label> + <div class="form-row-input"> + <input type="text" name="proxy_pass" value="${provider.proxy_pass}" size="40"> + </div> + <div style="clear: both"></div> + </div> + + <% + cls = "form-row" + %> + <div class="${cls}"> + <label>Debug:</label> + <div class="form-row-input"> + <input type="text" name="debug" value="${provider.debug}" size="40"> + </div> + <div style="clear: both"></div> + </div> + + <% + cls = "form-row" + %> + <div class="${cls}"> + <label>HTTPS connection factory:</label> + <div class="form-row-input"> + <input type="text" name="https_connection_factory" value="${provider.https_connection_factory}" size="40"> + </div> + <div style="clear: both"></div> + </div> + + <% + cls = "form-row" + %> + <div class="${cls}"> + <label>Path:</label> + <div class="form-row-input"> + <input type="text" name="path" id="path" value="${provider.path}" size="40"> + </div> + <div style="clear: both"></div> + </div> + + <div class="form-row"><input type="submit" value="Save"></div> + + </form> + + </div> + </div> +%endif diff -r b35215d0913e -r dc1a6f3a2c08 templates/cloud/view_provider.mako --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/templates/cloud/view_provider.mako Fri Nov 06 12:27:52 2009 -0500 @@ -0,0 +1,126 @@ +<%inherit file="/base.mako"/> + +<%def name="title()">Cloud provider</%def> + +<h2>Cloud provider details</h2> + +<ul class="manage-table-actions"> + <li> + <a class="action-button" href="${h.url_for( action='list' )}"> + <img src="${h.url_for('/static/images/silk/resultset_previous.png')}" /> + <span>Return to cloud management console</span> + </a> + </li> +</ul> + +%if provider: + ${view_provider( provider )} +%else: + There is no cloud provider under that name. +%endif + + + + +<%def name="view_provider( provider )"> + <table class="mange-table colored" border="0" cellspacing="0" cellpadding="0" width="100%"> + <tr> + <td> Cloud provider name: </td> + <td> + ${provider.name} + <a id="cp-popup" class="popup-arrow" style="display: none;">▼</a> + </td> + <td> + <div popupmenu="cp-popup"> + <a class="action-button" href="${h.url_for( action='edit_provider', id=trans.security.encode_id(provider.id) )}">Edit</a> + <a class="action-button" confirm="Are you sure you want to delete cloud provider '${provider.name}'?" href="${h.url_for( action='delete_provider', id=trans.security.encode_id(provider.id) )}">Delete</a> + </div> + </td> + </tr> + <tr> + <td> Last updated: </td> + <td> ${str(provider.update_time)[:16]} + <% + context.write( ' UTC (' ) + context.write( str(h.date.distance_of_time_in_words (provider.update_time, h.date.datetime.utcnow() ) ) ) + %> ago) + </td> + </tr> + <tr> + <td> Cloud provider type: </td> + <td> ${str(provider.type)[:16]}</td> + </tr> + %if provider.region_connection != None: + <tr> + <td> Region connection: </td> + <td> ${provider.region_connection} </td> + </tr> + %endif + %if provider.region_name != None: + <tr> + <td> Region name: </td> + <td> ${provider.region_name} </td> + </tr> + %endif + %if provider.region_endpoint != None: + <tr> + <td> Region endpoint: </td> + <td> ${provider.region_endpoint} </td> + </tr> + %endif + %if provider.is_secure != None: + <tr> + <td> Is secure: </td> + <td> ${provider.is_secure} </td> + </tr> + %endif + %if provider.host != None: + <tr> + <td> Host: </td> + <td> ${provider.host} </td> + </tr> + %endif + %if provider.port != None: + <tr> + <td> Port: </td> + <td> ${provider.port} </td> + </tr> + %endif + %if provider.proxy != None: + <tr> + <td> Proxy: </td> + <td> ${provider.proxy} </td> + </tr> + %endif + %if provider.proxy_port != None: + <tr> + <td> Proxy port: </td> + <td> ${provider.proxy_port} </td> + </tr> + %endif + %if provider.proxy_pass != None: + <tr> + <td> Proxy pass: </td> + <td> ${provider.proxy_pass} </td> + </tr> + %endif + %if provider.debug != None: + <tr> + <td> Debug: </td> + <td> ${provider.debug} </td> + </tr> + %endif + %if provider.https_connection_factory != None: + <tr> + <td> HTTPS connection factory: </td> + <td> ${provider.https_connection_factory} </td> + </tr> + %endif + %if provider.path != None: + <tr> + <td> Path: </td> + <td> ${provider.path} </td> + </tr> + %endif + </table> +</%def>

1 0

[hg] galaxy 3083: Bug fixes regarding registration of new provid...
by Greg Von Kuster 23 Nov '09

23 Nov '09

details: http://www.bx.psu.edu/hg/galaxy/rev/28252033d66a changeset: 3083:28252033d66a user: Enis Afgan <afgane(a)gmail.com> date: Thu Nov 05 11:27:00 2009 -0500 description: Bug fixes regarding registration of new providers and creation of new UCIs. diffstat: lib/galaxy/cloud/providers/ec2.py | 73 +++++++++--------- lib/galaxy/cloud/providers/eucalyptus.py | 76 +++++++++--------- lib/galaxy/model/migrate/versions/0014_cloud_tables.py | 53 +++++++----- lib/galaxy/web/controllers/cloud.py | 13 ++- templates/cloud/list_images.mako | 4 + 5 files changed, 118 insertions(+), 101 deletions(-) diffs (340 lines): diff -r 70ae74578254 -r 28252033d66a lib/galaxy/cloud/providers/ec2.py --- a/lib/galaxy/cloud/providers/ec2.py Wed Nov 04 17:19:18 2009 -0500 +++ b/lib/galaxy/cloud/providers/ec2.py Thu Nov 05 11:27:00 2009 -0500 @@ -402,19 +402,19 @@ model.CloudStore.c.status==store_states.CREATING, model.CloudStore.c.status==None ) ).all() for store in stores: - if self.type == store.uci.credentials.provider.type and store.volume_id != None: - log.debug( "[%s] Running general status update on store '%s'" % ( store.uci.credentials.provider.type, store.volume_id ) ) + if self.type == store.uci.credentials.provider.type: # and store.volume_id != None: + log.debug( "[%s] Running general status update on store with local database ID: '%s'" % ( store.uci.credentials.provider.type, store.id ) ) self.updateStore( store ) - else: - log.error( "[%s] There exists an entry for UCI (%s) storage volume without an ID. Storage volume might have been created with " - "cloud provider though. Manual check is recommended." % ( store.uci.credentials.provider.type, store.uci.name ) ) - store.uci.error = "There exists an entry in local database for a storage volume without an ID. Storage volume might have been created " \ - "with cloud provider though. Manual check is recommended. After understanding what happened, local database etry for given " \ - "storage volume should be updated." - store.status = store_states.ERROR - store.uci.state = uci_states.ERROR - store.uci.flush() - store.flush() +# else: +# log.error( "[%s] There exists an entry for UCI (%s) storage volume without an ID. Storage volume might have been created with " +# "cloud provider though. Manual check is recommended." % ( store.uci.credentials.provider.type, store.uci.name ) ) +# store.uci.error = "There exists an entry in local database for a storage volume without an ID. Storage volume might have been created " \ +# "with cloud provider though. Manual check is recommended. After understanding what happened, local database entry for given " \ +# "storage volume should be updated." +# store.status = store_states.ERROR +# store.uci.state = uci_states.ERROR +# store.uci.flush() +# store.flush() # Attempt at updating any zombie UCIs (i.e., instances that have been in SUBMITTED state for longer than expected - see below for exact time) zombies = model.UCI.filter_by( state=uci_states.SUBMITTED ).all() @@ -509,30 +509,31 @@ return None # Update store status in local DB with info from cloud provider - try: - if store.status != vl[0].status: - # In case something failed during creation of UCI but actual storage volume was created and yet - # UCI state remained as 'new', try to remedy this by updating UCI state here - if ( store.status == None ) and ( store.volume_id != None ): - uci.state = vl[0].status - uci.flush() - - store.status = vl[0].status - store.flush() - if store.i_id != vl[0].instance_id: - store.i_id = vl[0].instance_id - store.flush() - if store.attach_time != vl[0].attach_time: - store.attach_time = vl[0].attach_time - store.flush() - if store.device != vl[0].device: - store.device = vl[0].device - store.flush() - except boto.exception.EC2ResponseError, e: - log.error( "Updating status of volume(s) from cloud for UCI '%s' failed: " % ( uci.name, str(e) ) ) - uci.error( "Updating volume status from cloud failed: " + str(e) ) - uci.state( uci_states.ERROR ) - return None + if len(vl) > 0: + try: + if store.status != vl[0].status: + # In case something failed during creation of UCI but actual storage volume was created and yet + # UCI state remained as 'new', try to remedy this by updating UCI state here + if ( store.status == None ) and ( store.volume_id != None ): + uci.state = vl[0].status + uci.flush() + + store.status = vl[0].status + store.flush() + if store.i_id != vl[0].instance_id: + store.i_id = vl[0].instance_id + store.flush() + if store.attach_time != vl[0].attach_time: + store.attach_time = vl[0].attach_time + store.flush() + if store.device != vl[0].device: + store.device = vl[0].device + store.flush() + except boto.exception.EC2ResponseError, e: + log.error( "Updating status of volume(s) from cloud for UCI '%s' failed: " % ( uci.name, str(e) ) ) + uci.error( "Updating volume status from cloud failed: " + str(e) ) + uci.state( uci_states.ERROR ) + return None def processZombie( self, inst ): """ diff -r 70ae74578254 -r 28252033d66a lib/galaxy/cloud/providers/eucalyptus.py --- a/lib/galaxy/cloud/providers/eucalyptus.py Wed Nov 04 17:19:18 2009 -0500 +++ b/lib/galaxy/cloud/providers/eucalyptus.py Thu Nov 05 11:27:00 2009 -0500 @@ -368,19 +368,19 @@ model.CloudStore.c.status==store_states.CREATING, model.CloudStore.c.status==None ) ).all() for store in stores: - if self.type == store.uci.credentials.provider.type and store.volume_id != None: - log.debug( "[%s] Running general status update on store '%s'" % ( store.uci.credentials.provider.type, store.volume_id ) ) + if self.type == store.uci.credentials.provider.type: # and store.volume_id != None: + log.debug( "[%s] Running general status update on store with local database ID: '%s'" % ( store.uci.credentials.provider.type, store.id ) ) self.updateStore( store ) - else: - log.error( "[%s] There exists an entry for UCI (%s) storage volume without an ID. Storage volume might have been created with " - "cloud provider though. Manual check is recommended." % ( store.uci.credentials.provider.type, store.uci.name ) ) - store.uci.error = "There exists an entry in local database for a storage volume without an ID. Storage volume might have been created " \ - "with cloud provider though. Manual check is recommended. After understanding what happened, local database etry for given " \ - "storage volume should be updated." - store.status = store_states.ERROR - store.uci.state = uci_states.ERROR - store.uci.flush() - store.flush() +# else: +# log.error( "[%s] There exists an entry for UCI (%s) storage volume without an ID. Storage volume might have been created with " +# "cloud provider though. Manual check is recommended." % ( store.uci.credentials.provider.type, store.uci.name ) ) +# store.uci.error = "There exists an entry in local database for a storage volume without an ID. Storage volume might have been created " \ +# "with cloud provider though. Manual check is recommended. After understanding what happened, local database entry for given " \ +# "storage volume should be updated." +# store.status = store_states.ERROR +# store.uci.state = uci_states.ERROR +# store.uci.flush() +# store.flush() # Attempt at updating any zombie UCIs (i.e., instances that have been in SUBMITTED state for longer than expected - see below for exact time) zombies = model.UCI.filter_by( state=uci_states.SUBMITTED ).all() @@ -465,39 +465,41 @@ conn = self.get_connection_from_uci( uci ) try: - log.debug( "vol id: " % store.volume_id ) vl = conn.get_all_volumes( [store.volume_id] ) except boto.exception.EC2ResponseError, e: log.error( "Retrieving volume(s) from cloud for UCI '%s' failed: " % ( uci.name, str(e) ) ) uci.error( "Retrieving volume(s) from cloud failed: " + str(e) ) uci.state( uci_states.ERROR ) + uci.flush() return None # Update store status in local DB with info from cloud provider - try: - if store.status != vl[0].status: - # In case something failed during creation of UCI but actual storage volume was created and yet - # UCI state remained as 'new', try to remedy this by updating UCI state here - if ( store.status == None ) and ( store.volume_id != None ): - uci.state = vl[0].status - uci.flush() - - store.status = vl[0].status - store.flush() - if store.i_id != vl[0].instance_id: - store.i_id = vl[0].instance_id - store.flush() - if store.attach_time != vl[0].attach_time: - store.attach_time = vl[0].attach_time - store.flush() - if store.device != vl[0].device: - store.device = vl[0].device - store.flush() - except boto.exception.EC2ResponseError, e: - log.error( "Updating status of volume(s) from cloud for UCI '%s' failed: " % ( uci.name, str(e) ) ) - uci.error( "Updating volume status from cloud failed: " + str(e) ) - uci.state( uci_states.ERROR ) - return None + if len(vl) > 0: + try: + if store.status != vl[0].status: + # In case something failed during creation of UCI but actual storage volume was created and yet + # UCI state remained as 'new', try to remedy this by updating UCI state here + if ( store.status == None ) and ( store.volume_id != None ): + uci.state = vl[0].status + uci.flush() + + store.status = vl[0].status + store.flush() + if store.i_id != vl[0].instance_id: + store.i_id = vl[0].instance_id + store.flush() + if store.attach_time != vl[0].attach_time: + store.attach_time = vl[0].attach_time + store.flush() + if store.device != vl[0].device: + store.device = vl[0].device + store.flush() + except boto.exception.EC2ResponseError, e: + log.error( "Updating status of volume(s) from cloud for UCI '%s' failed: " % ( uci.name, str(e) ) ) + uci.error( "Updating volume status from cloud failed: " + str(e) ) + uci.state( uci_states.ERROR ) + uci.flush() + return None def processZombie( self, inst ): """ diff -r 70ae74578254 -r 28252033d66a lib/galaxy/model/migrate/versions/0014_cloud_tables.py --- a/lib/galaxy/model/migrate/versions/0014_cloud_tables.py Wed Nov 04 17:19:18 2009 -0500 +++ b/lib/galaxy/model/migrate/versions/0014_cloud_tables.py Thu Nov 05 11:27:00 2009 -0500 @@ -101,65 +101,72 @@ def upgrade(): metadata.reflect() + log.debug( "Creating cloud_image table." ) try: CloudImage_table.create() except Exception, e: log.debug( "Creating cloud_image table failed. Table probably exists already." ) + log.debug( "Creating cloud_uci table." ) try: UCI_table.create() except Exception, e: log.debug( "Creating UCI table failed. Table probably exists already." ) try: - CloudInstance_table.create() - except Exception, e: - log.debug( "Creating cloud_instance table failed. Table probably exists already." ) - try: - CloudStore_table.create() - except Exception: - log.debug( "Creating cloud_store table failed. Table probably exists already." ) - try: CloudUserCredentials_table.create() except Exception, e: log.debug( "Creating cloud_image table failed. Table probably exists already." ) + log.debug( "Creating cloud_provider table." ) try: CloudProvider_table.create() except Exception, e: log.debug( "Creating cloud_provider table failed. Table probably exists already." ) + log.debug( "Creating cloud_instance table." ) + #try: + CloudInstance_table.create() + #except Exception, e: + # log.debug( "Creating cloud_instance table failed. Table probably exists already." ) + #log.debug( "Creating cloud_store table." ) + #try: + CloudStore_table.create() + #except Exception: + # log.debug( "Creating cloud_store table failed. Table probably exists already." ) def downgrade(): metadata.reflect() try: - log.debug( "Would drop cloud_image table." ) -# CloudImage_table.drop() #Enable before release + #log.debug( "Would drop cloud_image table." ) + CloudImage_table.drop() #Enable before release except Exception, e: log.debug( "Dropping cloud_image table failed: %s" % str( e ) ) - try: - log.debug( "Would drop cloud_instance table." ) -# CloudInstance_table.drop() - except Exception, e: - log.debug( "Dropping cloud_instance table failed: %s" % str( e ) ) +# try: +# #log.debug( "Would drop cloud_instance table." ) +# print "inst" + CloudInstance_table.drop() +# except Exception, e: +# log.debug( "Dropping cloud_instance table failed: %s" % str( e ) ) - try: - log.debug( "Would drop cloud_store table." ) -# CloudStore_table.drop() - except Exception, e: - log.debug( "Dropping cloud_store table failed: %s" % str( e ) ) +# try: +# #log.debug( "Would drop cloud_store table." ) +# print "store" + CloudStore_table.drop() +# except Exception, e: +# log.debug( "Dropping cloud_store table failed: %s" % str( e ) ) try: log.debug( "Would drop cloud_user_credentials table." ) -# CloudUserCredentials_table.drop() #Enable before putting final version + CloudUserCredentials_table.drop() #Enable before putting final version except Exception, e: log.debug( "Dropping cloud_user_credentials table failed: %s" % str( e ) ) try: log.debug( "Would drop UCI table." ) -# UCI_table.drop() + UCI_table.drop() except Exception, e: log.debug( "Dropping UCI table failed: %s" % str( e ) ) try: -# log.debug( "Would drop cloud_provider table." ) + log.debug( "Would drop cloud_provider table." ) CloudProvider_table.drop() except Exception, e: log.debug( "Dropping cloud_provider table failed: %s" % str( e ) ) diff -r 70ae74578254 -r 28252033d66a lib/galaxy/web/controllers/cloud.py --- a/lib/galaxy/web/controllers/cloud.py Wed Nov 04 17:19:18 2009 -0500 +++ b/lib/galaxy/web/controllers/cloud.py Thu Nov 05 11:27:00 2009 -0500 @@ -626,12 +626,13 @@ proxy_user='', proxy_pass='', debug='', https_connection_factory='', path='' ): user = trans.get_user() error = {} - try: - is_secure = int(is_secure) - except ValueError: - pass if region_name or region_endpoint or name or is_secure or port or proxy or debug or path: + try: + is_secure = int(is_secure) + except ValueError: + error['is_secure_error'] = "Field 'is secure' can only take on a value '0' or '1'" + if trans.app.model.CloudProvider \ .filter_by (user=user, name=name) \ .first(): @@ -657,9 +658,11 @@ else: provider.region_endpoint = None - if is_secure=='0': + if is_secure==0: + log.debug("is_secure is false") provider.is_secure = False else: + log.debug("is_secure is true") provider.is_secure = True if host: diff -r 70ae74578254 -r 28252033d66a templates/cloud/list_images.mako --- a/templates/cloud/list_images.mako Wed Nov 04 17:19:18 2009 -0500 +++ b/templates/cloud/list_images.mako Thu Nov 05 11:27:00 2009 -0500 @@ -60,6 +60,10 @@ </tr> %endfor </table> +%else: + <h3>There are no registered machine images.</h3><br /> + <a href="${h.url_for( controller='cloud', action='addNewImage' )}" target="galaxy_main">Add machine image now?</a> + %endif

1 0

[hg] galaxy 3079: Added code to process zombie instances.
by Greg Von Kuster 23 Nov '09

23 Nov '09

details: http://www.bx.psu.edu/hg/galaxy/rev/dabbf7210eb7 changeset: 3079:dabbf7210eb7 user: Enis Afgan <afgane(a)gmail.com> date: Mon Nov 02 14:23:37 2009 -0500 description: Added code to process zombie instances. diffstat: lib/galaxy/cloud/providers/ec2.py | 160 +++++++++++++++++++++++++------ lib/galaxy/cloud/providers/eucalyptus.py | 160 +++++++++++++++++++++++-------- lib/galaxy/web/controllers/cloud.py | 14 ++- templates/cloud/configure_cloud.mako | 44 +++----- 4 files changed, 273 insertions(+), 105 deletions(-) diffs (581 lines): diff -r e19eef93584f -r dabbf7210eb7 lib/galaxy/cloud/providers/ec2.py --- a/lib/galaxy/cloud/providers/ec2.py Thu Oct 29 17:40:31 2009 -0400 +++ b/lib/galaxy/cloud/providers/ec2.py Mon Nov 02 14:23:37 2009 -0500 @@ -14,6 +14,7 @@ from boto.ec2.connection import EC2Connection from boto.ec2.regioninfo import RegionInfo import boto.exception +import boto import logging log = logging.getLogger( __name__ ) @@ -38,7 +39,8 @@ TERMINATED = "terminated", RUNNING = "running", PENDING = "pending", - SHUTTING_DOWN = "shutting-down" + SHUTTING_DOWN = "shutting-down", + ERROR = "error" ) store_states = Bunch( @@ -291,7 +293,8 @@ uci_wrapper.set_error( "EC2 response error when starting: " + str(e), True ) # Record newly available instance data into local Galaxy database l_time = datetime.utcnow() - uci_wrapper.set_launch_time( l_time, i_index=i_index ) # format_time( reservation.i_indexes[0].launch_time ) ) +# uci_wrapper.set_launch_time( l_time, i_index=i_index ) # format_time( reservation.i_indexes[0].launch_time ) ) + uci_wrapper.set_launch_time( self.format_time( reservation.instances[0].launch_time ), i_index=i_index ) if not uci_wrapper.uci_launch_time_set(): uci_wrapper.set_uci_launch_time( l_time ) try: @@ -384,6 +387,7 @@ Reason behind this method is to sync state of local DB and real-world resources """ log.debug( "Running general status update for EC2 UCIs..." ) + # Update instances instances = model.CloudInstance.filter( or_( model.CloudInstance.c.state==instance_states.RUNNING, model.CloudInstance.c.state==instance_states.PENDING, model.CloudInstance.c.state==instance_states.SHUTTING_DOWN ) ).all() @@ -392,6 +396,7 @@ log.debug( "[%s] Running general status update on instance '%s'" % ( inst.uci.credentials.provider.type, inst.instance_id ) ) self.updateInstance( inst ) + # Update storage volume(s) stores = model.CloudStore.filter( or_( model.CloudStore.c.status==store_states.IN_USE, model.CloudStore.c.status==store_states.CREATING, model.CloudStore.c.status==None ) ).all() @@ -399,6 +404,21 @@ if self.type == store.uci.credentials.provider.type: log.debug( "[%s] Running general status update on store '%s'" % ( store.uci.credentials.provider.type, store.volume_id ) ) self.updateStore( store ) + + # Attempt at updating any zombie UCIs (i.e., instances that have been in SUBMITTED state for longer than expected - see below for exact time) + zombies = model.UCI.filter_by( state=uci_states.SUBMITTED ).all() + for zombie in zombies: + z_instances = model.CloudInstance.filter_by( uci_id=zombie.id) \ + .filter( or_( model.CloudInstance.c.state != instance_states.TERMINATED, + model.CloudInstance.c.state == None ) ) \ + .all() + for z_inst in z_instances: + if self.type == z_inst.uci.credentials.provider.type: +# log.debug( "z_inst.id: '%s', state: '%s'" % ( z_inst.id, z_inst.state ) ) + td = datetime.utcnow() - z_inst.update_time + if td.seconds > 180: # if instance has been in SUBMITTED state for more than 3 minutes + log.debug( "[%s] Running zombie repair update on instance with DB id '%s'" % ( z_inst.uci.credentials.provider.type, z_inst.id ) ) + self.processZombie( z_inst ) def updateInstance( self, inst ): @@ -406,22 +426,8 @@ uci_id = inst.uci_id uci = model.UCI.get( uci_id ) uci.refresh() - a_key = uci.credentials.access_key - s_key = uci.credentials.secret_key - # Get connection - try: - region = RegionInfo( None, uci.credentials.provider.region_name, uci.credentials.provider.region_endpoint ) - conn = EC2Connection( aws_access_key_id=a_key, - aws_secret_access_key=s_key, - is_secure=uci.credentials.provider.is_secure, - region=region, - path=uci.credentials.provider.path ) - except boto.exception.EC2ResponseError, e: - log.error( "Establishing connection with cloud failed: %s" % str(e) ) - uci.error( "Establishing connection with cloud failed: " + str(e) ) - uci.state( uci_states.ERROR ) - return None - + conn = self.get_connection_from_uci( inst.uci ) + # Get reservations handle for given instance try: rl= conn.get_all_instances( [inst.instance_id] ) @@ -479,21 +485,7 @@ uci_id = store.uci_id uci = model.UCI.get( uci_id ) uci.refresh() - a_key = uci.credentials.access_key - s_key = uci.credentials.secret_key - # Get connection - try: - region = RegionInfo( None, uci.credentials.provider.region_name, uci.credentials.provider.region_endpoint ) - conn = EC2Connection( aws_access_key_id=a_key, - aws_secret_access_key=s_key, - is_secure=uci.credentials.provider.is_secure, - region=region, - path=uci.credentials.provider.path ) - except boto.exception.EC2ResponseError, e: - log.error( "Establishing connection with cloud failed: %s" % str(e) ) - uci.error( "Establishing connection with cloud failed: " + str(e) ) - uci.state( uci_states.ERROR ) - return None + conn = self.get_connection_from_uci( inst.uci ) # Get reservations handle for given store try: @@ -531,6 +523,106 @@ uci.state( uci_states.ERROR ) return None + def processZombie( self, inst ): + """ + Attempt at discovering if starting an instance was successful but local database was not updated + accordingly or if something else failed and instance was never started. Currently, no automatic + repairs are being attempted; instead, appropriate error messages are set. + """ + # Check if any instance-specific information was written to local DB; if 'yes', set instance and UCI's error message + # suggesting manual check. + if inst.launch_time != None or inst.reservation_id != None or inst.instance_id != None or inst.keypair_name != None: + # Try to recover state - this is best-case effort, so if something does not work immediately, not + # recovery steps are attempted. Recovery is based on hope that instance_id is available in local DB; if not, + # report as error. + # Fields attempting to be recovered are: reservation_id, keypair_name, instance status, and launch_time + if inst.instance_id != None: + conn = self.get_connection_from_uci( inst.uci ) + rl = conn.get_all_instances( [inst.instance_id] ) # reservation list + # Update local DB with relevant data from instance + if inst.reservation_id == None: + try: + inst.reservation_id = str(rl[0]).split(":")[1] + except: # something failed, so skip + pass + + if inst.keypair_name == None: + try: + inst.keypair_name = rl[0].instances[0].key_name + except: # something failed, so skip + pass + try: + state = rl[0].instances[0].update() + inst.state = state + inst.uci.state = state + inst.flush() + inst.uci.flush() + except: # something failed, so skip + pass + + if inst.launch_time == None: + try: + launch_time = self.format_time( rl[0].instances[0].launch_time ) + inst.launch_time = launch_time + inst.flush() + if inst.uci.launch_time == None: + inst.uci.launch_time = launch_time + inst.uci.flush() + except: # something failed, so skip + pass + else: + inst.error = "Starting a machine instance associated with UCI '" + str(inst.uci.name) + "' seems to have failed. " \ + "Because it appears that cloud instance might have gotten started, manual check is recommended." + inst.state = instance_states.ERROR + inst.uci.error = "Starting a machine instance (DB id: '"+str(inst.id)+"') associated with this UCI seems to have failed. " \ + "Because it appears that cloud instance might have gotten started, manual check is recommended." + inst.uci.state = uci_states.ERROR + log.error( "Starting a machine instance (DB id: '%s') associated with UCI '%s' seems to have failed. " \ + "Because it appears that cloud instance might have gotten started, manual check is recommended." + % ( inst.id, inst.uci.name ) ) + inst.flush() + inst.uci.flush() + + else: #Instance most likely never got processed, so set error message suggesting user to try starting instance again. + inst.error = "Starting a machine instance associated with UCI '" + str(inst.uci.name) + "' seems to have failed. " \ + "Because it appears that cloud instance never got started, it should be safe to reset state and try " \ + "starting the instance again." + inst.state = instance_states.ERROR + inst.uci.error = "Starting a machine instance (DB id: '"+str(inst.id)+"') associated with this UCI seems to have failed. " \ + "Because it appears that cloud instance never got started, it should be safe to reset state and try " \ + "starting the instance again." + inst.uci.state = uci_states.ERROR + log.error( "Starting a machine instance (DB id: '%s') associated with UCI '%s' seems to have failed. " \ + "Because it appears that cloud instance never got started, it should be safe to reset state and try " \ + "starting the instance again." % ( inst.id, inst.uci.name ) ) + inst.flush() + inst.uci.flush() +# uw = UCIwrapper( inst.uci ) +# log.debug( "Try automatically re-submitting UCI '%s'." % uw.get_name() ) + + def get_connection_from_uci( self, uci ): + """ + Establishes and returns connection to cloud provider. Information needed to do so is obtained + directly from uci database object. + """ + a_key = uci.credentials.access_key + s_key = uci.credentials.secret_key + # Get connection + try: + region = RegionInfo( None, uci.credentials.provider.region_name, uci.credentials.provider.region_endpoint ) + conn = EC2Connection( aws_access_key_id=a_key, + aws_secret_access_key=s_key, + is_secure=uci.credentials.provider.is_secure, + region=region, + path=uci.credentials.provider.path ) + except boto.exception.EC2ResponseError, e: + log.error( "Establishing connection with cloud failed: %s" % str(e) ) + uci.error( "Establishing connection with cloud failed: " + str(e) ) + uci.state( uci_states.ERROR ) + return None + + return conn + # def updateUCI( self, uci ): # """ # Runs a global status update on all storage volumes and all instances that are @@ -576,7 +668,7 @@ # --------- Helper methods ------------ - def format_time( time ): + def format_time( self, time ): dict = {'T':' ', 'Z':''} for i, j in dict.iteritems(): time = time.replace(i, j) diff -r e19eef93584f -r dabbf7210eb7 lib/galaxy/cloud/providers/eucalyptus.py --- a/lib/galaxy/cloud/providers/eucalyptus.py Thu Oct 29 17:40:31 2009 -0400 +++ b/lib/galaxy/cloud/providers/eucalyptus.py Mon Nov 02 14:23:37 2009 -0500 @@ -6,6 +6,7 @@ from galaxy.model import mapping from galaxy.datatypes.data import nice_size from galaxy.util.bunch import Bunch +from galaxy.cloud import UCIwrapper from Queue import Queue from sqlalchemy import or_, and_ @@ -13,6 +14,8 @@ galaxy.eggs.require("boto") from boto.ec2.connection import EC2Connection from boto.ec2.regioninfo import RegionInfo +import boto.exception +import boto import logging log = logging.getLogger( __name__ ) @@ -37,7 +40,8 @@ TERMINATED = "terminated", RUNNING = "running", PENDING = "pending", - SHUTTING_DOWN = "shutting-down" + SHUTTING_DOWN = "shutting-down", + ERROR = "error" ) store_states = Bunch( @@ -256,12 +260,13 @@ uci_wrapper.set_error( "EC2 response error when starting: " + str(e), True ) l_time = datetime.utcnow() - uci_wrapper.set_launch_time( l_time, i_index=i_index ) # format_time( reservation.i_indexes[0].launch_time ) ) +# uci_wrapper.set_launch_time( l_time, i_index=i_index ) + uci_wrapper.set_launch_time( self.format_time( reservation.instances[0].launch_time ), i_index=i_index ) if not uci_wrapper.uci_launch_time_set(): uci_wrapper.set_uci_launch_time( l_time ) try: uci_wrapper.set_reservation_id( i_index, str( reservation ).split(":")[1] ) - # TODO: if more than a single instance will be started through single reservation, change this reference to element [0] + # TODO: if more than a single instance will be started through single reservation, change this reference from element [0] i_id = str( reservation.instances[0]).split(":")[1] uci_wrapper.set_instance_id( i_index, i_id ) s = reservation.instances[0].state @@ -371,15 +376,16 @@ zombies = model.UCI.filter_by( state=uci_states.SUBMITTED ).all() for zombie in zombies: z_instances = model.CloudInstance.filter_by( uci_id=zombie.id) \ - .filter( or_( model.CloudInstance.c.state!=instance_states.TERMINATED, + .filter( or_( model.CloudInstance.c.state != instance_states.TERMINATED, model.CloudInstance.c.state == None ) ) \ .all() for z_inst in z_instances: if self.type == z_inst.uci.credentials.provider.type: - log.debug( "z_inst.id: '%s', state: '%s'" % ( z_inst.id, z_inst.state ) ) -# td = datetime.utcnow() - zombie.update_time -# if td.seconds > 180: # if instance has been in SUBMITTED state for more than 3 minutes -# log.debug( "[%s] Running zombie repair update on instance '%s'" % ( inst.uci.credentials.provider.type, inst.id ) ) +# log.debug( "z_inst.id: '%s', state: '%s'" % ( z_inst.id, z_inst.state ) ) + td = datetime.utcnow() - z_inst.update_time + if td.seconds > 180: # if instance has been in SUBMITTED state for more than 3 minutes + log.debug( "[%s] Running zombie repair update on instance with DB id '%s'" % ( z_inst.uci.credentials.provider.type, z_inst.id ) ) + self.processZombie( z_inst ) def updateInstance( self, inst ): @@ -387,22 +393,7 @@ uci_id = inst.uci_id uci = model.UCI.get( uci_id ) uci.refresh() - a_key = uci.credentials.access_key - s_key = uci.credentials.secret_key - # Get connection - try: - euca_region = RegionInfo( None, uci.credentials.provider.region_name, uci.credentials.provider.region_endpoint ) - conn = EC2Connection( aws_access_key_id=a_key, - aws_secret_access_key=s_key, - is_secure=uci.credentials.provider.is_secure, - port=uci.credentials.provider.port, - region=euca_region, - path=uci.credentials.provider.path ) - except boto.exception.EC2ResponseError, e: - log.error( "Establishing connection with cloud failed: %s" % str(e) ) - uci.error( "Establishing connection with cloud failed: " + str(e) ) - uci.state( uci_states.ERROR ) - return None + conn = self.get_connection_from_uci( uci ) # Get reservations handle for given instance try: @@ -461,22 +452,7 @@ uci_id = store.uci_id uci = model.UCI.get( uci_id ) uci.refresh() - a_key = uci.credentials.access_key - s_key = uci.credentials.secret_key - # Get connection - try: - euca_region = RegionInfo( None, uci.credentials.provider.region_name, uci.credentials.provider.region_endpoint ) - conn = EC2Connection( aws_access_key_id=a_key, - aws_secret_access_key=s_key, - is_secure=uci.credentials.provider.is_secure, - port=uci.credentials.provider.port, - region=euca_region, - path=uci.credentials.provider.path ) - except boto.exception.EC2ResponseError, e: - log.error( "Establishing connection with cloud failed: %s" % str(e) ) - uci.error( "Establishing connection with cloud failed: " + str(e) ) - uci.state( uci_states.ERROR ) - return None + conn = self.get_connection_from_uci( uci ) try: vl = conn.get_all_volumes( [store.volume_id] ) @@ -511,6 +487,108 @@ uci.error( "Updating volume status from cloud failed: " + str(e) ) uci.state( uci_states.ERROR ) return None + + def processZombie( self, inst ): + """ + Attempt at discovering if starting an instance was successful but local database was not updated + accordingly or if something else failed and instance was never started. Currently, no automatic + repairs are being attempted; instead, appropriate error messages are set. + """ + # Check if any instance-specific information was written to local DB; if 'yes', set instance and UCI's error message + # suggesting manual check. + if inst.launch_time != None or inst.reservation_id != None or inst.instance_id != None or inst.keypair_name != None: + # Try to recover state - this is best-case effort, so if something does not work immediately, not + # recovery steps are attempted. Recovery is based on hope that instance_id is available in local DB; if not, + # report as error. + # Fields attempting to be recovered are: reservation_id, keypair_name, instance status, and launch_time + if inst.instance_id != None: + conn = self.get_connection_from_uci( inst.uci ) + rl = conn.get_all_instances( [inst.instance_id] ) # reservation list + # Update local DB with relevant data from instance + if inst.reservation_id == None: + try: + inst.reservation_id = str(rl[0]).split(":")[1] + except: # something failed, so skip + pass + + if inst.keypair_name == None: + try: + inst.keypair_name = rl[0].instances[0].key_name + except: # something failed, so skip + pass + try: + state = rl[0].instances[0].update() + inst.state = state + inst.uci.state = state + inst.flush() + inst.uci.flush() + except: # something failed, so skip + pass + + if inst.launch_time == None: + try: + launch_time = self.format_time( rl[0].instances[0].launch_time ) + inst.launch_time = launch_time + inst.flush() + if inst.uci.launch_time == None: + inst.uci.launch_time = launch_time + inst.uci.flush() + except: # something failed, so skip + pass + else: + inst.error = "Starting a machine instance associated with UCI '" + str(inst.uci.name) + "' seems to have failed. " \ + "Because it appears that cloud instance might have gotten started, manual check is recommended." + inst.state = instance_states.ERROR + inst.uci.error = "Starting a machine instance (DB id: '"+str(inst.id)+"') associated with this UCI seems to have failed. " \ + "Because it appears that cloud instance might have gotten started, manual check is recommended." + inst.uci.state = uci_states.ERROR + log.error( "Starting a machine instance (DB id: '%s') associated with UCI '%s' seems to have failed. " \ + "Because it appears that cloud instance might have gotten started, manual check is recommended." + % ( inst.id, inst.uci.name ) ) + inst.flush() + inst.uci.flush() + + else: #Instance most likely never got processed, so set error message suggesting user to try starting instance again. + inst.error = "Starting a machine instance associated with UCI '" + str(inst.uci.name) + "' seems to have failed. " \ + "Because it appears that cloud instance never got started, it should be safe to reset state and try " \ + "starting the instance again." + inst.state = instance_states.ERROR + inst.uci.error = "Starting a machine instance (DB id: '"+str(inst.id)+"') associated with this UCI seems to have failed. " \ + "Because it appears that cloud instance never got started, it should be safe to reset state and try " \ + "starting the instance again." + inst.uci.state = uci_states.ERROR + log.error( "Starting a machine instance (DB id: '%s') associated with UCI '%s' seems to have failed. " \ + "Because it appears that cloud instance never got started, it should be safe to reset state and try " \ + "starting the instance again." % ( inst.id, inst.uci.name ) ) + inst.flush() + inst.uci.flush() +# uw = UCIwrapper( inst.uci ) +# log.debug( "Try automatically re-submitting UCI '%s'." % uw.get_name() ) + + def get_connection_from_uci( self, uci ): + """ + Establishes and returns connection to cloud provider. Information needed to do so is obtained + directly from uci database object. + """ + a_key = uci.credentials.access_key + s_key = uci.credentials.secret_key + # Get connection + try: + euca_region = RegionInfo( None, uci.credentials.provider.region_name, uci.credentials.provider.region_endpoint ) + conn = EC2Connection( aws_access_key_id=a_key, + aws_secret_access_key=s_key, + is_secure=uci.credentials.provider.is_secure, + port=uci.credentials.provider.port, + region=euca_region, + path=uci.credentials.provider.path ) + except boto.exception.EC2ResponseError, e: + log.error( "Establishing connection with cloud failed: %s" % str(e) ) + uci.error( "Establishing connection with cloud failed: " + str(e) ) + uci.state( uci_states.ERROR ) + return None + + return conn + # def updateUCI( self, uci ): # """ # Runs a global status update on all storage volumes and all instances that are @@ -556,7 +634,7 @@ # --------- Helper methods ------------ - def format_time( time ): + def format_time( self, time ): dict = {'T':' ', 'Z':''} for i, j in dict.iteritems(): time = time.replace(i, j) diff -r e19eef93584f -r dabbf7210eb7 lib/galaxy/web/controllers/cloud.py --- a/lib/galaxy/web/controllers/cloud.py Thu Oct 29 17:40:31 2009 -0400 +++ b/lib/galaxy/web/controllers/cloud.py Mon Nov 02 14:23:37 2009 -0500 @@ -151,7 +151,17 @@ stores = get_stores( trans, uci ) # Ensure instance is not already running (or related state) and store relevant data # into DB to initiate instance startup by cloud manager - if ( len(stores) is not 0 ) and ( uci.state == uci_states.AVAILABLE ): + if ( len(stores) is not 0 ) and \ + ( uci.state != uci_states.SUBMITTED ) and \ + ( uci.state != uci_states.SUBMITTED_UCI ) and \ + ( uci.state != uci_states.PENDING ) and \ + ( uci.state != uci_states.DELETING ) and \ + ( uci.state != uci_states.DELETING_UCI ) and \ + ( uci.state != uci_states.DELETED ) and \ + ( uci.state != uci_states.RUNNING ) and \ + ( uci.state != uci_states.NEW_UCI ) and \ + ( uci.state != uci_states.NEW ) and \ + ( uci.state != uci_states.ERROR ): instance = model.CloudInstance() instance.user = user instance.image = mi @@ -171,7 +181,7 @@ "instance description." ) return self.list( trans ) - trans.show_error_message( "Cannot start instance that is in state '%s'." % uci.state ) + error( "Cannot start instance that is in state '%s'." % uci.state ) return self.list( trans ) @web.expose diff -r e19eef93584f -r dabbf7210eb7 templates/cloud/configure_cloud.mako --- a/templates/cloud/configure_cloud.mako Thu Oct 29 17:40:31 2009 -0400 +++ b/templates/cloud/configure_cloud.mako Mon Nov 02 14:23:37 2009 -0500 @@ -24,34 +24,23 @@ $.getJSON( "${h.url_for( action='json_update' )}", {}, function ( data ) { for (var i in data) { var elem = '#' + data[i].id; - // Because of different list managing 'live' vs. 'available' instances, refresh entire - // page on necessary state change. + // Because of different list managing 'live' vs. 'available' instances, reload url on various state changes old_state = $(elem + "-state").text(); new_state = data[i].state; //console.log( "old_state[%d] = %s", i, old_state ); //console.log( "new_state[%d] = %s", i, new_state ); - if ( old_state=='pending' && new_state=='running' ) { - location.reload(true); + if ( ( old_state=='pending' && new_state=='running' ) || ( old_state=='shutting-down' && new_state=='available' ) || \ + ( old_state=='running' && new_state=='available' ) || ( old_state=='running' && new_state=='error' ) || \ + ( old_state=='pending' && new_state=='error' ) || ( old_state=='pending' && new_state=='available' ) || \ + ( old_state=='submitted' && new_state=='available' ) ) { + var url = "${h.url_for( controller='cloud', action='list')}"; + location.replace( url ); } else if ( ( old_state=='running' && new_state=='error' ) || ( old_state=='pending' && new_state=='error' ) || \ ( old_state=='submitted' && new_state=='error' ) || ( old_state=='submittedUCI' && new_state=='error' ) || \ ( old_state=='shutting-down' && new_state=='error' ) ) { - location.reload(true); - } - else if ( old_state=='shutting-down' && new_state=='available' ) { - location.reload(true); - } - else if ( old_state=='running' && new_state=='available' ) { - location.reload(true); - } - else if ( old_state=='running' && new_state=='error' ) { - location.reload(true); - } - else if ( old_state=='pending' && new_state=='error' ) { - location.reload(true); - } - else if ( old_state=='pending' && new_state=='available' ) { - location.reload(true); + var url = "${h.url_for( controller='cloud', action='list')}"; + location.replace( url ); } else if ( new_state=='shutting-down' || new_state=='shutting-downUCI' ) { $(elem + "-link").text( "" ); @@ -88,7 +77,6 @@ $(elem + "-launch_time").text( "N/A" ); } } - console.log(''); }); setTimeout("update_state()", 15000); } @@ -243,20 +231,20 @@ ${prevInstance.name} (${prevInstance.credentials.name}) <a id="pi-${i}-popup" class="popup-arrow" style="display: none;">▼</a> </td> - <td>${str(prevInstance.total_size)}</td>  + <td>${str(prevInstance.total_size)}</td> <td> <%state = str(prevInstance.state)%> %if state =='error': - <div id="short"> - <a onclick="document.getElementById('full').style.display = 'block'; - document.getElementById('short').style.display = 'none'; return 0" + <div id="${prevInstance.name}-short"> + <a onclick="document.getElementById('${prevInstance.name}-full').style.display = 'block'; + document.getElementById('${prevInstance.name}-short').style.display = 'none'; return 0" href="javascript:void(0)"> error </a> </div> - <div id="full" style="DISPLAY: none"> - <a onclick="document.getElementById('short').style.display = 'block'; - document.getElementById('full').style.display = 'none'; return 0;" + <div id="${prevInstance.name}-full" style="DISPLAY: none"> + <a onclick="document.getElementById('${prevInstance.name}-short').style.display = 'block'; + document.getElementById('${prevInstance.name}-full').style.display = 'none'; return 0;" href="javascript:void(0)"> error:</a><br /> ${str(prevInstance.error)}

1 0

[hg] galaxy 3080: Added machine image adding/deleting/viewing as...
by Greg Von Kuster 23 Nov '09

23 Nov '09

details: http://www.bx.psu.edu/hg/galaxy/rev/b7857b33dbd8 changeset: 3080:b7857b33dbd8 user: Enis Afgan <afgane(a)gmail.com> date: Mon Nov 02 17:11:03 2009 -0500 description: Added machine image adding/deleting/viewing as admin action. diffstat: lib/galaxy/web/controllers/cloud.py | 30 ++++++++++++--- templates/admin/index.mako | 9 ++++ templates/cloud/list_images.mako | 64 ++++++++++++++++++++++++++++++++ 3 files changed, 97 insertions(+), 6 deletions(-) diffs (138 lines): diff -r dabbf7210eb7 -r b7857b33dbd8 lib/galaxy/web/controllers/cloud.py --- a/lib/galaxy/web/controllers/cloud.py Mon Nov 02 14:23:37 2009 -0500 +++ b/lib/galaxy/web/controllers/cloud.py Mon Nov 02 17:11:03 2009 -0500 @@ -350,8 +350,8 @@ "<br />Note: you will be able to add more storage later", value='', error=vol_error ) ) @web.expose - @web.require_login( "add a cloud image" ) - #(a)web.require_admin + #(a)web.require_login( "add a cloud image" ) + @web.require_admin def addNewImage( self, trans, image_id='', manifest='', state=None ): error = None if image_id: @@ -373,14 +373,32 @@ trans.log_event( "New cloud image added: '%s'" % image.image_id ) trans.set_message( "Cloud image '%s' added." % image.image_id ) if state: - image.state= state - return self.list( trans ) - + image.state = state + images = trans.sa_session.query( model.CloudImage ).all() + return trans.fill_template( '/cloud/list_images.mako', images=images ) + return trans.show_form( web.FormBuilder( web.url_for(), "Add new cloud image", submit_text="Add" ) .add_text( "image_id", "Machine Image ID (AMI or EMI)", value='', error=error ) .add_text( "manifest", "Manifest", value='', error=error ) ) - + + @web.expose + @web.require_login( "use Galaxy cloud" ) + def listMachineImages( self, trans ): + images = trans.sa_session.query( model.CloudImage ).all() + return trans.fill_template( '/cloud/list_images.mako', images=images ) + + @web.expose + @web.require_admin + def deleteImage( self, trans, id=None ): + if not isinstance( id, int ): + id = trans.security.decode_id( id ) + + image = trans.sa_session.query( model.CloudImage ).get( id ) + image.delete() + image.flush() + return self.listMachineImages( trans ) + @web.expose @web.require_login( "use Galaxy cloud" ) def edit( self, trans, id, credName=None, accessKey=None, secretKey=None, edited=False ): diff -r dabbf7210eb7 -r b7857b33dbd8 templates/admin/index.mako --- a/templates/admin/index.mako Mon Nov 02 14:23:37 2009 -0500 +++ b/templates/admin/index.mako Mon Nov 02 17:11:03 2009 -0500 @@ -122,6 +122,15 @@ <div class="toolTitle"><a href="${h.url_for( controller='requests_admin', action='list')}" target="galaxy_main">Manage requests</a></div> </div> </div> + <div class="toolSectionTitle"> + <span>Cloud</span> + </div> + <div class="toolSectionBody"> + <div class="toolSectionBg"> + <div class="toolTitle"><a href="${h.url_for( controller='cloud', action='listMachineImages' )}" target="galaxy_main">List machine images</a></div> + <div class="toolTitle"><a href="${h.url_for( controller='cloud', action='addNewImage' )}" target="galaxy_main">Add machine image</a></div> + </div> + </div> </div> </div> </div> diff -r dabbf7210eb7 -r b7857b33dbd8 templates/cloud/list_images.mako --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/templates/cloud/list_images.mako Mon Nov 02 17:11:03 2009 -0500 @@ -0,0 +1,64 @@ +<%inherit file="/base.mako"/> + +<%def name="title()">Cloud home</%def> + +%if message: +<% + try: + messagetype + except: + messagetype = "done" +%> + +<p /> +<div class="${messagetype}message"> + ${message} +</div> +%endif + +%if images: + <h2>List of registered machine images:</h2> + + <table class="mange-table colored" border="0" cellspacing="0" cellpadding="0" width="100%"> + <colgroup width="2%"></colgroup> + <colgroup width="13%"></colgroup> + <colgroup width="70%"></colgroup> + <colgroup width="5%"></colgroup> + + <tr class="header"> + <th>#</th> + <th>Machime image ID</th> + <th>Manifest</th> + <th>Delete</th> + <th></th> + </tr> + %for i, image in enumerate( images ): + <tr> + <td>${i+1}</td> + <td> + %if image.image_id: + ${image.image_id} + %else: + N/A + %endif + </td> + <td> + %if image.manifest: + ${image.manifest} + %else: + N/A + %endif + </td> + <td><div align="center"> + <a confirm="Are you sure you want to delete machine image '${image.image_id}'?" + href="${h.url_for( controller='cloud', action='deleteImage', id=trans.security.encode_id(image.id) )}">x</a> + </div></td> + </tr> + %endfor + </table> +%endif + + + + +

1 0

[hg] galaxy 3081: Added ability for administrators to edit regis...
by Greg Von Kuster 23 Nov '09

23 Nov '09

details: http://www.bx.psu.edu/hg/galaxy/rev/89276f68513a changeset: 3081:89276f68513a user: Enis Afgan <afgane(a)gmail.com> date: Tue Nov 03 13:20:26 2009 -0500 description: Added ability for administrators to edit registered machine image values. diffstat: lib/galaxy/web/controllers/cloud.py | 54 +++++++++++++++++++++++---- templates/cloud/configure_cloud.mako | 10 +---- templates/cloud/edit_image.mako | 64 ++++++++++++++++++++++++++++++++ templates/cloud/list_images.mako | 12 ++++-- 4 files changed, 119 insertions(+), 21 deletions(-) diffs (204 lines): diff -r b7857b33dbd8 -r 89276f68513a lib/galaxy/web/controllers/cloud.py --- a/lib/galaxy/web/controllers/cloud.py Mon Nov 02 17:11:03 2009 -0500 +++ b/lib/galaxy/web/controllers/cloud.py Tue Nov 03 13:20:26 2009 -0500 @@ -350,16 +350,17 @@ "<br />Note: you will be able to add more storage later", value='', error=vol_error ) ) @web.expose - #(a)web.require_login( "add a cloud image" ) @web.require_admin def addNewImage( self, trans, image_id='', manifest='', state=None ): - error = None + id_error = None + manifest_error = None if image_id: - if len( image_id ) > 255: - error = "Image ID name exceeds maximum allowable length." + if image_id=='' or len( image_id ) > 255: + id_error = "Image ID must be between 1 and 255 characters long." elif trans.app.model.CloudUserCredentials.filter( trans.app.model.CloudImage.table.c.image_id==image_id ).first(): - error = "Image with that ID is already registered." + id_error = "Image with ID '" + image_id + "' is already registered. \ + Please choose another ID.ga" else: # Create new image image = model.CloudImage() @@ -379,8 +380,8 @@ return trans.show_form( web.FormBuilder( web.url_for(), "Add new cloud image", submit_text="Add" ) - .add_text( "image_id", "Machine Image ID (AMI or EMI)", value='', error=error ) - .add_text( "manifest", "Manifest", value='', error=error ) ) + .add_text( "image_id", "Machine Image ID (AMI or EMI)", value='', error=id_error ) + .add_text( "manifest", "Manifest", value='', error=manifest_error ) ) @web.expose @web.require_login( "use Galaxy cloud" ) @@ -398,7 +399,44 @@ image.delete() image.flush() return self.listMachineImages( trans ) - + + @web.expose + @web.require_admin + def editImage( self, trans, image_id, manifest, id=None, edited=False ): + error = {} + if not isinstance( id, int ): + id = trans.security.decode_id( id ) + + if not edited: + image = trans.sa_session.query( model.CloudImage ).get( id ) + return trans.fill_template( "cloud/edit_image.mako", + image = image, + error = error + ) + else: + image = trans.sa_session.query( model.CloudImage ).get( id ) + if image_id=='' or len( image_id ) > 255: + error['id_error'] = "Image ID must be between 1 and 255 characters in length." + elif trans.app.model.CloudImage \ + .filter( and_( trans.app.model.CloudImage.table.c.id != image.id, trans.app.model.CloudImage.table.c.image_id==image_id ) ) \ + .first(): + error['id_error'] = "Image with ID '" + image_id + "' already exist. Please choose an alternative name." + if error: + return trans.fill_template( "cloud/edit_image.mako", + image = image, + error = error + ) + else: + image.image_id = image_id + image.manifest = manifest + # Persist + session = trans.sa_session + session.save_or_update( image ) + session.flush() + # Log and display the management page + trans.set_message( "Image '%s' edited." % image.image_id ) + return self.listMachineImages( trans ) + @web.expose @web.require_login( "use Galaxy cloud" ) def edit( self, trans, id, credName=None, accessKey=None, secretKey=None, edited=False ): diff -r b7857b33dbd8 -r 89276f68513a templates/cloud/configure_cloud.mako --- a/templates/cloud/configure_cloud.mako Mon Nov 02 17:11:03 2009 -0500 +++ b/templates/cloud/configure_cloud.mako Tue Nov 03 13:20:26 2009 -0500 @@ -284,12 +284,4 @@ or <a href="http://aws.amazon.com/" target="_blank"> open AWS account with Amazon</a>. - -%endif - -<p /><br /> -<ul class="manage-table-actions"> - <li> - <a class="action-button" href="${h.url_for( action='addNewImage' )}"><span>Add new image</span></a> - </li> -</ul> \ No newline at end of file +%endif \ No newline at end of file diff -r b7857b33dbd8 -r 89276f68513a templates/cloud/edit_image.mako --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/templates/cloud/edit_image.mako Tue Nov 03 13:20:26 2009 -0500 @@ -0,0 +1,64 @@ +<% _=n_ %> +<%inherit file="/base.mako"/> +<%def name="title()">Edit machine image</%def> + +<%def name="javascripts()"> +${parent.javascripts()} +<script type="text/javascript"> +$(function(){ + $("input:text:first").focus(); +}) +</script> +</%def> + +%if header: + ${header} +%endif + +%if image: + +<div class="form"> + <div class="form-title">Edit image</div> + <div class="form-body"> + <form name="edit_image" action="${h.url_for( action='editImage', id=trans.security.encode_id(image.id), edited="true" )}" method="post" > + + <% + cls = "form-row" + if error.has_key('id_error'): + cls += " form-row-error" + %> + <div class="${cls}"> + <label>Machine Image ID (AMI or EMI):</label> + <div class="form-row-input"> + <input type="text" name="image_id" value="${image.image_id}" size="40"> + </div> + %if error.has_key('id_error'): + <div class="form-row-error-message">${error['id_error']}</div> + %endif + <div style="clear: both"></div> + </div> + <% + cls = "form-row" + if error.has_key('manifest_error'): + cls += " form-row-error" + %> + <div class="${cls}"> + <label>Manifest:</label> + <div class="form-row-input"> + <input type="text" name="manifest" value="${image.manifest}" size="40"> + </div> + %if error.has_key('manifest_error'): + <div class="form-row-error-message">${error['manifest_error']}</div> + %endif + <div style="clear: both"></div> + </div> + + + <div class="form-row"><input type="submit" value="Save"></div> + </form> + </div> +</div> + +%else: + Specified machine image could not be found. +%endif diff -r b7857b33dbd8 -r 89276f68513a templates/cloud/list_images.mako --- a/templates/cloud/list_images.mako Mon Nov 02 17:11:03 2009 -0500 +++ b/templates/cloud/list_images.mako Tue Nov 03 13:20:26 2009 -0500 @@ -24,11 +24,12 @@ <colgroup width="13%"></colgroup> <colgroup width="70%"></colgroup> <colgroup width="5%"></colgroup> - + <colgroup width="5%"></colgroup> <tr class="header"> <th>#</th> <th>Machime image ID</th> <th>Manifest</th> + <th>Edit</th> <th>Delete</th> <th></th> </tr> @@ -49,10 +50,13 @@ N/A %endif </td> - <td><div align="center"> - <a confirm="Are you sure you want to delete machine image '${image.image_id}'?" + <td> + <a href="${h.url_for( controller='cloud', action='editImage', image_id=image.image_id, manifest=image.manifest, id=trans.security.encode_id(image.id) )}">e</a> + </td> + <td> + <a confirm="Are you sure you want to delete machine image '${image.image_id}'? Note that this may result in users' UCI's not to work any more!" href="${h.url_for( controller='cloud', action='deleteImage', id=trans.security.encode_id(image.id) )}">x</a> - </div></td> + </td> </tr> %endfor </table>

1 0