details: http://www.bx.psu.edu/hg/galaxy/rev/c78e8e1514e4 changeset: 3088:c78e8e1514e4 user: Enis Afgan <afgane@gmail.com> date: Tue Nov 10 18:36:21 2009 -0500 description: Fixed bugs leading to cloud provder code for EC2 and EPC to be very functional. diffstat: lib/galaxy/cloud/__init__.py | 39 ++++- lib/galaxy/cloud/providers/ec2.py | 210 ++++++++++++++++++++-------------- lib/galaxy/cloud/providers/eucalyptus.py | 162 ++++++++++++++++---------- lib/galaxy/web/controllers/cloud.py | 88 +++++++------- templates/cloud/configure_cloud.mako | 42 +++--- templates/cloud/viewInstance.mako | 21 +- 6 files changed, 325 insertions(+), 237 deletions(-) diffs (798 lines): diff -r 7c4cfc243cc3 -r c78e8e1514e4 lib/galaxy/cloud/__init__.py --- a/lib/galaxy/cloud/__init__.py Mon Nov 09 18:55:14 2009 -0500 +++ b/lib/galaxy/cloud/__init__.py Tue Nov 10 18:36:21 2009 -0500 @@ -34,6 +34,14 @@ ERROR = "error", CREATING = "creating" ) +instance_states = Bunch( + TERMINATED = "terminated", + SUBMITTED = "submitted", + RUNNING = "running", + PENDING = "pending", + SHUTTING_DOWN = "shutting-down", + ERROR = "error" +) class CloudManager( object ): """ @@ -338,14 +346,23 @@ def set_error( self, error, set_state=False ): """ - Sets error field of given UCI in local Galaxy database. If set_state is set to 'true', - method also sets state of give UCI to 'error' + Sets error field of given UCI in local Galaxy database as well as any instances associated with + this UCI whose state is 'None' or 'SUBMITTED'. If set_state is set to 'true', + method also sets state of give UCI and corresponding instances to 'error' """ uci = model.UCI.get( self.uci_id ) uci.refresh() uci.error = error if set_state: uci.state = uci_states.ERROR + instances = model.CloudInstance \ + .filter_by( uci=uci ) \ + .filter( or_( model.CloudInstance.c.state==None, model.CloudInstance.c.state==instance_states.SUBMITTED ) ) \ + .all() + for i in instances: + i.error = error + i.state = instance_states.ERROR + i.flush() uci.flush() # --------- Getter methods ----------------- @@ -358,6 +375,15 @@ # cred = model.CloudUserCredentials.get( cred_id ) return uci.credentials.provider.type + def get_type( self, i_index ): + instance = model.CloudInstance.get( i_index ) + return instance.type + + def get_state( self ): + uci = model.UCI.get( self.uci_id ) + uci.refresh() + return uci.state + def get_instances_indexes( self, state=None ): """ Returns indexes of instances associated with given UCI as they are stored in local Galaxy database and @@ -372,15 +398,6 @@ return il - def get_type( self, i_index ): - instance = model.CloudInstance.get( i_index ) - return instance.type - - def get_state( self ): - uci = model.UCI.get( self.uci_id ) - uci.refresh() - return uci.state - def get_instance_state( self, instance_id ): uci = model.UCI.get( self.uci_id ) uci.refresh() diff -r 7c4cfc243cc3 -r c78e8e1514e4 lib/galaxy/cloud/providers/ec2.py --- a/lib/galaxy/cloud/providers/ec2.py Mon Nov 09 18:55:14 2009 -0500 +++ b/lib/galaxy/cloud/providers/ec2.py Tue Nov 10 18:36:21 2009 -0500 @@ -37,6 +37,7 @@ instance_states = Bunch( TERMINATED = "terminated", + SUBMITTED = "submitted", RUNNING = "running", PENDING = "pending", SHUTTING_DOWN = "shutting-down", @@ -119,49 +120,73 @@ def check_key_pair( self, uci_wrapper, conn ): """ - Generate keypair using user's default credentials + Generate key pair using user's credentials """ - log.debug( "Getting user's key pair: '%s'" % self.key_pair ) + kp = None + kp_name = uci_wrapper.get_name().replace(' ','_') + "_kp" + log.debug( "Checking user's key pair: '%s'" % kp_name ) try: - kp = conn.get_key_pair( self.key_pair ) - uci_kp = uci_wrapper.get_key_pair_name() + kp = conn.get_key_pair( kp_name ) + uci_kp_name = uci_wrapper.get_key_pair_name() uci_material = uci_wrapper.get_key_pair_material() - if kp.name != uci_kp or uci_material == None: - try: # key pair exists on the cloud but not in local database, so re-generate it (i.e., delete and then create) - conn.delete_key_pair( self.key_pair ) - except boto.exception.EC2ResponseError: - pass - kp = self.create_key_pair( conn ) + if kp != None: + if kp.name != uci_kp_name or uci_material == None: + # key pair exists on the cloud but not in local database, so re-generate it (i.e., delete and then create) + try: + conn.delete_key_pair( kp_name ) + kp = self.create_key_pair( conn, kp_name ) + uci_wrapper.set_key_pair( kp.name, kp.material ) + except boto.exception.EC2ResponseError, e: + log.error( "EC2 response error when deleting key pair: '%s'" % e ) + uci_wrapper.set_error( "EC2 response error while deleting key pair: " + str( e ), True ) + else: + try: + kp = self.create_key_pair( conn, kp_name ) + uci_wrapper.set_key_pair( kp.name, kp.material ) + except boto.exception.EC2ResponseError, e: + log.error( "EC2 response error when creating key pair: '%s'" % e ) + uci_wrapper.set_error( "EC2 response error while creating key pair: " + str( e ), True ) + except Exception, ex: + log.error( "Exception when creating key pair: '%s'" % e ) + uci_wrapper.set_error( "Error while creating key pair: " + str( e ), True ) + + except boto.exception.EC2ResponseError, e: # No keypair under this name exists so create it + if e.code == 'InvalidKeyPair.NotFound': + log.info( "No keypair found, creating keypair '%s'" % kp_name ) + kp = self.create_key_pair( conn, kp_name ) uci_wrapper.set_key_pair( kp.name, kp.material ) else: - return kp.name - except boto.exception.EC2ResponseError, e: # No keypair under this name exists so create it - if e.code == 'InvalidKeyPair.NotFound': - log.info( "No keypair found, creating keypair '%s'" % self.key_pair ) - kp = self.create_key_pair( conn ) - uci_wrapper.set_key_pair( kp.name, kp.material ) - else: - log.error( "EC2 response error: '%s'" % e ) - uci_wrapper.set_error( "EC2 response error while creating key pair: " + str( e ), True ) + log.error( "EC2 response error while retrieving key pair: '%s'" % e ) + uci_wrapper.set_error( "Cloud provider response error while retrieving key pair: " + str( e ), True ) if kp != None: return kp.name else: return None - def create_key_pair( self, conn ): + def create_key_pair( self, conn, kp_name ): try: - return conn.create_key_pair( self.key_pair ) + return conn.create_key_pair( kp_name ) except boto.exception.EC2ResponseError, e: return None - def get_mi_id( self, type ): + def get_mi_id( self, uci_wrapper, i_index ): """ Get appropriate machine image (mi) based on instance size. - TODO: Dummy method - need to implement logic - For valid sizes, see http://aws.amazon.com/ec2/instance-types/ """ - return model.CloudImage.filter( model.CloudImage.table.c.id==2 ).first().image_id + i_type = uci_wrapper.get_type( i_index ) + if i_type=='m1.small' or i_type=='c1.medium': + arch = 'i386' + else: + arch = 'x86_64' + + mi = model.CloudImage.filter_by( deleted=False, provider_type=self.type, architecture=arch ).first() + if mi: + return mi.image_id + else: + log.error( "Machine image could not be retrieved for UCI '%s'." % uci_wrapper.get_name() ) + uci_wrapper.set_error( "Machine image could not be retrieved. Contact site administrator to ensure needed machine image is registered.", True ) + return None def shutdown( self ): """Attempts to gracefully shut down the monitor thread""" @@ -267,71 +292,82 @@ conn = self.get_connection( uci_wrapper ) self.check_key_pair( uci_wrapper, conn ) - i_indexes = uci_wrapper.get_instances_indexes( state=None ) # Get indexes of i_indexes associated with this UCI whose state is 'None' - log.debug( "Starting instances with IDs: '%s' associated with UCI '%s' " % ( uci_wrapper.get_name(), i_indexes ) ) - for i_index in i_indexes: - mi_id = self.get_mi_id( uci_wrapper.get_type( i_index ) ) - uci_wrapper.set_mi( i_index, mi_id ) + i_indexes = uci_wrapper.get_instances_indexes( state=instance_states.SUBMITTED ) # Get indexes of i_indexes associated with this UCI that are in 'submitted' state + log.debug( "Starting instances with IDs: '%s' associated with UCI '%s' " % ( i_indexes, uci_wrapper.get_name(), ) ) + if len( i_indexes ) > 0: + for i_index in i_indexes: + # Get machine image for current instance + mi_id = self.get_mi_id( uci_wrapper, i_index ) + uci_wrapper.set_mi( i_index, mi_id ) + if uci_wrapper.get_key_pair_name() == None: + log.error( "Key pair for UCI '%s' is None." % uci_wrapper.get_name() ) + uci_wrapper.set_error( "Key pair not found. Try resetting the state and starting the instance again.", True ) + return - # Check if galaxy security group exists (and create it if it does not) - log.debug( "Setting up '%s' security group." % self.security_group ) - try: - conn.get_all_security_groups( [self.security_group] ) # security groups - except boto.exception.EC2ResponseError, e: - if e.code == 'InvalidGroup.NotFound': - log.info( "No security group found, creating security group '%s'" % self.security_group ) + # Check if galaxy security group exists (and create it if it does not) + log.debug( "Setting up '%s' security group." % self.security_group ) + try: + conn.get_all_security_groups( [self.security_group] ) # security groups + except boto.exception.EC2ResponseError, e: + if e.code == 'InvalidGroup.NotFound': + log.info( "No security group found, creating security group '%s'" % self.security_group ) + try: + gSecurityGroup = conn.create_security_group(self.security_group, 'Security group for Galaxy.') + gSecurityGroup.authorize( 'tcp', 80, 80, '0.0.0.0/0' ) # Open HTTP port + gSecurityGroup.authorize( 'tcp', 22, 22, '0.0.0.0/0' ) # Open SSH port + except boto.exception.EC2ResponseError, ex: + log.error( "EC2 response error while creating security group: '%s'" % e ) + uci_wrapper.set_error( "EC2 response error while creating security group: " + str( e ), True ) + else: + log.error( "EC2 response error while retrieving security group: '%s'" % e ) + uci_wrapper.set_error( "EC2 response error while retrieving security group: " + str( e ), True ) + + + if uci_wrapper.get_state() != uci_states.ERROR: + # Start an instance + log.debug( "Starting instance for UCI '%s'" % uci_wrapper.get_name() ) + #TODO: Once multiple volumes can be attached to a single instance, update 'userdata' composition + userdata = uci_wrapper.get_store_volume_id()+"|"+uci_wrapper.get_access_key()+"|"+uci_wrapper.get_secret_key() + log.debug( "Using following command: conn.run_instances( image_id='%s', key_name='%s', security_groups='%s', user_data=[OMITTED], instance_type='%s', placement='%s' )" + % ( mi_id, uci_wrapper.get_key_pair_name(), self.security_group, uci_wrapper.get_type( i_index ), uci_wrapper.get_uci_availability_zone() ) ) + reservation = None try: - gSecurityGroup = conn.create_security_group(self.security_group, 'Security group for Galaxy.') - gSecurityGroup.authorize( 'tcp', 80, 80, '0.0.0.0/0' ) # Open HTTP port - gSecurityGroup.authorize( 'tcp', 22, 22, '0.0.0.0/0' ) # Open SSH port - except boto.exception.EC2ResponseError, ex: - log.error( "EC2 response error while creating security group: '%s'" % e ) - uci_wrapper.set_error( "EC2 response error while creating security group: " + str( e ), True ) + reservation = conn.run_instances( image_id=mi_id, + key_name=uci_wrapper.get_key_pair_name(), + security_groups=[self.security_group], + user_data=userdata, + instance_type=uci_wrapper.get_type( i_index ), + placement=uci_wrapper.get_uci_availability_zone() ) + except boto.exception.EC2ResponseError, e: + log.error( "EC2 response error when starting UCI '%s': '%s'" % ( uci_wrapper.get_name(), str(e) ) ) + uci_wrapper.set_error( "EC2 response error when starting: " + str(e), True ) + except Exception, ex: + log.error( "Error when starting UCI '%s': '%s'" % ( uci_wrapper.get_name(), str( ex ) ) ) + uci_wrapper.set_error( "Cloud provider error when starting: " + str( ex ), True ) + # Record newly available instance data into local Galaxy database + if reservation: + uci_wrapper.set_launch_time( self.format_time( reservation.instances[0].launch_time ), i_index=i_index ) + if not uci_wrapper.uci_launch_time_set(): + uci_wrapper.set_uci_launch_time( self.format_time( reservation.instances[0].launch_time ) ) + try: + uci_wrapper.set_reservation_id( i_index, str( reservation ).split(":")[1] ) + # TODO: if more than a single instance will be started through single reservation, change this reference to element [0] + i_id = str( reservation.instances[0]).split(":")[1] + uci_wrapper.set_instance_id( i_index, i_id ) + s = reservation.instances[0].state + uci_wrapper.change_state( s, i_id, s ) + uci_wrapper.set_security_group_name( self.security_group, i_id=i_id ) + log.debug( "Instance of UCI '%s' started, current state: '%s'" % ( uci_wrapper.get_name(), uci_wrapper.get_state() ) ) + except boto.exception.EC2ResponseError, e: + log.error( "EC2 response error when retrieving instance information for UCI '%s': '%s'" % ( uci_wrapper.get_name(), str(e) ) ) + uci_wrapper.set_error( "EC2 response error when retrieving instance information: " + str(e), True ) else: - log.error( "EC2 response error while retrieving security group: '%s'" % e ) - uci_wrapper.set_error( "EC2 response error while retrieving security group: " + str( e ), True ) - - - if uci_wrapper.get_state() != uci_states.ERROR: - # Start an instance - log.debug( "Starting instance for UCI '%s'" % uci_wrapper.get_name() ) - #TODO: Once multiple volumes can be attached to a single instance, update 'userdata' composition - userdata = uci_wrapper.get_store_volume_id()+"|"+uci_wrapper.get_access_key()+"|"+uci_wrapper.get_secret_key() - log.debug( 'Using following command: conn.run_instances( image_id=%s, key_name=%s, security_groups=[%s], user_data=[OMITTED], instance_type=%s, placement=%s )' - % ( mi_id, uci_wrapper.get_key_pair_name( i_index ), self.security_group, uci_wrapper.get_type( i_index ), uci_wrapper.get_uci_availability_zone() ) ) - reservation = None - try: - reservation = conn.run_instances( image_id=mi_id, - key_name=uci_wrapper.get_key_pair_name( i_index ), - security_groups=[self.security_group], - user_data=userdata, - instance_type=uci_wrapper.get_type( i_index ), - placement=uci_wrapper.get_uci_availability_zone() ) - except boto.exception.EC2ResponseError, e: - log.error( "EC2 response error when starting UCI '%s': '%s'" % ( uci_wrapper.get_name(), str(e) ) ) - uci_wrapper.set_error( "EC2 response error when starting: " + str(e), True ) - except Exception, ex: - log.error( "Error when starting UCI '%s': '%s'" % ( uci_wrapper.get_name(), str( ex ) ) ) - uci_wrapper.set_error( "Cloud provider error when starting: " + str( ex ), True ) - # Record newly available instance data into local Galaxy database -# l_time = datetime.utcnow() -# uci_wrapper.set_launch_time( l_time, i_index=i_index ) # format_time( reservation.i_indexes[0].launch_time ) ) - if reservation: - uci_wrapper.set_launch_time( self.format_time( reservation.instances[0].launch_time ), i_index=i_index ) - if not uci_wrapper.uci_launch_time_set(): - uci_wrapper.set_uci_launch_time( self.format_time( reservation.instances[0].launch_time ) ) - try: - uci_wrapper.set_reservation_id( i_index, str( reservation ).split(":")[1] ) - # TODO: if more than a single instance will be started through single reservation, change this reference to element [0] - i_id = str( reservation.instances[0]).split(":")[1] - uci_wrapper.set_instance_id( i_index, i_id ) - s = reservation.instances[0].state - uci_wrapper.change_state( s, i_id, s ) - uci_wrapper.set_security_group_name( self.security_group, i_id=i_id ) - log.debug( "Instance of UCI '%s' started, current state: '%s'" % ( uci_wrapper.get_name(), uci_wrapper.get_state() ) ) - except boto.exception.EC2ResponseError, e: - log.error( "EC2 response error when retrieving instance information for UCI '%s': '%s'" % ( uci_wrapper.get_name(), str(e) ) ) - uci_wrapper.set_error( "EC2 response error when retrieving instance information: " + str(e), True ) + log.error( "UCI '%s' is in 'error' state, starting instance was aborted." % uci_wrapper.get_name() ) + else: + log.error( "No instances were found for UCI '%s'" % uci_wrapper.get_name() ) + uci_wrapper.set_error( "EC2 response error when retrieving instance information: " + str(e), True ) + else: + log.error( "UCI '%s' is in 'error' state, starting instance was aborted." % uci_wrapper.get_name() ) def stopUCI( self, uci_wrapper): """ diff -r 7c4cfc243cc3 -r c78e8e1514e4 lib/galaxy/cloud/providers/eucalyptus.py --- a/lib/galaxy/cloud/providers/eucalyptus.py Mon Nov 09 18:55:14 2009 -0500 +++ b/lib/galaxy/cloud/providers/eucalyptus.py Tue Nov 10 18:36:21 2009 -0500 @@ -38,6 +38,7 @@ instance_states = Bunch( TERMINATED = "terminated", + SUBMITTED = "submitted", RUNNING = "running", PENDING = "pending", SHUTTING_DOWN = "shutting-down", @@ -58,7 +59,6 @@ def __init__( self, app ): self.type = "eucalyptus" # cloud provider type (e.g., ec2, eucalyptus, opennebula) self.zone = "epc" - self.key_pair = "galaxy-keypair" self.queue = Queue() self.threads = [] @@ -123,49 +123,72 @@ """ Generate key pair using user's credentials """ - log.debug( "Getting user's key pair: '%s'" % self.key_pair ) + kp = None + kp_name = uci_wrapper.get_name().replace(' ','_') + "_kp" + log.debug( "Checking user's key pair: '%s'" % kp_name ) try: - kp = conn.get_key_pair( self.key_pair ) - uci_kp = uci_wrapper.get_key_pair_name() + kp = conn.get_key_pair( kp_name ) + uci_kp_name = uci_wrapper.get_key_pair_name() uci_material = uci_wrapper.get_key_pair_material() - if kp.name != uci_kp or uci_material == None: - try: # key pair exists on the cloud but not in local database, so re-generate it (i.e., delete and then create) - conn.delete_key_pair( self.key_pair ) - except boto.exception.EC2ResponseError: - pass - kp = self.create_key_pair( conn ) + if kp != None: + if kp.name != uci_kp_name or uci_material == None: + # key pair exists on the cloud but not in local database, so re-generate it (i.e., delete and then create) + try: + conn.delete_key_pair( kp_name ) + kp = self.create_key_pair( conn, kp_name ) + uci_wrapper.set_key_pair( kp.name, kp.material ) + except boto.exception.EC2ResponseError, e: + log.error( "EC2 response error when deleting key pair: '%s'" % e ) + uci_wrapper.set_error( "Cloud provider response error while deleting key pair: " + str( e ), True ) + else: + try: + kp = self.create_key_pair( conn, kp_name ) + uci_wrapper.set_key_pair( kp.name, kp.material ) + except boto.exception.EC2ResponseError, e: + log.error( "EC2 response error when creating key pair: '%s'" % e ) + uci_wrapper.set_error( "Cloud provider response error while creating key pair: " + str( e ), True ) + except Exception, ex: + log.error( "Exception when creating key pair: '%s'" % e ) + uci_wrapper.set_error( "Error while creating key pair: " + str( e ), True ) + + except boto.exception.EC2ResponseError, e: # No keypair under this name exists so create it + if e.code == 'InvalidKeyPair.NotFound': + log.info( "No keypair found, creating keypair '%s'" % kp_name ) + kp = self.create_key_pair( conn, kp_name ) uci_wrapper.set_key_pair( kp.name, kp.material ) else: - return kp.name - except boto.exception.EC2ResponseError, e: # No keypair under this name exists so create it - if e.code == 'InvalidKeyPair.NotFound': - log.info( "No keypair found, creating keypair '%s'" % self.key_pair ) - kp = self.create_key_pair( conn ) - uci_wrapper.set_key_pair( kp.name, kp.material ) - else: - log.error( "EC2 response error: '%s'" % e ) - uci_wrapper.set_error( "Cloud provider response error while creating key pair: " + str( e ), True ) + log.error( "EC2 response error while retrieving key pair: '%s'" % e ) + uci_wrapper.set_error( "Cloud provider response error while retrieving key pair: " + str( e ), True ) if kp != None: return kp.name else: return None - def create_key_pair( self, conn ): + def create_key_pair( self, conn, kp_name ): try: - return conn.create_key_pair( self.key_pair ) + return conn.create_key_pair( kp_name ) except boto.exception.EC2ResponseError, e: return None - def get_mi_id( self, type ): + def get_mi_id( self, uci_wrapper, i_index ): """ Get appropriate machine image (mi) based on instance size. - TODO: Dummy method - need to implement logic - For valid sizes, see http://aws.amazon.com/ec2/instance-types/ """ - log.debug( "image id: '%s'" % model.CloudImage.get( 1 ).image_id ) - return model.CloudImage.get( 1 ).image_id - + i_type = uci_wrapper.get_type( i_index ) + if i_type=='m1.small' or i_type=='c1.medium': + arch = 'i386' + else: + arch = 'x86_64' + + mi = model.CloudImage.filter_by( deleted=False, provider_type=self.type, architecture=arch ).first() + if mi: + return mi.image_id + else: + log.error( "Machine image could not be retrieved for UCI '%s'." % uci_wrapper.get_name() ) + uci_wrapper.set_error( "Machine image could not be retrieved. Contact site administrator to ensure needed machine image is registered.", True ) + return None + def shutdown( self ): """Attempts to gracefully shut down the monitor thread""" log.info( "sending stop signal to worker threads in eucalyptus cloud manager" ) @@ -253,47 +276,56 @@ if uci_wrapper.get_state() != uci_states.ERROR: conn = self.get_connection( uci_wrapper ) self.check_key_pair( uci_wrapper, conn ) + if uci_wrapper.get_key_pair_name() == None: + log.error( "Key pair for UCI '%s' is NULL." % uci_wrapper.get_name() ) + uci_wrapper.set_error( "Key pair not found. Try resetting the state and starting the instance again.", True ) + return - i_indexes = uci_wrapper.get_instances_indexes( state=None ) # Get indexes of i_indexes associated with this UCI - for i_index in i_indexes: - mi_id = self.get_mi_id( uci_wrapper.get_type( i_index ) ) - log.debug( "mi_id: %s, uci_wrapper.get_key_pair_name(): %s" % ( mi_id, uci_wrapper.get_key_pair_name() ) ) - uci_wrapper.set_mi( i_index, mi_id ) - - if uci_wrapper.get_state() != uci_states.ERROR and uci_wrapper.get_key_pair_name() != None: - log.debug( "Starting UCI instance '%s'" % uci_wrapper.get_name() ) - log.debug( 'Using following command: conn.run_instances( image_id=%s, key_name=%s )' % ( mi_id, uci_wrapper.get_key_pair_name() ) ) - reservation = None - try: - reservation = conn.run_instances( image_id=mi_id, key_name=uci_wrapper.get_key_pair_name() ) - #reservation = conn.run_instances( image_id=instance.image, key_name=instance.keypair_name, security_groups=['galaxy'], instance_type=instance.type, placement=instance.availability_zone ) - except boto.exception.EC2ResponseError, e: - log.error( "EC2 response error when starting UCI '%s': '%s'" % ( uci_wrapper.get_name(), str( e ) ) ) - uci_wrapper.set_error( "Cloud provider response error when starting: " + str( e ), True ) - except Exception, ex: - log.error( "Error when starting UCI '%s': '%s'" % ( uci_wrapper.get_name(), str( ex ) ) ) - uci_wrapper.set_error( "Cloud provider error when starting: " + str( ex ), True ) -# l_time = datetime.utcnow() -# uci_wrapper.set_launch_time( l_time, i_index=i_index ) - if reservation: - uci_wrapper.set_launch_time( self.format_time( reservation.instances[0].launch_time ), i_index=i_index ) - if not uci_wrapper.uci_launch_time_set(): - uci_wrapper.set_uci_launch_time( self.format_time( reservation.instances[0].launch_time ) ) + i_indexes = uci_wrapper.get_instances_indexes( state=instance_states.SUBMITTED ) # Get indexes of i_indexes associated with this UCI that are in 'submitted' state + if len( i_indexes ) > 0: + for i_index in i_indexes: + mi_id = self.get_mi_id( uci_wrapper, i_index ) + log.debug( "mi_id: %s, uci_wrapper.get_key_pair_name(): %s" % ( mi_id, uci_wrapper.get_key_pair_name() ) ) + uci_wrapper.set_mi( i_index, mi_id ) + + if uci_wrapper.get_state() != uci_states.ERROR and uci_wrapper.get_key_pair_name() != None: + log.debug( "Starting UCI instance '%s'" % uci_wrapper.get_name() ) + log.debug( "Using following command: conn.run_instances( image_id='%s', key_name='%s', instance_type='%s' )" + % ( mi_id, uci_wrapper.get_key_pair_name(), uci_wrapper.get_type( i_index ) ) ) + reservation = None try: - uci_wrapper.set_reservation_id( i_index, str( reservation ).split(":")[1] ) - # TODO: if more than a single instance will be started through single reservation, change this reference from element [0] - i_id = str( reservation.instances[0]).split(":")[1] - uci_wrapper.set_instance_id( i_index, i_id ) - s = reservation.instances[0].state - uci_wrapper.change_state( s, i_id, s ) - log.debug( "Instance of UCI '%s' started, current state: '%s'" % ( uci_wrapper.get_name(), uci_wrapper.get_state() ) ) + reservation = conn.run_instances( image_id=mi_id, + key_name=uci_wrapper.get_key_pair_name(), + instance_type=uci_wrapper.get_type( i_index ) ) except boto.exception.EC2ResponseError, e: - log.error( "EC2 response error when retrieving instance information for UCI '%s': '%s'" % ( uci_wrapper.get_name(), str(e) ) ) - uci_wrapper.set_error( "Cloud provider response error when retrieving instance information: " + str(e), True ) - - if uci_wrapper.get_key_pair_name() == None: - log.debug( "Key pair for UCI '%s' is NULL." % uci_wrapper.get_name() ) - uci_wrapper.set_error( "Key pair not found. Try resetting the state and starting the instance again.", True ) + log.error( "EC2 response error when starting UCI '%s': '%s'" % ( uci_wrapper.get_name(), str( e ) ) ) + uci_wrapper.set_error( "Cloud provider response error when starting: " + str( e ), True ) + except Exception, ex: + log.error( "Error when starting UCI '%s': '%s'" % ( uci_wrapper.get_name(), str( ex ) ) ) + uci_wrapper.set_error( "Cloud provider error when starting: " + str( ex ), True ) + # Record newly available instance data into local Galaxy database + if reservation: + uci_wrapper.set_launch_time( self.format_time( reservation.instances[0].launch_time ), i_index=i_index ) + if not uci_wrapper.uci_launch_time_set(): + uci_wrapper.set_uci_launch_time( self.format_time( reservation.instances[0].launch_time ) ) + try: + uci_wrapper.set_reservation_id( i_index, str( reservation ).split(":")[1] ) + # TODO: if more than a single instance will be started through single reservation, change this reference from element [0] + i_id = str( reservation.instances[0]).split(":")[1] + uci_wrapper.set_instance_id( i_index, i_id ) + s = reservation.instances[0].state + uci_wrapper.change_state( s, i_id, s ) + log.debug( "Instance of UCI '%s' started, current state: '%s'" % ( uci_wrapper.get_name(), uci_wrapper.get_state() ) ) + except boto.exception.EC2ResponseError, e: + log.error( "EC2 response error when retrieving instance information for UCI '%s': '%s'" % ( uci_wrapper.get_name(), str(e) ) ) + uci_wrapper.set_error( "Cloud provider response error when retrieving instance information: " + str(e), True ) + else: + log.error( "UCI '%s' is in 'error' state, starting instance was aborted." % uci_wrapper.get_name() ) + else: + log.error( "No instances were found for UCI '%s'" % uci_wrapper.get_name() ) + uci_wrapper.set_error( "EC2 response error when retrieving instance information: " + str(e), True ) + else: + log.error( "UCI '%s' is in 'error' state, starting instance was aborted." % uci_wrapper.get_name() ) def stopUCI( self, uci_wrapper): """ diff -r 7c4cfc243cc3 -r c78e8e1514e4 lib/galaxy/web/controllers/cloud.py --- a/lib/galaxy/web/controllers/cloud.py Mon Nov 09 18:55:14 2009 -0500 +++ b/lib/galaxy/web/controllers/cloud.py Tue Nov 10 18:36:21 2009 -0500 @@ -49,9 +49,11 @@ instance_states = Bunch( TERMINATED = "terminated", + SUBMITTED = "submitted", RUNNING = "running", PENDING = "pending", - SHUTTING_DOWN = "shutting-down" + SHUTTING_DOWN = "shutting-down", + ERROR = "error" ) store_states = Bunch( @@ -141,15 +143,14 @@ """ user = trans.get_user() uci = get_uci( trans, id ) -# mi = get_mi( trans, uci, type ) stores = get_stores( trans, uci ) # Ensure instance is available and then store relevant data # into DB to initiate instance startup by cloud manager if ( len(stores) is not 0 ) and ( uci.state == uci_states.AVAILABLE ): instance = model.CloudInstance() instance.user = user -# instance.image = mi instance.uci = uci + instance.state = instance_states.SUBMITTED instance.availability_zone = stores[0].availability_zone # Bc. all EBS volumes need to be in the same avail. zone, just check 1st instance.type = type uci.state = uci_states.SUBMITTED_UCI @@ -253,39 +254,6 @@ inst_error = vol_error = cred_error = None error = {} user = trans.get_user() - storedCreds = trans.sa_session.query( model.CloudUserCredentials ).filter_by( user=user, deleted=False ).all() - if len( storedCreds ) == 0: - return trans.show_error_message( "You must register credentials before configuring a Galaxy cloud instance." ) - # Create dict mapping of cloud-providers-to-zones available by those providers - providersToZones = {} - for storedCred in storedCreds: - zones = None - conn = get_connection( trans, storedCred ) - if conn != None: - avail_zones = [] - try: - zones = conn.get_all_zones() - for zone in zones: - zone = str( zone ).split(':')[1] - avail_zones.append( zone ) - providersToZones[storedCred.name] = avail_zones - except boto.exception.EC2ResponseError, e: - log.error( "Retrieving zones for credentials '%s' failed." % storedCred.name ) - providersToZones[storedCred.name] = ['Unknown provider zone'] - else: - providersToZones[storedCred.name] = ['Unknown provider zone'] - - # Hard-coded solution -# if storedCred.provider.storedCred.provider.region_name == 'us-east-1': -# ec2_zones = ['us-east-1a', 'us-east-1b', 'us-east-1c', 'us-east-1d'] -# providersToZones[storedCred.name] = ec2_zones -# elif storedCred.provider.region_name == 'eu-west-1': -# ec2_zones = ['eu-west-1a', 'eu-west-1b'] -# providersToZones[storedCred.name] = ec2_zones -# elif storedCred.provider.type == 'eucalyptus': -# providersToZones[storedCred.name] = ['epc'] -# else: -# providersToZones[storedCred.name] = ['Unknown provider zone'] if instanceName: # Check if volume size is entered as an integer @@ -337,14 +305,48 @@ except AttributeError, ae: inst_error = "No registered cloud images. You must contact administrator to add some before proceeding." log.debug("AttributeError when registering new UCI '%s': %s " % ( instanceName, str( ae ) ) ) + else: + storedCreds = trans.sa_session.query( model.CloudUserCredentials ).filter_by( user=user, deleted=False ).all() + if len( storedCreds ) == 0: + return trans.show_error_message( "You must register credentials before configuring a Galaxy cloud instance." ) + # Create dict mapping of cloud-providers-to-zones available by those providers + providersToZones = {} + for storedCred in storedCreds: + zones = None + conn = get_connection( trans, storedCred ) + if conn != None: + avail_zones = [] + try: + zones = conn.get_all_zones() + for z in zones: + z = str( z ).split(':')[1] + avail_zones.append( z ) + providersToZones[storedCred.name] = avail_zones + except boto.exception.EC2ResponseError, e: + log.error( "Retrieving zones for credentials '%s' failed: %s" % ( storedCred.name, e ) ) + providersToZones[storedCred.name] = [ "Retrieving zones failed: " + str( e ) ] + else: + providersToZones[storedCred.name] = ['Connection with cloud provider could not be established.'] - return trans.fill_template( "cloud/configure_uci.mako", - instanceName = instanceName, - credName = storedCreds, - volSize = volSize, - zone = zone, - error = error, - providersToZones = providersToZones ) + # Hard-coded solution + # if storedCred.provider.storedCred.provider.region_name == 'us-east-1': + # ec2_zones = ['us-east-1a', 'us-east-1b', 'us-east-1c', 'us-east-1d'] + # providersToZones[storedCred.name] = ec2_zones + # elif storedCred.provider.region_name == 'eu-west-1': + # ec2_zones = ['eu-west-1a', 'eu-west-1b'] + # providersToZones[storedCred.name] = ec2_zones + # elif storedCred.provider.type == 'eucalyptus': + # providersToZones[storedCred.name] = ['epc'] + # else: + # providersToZones[storedCred.name] = ['Unknown provider zone'] + + return trans.fill_template( "cloud/configure_uci.mako", + instanceName = instanceName, + credName = storedCreds, + volSize = volSize, + zone = zone, + error = error, + providersToZones = providersToZones ) @web.expose @web.require_admin diff -r 7c4cfc243cc3 -r c78e8e1514e4 templates/cloud/configure_cloud.mako --- a/templates/cloud/configure_cloud.mako Mon Nov 09 18:55:14 2009 -0500 +++ b/templates/cloud/configure_cloud.mako Tue Nov 10 18:36:21 2009 -0500 @@ -28,14 +28,15 @@ old_state = $(elem + "-state").text(); prev_old_state = $(elem + "-state-p").text(); new_state = data[i].state; - //console.log( "old_state[%d] = %s", i, old_state ); - //console.log( "prev_old_state[%d] = %s", i, prev_old_state ); - //console.log( "new_state[%d] = %s", i, new_state ); + console.log( "old_state[%d] = %s", i, old_state ); + console.log( "prev_old_state[%d] = %s", i, prev_old_state ); + console.log( "new_state[%d] = %s", i, new_state ); if ( ( old_state=='pending' && new_state=='running' ) || ( old_state=='shutting-down' && new_state=='available' ) || \ ( old_state=='running' && new_state=='available' ) || ( old_state=='running' && new_state=='error' ) || \ ( old_state=='pending' && new_state=='error' ) || ( old_state=='pending' && new_state=='available' ) || \ ( old_state=='submitted' && new_state=='available' ) || ( prev_old_state.match('newUCI') && new_state=='available' ) || \ - ( prev_old_state.match('new') && new_state=='available' ) ) { + ( prev_old_state.match('new') && new_state=='available' ) || ( prev_old_state.match('deletingUCI') && new_state=='deleted' ) || \ + ( prev_old_state.match('deleting') && new_state=='deleted' ) ) { var url = "${h.url_for( controller='cloud', action='list')}"; location.replace( url ); } @@ -50,7 +51,8 @@ var url = "${h.url_for( controller='cloud', action='list')}"; location.replace( url ); } - else if ( new_state=='shutting-down' || new_state=='shutting-downUCI' ) { + + if ( new_state=='shutting-down' || new_state=='shutting-downUCI' ) { $(elem + "-link").text( "" ); } @@ -198,7 +200,7 @@ <colgroup width="25%"></colgroup> <colgroup width="10%"></colgroup> <tr class="header"> - <th>Instance name (credentials)</th> + <th>Live instance name (credentials)</th> <th>Storage size (GB)</th> <th>State</th> <th>Alive since</th> @@ -255,24 +257,23 @@ </tr> %endif </table> - + <p /> ## ***************************************************** ## Manage previously configured instances - <table class="mange-table noHR" border="0" cellspacing="0" cellpadding="0" width="100%"> + ## <table class="mange-table noHR" border="0" cellspacing="0" cellpadding="0" width="100%"> + <table class="mange-table colored" border="0" cellspacing="0" cellpadding="0" width="100%"> <colgroup width="40%"></colgroup> <colgroup width="15%"></colgroup> <colgroup width="10%"></colgroup> - <colgroup width="35%"></colgroup> - ##<tr class="header"> - ##<th>Previously configured instances</th> - ##<th>Storage size (GB)</th> - ##<th>State</th> - ##<th>Alive since</th> - ##<th></th> - ##<th></th> - ##<th></th> - ##<th></th> - ##</tr> + <colgroup width="25%"></colgroup> + <colgroup width="10%"></colgroup> + <tr class="header"> + <th>Configured instance name (credentials)</th> + <th>Storage size (GB)</th> + <th>State</th> + <th></th> + <th></th> + </tr> %if prevInstances: %for i, prevInstance in enumerate( prevInstances ): @@ -307,8 +308,7 @@ ${str(prevInstance.state)} %endif </td> - <td>N/A</td> - <td> + <td> <div popupmenu="pi-${i}-popup"> <a class="action-button" href="${h.url_for( action='start', id=trans.security.encode_id(prevInstance.id), type='m1.small' )}"> Start m1.small</a> <a class="action-button" href="${h.url_for( action='start', id=trans.security.encode_id(prevInstance.id), type='c1.medium' )}"> Start c1.medium</a> diff -r 7c4cfc243cc3 -r c78e8e1514e4 templates/cloud/viewInstance.mako --- a/templates/cloud/viewInstance.mako Mon Nov 09 18:55:14 2009 -0500 +++ b/templates/cloud/viewInstance.mako Tue Nov 10 18:36:21 2009 -0500 @@ -115,21 +115,22 @@ %if liveInstance.uci.key_pair_material != None: <tr> <td> Keypair material:</td> - <div id="shortComment2"> - <a onclick="document.getElementById('fullComment2').style.display = 'block'; - document.getElementById('shortComment2').style.display = 'none'; return 0" + <td> + <div id="short"> + <a onclick="document.getElementById('full').style.display = 'block'; + document.getElementById('short').style.display = 'none'; return 0" href="javascript:void(0)"> + Show </a> </div> - <div id="fullComment2" style="DISPLAY: none"> - <nobr><b>${liveInstance.uci.key_pair_material}</b></nobr><br/> - <a onclick="document.getElementById('shortComment2').style.display = 'block'; - document.getElementById('fullComment2').style.display = 'none'; return 0;" + <div id="full" style="DISPLAY: none"> + <a onclick="document.getElementById('short').style.display = 'block'; + document.getElementById('full').style.display = 'none'; return 0;" href="javascript:void(0)"> - - Hide - </a> - </div> + - Hide</a> + ${liveInstance.uci.key_pair_material}<br/> + </div> + </td> </tr> %endif