details: http://www.bx.psu.edu/hg/galaxy/rev/7c4cfc243cc3 changeset: 3087:7c4cfc243cc3 user: Enis Afgan <afgane@gmail.com> date: Mon Nov 09 18:55:14 2009 -0500 description: Changed DB tables to support ghost deletion. Changed keypair saving location to UCI table. Adjusted code accordingly. *Have not tested with cloud providers yet thoughtadd templates/cloud/add_image.mako diffstat: lib/galaxy/cloud/__init__.py | 86 ++++++++--- lib/galaxy/cloud/providers/ec2.py | 146 +++++++++++--------- lib/galaxy/cloud/providers/eucalyptus.py | 130 ++++++++++-------- lib/galaxy/model/__init__.py | 1 - lib/galaxy/model/mapping.py | 46 +++-- lib/galaxy/model/migrate/versions/0014_cloud_tables.py | 44 ++++-- lib/galaxy/web/controllers/cloud.py | 126 +++++++++-------- templates/cloud/add_image.mako | 98 ++++++++++++++ templates/cloud/configure_cloud.mako | 5 +- templates/cloud/edit_image.mako | 34 ++++- templates/cloud/list_images.mako | 22 ++- templates/cloud/view.mako | 6 +- templates/cloud/viewInstance.mako | 14 +- 13 files changed, 501 insertions(+), 257 deletions(-) diffs (1468 lines): diff -r 5963dd169715 -r 7c4cfc243cc3 lib/galaxy/cloud/__init__.py --- a/lib/galaxy/cloud/__init__.py Fri Nov 06 13:45:57 2009 -0500 +++ b/lib/galaxy/cloud/__init__.py Mon Nov 09 18:55:14 2009 -0500 @@ -23,6 +23,7 @@ NEW = "new", DELETING_UCI = "deletingUCI", DELETING = "deleting", + DELETED = "deleted", SUBMITTED_UCI = "submittedUCI", SUBMITTED = "submitted", SHUTTING_DOWN_UCI = "shutting-downUCI", @@ -139,16 +140,15 @@ for uci_wrapper in new_requests: session.clear() - self.provider.put( uci_wrapper ) + self.put( uci_wrapper ) # Done with the session mapping.Session.remove() - def put( self, job_id, tool ): - """Add a job to the queue (by job identifier)""" - if not self.track_jobs_in_database: - self.queue.put( ( job_id, tool.id ) ) - self.sleeper.wake() + def put( self, uci_wrapper ): + """Add a request to the queue.""" + self.provider.put( uci_wrapper ) + self.sleeper.wake() def shutdown( self ): """Attempts to gracefully shut down the worker thread""" @@ -156,7 +156,7 @@ # We're not the real queue, do nothing return else: - log.info( "sending stop signal to worker thread" ) + log.info( "Sending stop signal to worker thread" ) self.running = False self.sleeper.wake() log.info( "cloud manager stopped" ) @@ -175,7 +175,7 @@ """ Sets state for UCI and/or UCI's instance with instance_id as provided by cloud provider and stored in local Galaxy database. - Need to provide either state for the UCI or instance_id and it's state or all arguments. + Need to provide either: (1) state for the UCI, or (2) instance_id and it's state, or (3) all arguments. """ # log.debug( "Changing state - new uci_state: %s, instance_id: %s, i_state: %s" % ( uci_state, instance_id, i_state ) ) if uci_state is not None: @@ -198,22 +198,22 @@ instance.image = mi instance.flush() - def set_key_pair( self, i_index, key_name, key_material=None ): + def set_key_pair( self, key_name, key_material=None ): """ - Single UCI may instantiate many instances, i_index refers to the numeric index - of instance controlled by this UCI as it is stored in local DB (see get_instances_ids()). + Sets key pair value for current UCI. """ - instance = model.CloudInstance.get( i_index ) - instance.keypair_name = key_name + uci = model.UCI.get( self.uci_id ) + uci.refresh() + uci.key_pair_name = key_name if key_material is not None: - instance.keypair_material = key_material - instance.flush() + uci.key_pair_material = key_material + uci.flush() def set_launch_time( self, launch_time, i_index=None, i_id=None ): """ - Stores launch time in local database for instance with specified index (as it is stored in local - Galaxy database) or with specified instance ID (as obtained from the cloud provider AND stored - in local Galaxy Database). Only one of i_index or i_id needs to be provided. + Stores launch time in local database for instance with specified index - i_index (as it is stored in local + Galaxy database) or with specified instance ID - i_id (as obtained from the cloud provider AND stored + in local Galaxy Database). Either 'i_index' or 'i_id' needs to be provided. """ if i_index != None: instance = model.CloudInstance.get( i_index ) @@ -231,6 +231,11 @@ uci.flush() def set_stop_time( self, stop_time, i_index=None, i_id=None ): + """ + Stores stop time in local database for instance with specified index - i_index (as it is stored in local + Galaxy database) or with specified instance ID - i_id (as obtained from the cloud provider AND stored + in local Galaxy Database). Either 'i_index' or 'i_id' needs to be provided. + """ if i_index != None: instance = model.CloudInstance.get( i_index ) instance.stop_time = stop_time @@ -245,6 +250,21 @@ uci.refresh() uci.launch_time = None uci.flush() + + def set_security_group_name( self, security_group_name, i_index=None, i_id=None ): + """ + Stores security group name in local database for instance with specified index - i_index (as it is stored in local + Galaxy database) or with specified instance ID - i_id (as obtained from the cloud provider AND stored + in local Galaxy Database). Either 'i_index' or 'i_id' needs to be provided. + """ + if i_index != None: + instance = model.CloudInstance.get( i_index ) + instance.security_group = security_group_name + instance.flush() + elif i_id != None: + instance = model.CloudInstance.filter_by( uci_id=self.uci_id, instance_id=i_id).first() + instance.security_group = security_group_name + instance.flush() def set_reservation_id( self, i_index, reservation_id ): instance = model.CloudInstance.get( i_index ) @@ -382,19 +402,35 @@ uci.refresh() return uci.name - def get_key_pair_name( self, i_index=None, i_id=None ): + def get_key_pair_name( self ): + """ + Returns keypair name associated with given UCI. + """ + uci = model.UCI.get( self.uci_id ) + uci.refresh() + return uci.key_pair_name + + def get_key_pair_material( self ): + """ + Returns keypair material (i.e., private key) associated with given UCI. + """ + uci = model.UCI.get( self.uci_id ) + uci.refresh() + return uci.key_pair_material + + def get_security_group_name( self, i_index=None, i_id=None ): """ Given EITHER instance index as it is stored in local Galaxy database OR instance ID as it is - obtained from cloud provider and stored in local Galaxy database, return keypair name assocaited + obtained from cloud provider and stored in local Galaxy database, return security group name associated with given instance. """ if i_index != None: instance = model.CloudInstance.get( i_index ) - return instance.keypair_name + return instance.security_group elif i_id != None: instance = model.CloudInstance.filter_by( uci_id=self.uci_id, instance_id=i_id).first() - return instance.keypair_name - + return instance.security_group + def get_access_key( self ): uci = model.UCI.get( self.uci_id ) uci.refresh() @@ -469,8 +505,8 @@ def delete( self ): uci = model.UCI.get( self.uci_id ) uci.refresh() -# uci.delete() - uci.state = 'deleted' # for bookkeeping reasons, mark as deleted but don't actually delete. + uci.state = uci_states.DELETED # for bookkeeping reasons, mark as deleted but don't actually delete. + uci.deleted = True uci.flush() class CloudProvider( object ): diff -r 5963dd169715 -r 7c4cfc243cc3 lib/galaxy/cloud/providers/ec2.py --- a/lib/galaxy/cloud/providers/ec2.py Fri Nov 06 13:45:57 2009 -0500 +++ b/lib/galaxy/cloud/providers/ec2.py Mon Nov 09 18:55:14 2009 -0500 @@ -58,11 +58,12 @@ self.type = "ec2" # cloud provider type (e.g., ec2, eucalyptus, opennebula) self.zone = "us-east-1a" self.key_pair = "galaxy-keypair" + self.security_group = "galaxyWeb" self.queue = Queue() self.threads = [] nworkers = 5 - log.info( "Starting EC2 cloud controller workers" ) + log.info( "Starting EC2 cloud controller workers..." ) for i in range( nworkers ): worker = threading.Thread( target=self.run_next ) worker.start() @@ -116,28 +117,43 @@ return conn - def set_keypair( self, uci_wrapper, conn ): + def check_key_pair( self, uci_wrapper, conn ): """ Generate keypair using user's default credentials """ - log.debug( "Getting user's keypair" ) - instances = uci_wrapper.get_instances_indexes() + log.debug( "Getting user's key pair: '%s'" % self.key_pair ) try: kp = conn.get_key_pair( self.key_pair ) - for inst in instances: - uci_wrapper.set_key_pair( inst, kp.name ) - return kp.name + uci_kp = uci_wrapper.get_key_pair_name() + uci_material = uci_wrapper.get_key_pair_material() + if kp.name != uci_kp or uci_material == None: + try: # key pair exists on the cloud but not in local database, so re-generate it (i.e., delete and then create) + conn.delete_key_pair( self.key_pair ) + except boto.exception.EC2ResponseError: + pass + kp = self.create_key_pair( conn ) + uci_wrapper.set_key_pair( kp.name, kp.material ) + else: + return kp.name except boto.exception.EC2ResponseError, e: # No keypair under this name exists so create it if e.code == 'InvalidKeyPair.NotFound': log.info( "No keypair found, creating keypair '%s'" % self.key_pair ) - kp = conn.create_key_pair( self.key_pair ) - for inst in instances: - uci_wrapper.set_key_pair( inst, kp.name, kp.material ) + kp = self.create_key_pair( conn ) + uci_wrapper.set_key_pair( kp.name, kp.material ) else: log.error( "EC2 response error: '%s'" % e ) - uci_wrapper.set_error( "EC2 response error while creating key pair: " + str(e), True ) - - return kp.name + uci_wrapper.set_error( "EC2 response error while creating key pair: " + str( e ), True ) + + if kp != None: + return kp.name + else: + return None + + def create_key_pair( self, conn ): + try: + return conn.create_key_pair( self.key_pair ) + except boto.exception.EC2ResponseError, e: + return None def get_mi_id( self, type ): """ @@ -170,7 +186,6 @@ log.info( "Availability zone for UCI (i.e., storage volume) was not selected, using default zone: %s" % self.zone ) uci_wrapper.set_store_availability_zone( self.zone ) - #TODO: check if volume associated with UCI already exists (if server crashed for example) and don't recreate it log.info( "Creating volume in zone '%s'..." % uci_wrapper.get_uci_availability_zone() ) # Because only 1 storage volume may be created at UCI config time, index of this storage volume in local Galaxy DB w.r.t # current UCI is 0, so reference it in following methods @@ -196,7 +211,7 @@ else: uci_wrapper.change_state( uci_state=uci_states.ERROR ) uci_wrapper.set_store_status( vol.id, uci_states.ERROR ) - uci_wrapper.set_error( "Volume '%s' not found by cloud provider after being created" % vol.id ) + uci_wrapper.set_error( "Volume '%s' not found by EC2 after being created" % vol.id ) def deleteUCI( self, uci_wrapper ): """ @@ -215,14 +230,13 @@ log.debug( "Deleting volume with id='%s'" % v.volume_id ) if conn.delete_volume( v.volume_id ): deletedList.append( v.volume_id ) - v.delete() + v.deleted = True v.flush() count += 1 else: failedList.append( v.volume_id ) # Delete UCI if all of associated - log.debug( "count=%s, len(vl)=%s" % (count, len( vl ) ) ) if count == len( vl ): uci_wrapper.delete() else: @@ -249,31 +263,34 @@ """ Starts instance(s) of given UCI on the cloud. """ - conn = self.get_connection( uci_wrapper ) - if uci_wrapper.get_state() != uci_states.ERROR: - self.set_keypair( uci_wrapper, conn ) - i_indexes = uci_wrapper.get_instances_indexes() # Get indexes of i_indexes associated with this UCI whose state is 'None' - log.debug( "Starting instances with IDs: '%s' associated with UCI '%s' " % ( uci_wrapper.get_name(), i_indexes ) ) - - if uci_wrapper.get_state() != uci_states.ERROR: + conn = self.get_connection( uci_wrapper ) + self.check_key_pair( uci_wrapper, conn ) + + i_indexes = uci_wrapper.get_instances_indexes( state=None ) # Get indexes of i_indexes associated with this UCI whose state is 'None' + log.debug( "Starting instances with IDs: '%s' associated with UCI '%s' " % ( uci_wrapper.get_name(), i_indexes ) ) for i_index in i_indexes: mi_id = self.get_mi_id( uci_wrapper.get_type( i_index ) ) uci_wrapper.set_mi( i_index, mi_id ) # Check if galaxy security group exists (and create it if it does not) - security_group = 'galaxyWeb' - log.debug( "Setting up '%s' security group." % security_group ) - sgs = conn.get_all_security_groups() # security groups - gsgt = False # galaxy security group test - for sg in sgs: - if sg.name == security_group: - gsgt = True - # If security group does not exist, create it - if not gsgt: - gSecurityGroup = conn.create_security_group(security_group, 'Security group for Galaxy.') - gSecurityGroup.authorize( 'tcp', 80, 80, '0.0.0.0/0' ) # Open HTTP port - gSecurityGroup.authorize( 'tcp', 22, 22, '0.0.0.0/0' ) # Open SSH port + log.debug( "Setting up '%s' security group." % self.security_group ) + try: + conn.get_all_security_groups( [self.security_group] ) # security groups + except boto.exception.EC2ResponseError, e: + if e.code == 'InvalidGroup.NotFound': + log.info( "No security group found, creating security group '%s'" % self.security_group ) + try: + gSecurityGroup = conn.create_security_group(self.security_group, 'Security group for Galaxy.') + gSecurityGroup.authorize( 'tcp', 80, 80, '0.0.0.0/0' ) # Open HTTP port + gSecurityGroup.authorize( 'tcp', 22, 22, '0.0.0.0/0' ) # Open SSH port + except boto.exception.EC2ResponseError, ex: + log.error( "EC2 response error while creating security group: '%s'" % e ) + uci_wrapper.set_error( "EC2 response error while creating security group: " + str( e ), True ) + else: + log.error( "EC2 response error while retrieving security group: '%s'" % e ) + uci_wrapper.set_error( "EC2 response error while retrieving security group: " + str( e ), True ) + if uci_wrapper.get_state() != uci_states.ERROR: # Start an instance @@ -281,34 +298,40 @@ #TODO: Once multiple volumes can be attached to a single instance, update 'userdata' composition userdata = uci_wrapper.get_store_volume_id()+"|"+uci_wrapper.get_access_key()+"|"+uci_wrapper.get_secret_key() log.debug( 'Using following command: conn.run_instances( image_id=%s, key_name=%s, security_groups=[%s], user_data=[OMITTED], instance_type=%s, placement=%s )' - % ( mi_id, uci_wrapper.get_key_pair_name( i_index ), [security_group], uci_wrapper.get_type( i_index ), uci_wrapper.get_uci_availability_zone() ) ) + % ( mi_id, uci_wrapper.get_key_pair_name( i_index ), self.security_group, uci_wrapper.get_type( i_index ), uci_wrapper.get_uci_availability_zone() ) ) + reservation = None try: reservation = conn.run_instances( image_id=mi_id, - key_name=uci_wrapper.get_key_pair_name( i_index ), - security_groups=[security_group], - user_data=userdata, - instance_type=uci_wrapper.get_type( i_index ), - placement=uci_wrapper.get_uci_availability_zone() ) + key_name=uci_wrapper.get_key_pair_name( i_index ), + security_groups=[self.security_group], + user_data=userdata, + instance_type=uci_wrapper.get_type( i_index ), + placement=uci_wrapper.get_uci_availability_zone() ) except boto.exception.EC2ResponseError, e: log.error( "EC2 response error when starting UCI '%s': '%s'" % ( uci_wrapper.get_name(), str(e) ) ) uci_wrapper.set_error( "EC2 response error when starting: " + str(e), True ) + except Exception, ex: + log.error( "Error when starting UCI '%s': '%s'" % ( uci_wrapper.get_name(), str( ex ) ) ) + uci_wrapper.set_error( "Cloud provider error when starting: " + str( ex ), True ) # Record newly available instance data into local Galaxy database - l_time = datetime.utcnow() +# l_time = datetime.utcnow() # uci_wrapper.set_launch_time( l_time, i_index=i_index ) # format_time( reservation.i_indexes[0].launch_time ) ) - uci_wrapper.set_launch_time( self.format_time( reservation.instances[0].launch_time ), i_index=i_index ) - if not uci_wrapper.uci_launch_time_set(): - uci_wrapper.set_uci_launch_time( l_time ) - try: - uci_wrapper.set_reservation_id( i_index, str( reservation ).split(":")[1] ) - # TODO: if more than a single instance will be started through single reservation, change this reference to element [0] - i_id = str( reservation.instances[0]).split(":")[1] - uci_wrapper.set_instance_id( i_index, i_id ) - s = reservation.instances[0].state - uci_wrapper.change_state( s, i_id, s ) - log.debug( "Instance of UCI '%s' started, current state: '%s'" % ( uci_wrapper.get_name(), uci_wrapper.get_state() ) ) - except boto.exception.EC2ResponseError, e: - log.error( "EC2 response error when retrieving instance information for UCI '%s': '%s'" % ( uci_wrapper.get_name(), str(e) ) ) - uci_wrapper.set_error( "EC2 response error when retrieving instance information: " + str(e), True ) + if reservation: + uci_wrapper.set_launch_time( self.format_time( reservation.instances[0].launch_time ), i_index=i_index ) + if not uci_wrapper.uci_launch_time_set(): + uci_wrapper.set_uci_launch_time( self.format_time( reservation.instances[0].launch_time ) ) + try: + uci_wrapper.set_reservation_id( i_index, str( reservation ).split(":")[1] ) + # TODO: if more than a single instance will be started through single reservation, change this reference to element [0] + i_id = str( reservation.instances[0]).split(":")[1] + uci_wrapper.set_instance_id( i_index, i_id ) + s = reservation.instances[0].state + uci_wrapper.change_state( s, i_id, s ) + uci_wrapper.set_security_group_name( self.security_group, i_id=i_id ) + log.debug( "Instance of UCI '%s' started, current state: '%s'" % ( uci_wrapper.get_name(), uci_wrapper.get_state() ) ) + except boto.exception.EC2ResponseError, e: + log.error( "EC2 response error when retrieving instance information for UCI '%s': '%s'" % ( uci_wrapper.get_name(), str(e) ) ) + uci_wrapper.set_error( "EC2 response error when retrieving instance information: " + str(e), True ) def stopUCI( self, uci_wrapper): """ @@ -544,11 +567,11 @@ """ # Check if any instance-specific information was written to local DB; if 'yes', set instance and UCI's error message # suggesting manual check. - if inst.launch_time != None or inst.reservation_id != None or inst.instance_id != None or inst.keypair_name != None: + if inst.launch_time != None or inst.reservation_id != None or inst.instance_id != None: # Try to recover state - this is best-case effort, so if something does not work immediately, not # recovery steps are attempted. Recovery is based on hope that instance_id is available in local DB; if not, # report as error. - # Fields attempting to be recovered are: reservation_id, keypair_name, instance status, and launch_time + # Fields attempting to be recovered are: reservation_id, instance status, and launch_time if inst.instance_id != None: conn = self.get_connection_from_uci( inst.uci ) rl = conn.get_all_instances( [inst.instance_id] ) # reservation list @@ -559,11 +582,6 @@ except: # something failed, so skip pass - if inst.keypair_name == None: - try: - inst.keypair_name = rl[0].instances[0].key_name - except: # something failed, so skip - pass try: state = rl[0].instances[0].update() inst.state = state diff -r 5963dd169715 -r 7c4cfc243cc3 lib/galaxy/cloud/providers/eucalyptus.py --- a/lib/galaxy/cloud/providers/eucalyptus.py Fri Nov 06 13:45:57 2009 -0500 +++ b/lib/galaxy/cloud/providers/eucalyptus.py Mon Nov 09 18:55:14 2009 -0500 @@ -63,7 +63,7 @@ self.threads = [] nworkers = 5 - log.info( "Starting eucalyptus cloud controller workers" ) + log.info( "Starting eucalyptus cloud controller workers..." ) for i in range( nworkers ): worker = threading.Thread( target=self.run_next ) worker.start() @@ -74,7 +74,6 @@ """Run the next job, waiting until one is available if necessary""" cnt = 0 while 1: - #log.debug( '[%d] run_next->queue.qsize(): %s' % ( cnt, self.queue.qsize() ) ) uci_wrapper = self.queue.get() uci_state = uci_wrapper.get_state() if uci_state is self.STOP_SIGNAL: @@ -90,7 +89,7 @@ elif uci_state==uci_states.SHUTTING_DOWN: self.stopUCI( uci_wrapper ) except: - log.exception( "Uncaught exception executing request." ) + log.exception( "Uncaught exception executing cloud request." ) cnt += 1 def get_connection( self, uci_wrapper ): @@ -120,28 +119,43 @@ return conn - def set_keypair( self, uci_wrapper, conn ): + def check_key_pair( self, uci_wrapper, conn ): """ - Generate keypair using user's default credentials + Generate key pair using user's credentials """ - log.debug( "Getting user's keypair: '%s'" % self.key_pair ) - instances = uci_wrapper.get_instances_indexes() + log.debug( "Getting user's key pair: '%s'" % self.key_pair ) try: kp = conn.get_key_pair( self.key_pair ) - for inst in instances: - uci_wrapper.set_key_pair( inst, kp.name ) - return kp.name + uci_kp = uci_wrapper.get_key_pair_name() + uci_material = uci_wrapper.get_key_pair_material() + if kp.name != uci_kp or uci_material == None: + try: # key pair exists on the cloud but not in local database, so re-generate it (i.e., delete and then create) + conn.delete_key_pair( self.key_pair ) + except boto.exception.EC2ResponseError: + pass + kp = self.create_key_pair( conn ) + uci_wrapper.set_key_pair( kp.name, kp.material ) + else: + return kp.name except boto.exception.EC2ResponseError, e: # No keypair under this name exists so create it if e.code == 'InvalidKeyPair.NotFound': log.info( "No keypair found, creating keypair '%s'" % self.key_pair ) - kp = conn.create_key_pair( self.key_pair ) - for inst in instances: - uci_wrapper.set_key_pair( inst, kp.name, kp.material ) + kp = self.create_key_pair( conn ) + uci_wrapper.set_key_pair( kp.name, kp.material ) else: log.error( "EC2 response error: '%s'" % e ) - uci_wrapper.set_error( "EC2 response error while creating key pair: " + str(e), True ) + uci_wrapper.set_error( "Cloud provider response error while creating key pair: " + str( e ), True ) - return kp.name + if kp != None: + return kp.name + else: + return None + + def create_key_pair( self, conn ): + try: + return conn.create_key_pair( self.key_pair ) + except boto.exception.EC2ResponseError, e: + return None def get_mi_id( self, type ): """ @@ -203,7 +217,7 @@ log.debug( "Deleting volume with id='%s'" % v.volume_id ) if conn.delete_volume( v.volume_id ): deletedList.append( v.volume_id ) - v.delete() + v.deleted = True v.flush() count += 1 else: @@ -236,46 +250,50 @@ """ Starts instance(s) of given UCI on the cloud. """ - conn = self.get_connection( uci_wrapper ) -# if uci_wrapper.get_state() != uci_states.ERROR: - self.set_keypair( uci_wrapper, conn ) - - i_indexes = uci_wrapper.get_instances_indexes() # Get indexes of i_indexes associated with this UCI - - if uci_wrapper.get_state() != uci_states.ERROR: + conn = self.get_connection( uci_wrapper ) + self.check_key_pair( uci_wrapper, conn ) + + i_indexes = uci_wrapper.get_instances_indexes( state=None ) # Get indexes of i_indexes associated with this UCI for i_index in i_indexes: mi_id = self.get_mi_id( uci_wrapper.get_type( i_index ) ) - log.debug( "mi_id: %s, uci_wrapper.get_key_pair_name( i_index ): %s" % ( mi_id, uci_wrapper.get_key_pair_name( i_index ) ) ) + log.debug( "mi_id: %s, uci_wrapper.get_key_pair_name(): %s" % ( mi_id, uci_wrapper.get_key_pair_name() ) ) uci_wrapper.set_mi( i_index, mi_id ) - if uci_wrapper.get_state() != uci_states.ERROR: + if uci_wrapper.get_state() != uci_states.ERROR and uci_wrapper.get_key_pair_name() != None: log.debug( "Starting UCI instance '%s'" % uci_wrapper.get_name() ) - log.debug( 'Using following command: conn.run_instances( image_id=%s, key_name=%s )' % ( mi_id, uci_wrapper.get_key_pair_name( i_index ) ) ) + log.debug( 'Using following command: conn.run_instances( image_id=%s, key_name=%s )' % ( mi_id, uci_wrapper.get_key_pair_name() ) ) + reservation = None try: - reservation = conn.run_instances( image_id=mi_id, key_name=uci_wrapper.get_key_pair_name( i_index ) ) + reservation = conn.run_instances( image_id=mi_id, key_name=uci_wrapper.get_key_pair_name() ) #reservation = conn.run_instances( image_id=instance.image, key_name=instance.keypair_name, security_groups=['galaxy'], instance_type=instance.type, placement=instance.availability_zone ) except boto.exception.EC2ResponseError, e: - log.error( "EC2 response error when starting UCI '%s': '%s'" % ( uci_wrapper.get_name(), str(e) ) ) - uci_wrapper.set_error( "EC2 response error when starting: " + str(e), True ) - - l_time = datetime.utcnow() + log.error( "EC2 response error when starting UCI '%s': '%s'" % ( uci_wrapper.get_name(), str( e ) ) ) + uci_wrapper.set_error( "Cloud provider response error when starting: " + str( e ), True ) + except Exception, ex: + log.error( "Error when starting UCI '%s': '%s'" % ( uci_wrapper.get_name(), str( ex ) ) ) + uci_wrapper.set_error( "Cloud provider error when starting: " + str( ex ), True ) +# l_time = datetime.utcnow() # uci_wrapper.set_launch_time( l_time, i_index=i_index ) - uci_wrapper.set_launch_time( self.format_time( reservation.instances[0].launch_time ), i_index=i_index ) - if not uci_wrapper.uci_launch_time_set(): - uci_wrapper.set_uci_launch_time( l_time ) - try: - uci_wrapper.set_reservation_id( i_index, str( reservation ).split(":")[1] ) - # TODO: if more than a single instance will be started through single reservation, change this reference from element [0] - i_id = str( reservation.instances[0]).split(":")[1] - uci_wrapper.set_instance_id( i_index, i_id ) - s = reservation.instances[0].state - uci_wrapper.change_state( s, i_id, s ) - log.debug( "Instance of UCI '%s' started, current state: '%s'" % ( uci_wrapper.get_name(), uci_wrapper.get_state() ) ) - except boto.exception.EC2ResponseError, e: - log.error( "EC2 response error when retrieving instance information for UCI '%s': '%s'" % ( uci_wrapper.get_name(), str(e) ) ) - uci_wrapper.set_error( "EC2 response error when retrieving instance information: " + str(e), True ) - + if reservation: + uci_wrapper.set_launch_time( self.format_time( reservation.instances[0].launch_time ), i_index=i_index ) + if not uci_wrapper.uci_launch_time_set(): + uci_wrapper.set_uci_launch_time( self.format_time( reservation.instances[0].launch_time ) ) + try: + uci_wrapper.set_reservation_id( i_index, str( reservation ).split(":")[1] ) + # TODO: if more than a single instance will be started through single reservation, change this reference from element [0] + i_id = str( reservation.instances[0]).split(":")[1] + uci_wrapper.set_instance_id( i_index, i_id ) + s = reservation.instances[0].state + uci_wrapper.change_state( s, i_id, s ) + log.debug( "Instance of UCI '%s' started, current state: '%s'" % ( uci_wrapper.get_name(), uci_wrapper.get_state() ) ) + except boto.exception.EC2ResponseError, e: + log.error( "EC2 response error when retrieving instance information for UCI '%s': '%s'" % ( uci_wrapper.get_name(), str(e) ) ) + uci_wrapper.set_error( "Cloud provider response error when retrieving instance information: " + str(e), True ) + + if uci_wrapper.get_key_pair_name() == None: + log.debug( "Key pair for UCI '%s' is NULL." % uci_wrapper.get_name() ) + uci_wrapper.set_error( "Key pair not found. Try resetting the state and starting the instance again.", True ) def stopUCI( self, uci_wrapper): """ @@ -294,7 +312,7 @@ notStopped = [] for r in rl: for inst in r.instances: - log.debug( "Sending stop signal to instance '%s' associated with reservation '%s'." % ( inst, r ) ) + log.debug( "Sending stop signal to instance '%s' associated with reservation '%s' (UCI: %s)." % ( inst, r, uci_wrapper.get_name() ) ) inst.stop() uci_wrapper.set_stop_time( datetime.utcnow(), i_id=inst.id ) uci_wrapper.change_state( instance_id=inst.id, i_state=inst.update() ) @@ -385,16 +403,17 @@ # Attempt at updating any zombie UCIs (i.e., instances that have been in SUBMITTED state for longer than expected - see below for exact time) zombies = model.UCI.filter_by( state=uci_states.SUBMITTED ).all() for zombie in zombies: - z_instances = model.CloudInstance.filter_by( uci_id=zombie.id) \ - .filter( or_( model.CloudInstance.c.state != instance_states.TERMINATED, - model.CloudInstance.c.state == None ) ) \ + log.debug( "zombie UCI: %s" % zombie.name ) + z_instances = model.CloudInstance \ + .filter_by( uci_id=zombie.id, state=None ) \ .all() for z_inst in z_instances: if self.type == z_inst.uci.credentials.provider.type: # log.debug( "z_inst.id: '%s', state: '%s'" % ( z_inst.id, z_inst.state ) ) td = datetime.utcnow() - z_inst.update_time + log.debug( "z_inst.id: %s, time delta is %s sec" % ( z_inst.id, td.seconds ) ) if td.seconds > 180: # if instance has been in SUBMITTED state for more than 3 minutes - log.debug( "[%s] Running zombie repair update on instance with DB id '%s'" % ( z_inst.uci.credentials.provider.type, z_inst.id ) ) + log.debug( "[%s](td=%s) Running zombie repair update on instance with DB id '%s'" % ( z_inst.uci.credentials.provider.type, td.seconds, z_inst.id ) ) self.processZombie( z_inst ) def updateInstance( self, inst ): @@ -509,11 +528,11 @@ """ # Check if any instance-specific information was written to local DB; if 'yes', set instance and UCI's error message # suggesting manual check. - if inst.launch_time != None or inst.reservation_id != None or inst.instance_id != None or inst.keypair_name != None: + if inst.launch_time != None or inst.reservation_id != None or inst.instance_id != None: # Try to recover state - this is best-case effort, so if something does not work immediately, not # recovery steps are attempted. Recovery is based on hope that instance_id is available in local DB; if not, # report as error. - # Fields attempting to be recovered are: reservation_id, keypair_name, instance status, and launch_time + # Fields attempting to be recovered are: reservation_id, instance status, and launch_time if inst.instance_id != None: conn = self.get_connection_from_uci( inst.uci ) rl = conn.get_all_instances( [inst.instance_id] ) # reservation list @@ -524,11 +543,6 @@ except: # something failed, so skip pass - if inst.keypair_name == None: - try: - inst.keypair_name = rl[0].instances[0].key_name - except: # something failed, so skip - pass try: state = rl[0].instances[0].update() inst.state = state diff -r 5963dd169715 -r 7c4cfc243cc3 lib/galaxy/model/__init__.py --- a/lib/galaxy/model/__init__.py Fri Nov 06 13:45:57 2009 -0500 +++ b/lib/galaxy/model/__init__.py Mon Nov 09 18:55:14 2009 -0500 @@ -951,7 +951,6 @@ self.instance_id = None self.mi = None self.state = None - self.keypair_name = None self.public_dns = None self.availability_zone = None diff -r 5963dd169715 -r 7c4cfc243cc3 lib/galaxy/model/mapping.py --- a/lib/galaxy/model/mapping.py Fri Nov 06 13:45:57 2009 -0500 +++ b/lib/galaxy/model/mapping.py Mon Nov 09 18:55:14 2009 -0500 @@ -390,9 +390,12 @@ Column( "id", Integer, primary_key=True ), Column( "create_time", DateTime, default=now ), Column( "update_time", DateTime, default=now, onupdate=now ), + Column( "provider_type", TEXT ), Column( "image_id", TEXT, nullable=False ), Column( "manifest", TEXT ), - Column( "state", TEXT ) ) + Column( "state", TEXT ), + Column( "architecture", TEXT ), + Column( "deleted", Boolean, default=False ) ) """ UserConfiguredInstance (UCI) table """ UCI.table = Table( "cloud_uci", metadata, @@ -401,11 +404,14 @@ Column( "update_time", DateTime, default=now, onupdate=now ), Column( "user_id", Integer, ForeignKey( "galaxy_user.id" ), index=True, nullable=False ), Column( "credentials_id", Integer, ForeignKey( "cloud_user_credentials.id" ), index=True ), + Column( "key_pair_name", TEXT ), + Column( "key_pair_material", TEXT ), Column( "name", TEXT ), Column( "state", TEXT ), Column( "error", TEXT ), Column( "total_size", Integer ), - Column( "launch_time", DateTime ) ) + Column( "launch_time", DateTime ), + Column( "deleted", Boolean, default=False ) ) CloudInstance.table = Table( "cloud_instance", metadata, Column( "id", Integer, primary_key=True ), @@ -418,13 +424,12 @@ Column( "type", TEXT ), Column( "reservation_id", TEXT ), Column( "instance_id", TEXT ), - Column( "mi_id", TEXT, ForeignKey( "cloud_image.image_id" ), index=True, nullable=False ), + Column( "mi_id", TEXT, ForeignKey( "cloud_image.image_id" ), index=True ), Column( "state", TEXT ), Column( "error", TEXT ), Column( "public_dns", TEXT ), Column( "private_dns", TEXT ), - Column( "keypair_name", TEXT ), - Column( "keypair_material", TEXT ), + Column( "security_group", TEXT ), Column( "availability_zone", TEXT ) ) CloudStore.table = Table( "cloud_store", metadata, @@ -437,10 +442,22 @@ Column( "volume_id", TEXT ), Column( "size", Integer, nullable=False ), Column( "availability_zone", TEXT ), - Column( "i_id", TEXT, ForeignKey( "cloud_instance.instance_id" ), index=True ), + Column( "i_id", TEXT, ForeignKey( "cloud_instance.instance_id" ) ), Column( "status", TEXT ), Column( "device", TEXT ), - Column( "space_consumed", Integer ) ) + Column( "space_consumed", Integer ), + Column( "deleted", Boolean, default=False ) ) + +CloudUserCredentials.table = Table( "cloud_user_credentials", metadata, + Column( "id", Integer, primary_key=True ), + Column( "create_time", DateTime, default=now ), + Column( "update_time", DateTime, default=now, onupdate=now ), + Column( "user_id", Integer, ForeignKey( "galaxy_user.id" ), index=True, nullable=False ), + Column( "provider_id", Integer, ForeignKey( "cloud_provider.id" ), index=True, nullable=False ), + Column( "name", TEXT ), + Column( "access_key", TEXT ), + Column( "secret_key", TEXT ), + Column( "deleted", Boolean, default=False ) ) CloudProvider.table = Table( "cloud_provider", metadata, Column( "id", Integer, primary_key=True ), @@ -461,19 +478,8 @@ Column( "proxy_pass", TEXT ), Column( "debug", Integer ), Column( "https_connection_factory", TEXT ), - Column( "path", TEXT ) ) - -CloudUserCredentials.table = Table( "cloud_user_credentials", metadata, - Column( "id", Integer, primary_key=True ), - Column( "create_time", DateTime, default=now ), - Column( "update_time", DateTime, default=now, onupdate=now ), - Column( "user_id", Integer, ForeignKey( "galaxy_user.id" ), index=True, nullable=False ), - Column( "name", TEXT ), - Column( "access_key", TEXT ), - Column( "secret_key", TEXT ), - Column( "defaultCred", Boolean, default=False ), - Column( "provider_id", Integer, ForeignKey( "cloud_provider.id" ), index=True, nullable=False ) ) - + Column( "path", TEXT ), + Column( "deleted", Boolean, default=False ) ) # *************************************************************************** StoredWorkflow.table = Table( "stored_workflow", metadata, diff -r 5963dd169715 -r 7c4cfc243cc3 lib/galaxy/model/migrate/versions/0014_cloud_tables.py --- a/lib/galaxy/model/migrate/versions/0014_cloud_tables.py Fri Nov 06 13:45:57 2009 -0500 +++ b/lib/galaxy/model/migrate/versions/0014_cloud_tables.py Mon Nov 09 18:55:14 2009 -0500 @@ -12,25 +12,38 @@ metadata = MetaData( migrate_engine ) +def display_migration_details(): + print + print "========================================" + print "This script adds tables needed for Galaxy cloud functionality." + print "========================================" + CloudImage_table = Table( "cloud_image", metadata, Column( "id", Integer, primary_key=True ), Column( "create_time", DateTime, default=now ), Column( "update_time", DateTime, default=now, onupdate=now ), + Column( "provider_type", TEXT ), Column( "image_id", TEXT, nullable=False ), Column( "manifest", TEXT ), - Column( "state", TEXT ) ) + Column( "state", TEXT ), + Column( "architecture", TEXT ), + Column( "deleted", Boolean, default=False ) ) +""" UserConfiguredInstance (UCI) table """ UCI_table = Table( "cloud_uci", metadata, Column( "id", Integer, primary_key=True ), Column( "create_time", DateTime, default=now ), Column( "update_time", DateTime, default=now, onupdate=now ), Column( "user_id", Integer, ForeignKey( "galaxy_user.id" ), index=True, nullable=False ), Column( "credentials_id", Integer, ForeignKey( "cloud_user_credentials.id" ), index=True ), + Column( "key_pair_name", TEXT ), + Column( "key_pair_material", TEXT ), Column( "name", TEXT ), Column( "state", TEXT ), Column( "error", TEXT ), Column( "total_size", Integer ), - Column( "launch_time", DateTime ) ) + Column( "launch_time", DateTime ), + Column( "deleted", Boolean, default=False ) ) CloudInstance_table = Table( "cloud_instance", metadata, Column( "id", Integer, primary_key=True ), @@ -39,17 +52,16 @@ Column( "launch_time", DateTime ), Column( "stop_time", DateTime ), Column( "user_id", Integer, ForeignKey( "galaxy_user.id" ), index=True, nullable=False ), - Column( "uci_id", Integer, ForeignKey( "uci.id" ), index=True ), + Column( "uci_id", Integer, ForeignKey( "cloud_uci.id" ), index=True ), Column( "type", TEXT ), Column( "reservation_id", TEXT ), Column( "instance_id", TEXT ), - Column( "mi_id", TEXT, ForeignKey( "cloud_image.image_id" ), index=True, nullable=False ), + Column( "mi_id", TEXT, ForeignKey( "cloud_image.image_id" ), index=True ), Column( "state", TEXT ), Column( "error", TEXT ), Column( "public_dns", TEXT ), Column( "private_dns", TEXT ), - Column( "keypair_name", TEXT ), - Column( "keypair_material", TEXT ), + Column( "security_group", TEXT ), Column( "availability_zone", TEXT ) ) CloudStore_table = Table( "cloud_store", metadata, @@ -58,25 +70,26 @@ Column( "update_time", DateTime, default=now, onupdate=now ), Column( "attach_time", DateTime ), Column( "user_id", Integer, ForeignKey( "galaxy_user.id" ), index=True, nullable=False ), - Column( "uci_id", Integer, ForeignKey( "uci.id" ), index=True, nullable=False ), + Column( "uci_id", Integer, ForeignKey( "cloud_uci.id" ), index=True, nullable=False ), Column( "volume_id", TEXT ), Column( "size", Integer, nullable=False ), Column( "availability_zone", TEXT ), - Column( "i_id", TEXT, ForeignKey( "cloud_instance.instance_id" ), index=True ), + Column( "i_id", TEXT, ForeignKey( "cloud_instance.instance_id" ) ), Column( "status", TEXT ), Column( "device", TEXT ), - Column( "space_consumed", Integer ) ) + Column( "space_consumed", Integer ), + Column( "deleted", Boolean, default=False ) ) CloudUserCredentials_table = Table( "cloud_user_credentials", metadata, Column( "id", Integer, primary_key=True ), Column( "create_time", DateTime, default=now ), Column( "update_time", DateTime, default=now, onupdate=now ), Column( "user_id", Integer, ForeignKey( "galaxy_user.id" ), index=True, nullable=False ), + Column( "provider_id", Integer, ForeignKey( "cloud_provider.id" ), index=True, nullable=False ), Column( "name", TEXT ), Column( "access_key", TEXT ), Column( "secret_key", TEXT ), - Column( "defaultCred", Boolean, default=False ), - Column( "provider_id", Integer, ForeignKey( "cloud_provider.id" ), index=True, nullable=False ) ) + Column( "deleted", Boolean, default=False ) ) CloudProvider_table = Table( "cloud_provider", metadata, Column( "id", Integer, primary_key=True ), @@ -97,17 +110,20 @@ Column( "proxy_pass", TEXT ), Column( "debug", Integer ), Column( "https_connection_factory", TEXT ), - Column( "path", TEXT ) ) + Column( "path", TEXT ), + Column( "deleted", Boolean, default=False ) ) def upgrade(): + display_migration_details() + # Load existing tables metadata.reflect() CloudImage_table.create() UCI_table.create() CloudUserCredentials_table.create() + CloudStore_table.create() + CloudInstance_table.create() CloudProvider_table.create() - CloudInstance_table.create() - CloudStore_table.create() def downgrade(): metadata.reflect() diff -r 5963dd169715 -r 7c4cfc243cc3 lib/galaxy/web/controllers/cloud.py --- a/lib/galaxy/web/controllers/cloud.py Fri Nov 06 13:45:57 2009 -0500 +++ b/lib/galaxy/web/controllers/cloud.py Mon Nov 09 18:55:14 2009 -0500 @@ -76,11 +76,13 @@ cloudCredentials = trans.sa_session.query( model.CloudUserCredentials ) \ .filter_by( user=user ) \ + .filter( model.CloudUserCredentials.c.deleted != True ) \ .order_by( model.CloudUserCredentials.c.name ) \ .all() cloudProviders = trans.sa_session.query( model.CloudProvider ) \ .filter_by( user=user ) \ + .filter( model.CloudProvider.c.deleted != True ) \ .order_by( model.CloudProvider.c.name ) \ .all() @@ -131,25 +133,6 @@ prevInstances = prevInstances, cloudProviders = cloudProviders ) - @web.require_login( "use Galaxy cloud" ) - def makeDefault( self, trans, id=None ): - """ - Set current credentials as default. - *NOT USED* - """ - currentDefault = get_default_credentials (trans) - if currentDefault: - currentDefault.defaultCred = False - - newDefault = get_stored_credentials( trans, id ) - newDefault.defaultCred = True - trans.sa_session.flush() - trans.set_message( "Credentials '%s' set as default." % newDefault.name ) - - # TODO: Fix bug that when this function returns, top Galaxy tab bar is missing from the webpage - return self.list( trans ) #trans.fill_template( "cloud/configure_cloud.mako", - #awsCredentials = awsCredentials ) - @web.expose @web.require_login( "start Galaxy cloud instance" ) def start( self, trans, id, type='m1.small' ): @@ -158,14 +141,14 @@ """ user = trans.get_user() uci = get_uci( trans, id ) - mi = get_mi( trans, uci, type ) +# mi = get_mi( trans, uci, type ) stores = get_stores( trans, uci ) # Ensure instance is available and then store relevant data # into DB to initiate instance startup by cloud manager if ( len(stores) is not 0 ) and ( uci.state == uci_states.AVAILABLE ): instance = model.CloudInstance() instance.user = user - instance.image = mi +# instance.image = mi instance.uci = uci instance.availability_zone = stores[0].availability_zone # Bc. all EBS volumes need to be in the same avail. zone, just check 1st instance.type = type @@ -177,9 +160,9 @@ session.flush() # Log trans.log_event ("User initiated starting of UCI '%s'." % uci.name ) - trans.set_message( "Galaxy instance started. NOTE: Please wait about 3-5 minutes for the instance to " - "start up and then refresh this page. A button to connect to the instance will then appear alongside " - "instance description." ) + trans.set_message( "Galaxy instance started. NOTE: Please wait about 5 minutes for the instance to " + "start up. A button to connect to the instance will appear alongside " + "instance description once cloud instance of Galaxy is ready." ) return self.list( trans ) if len(stores) == 0: @@ -270,10 +253,10 @@ inst_error = vol_error = cred_error = None error = {} user = trans.get_user() - storedCreds = trans.sa_session.query( model.CloudUserCredentials ).filter_by( user=user ).all() + storedCreds = trans.sa_session.query( model.CloudUserCredentials ).filter_by( user=user, deleted=False ).all() if len( storedCreds ) == 0: return trans.show_error_message( "You must register credentials before configuring a Galaxy cloud instance." ) - # Create dict mapping of cloud providers to zones available by those providers + # Create dict mapping of cloud-providers-to-zones available by those providers providersToZones = {} for storedCred in storedCreds: zones = None @@ -314,8 +297,7 @@ # Create new user configured instance try: if trans.app.model.UCI \ - .filter_by (user=user) \ - .filter( and_( trans.app.model.UCI.table.c.name==instanceName, trans.app.model.UCI.table.c.state!=uci_states.DELETED ) ) \ + .filter_by (user=user, deleted=False, name=instanceName ) \ .first(): error['inst_error'] = "An instance with that name already exist." elif instanceName=='' or len( instanceName ) > 255: @@ -327,7 +309,7 @@ elif ( int( volSize ) < 1 ) or ( int( volSize ) > 1000 ): error['vol_error'] = "Volume size must be integer value between 1 and 1000." elif zone=='': - error['zone_error'] = "You must select zone where this UCI will be registered." + error['zone_error'] = "You must select a zone where this UCI will be registered." else: # Capture user configured instance information uci = model.UCI() @@ -349,14 +331,12 @@ session.save_or_update( storage ) session.flush() # Log and display the management page - trans.log_event( "User configured new cloud instance" ) + trans.log_event( "User configured new cloud instance: '%s'" % instanceName ) trans.set_message( "New Galaxy instance '%s' configured. Once instance status shows 'available' you will be able to start the instance." % instanceName ) return self.list( trans ) - except ValueError: - vol_error = "Volume size must be specified as an integer value only, between 1 and 1000." except AttributeError, ae: inst_error = "No registered cloud images. You must contact administrator to add some before proceeding." - log.debug("AttributeError: %s " % str( ae ) ) + log.debug("AttributeError when registering new UCI '%s': %s " % ( instanceName, str( ae ) ) ) return trans.fill_template( "cloud/configure_uci.mako", instanceName = instanceName, @@ -368,21 +348,29 @@ @web.expose @web.require_admin - def addNewImage( self, trans, image_id='', manifest='', state=None ): - id_error = None - manifest_error = None - if image_id: - if image_id=='' or len( image_id ) > 255: - id_error = "Image ID must be between 1 and 255 characters long." - elif trans.app.model.CloudUserCredentials.filter( - trans.app.model.CloudImage.table.c.image_id==image_id ).first(): - id_error = "Image with ID '" + image_id + "' is already registered. \ - Please choose another ID.ga" + def addNewImage( self, trans, provider_type='', image_id='', manifest='', architecture='', state=None ): + #id_error = arch_error = provider_error = manifest_error = None + error = {} + if provider_type or image_id or manifest or architecture: + if provider_type=='': + error['provider_error'] = "You must select cloud provider type for this machine image." + elif image_id=='' or len( image_id ) > 255: + error['id_error'] = "Image ID must be between 1 and 255 characters long." + elif trans.app.model.CloudUserCredentials \ + .filter_by( deleted=False ) \ + .filter( trans.app.model.CloudImage.table.c.image_id == image_id ) \ + .first(): + error['id_error'] = "Image with ID '" + image_id + "' is already registered. \ + Please choose another ID." + elif architecture=='': + error['arch_error'] = "You must select architecture type for this machine image." else: # Create new image image = model.CloudImage() + image.provider_type = provider_type image.image_id = image_id image.manifest = manifest + image.architecture = architecture # Persist session = trans.sa_session session.save_or_update( image ) @@ -394,16 +382,24 @@ image.state = state images = trans.sa_session.query( model.CloudImage ).all() return trans.fill_template( '/cloud/list_images.mako', images=images ) - - return trans.show_form( - web.FormBuilder( web.url_for(), "Add new cloud image", submit_text="Add" ) - .add_text( "image_id", "Machine Image ID (AMI or EMI)", value='', error=id_error ) - .add_text( "manifest", "Manifest", value='', error=manifest_error ) ) + + return trans.fill_template( "cloud/add_image.mako", + provider_type = provider_type, + image_id = image_id, + manifest = manifest, + architecture = architecture, + error = error ) +# return trans.show_form( +# web.FormBuilder( web.url_for(), "Add new cloud image", submit_text="Add" ) +# .add_text( "provider_type", "Provider type", value='ec2 or eucalyptus', error=provider_error ) +# .add_text( "image_id", "Machine Image ID (AMI or EMI)", value='', error=id_error ) +# .add_text( "manifest", "Manifest", value='', error=manifest_error ) +# .add_text( "architecture", "Architecture", value='i386 or x86_64', error=arch_error ) ) @web.expose @web.require_login( "use Galaxy cloud" ) def listMachineImages( self, trans ): - images = trans.sa_session.query( model.CloudImage ).all() + images = trans.sa_session.query( model.CloudImage ).filter( trans.app.model.CloudImage.table.c.deleted != True ).all() return trans.fill_template( '/cloud/list_images.mako', images=images ) @web.expose @@ -413,13 +409,13 @@ id = trans.security.decode_id( id ) image = trans.sa_session.query( model.CloudImage ).get( id ) - image.delete() + image.deleted = True image.flush() return self.listMachineImages( trans ) @web.expose @web.require_admin - def editImage( self, trans, image_id, manifest, id=None, edited=False ): + def editImage( self, trans, provider_type='', image_id='', manifest='', architecture='', id='', edited=False ): error = {} if not isinstance( id, int ): id = trans.security.decode_id( id ) @@ -435,9 +431,12 @@ if image_id=='' or len( image_id ) > 255: error['id_error'] = "Image ID must be between 1 and 255 characters in length." elif trans.app.model.CloudImage \ + .filter_by( deleted=False ) \ .filter( and_( trans.app.model.CloudImage.table.c.id != image.id, trans.app.model.CloudImage.table.c.image_id==image_id ) ) \ .first(): error['id_error'] = "Image with ID '" + image_id + "' already exist. Please choose an alternative name." + elif architecture=='' or len( architecture ) > 255: + error['arch_error'] = "Architecture type must be between 1 and 255 characters long." if error: return trans.fill_template( "cloud/edit_image.mako", image = image, @@ -446,12 +445,13 @@ else: image.image_id = image_id image.manifest = manifest + image.architecture = architecture # Persist session = trans.sa_session session.save_or_update( image ) session.flush() # Log and display the management page - trans.set_message( "Image '%s' edited." % image.image_id ) + trans.set_message( "Machine image '%s' edited." % image.image_id ) return self.listMachineImages( trans ) @web.expose @@ -548,8 +548,10 @@ if credName or providerName or accessKey or secretKey: if credName=='' or len( credName ) > 255: error['cred_error'] = "Credentials name must be between 1 and 255 characters in length." - elif trans.app.model.CloudUserCredentials.filter_by( user=user ).filter( - trans.app.model.CloudUserCredentials.table.c.name==credName ).first(): + elif trans.app.model.CloudUserCredentials \ + .filter_by( user=user, deleted=False ) \ + .filter( trans.app.model.CloudUserCredentials.table.c.name == credName ) \ + .first(): error['cred_error'] = "Credentials with that name already exist." elif providerName=='': error['provider_error'] = "You must select cloud provider associated with these credentials." @@ -627,13 +629,12 @@ stored = get_stored_credentials( trans, id ) UCIs = trans.sa_session.query( model.UCI ) \ .filter_by( user=user, credentials_id=stored.id ) \ - .filter( model.UCI.c.state!=uci_states.DELETED ) \ + .filter( model.UCI.c.deleted != True ) \ .all() if len(UCIs) == 0: # Delete and save - sess = trans.sa_session - sess.delete( stored ) + stored.deleted = True stored.flush() # Display the management page trans.set_message( "Credentials '%s' deleted." % stored.name ) @@ -658,6 +659,7 @@ if trans.app.model.CloudProvider \ .filter_by (user=user, name=name) \ + .filter( model.CloudProvider.c.deleted != True ) \ .first(): error['name_error'] = "A provider with that name already exist." elif name=='' or len( name ) > 255: @@ -889,13 +891,13 @@ provider = get_provider_by_id( trans, id ) creds = trans.sa_session.query( model.CloudUserCredentials ) \ .filter_by( user=user, provider_id=provider.id ) \ - .filter( model.UCI.c.state!=uci_states.DELETED ) \ + .filter( model.CloudUserCredentials.c.deleted != True ) \ .all() - + if len( creds ) == 0: # Delete and save - sess = trans.sa_session - sess.delete( provider ) + #sess = trans.sa_session + provider.deleted = True provider.flush() # Display the management page trans.set_message( "Cloud provider '%s' deleted." % provider.name ) @@ -908,7 +910,7 @@ @web.json def json_update( self, trans ): user = trans.get_user() - UCIs = trans.sa_session.query( model.UCI ).filter_by( user=user ).filter( model.UCI.c.state != uci_states.DELETED ).all() + UCIs = trans.sa_session.query( model.UCI ).filter_by( user=user ).filter( model.UCI.c.deleted != True ).all() insd = {} # instance name-state dict for uci in UCIs: dict = {} diff -r 5963dd169715 -r 7c4cfc243cc3 templates/cloud/add_image.mako --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/templates/cloud/add_image.mako Mon Nov 09 18:55:14 2009 -0500 @@ -0,0 +1,98 @@ +<% _=n_ %> +<%inherit file="/base.mako"/> +<%def name="title()">Add machine image</%def> + +<%def name="javascripts()"> +${parent.javascripts()} +<script type="text/javascript"> +$(function(){ + //$("input:text:first").focus(); +}) +</script> +</%def> + +%if header: + ${header} +%endif + +<div class="form"> + <div class="form-title">Add machine image</div> + <div class="form-body"> + <form name="add_image" action="${h.url_for( action='addNewImage' )}" method="post" > + <% + cls = "form-row" + if error.has_key('provider_error'): + cls += " form-row-error" + %> + <div class="${cls}"> + <label>Cloud provider type:</label> + <div class="form-row-input"> + <select name="provider_type" style="width:40em"> + <option value="">Select Provider Type...</option> + <option value="eucalyptus">Eucalyptus</option> + <option value="ec2">Amazon EC2</option> + </select> + </div> + %if error.has_key('provider_error'): + <div class="form-row-error-message">${error['provider_error']}</div> + %endif + <div style="clear: both"></div> + </div> + + <% + cls = "form-row" + if error.has_key('id_error'): + cls += " form-row-error" + %> + <div class="${cls}"> + <label>Machine Image ID (AMI or EMI):</label> + <div class="form-row-input"> + <input type="text" name="image_id" value="${image_id}" size="40"> + </div> + %if error.has_key('id_error'): + <div class="form-row-error-message">${error['id_error']}</div> + %endif + <div style="clear: both"></div> + </div> + + <% + cls = "form-row" + if error.has_key('manifest_error'): + cls += " form-row-error" + %> + <div class="${cls}"> + <label>Manifest:</label> + <div class="form-row-input"> + <input type="text" name="manifest" value="${manifest}" size="40"> + </div> + %if error.has_key('manifest_error'): + <div class="form-row-error-message">${error['manifest_error']}</div> + %endif + <div style="clear: both"></div> + </div> + + + <% + cls = "form-row" + if error.has_key('arch_error'): + cls += " form-row-error" + %> + <div class="${cls}"> + <label>Image architecture:</label> + <div class="form-row-input"> + <select name="architecture" style="width:40em"> + <option value="">Select Architecture Type...</option> + <option value="i386">i386 (32 bit)</option> + <option value="x86_64">x86_64 (64 bit)</option> + </select> + </div> + %if error.has_key('arch_error'): + <div class="form-row-error-message">${error['arch_error']}</div> + %endif + <div style="clear: both"></div> + </div> + + <div class="form-row"><input type="submit" value="Add"></div> + </form> + </div> +</div> \ No newline at end of file diff -r 5963dd169715 -r 7c4cfc243cc3 templates/cloud/configure_cloud.mako --- a/templates/cloud/configure_cloud.mako Fri Nov 06 13:45:57 2009 -0500 +++ b/templates/cloud/configure_cloud.mako Mon Nov 09 18:55:14 2009 -0500 @@ -42,8 +42,11 @@ else if ( ( old_state=='running' && new_state=='error' ) || ( old_state=='pending' && new_state=='error' ) || \ ( old_state=='submitted' && new_state=='error' ) || ( old_state=='submittedUCI' && new_state=='error' ) || \ ( old_state=='shutting-down' && new_state=='error' ) || ( prev_old_state.match('newUCI') && new_state=='error' ) || \ - ( prev_old_state.match('new') && new_state=='error' ) || ( prev_old_state.match('available') && new_state=='error' ) || \ + ( prev_old_state.match('new') && new_state=='error' ) || \ ( prev_old_state.match('deleting') && new_state=='error' ) || ( prev_old_state.match('deletingUCI') && new_state=='error' ) ) { + // TODO: Following clause causes constant page refresh for an exception thrown as a result of instance not starting correctly - need alternative method! + //( prev_old_state.match('available') && new_state=='error' ) || \ + var url = "${h.url_for( controller='cloud', action='list')}"; location.replace( url ); } diff -r 5963dd169715 -r 7c4cfc243cc3 templates/cloud/edit_image.mako --- a/templates/cloud/edit_image.mako Fri Nov 06 13:45:57 2009 -0500 +++ b/templates/cloud/edit_image.mako Mon Nov 09 18:55:14 2009 -0500 @@ -21,8 +21,22 @@ <div class="form-title">Edit image</div> <div class="form-body"> <form name="edit_image" action="${h.url_for( action='editImage', id=trans.security.encode_id(image.id), edited="true" )}" method="post" > - - <% + <% + cls = "form-row" + if error.has_key('provider_error'): + cls += " form-row-error" + %> + <div class="${cls}"> + <label>Provider type:</label> + <div class="form-row-input"> + ${image.provider_type} + </div> + %if error.has_key('provider_error'): + <div class="form-row-error-message">${error['provider_error']}</div> + %endif + <div style="clear: both"></div> + </div> + <% cls = "form-row" if error.has_key('id_error'): cls += " form-row-error" @@ -52,7 +66,21 @@ %endif <div style="clear: both"></div> </div> - + <% + cls = "form-row" + if error.has_key('arch_error'): + cls += " form-row-error" + %> + <div class="${cls}"> + <label>Architecture:</label> + <div class="form-row-input"> + <input type="text" name="architecture" value="${image.architecture}" size="40"> + </div> + %if error.has_key('arch_error'): + <div class="form-row-error-message">${error['arch_error']}</div> + %endif + <div style="clear: both"></div> + </div> <div class="form-row"><input type="submit" value="Save"></div> </form> diff -r 5963dd169715 -r 7c4cfc243cc3 templates/cloud/list_images.mako --- a/templates/cloud/list_images.mako Fri Nov 06 13:45:57 2009 -0500 +++ b/templates/cloud/list_images.mako Mon Nov 09 18:55:14 2009 -0500 @@ -21,14 +21,18 @@ <table class="mange-table colored" border="0" cellspacing="0" cellpadding="0" width="100%"> <colgroup width="2%"></colgroup> + <colgroup width="10%"></colgroup> <colgroup width="13%"></colgroup> - <colgroup width="70%"></colgroup> + <colgroup width="55%"></colgroup> + <colgroup width="10%"></colgroup> <colgroup width="5%"></colgroup> <colgroup width="5%"></colgroup> <tr class="header"> <th>#</th> - <th>Machime image ID</th> + <th>Provider type</th> + <th>Machime image ID</th> <th>Manifest</th> + <th>Architecture</th> <th>Edit</th> <th>Delete</th> <th></th> @@ -37,6 +41,13 @@ <tr> <td>${i+1}</td> <td> + %if image.provider_type: + ${image.provider_type} + %else: + N/A + %endif + </td> + <td> %if image.image_id: ${image.image_id} %else: @@ -51,6 +62,13 @@ %endif </td> <td> + %if image.architecture: + ${image.architecture} + %else: + N/A + %endif + </td> + <td> <a href="${h.url_for( controller='cloud', action='editImage', image_id=image.image_id, manifest=image.manifest, id=trans.security.encode_id(image.id) )}">e</a> </td> <td> diff -r 5963dd169715 -r 7c4cfc243cc3 templates/cloud/view.mako --- a/templates/cloud/view.mako Fri Nov 06 13:45:57 2009 -0500 +++ b/templates/cloud/view.mako Mon Nov 09 18:55:14 2009 -0500 @@ -48,11 +48,11 @@ </tr> <tr> <td> Cloud provider type: </td> - <td> ${str(credDetails.provider.type)[:16]}</td> + <td> ${str(credDetails.provider.type)}</td> </tr> <tr> <td> Cloud provider name: </td> - <td> ${str(credDetails.provider.name)[:16]}</td> + <td> ${str(credDetails.provider.name)}</td> </tr> <tr> <td> Access key: </td> @@ -76,7 +76,7 @@ href="javascript:void(0)"> - Hide </a><br /> - <nobr><b>${credDetails.secret_key}</b></nobr><br/> + <nobr>${credDetails.secret_key}</nobr><br/> </div> </td> </tr> diff -r 5963dd169715 -r 7c4cfc243cc3 templates/cloud/viewInstance.mako --- a/templates/cloud/viewInstance.mako Fri Nov 06 13:45:57 2009 -0500 +++ b/templates/cloud/viewInstance.mako Mon Nov 09 18:55:14 2009 -0500 @@ -94,19 +94,25 @@ <td> ${liveInstance.private_dns}</td> </tr> %endif + %if liveInstance.security_group != None: + <tr> + <td> Security group zone:</td> + <td> ${liveInstance.security_group} </td> + </tr> + %endif %if liveInstance.availability_zone != None: <tr> <td> Availabilty zone:</td> <td> ${liveInstance.availability_zone} </td> </tr> %endif - %if liveInstance.keypair_name != None: + %if liveInstance.uci.key_pair_name != None: <tr> <td> Keypair file name:</td> - <td> ${liveInstance.keypair_name} </td> + <td> ${liveInstance.uci.key_pair_name} </td> </tr> %endif - %if liveInstance.keypair_material != None: + %if liveInstance.uci.key_pair_material != None: <tr> <td> Keypair material:</td> <div id="shortComment2"> @@ -117,7 +123,7 @@ </a> </div> <div id="fullComment2" style="DISPLAY: none"> - <nobr><b>${liveInstance.keypair_material}</b></nobr><br/> + <nobr><b>${liveInstance.uci.key_pair_material}</b></nobr><br/> <a onclick="document.getElementById('shortComment2').style.display = 'block'; document.getElementById('fullComment2').style.display = 'none'; return 0;" href="javascript:void(0)">