details: http://www.bx.psu.edu/hg/galaxy/rev/70ae74578254 changeset: 3082:70ae74578254 user: Enis Afgan <afgane@gmail.com> date: Wed Nov 04 17:19:18 2009 -0500 description: Bug fixes and minor updates. diffstat: lib/galaxy/cloud/providers/ec2.py | 20 ++++- lib/galaxy/cloud/providers/eucalyptus.py | 36 ++++++--- lib/galaxy/web/controllers/cloud.py | 122 ++++++++++-------------------- templates/cloud/configure_cloud.mako | 20 +++- 4 files changed, 96 insertions(+), 102 deletions(-) diffs (503 lines): diff -r 89276f68513a -r 70ae74578254 lib/galaxy/cloud/providers/ec2.py --- a/lib/galaxy/cloud/providers/ec2.py Tue Nov 03 13:20:26 2009 -0500 +++ b/lib/galaxy/cloud/providers/ec2.py Wed Nov 04 17:19:18 2009 -0500 @@ -45,7 +45,8 @@ store_states = Bunch( IN_USE = "in-use", - CREATING = "creating" + CREATING = "creating", + ERROR = "error" ) class EC2CloudProvider( object ): @@ -94,7 +95,7 @@ """ Establishes EC2 cloud connection using user's credentials associated with given UCI """ - log.debug( '##### Establishing EC2 cloud connection' ) + log.debug( 'Establishing %s cloud connection' % self.type ) provider = uci_wrapper.get_provider() try: region = RegionInfo( None, provider.region_name, provider.region_endpoint ) @@ -276,7 +277,7 @@ if uci_wrapper.get_state() != uci_states.ERROR: # Start an instance - log.debug( "***** Starting instance for UCI '%s'" % uci_wrapper.get_name() ) + log.debug( "Starting instance for UCI '%s'" % uci_wrapper.get_name() ) #TODO: Once multiple volumes can be attached to a single instance, update 'userdata' composition userdata = uci_wrapper.get_store_volume_id()+"|"+uci_wrapper.get_access_key()+"|"+uci_wrapper.get_secret_key() log.debug( 'Using following command: conn.run_instances( image_id=%s, key_name=%s, security_groups=[%s], user_data=[OMITTED], instance_type=%s, placement=%s )' @@ -401,9 +402,19 @@ model.CloudStore.c.status==store_states.CREATING, model.CloudStore.c.status==None ) ).all() for store in stores: - if self.type == store.uci.credentials.provider.type: + if self.type == store.uci.credentials.provider.type and store.volume_id != None: log.debug( "[%s] Running general status update on store '%s'" % ( store.uci.credentials.provider.type, store.volume_id ) ) self.updateStore( store ) + else: + log.error( "[%s] There exists an entry for UCI (%s) storage volume without an ID. Storage volume might have been created with " + "cloud provider though. Manual check is recommended." % ( store.uci.credentials.provider.type, store.uci.name ) ) + store.uci.error = "There exists an entry in local database for a storage volume without an ID. Storage volume might have been created " \ + "with cloud provider though. Manual check is recommended. After understanding what happened, local database etry for given " \ + "storage volume should be updated." + store.status = store_states.ERROR + store.uci.state = uci_states.ERROR + store.uci.flush() + store.flush() # Attempt at updating any zombie UCIs (i.e., instances that have been in SUBMITTED state for longer than expected - see below for exact time) zombies = model.UCI.filter_by( state=uci_states.SUBMITTED ).all() @@ -605,6 +616,7 @@ Establishes and returns connection to cloud provider. Information needed to do so is obtained directly from uci database object. """ + log.debug( 'Establishing %s cloud connection' % self.type ) a_key = uci.credentials.access_key s_key = uci.credentials.secret_key # Get connection diff -r 89276f68513a -r 70ae74578254 lib/galaxy/cloud/providers/eucalyptus.py --- a/lib/galaxy/cloud/providers/eucalyptus.py Tue Nov 03 13:20:26 2009 -0500 +++ b/lib/galaxy/cloud/providers/eucalyptus.py Wed Nov 04 17:19:18 2009 -0500 @@ -46,7 +46,8 @@ store_states = Bunch( IN_USE = "in-use", - CREATING = "creating" + CREATING = "creating", + ERROR = "error" ) class EucalyptusCloudProvider( object ): @@ -96,7 +97,7 @@ """ Establishes eucalyptus cloud connection using user's credentials associated with given UCI """ - log.debug( '##### Establishing eucalyptus cloud connection' ) + log.debug( 'Establishing %s cloud connection.' % self.type ) provider = uci_wrapper.get_provider() try: euca_region = RegionInfo( None, provider.region_name, provider.region_endpoint ) @@ -170,14 +171,14 @@ and registers relevant information in Galaxy database. """ conn = self.get_connection( uci_wrapper ) + + # Because only 1 storage volume may be created at UCI config time, index of this storage volume in local Galaxy DB w.r.t + # current UCI is 0; therefore, it can be referenced in following code + log.info( "Creating volume in zone '%s'..." % uci_wrapper.get_uci_availability_zone() ) if uci_wrapper.get_uci_availability_zone()=='': log.info( "Availability zone for UCI (i.e., storage volume) was not selected, using default zone: %s" % self.zone ) uci_wrapper.set_store_availability_zone( self.zone ) - #TODO: check if volume associated with UCI already exists (if server crashed for example) and don't recreate it - log.info( "Creating volume in zone '%s'..." % uci_wrapper.get_uci_availability_zone() ) - # Because only 1 storage volume may be created at UCI config time, index of this storage volume in local Galaxy DB w.r.t - # current UCI is 0, so reference it in following methods vol = conn.create_volume( uci_wrapper.get_store_size( 0 ), uci_wrapper.get_uci_availability_zone(), snapshot=None ) uci_wrapper.set_store_volume_id( 0, vol.id ) @@ -209,15 +210,14 @@ failedList.append( v.volume_id ) # Delete UCI if all of associated - log.debug( "count=%s, len(vl)=%s" % (count, len( vl ) ) ) if count == len( vl ): uci_wrapper.delete() else: log.error( "Deleting following volume(s) failed: %s. However, these volumes were successfully deleted: %s. \ - MANUAL intervention and processing needed." % ( failedList, deletedList ) ) + Manual intervention and processing needed." % ( str( failedList ), str( deletedList ) ) ) uci_wrapper.change_state( uci_state=uci_states.ERROR ) - uci_wrapper.set_error( "Deleting following volume(s) failed: "+failedList+". However, these volumes were successfully deleted: "+deletedList+". \ - MANUAL intervention and processing needed." ) + uci_wrapper.set_error( "Deleting following volume(s) failed: "+str(failedList)+". However, these volumes were \ + successfully deleted: "+str(deletedList)+". Manual intervention and processing needed." ) def addStorageToUCI( self, name ): """ Adds more storage to specified UCI """ @@ -250,7 +250,7 @@ uci_wrapper.set_mi( i_index, mi_id ) if uci_wrapper.get_state() != uci_states.ERROR: - log.debug( "***** Starting UCI instance '%s'" % uci_wrapper.get_name() ) + log.debug( "Starting UCI instance '%s'" % uci_wrapper.get_name() ) log.debug( 'Using following command: conn.run_instances( image_id=%s, key_name=%s )' % ( mi_id, uci_wrapper.get_key_pair_name( i_index ) ) ) try: reservation = conn.run_instances( image_id=mi_id, key_name=uci_wrapper.get_key_pair_name( i_index ) ) @@ -368,9 +368,19 @@ model.CloudStore.c.status==store_states.CREATING, model.CloudStore.c.status==None ) ).all() for store in stores: - if self.type == store.uci.credentials.provider.type: + if self.type == store.uci.credentials.provider.type and store.volume_id != None: log.debug( "[%s] Running general status update on store '%s'" % ( store.uci.credentials.provider.type, store.volume_id ) ) self.updateStore( store ) + else: + log.error( "[%s] There exists an entry for UCI (%s) storage volume without an ID. Storage volume might have been created with " + "cloud provider though. Manual check is recommended." % ( store.uci.credentials.provider.type, store.uci.name ) ) + store.uci.error = "There exists an entry in local database for a storage volume without an ID. Storage volume might have been created " \ + "with cloud provider though. Manual check is recommended. After understanding what happened, local database etry for given " \ + "storage volume should be updated." + store.status = store_states.ERROR + store.uci.state = uci_states.ERROR + store.uci.flush() + store.flush() # Attempt at updating any zombie UCIs (i.e., instances that have been in SUBMITTED state for longer than expected - see below for exact time) zombies = model.UCI.filter_by( state=uci_states.SUBMITTED ).all() @@ -455,6 +465,7 @@ conn = self.get_connection_from_uci( uci ) try: + log.debug( "vol id: " % store.volume_id ) vl = conn.get_all_volumes( [store.volume_id] ) except boto.exception.EC2ResponseError, e: log.error( "Retrieving volume(s) from cloud for UCI '%s' failed: " % ( uci.name, str(e) ) ) @@ -570,6 +581,7 @@ Establishes and returns connection to cloud provider. Information needed to do so is obtained directly from uci database object. """ + log.debug( 'Establishing %s cloud connection.' % self.type ) a_key = uci.credentials.access_key s_key = uci.credentials.secret_key # Get connection diff -r 89276f68513a -r 70ae74578254 lib/galaxy/web/controllers/cloud.py --- a/lib/galaxy/web/controllers/cloud.py Tue Nov 03 13:20:26 2009 -0500 +++ b/lib/galaxy/web/controllers/cloud.py Wed Nov 04 17:19:18 2009 -0500 @@ -52,11 +52,14 @@ SHUTTING_DOWN = "shutting-down" ) +store_states = Bunch( + IN_USE = "in-use", + CREATING = "creating", + ERROR = "error" +) + class CloudController( BaseController ): - -# def __init__( self ): -# self.cloudManager = CloudManager() - + @web.expose def index( self, trans ): return trans.fill_template( "cloud/index.mako" ) @@ -76,22 +79,22 @@ liveInstances = trans.sa_session.query( model.UCI ) \ .filter_by( user=user ) \ - .filter( or_( model.UCI.c.state==uci_states.RUNNING, #"running", - model.UCI.c.state==uci_states.PENDING, #"pending", - model.UCI.c.state==uci_states.SUBMITTED, #"submitted", - model.UCI.c.state==uci_states.SUBMITTED_UCI, #"submittedUCI", - model.UCI.c.state==uci_states.SHUTTING_DOWN, #"shutting-down", + .filter( or_( model.UCI.c.state==uci_states.RUNNING, + model.UCI.c.state==uci_states.PENDING, + model.UCI.c.state==uci_states.SUBMITTED, + model.UCI.c.state==uci_states.SUBMITTED_UCI, + model.UCI.c.state==uci_states.SHUTTING_DOWN, model.UCI.c.state==uci_states.SHUTTING_DOWN_UCI ) ) \ .order_by( desc( model.UCI.c.update_time ) ) \ .all() prevInstances = trans.sa_session.query( model.UCI ) \ .filter_by( user=user ) \ - .filter( or_( model.UCI.c.state==uci_states.AVAILABLE, #"available", - model.UCI.c.state==uci_states.NEW, #"new", - model.UCI.c.state==uci_states.NEW_UCI, #"newUCI", - model.UCI.c.state==uci_states.ERROR, #"error", - model.UCI.c.state==uci_states.DELETING, #"deleting", + .filter( or_( model.UCI.c.state==uci_states.AVAILABLE, + model.UCI.c.state==uci_states.NEW, + model.UCI.c.state==uci_states.NEW_UCI, + model.UCI.c.state==uci_states.ERROR, + model.UCI.c.state==uci_states.DELETING, model.UCI.c.state==uci_states.DELETING_UCI ) ) \ .order_by( desc( model.UCI.c.update_time ) ) \ .all() @@ -99,8 +102,8 @@ # Check after update there are instances in pending state; if so, display message pendingInstances = trans.sa_session.query( model.UCI ) \ .filter_by( user=user ) \ - .filter( or_( model.UCI.c.state==uci_states.PENDING, #"pending" , \ - model.UCI.c.state==uci_states.SUBMITTED, #"submitted" , \ + .filter( or_( model.UCI.c.state==uci_states.PENDING, + model.UCI.c.state==uci_states.SUBMITTED, model.UCI.c.state==uci_states.SUBMITTED_UCI ) ) \ .all() if pendingInstances: @@ -124,6 +127,7 @@ def makeDefault( self, trans, id=None ): """ Set current credentials as default. + *NOT USED* """ currentDefault = get_default_credentials (trans) if currentDefault: @@ -137,8 +141,7 @@ # TODO: Fix bug that when this function returns, top Galaxy tab bar is missing from the webpage return self.list( trans ) #trans.fill_template( "cloud/configure_cloud.mako", #awsCredentials = awsCredentials ) - - + @web.expose @web.require_login( "start Galaxy cloud instance" ) def start( self, trans, id, type='m1.small' ): @@ -149,19 +152,9 @@ uci = get_uci( trans, id ) mi = get_mi( trans, uci, type ) stores = get_stores( trans, uci ) - # Ensure instance is not already running (or related state) and store relevant data + # Ensure instance is available and then store relevant data # into DB to initiate instance startup by cloud manager - if ( len(stores) is not 0 ) and \ - ( uci.state != uci_states.SUBMITTED ) and \ - ( uci.state != uci_states.SUBMITTED_UCI ) and \ - ( uci.state != uci_states.PENDING ) and \ - ( uci.state != uci_states.DELETING ) and \ - ( uci.state != uci_states.DELETING_UCI ) and \ - ( uci.state != uci_states.DELETED ) and \ - ( uci.state != uci_states.RUNNING ) and \ - ( uci.state != uci_states.NEW_UCI ) and \ - ( uci.state != uci_states.NEW ) and \ - ( uci.state != uci_states.ERROR ): + if ( len(stores) is not 0 ) and ( uci.state == uci_states.AVAILABLE ): instance = model.CloudInstance() instance.user = user instance.image = mi @@ -181,7 +174,10 @@ "instance description." ) return self.list( trans ) - error( "Cannot start instance that is in state '%s'." % uci.state ) + if len(stores) == 0: + error( "This instance does not have any storage volumes associated it and thus cannot be started." ) + else: + error( "Cannot start instance that is in state '%s'." % uci.state ) return self.list( trans ) @web.expose @@ -228,7 +224,10 @@ trans.set_message( "Galaxy instance '%s' marked for deletion." % name ) return self.list( trans ) - trans.set_message( "Instance '%s' is already marked for deletion." % uci.name ) + if uci.state != uci_states.ERROR: + trans.set_message( "Cannot delete instance in state ERROR." ) + else: + trans.set_message( "Instance '%s' is already marked for deletion." % uci.name ) return self.list( trans ) @web.expose @@ -251,8 +250,6 @@ .filter_by( user=user, state=instance_states.TERMINATED, uci_id=id ) \ .order_by( desc( model.CloudInstance.c.update_time ) ) \ .all() - - log.debug( "id: %s" % id ) return trans.fill_template( "cloud/view_usage.mako", prevInstances = prevInstances ) @@ -284,6 +281,12 @@ providersToZones[storedCred.name] = ['Unknown provider zone'] if instanceName: + # Check if volume size is entered as an integer + try: + volSize = int( volSize ) + except ValueError: + error['vol_error'] = "Volume size must be integer value between 1 and 1000." + # Create new user configured instance try: if trans.app.model.UCI \ @@ -299,9 +302,6 @@ error['vol_error'] = "You must specify volume size as an integer value between 1 and 1000." elif ( int( volSize ) < 1 ) or ( int( volSize ) > 1000 ): error['vol_error'] = "Volume size must be integer value between 1 and 1000." -# elif type( volSize ) != type( 1 ): # Check if volSize is int -# log.debug( "volSize='%s'" % volSize ) -# error['vol_error'] = "Volume size must be integer value between 1 and 1000." elif zone=='': error['zone_error'] = "You must select zone where this UCI will be registered." else: @@ -311,14 +311,14 @@ uci.credentials = trans.app.model.CloudUserCredentials.filter( trans.app.model.CloudUserCredentials.table.c.name==credName ).first() uci.user= user - uci.total_size = volSize # This is OK now because new instance is being created. + uci.total_size = volSize # This is OK now because new instance is being created and only one storage volume can be created at UCI creation time uci.state = uci_states.NEW_UCI storage = model.CloudStore() storage.user = user storage.uci = uci storage.size = volSize - storage.availability_zone = zone # TODO: Give user choice here. Also, enable region selection. + storage.availability_zone = zone # Persist session = trans.sa_session session.save_or_update( uci ) @@ -333,7 +333,7 @@ except AttributeError, ae: inst_error = "No registered cloud images. You must contact administrator to add some before proceeding." log.debug("AttributeError: %s " % str( ae ) ) - + return trans.fill_template( "cloud/configure_uci.mako", instanceName = instanceName, credName = storedCreds, @@ -341,14 +341,7 @@ zone = zone, error = error, providersToZones = providersToZones ) - - return trans.show_form( - web.FormBuilder( web.url_for(), "Configure new instance", submit_text="Add" ) - .add_text( "instanceName", "Instance name", value="Unnamed instance", error=inst_error ) - .add_text( "credName", "Name of registered credentials to use", value="", error=cred_error ) - .add_text( "volSize", "Permanent storage size (1GB - 1000GB)" - "<br />Note: you will be able to add more storage later", value='', error=vol_error ) ) - + @web.expose @web.require_admin def addNewImage( self, trans, image_id='', manifest='', state=None ): @@ -851,6 +844,7 @@ user = trans.get_user() stores = trans.sa_session.query( model.CloudStore ) \ .filter_by( user=user, uci_id=uci.id ) \ + .filter( model.CloudStore.c.status != store_states.ERROR ) \ .all() return stores @@ -868,19 +862,6 @@ return instances -def get_cloud_instance( conn, instance_id ): - """ - Returns a cloud instance representation of the instance id, i.e., cloud instance object that cloud API can be invoked on - """ - # get_all_instances func. takes a list of desired instance id's, so create a list first - idLst = list() - idLst.append( instance_id ) - # Retrieve cloud instance based on passed instance id. get_all_instances( idLst ) method returns reservation ID. Because - # we are passing only 1 ID, we can retrieve only the first element of the returning list. Furthermore, because (for now!) - # only 1 instance corresponds each individual reservation, grab only the first element of the returned list of instances. - cloudInstance = conn.get_all_instances( [instance_id] )[0].instances[0] - return cloudInstance - def get_connection( trans, credName ): """ Establishes EC2 connection using user's default credentials @@ -900,22 +881,3 @@ else: error( "You must specify default credentials before starting an instance." ) return 0 - -def get_keypair_name( trans ): - """ - Generate keypair using user's default credentials - """ - conn = get_connection( trans ) - - log.debug( "Getting user's keypair" ) - key_pair = conn.get_key_pair( 'galaxy-keypair' ) - - try: - return key_pair.name - except AttributeError: # No keypair under this name exists so create it - log.debug( 'No keypair found, creating keypair' ) - key_pair = conn.create_key_pair( 'galaxy-keypair' ) - # TODO: Store key_pair.material into instance table - this is the only time private key can be retrieved - # Actually, probably return key_pair to calling method and store name & key from there... - - return key_pair.name diff -r 89276f68513a -r 70ae74578254 templates/cloud/configure_cloud.mako --- a/templates/cloud/configure_cloud.mako Tue Nov 03 13:20:26 2009 -0500 +++ b/templates/cloud/configure_cloud.mako Wed Nov 04 17:19:18 2009 -0500 @@ -26,19 +26,24 @@ var elem = '#' + data[i].id; // Because of different list managing 'live' vs. 'available' instances, reload url on various state changes old_state = $(elem + "-state").text(); + prev_old_state = $(elem + "-state-p").text(); new_state = data[i].state; //console.log( "old_state[%d] = %s", i, old_state ); + //console.log( "prev_old_state[%d] = %s", i, prev_old_state ); //console.log( "new_state[%d] = %s", i, new_state ); if ( ( old_state=='pending' && new_state=='running' ) || ( old_state=='shutting-down' && new_state=='available' ) || \ ( old_state=='running' && new_state=='available' ) || ( old_state=='running' && new_state=='error' ) || \ ( old_state=='pending' && new_state=='error' ) || ( old_state=='pending' && new_state=='available' ) || \ - ( old_state=='submitted' && new_state=='available' ) ) { + ( old_state=='submitted' && new_state=='available' ) || ( prev_old_state.match('newUCI') && new_state=='available' ) || \ + ( prev_old_state.match('new') && new_state=='available' ) ) { var url = "${h.url_for( controller='cloud', action='list')}"; location.replace( url ); } else if ( ( old_state=='running' && new_state=='error' ) || ( old_state=='pending' && new_state=='error' ) || \ ( old_state=='submitted' && new_state=='error' ) || ( old_state=='submittedUCI' && new_state=='error' ) || \ - ( old_state=='shutting-down' && new_state=='error' ) ) { + ( old_state=='shutting-down' && new_state=='error' ) || ( prev_old_state.match('newUCI') && new_state=='error' ) || \ + ( prev_old_state.match('new') && new_state=='error' ) || ( prev_old_state.match('available') && new_state=='error' ) || \ + ( prev_old_state.match('deleting') && new_state=='error' ) || ( prev_old_state.match('deletingUCI') && new_state=='error' ) ) { var url = "${h.url_for( controller='cloud', action='list')}"; location.replace( url ); } @@ -56,13 +61,13 @@ if( !dns ) { $(uci+"-link").text( 'Galaxy starting...' ); // http://stackoverflow.com/questions/275931/how-do-you-make-an-element-flash-i... - $(uci+"-link").stop().animate({ fontSize: "14px" }, 1000).animate({ fontSize: "12px" }, 1000); + //$(uci+"-link").stop().animate({ fontSize: "14px" }, 1000).animate({ fontSize: "12px" }, 1000); } else { $(uci+"-link").html( '<div align="right"><a class="action-button" href="http://'+dns+'" target="_blank">' + '<span>Access Galaxy</span>'+ '<img src="' + "${h.url_for( '/static/images/silk/resultset_next.png' )}" + '" /></div>' ); - $(uci+"-link").stop().animate({ fontSize: "14px" }, 1000).animate({ fontSize: "12px" }, 1000); + //$(uci+"-link").stop().animate({ fontSize: "14px" }, 1000).animate({ fontSize: "12px" }, 1000); } } }); @@ -92,6 +97,7 @@ %if cloudCredentials: ## Manage user credentials + <h3>Your registered credentials</h3> <ul class="manage-table-actions"> <li> <a class="action-button" href="${h.url_for( action='add' )}"> @@ -131,7 +137,7 @@ ## ***************************************************** ## Manage live instances <p /> - <h2>Manage your cloud instances</h2> + <h3>Manage your cloud instances</h3> <ul class="manage-table-actions"> <li> <a class="action-button" href="${h.url_for( action='configureNew' )}"> @@ -232,7 +238,7 @@ <a id="pi-${i}-popup" class="popup-arrow" style="display: none;">▼</a> </td> <td>${str(prevInstance.total_size)}</td> - <td> + <td id="${ prevInstance.id }-state-p"> <%state = str(prevInstance.state)%> %if state =='error': <div id="${prevInstance.name}-short"> @@ -249,7 +255,9 @@ error:</a><br /> ${str(prevInstance.error)} <p /> + <div style="font-size:10px;"> <a href="${h.url_for( action='set_uci_state', id=trans.security.encode_id(prevInstance.id), state='available' )}">reset state</a> + </div> </div> %else: ${str(prevInstance.state)}