details: http://www.bx.psu.edu/hg/galaxy/rev/a620afab791d changeset: 3056:a620afab791d user: Enis Afgan <afgane@gmail.com> date: Thu Aug 27 17:36:03 2009 -0400 description: In the process of adding and integrating cloud storage DB table. Currently, broken. diffstat: lib/galaxy/model/__init__.py | 18 ++- lib/galaxy/model/mapping.py | 67 ++++++++---- lib/galaxy/model/migrate/versions/0014_cloud_tables.py | 102 +++++++++++++------- lib/galaxy/web/controllers/cloud.py | 85 ++++++++++------ templates/cloud/configure_cloud.mako | 24 ++-- 5 files changed, 189 insertions(+), 107 deletions(-) diffs (554 lines): diff -r c28dd8c0a231 -r a620afab791d lib/galaxy/model/__init__.py --- a/lib/galaxy/model/__init__.py Wed Aug 26 17:34:58 2009 -0400 +++ b/lib/galaxy/model/__init__.py Thu Aug 27 17:36:03 2009 -0400 @@ -932,24 +932,34 @@ self.galaxy_session = galaxy_session self.history = history -class UserInstances( object ): +class CloudInstance( object ): def __init__( self ): self.id = None self.user = None self.name = None self.instance_id = None - self.ami = None + self.mi = None self.state = None + self.keypair_name = None self.public_dns = None self.availability_zone = None -class CloudImages( object ): +class CloudImage( object ): def __init__( self ): self.id = None self.instance_id = None self.state = None -class StoredUserCredentials( object ): +class CloudStorage( object ): + def __init__( self ): + self.id = None + self.volume_id = None + self.i_id = None + self.user = None + self.size = None + self.availability_zone = None + +class CloudUserCredentials( object ): def __init__( self ): self.id = None self.user = None diff -r c28dd8c0a231 -r a620afab791d lib/galaxy/model/mapping.py --- a/lib/galaxy/model/mapping.py Wed Aug 26 17:34:58 2009 -0400 +++ b/lib/galaxy/model/mapping.py Thu Aug 27 17:36:03 2009 -0400 @@ -386,31 +386,47 @@ # *************************************************************************** # *************************** Cloud tables*********************************** # *************************************************************************** -UserInstances.table = Table( "user_instances", metadata, +CloudImage.table = Table( "cloud_image", metadata, Column( "id", Integer, primary_key=True ), Column( "create_time", DateTime, default=now ), - Column( "launch_time", DateTime, onupdate=now ), - Column( "user_id", Integer, ForeignKey( "galaxy_user.id" ), index=True, nullable=False ), - Column( "name", TEXT ), - Column( "reservation_id", TEXT ), - Column( "instance_id", TEXT ), - Column( "ami", TEXT, ForeignKey( "cloud_images.image_id" ), nullable=False ), - Column( "state", TEXT ), - Column( "public_dns", TEXT ), - Column( "private_dns", TEXT ), - Column( "keypair_fingerprint", TEXT ), - Column( "keypair_material", TEXT ), - Column( "availability_zone", TEXT ) ) - -CloudImages.table = Table( "cloud_images", metadata, - Column( "id", Integer, primary_key=True ), - Column( "create_time", DateTime, default=now ), - Column( "upadte_time", DateTime, default=now, onupdate=now ), + Column( "update_time", DateTime, default=now, onupdate=now ), Column( "image_id", TEXT, nullable=False ), Column( "manifest", TEXT ), Column( "state", TEXT ) ) -StoredUserCredentials.table = Table( "stored_user_credentials", metadata, +CloudInstance.table = Table( "cloud_instance", metadata, + Column( "id", Integer, primary_key=True ), + Column( "create_time", DateTime, default=now ), + Column( "update_time", DateTime, default=now, onupdate=now ), + Column( "launch_time", DateTime ), + Column( "user_id", Integer, ForeignKey( "galaxy_user.id" ), index=True, nullable=False ), + Column( "name", TEXT ), + Column( "type", TEXT ), + Column( "reservation_id", TEXT ), + Column( "instance_id", TEXT ), + Column( "mi", TEXT, ForeignKey( "cloud_image.image_id" ), index=True, nullable=False ), + Column( "state", TEXT ), + Column( "public_dns", TEXT ), + Column( "private_dns", TEXT ), + Column( "keypair_name", TEXT ), + Column( "availability_zone", TEXT ) ) + +CloudStorage.table = Table( "cloud_storage", metadata, + Column( "id", Integer, primary_key=True ), + Column( "create_time", DateTime, default=now ), + Column( "update_time", DateTime, default=now, onupdate=now ), + Column( "attach_time", DateTime ), + Column( "user_id", Integer, ForeignKey( "galaxy_user.id" ), index=True, nullable=False ), + Column( "volume_id", TEXT, nullable=False ), + Column( "size", Integer, nullable=False ), + Column( "availability_zone", TEXT, nullable=False ), + Column( "i_id", TEXT, ForeignKey( "cloud_instance.instance_id" ), index=True ), + Column( "status", TEXT ), + Column( "device", TEXT ), + Column( "space_consumed", Integer ) + ) + +CloudUserCredentials.table = Table( "cloud_user_credentials", metadata, Column( "id", Integer, primary_key=True ), Column( "create_time", DateTime, default=now ), Column( "update_time", DateTime, default=now, onupdate=now ), @@ -929,14 +945,19 @@ # ************************************************************ # vvvvvvvvvvvvvvvv Start cloud table mappings vvvvvvvvvvvvvvvv # ************************************************************ -assign_mapper( context, UserInstances, UserInstances.table, +assign_mapper( context, CloudImage, CloudImage.table ) + +assign_mapper( context, CloudInstance, CloudInstance.table, properties=dict( user=relation( User ), - cloud_image=relation( CloudImages ) + image=relation( CloudImage ) ) ) -assign_mapper( context, CloudImages, CloudImages.table ) +assign_mapper( context, CloudStorage, CloudStorage.table, + properties=dict( user=relation( User ), + instance=relation( CloudInstance, backref='cloud_instance' ) + ) ) -assign_mapper( context, StoredUserCredentials, StoredUserCredentials.table, +assign_mapper( context, CloudUserCredentials, CloudUserCredentials.table, properties=dict( user=relation( User) ) ) # ^^^^^^^^^^^^^^^ End cloud table mappings ^^^^^^^^^^^^^^^^^^ diff -r c28dd8c0a231 -r a620afab791d lib/galaxy/model/migrate/versions/0014_cloud_tables.py --- a/lib/galaxy/model/migrate/versions/0014_cloud_tables.py Wed Aug 26 17:34:58 2009 -0400 +++ b/lib/galaxy/model/migrate/versions/0014_cloud_tables.py Thu Aug 27 17:36:03 2009 -0400 @@ -12,7 +12,47 @@ metadata = MetaData( migrate_engine ) -Credentials_table = Table( "stored_user_credentials", metadata, +CloudImage_table = Table( "cloud_image", metadata, + Column( "id", Integer, primary_key=True ), + Column( "create_time", DateTime, default=now ), + Column( "update_time", DateTime, default=now, onupdate=now ), + Column( "image_id", TEXT, nullable=False ), + Column( "manifest", TEXT ), + Column( "state", TEXT ) ) + +CloudInstance_table = Table( "cloud_instance", metadata, + Column( "id", Integer, primary_key=True ), + Column( "create_time", DateTime, default=now ), + Column( "update_time", DateTime, default=now, onupdate=now ), + Column( "launch_time", DateTime ), + Column( "user_id", Integer, ForeignKey( "galaxy_user.id" ), index=True, nullable=False ), + Column( "name", TEXT ), + Column( "type", TEXT ), + Column( "reservation_id", TEXT ), + Column( "instance_id", TEXT ), + Column( "mi", TEXT, ForeignKey( "cloud_image.image_id" ), index=True, nullable=False ), + Column( "state", TEXT ), + Column( "public_dns", TEXT ), + Column( "private_dns", TEXT ), + Column( "keypair_name", TEXT ), + Column( "availability_zone", TEXT ) ) + +CloudStorage_table = Table( "cloud_storage", metadata, + Column( "id", Integer, primary_key=True ), + Column( "create_time", DateTime, default=now ), + Column( "update_time", DateTime, default=now, onupdate=now ), + Column( "attach_time", DateTime ), + Column( "user_id", Integer, ForeignKey( "galaxy_user.id" ), index=True, nullable=False ), + Column( "volume_id", TEXT, nullable=False ), + Column( "size", Integer, nullable=False ), + Column( "availability_zone", TEXT, nullable=False ), + Column( "i_id", TEXT, ForeignKey( "cloud_instance.instance_id" ), index=True ), + Column( "status", TEXT ), + Column( "device", TEXT ), + Column( "space_consumed", Integer ) + ) + +CloudUserCredentials_table = Table( "cloud_user_credentials", metadata, Column( "id", Integer, primary_key=True ), Column( "create_time", DateTime, default=now ), Column( "update_time", DateTime, default=now, onupdate=now ), @@ -20,49 +60,41 @@ Column( "name", TEXT), Column( "access_key", TEXT), Column( "secret_key", TEXT), - Column( "defaultCred", Boolean, default=False ) ) - -UserInstances_table = Table ( "user_instances", metadata, - Column( "id", Integer, primary_key=True ), - Column( "create_time", DateTime, default=now ), - Column( "launch_time", DateTime, onupdate=now ), - Column( "user_id", Integer, ForeignKey( "galaxy_user.id" ), index=True, nullable=False ), - Column( "name", TEXT ), - Column( "reservation_id", TEXT ), - Column( "instance_id", TEXT ), - Column( "ami", TEXT, ForeignKey( "cloud_images.image_id" ), nullable=False ), - Column( "state", TEXT ), - Column( "public_dns", TEXT ), - Column( "private_dns", TEXT ), - Column( "keypair_fingerprint", TEXT ), - Column( "keypair_material", TEXT ), - Column( "availability_zone", TEXT ) ) - -CloudImages_table = Table( "cloud_images", metadata, - Column( "id", Integer, primary_key=True ), - Column( "create_time", DateTime, default=now ), - Column( "upadte_time", DateTime, default=now, onupdate=now ), - Column( "image_id", TEXT, nullable=False ), - Column( "manifest", TEXT ), - Column( "state", TEXT ) ) - + Column( "defaultCred", Boolean, default=False) + ) def upgrade(): metadata.reflect() - Credentials_table.create() - UserInstances_table.create() + CloudUserCredentials_table.create() + CloudInstance_table.create() + CloudStorage_table.create() try: - CloudImages_table.create() + CloudImage_table.create() except Exception, e: - log.debug( "Creating CloudImages table failed. Table probably exists already." ) + log.debug( "Creating cloud_image table failed. Table probably exists already." ) def downgrade(): metadata.reflect() try: - Credentials_table.drop() + CloudImage_table.drop() except Exception, e: - log.debug( "Dropping stored_user_credentials table failed: %s" % str( e ) ) + log.debug( "Dropping cloud_image table failed: %s" % str( e ) ) + + try: + CloudInstance_table.drop() + except Exception, e: + log.debug( "Dropping cloud_instance table failed: %s" % str( e ) ) try: - UserInstances_table.drop() + CloudStorage_table.drop() except Exception, e: - log.debug( "Dropping user_instances table failed: %s" % str( e ) ) + log.debug( "Dropping cloud_user_credentials table failed: %s" % str( e ) ) + + try: + log.deboug( "Would drop cloud user credentials table." ) + #CloudUserCredentials_table.drop() + except Exception, e: + log.debug( "Dropping cloud_user_credentials table failed: %s" % str( e ) ) + + + + diff -r c28dd8c0a231 -r a620afab791d lib/galaxy/web/controllers/cloud.py --- a/lib/galaxy/web/controllers/cloud.py Wed Aug 26 17:34:58 2009 -0400 +++ b/lib/galaxy/web/controllers/cloud.py Thu Aug 27 17:36:03 2009 -0400 @@ -36,23 +36,23 @@ Render cloud main page (management of cloud resources) """ user = trans.get_user() - awsCredentials = trans.sa_session.query( model.StoredUserCredentials ) \ - .order_by( desc( model.StoredUserCredentials.c.update_time ) ) \ + cloudCredentials = trans.sa_session.query( model.CloudUserCredentials ) \ + .order_by( desc( model.CloudUserCredentials.c.update_time ) ) \ .all() - prevInstances = trans.sa_session.query( model.UserInstances ) \ + prevInstances = trans.sa_session.query( model.CloudInstance ) \ .filter_by( user=user, state="available" ) \ - .order_by( desc( model.UserInstances.c.create_time ) ) \ + .order_by( desc( model.CloudInstance.c.create_time ) ) \ .all() #TODO: diff between live and previous instances - liveInstances = trans.sa_session.query( model.UserInstances ) \ + liveInstances = trans.sa_session.query( model.CloudInstance ) \ .filter_by( user=user ) \ - .filter( or_(model.UserInstances.c.state=="running", model.UserInstances.c.state=="pending") ) \ - .order_by( desc( model.UserInstances.c.create_time ) ) \ + .filter( or_(model.CloudInstance.c.state=="running", model.CloudInstance.c.state=="pending") ) \ + .order_by( desc( model.CloudInstance.c.create_time ) ) \ .all() return trans.fill_template( "cloud/configure_cloud.mako", - awsCredentials = awsCredentials, + cloudCredentials = cloudCredentials, prevInstances = prevInstances, liveInstances = liveInstances ) @@ -60,9 +60,9 @@ @web.require_login( "use Galaxy cloud" ) def makeDefault( self, trans, id=None ): """ - Set current credentials as default to be used for acquiring cloud resources. + Set current credentials as default. """ - currentDefault = get_defalt_credentials (trans) + currentDefault = get_default_credentials (trans) if currentDefault: currentDefault.defaultCred = False @@ -177,8 +177,11 @@ return trans.response.send_redirect( url_for( controller='workflow' ) ) @web.expose - @web.require_login( "use Galaxy cloud instance" ) + @web.require_login( "start Galaxy cloud instance" ) def start( self, trans, id ): + """ + Start a new cloud resource instance + """ instance = get_instance( trans, id ) @@ -189,6 +192,9 @@ @web.expose @web.require_login( "stop Galaxy cloud instance" ) def stop( self, trans, id ): + """ + Stop a cloud resource instance + """ instance = get_instance( trans, id ) @@ -207,7 +213,7 @@ return self.list( trans ) @web.expose - @web.require_login( "delete Galaxy cloud instance" ) + @web.require_login( "add instance storage" ) def addStorage( self, trans, id ): instance = get_instance( trans, id ) @@ -224,33 +230,43 @@ """ user = trans.get_user() # TODO: If more images are made available, must add code to choose between those - ami = trans.app.model.CloudImages.filter( - trans.app.model.CloudImages.table.c.id==1).first() + mi = trans.app.model.CloudImage.filter( + trans.app.model.CloudImage.table.c.id==1).first() + inst_error = vol_error = None if instanceName: # Create new user configured instance try: if len( instanceName ) > 255: inst_error = "Instance name exceeds maximum allowable length." - elif trans.app.model.UserInstances.filter( - trans.app.model.UserInstances.table.c.name==instanceName ).first(): + elif trans.app.model.CloudInstance.filter( + trans.app.model.CloudInstance.table.c.name==instanceName ).first(): inst_error = "An instance with that name already exist." elif int( volSize ) > 1000: vol_error = "Volume size cannot exceed 1000GB. You must specify an integer between 1 and 1000." elif int( volSize ) < 1: vol_error = "Volume size cannot be less than 1GB. You must specify an integer between 1 and 1000." else: - instance = model.UserInstances() + instance = model.CloudInstance() instance.user = user instance.name = instanceName - instance.ami = ami.image_id - # Valid states include: "available", "running" or "pending" + instance.mi = mi.image_id + # TODO: get state from AWS - also, update state on page update + # Currently, valid states include: "available", "running" or "pending" instance.state = "available" - #TODO: include storage volume size code + # Capture storage related information + storage = model.CloudStorage() + storage.user = user + storage.size = volSize + log.debug("******** Size: %s" % storage.size ) + # TODO: get correct values from AWS + storage.volume_id = "made up" + storage.availability_zone = "avail zone" # Persist session = trans.sa_session session.save_or_update( instance ) + session.save_or_update( storage ) session.flush() # Log and display the management page trans.log_event( "User configured new cloud resource instance" ) @@ -258,7 +274,10 @@ return self.list( trans ) except ValueError: vol_error = "Volume size must be specified as an integer value only, between 1 and 1000." - + except AttributeError, ae: + inst_error = "No registered cloud images. You must contact administrator to add some before proceeding." + log.debug("AttributeError: %s " % str( ae ) ) + return trans.show_form( web.FormBuilder( web.url_for(), "Configure new instance", submit_text="Add" ) .add_text( "instanceName", "Instance name", value="Unnamed instance", error=inst_error ) @@ -273,12 +292,12 @@ if image_id: if len( image_id ) > 255: error = "Image ID name exceeds maximum allowable length." - elif trans.app.model.StoredUserCredentials.filter( - trans.app.model.CloudImages.table.c.image_id==image_id ).first(): + elif trans.app.model.CloudUserCredentials.filter( + trans.app.model.CloudImage.table.c.image_id==image_id ).first(): error = "Image with that ID is already registered." else: # Create new image - image = model.CloudImages() + image = model.CloudImage() image.image_id = image_id image.manifest = manifest # Persist @@ -327,19 +346,19 @@ @web.require_login( "add credentials" ) def add( self, trans, credName='', accessKey='', secretKey='', defaultCred=True ): """ - Add user's AWS credentials stored under name `credName`. + Add user's cloud credentials stored under name `credName`. """ user = trans.get_user() cred_error = accessKey_error = secretKey_error = None if credName: if len( credName ) > 255: cred_error = "Credentials name exceeds maximum allowable length." - elif trans.app.model.StoredUserCredentials.filter( - trans.app.model.StoredUserCredentials.table.c.name==credName ).first(): + elif trans.app.model.CloudUserCredentials.filter( + trans.app.model.CloudUserCredentials.table.c.name==credName ).first(): cred_error = "Credentials with that name already exist." else: # Create new user stored credentials - credentials = model.StoredUserCredentials() + credentials = model.CloudUserCredentials() credentials.name = credName credentials.user = user credentials.access_key = accessKey @@ -853,7 +872,7 @@ if not isinstance( id, int ): id = trans.security.decode_id( id ) - stored = trans.sa_session.query( model.StoredUserCredentials ).get( id ) + stored = trans.sa_session.query( model.CloudUserCredentials ).get( id ) if not stored: error( "Credentials not found" ) # Verify ownership @@ -865,13 +884,13 @@ # Looks good return stored -def get_defalt_credentials( trans, check_ownership=True ): +def get_default_credentials( trans, check_ownership=True ): """ Get a StoredUserCredntials from the database by 'default' setting, verifying ownership. """ user = trans.get_user() # Load credentials from database - stored = trans.sa_session.query( model.StoredUserCredentials ) \ + stored = trans.sa_session.query( model.CloudUserCredentials ) \ .filter_by (user=user, defaultCred=True) \ .first() @@ -879,11 +898,11 @@ def get_instance( trans, id, check_ownership=True ): """ - Get a UserInstances from the database by id, verifying ownership. + Get a CloudInstance from the database by id, verifying ownership. """ id = trans.security.decode_id( id ) - live = trans.sa_session.query( model.UserInstances ).get( id ) + live = trans.sa_session.query( model.CloudInstance ).get( id ) if not live: error( "Instance not found" ) # Verify ownership diff -r c28dd8c0a231 -r a620afab791d templates/cloud/configure_cloud.mako --- a/templates/cloud/configure_cloud.mako Wed Aug 26 17:34:58 2009 -0400 +++ b/templates/cloud/configure_cloud.mako Thu Aug 27 17:36:03 2009 -0400 @@ -20,7 +20,7 @@ <h2>Galaxy in the clouds</h2> -%if awsCredentials: +%if cloudCredentials: ## Manage user credentials <ul class="manage-table-actions"> <li> @@ -37,28 +37,28 @@ <th>Default</th> <th></th> </tr> - %for i, awsCredential in enumerate( awsCredentials ): + %for i, cloudCredential in enumerate( cloudCredentials ): <tr> <td> - ${awsCredential.name} + ${cloudCredential.name} <a id="cr-${i}-popup" class="popup-arrow" style="display: none;">▼</a> </td> ## Comment <td>${len(workflow.latest_workflow.steps)}</td> - ##<td>${str(awsCredential.update_time)[:19]}</td> + ##<td>${str(cloudCredential.update_time)[:19]}</td> <td> - ##${str(awsCredential.defaultCred)} + ##${str(cloudCredential.defaultCred)} <% - if awsCredential.defaultCred: + if cloudCredential.defaultCred: context.write('*') %> </td> <td> <div popupmenu="cr-${i}-popup"> - <a class="action-button" href="${h.url_for( action='view', id=trans.security.encode_id(awsCredential.id) )}">View</a> - <a class="action-button" href="${h.url_for( action='rename', id=trans.security.encode_id(awsCredential.id) )}">Rename</a> - <a class="action-button" href="${h.url_for( action='makeDefault', id=trans.security.encode_id(awsCredential.id) )}" target="_parent">Make default</a> - <a class="action-button" confirm="Are you sure you want to delete credentials '${awsCredential.name}'?" href="${h.url_for( action='delete', id=trans.security.encode_id(awsCredential.id) )}">Delete</a> + <a class="action-button" href="${h.url_for( action='view', id=trans.security.encode_id(cloudCredential.id) )}">View</a> + <a class="action-button" href="${h.url_for( action='rename', id=trans.security.encode_id(cloudCredential.id) )}">Rename</a> + <a class="action-button" href="${h.url_for( action='makeDefault', id=trans.security.encode_id(cloudCredential.id) )}" target="_parent">Make default</a> + <a class="action-button" confirm="Are you sure you want to delete credentials '${cloudCredential.name}'?" href="${h.url_for( action='delete', id=trans.security.encode_id(cloudCredential.id) )}">Delete</a> </div> </td> </tr> @@ -97,7 +97,7 @@ ${liveInstance.name} <a id="li-${i}-popup" class="popup-arrow" style="display: none;">▼</a> </td> - <td>${str(liveInstance.name)}</td> <!--TODO:Replace with vol size once available--> + <td>${str(liveInstance.size)}</td> <!--TODO:Replace with vol size once available--> <td>${str(liveInstance.state)}</td> <td> ${str(liveInstance.launch_time)[:16]} @@ -161,7 +161,7 @@ <a id="pi-${i}-popup" class="popup-arrow" style="display: none;">▼</a> </td> ## Comment <td>${len(workflow.latest_workflow.steps)}</td> - <td>${str(prevInstance.name)}</td> <!-- TODO: Change to show vol size once available--> + <td>${str(prevInstance.CloudStorage.size)}</td> <!-- TODO: Change to show vol size once available--> <td>${str(prevInstance.state)}</td> <td> <div popupmenu="pi-${i}-popup">