details: http://www.bx.psu.edu/hg/galaxy/rev/a47ad6a21a5d changeset: 3053:a47ad6a21a5d user: Enis Afgan <afgane@gmail.com> date: Tue Aug 25 17:42:34 2009 -0400 description: Added cloud resource management tables and some code. diffstat: lib/galaxy/model/__init__.py | 16 ++ lib/galaxy/model/mapping.py | 40 +++++ lib/galaxy/model/migrate/versions/0014_cloud_tables.py | 68 +++++++++ lib/galaxy/model/migrate/versions/0014_credentials_table.py | 31 ---- lib/galaxy/web/controllers/cloud.py | 88 ++++++++++++- templates/cloud/configure_cloud.mako | 118 +++++++++++++++- 6 files changed, 317 insertions(+), 44 deletions(-) diffs (458 lines): diff -r 6b3f453078c4 -r a47ad6a21a5d lib/galaxy/model/__init__.py --- a/lib/galaxy/model/__init__.py Mon Aug 24 17:46:22 2009 -0400 +++ b/lib/galaxy/model/__init__.py Tue Aug 25 17:42:34 2009 -0400 @@ -932,6 +932,22 @@ self.galaxy_session = galaxy_session self.history = history +class UserInstances( object ): + def __init__( self ): + self.id = None + self.user = None + self.name = None + self.instance_id = None + self.state = None + self.public_dns = None + self.availability_zone = None + +class CloudImages( object ): + def __init__( self ): + self.id = None + self.instance_id = None + self.state = None + class StoredUserCredentials( object ): def __init__( self ): self.id = None diff -r 6b3f453078c4 -r a47ad6a21a5d lib/galaxy/model/mapping.py --- a/lib/galaxy/model/mapping.py Mon Aug 24 17:46:22 2009 -0400 +++ b/lib/galaxy/model/mapping.py Tue Aug 25 17:42:34 2009 -0400 @@ -379,6 +379,37 @@ Column( "session_id", Integer, ForeignKey( "galaxy_session.id" ), index=True ), Column( "history_id", Integer, ForeignKey( "history.id" ), index=True ) ) + + + + +# *************************************************************************** +# *************************** Cloud tables*********************************** +# *************************************************************************** +UserInstances.table = Table( "user_instances", metadata, + Column( "id", Integer, primary_key=True ), + Column( "create_time", DateTime, default=now ), + Column( "launch_time", DateTime, onupdate=now ), + Column( "user_id", Integer, ForeignKey( "galaxy_user.id" ), index=True, nullable=False ), + Column( "name", TEXT ), + Column( "reservation_id", TEXT ), + Column( "instance_id", TEXT ), + Column( "ami", TEXT, ForeignKey( "cloud_images.image_id" ), nullable=False ), + Column( "state", TEXT ), + Column( "public_dns", TEXT ), + Column( "private_dns", TEXT ), + Column( "keypair_fingerprint", TEXT ), + Column( "keypair_material", TEXT ), + Column( "availability_zone", TEXT ) ) + +CloudImages.table = Table( "cloud_images", metadata, + Column( "id", Integer, primary_key=True ), + Column( "create_time", DateTime, default=now ), + Column( "upadte_time", DateTime, default=now, onupdate=now ), + Column( "image_id", TEXT, nullable=False ), + Column( "manifest", TEXT ), + Column( "state", TEXT ) ) + StoredUserCredentials.table = Table( "stored_user_credentials", metadata, Column( "id", Integer, primary_key=True ), Column( "create_time", DateTime, default=now ), @@ -895,9 +926,18 @@ output_step=relation( WorkflowStep, backref="output_connections", cascade="all", primaryjoin=( WorkflowStepConnection.table.c.output_step_id == WorkflowStep.table.c.id ) ) ) ) +# vvvvvvvvvvvvvvvv Start cloud table mappings vvvvvvvvvvvvvvvv +assign_mapper( context, UserInstances, UserInstances.table, + properties=dict( user=relation( User ), + cloud_image=relation( CloudImages ) + ) ) + +assign_mapper( context, CloudImages, CloudImages.table ) + assign_mapper( context, StoredUserCredentials, StoredUserCredentials.table, properties=dict( user=relation( User) ) ) +# ^^^^^^^^^^^^^^^ End cloud table mappings ^^^^^^^^^^^^^^^^^^ assign_mapper( context, StoredWorkflow, StoredWorkflow.table, properties=dict( user=relation( User ), diff -r 6b3f453078c4 -r a47ad6a21a5d lib/galaxy/model/migrate/versions/0014_cloud_tables.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/lib/galaxy/model/migrate/versions/0014_cloud_tables.py Tue Aug 25 17:42:34 2009 -0400 @@ -0,0 +1,68 @@ +from sqlalchemy import * +from migrate import * + +import datetime +now = datetime.datetime.utcnow + +# Need our custom types, but don't import anything else from model +from galaxy.model.custom_types import * + +import logging +log = logging.getLogger( __name__ ) + +metadata = MetaData( migrate_engine ) + +Credentials_table = Table( "stored_user_credentials", metadata, + Column( "id", Integer, primary_key=True ), + Column( "create_time", DateTime, default=now ), + Column( "update_time", DateTime, default=now, onupdate=now ), + Column( "user_id", Integer, ForeignKey( "galaxy_user.id" ), index=True, nullable=False ), + Column( "name", TEXT), + Column( "access_key", TEXT), + Column( "secret_key", TEXT), + Column( "defaultCred", Boolean, default=False ) ) + +UserInstances_table = Table ( "user_instances", metadata, + Column( "id", Integer, primary_key=True ), + Column( "create_time", DateTime, default=now ), + Column( "launch_time", DateTime, onupdate=now ), + Column( "user_id", Integer, ForeignKey( "galaxy_user.id" ), index=True, nullable=False ), + Column( "name", TEXT ), + Column( "reservation_id", TEXT ), + Column( "instance_id", TEXT ), + Column( "ami", TEXT, ForeignKey( "cloud_images.image_id" ), nullable=False ), + Column( "state", TEXT ), + Column( "public_dns", TEXT ), + Column( "private_dns", TEXT ), + Column( "keypair_fingerprint", TEXT ), + Column( "keypair_material", TEXT ), + Column( "availability_zone", TEXT ) ) + +CloudImages_table = Table( "cloud_images", metadata, + Column( "id", Integer, primary_key=True ), + Column( "create_time", DateTime, default=now ), + Column( "upadte_time", DateTime, default=now, onupdate=now ), + Column( "image_id", TEXT, nullable=False ), + Column( "manifest", TEXT ), + Column( "state", TEXT ) ) + +def upgrade(): + metadata.reflect() + Credentials_table.create() + UserInstances_table.create() + try: + CloudImages_table.create() + except Exception, e: + log.debug( "Creating CloudImages table failed. Table probably exists already." ) + +def downgrade(): + metadata.reflect() + try: + Credentials_table.drop() + except Exception, e: + log.debug( "Dropping stored_user_credentials table failed: %s" % str( e ) ) + + try: + UserInstances_table.drop() + except Exception, e: + log.debug( "Dropping user_instances table failed: %s" % str( e ) ) diff -r 6b3f453078c4 -r a47ad6a21a5d lib/galaxy/model/migrate/versions/0014_credentials_table.py --- a/lib/galaxy/model/migrate/versions/0014_credentials_table.py Mon Aug 24 17:46:22 2009 -0400 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,31 +0,0 @@ -from sqlalchemy import * -from migrate import * - -import datetime -now = datetime.datetime.utcnow - -# Need our custom types, but don't import anything else from model -from galaxy.model.custom_types import * - -import logging -log = logging.getLogger( __name__ ) - -metadata = MetaData( migrate_engine ) - -Credentials_table = Table( "stored_user_credentials", metadata, - Column( "id", Integer, primary_key=True ), - Column( "create_time", DateTime, default=now ), - Column( "update_time", DateTime, default=now, onupdate=now ), - Column( "user_id", Integer, ForeignKey( "galaxy_user.id" ), index=True, nullable=False ), - Column( "name", TEXT), - Column( "access_key", TEXT), - Column( "secret_key", TEXT), - Column( "defaultCred", Boolean, default=False ) ) - -def upgrade(): - metadata.reflect() - Credentials_table.create() - -def downgrade(): - metadata.reflect() - Credentials_table.drop() diff -r 6b3f453078c4 -r a47ad6a21a5d lib/galaxy/web/controllers/cloud.py --- a/lib/galaxy/web/controllers/cloud.py Mon Aug 24 17:46:22 2009 -0400 +++ b/lib/galaxy/web/controllers/cloud.py Tue Aug 25 17:42:34 2009 -0400 @@ -36,12 +36,17 @@ Render cloud main page (management of cloud resources) """ user = trans.get_user() - awsCredentials = trans.sa_session.query ( model.StoredUserCredentials ) \ + awsCredentials = trans.sa_session.query( model.StoredUserCredentials ) \ .order_by( desc( model.StoredUserCredentials.c.update_time ) ) \ .all() + prevInstances = trans.sa_session.query( model.UserInstances ) \ + .order_by( desc( model.UserInstances.c.create_time ) ) \ + .all() #TODO: diff between live and previous instances + return trans.fill_template( "cloud/configure_cloud.mako", - awsCredentials = awsCredentials ) + awsCredentials = awsCredentials, + prevInstances = prevInstances ) @web.expose @web.require_login( "use Galaxy cloud" ) @@ -162,7 +167,82 @@ session.flush() # Redirect to load galaxy frames. return trans.response.send_redirect( url_for( controller='workflow' ) ) - + + + @web.expose + @web.require_login( "use Galaxy cloud" ) + def configureNew( self, trans, instanceName='', volSize=''): + """ + Configure and add new cloud instance to user's instance pool + """ + user = trans.get_user() + ami = trans.app.model.CloudImages.filter( + trans.app.model.CloudImages.table.c.id==1).first() + log.debug(ami.image_id) + inst_error = vol_error = None + if instanceName: + # Create new user configured instance + if len( instanceName ) > 255: + inst_error = "Instance name exceeds maximum allowable length." + elif trans.app.model.UserInstances.filter( + trans.app.model.UserInstances.table.c.name==instanceName ).first(): + inst_error = "An instance with that name already exist." + elif int( volSize ) > 1000: + vol_error = "Volume size cannot exceed 1000GB. You must specify an integer between 1 and 1000." + elif int( volSize ) < 1: + vol_error = "Volume size cannot be less than 1GB. You must specify an integer between 1 and 1000." + else: + instance = model.UserInstances() + instance.user_id = user + instance.name = instanceName + instance.ami = ami.image_id + #TODO: include storage volume size code + # Persist + session = trans.sa_session + session.save_or_update( instance ) + session.flush() + # Log and display the management page + trans.log_event( "User configured new cloud resource instance" ) + trans.set_message( "Instance '%s' configured" % credentials.name ) + return self.list( trans ) + + return trans.show_form( + web.FormBuilder( web.url_for(), "Configure new instance", submit_text="Add" ) + .add_text( "instanceName", "Instance name", value="Unnamed instance", error=inst_error ) + .add_text( "volSize", "Permanent storage size (1GB - 1000GB)", value='', error=vol_error ) ) + + @web.expose + @web.require_login( "add a cloud image" ) + #@web.require_admin + def addNewImage( self, trans, image_id='', manifest='', state=None ): + error = None + if image_id: + if len( image_id ) > 255: + error = "Image ID name exceeds maximum allowable length." + elif trans.app.model.StoredUserCredentials.filter( + trans.app.model.CloudImages.table.c.image_id==image_id ).first(): + error = "Image with that ID is already registered." + else: + # Create new image + image = model.CloudImages() + image.image_id = image_id + image.manifest = manifest + # Persist + session = trans.sa_session + session.save_or_update( image ) + session.flush() + # Log and display the management page + trans.log_event( "New cloud image added: '%s'" % image.image_id ) + trans.set_message( "Cloud image '%s' added." % image.image_id ) + if state: + image.state= state + return self.list( trans ) + + return trans.show_form( + web.FormBuilder( web.url_for(), "Add new cloud image", submit_text="Add" ) + .add_text( "image_id", "Image ID", value='', error=error ) + .add_text( "manifest", "Manifest", value='', error=error ) ) + @web.expose @web.require_login( "use Galaxy cloud" ) def rename( self, trans, id, new_name=None ): @@ -178,7 +258,7 @@ @web.expose @web.require_login( "add credentials" ) - def add( self, trans, credName='', accessKey='', secretKey='', defaultCred=False ): + def add( self, trans, credName='', accessKey='', secretKey='', defaultCred=True ): """ Add user's AWS credentials stored under name `credName`. """ diff -r 6b3f453078c4 -r a47ad6a21a5d templates/cloud/configure_cloud.mako --- a/templates/cloud/configure_cloud.mako Mon Aug 24 17:46:22 2009 -0400 +++ b/templates/cloud/configure_cloud.mako Tue Aug 25 17:42:34 2009 -0400 @@ -21,6 +21,7 @@ <h2>Galaxy in the clouds</h2> %if awsCredentials: + ## Manage user credentials <ul class="manage-table-actions"> <li> <a class="action-button" href="${h.url_for( action='add' )}"> @@ -45,12 +46,11 @@ ## Comment <td>${len(workflow.latest_workflow.steps)}</td> ##<td>${str(awsCredential.update_time)[:19]}</td> <td> - ${str(awsCredential.defaultCred)} + ##${str(awsCredential.defaultCred)} <% - c=str(awsCredential.defaultCred) + if awsCredential.defaultCred: + context.write('*') %> - - ##${dflt(cred=c)} </td> <td> @@ -64,6 +64,105 @@ </tr> %endfor </table> + + ## ***************************************************** + ## Manage live instances + <p /> + <h2>Manage cloud instances</h2> + <ul class="manage-table-actions"> + <li> + <a class="action-button" href="${h.url_for( action='configureNew' )}"> + <img src="${h.url_for('/static/images/silk/add.png')}" /> + <span>Configure new instance</span> + </a> + </li> + </ul> + + <table class="mange-table colored" border="0" cellspacing="0" cellpadding="0" width="100%"> + <tr class="header"> + <th>Live instances</th> + <th>Alive since</th> + <th></th> + </tr> + %for i, awsCredential in enumerate( awsCredentials ): + <tr> + <td> + ${awsCredential.name} + <a id="wf-${i}-popup" class="popup-arrow" style="display: none;">▼</a> + </td> + <td> + ##${str(awsCredential.update_time)[:19]} + <% + from datetime import datetime + from datetime import timedelta + + # DB stores all times in GMT, so adjust for difference (4 hours) + adjustedStarttime = awsCredential.update_time - timedelta(hours=4) + + # (NOT CURRENTLY USED BLOCK OF CODE) Calculate time difference from now + delta = datetime.now() - adjustedStarttime + #context.write( str(datetime.utcnow() ) ) + #context.write( str(delta) ) + + # This is where current time and since duration is calculated + context.write( str( awsCredential.update_time ) ) + context.write( ' UTC (' ) + context.write( str(h.date.distance_of_time_in_words (awsCredential.update_time, h.date.datetime.utcnow() ) ) ) + %>) + </td> + + <td> + <div popupmenu="wf-${i}-popup"> + <a class="action-button" href="${h.url_for( action='view', id=trans.security.encode_id(awsCredential.id) )}">View</a> + <a class="action-button" href="${h.url_for( action='rename', id=trans.security.encode_id(awsCredential.id) )}">Rename</a> + <a class="action-button" href="${h.url_for( action='makeDefault', id=trans.security.encode_id(awsCredential.id) )}" target="_parent">Make default</a> + <a class="action-button" confirm="Are you sure you want to delete workflow '${awsCredential.name}'?" href="${h.url_for( action='delete', id=trans.security.encode_id(awsCredential.id) )}">Delete</a> + </div> + </td> + </tr> + %endfor + </table> + + ## ***************************************************** + ## Manage previously configured instances + <p /> <p /> + <table class="mange-table colored" border="0" cellspacing="0" cellpadding="0" width="100%"> + <tr class="header"> + <th>Previously configured instances</th> + <th>Action</th> + <th></th> + </tr> + + %if prevInstances: + %for i, prevInstance in enumerate( prevInstances ): + <tr> + <td> + ${prevInstance.name} + <a id="wf-${i}-popup" class="popup-arrow" style="display: none;">▼</a> + </td> + ## Comment <td>${len(workflow.latest_workflow.steps)}</td> + ##<td>${str(awsCredential.update_time)[:19]}</td> + <td> + <a class="action-button" href="${h.url_for( action='start', id=trans.security.encode_id(awsCredential.id) )}">Start</a> + </td> + + <td> + <div popupmenu="wf-${i}-popup"> + <a class="action-button" href="${h.url_for( action='view', id=trans.security.encode_id(awsCredential.id) )}">View</a> + <a class="action-button" href="${h.url_for( action='rename', id=trans.security.encode_id(awsCredential.id) )}">Rename</a> + <a class="action-button" href="${h.url_for( action='makeDefault', id=trans.security.encode_id(awsCredential.id) )}" target="_parent">Make default</a> + <a class="action-button" confirm="Are you sure you want to delete workflow '${awsCredential.name}'?" href="${h.url_for( action='delete', id=trans.security.encode_id(awsCredential.id) )}">Delete</a> + </div> + </td> + </tr> + %endfor + %else: + <tr> + <td>You have no previously configured instances.</td> + </tr> + %endif + </table> + %else: You have no AWS credentials associated with your Galaxy account: <a class="action-button" href="${h.url_for( action='add' )}"> @@ -76,8 +175,9 @@ %endif -<%def name="dflt(cred)"> - %if cred: - default - %endif -</%def> +<p /><br /> +<ul class="manage-table-actions"> + <li> + <a class="action-button" href="${h.url_for( action='addNewImage' )}"><span>Add new image</span></a> + </li> +</ul> \ No newline at end of file