details: http://www.bx.psu.edu/hg/galaxy/rev/f87f1eed6e0c changeset: 3057:f87f1eed6e0c user: Enis Afgan <afgane@gmail.com> date: Fri Aug 28 17:45:40 2009 -0400 description: Cloud UI & DB connected and interfaced. diffstat: lib/galaxy/model/__init__.py | 21 ++- lib/galaxy/model/mapping.py | 33 ++++- lib/galaxy/model/migrate/versions/0014_cloud_tables.py | 48 +++++-- lib/galaxy/web/controllers/cloud.py | 138 ++++++++++++++++------ templates/cloud/configure_cloud.mako | 14 +- 5 files changed, 183 insertions(+), 71 deletions(-) diffs (540 lines): diff -r a620afab791d -r f87f1eed6e0c lib/galaxy/model/__init__.py --- a/lib/galaxy/model/__init__.py Thu Aug 27 17:36:03 2009 -0400 +++ b/lib/galaxy/model/__init__.py Fri Aug 28 17:45:40 2009 -0400 @@ -931,7 +931,18 @@ def __init__( self, galaxy_session, history ): self.galaxy_session = galaxy_session self.history = history - + +class CloudImage( object ): + def __init__( self ): + self.id = None + self.instance_id = None + self.state = None + +class UCI( object ): + def __init__( self ): + self.id = None + self.user = None + class CloudInstance( object ): def __init__( self ): self.id = None @@ -944,13 +955,7 @@ self.public_dns = None self.availability_zone = None -class CloudImage( object ): - def __init__( self ): - self.id = None - self.instance_id = None - self.state = None - -class CloudStorage( object ): +class CloudStore( object ): def __init__( self ): self.id = None self.volume_id = None diff -r a620afab791d -r f87f1eed6e0c lib/galaxy/model/mapping.py --- a/lib/galaxy/model/mapping.py Thu Aug 27 17:36:03 2009 -0400 +++ b/lib/galaxy/model/mapping.py Fri Aug 28 17:45:40 2009 -0400 @@ -394,29 +394,42 @@ Column( "manifest", TEXT ), Column( "state", TEXT ) ) +""" UserConfiguredInstance (UCI) table """ +UCI.table = Table( "uci", metadata, + Column( "id", Integer, primary_key=True ), + Column( "create_time", DateTime, default=now ), + Column( "update_time", DateTime, default=now, onupdate=now ), + Column( "user_id", Integer, ForeignKey( "galaxy_user.id" ), index=True, nullable=False ), + Column( "name", TEXT ), + Column( "state", TEXT ), + Column( "total_size", Integer ), + Column( "launch_time", DateTime ) ) + CloudInstance.table = Table( "cloud_instance", metadata, Column( "id", Integer, primary_key=True ), Column( "create_time", DateTime, default=now ), Column( "update_time", DateTime, default=now, onupdate=now ), Column( "launch_time", DateTime ), + Column( "stop_time", DateTime ), Column( "user_id", Integer, ForeignKey( "galaxy_user.id" ), index=True, nullable=False ), - Column( "name", TEXT ), + Column( "uci_id", Integer, ForeignKey( "uci.id" ), index=True ), Column( "type", TEXT ), Column( "reservation_id", TEXT ), Column( "instance_id", TEXT ), - Column( "mi", TEXT, ForeignKey( "cloud_image.image_id" ), index=True, nullable=False ), + Column( "mi_id", TEXT, ForeignKey( "cloud_image.image_id" ), index=True, nullable=False ), Column( "state", TEXT ), Column( "public_dns", TEXT ), Column( "private_dns", TEXT ), Column( "keypair_name", TEXT ), Column( "availability_zone", TEXT ) ) -CloudStorage.table = Table( "cloud_storage", metadata, +CloudStore.table = Table( "cloud_store", metadata, Column( "id", Integer, primary_key=True ), Column( "create_time", DateTime, default=now ), Column( "update_time", DateTime, default=now, onupdate=now ), Column( "attach_time", DateTime ), Column( "user_id", Integer, ForeignKey( "galaxy_user.id" ), index=True, nullable=False ), + Column( "uci_id", Integer, ForeignKey( "uci.id" ), index=True, nullable=False ), Column( "volume_id", TEXT, nullable=False ), Column( "size", Integer, nullable=False ), Column( "availability_zone", TEXT, nullable=False ), @@ -947,19 +960,25 @@ # ************************************************************ assign_mapper( context, CloudImage, CloudImage.table ) +assign_mapper( context, UCI, UCI.table, + properties=dict( user=relation( User ), + instance=relation( CloudInstance, backref='uci' ), + store=relation( CloudStore, backref='uci' ) + ) ) + assign_mapper( context, CloudInstance, CloudInstance.table, properties=dict( user=relation( User ), image=relation( CloudImage ) ) ) -assign_mapper( context, CloudStorage, CloudStorage.table, +assign_mapper( context, CloudStore, CloudStore.table, properties=dict( user=relation( User ), - instance=relation( CloudInstance, backref='cloud_instance' ) + i=relation( CloudInstance ) ) ) assign_mapper( context, CloudUserCredentials, CloudUserCredentials.table, - properties=dict( user=relation( User) ) - ) + properties=dict( user=relation( User) + ) ) # ^^^^^^^^^^^^^^^ End cloud table mappings ^^^^^^^^^^^^^^^^^^ assign_mapper( context, StoredWorkflow, StoredWorkflow.table, diff -r a620afab791d -r f87f1eed6e0c lib/galaxy/model/migrate/versions/0014_cloud_tables.py --- a/lib/galaxy/model/migrate/versions/0014_cloud_tables.py Thu Aug 27 17:36:03 2009 -0400 +++ b/lib/galaxy/model/migrate/versions/0014_cloud_tables.py Fri Aug 28 17:45:40 2009 -0400 @@ -20,29 +20,41 @@ Column( "manifest", TEXT ), Column( "state", TEXT ) ) +UCI_table = Table( "uci", metadata, + Column( "id", Integer, primary_key=True ), + Column( "create_time", DateTime, default=now ), + Column( "update_time", DateTime, default=now, onupdate=now ), + Column( "user_id", Integer, ForeignKey( "galaxy_user.id" ), index=True, nullable=False ), + Column( "name", TEXT ), + Column( "state", TEXT ), + Column( "total_size", Integer ), + Column( "launch_time", DateTime ) ) + CloudInstance_table = Table( "cloud_instance", metadata, Column( "id", Integer, primary_key=True ), Column( "create_time", DateTime, default=now ), Column( "update_time", DateTime, default=now, onupdate=now ), Column( "launch_time", DateTime ), + Column( "stop_time", DateTime ), Column( "user_id", Integer, ForeignKey( "galaxy_user.id" ), index=True, nullable=False ), - Column( "name", TEXT ), + Column( "uci_id", Integer, ForeignKey( "uci.id" ), index=True ), Column( "type", TEXT ), Column( "reservation_id", TEXT ), Column( "instance_id", TEXT ), - Column( "mi", TEXT, ForeignKey( "cloud_image.image_id" ), index=True, nullable=False ), + Column( "mi_id", TEXT, ForeignKey( "cloud_image.image_id" ), index=True, nullable=False ), Column( "state", TEXT ), Column( "public_dns", TEXT ), Column( "private_dns", TEXT ), Column( "keypair_name", TEXT ), Column( "availability_zone", TEXT ) ) -CloudStorage_table = Table( "cloud_storage", metadata, +CloudStore_table = Table( "cloud_store", metadata, Column( "id", Integer, primary_key=True ), Column( "create_time", DateTime, default=now ), Column( "update_time", DateTime, default=now, onupdate=now ), Column( "attach_time", DateTime ), Column( "user_id", Integer, ForeignKey( "galaxy_user.id" ), index=True, nullable=False ), + Column( "uci_id", Integer, ForeignKey( "uci.id" ), index=True, nullable=False ), Column( "volume_id", TEXT, nullable=False ), Column( "size", Integer, nullable=False ), Column( "availability_zone", TEXT, nullable=False ), @@ -62,20 +74,26 @@ Column( "secret_key", TEXT), Column( "defaultCred", Boolean, default=False) ) + def upgrade(): metadata.reflect() - CloudUserCredentials_table.create() - CloudInstance_table.create() - CloudStorage_table.create() try: CloudImage_table.create() except Exception, e: log.debug( "Creating cloud_image table failed. Table probably exists already." ) - + UCI_table.create() + CloudInstance_table.create() + CloudStore_table.create() + try: + CloudUserCredentials_table.create() + except Exception, e: + log.debug( "Creating cloud_image table failed. Table probably exists already." ) + def downgrade(): metadata.reflect() try: - CloudImage_table.drop() + log.deboug( "Would drop cloud_image table." ) + #CloudImage_table.drop() #Enable before putting final version except Exception, e: log.debug( "Dropping cloud_image table failed: %s" % str( e ) ) @@ -85,16 +103,20 @@ log.debug( "Dropping cloud_instance table failed: %s" % str( e ) ) try: - CloudStorage_table.drop() + CloudStore_table.drop() + except Exception, e: + log.debug( "Dropping cloud_store table failed: %s" % str( e ) ) + + try: + log.deboug( "Would drop cloud_user_credentials table." ) + #CloudUserCredentials_table.drop() #Enable before putting final version except Exception, e: log.debug( "Dropping cloud_user_credentials table failed: %s" % str( e ) ) try: - log.deboug( "Would drop cloud user credentials table." ) - #CloudUserCredentials_table.drop() + UCI_table.drop() except Exception, e: - log.debug( "Dropping cloud_user_credentials table failed: %s" % str( e ) ) - + log.debug( "Dropping UCI table failed: %s" % str( e ) ) diff -r a620afab791d -r f87f1eed6e0c lib/galaxy/web/controllers/cloud.py --- a/lib/galaxy/web/controllers/cloud.py Thu Aug 27 17:36:03 2009 -0400 +++ b/lib/galaxy/web/controllers/cloud.py Fri Aug 28 17:45:40 2009 -0400 @@ -14,6 +14,7 @@ from galaxy.workflow.modules import * from galaxy.model.mapping import desc from galaxy.model.orm import * +from datetime import datetime # Required for Cloud tab import galaxy.eggs @@ -40,15 +41,21 @@ .order_by( desc( model.CloudUserCredentials.c.update_time ) ) \ .all() - prevInstances = trans.sa_session.query( model.CloudInstance ) \ + prevInstances = trans.sa_session.query( model.UCI ) \ .filter_by( user=user, state="available" ) \ - .order_by( desc( model.CloudInstance.c.create_time ) ) \ + .order_by( desc( model.UCI.c.update_time ) ) \ .all() #TODO: diff between live and previous instances liveInstances = trans.sa_session.query( model.CloudInstance ) \ .filter_by( user=user ) \ .filter( or_(model.CloudInstance.c.state=="running", model.CloudInstance.c.state=="pending") ) \ - .order_by( desc( model.CloudInstance.c.create_time ) ) \ + .order_by( desc( model.CloudInstance.c.launch_time ) ) \ + .all() + + liveInstances = trans.sa_session.query( model.UCI ) \ + .filter_by( user=user ) \ + .filter( or_(model.UCI.c.state=="running", model.UCI.c.state=="pending") ) \ + .order_by( desc( model.UCI.c.launch_time ) ) \ .all() return trans.fill_template( "cloud/configure_cloud.mako", @@ -178,15 +185,35 @@ @web.expose @web.require_login( "start Galaxy cloud instance" ) - def start( self, trans, id ): + def start( self, trans, id, size='small' ): """ Start a new cloud resource instance """ - instance = get_instance( trans, id ) + user = trans.get_user() + mi = get_mi( trans, size ) + uci = get_uci( trans, id ) + stores = get_stores( trans, uci ) #TODO: handle list! + instance = model.CloudInstance() + instance.user = user + instance.image = mi + instance.uci = uci + # TODO: get real value from AWS + instance.state = "pending" + uci.state = instance.state + uci.launch_time = datetime.utcnow() + instance.launch_time = datetime.utcnow() + instance.availability_zone = stores.availability_zone + instance.type = size - error( "Starting instance '%s' is not supported yet." % instance.name ) + # Persist + session = trans.sa_session + session.save_or_update( instance ) + session.flush() + #error( "Starting instance '%s' is not supported yet." % uci.name ) + trans.log_event( "User started cloud instance '%s'" % uci.name ) + trans.set_message( "Galaxy instance '%s' started." % uci.name ) return self.list( trans ) @web.expose @@ -195,17 +222,27 @@ """ Stop a cloud resource instance """ - instance = get_instance( trans, id ) + uci = get_uci( trans, id ) + instances = get_instances( trans, uci ) #TODO: handle list! - - error( "Stopping instance '%s' is not supported yet." % instance.name ) + instances.state = 'done' + instances.stop_time = datetime.utcnow() + uci.state = 'available' + uci.launch_time = None + # Persist + session = trans.sa_session + session.save_or_update( uci ) + session.save_or_update( instances ) + session.flush() + trans.log_event( "User stopped cloud instance '%s'" % uci.name ) + trans.set_message( "Galaxy instance '%s' stopped." % uci.name ) return self.list( trans ) @web.expose @web.require_login( "delete Galaxy cloud instance" ) def deleteInstance( self, trans, id ): - instance = get_instance( trans, id ) + instance = get_uci( trans, id ) error( "Deleting instance '%s' is not supported yet." % instance.name ) @@ -215,7 +252,7 @@ @web.expose @web.require_login( "add instance storage" ) def addStorage( self, trans, id ): - instance = get_instance( trans, id ) + instance = get_uci( trans, id ) error( "Adding storage to instance '%s' is not supported yet." % instance.name ) @@ -229,48 +266,42 @@ Configure and add new cloud instance to user's instance pool """ user = trans.get_user() - # TODO: If more images are made available, must add code to choose between those - mi = trans.app.model.CloudImage.filter( - trans.app.model.CloudImage.table.c.id==1).first() - - inst_error = vol_error = None if instanceName: # Create new user configured instance try: if len( instanceName ) > 255: inst_error = "Instance name exceeds maximum allowable length." - elif trans.app.model.CloudInstance.filter( - trans.app.model.CloudInstance.table.c.name==instanceName ).first(): + elif trans.app.model.UCI.filter( + trans.app.model.UCI.table.c.name==instanceName ).first(): inst_error = "An instance with that name already exist." elif int( volSize ) > 1000: vol_error = "Volume size cannot exceed 1000GB. You must specify an integer between 1 and 1000." elif int( volSize ) < 1: vol_error = "Volume size cannot be less than 1GB. You must specify an integer between 1 and 1000." else: - instance = model.CloudInstance() - instance.user = user - instance.name = instanceName - instance.mi = mi.image_id - # TODO: get state from AWS - also, update state on page update - # Currently, valid states include: "available", "running" or "pending" - instance.state = "available" - # Capture storage related information - storage = model.CloudStorage() + # Capture user configured instance information + uci = model.UCI() + uci.name = instanceName + uci.user= user + uci.state = "available" # Valid states include: "available", "running" or "pending" + uci.total_size = volSize # This is OK now because new instance is being created. + # Capture store related information + storage = model.CloudStore() storage.user = user + storage.uci = uci storage.size = volSize - log.debug("******** Size: %s" % storage.size ) # TODO: get correct values from AWS storage.volume_id = "made up" storage.availability_zone = "avail zone" # Persist session = trans.sa_session - session.save_or_update( instance ) + session.save_or_update( uci ) session.save_or_update( storage ) session.flush() # Log and display the management page - trans.log_event( "User configured new cloud resource instance" ) - trans.set_message( "Instance '%s' configured" % instance.name ) + trans.log_event( "User configured new cloud instance" ) + trans.set_message( "New Galaxy instance '%s' configured." % uci.name ) return self.list( trans ) except ValueError: vol_error = "Volume size must be specified as an integer value only, between 1 and 1000." @@ -332,7 +363,7 @@ @web.expose @web.require_login( "use Galaxy cloud" ) def renameInstance( self, trans, id, new_name=None ): - instance = get_instance( trans, id ) + instance = get_uci( trans, id ) if new_name is not None: instance.name = new_name trans.sa_session.flush() @@ -398,7 +429,7 @@ """ View details about running instance """ - instance = get_instance( trans, id ) + instance = get_uci( trans, id ) log.debug ( instance.name ) return trans.fill_template( "cloud/viewInstance.mako", @@ -896,15 +927,15 @@ return stored -def get_instance( trans, id, check_ownership=True ): +def get_uci( trans, id, check_ownership=True ): """ - Get a CloudInstance from the database by id, verifying ownership. + Get a UCI from the database by id, verifying ownership. """ id = trans.security.decode_id( id ) - live = trans.sa_session.query( model.CloudInstance ).get( id ) + live = trans.sa_session.query( model.UCI ).get( id ) if not live: - error( "Instance not found" ) + error( "Galaxy instance not found." ) # Verify ownership user = trans.get_user() if not user: @@ -914,6 +945,39 @@ # Looks good return live +def get_mi( trans, size='small' ): + """ + Get appropriate machine image (mi) based on instance size. + TODO: Dummy method - need to implement logic + For valid sizes, see http://aws.amazon.com/ec2/instance-types/ + """ + return trans.app.model.CloudImage.filter( + trans.app.model.CloudImage.table.c.id==1).first() + +def get_stores( trans, uci ): + """ + Get store objects/tables that are connected to uci object/table + """ + user = trans.get_user() + stores = trans.sa_session.query( model.CloudStore ) \ + .filter_by( user=user, uci_id=uci.id ) \ + .first() + #.all() #TODO: return all but need to edit calling method(s) to handle list + + return stores + +def get_instances( trans, uci ): + """ + Get instance objects/tables that are connected to uci object/table + """ + user = trans.get_user() + instances = trans.sa_session.query( model.CloudInstance ) \ + .filter_by( user=user, uci_id=uci.id ) \ + .first() + #.all() #TODO: return all but need to edit calling method(s) to handle list + + return instances + def attach_ordered_steps( workflow, steps ): ordered_steps = order_workflow_steps( steps ) if ordered_steps: diff -r a620afab791d -r f87f1eed6e0c templates/cloud/configure_cloud.mako --- a/templates/cloud/configure_cloud.mako Thu Aug 27 17:36:03 2009 -0400 +++ b/templates/cloud/configure_cloud.mako Fri Aug 28 17:45:40 2009 -0400 @@ -85,7 +85,7 @@ <colgroup width="35%"></colgroup> <tr class="header"> <th>Live instances</th> - <th>Volume size (GB)</th> + <th>Storage size (GB)</th> <th>State</th> <th>Alive since</th> <th></th> @@ -97,7 +97,7 @@ ${liveInstance.name} <a id="li-${i}-popup" class="popup-arrow" style="display: none;">▼</a> </td> - <td>${str(liveInstance.size)}</td> <!--TODO:Replace with vol size once available--> + <td>${str(liveInstance.total_size)}</td> <!--TODO:Replace with vol size once available--> <td>${str(liveInstance.state)}</td> <td> ${str(liveInstance.launch_time)[:16]} @@ -106,7 +106,7 @@ from datetime import timedelta # DB stores all times in GMT, so adjust for difference (4 hours) - adjustedStarttime = liveInstance.launch_time - timedelta(hours=4) + adjustedStarttime = liveInstance.update_time - timedelta(hours=4) # (NOT CURRENTLY USED BLOCK OF CODE) Calculate time difference from now delta = datetime.now() - adjustedStarttime @@ -117,6 +117,8 @@ #context.write( str( liveInstance.launch_time ) ) context.write( ' UTC (' ) context.write( str(h.date.distance_of_time_in_words (liveInstance.launch_time, h.date.datetime.utcnow() ) ) ) + + %>) </td> <td> @@ -144,7 +146,7 @@ <colgroup width="35%"></colgroup> <tr class="header"> <th>Previously configured instances</th> - ##<th>Volume size (GB)</th> + ##<th>Storage size (GB)</th> ##<th>State</th> ##<th>Alive since</th> <th></th> @@ -160,9 +162,9 @@ ${prevInstance.name} <a id="pi-${i}-popup" class="popup-arrow" style="display: none;">▼</a> </td> - ## Comment <td>${len(workflow.latest_workflow.steps)}</td> - <td>${str(prevInstance.CloudStorage.size)}</td> <!-- TODO: Change to show vol size once available--> + <td>${str(prevInstance.total_size)}</td> <!-- TODO: Change to show vol size once available--> <td>${str(prevInstance.state)}</td> + <td>N/A</td> <td> <div popupmenu="pi-${i}-popup"> <a class="action-button" href="${h.url_for( action='start', id=trans.security.encode_id(prevInstance.id) )}">Start</a>