- galaxy-dev - lists.galaxyproject.org

[hg] galaxy 3063: Verified much of the code regarding cloud conn...
by Greg Von Kuster 23 Nov '09

23 Nov '09

details: http://www.bx.psu.edu/hg/galaxy/rev/5b18a134bf56 changeset: 3063:5b18a134bf56 user: Enis Afgan <afgane(a)gmail.com> date: Tue Sep 15 16:53:01 2009 -0400 description: Verified much of the code regarding cloud connectivity functionality w/ Eucalyptus. diffstat: lib/galaxy/web/controllers/cloud.py | 130 ++++++++++++++++++++++--------- templates/cloud/configure_cloud.mako | 2 +- templates/root/index.mako | 16 ++-- 3 files changed, 100 insertions(+), 48 deletions(-) diffs (294 lines): diff -r 52318cf73454 -r 5b18a134bf56 lib/galaxy/web/controllers/cloud.py --- a/lib/galaxy/web/controllers/cloud.py Mon Sep 14 15:08:23 2009 -0400 +++ b/lib/galaxy/web/controllers/cloud.py Tue Sep 15 16:53:01 2009 -0400 @@ -42,7 +42,7 @@ .filter_by( user=user, state="pending" ) \ .all() - for i in range( len( pendingInstances ) ): + for i in range( len ( pendingInstances ) ): update_instance_state( trans, pendingInstances[i].id ) cloudCredentials = trans.sa_session.query( model.CloudUserCredentials ) \ @@ -60,7 +60,16 @@ .filter_by( user=user, state="available" ) \ .order_by( desc( model.UCI.c.update_time ) ) \ .all() - + + # Check after update there are instances in pending state; if so, display message + # TODO: Auto-refresh once instance is running + pendingInstances = trans.sa_session.query( model.UCI ) \ + .filter_by( user=user, state="pending" ) \ + .all() + if pendingInstances: + trans.set_message( "Galaxy instance started. NOTE: Please wait about 3-5 minutes for the instance to " + "start up and then refresh this page. A button to connect to the instance will then appear alongside " + "instance description." ) return trans.fill_template( "cloud/configure_cloud.mako", cloudCredentials = cloudCredentials, liveInstances = liveInstances, @@ -88,7 +97,7 @@ @web.expose @web.require_login( "start Galaxy cloud instance" ) - def start( self, trans, id, size='' ): + def start( self, trans, id, size='small' ): """ Start a new cloud resource instance """ @@ -108,26 +117,32 @@ instance.availability_zone = stores[0].availability_zone # Bc. all EBS volumes need to be in the same avail. zone, just check 1st instance.type = size conn = get_connection( trans ) +# log.debug( '***** Setting up security group' ) # If not existent, setup galaxy security group - try: - gSecurityGroup = conn.create_security_group('galaxy', 'Security group for Galaxy.') - gSecurityGroup.authorize( 'tcp', 80, 80, '0.0.0.0/0' ) # Open HTTP port - gSecurityGroup.authorize( 'tcp', 22, 22, '0.0.0.0/0' ) # Open SSH port - except: - pass +# try: +# gSecurityGroup = conn.create_security_group('galaxy', 'Security group for Galaxy.') +# gSecurityGroup.authorize( 'tcp', 80, 80, '0.0.0.0/0' ) # Open HTTP port +# gSecurityGroup.authorize( 'tcp', 22, 22, '0.0.0.0/0' ) # Open SSH port +# except: +# pass # sgs = conn.get_all_security_groups() # for i in range( len( sgs ) ): # if sgs[i].name == "galaxy": # sg.append( sgs[i] ) # break # only 1 security group w/ this name can exist, so continue + log.debug( '***** Starting an instance' ) + log.debug( 'Using following command: conn.run_instances( image_id=%s, key_name=%s )' % ( instance.image.image_id, instance.keypair_name ) ) + reservation = conn.run_instances( image_id=instance.image.image_id, key_name=instance.keypair_name ) #reservation = conn.run_instances( image_id=instance.image, key_name=instance.keypair_name, security_groups=['galaxy'], instance_type=instance.type, placement=instance.availability_zone ) instance.launch_time = datetime.utcnow() uci.launch_time = instance.launch_time - #instance.reservation = str( reservation.instances[0] ) - instance.state = "pending" - #instance.state = reservation.instances[0].state + instance.reservation_id = str( reservation ).split(":")[1] + instance.instance_id = str( reservation.instances[0]).split(":")[1] +# instance.state = "pending" + instance.state = reservation.instances[0].state uci.state = instance.state + # TODO: After instance boots up, need to update status, DNS and attach EBS # Persist @@ -135,10 +150,7 @@ session.save_or_update( instance ) session.flush() - trans.log_event( "User started cloud instance '%s'" % uci.name ) - trans.set_message( "Galaxy instance '%s' started. NOTE: Please wait about 2-3 minutes for the instance to " - "start up and then refresh this page. A button to connect to the instance will then appear alongside " - "instance description." % uci.name ) + trans.log_event ("Started new instance. Reservation ID: '%s', Instance ID: '%s'" % (instance.reservation_id, instance.instance_id ) ) return self.list( trans ) # return trans.show_form( @@ -151,21 +163,49 @@ @web.require_login( "stop Galaxy cloud instance" ) def stop( self, trans, id ): """ - Stop a cloud instance. This implies stopping Galaxy servcer and unmounting relevant file system(s) + Stop a cloud instance. This implies stopping Galaxy server and disconnecting/unmounting relevant file system(s). """ uci = get_uci( trans, id ) - instances = get_instances( trans, uci ) #TODO: handle list! + dbInstances = get_instances( trans, uci ) #TODO: handle list! - instances.stop() - instances.state = 'terminated' - instances.stop_time = datetime.utcnow() + conn = get_connection( trans ) + # Get actual cloud instance object + cloudInstance = get_cloud_instance( conn, dbInstances.instance_id ) + + # TODO: Detach persistent storage volume(s) from instance and update volume data in local database + stores = get_stores( trans, uci ) + for i, store in enumerate( stores ): + log.debug( "Detaching volume '%s' to instance '%s'." % ( store.volume_id, dbInstances.instance_id ) ) + mntDevice = store.device + volStat = None +# try: +# volStat = conn.detach_volume( store.volume_id, dbInstances.instance_id, mntDevice ) +# except: +# log.debug ( 'Error detaching volume; still going to try and stop instance %s.' % dbInstances.instance_id ) + store.attach_time = None + store.device = None + store.i_id = None + store.status = volStat + log.debug ( '***** volume status: %s' % volStat ) + + + # Stop the instance and update status in local database + cloudInstance.stop() + dbInstances.stop_time = datetime.utcnow() + while cloudInstance.state != 'terminated': + log.debug( "Stopping instance %s state; current state: %s" % ( str( cloudInstance ).split(":")[1], cloudInstance.state ) ) + time.sleep(2) + cloudInstance.update() + dbInstances.state = cloudInstance.state + # Reset relevant UCI fields uci.state = 'available' uci.launch_time = None # Persist session = trans.sa_session - session.save_or_update( instances ) +# session.save_or_update( stores ) + session.save_or_update( dbInstances ) # TODO: Is this going to work w/ multiple instances stored in dbInstances variable? session.save_or_update( uci ) session.flush() trans.log_event( "User stopped cloud instance '%s'" % uci.name ) @@ -891,7 +931,7 @@ For valid sizes, see http://aws.amazon.com/ec2/instance-types/ """ return trans.app.model.CloudImage.filter( - trans.app.model.CloudImage.table.c.id==1).first() + trans.app.model.CloudImage.table.c.id==2).first() def get_stores( trans, uci ): """ @@ -917,30 +957,33 @@ return instances - def get_cloud_instance( conn, instance_id ): """ - Returns a cloud instance representation of the instance id, i.e., cloud instance objects that cloud API can be invoked on + Returns a cloud instance representation of the instance id, i.e., cloud instance object that cloud API can be invoked on """ # get_all_instances func. takes a list of desired instance id's, so create a list first idLst = list() idLst.append( instance_id ) - # Retrieve cloud instance based on passed instance id - cloudInstance = conn.get_all_instances( idLst ) - return cloudInstance[0] + # Retrieve cloud instance based on passed instance id. get_all_instances( idLst ) method returns reservation ID. Because + # we are passing only 1 ID, we can retrieve only the first element of the returning list. Furthermore, because (for now!) + # only 1 instance corresponds each individual reservation, grab only the first element of the returned list of instances. + cloudInstance = conn.get_all_instances( idLst )[0].instances[0] + return cloudInstance def get_connection( trans ): """ Establishes EC2 conncection using user's default credentials """ + log.debug( '##### Establishing cloud connection.' ) user = trans.get_user() creds = trans.sa_session.query(model.CloudUserCredentials).filter_by(user=user, defaultCred=True).first() if creds: a_key = creds.access_key s_key = creds.secret_key + # Amazon EC2 #conn = EC2Connection( a_key, s_key ) + # Eucalyptus Public Cloud euca_region = RegionInfo(None, "eucalyptus", "mayhem9.cs.ucsb.edu") - #conn = EC2Connection(aws_access_key_id="2s42fQmcCu4WBpS3RJ9e5g", aws_secret_access_key="2iEzpThjZQttuvWYXL-0nRUuurzl2dump2drwg", is_secure=False, port=8773, region=euca_region, path="/services/Eucalyptus") conn = EC2Connection(aws_access_key_id=a_key, aws_secret_access_key=s_key, is_secure=False, port=8773, region=euca_region, path="/services/Eucalyptus") return conn else: @@ -952,11 +995,16 @@ Generate keypair using user's default credentials """ conn = get_connection( trans ) + + log.debug( "Getting user's keypair" ) + key_pair = conn.get_key_pair( 'galaxy-keypair' ) + try: - key_pair = conn.get_key_pair( 'galaxy-keypair' ) - except: # No keypair under this name exists so create it + return key_pair.name + except AttributeError: # No keypair under this name exists so create it + log.debug( 'No keypair found, creating keypair' ) key_pair = conn.create_key_pair( 'galaxy-keypair' ) - + return key_pair.name def update_instance_state( trans, id ): @@ -970,12 +1018,12 @@ oldState = dbInstances.state # Establish connection with cloud conn = get_connection( trans ) - # Get actual instance from the cloud + # Get actual cloud instance object cloudInstance = get_cloud_instance( conn, dbInstances.instance_id ) - # Update status of instance + # Update instance status cloudInstance.update() dbInstances.state = cloudInstance.state - log.debug( "Processing instance %i, current instance state: %s" % i, cloudInstance.state ) + log.debug( "Updating instance %s state; current state: %s" % ( str( cloudInstance ).split(":")[1], cloudInstance.state ) ) # Update state of UCI (TODO: once more than 1 instance is assoc. w/ 1 UCI, this will be need to be updated differently) uci.state = dbInstances.state # Persist @@ -1004,13 +1052,17 @@ dbInstance.public_dns = cloudInstance.dns_name dbInstance.private_dns = cloudInstance.private_dns_name - # TODO: connect EBS volume to instance # Attach storage volume(s) to instance stores = get_stores( trans, uci ) for i, store in enumerate( stores ): - log.debug( "Attaching volume %s to instance %s." % store.volume_id, store.uci_id ) - mtnDevice = '/dev/sdb'+str(i) - conn.attach_volume( store.volume_id, store.uci_id, mntDevice ) + log.debug( "Attaching volume '%s' to instance '%s'." % ( store.volume_id, dbInstance.instance_id ) ) + mntDevice = '/dev/sdb'+str(i) + volStat = conn.attach_volume( store.volume_id, dbInstance.instance_id, mntDevice ) + store.attach_time = datetime.utcnow() + store.device = mntDevice + store.i_id = dbInstance.instance_id + store.status = volStat + log.debug ( '***** volume status: %s' % volStat ) # Wait until instances have attached and add file system diff -r 52318cf73454 -r 5b18a134bf56 templates/cloud/configure_cloud.mako --- a/templates/cloud/configure_cloud.mako Mon Sep 14 15:08:23 2009 -0400 +++ b/templates/cloud/configure_cloud.mako Tue Sep 15 16:53:01 2009 -0400 @@ -136,7 +136,7 @@ </td> <td> <div popupmenu="li-${i}-popup"> - <a class="action-button" confirm="Are you sure you want to stop instance '${liveInstance.name}'?" href="${h.url_for( action='stop', id=trans.security.encode_id(liveInstance.id) )}">Stop</a> + <a class="action-button" confirm="Are you sure you want to stop instance '${liveInstance.name}'? Please note that this may take up to 1 minute during which time the page will not refresh." href="${h.url_for( action='stop', id=trans.security.encode_id(liveInstance.id) )}">Stop</a> <a class="action-button" href="${h.url_for( action='renameInstance', id=trans.security.encode_id(liveInstance.id) )}">Rename</a> <a class="action-button" href="${h.url_for( action='viewInstance', id=trans.security.encode_id(liveInstance.id) )}">View details</a> </div> diff -r 52318cf73454 -r 5b18a134bf56 templates/root/index.mako --- a/templates/root/index.mako Mon Sep 14 15:08:23 2009 -0400 +++ b/templates/root/index.mako Tue Sep 15 16:53:01 2009 -0400 @@ -2,14 +2,14 @@ <%def name="init()"> <% -if trans.app.config.cloud_controller_instance: - self.has_left_panel=False - self.has_right_panel=False - self.active_view="cloud" -else: - self.has_left_panel=True - self.has_right_panel=True - self.active_view="analysis" + if trans.app.config.cloud_controller_instance: + self.has_left_panel=False + self.has_right_panel=False + self.active_view="cloud" + else: + self.has_left_panel=True + self.has_right_panel=True + self.active_view="analysis" %> %if trans.app.config.require_login and not trans.user: <script type="text/javascript">

1 0

[hg] galaxy 3062: Cloud tab is now enabled/disabled through conf...
by Greg Von Kuster 23 Nov '09

23 Nov '09

details: http://www.bx.psu.edu/hg/galaxy/rev/52318cf73454 changeset: 3062:52318cf73454 user: Enis Afgan <afgane(a)gmail.com> date: Mon Sep 14 15:08:23 2009 -0400 description: Cloud tab is now enabled/disabled through configuration option in universe_wsgi.ini file diffstat: lib/galaxy/config.py | 2 ++ lib/galaxy/web/controllers/cloud.py | 6 +++++- templates/base_panels.mako | 18 ++++++++++-------- templates/root/index.mako | 34 ++++++++++++++++++++-------------- 4 files changed, 37 insertions(+), 23 deletions(-) diffs (121 lines): diff -r 2d1b957b8448 -r 52318cf73454 lib/galaxy/config.py --- a/lib/galaxy/config.py Fri Sep 11 15:25:21 2009 -0400 +++ b/lib/galaxy/config.py Mon Sep 14 15:08:23 2009 -0400 @@ -103,6 +103,8 @@ except ConfigParser.NoSectionError: self.tool_runners = [] self.datatypes_config = kwargs.get( 'datatypes_config_file', 'datatypes_conf.xml' ) + # Cloud configuration options + self.cloud_controller_instance = string_as_bool( kwargs.get( 'cloud_controller_instance', 'False' ) ) def get( self, key, default ): return self.config_dict.get( key, default ) def get_bool( self, key, default ): diff -r 2d1b957b8448 -r 52318cf73454 lib/galaxy/web/controllers/cloud.py --- a/lib/galaxy/web/controllers/cloud.py Fri Sep 11 15:25:21 2009 -0400 +++ b/lib/galaxy/web/controllers/cloud.py Mon Sep 14 15:08:23 2009 -0400 @@ -20,6 +20,7 @@ import galaxy.eggs galaxy.eggs.require("boto") from boto.ec2.connection import EC2Connection +from boto.ec2.regioninfo import RegionInfo import logging log = logging.getLogger( __name__ ) @@ -937,7 +938,10 @@ if creds: a_key = creds.access_key s_key = creds.secret_key - conn = EC2Connection( a_key, s_key ) + #conn = EC2Connection( a_key, s_key ) + euca_region = RegionInfo(None, "eucalyptus", "mayhem9.cs.ucsb.edu") + #conn = EC2Connection(aws_access_key_id="2s42fQmcCu4WBpS3RJ9e5g", aws_secret_access_key="2iEzpThjZQttuvWYXL-0nRUuurzl2dump2drwg", is_secure=False, port=8773, region=euca_region, path="/services/Eucalyptus") + conn = EC2Connection(aws_access_key_id=a_key, aws_secret_access_key=s_key, is_secure=False, port=8773, region=euca_region, path="/services/Eucalyptus") return conn else: error( "You must specify default credentials before starting an instance." ) diff -r 2d1b957b8448 -r 52318cf73454 templates/base_panels.mako --- a/templates/base_panels.mako Fri Sep 11 15:25:21 2009 -0400 +++ b/templates/base_panels.mako Mon Sep 14 15:08:23 2009 -0400 @@ -16,7 +16,7 @@ </%def> ## Default title -<%def name="title()">Galaxy Cloud</%def> +<%def name="title()">Galaxy</%def> ## Default stylesheets <%def name="stylesheets()"> @@ -135,13 +135,15 @@ <td class="${cls}" style="${style}"><a target="${target}" href="${href}">${display}</a></td> </%def> - ##${tab( "analysis", "Analyze Data", h.url_for( controller='root', action='index' ))} - - ##${tab( "workflow", "Workflow", h.url_for( controller='workflow', action='index' ))} - - ##${tab( "libraries", "Data Libraries", h.url_for( controller='library', action='index' ))} - - ${tab( "cloud", "Cloud", h.url_for( controller='cloud', action='index' ))} + %if app.config.cloud_controller_instance: + ${tab( "cloud", "Cloud", h.url_for( controller='cloud', action='index' ))} + %else: + ${tab( "analysis", "Analyze Data", h.url_for( controller='root', action='index' ))} + + ${tab( "workflow", "Workflow", h.url_for( controller='workflow', action='index' ))} + + ${tab( "libraries", "Data Libraries", h.url_for( controller='library', action='index' ))} + %endif %if trans.request_types(): <td class="tab"> diff -r 2d1b957b8448 -r 52318cf73454 templates/root/index.mako --- a/templates/root/index.mako Fri Sep 11 15:25:21 2009 -0400 +++ b/templates/root/index.mako Mon Sep 14 15:08:23 2009 -0400 @@ -2,9 +2,14 @@ <%def name="init()"> <% - self.has_left_panel=False - self.has_right_panel=False - self.active_view="cloud" +if trans.app.config.cloud_controller_instance: + self.has_left_panel=False + self.has_right_panel=False + self.active_view="cloud" +else: + self.has_left_panel=True + self.has_right_panel=True + self.active_view="analysis" %> %if trans.app.config.require_login and not trans.user: <script type="text/javascript"> @@ -28,17 +33,18 @@ ## If a specific tool id was specified, load it in the middle frame <% - if trans.app.config.require_login and not trans.user: - center_url = h.url_for( controller='user', action='login' ) - elif tool_id is not None: - center_url = h.url_for( 'tool_runner', tool_id=tool_id, from_noframe=True ) - elif workflow_id is not None: - center_url = h.url_for( controller='workflow', action='run', id=workflow_id ) - elif m_c is not None: - center_url = h.url_for( controller=m_c, action=m_a ) - else: - #center_url = h.url_for( '/static/welcome.html' ) - center_url = h.url_for( controller='cloud', action='list' ) +if trans.app.config.require_login and not trans.user: + center_url = h.url_for( controller='user', action='login' ) +elif tool_id is not None: + center_url = h.url_for( 'tool_runner', tool_id=tool_id, from_noframe=True ) +elif workflow_id is not None: + center_url = h.url_for( controller='workflow', action='run', id=workflow_id ) +elif m_c is not None: + center_url = h.url_for( controller=m_c, action=m_a ) +elif trans.app.config.cloud_controller_instance: + center_url = h.url_for( controller='cloud', action='list' ) +else: + center_url = h.url_for( '/static/welcome.html' ) %> <iframe name="galaxy_main" id="galaxy_main" frameborder="0" style="position: absolute; width: 100%; height: 100%;" src="${center_url}"> </iframe>

1 0

[hg] galaxy 3060: Added most of the functionality to connect to ...
by Greg Von Kuster 23 Nov '09

23 Nov '09

details: http://www.bx.psu.edu/hg/galaxy/rev/81e8a5e42900 changeset: 3060:81e8a5e42900 user: afgane(a)enis-afgans-macbook-pro.local date: Fri Sep 04 09:54:27 2009 -0400 description: Added most of the functionality to connect to EC2. Have not tested it though. diffstat: lib/galaxy/model/migrate/versions/0014_cloud_tables.py | 6 +- lib/galaxy/web/controllers/cloud.py | 337 ++++++++++++---------- static/june_2007_style/blue/base.css | 9 + templates/base_panels.mako | 8 +- templates/cloud/configure_cloud.mako | 56 ++- templates/cloud/viewInstance.mako | 48 +-- templates/root/index.mako | 9 +- 7 files changed, 258 insertions(+), 215 deletions(-) diffs (728 lines): diff -r f87f1eed6e0c -r 81e8a5e42900 lib/galaxy/model/migrate/versions/0014_cloud_tables.py --- a/lib/galaxy/model/migrate/versions/0014_cloud_tables.py Fri Aug 28 17:45:40 2009 -0400 +++ b/lib/galaxy/model/migrate/versions/0014_cloud_tables.py Fri Sep 04 09:54:27 2009 -0400 @@ -92,8 +92,8 @@ def downgrade(): metadata.reflect() try: - log.deboug( "Would drop cloud_image table." ) - #CloudImage_table.drop() #Enable before putting final version + log.debug( "Would drop cloud_image table." ) + CloudImage_table.drop() #Enable before putting final version except Exception, e: log.debug( "Dropping cloud_image table failed: %s" % str( e ) ) @@ -108,7 +108,7 @@ log.debug( "Dropping cloud_store table failed: %s" % str( e ) ) try: - log.deboug( "Would drop cloud_user_credentials table." ) + log.debug( "Would drop cloud_user_credentials table." ) #CloudUserCredentials_table.drop() #Enable before putting final version except Exception, e: log.debug( "Dropping cloud_user_credentials table failed: %s" % str( e ) ) diff -r f87f1eed6e0c -r 81e8a5e42900 lib/galaxy/web/controllers/cloud.py --- a/lib/galaxy/web/controllers/cloud.py Fri Aug 28 17:45:40 2009 -0400 +++ b/lib/galaxy/web/controllers/cloud.py Fri Sep 04 09:54:27 2009 -0400 @@ -37,31 +37,33 @@ Render cloud main page (management of cloud resources) """ user = trans.get_user() + pendingInstances = trans.sa_session.query( model.UCI ) \ + .filter_by( user=user, state="pending" ) \ + .all() + + for i in range( len( pendingInstances ) ): + update_instance_state( trans, pendingInstances[i].id ) + cloudCredentials = trans.sa_session.query( model.CloudUserCredentials ) \ + .filter_by( user=user ) \ .order_by( desc( model.CloudUserCredentials.c.update_time ) ) \ .all() - prevInstances = trans.sa_session.query( model.UCI ) \ - .filter_by( user=user, state="available" ) \ - .order_by( desc( model.UCI.c.update_time ) ) \ - .all() #TODO: diff between live and previous instances - - liveInstances = trans.sa_session.query( model.CloudInstance ) \ - .filter_by( user=user ) \ - .filter( or_(model.CloudInstance.c.state=="running", model.CloudInstance.c.state=="pending") ) \ - .order_by( desc( model.CloudInstance.c.launch_time ) ) \ - .all() - liveInstances = trans.sa_session.query( model.UCI ) \ .filter_by( user=user ) \ .filter( or_(model.UCI.c.state=="running", model.UCI.c.state=="pending") ) \ .order_by( desc( model.UCI.c.launch_time ) ) \ .all() - + + prevInstances = trans.sa_session.query( model.UCI ) \ + .filter_by( user=user, state="available" ) \ + .order_by( desc( model.UCI.c.update_time ) ) \ + .all() + return trans.fill_template( "cloud/configure_cloud.mako", cloudCredentials = cloudCredentials, - prevInstances = prevInstances, - liveInstances = liveInstances ) + liveInstances = liveInstances, + prevInstances = prevInstances ) @web.expose @web.require_login( "use Galaxy cloud" ) @@ -77,162 +79,94 @@ newDefault.defaultCred = True trans.sa_session.flush() trans.set_message( "Credentials '%s' set as default." % newDefault.name ) - + + # TODO: Fix bug that when this function returns, top Galaxy tab bar is missing from the webpage return self.list( trans ) #trans.fill_template( "cloud/configure_cloud.mako", #awsCredentials = awsCredentials ) - - @web.expose - @web.require_login( "use Galaxy workflows" ) - def list_for_run( self, trans ): - """ - Render workflow list for analysis view (just allows running workflow - or switching to management view) - """ - user = trans.get_user() - workflows = trans.sa_session.query( model.StoredWorkflow ) \ - .filter_by( user=user, deleted=False ) \ - .order_by( desc( model.StoredWorkflow.c.update_time ) ) \ - .all() - shared_by_others = trans.sa_session \ - .query( model.StoredWorkflowUserShareAssociation ) \ - .filter_by( user=user ) \ - .filter( model.StoredWorkflow.c.deleted == False ) \ - .order_by( desc( model.StoredWorkflow.c.update_time ) ) \ - .all() - return trans.fill_template( "workflow/list_for_run.mako", - workflows = workflows, - shared_by_others = shared_by_others ) - - @web.expose - @web.require_login( "use Galaxy workflows" ) - def share( self, trans, id, email="" ): - msg = mtype = None - # Load workflow from database - stored = get_stored_workflow( trans, id ) - if email: - other = model.User.filter( and_( model.User.table.c.email==email, - model.User.table.c.deleted==False ) ).first() - if not other: - mtype = "error" - msg = ( "User '%s' does not exist" % email ) - elif other == trans.get_user(): - mtype = "error" - msg = ( "You cannot share a workflow with yourself" ) - elif trans.sa_session.query( model.StoredWorkflowUserShareAssociation ) \ - .filter_by( user=other, stored_workflow=stored ).count() > 0: - mtype = "error" - msg = ( "Workflow already shared with '%s'" % email ) - else: - share = model.StoredWorkflowUserShareAssociation() - share.stored_workflow = stored - share.user = other - session = trans.sa_session - session.save_or_update( share ) - session.flush() - trans.set_message( "Workflow '%s' shared with user '%s'" % ( stored.name, other.email ) ) - return trans.response.send_redirect( url_for( controller='workflow', action='sharing', id=id ) ) - return trans.fill_template( "workflow/share.mako", - message = msg, - messagetype = mtype, - stored=stored, - email=email ) - - @web.expose - @web.require_login( "use Galaxy workflows" ) - def sharing( self, trans, id, **kwargs ): - session = trans.sa_session - stored = get_stored_workflow( trans, id ) - if 'enable_import_via_link' in kwargs: - stored.importable = True - stored.flush() - elif 'disable_import_via_link' in kwargs: - stored.importable = False - stored.flush() - elif 'unshare_user' in kwargs: - user = session.query( model.User ).get( trans.security.decode_id( kwargs['unshare_user' ] ) ) - if not user: - error( "User not found for provided id" ) - association = session.query( model.StoredWorkflowUserShareAssociation ) \ - .filter_by( user=user, stored_workflow=stored ).one() - session.delete( association ) - session.flush() - return trans.fill_template( "workflow/sharing.mako", - stored=stored ) - - @web.expose - @web.require_login( "use Galaxy workflows" ) - def imp( self, trans, id, **kwargs ): - session = trans.sa_session - stored = get_stored_workflow( trans, id, check_ownership=False ) - if stored.importable == False: - error( "The owner of this workflow has disabled imports via this link" ) - elif stored.user == trans.user: - error( "You are already the owner of this workflow, can't import" ) - elif stored.deleted: - error( "This workflow has been deleted, can't import" ) - elif session.query( model.StoredWorkflowUserShareAssociation ) \ - .filter_by( user=trans.user, stored_workflow=stored ).count() > 0: - error( "This workflow is already shared with you" ) - else: - share = model.StoredWorkflowUserShareAssociation() - share.stored_workflow = stored - share.user = trans.user - session = trans.sa_session - session.save_or_update( share ) - session.flush() - # Redirect to load galaxy frames. - return trans.response.send_redirect( url_for( controller='workflow' ) ) - + + @web.expose @web.require_login( "start Galaxy cloud instance" ) - def start( self, trans, id, size='small' ): + def start( self, trans, id, size='' ): """ Start a new cloud resource instance """ + # TODO: Add choice of instance size before starting one + #if size: user = trans.get_user() mi = get_mi( trans, size ) uci = get_uci( trans, id ) - stores = get_stores( trans, uci ) #TODO: handle list! + stores = get_stores( trans, uci ) + log.debug(self.app.config.job_working_directory) + if len(stores) is not 0: + instance = model.CloudInstance() + instance.user = user + instance.image = mi + instance.uci = uci + instance.keypair_name = get_keypair_name( trans ) + instance.availability_zone = stores[0].availability_zone # Bc. all EBS volumes need to be in the same avail. zone, just check 1st + instance.type = size + conn = get_connection( trans ) + # Get or setup appropriate security group + sg = list() # security group list + try: + gSecurityGroup = conn.create_security_group('galaxy', 'Galaxy security group') + gSecurityGroup.authorize( 'tcp', 80, 80, '0.0.0.0/0' ) + sg.append( gSecurityGroup ) + except: + sgs = rs = conn.get_all_security_groups() + for i in range( len( sgs ) ): + if sgs[i].name == "galaxy": + sg.append( sgs[i] ) + break # only 1 security group w/ this name can exist, so continue + + #reservation = conn.run_instances( image_id=instance.image, key_name=instance.keypair_name, security_groups=sg, instance_type=instance.type, placement=instance.availability_zone ) + instance.launch_time = datetime.utcnow() + uci.launch_time = instance.launch_time + #instance.reservation = str( reservation.instances[0] ) + instance.state = "pending" + #instance.state = reservation.instances[0].state + uci.state = instance.state + # TODO: After instance boots up, need to update status, DNS and attach EBS + + # Persist + session = trans.sa_session + session.save_or_update( instance ) + session.flush() + #error( "Starting instance '%s' is not supported yet." % uci.name ) + + trans.log_event( "User started cloud instance '%s'" % uci.name ) + trans.set_message( "Galaxy instance '%s' started. NOTE: Please wait about 2-3 minutes for the instance to " + "start up and then refresh this page. A button to connect to the instance will then appear alongside " + "instance description." % uci.name ) + return self.list( trans ) - instance = model.CloudInstance() - instance.user = user - instance.image = mi - instance.uci = uci - # TODO: get real value from AWS - instance.state = "pending" - uci.state = instance.state - uci.launch_time = datetime.utcnow() - instance.launch_time = datetime.utcnow() - instance.availability_zone = stores.availability_zone - instance.type = size - - # Persist - session = trans.sa_session - session.save_or_update( instance ) - session.flush() - #error( "Starting instance '%s' is not supported yet." % uci.name ) +# return trans.show_form( +# web.FormBuilder( web.url_for(), "Start instance size", submit_text="Start" ) +# .add_input( "radio","Small","size", value='small' ) +# .add_input( "radio","Medium","size", value='medium' ) ) - trans.log_event( "User started cloud instance '%s'" % uci.name ) - trans.set_message( "Galaxy instance '%s' started." % uci.name ) - return self.list( trans ) @web.expose @web.require_login( "stop Galaxy cloud instance" ) def stop( self, trans, id ): """ - Stop a cloud resource instance + Stop a cloud instance. This implies stopping Galaxy servcer and unmounting relevant file system(s) """ uci = get_uci( trans, id ) instances = get_instances( trans, uci ) #TODO: handle list! - instances.state = 'done' + instances.stop() + instances.state = 'terminated' instances.stop_time = datetime.utcnow() + # Reset relevant UCI fields uci.state = 'available' uci.launch_time = None + # Persist session = trans.sa_session + session.save_or_update( instances ) session.save_or_update( uci ) - session.save_or_update( instances ) session.flush() trans.log_event( "User stopped cloud instance '%s'" % uci.name ) trans.set_message( "Galaxy instance '%s' stopped." % uci.name ) @@ -429,11 +363,11 @@ """ View details about running instance """ - instance = get_uci( trans, id ) - log.debug ( instance.name ) + uci = get_uci( trans, id ) + instances = get_instances( trans, uci ) # TODO: Handle list (will probably need to be done in mako template) return trans.fill_template( "cloud/viewInstance.mako", - liveInstance = instance ) + liveInstance = instances ) @web.expose @@ -931,7 +865,10 @@ """ Get a UCI from the database by id, verifying ownership. """ - id = trans.security.decode_id( id ) + # Check if 'id' is in int (i.e., it was called from this program) or + # it was passed from the web (in which case decode it) + if not isinstance( id, int ): + id = trans.security.decode_id( id ) live = trans.sa_session.query( model.UCI ).get( id ) if not live: @@ -956,28 +893,121 @@ def get_stores( trans, uci ): """ - Get store objects/tables that are connected to uci object/table + Get stores objects that are connected to uci object """ user = trans.get_user() stores = trans.sa_session.query( model.CloudStore ) \ .filter_by( user=user, uci_id=uci.id ) \ - .first() - #.all() #TODO: return all but need to edit calling method(s) to handle list + .all() return stores def get_instances( trans, uci ): """ - Get instance objects/tables that are connected to uci object/table + Get objects of instances that are pending or running and are connected to uci object """ user = trans.get_user() instances = trans.sa_session.query( model.CloudInstance ) \ .filter_by( user=user, uci_id=uci.id ) \ + .filter( or_(model.CloudInstance.table.c.state=="running", model.CloudInstance.table.c.state=="pending" ) ) \ .first() #.all() #TODO: return all but need to edit calling method(s) to handle list return instances + +def get_cloud_instance( conn, instance_id ): + """ + Returns a cloud instance representation of the instance id, i.e., cloud instance objects that cloud API can be invoked on + """ + # get_all_instances func. takes a list of desired instance id's, so create a list first + idLst = list() + idLst.append( instance_id ) + # Retrieve cloud instance based on passed instance id + cloudInstance = conn.get_all_instances( idLst ) + return cloudInstance[0] + +def get_connection( trans ): + """ + Establishes EC2 conncection using user's default credentials + """ + user = trans.get_user() + creds = trans.sa_session.query(model.CloudUserCredentials).filter_by(user=user, defaultCred=True).first() + if creds: + a_key = creds.access_key + s_key = creds.secret_key + conn = EC2Connection( a_key, s_key ) + return conn + else: + error( "You must specify default credentials before starting an instance." ) + return 0 + +def get_keypair_name( trans ): + """ + Generate X.509 Certificate (i.e., keypair) using user's default credentials + """ + conn = get_connection( trans ) + try: + key_pair = conn.get_key_pair( 'galaxy-keypair' ) + except: # No keypair under this name exists so create it + key_pair = conn.create_key_pair( 'galaxy-keypair' ) +# log.debug( key_pair.name ) + + return key_pair.name + +def update_instance_state( trans, id ): + """ + Update state of instances associated with given UCI id and store it in local database. Also update + state of the given UCI. + """ + uci = get_uci( trans, id ) + # Get list of instances associated with given uci as they are stored in local database + dbInstances = get_instances( trans, uci ) # TODO: handle list (currently only 1 instance can correspond to 1 UCI) + oldState = dbInstances.state + # Establish connection with cloud + conn = get_connection( trans ) + # Get actual instance from the cloud + cloudInstance = get_cloud_instance( conn, dbInstances.instance_id ) + # Update status of instance + cloudInstance.update() + dbInstances.state = cloudInstance.state + log.debug( "Processing instance %i, instance current state: %s" % i, cloudInstance.state ) + + # If instance is now running, update/process instance (i.e., mount file system, start Galaxy, update DB with DNS) + if oldState=="pending" and dbInstances.state=="running": + update_instance( trans, dbInstances, cloudInstance ) + + # Update state of UCI (TODO: once more than 1 instance is assoc. w/ 1 UCI, this will be need to be updated differently) + uci.state = dbInstances.state + + # Persist + session = trans.sa_session + session.save_or_update( dbInstances ) + session.save_or_update( uci ) + session.flush() + + +def update_instance( trans, dbInstance, cloudInstance ): + """ + Update instance: mount file system, start Galaxy and update local DB w/ DNS info + + Keyword arguments: + trans -- current transaction + dbInstance -- object of 'instance' as it is stored in local database + cloudInstance -- object of 'instance' as it resides in the cloud. Functions supported by the cloud API can be + instantiated directly on this object. + """ + dbInstance.public_dns = cloudInstance.dns_name + dbInstance.private_dns = cloudInstance.private_dns_name + + # TODO: mount storage + # TODO: start Galaxy + + # Persist + session = trans.sa_session + session.save_or_update( dbInstance ) + session.flush() + def attach_ordered_steps( workflow, steps ): ordered_steps = order_workflow_steps( steps ) if ordered_steps: @@ -1104,3 +1134,8 @@ cleanup( prefix, input.cases[current_case].inputs, group_values ) cleanup( "", inputs, values ) return associations + + + + + diff -r f87f1eed6e0c -r 81e8a5e42900 static/june_2007_style/blue/base.css --- a/static/june_2007_style/blue/base.css Fri Aug 28 17:45:40 2009 -0400 +++ b/static/june_2007_style/blue/base.css Fri Sep 04 09:54:27 2009 -0400 @@ -57,6 +57,15 @@ table.colored tr.header{background:#ebd9b2;background-image:url(form_title_bg.png);background-repeat:repeat-x;background-position:top;border-bottom:solid #d8b365 1px;font-weight:bold;} table.colored tr{background:white;} table.colored tr.odd_row{background:#DADFEF;} +table.noBottomHR{border-top:solid #d8b365 1px;} +table.noBottomHR td,table.noBottomHR th{text-align:left;padding:5px;} +table.noBottomHR tr.header{background:#ebd9b2;background-image:url(form_title_bg.png);background-repeat:repeat-x;background-position:top;border-bottom:solid #d8b365 1px;font-weight:bold;} +table.noBottomHR tr{background:white;} +table.noBottomHR tr.odd_row{background:#DADFEF;} +table.noHR td,table.noHR th{text-align:left;padding:5px;} +table.noHR tr.header{background:#ebd9b2;background-image:url(form_title_bg.png);background-repeat:repeat-x;background-position:top;border-bottom:solid #d8b365 1px;font-weight:bold;} +table.noHR tr{background:white;} +table.noHR tr.odd_row{background:#DADFEF;} div.debug{margin:10px;padding:5px;background:#FFFF99;border:solid #FFFF33 1px;color:black;} div.odd_row{background:#DADFEF;} #footer{display:none;} diff -r f87f1eed6e0c -r 81e8a5e42900 templates/base_panels.mako --- a/templates/base_panels.mako Fri Aug 28 17:45:40 2009 -0400 +++ b/templates/base_panels.mako Fri Sep 04 09:54:27 2009 -0400 @@ -16,7 +16,7 @@ </%def> ## Default title -<%def name="title()">Galaxy</%def> +<%def name="title()">Galaxy Cloud</%def> ## Default stylesheets <%def name="stylesheets()"> @@ -135,11 +135,11 @@ <td class="${cls}" style="${style}"><a target="${target}" href="${href}">${display}</a></td> </%def> - ${tab( "analysis", "Analyze Data", h.url_for( controller='root', action='index' ))} + ##${tab( "analysis", "Analyze Data", h.url_for( controller='root', action='index' ))} - ${tab( "workflow", "Workflow", h.url_for( controller='workflow', action='index' ))} + ##${tab( "workflow", "Workflow", h.url_for( controller='workflow', action='index' ))} - ${tab( "libraries", "Data Libraries", h.url_for( controller='library', action='index' ))} + ##${tab( "libraries", "Data Libraries", h.url_for( controller='library', action='index' ))} ${tab( "cloud", "Cloud", h.url_for( controller='cloud', action='index' ))} diff -r f87f1eed6e0c -r 81e8a5e42900 templates/cloud/configure_cloud.mako --- a/templates/cloud/configure_cloud.mako Fri Aug 28 17:45:40 2009 -0400 +++ b/templates/cloud/configure_cloud.mako Fri Sep 04 09:54:27 2009 -0400 @@ -82,12 +82,14 @@ <colgroup width="40%"></colgroup> <colgroup width="15%"></colgroup> <colgroup width="10%"></colgroup> - <colgroup width="35%"></colgroup> + <colgroup width="25%"></colgroup> + <colgroup width="10%"></colgroup> <tr class="header"> <th>Live instances</th> <th>Storage size (GB)</th> <th>State</th> <th>Alive since</th> + <th></th> <th></th> </tr> %if liveInstances: @@ -97,30 +99,40 @@ ${liveInstance.name} <a id="li-${i}-popup" class="popup-arrow" style="display: none;">▼</a> </td> - <td>${str(liveInstance.total_size)}</td>  + <td>${str(liveInstance.total_size)}</td> <td>${str(liveInstance.state)}</td> <td> - ${str(liveInstance.launch_time)[:16]} + ##${str(liveInstance.launch_time)[:16]} <% - from datetime import datetime - from datetime import timedelta + #from datetime import datetime + #from datetime import timedelta # DB stores all times in GMT, so adjust for difference (4 hours) - adjustedStarttime = liveInstance.update_time - timedelta(hours=4) + #adjustedStarttime = liveInstance.update_time - timedelta(hours=4) - # (NOT CURRENTLY USED BLOCK OF CODE) Calculate time difference from now - delta = datetime.now() - adjustedStarttime + #delta = datetime.now() - adjustedStarttime #context.write( str(datetime.utcnow() ) ) #context.write( str(delta) ) # This is where current time and since duration is calculated - #context.write( str( liveInstance.launch_time ) ) - context.write( ' UTC (' ) - context.write( str(h.date.distance_of_time_in_words (liveInstance.launch_time, h.date.datetime.utcnow() ) ) ) - - - %>) + if liveInstance.launch_time is None: + context.write( 'N/A' ) + else: + context.write( str( liveInstance.launch_time )[:16] ) + context.write( ' UTC (' ) + context.write( str(h.date.distance_of_time_in_words (liveInstance.launch_time, h.date.datetime.utcnow() ) ) ) + context.write( ')' ) + %> </td> + <td><div align="right"> + %for i, instance in enumerate( liveInstance.instance ): + %if instance.state == "running": + <a class="action-button" href="http://${instance.public_dns}" target="_blank"> + <span>Connect to!</span> + <img src="${h.url_for('/static/images/silk/resultset_next.png')}" /></a></div> + %endif + %endfor + </td> <td> <div popupmenu="li-${i}-popup"> <a class="action-button" confirm="Are you sure you want to stop instance '${liveInstance.name}'?" href="${h.url_for( action='stop', id=trans.security.encode_id(liveInstance.id) )}">Stop</a> @@ -139,21 +151,21 @@ ## ***************************************************** ## Manage previously configured instances - <table class="mange-table colored" border="0" cellspacing="0" cellpadding="0" width="100%"> + <table class="mange-table noHR" border="0" cellspacing="0" cellpadding="0" width="100%"> <colgroup width="40%"></colgroup> <colgroup width="15%"></colgroup> <colgroup width="10%"></colgroup> <colgroup width="35%"></colgroup> - <tr class="header"> - <th>Previously configured instances</th> + ##<tr class="header"> + ##<th>Previously configured instances</th> ##<th>Storage size (GB)</th> ##<th>State</th> ##<th>Alive since</th> - <th></th> - <th></th> - <th></th> - <th></th> - </tr> + ##<th></th> + ##<th></th> + ##<th></th> + ##<th></th> + ##</tr> %if prevInstances: %for i, prevInstance in enumerate( prevInstances ): diff -r f87f1eed6e0c -r 81e8a5e42900 templates/cloud/viewInstance.mako --- a/templates/cloud/viewInstance.mako Fri Aug 28 17:45:40 2009 -0400 +++ b/templates/cloud/viewInstance.mako Fri Sep 04 09:54:27 2009 -0400 @@ -1,6 +1,13 @@ <%inherit file="/base.mako"/> <%def name="title()">Live instance details</%def> +<% + # Because of the one-to-many relationship between liveInstance (i.e., UCI) and actual instances, need to know + # which one is currently active. Because only one instance of UCI can be alive at any point in time, simply + # select the most recent one. + # TODO: Once individual UCI's will be able to start more than one instance, this will need to be fixed + #i_id = len(liveInstance.instance) - 1 +%> <h2>Live instance details</h2> @@ -18,22 +25,22 @@ <tr> <td> Instance name: </td> <td> - ${liveInstance.name} + ${liveInstance.uci.name} <a id="li-popup" class="popup-arrow" style="display: none;">▼</a> </td> <td> <div popupmenu="li-popup"> - <a class="action-button" href="${h.url_for( action='renameInstance', id=trans.security.encode_id(liveInstance.id) )}">Rename</a> - <a class="action-button" confirm="Are you sure you want to stop instance '${liveInstance.name}'?" href="${h.url_for( action='stop', id=trans.security.encode_id(liveInstance.id) )}">Stop</a> + <a class="action-button" href="${h.url_for( action='renameInstance', id=trans.security.encode_id(liveInstance.uci.id) )}">Rename</a> + <a class="action-button" confirm="Are you sure you want to stop instance '${liveInstance.uci.name}'?" href="${h.url_for( action='stop', id=trans.security.encode_id(liveInstance.uci.id) )}">Stop</a> </div> </td> </tr> <tr> <td> Date created: </td> - <td> ${str(liveInstance.create_time)[:16]} + <td> ${str(liveInstance.uci.create_time)[:16]} <% context.write( ' UTC (' ) - context.write( str(h.date.distance_of_time_in_words (liveInstance.create_time, h.date.datetime.utcnow() ) ) ) + context.write( str(h.date.distance_of_time_in_words (liveInstance.uci.create_time, h.date.datetime.utcnow() ) ) ) %> ago) </td> </tr> @@ -60,7 +67,7 @@ %endif <tr> <td> AMI: </td> - <td> ${liveInstance.ami} </td> + <td> ${liveInstance.mi_id} </td> </tr> <tr> <td> State:</td> @@ -82,34 +89,13 @@ <td> ${liveInstance.availability_zone} </td> </tr> %endif - %if liveInstance.keypair_fingerprint != None: + %if liveInstance.keypair_name != None: <tr> - <td> Keypair fingerprint:</td> - <td> ${liveInstance.keypair_fingerprint} </td> + <td> Keypair file name:</td> + <td> ${liveInstance.keypair_name} </td> </tr> %endif - %if liveInstance.keypair_material != None: - <tr> - <td> Keypair private key:</td> - <td> - <div id="shortComment2"> - <a onclick="document.getElementById('fullComment2').style.display = 'block'; - document.getElementById('shortComment2').style.display = 'none'; return 0" - href="javascript:void(0)"> - + Show - </a> - </div> - <div id="fullComment2" style="DISPLAY: none"> - <nobr><b>${liveInstance.keypair_material}</b></nobr><br/> - <a onclick="document.getElementById('shortComment2').style.display = 'block'; - document.getElementById('fullComment2').style.display = 'none'; return 0;" - href="javascript:void(0)"> - - Hide - </a> - </div> - </td> - </tr> - %endif + </table> %else: There is no live instance under that name. diff -r f87f1eed6e0c -r 81e8a5e42900 templates/root/index.mako --- a/templates/root/index.mako Fri Aug 28 17:45:40 2009 -0400 +++ b/templates/root/index.mako Fri Sep 04 09:54:27 2009 -0400 @@ -2,9 +2,9 @@ <%def name="init()"> <% - self.has_left_panel=True - self.has_right_panel=True - self.active_view="analysis" + self.has_left_panel=False + self.has_right_panel=False + self.active_view="cloud" %> %if trans.app.config.require_login and not trans.user: <script type="text/javascript"> @@ -37,7 +37,8 @@ elif m_c is not None: center_url = h.url_for( controller=m_c, action=m_a ) else: - center_url = h.url_for( '/static/welcome.html' ) + #center_url = h.url_for( '/static/welcome.html' ) + center_url = h.url_for( controller='cloud', action='list' ) %> <iframe name="galaxy_main" id="galaxy_main" frameborder="0" style="position: absolute; width: 100%; height: 100%;" src="${center_url}"> </iframe>

1 0

[hg] galaxy 3061: Merge
by Greg Von Kuster 23 Nov '09

23 Nov '09

details: http://www.bx.psu.edu/hg/galaxy/rev/2d1b957b8448 changeset: 3061:2d1b957b8448 user: Enis Afgan <afgane(a)gmail.com> date: Fri Sep 11 15:25:21 2009 -0400 description: Merge diffstat: lib/galaxy/web/controllers/cloud.py | 68 ++++++++++++++++++++-------------- templates/cloud/configure_cloud.mako | 3 +- 2 files changed, 42 insertions(+), 29 deletions(-) diffs (161 lines): diff -r 81e8a5e42900 -r 2d1b957b8448 lib/galaxy/web/controllers/cloud.py --- a/lib/galaxy/web/controllers/cloud.py Fri Sep 04 09:54:27 2009 -0400 +++ b/lib/galaxy/web/controllers/cloud.py Fri Sep 11 15:25:21 2009 -0400 @@ -107,20 +107,20 @@ instance.availability_zone = stores[0].availability_zone # Bc. all EBS volumes need to be in the same avail. zone, just check 1st instance.type = size conn = get_connection( trans ) - # Get or setup appropriate security group - sg = list() # security group list + # If not existent, setup galaxy security group try: - gSecurityGroup = conn.create_security_group('galaxy', 'Galaxy security group') - gSecurityGroup.authorize( 'tcp', 80, 80, '0.0.0.0/0' ) - sg.append( gSecurityGroup ) + gSecurityGroup = conn.create_security_group('galaxy', 'Security group for Galaxy.') + gSecurityGroup.authorize( 'tcp', 80, 80, '0.0.0.0/0' ) # Open HTTP port + gSecurityGroup.authorize( 'tcp', 22, 22, '0.0.0.0/0' ) # Open SSH port except: - sgs = rs = conn.get_all_security_groups() - for i in range( len( sgs ) ): - if sgs[i].name == "galaxy": - sg.append( sgs[i] ) - break # only 1 security group w/ this name can exist, so continue + pass +# sgs = conn.get_all_security_groups() +# for i in range( len( sgs ) ): +# if sgs[i].name == "galaxy": +# sg.append( sgs[i] ) +# break # only 1 security group w/ this name can exist, so continue - #reservation = conn.run_instances( image_id=instance.image, key_name=instance.keypair_name, security_groups=sg, instance_type=instance.type, placement=instance.availability_zone ) + #reservation = conn.run_instances( image_id=instance.image, key_name=instance.keypair_name, security_groups=['galaxy'], instance_type=instance.type, placement=instance.availability_zone ) instance.launch_time = datetime.utcnow() uci.launch_time = instance.launch_time #instance.reservation = str( reservation.instances[0] ) @@ -133,7 +133,6 @@ session = trans.sa_session session.save_or_update( instance ) session.flush() - #error( "Starting instance '%s' is not supported yet." % uci.name ) trans.log_event( "User started cloud instance '%s'" % uci.name ) trans.set_message( "Galaxy instance '%s' started. NOTE: Please wait about 2-3 minutes for the instance to " @@ -225,9 +224,11 @@ storage.user = user storage.uci = uci storage.size = volSize - # TODO: get correct values from AWS + storage.availability_zone = "us-east-1a" # TODO: Give user choice here. Also, enable region selection. + conn = get_connection( trans ) + #conn.create_volume( volSize, storage.availability_zone, snapshot=None ) + # TODO: get correct value from AWS storage.volume_id = "made up" - storage.availability_zone = "avail zone" # Persist session = trans.sa_session session.save_or_update( uci ) @@ -863,7 +864,7 @@ def get_uci( trans, id, check_ownership=True ): """ - Get a UCI from the database by id, verifying ownership. + Get a UCI object from the database by id, verifying ownership. """ # Check if 'id' is in int (i.e., it was called from this program) or # it was passed from the web (in which case decode it) @@ -944,20 +945,19 @@ def get_keypair_name( trans ): """ - Generate X.509 Certificate (i.e., keypair) using user's default credentials + Generate keypair using user's default credentials """ conn = get_connection( trans ) try: key_pair = conn.get_key_pair( 'galaxy-keypair' ) except: # No keypair under this name exists so create it key_pair = conn.create_key_pair( 'galaxy-keypair' ) -# log.debug( key_pair.name ) return key_pair.name def update_instance_state( trans, id ): """ - Update state of instances associated with given UCI id and store it in local database. Also update + Update state of instances associated with given UCI id and store state in local database. Also update state of the given UCI. """ uci = get_uci( trans, id ) @@ -971,36 +971,48 @@ # Update status of instance cloudInstance.update() dbInstances.state = cloudInstance.state - log.debug( "Processing instance %i, instance current state: %s" % i, cloudInstance.state ) - - # If instance is now running, update/process instance (i.e., mount file system, start Galaxy, update DB with DNS) - if oldState=="pending" and dbInstances.state=="running": - update_instance( trans, dbInstances, cloudInstance ) - + log.debug( "Processing instance %i, current instance state: %s" % i, cloudInstance.state ) # Update state of UCI (TODO: once more than 1 instance is assoc. w/ 1 UCI, this will be need to be updated differently) uci.state = dbInstances.state - # Persist session = trans.sa_session session.save_or_update( dbInstances ) session.save_or_update( uci ) session.flush() + # If instance is now running, update/process instance (i.e., mount file system, start Galaxy, update DB with DNS) + if oldState=="pending" and dbInstances.state=="running": + update_instance( trans, dbInstances, cloudInstance, conn, uci ) -def update_instance( trans, dbInstance, cloudInstance ): + +def update_instance( trans, dbInstance, cloudInstance, conn, uci ): """ - Update instance: mount file system, start Galaxy and update local DB w/ DNS info + Update instance: connect EBS volume, mount file system, start Galaxy, and update local DB w/ DNS info Keyword arguments: trans -- current transaction dbInstance -- object of 'instance' as it is stored in local database cloudInstance -- object of 'instance' as it resides in the cloud. Functions supported by the cloud API can be instantiated directly on this object. + conn -- cloud connection object + uci -- UCI object """ dbInstance.public_dns = cloudInstance.dns_name dbInstance.private_dns = cloudInstance.private_dns_name - # TODO: mount storage + # TODO: connect EBS volume to instance + # Attach storage volume(s) to instance + stores = get_stores( trans, uci ) + for i, store in enumerate( stores ): + log.debug( "Attaching volume %s to instance %s." % store.volume_id, store.uci_id ) + mtnDevice = '/dev/sdb'+str(i) + conn.attach_volume( store.volume_id, store.uci_id, mntDevice ) + + # Wait until instances have attached and add file system + + + + # TODO: mount storage through ZFS # TODO: start Galaxy # Persist diff -r 81e8a5e42900 -r 2d1b957b8448 templates/cloud/configure_cloud.mako --- a/templates/cloud/configure_cloud.mako Fri Sep 04 09:54:27 2009 -0400 +++ b/templates/cloud/configure_cloud.mako Fri Sep 11 15:25:21 2009 -0400 @@ -125,7 +125,8 @@ %> </td> <td><div align="right"> - %for i, instance in enumerate( liveInstance.instance ): + %for j, instance in enumerate( liveInstance.instance ): + ## TODO: Once more instances will be running under the same liveInstance, additional logic will need to be added to account for that %if instance.state == "running": <a class="action-button" href="http://${instance.public_dns}" target="_blank"> <span>Connect to!</span>

1 0

[hg] galaxy 3059: Minor clean up.
by Greg Von Kuster 23 Nov '09

23 Nov '09

details: http://www.bx.psu.edu/hg/galaxy/rev/5f39f1dd8f57 changeset: 3059:5f39f1dd8f57 user: Enis Afgan <afgane(a)gmail.com> date: Fri Sep 11 15:02:15 2009 -0400 description: Minor clean up. diffstat: lib/galaxy/web/controllers/cloud.py | 68 ++++++++++++++++++++-------------- templates/cloud/configure_cloud.mako | 3 +- 2 files changed, 42 insertions(+), 29 deletions(-) diffs (161 lines): diff -r 0d8572e68b34 -r 5f39f1dd8f57 lib/galaxy/web/controllers/cloud.py --- a/lib/galaxy/web/controllers/cloud.py Tue Sep 08 09:50:05 2009 -0400 +++ b/lib/galaxy/web/controllers/cloud.py Fri Sep 11 15:02:15 2009 -0400 @@ -107,20 +107,20 @@ instance.availability_zone = stores[0].availability_zone # Bc. all EBS volumes need to be in the same avail. zone, just check 1st instance.type = size conn = get_connection( trans ) - # Get or setup appropriate security group - sg = list() # security group list + # If not existent, setup galaxy security group try: - gSecurityGroup = conn.create_security_group('galaxy', 'Galaxy security group') - gSecurityGroup.authorize( 'tcp', 80, 80, '0.0.0.0/0' ) - sg.append( gSecurityGroup ) + gSecurityGroup = conn.create_security_group('galaxy', 'Security group for Galaxy.') + gSecurityGroup.authorize( 'tcp', 80, 80, '0.0.0.0/0' ) # Open HTTP port + gSecurityGroup.authorize( 'tcp', 22, 22, '0.0.0.0/0' ) # Open SSH port except: - sgs = rs = conn.get_all_security_groups() - for i in range( len( sgs ) ): - if sgs[i].name == "galaxy": - sg.append( sgs[i] ) - break # only 1 security group w/ this name can exist, so continue + pass +# sgs = conn.get_all_security_groups() +# for i in range( len( sgs ) ): +# if sgs[i].name == "galaxy": +# sg.append( sgs[i] ) +# break # only 1 security group w/ this name can exist, so continue - #reservation = conn.run_instances( image_id=instance.image, key_name=instance.keypair_name, security_groups=sg, instance_type=instance.type, placement=instance.availability_zone ) + #reservation = conn.run_instances( image_id=instance.image, key_name=instance.keypair_name, security_groups=['galaxy'], instance_type=instance.type, placement=instance.availability_zone ) instance.launch_time = datetime.utcnow() uci.launch_time = instance.launch_time #instance.reservation = str( reservation.instances[0] ) @@ -133,7 +133,6 @@ session = trans.sa_session session.save_or_update( instance ) session.flush() - #error( "Starting instance '%s' is not supported yet." % uci.name ) trans.log_event( "User started cloud instance '%s'" % uci.name ) trans.set_message( "Galaxy instance '%s' started. NOTE: Please wait about 2-3 minutes for the instance to " @@ -225,9 +224,11 @@ storage.user = user storage.uci = uci storage.size = volSize - # TODO: get correct values from AWS + storage.availability_zone = "us-east-1a" # TODO: Give user choice here. Also, enable region selection. + conn = get_connection( trans ) + #conn.create_volume( volSize, storage.availability_zone, snapshot=None ) + # TODO: get correct value from AWS storage.volume_id = "made up" - storage.availability_zone = "avail zone" # Persist session = trans.sa_session session.save_or_update( uci ) @@ -863,7 +864,7 @@ def get_uci( trans, id, check_ownership=True ): """ - Get a UCI from the database by id, verifying ownership. + Get a UCI object from the database by id, verifying ownership. """ # Check if 'id' is in int (i.e., it was called from this program) or # it was passed from the web (in which case decode it) @@ -944,20 +945,19 @@ def get_keypair_name( trans ): """ - Generate X.509 Certificate (i.e., keypair) using user's default credentials + Generate keypair using user's default credentials """ conn = get_connection( trans ) try: key_pair = conn.get_key_pair( 'galaxy-keypair' ) except: # No keypair under this name exists so create it key_pair = conn.create_key_pair( 'galaxy-keypair' ) -# log.debug( key_pair.name ) return key_pair.name def update_instance_state( trans, id ): """ - Update state of instances associated with given UCI id and store it in local database. Also update + Update state of instances associated with given UCI id and store state in local database. Also update state of the given UCI. """ uci = get_uci( trans, id ) @@ -971,36 +971,48 @@ # Update status of instance cloudInstance.update() dbInstances.state = cloudInstance.state - log.debug( "Processing instance %i, instance current state: %s" % i, cloudInstance.state ) - - # If instance is now running, update/process instance (i.e., mount file system, start Galaxy, update DB with DNS) - if oldState=="pending" and dbInstances.state=="running": - update_instance( trans, dbInstances, cloudInstance ) - + log.debug( "Processing instance %i, current instance state: %s" % i, cloudInstance.state ) # Update state of UCI (TODO: once more than 1 instance is assoc. w/ 1 UCI, this will be need to be updated differently) uci.state = dbInstances.state - # Persist session = trans.sa_session session.save_or_update( dbInstances ) session.save_or_update( uci ) session.flush() + # If instance is now running, update/process instance (i.e., mount file system, start Galaxy, update DB with DNS) + if oldState=="pending" and dbInstances.state=="running": + update_instance( trans, dbInstances, cloudInstance, conn, uci ) -def update_instance( trans, dbInstance, cloudInstance ): + +def update_instance( trans, dbInstance, cloudInstance, conn, uci ): """ - Update instance: mount file system, start Galaxy and update local DB w/ DNS info + Update instance: connect EBS volume, mount file system, start Galaxy, and update local DB w/ DNS info Keyword arguments: trans -- current transaction dbInstance -- object of 'instance' as it is stored in local database cloudInstance -- object of 'instance' as it resides in the cloud. Functions supported by the cloud API can be instantiated directly on this object. + conn -- cloud connection object + uci -- UCI object """ dbInstance.public_dns = cloudInstance.dns_name dbInstance.private_dns = cloudInstance.private_dns_name - # TODO: mount storage + # TODO: connect EBS volume to instance + # Attach storage volume(s) to instance + stores = get_stores( trans, uci ) + for i, store in enumerate( stores ): + log.debug( "Attaching volume %s to instance %s." % store.volume_id, store.uci_id ) + mtnDevice = '/dev/sdb'+str(i) + conn.attach_volume( store.volume_id, store.uci_id, mntDevice ) + + # Wait until instances have attached and add file system + + + + # TODO: mount storage through ZFS # TODO: start Galaxy # Persist diff -r 0d8572e68b34 -r 5f39f1dd8f57 templates/cloud/configure_cloud.mako --- a/templates/cloud/configure_cloud.mako Tue Sep 08 09:50:05 2009 -0400 +++ b/templates/cloud/configure_cloud.mako Fri Sep 11 15:02:15 2009 -0400 @@ -125,7 +125,8 @@ %> </td> <td><div align="right"> - %for i, instance in enumerate( liveInstance.instance ): + %for j, instance in enumerate( liveInstance.instance ): + ## TODO: Once more instances will be running under the same liveInstance, additional logic will need to be added to account for that %if instance.state == "running": <a class="action-button" href="http://${instance.public_dns}" target="_blank"> <span>Connect to!</span>

1 0

[hg] galaxy 3058: Added most of code needed to interact with EC2...
by Greg Von Kuster 23 Nov '09

23 Nov '09

details: http://www.bx.psu.edu/hg/galaxy/rev/0d8572e68b34 changeset: 3058:0d8572e68b34 user: Enis Afgan <afgane(a)gmail.com> date: Tue Sep 08 09:50:05 2009 -0400 description: Added most of code needed to interact with EC2 (have not tested it though). Also, modified main page to directly point to cloud tab. diffstat: lib/galaxy/web/controllers/cloud.py | 337 ++++++++++++++++++++---------------- static/june_2007_style/blue/base.css | 9 + templates/base_panels.mako | 8 +- templates/cloud/configure_cloud.mako | 56 +++-- templates/cloud/viewInstance.mako | 48 +--- templates/root/index.mako | 9 +- 6 files changed, 255 insertions(+), 212 deletions(-) diffs (705 lines): diff -r f87f1eed6e0c -r 0d8572e68b34 lib/galaxy/web/controllers/cloud.py --- a/lib/galaxy/web/controllers/cloud.py Fri Aug 28 17:45:40 2009 -0400 +++ b/lib/galaxy/web/controllers/cloud.py Tue Sep 08 09:50:05 2009 -0400 @@ -37,31 +37,33 @@ Render cloud main page (management of cloud resources) """ user = trans.get_user() + pendingInstances = trans.sa_session.query( model.UCI ) \ + .filter_by( user=user, state="pending" ) \ + .all() + + for i in range( len( pendingInstances ) ): + update_instance_state( trans, pendingInstances[i].id ) + cloudCredentials = trans.sa_session.query( model.CloudUserCredentials ) \ + .filter_by( user=user ) \ .order_by( desc( model.CloudUserCredentials.c.update_time ) ) \ .all() - prevInstances = trans.sa_session.query( model.UCI ) \ - .filter_by( user=user, state="available" ) \ - .order_by( desc( model.UCI.c.update_time ) ) \ - .all() #TODO: diff between live and previous instances - - liveInstances = trans.sa_session.query( model.CloudInstance ) \ - .filter_by( user=user ) \ - .filter( or_(model.CloudInstance.c.state=="running", model.CloudInstance.c.state=="pending") ) \ - .order_by( desc( model.CloudInstance.c.launch_time ) ) \ - .all() - liveInstances = trans.sa_session.query( model.UCI ) \ .filter_by( user=user ) \ .filter( or_(model.UCI.c.state=="running", model.UCI.c.state=="pending") ) \ .order_by( desc( model.UCI.c.launch_time ) ) \ .all() - + + prevInstances = trans.sa_session.query( model.UCI ) \ + .filter_by( user=user, state="available" ) \ + .order_by( desc( model.UCI.c.update_time ) ) \ + .all() + return trans.fill_template( "cloud/configure_cloud.mako", cloudCredentials = cloudCredentials, - prevInstances = prevInstances, - liveInstances = liveInstances ) + liveInstances = liveInstances, + prevInstances = prevInstances ) @web.expose @web.require_login( "use Galaxy cloud" ) @@ -77,162 +79,94 @@ newDefault.defaultCred = True trans.sa_session.flush() trans.set_message( "Credentials '%s' set as default." % newDefault.name ) - + + # TODO: Fix bug that when this function returns, top Galaxy tab bar is missing from the webpage return self.list( trans ) #trans.fill_template( "cloud/configure_cloud.mako", #awsCredentials = awsCredentials ) - - @web.expose - @web.require_login( "use Galaxy workflows" ) - def list_for_run( self, trans ): - """ - Render workflow list for analysis view (just allows running workflow - or switching to management view) - """ - user = trans.get_user() - workflows = trans.sa_session.query( model.StoredWorkflow ) \ - .filter_by( user=user, deleted=False ) \ - .order_by( desc( model.StoredWorkflow.c.update_time ) ) \ - .all() - shared_by_others = trans.sa_session \ - .query( model.StoredWorkflowUserShareAssociation ) \ - .filter_by( user=user ) \ - .filter( model.StoredWorkflow.c.deleted == False ) \ - .order_by( desc( model.StoredWorkflow.c.update_time ) ) \ - .all() - return trans.fill_template( "workflow/list_for_run.mako", - workflows = workflows, - shared_by_others = shared_by_others ) - - @web.expose - @web.require_login( "use Galaxy workflows" ) - def share( self, trans, id, email="" ): - msg = mtype = None - # Load workflow from database - stored = get_stored_workflow( trans, id ) - if email: - other = model.User.filter( and_( model.User.table.c.email==email, - model.User.table.c.deleted==False ) ).first() - if not other: - mtype = "error" - msg = ( "User '%s' does not exist" % email ) - elif other == trans.get_user(): - mtype = "error" - msg = ( "You cannot share a workflow with yourself" ) - elif trans.sa_session.query( model.StoredWorkflowUserShareAssociation ) \ - .filter_by( user=other, stored_workflow=stored ).count() > 0: - mtype = "error" - msg = ( "Workflow already shared with '%s'" % email ) - else: - share = model.StoredWorkflowUserShareAssociation() - share.stored_workflow = stored - share.user = other - session = trans.sa_session - session.save_or_update( share ) - session.flush() - trans.set_message( "Workflow '%s' shared with user '%s'" % ( stored.name, other.email ) ) - return trans.response.send_redirect( url_for( controller='workflow', action='sharing', id=id ) ) - return trans.fill_template( "workflow/share.mako", - message = msg, - messagetype = mtype, - stored=stored, - email=email ) - - @web.expose - @web.require_login( "use Galaxy workflows" ) - def sharing( self, trans, id, **kwargs ): - session = trans.sa_session - stored = get_stored_workflow( trans, id ) - if 'enable_import_via_link' in kwargs: - stored.importable = True - stored.flush() - elif 'disable_import_via_link' in kwargs: - stored.importable = False - stored.flush() - elif 'unshare_user' in kwargs: - user = session.query( model.User ).get( trans.security.decode_id( kwargs['unshare_user' ] ) ) - if not user: - error( "User not found for provided id" ) - association = session.query( model.StoredWorkflowUserShareAssociation ) \ - .filter_by( user=user, stored_workflow=stored ).one() - session.delete( association ) - session.flush() - return trans.fill_template( "workflow/sharing.mako", - stored=stored ) - - @web.expose - @web.require_login( "use Galaxy workflows" ) - def imp( self, trans, id, **kwargs ): - session = trans.sa_session - stored = get_stored_workflow( trans, id, check_ownership=False ) - if stored.importable == False: - error( "The owner of this workflow has disabled imports via this link" ) - elif stored.user == trans.user: - error( "You are already the owner of this workflow, can't import" ) - elif stored.deleted: - error( "This workflow has been deleted, can't import" ) - elif session.query( model.StoredWorkflowUserShareAssociation ) \ - .filter_by( user=trans.user, stored_workflow=stored ).count() > 0: - error( "This workflow is already shared with you" ) - else: - share = model.StoredWorkflowUserShareAssociation() - share.stored_workflow = stored - share.user = trans.user - session = trans.sa_session - session.save_or_update( share ) - session.flush() - # Redirect to load galaxy frames. - return trans.response.send_redirect( url_for( controller='workflow' ) ) - + + @web.expose @web.require_login( "start Galaxy cloud instance" ) - def start( self, trans, id, size='small' ): + def start( self, trans, id, size='' ): """ Start a new cloud resource instance """ + # TODO: Add choice of instance size before starting one + #if size: user = trans.get_user() mi = get_mi( trans, size ) uci = get_uci( trans, id ) - stores = get_stores( trans, uci ) #TODO: handle list! + stores = get_stores( trans, uci ) + log.debug(self.app.config.job_working_directory) + if len(stores) is not 0: + instance = model.CloudInstance() + instance.user = user + instance.image = mi + instance.uci = uci + instance.keypair_name = get_keypair_name( trans ) + instance.availability_zone = stores[0].availability_zone # Bc. all EBS volumes need to be in the same avail. zone, just check 1st + instance.type = size + conn = get_connection( trans ) + # Get or setup appropriate security group + sg = list() # security group list + try: + gSecurityGroup = conn.create_security_group('galaxy', 'Galaxy security group') + gSecurityGroup.authorize( 'tcp', 80, 80, '0.0.0.0/0' ) + sg.append( gSecurityGroup ) + except: + sgs = rs = conn.get_all_security_groups() + for i in range( len( sgs ) ): + if sgs[i].name == "galaxy": + sg.append( sgs[i] ) + break # only 1 security group w/ this name can exist, so continue + + #reservation = conn.run_instances( image_id=instance.image, key_name=instance.keypair_name, security_groups=sg, instance_type=instance.type, placement=instance.availability_zone ) + instance.launch_time = datetime.utcnow() + uci.launch_time = instance.launch_time + #instance.reservation = str( reservation.instances[0] ) + instance.state = "pending" + #instance.state = reservation.instances[0].state + uci.state = instance.state + # TODO: After instance boots up, need to update status, DNS and attach EBS + + # Persist + session = trans.sa_session + session.save_or_update( instance ) + session.flush() + #error( "Starting instance '%s' is not supported yet." % uci.name ) + + trans.log_event( "User started cloud instance '%s'" % uci.name ) + trans.set_message( "Galaxy instance '%s' started. NOTE: Please wait about 2-3 minutes for the instance to " + "start up and then refresh this page. A button to connect to the instance will then appear alongside " + "instance description." % uci.name ) + return self.list( trans ) - instance = model.CloudInstance() - instance.user = user - instance.image = mi - instance.uci = uci - # TODO: get real value from AWS - instance.state = "pending" - uci.state = instance.state - uci.launch_time = datetime.utcnow() - instance.launch_time = datetime.utcnow() - instance.availability_zone = stores.availability_zone - instance.type = size - - # Persist - session = trans.sa_session - session.save_or_update( instance ) - session.flush() - #error( "Starting instance '%s' is not supported yet." % uci.name ) +# return trans.show_form( +# web.FormBuilder( web.url_for(), "Start instance size", submit_text="Start" ) +# .add_input( "radio","Small","size", value='small' ) +# .add_input( "radio","Medium","size", value='medium' ) ) - trans.log_event( "User started cloud instance '%s'" % uci.name ) - trans.set_message( "Galaxy instance '%s' started." % uci.name ) - return self.list( trans ) @web.expose @web.require_login( "stop Galaxy cloud instance" ) def stop( self, trans, id ): """ - Stop a cloud resource instance + Stop a cloud instance. This implies stopping Galaxy servcer and unmounting relevant file system(s) """ uci = get_uci( trans, id ) instances = get_instances( trans, uci ) #TODO: handle list! - instances.state = 'done' + instances.stop() + instances.state = 'terminated' instances.stop_time = datetime.utcnow() + # Reset relevant UCI fields uci.state = 'available' uci.launch_time = None + # Persist session = trans.sa_session + session.save_or_update( instances ) session.save_or_update( uci ) - session.save_or_update( instances ) session.flush() trans.log_event( "User stopped cloud instance '%s'" % uci.name ) trans.set_message( "Galaxy instance '%s' stopped." % uci.name ) @@ -429,11 +363,11 @@ """ View details about running instance """ - instance = get_uci( trans, id ) - log.debug ( instance.name ) + uci = get_uci( trans, id ) + instances = get_instances( trans, uci ) # TODO: Handle list (will probably need to be done in mako template) return trans.fill_template( "cloud/viewInstance.mako", - liveInstance = instance ) + liveInstance = instances ) @web.expose @@ -931,7 +865,10 @@ """ Get a UCI from the database by id, verifying ownership. """ - id = trans.security.decode_id( id ) + # Check if 'id' is in int (i.e., it was called from this program) or + # it was passed from the web (in which case decode it) + if not isinstance( id, int ): + id = trans.security.decode_id( id ) live = trans.sa_session.query( model.UCI ).get( id ) if not live: @@ -956,28 +893,121 @@ def get_stores( trans, uci ): """ - Get store objects/tables that are connected to uci object/table + Get stores objects that are connected to uci object """ user = trans.get_user() stores = trans.sa_session.query( model.CloudStore ) \ .filter_by( user=user, uci_id=uci.id ) \ - .first() - #.all() #TODO: return all but need to edit calling method(s) to handle list + .all() return stores def get_instances( trans, uci ): """ - Get instance objects/tables that are connected to uci object/table + Get objects of instances that are pending or running and are connected to uci object """ user = trans.get_user() instances = trans.sa_session.query( model.CloudInstance ) \ .filter_by( user=user, uci_id=uci.id ) \ + .filter( or_(model.CloudInstance.table.c.state=="running", model.CloudInstance.table.c.state=="pending" ) ) \ .first() #.all() #TODO: return all but need to edit calling method(s) to handle list return instances + +def get_cloud_instance( conn, instance_id ): + """ + Returns a cloud instance representation of the instance id, i.e., cloud instance objects that cloud API can be invoked on + """ + # get_all_instances func. takes a list of desired instance id's, so create a list first + idLst = list() + idLst.append( instance_id ) + # Retrieve cloud instance based on passed instance id + cloudInstance = conn.get_all_instances( idLst ) + return cloudInstance[0] + +def get_connection( trans ): + """ + Establishes EC2 conncection using user's default credentials + """ + user = trans.get_user() + creds = trans.sa_session.query(model.CloudUserCredentials).filter_by(user=user, defaultCred=True).first() + if creds: + a_key = creds.access_key + s_key = creds.secret_key + conn = EC2Connection( a_key, s_key ) + return conn + else: + error( "You must specify default credentials before starting an instance." ) + return 0 + +def get_keypair_name( trans ): + """ + Generate X.509 Certificate (i.e., keypair) using user's default credentials + """ + conn = get_connection( trans ) + try: + key_pair = conn.get_key_pair( 'galaxy-keypair' ) + except: # No keypair under this name exists so create it + key_pair = conn.create_key_pair( 'galaxy-keypair' ) +# log.debug( key_pair.name ) + + return key_pair.name + +def update_instance_state( trans, id ): + """ + Update state of instances associated with given UCI id and store it in local database. Also update + state of the given UCI. + """ + uci = get_uci( trans, id ) + # Get list of instances associated with given uci as they are stored in local database + dbInstances = get_instances( trans, uci ) # TODO: handle list (currently only 1 instance can correspond to 1 UCI) + oldState = dbInstances.state + # Establish connection with cloud + conn = get_connection( trans ) + # Get actual instance from the cloud + cloudInstance = get_cloud_instance( conn, dbInstances.instance_id ) + # Update status of instance + cloudInstance.update() + dbInstances.state = cloudInstance.state + log.debug( "Processing instance %i, instance current state: %s" % i, cloudInstance.state ) + + # If instance is now running, update/process instance (i.e., mount file system, start Galaxy, update DB with DNS) + if oldState=="pending" and dbInstances.state=="running": + update_instance( trans, dbInstances, cloudInstance ) + + # Update state of UCI (TODO: once more than 1 instance is assoc. w/ 1 UCI, this will be need to be updated differently) + uci.state = dbInstances.state + + # Persist + session = trans.sa_session + session.save_or_update( dbInstances ) + session.save_or_update( uci ) + session.flush() + + +def update_instance( trans, dbInstance, cloudInstance ): + """ + Update instance: mount file system, start Galaxy and update local DB w/ DNS info + + Keyword arguments: + trans -- current transaction + dbInstance -- object of 'instance' as it is stored in local database + cloudInstance -- object of 'instance' as it resides in the cloud. Functions supported by the cloud API can be + instantiated directly on this object. + """ + dbInstance.public_dns = cloudInstance.dns_name + dbInstance.private_dns = cloudInstance.private_dns_name + + # TODO: mount storage + # TODO: start Galaxy + + # Persist + session = trans.sa_session + session.save_or_update( dbInstance ) + session.flush() + def attach_ordered_steps( workflow, steps ): ordered_steps = order_workflow_steps( steps ) if ordered_steps: @@ -1104,3 +1134,8 @@ cleanup( prefix, input.cases[current_case].inputs, group_values ) cleanup( "", inputs, values ) return associations + + + + + diff -r f87f1eed6e0c -r 0d8572e68b34 static/june_2007_style/blue/base.css --- a/static/june_2007_style/blue/base.css Fri Aug 28 17:45:40 2009 -0400 +++ b/static/june_2007_style/blue/base.css Tue Sep 08 09:50:05 2009 -0400 @@ -57,6 +57,15 @@ table.colored tr.header{background:#ebd9b2;background-image:url(form_title_bg.png);background-repeat:repeat-x;background-position:top;border-bottom:solid #d8b365 1px;font-weight:bold;} table.colored tr{background:white;} table.colored tr.odd_row{background:#DADFEF;} +table.noBottomHR{border-top:solid #d8b365 1px;} +table.noBottomHR td,table.noBottomHR th{text-align:left;padding:5px;} +table.noBottomHR tr.header{background:#ebd9b2;background-image:url(form_title_bg.png);background-repeat:repeat-x;background-position:top;border-bottom:solid #d8b365 1px;font-weight:bold;} +table.noBottomHR tr{background:white;} +table.noBottomHR tr.odd_row{background:#DADFEF;} +table.noHR td,table.noHR th{text-align:left;padding:5px;} +table.noHR tr.header{background:#ebd9b2;background-image:url(form_title_bg.png);background-repeat:repeat-x;background-position:top;border-bottom:solid #d8b365 1px;font-weight:bold;} +table.noHR tr{background:white;} +table.noHR tr.odd_row{background:#DADFEF;} div.debug{margin:10px;padding:5px;background:#FFFF99;border:solid #FFFF33 1px;color:black;} div.odd_row{background:#DADFEF;} #footer{display:none;} diff -r f87f1eed6e0c -r 0d8572e68b34 templates/base_panels.mako --- a/templates/base_panels.mako Fri Aug 28 17:45:40 2009 -0400 +++ b/templates/base_panels.mako Tue Sep 08 09:50:05 2009 -0400 @@ -16,7 +16,7 @@ </%def> ## Default title -<%def name="title()">Galaxy</%def> +<%def name="title()">Galaxy Cloud</%def> ## Default stylesheets <%def name="stylesheets()"> @@ -135,11 +135,11 @@ <td class="${cls}" style="${style}"><a target="${target}" href="${href}">${display}</a></td> </%def> - ${tab( "analysis", "Analyze Data", h.url_for( controller='root', action='index' ))} + ##${tab( "analysis", "Analyze Data", h.url_for( controller='root', action='index' ))} - ${tab( "workflow", "Workflow", h.url_for( controller='workflow', action='index' ))} + ##${tab( "workflow", "Workflow", h.url_for( controller='workflow', action='index' ))} - ${tab( "libraries", "Data Libraries", h.url_for( controller='library', action='index' ))} + ##${tab( "libraries", "Data Libraries", h.url_for( controller='library', action='index' ))} ${tab( "cloud", "Cloud", h.url_for( controller='cloud', action='index' ))} diff -r f87f1eed6e0c -r 0d8572e68b34 templates/cloud/configure_cloud.mako --- a/templates/cloud/configure_cloud.mako Fri Aug 28 17:45:40 2009 -0400 +++ b/templates/cloud/configure_cloud.mako Tue Sep 08 09:50:05 2009 -0400 @@ -82,12 +82,14 @@ <colgroup width="40%"></colgroup> <colgroup width="15%"></colgroup> <colgroup width="10%"></colgroup> - <colgroup width="35%"></colgroup> + <colgroup width="25%"></colgroup> + <colgroup width="10%"></colgroup> <tr class="header"> <th>Live instances</th> <th>Storage size (GB)</th> <th>State</th> <th>Alive since</th> + <th></th> <th></th> </tr> %if liveInstances: @@ -97,30 +99,40 @@ ${liveInstance.name} <a id="li-${i}-popup" class="popup-arrow" style="display: none;">▼</a> </td> - <td>${str(liveInstance.total_size)}</td>  + <td>${str(liveInstance.total_size)}</td> <td>${str(liveInstance.state)}</td> <td> - ${str(liveInstance.launch_time)[:16]} + ##${str(liveInstance.launch_time)[:16]} <% - from datetime import datetime - from datetime import timedelta + #from datetime import datetime + #from datetime import timedelta # DB stores all times in GMT, so adjust for difference (4 hours) - adjustedStarttime = liveInstance.update_time - timedelta(hours=4) + #adjustedStarttime = liveInstance.update_time - timedelta(hours=4) - # (NOT CURRENTLY USED BLOCK OF CODE) Calculate time difference from now - delta = datetime.now() - adjustedStarttime + #delta = datetime.now() - adjustedStarttime #context.write( str(datetime.utcnow() ) ) #context.write( str(delta) ) # This is where current time and since duration is calculated - #context.write( str( liveInstance.launch_time ) ) - context.write( ' UTC (' ) - context.write( str(h.date.distance_of_time_in_words (liveInstance.launch_time, h.date.datetime.utcnow() ) ) ) - - - %>) + if liveInstance.launch_time is None: + context.write( 'N/A' ) + else: + context.write( str( liveInstance.launch_time )[:16] ) + context.write( ' UTC (' ) + context.write( str(h.date.distance_of_time_in_words (liveInstance.launch_time, h.date.datetime.utcnow() ) ) ) + context.write( ')' ) + %> </td> + <td><div align="right"> + %for i, instance in enumerate( liveInstance.instance ): + %if instance.state == "running": + <a class="action-button" href="http://${instance.public_dns}" target="_blank"> + <span>Connect to!</span> + <img src="${h.url_for('/static/images/silk/resultset_next.png')}" /></a></div> + %endif + %endfor + </td> <td> <div popupmenu="li-${i}-popup"> <a class="action-button" confirm="Are you sure you want to stop instance '${liveInstance.name}'?" href="${h.url_for( action='stop', id=trans.security.encode_id(liveInstance.id) )}">Stop</a> @@ -139,21 +151,21 @@ ## ***************************************************** ## Manage previously configured instances - <table class="mange-table colored" border="0" cellspacing="0" cellpadding="0" width="100%"> + <table class="mange-table noHR" border="0" cellspacing="0" cellpadding="0" width="100%"> <colgroup width="40%"></colgroup> <colgroup width="15%"></colgroup> <colgroup width="10%"></colgroup> <colgroup width="35%"></colgroup> - <tr class="header"> - <th>Previously configured instances</th> + ##<tr class="header"> + ##<th>Previously configured instances</th> ##<th>Storage size (GB)</th> ##<th>State</th> ##<th>Alive since</th> - <th></th> - <th></th> - <th></th> - <th></th> - </tr> + ##<th></th> + ##<th></th> + ##<th></th> + ##<th></th> + ##</tr> %if prevInstances: %for i, prevInstance in enumerate( prevInstances ): diff -r f87f1eed6e0c -r 0d8572e68b34 templates/cloud/viewInstance.mako --- a/templates/cloud/viewInstance.mako Fri Aug 28 17:45:40 2009 -0400 +++ b/templates/cloud/viewInstance.mako Tue Sep 08 09:50:05 2009 -0400 @@ -1,6 +1,13 @@ <%inherit file="/base.mako"/> <%def name="title()">Live instance details</%def> +<% + # Because of the one-to-many relationship between liveInstance (i.e., UCI) and actual instances, need to know + # which one is currently active. Because only one instance of UCI can be alive at any point in time, simply + # select the most recent one. + # TODO: Once individual UCI's will be able to start more than one instance, this will need to be fixed + #i_id = len(liveInstance.instance) - 1 +%> <h2>Live instance details</h2> @@ -18,22 +25,22 @@ <tr> <td> Instance name: </td> <td> - ${liveInstance.name} + ${liveInstance.uci.name} <a id="li-popup" class="popup-arrow" style="display: none;">▼</a> </td> <td> <div popupmenu="li-popup"> - <a class="action-button" href="${h.url_for( action='renameInstance', id=trans.security.encode_id(liveInstance.id) )}">Rename</a> - <a class="action-button" confirm="Are you sure you want to stop instance '${liveInstance.name}'?" href="${h.url_for( action='stop', id=trans.security.encode_id(liveInstance.id) )}">Stop</a> + <a class="action-button" href="${h.url_for( action='renameInstance', id=trans.security.encode_id(liveInstance.uci.id) )}">Rename</a> + <a class="action-button" confirm="Are you sure you want to stop instance '${liveInstance.uci.name}'?" href="${h.url_for( action='stop', id=trans.security.encode_id(liveInstance.uci.id) )}">Stop</a> </div> </td> </tr> <tr> <td> Date created: </td> - <td> ${str(liveInstance.create_time)[:16]} + <td> ${str(liveInstance.uci.create_time)[:16]} <% context.write( ' UTC (' ) - context.write( str(h.date.distance_of_time_in_words (liveInstance.create_time, h.date.datetime.utcnow() ) ) ) + context.write( str(h.date.distance_of_time_in_words (liveInstance.uci.create_time, h.date.datetime.utcnow() ) ) ) %> ago) </td> </tr> @@ -60,7 +67,7 @@ %endif <tr> <td> AMI: </td> - <td> ${liveInstance.ami} </td> + <td> ${liveInstance.mi_id} </td> </tr> <tr> <td> State:</td> @@ -82,34 +89,13 @@ <td> ${liveInstance.availability_zone} </td> </tr> %endif - %if liveInstance.keypair_fingerprint != None: + %if liveInstance.keypair_name != None: <tr> - <td> Keypair fingerprint:</td> - <td> ${liveInstance.keypair_fingerprint} </td> + <td> Keypair file name:</td> + <td> ${liveInstance.keypair_name} </td> </tr> %endif - %if liveInstance.keypair_material != None: - <tr> - <td> Keypair private key:</td> - <td> - <div id="shortComment2"> - <a onclick="document.getElementById('fullComment2').style.display = 'block'; - document.getElementById('shortComment2').style.display = 'none'; return 0" - href="javascript:void(0)"> - + Show - </a> - </div> - <div id="fullComment2" style="DISPLAY: none"> - <nobr><b>${liveInstance.keypair_material}</b></nobr><br/> - <a onclick="document.getElementById('shortComment2').style.display = 'block'; - document.getElementById('fullComment2').style.display = 'none'; return 0;" - href="javascript:void(0)"> - - Hide - </a> - </div> - </td> - </tr> - %endif + </table> %else: There is no live instance under that name. diff -r f87f1eed6e0c -r 0d8572e68b34 templates/root/index.mako --- a/templates/root/index.mako Fri Aug 28 17:45:40 2009 -0400 +++ b/templates/root/index.mako Tue Sep 08 09:50:05 2009 -0400 @@ -2,9 +2,9 @@ <%def name="init()"> <% - self.has_left_panel=True - self.has_right_panel=True - self.active_view="analysis" + self.has_left_panel=False + self.has_right_panel=False + self.active_view="cloud" %> %if trans.app.config.require_login and not trans.user: <script type="text/javascript"> @@ -37,7 +37,8 @@ elif m_c is not None: center_url = h.url_for( controller=m_c, action=m_a ) else: - center_url = h.url_for( '/static/welcome.html' ) + #center_url = h.url_for( '/static/welcome.html' ) + center_url = h.url_for( controller='cloud', action='list' ) %> <iframe name="galaxy_main" id="galaxy_main" frameborder="0" style="position: absolute; width: 100%; height: 100%;" src="${center_url}"> </iframe>

1 0

[hg] galaxy 3056: In the process of adding and integrating cloud...
by Greg Von Kuster 23 Nov '09

23 Nov '09

details: http://www.bx.psu.edu/hg/galaxy/rev/a620afab791d changeset: 3056:a620afab791d user: Enis Afgan <afgane(a)gmail.com> date: Thu Aug 27 17:36:03 2009 -0400 description: In the process of adding and integrating cloud storage DB table. Currently, broken. diffstat: lib/galaxy/model/__init__.py | 18 ++- lib/galaxy/model/mapping.py | 67 ++++++++---- lib/galaxy/model/migrate/versions/0014_cloud_tables.py | 102 +++++++++++++------- lib/galaxy/web/controllers/cloud.py | 85 ++++++++++------ templates/cloud/configure_cloud.mako | 24 ++-- 5 files changed, 189 insertions(+), 107 deletions(-) diffs (554 lines): diff -r c28dd8c0a231 -r a620afab791d lib/galaxy/model/__init__.py --- a/lib/galaxy/model/__init__.py Wed Aug 26 17:34:58 2009 -0400 +++ b/lib/galaxy/model/__init__.py Thu Aug 27 17:36:03 2009 -0400 @@ -932,24 +932,34 @@ self.galaxy_session = galaxy_session self.history = history -class UserInstances( object ): +class CloudInstance( object ): def __init__( self ): self.id = None self.user = None self.name = None self.instance_id = None - self.ami = None + self.mi = None self.state = None + self.keypair_name = None self.public_dns = None self.availability_zone = None -class CloudImages( object ): +class CloudImage( object ): def __init__( self ): self.id = None self.instance_id = None self.state = None -class StoredUserCredentials( object ): +class CloudStorage( object ): + def __init__( self ): + self.id = None + self.volume_id = None + self.i_id = None + self.user = None + self.size = None + self.availability_zone = None + +class CloudUserCredentials( object ): def __init__( self ): self.id = None self.user = None diff -r c28dd8c0a231 -r a620afab791d lib/galaxy/model/mapping.py --- a/lib/galaxy/model/mapping.py Wed Aug 26 17:34:58 2009 -0400 +++ b/lib/galaxy/model/mapping.py Thu Aug 27 17:36:03 2009 -0400 @@ -386,31 +386,47 @@ # *************************************************************************** # *************************** Cloud tables*********************************** # *************************************************************************** -UserInstances.table = Table( "user_instances", metadata, +CloudImage.table = Table( "cloud_image", metadata, Column( "id", Integer, primary_key=True ), Column( "create_time", DateTime, default=now ), - Column( "launch_time", DateTime, onupdate=now ), - Column( "user_id", Integer, ForeignKey( "galaxy_user.id" ), index=True, nullable=False ), - Column( "name", TEXT ), - Column( "reservation_id", TEXT ), - Column( "instance_id", TEXT ), - Column( "ami", TEXT, ForeignKey( "cloud_images.image_id" ), nullable=False ), - Column( "state", TEXT ), - Column( "public_dns", TEXT ), - Column( "private_dns", TEXT ), - Column( "keypair_fingerprint", TEXT ), - Column( "keypair_material", TEXT ), - Column( "availability_zone", TEXT ) ) - -CloudImages.table = Table( "cloud_images", metadata, - Column( "id", Integer, primary_key=True ), - Column( "create_time", DateTime, default=now ), - Column( "upadte_time", DateTime, default=now, onupdate=now ), + Column( "update_time", DateTime, default=now, onupdate=now ), Column( "image_id", TEXT, nullable=False ), Column( "manifest", TEXT ), Column( "state", TEXT ) ) -StoredUserCredentials.table = Table( "stored_user_credentials", metadata, +CloudInstance.table = Table( "cloud_instance", metadata, + Column( "id", Integer, primary_key=True ), + Column( "create_time", DateTime, default=now ), + Column( "update_time", DateTime, default=now, onupdate=now ), + Column( "launch_time", DateTime ), + Column( "user_id", Integer, ForeignKey( "galaxy_user.id" ), index=True, nullable=False ), + Column( "name", TEXT ), + Column( "type", TEXT ), + Column( "reservation_id", TEXT ), + Column( "instance_id", TEXT ), + Column( "mi", TEXT, ForeignKey( "cloud_image.image_id" ), index=True, nullable=False ), + Column( "state", TEXT ), + Column( "public_dns", TEXT ), + Column( "private_dns", TEXT ), + Column( "keypair_name", TEXT ), + Column( "availability_zone", TEXT ) ) + +CloudStorage.table = Table( "cloud_storage", metadata, + Column( "id", Integer, primary_key=True ), + Column( "create_time", DateTime, default=now ), + Column( "update_time", DateTime, default=now, onupdate=now ), + Column( "attach_time", DateTime ), + Column( "user_id", Integer, ForeignKey( "galaxy_user.id" ), index=True, nullable=False ), + Column( "volume_id", TEXT, nullable=False ), + Column( "size", Integer, nullable=False ), + Column( "availability_zone", TEXT, nullable=False ), + Column( "i_id", TEXT, ForeignKey( "cloud_instance.instance_id" ), index=True ), + Column( "status", TEXT ), + Column( "device", TEXT ), + Column( "space_consumed", Integer ) + ) + +CloudUserCredentials.table = Table( "cloud_user_credentials", metadata, Column( "id", Integer, primary_key=True ), Column( "create_time", DateTime, default=now ), Column( "update_time", DateTime, default=now, onupdate=now ), @@ -929,14 +945,19 @@ # ************************************************************ # vvvvvvvvvvvvvvvv Start cloud table mappings vvvvvvvvvvvvvvvv # ************************************************************ -assign_mapper( context, UserInstances, UserInstances.table, +assign_mapper( context, CloudImage, CloudImage.table ) + +assign_mapper( context, CloudInstance, CloudInstance.table, properties=dict( user=relation( User ), - cloud_image=relation( CloudImages ) + image=relation( CloudImage ) ) ) -assign_mapper( context, CloudImages, CloudImages.table ) +assign_mapper( context, CloudStorage, CloudStorage.table, + properties=dict( user=relation( User ), + instance=relation( CloudInstance, backref='cloud_instance' ) + ) ) -assign_mapper( context, StoredUserCredentials, StoredUserCredentials.table, +assign_mapper( context, CloudUserCredentials, CloudUserCredentials.table, properties=dict( user=relation( User) ) ) # ^^^^^^^^^^^^^^^ End cloud table mappings ^^^^^^^^^^^^^^^^^^ diff -r c28dd8c0a231 -r a620afab791d lib/galaxy/model/migrate/versions/0014_cloud_tables.py --- a/lib/galaxy/model/migrate/versions/0014_cloud_tables.py Wed Aug 26 17:34:58 2009 -0400 +++ b/lib/galaxy/model/migrate/versions/0014_cloud_tables.py Thu Aug 27 17:36:03 2009 -0400 @@ -12,7 +12,47 @@ metadata = MetaData( migrate_engine ) -Credentials_table = Table( "stored_user_credentials", metadata, +CloudImage_table = Table( "cloud_image", metadata, + Column( "id", Integer, primary_key=True ), + Column( "create_time", DateTime, default=now ), + Column( "update_time", DateTime, default=now, onupdate=now ), + Column( "image_id", TEXT, nullable=False ), + Column( "manifest", TEXT ), + Column( "state", TEXT ) ) + +CloudInstance_table = Table( "cloud_instance", metadata, + Column( "id", Integer, primary_key=True ), + Column( "create_time", DateTime, default=now ), + Column( "update_time", DateTime, default=now, onupdate=now ), + Column( "launch_time", DateTime ), + Column( "user_id", Integer, ForeignKey( "galaxy_user.id" ), index=True, nullable=False ), + Column( "name", TEXT ), + Column( "type", TEXT ), + Column( "reservation_id", TEXT ), + Column( "instance_id", TEXT ), + Column( "mi", TEXT, ForeignKey( "cloud_image.image_id" ), index=True, nullable=False ), + Column( "state", TEXT ), + Column( "public_dns", TEXT ), + Column( "private_dns", TEXT ), + Column( "keypair_name", TEXT ), + Column( "availability_zone", TEXT ) ) + +CloudStorage_table = Table( "cloud_storage", metadata, + Column( "id", Integer, primary_key=True ), + Column( "create_time", DateTime, default=now ), + Column( "update_time", DateTime, default=now, onupdate=now ), + Column( "attach_time", DateTime ), + Column( "user_id", Integer, ForeignKey( "galaxy_user.id" ), index=True, nullable=False ), + Column( "volume_id", TEXT, nullable=False ), + Column( "size", Integer, nullable=False ), + Column( "availability_zone", TEXT, nullable=False ), + Column( "i_id", TEXT, ForeignKey( "cloud_instance.instance_id" ), index=True ), + Column( "status", TEXT ), + Column( "device", TEXT ), + Column( "space_consumed", Integer ) + ) + +CloudUserCredentials_table = Table( "cloud_user_credentials", metadata, Column( "id", Integer, primary_key=True ), Column( "create_time", DateTime, default=now ), Column( "update_time", DateTime, default=now, onupdate=now ), @@ -20,49 +60,41 @@ Column( "name", TEXT), Column( "access_key", TEXT), Column( "secret_key", TEXT), - Column( "defaultCred", Boolean, default=False ) ) - -UserInstances_table = Table ( "user_instances", metadata, - Column( "id", Integer, primary_key=True ), - Column( "create_time", DateTime, default=now ), - Column( "launch_time", DateTime, onupdate=now ), - Column( "user_id", Integer, ForeignKey( "galaxy_user.id" ), index=True, nullable=False ), - Column( "name", TEXT ), - Column( "reservation_id", TEXT ), - Column( "instance_id", TEXT ), - Column( "ami", TEXT, ForeignKey( "cloud_images.image_id" ), nullable=False ), - Column( "state", TEXT ), - Column( "public_dns", TEXT ), - Column( "private_dns", TEXT ), - Column( "keypair_fingerprint", TEXT ), - Column( "keypair_material", TEXT ), - Column( "availability_zone", TEXT ) ) - -CloudImages_table = Table( "cloud_images", metadata, - Column( "id", Integer, primary_key=True ), - Column( "create_time", DateTime, default=now ), - Column( "upadte_time", DateTime, default=now, onupdate=now ), - Column( "image_id", TEXT, nullable=False ), - Column( "manifest", TEXT ), - Column( "state", TEXT ) ) - + Column( "defaultCred", Boolean, default=False) + ) def upgrade(): metadata.reflect() - Credentials_table.create() - UserInstances_table.create() + CloudUserCredentials_table.create() + CloudInstance_table.create() + CloudStorage_table.create() try: - CloudImages_table.create() + CloudImage_table.create() except Exception, e: - log.debug( "Creating CloudImages table failed. Table probably exists already." ) + log.debug( "Creating cloud_image table failed. Table probably exists already." ) def downgrade(): metadata.reflect() try: - Credentials_table.drop() + CloudImage_table.drop() except Exception, e: - log.debug( "Dropping stored_user_credentials table failed: %s" % str( e ) ) + log.debug( "Dropping cloud_image table failed: %s" % str( e ) ) + + try: + CloudInstance_table.drop() + except Exception, e: + log.debug( "Dropping cloud_instance table failed: %s" % str( e ) ) try: - UserInstances_table.drop() + CloudStorage_table.drop() except Exception, e: - log.debug( "Dropping user_instances table failed: %s" % str( e ) ) + log.debug( "Dropping cloud_user_credentials table failed: %s" % str( e ) ) + + try: + log.deboug( "Would drop cloud user credentials table." ) + #CloudUserCredentials_table.drop() + except Exception, e: + log.debug( "Dropping cloud_user_credentials table failed: %s" % str( e ) ) + + + + diff -r c28dd8c0a231 -r a620afab791d lib/galaxy/web/controllers/cloud.py --- a/lib/galaxy/web/controllers/cloud.py Wed Aug 26 17:34:58 2009 -0400 +++ b/lib/galaxy/web/controllers/cloud.py Thu Aug 27 17:36:03 2009 -0400 @@ -36,23 +36,23 @@ Render cloud main page (management of cloud resources) """ user = trans.get_user() - awsCredentials = trans.sa_session.query( model.StoredUserCredentials ) \ - .order_by( desc( model.StoredUserCredentials.c.update_time ) ) \ + cloudCredentials = trans.sa_session.query( model.CloudUserCredentials ) \ + .order_by( desc( model.CloudUserCredentials.c.update_time ) ) \ .all() - prevInstances = trans.sa_session.query( model.UserInstances ) \ + prevInstances = trans.sa_session.query( model.CloudInstance ) \ .filter_by( user=user, state="available" ) \ - .order_by( desc( model.UserInstances.c.create_time ) ) \ + .order_by( desc( model.CloudInstance.c.create_time ) ) \ .all() #TODO: diff between live and previous instances - liveInstances = trans.sa_session.query( model.UserInstances ) \ + liveInstances = trans.sa_session.query( model.CloudInstance ) \ .filter_by( user=user ) \ - .filter( or_(model.UserInstances.c.state=="running", model.UserInstances.c.state=="pending") ) \ - .order_by( desc( model.UserInstances.c.create_time ) ) \ + .filter( or_(model.CloudInstance.c.state=="running", model.CloudInstance.c.state=="pending") ) \ + .order_by( desc( model.CloudInstance.c.create_time ) ) \ .all() return trans.fill_template( "cloud/configure_cloud.mako", - awsCredentials = awsCredentials, + cloudCredentials = cloudCredentials, prevInstances = prevInstances, liveInstances = liveInstances ) @@ -60,9 +60,9 @@ @web.require_login( "use Galaxy cloud" ) def makeDefault( self, trans, id=None ): """ - Set current credentials as default to be used for acquiring cloud resources. + Set current credentials as default. """ - currentDefault = get_defalt_credentials (trans) + currentDefault = get_default_credentials (trans) if currentDefault: currentDefault.defaultCred = False @@ -177,8 +177,11 @@ return trans.response.send_redirect( url_for( controller='workflow' ) ) @web.expose - @web.require_login( "use Galaxy cloud instance" ) + @web.require_login( "start Galaxy cloud instance" ) def start( self, trans, id ): + """ + Start a new cloud resource instance + """ instance = get_instance( trans, id ) @@ -189,6 +192,9 @@ @web.expose @web.require_login( "stop Galaxy cloud instance" ) def stop( self, trans, id ): + """ + Stop a cloud resource instance + """ instance = get_instance( trans, id ) @@ -207,7 +213,7 @@ return self.list( trans ) @web.expose - @web.require_login( "delete Galaxy cloud instance" ) + @web.require_login( "add instance storage" ) def addStorage( self, trans, id ): instance = get_instance( trans, id ) @@ -224,33 +230,43 @@ """ user = trans.get_user() # TODO: If more images are made available, must add code to choose between those - ami = trans.app.model.CloudImages.filter( - trans.app.model.CloudImages.table.c.id==1).first() + mi = trans.app.model.CloudImage.filter( + trans.app.model.CloudImage.table.c.id==1).first() + inst_error = vol_error = None if instanceName: # Create new user configured instance try: if len( instanceName ) > 255: inst_error = "Instance name exceeds maximum allowable length." - elif trans.app.model.UserInstances.filter( - trans.app.model.UserInstances.table.c.name==instanceName ).first(): + elif trans.app.model.CloudInstance.filter( + trans.app.model.CloudInstance.table.c.name==instanceName ).first(): inst_error = "An instance with that name already exist." elif int( volSize ) > 1000: vol_error = "Volume size cannot exceed 1000GB. You must specify an integer between 1 and 1000." elif int( volSize ) < 1: vol_error = "Volume size cannot be less than 1GB. You must specify an integer between 1 and 1000." else: - instance = model.UserInstances() + instance = model.CloudInstance() instance.user = user instance.name = instanceName - instance.ami = ami.image_id - # Valid states include: "available", "running" or "pending" + instance.mi = mi.image_id + # TODO: get state from AWS - also, update state on page update + # Currently, valid states include: "available", "running" or "pending" instance.state = "available" - #TODO: include storage volume size code + # Capture storage related information + storage = model.CloudStorage() + storage.user = user + storage.size = volSize + log.debug("******** Size: %s" % storage.size ) + # TODO: get correct values from AWS + storage.volume_id = "made up" + storage.availability_zone = "avail zone" # Persist session = trans.sa_session session.save_or_update( instance ) + session.save_or_update( storage ) session.flush() # Log and display the management page trans.log_event( "User configured new cloud resource instance" ) @@ -258,7 +274,10 @@ return self.list( trans ) except ValueError: vol_error = "Volume size must be specified as an integer value only, between 1 and 1000." - + except AttributeError, ae: + inst_error = "No registered cloud images. You must contact administrator to add some before proceeding." + log.debug("AttributeError: %s " % str( ae ) ) + return trans.show_form( web.FormBuilder( web.url_for(), "Configure new instance", submit_text="Add" ) .add_text( "instanceName", "Instance name", value="Unnamed instance", error=inst_error ) @@ -273,12 +292,12 @@ if image_id: if len( image_id ) > 255: error = "Image ID name exceeds maximum allowable length." - elif trans.app.model.StoredUserCredentials.filter( - trans.app.model.CloudImages.table.c.image_id==image_id ).first(): + elif trans.app.model.CloudUserCredentials.filter( + trans.app.model.CloudImage.table.c.image_id==image_id ).first(): error = "Image with that ID is already registered." else: # Create new image - image = model.CloudImages() + image = model.CloudImage() image.image_id = image_id image.manifest = manifest # Persist @@ -327,19 +346,19 @@ @web.require_login( "add credentials" ) def add( self, trans, credName='', accessKey='', secretKey='', defaultCred=True ): """ - Add user's AWS credentials stored under name `credName`. + Add user's cloud credentials stored under name `credName`. """ user = trans.get_user() cred_error = accessKey_error = secretKey_error = None if credName: if len( credName ) > 255: cred_error = "Credentials name exceeds maximum allowable length." - elif trans.app.model.StoredUserCredentials.filter( - trans.app.model.StoredUserCredentials.table.c.name==credName ).first(): + elif trans.app.model.CloudUserCredentials.filter( + trans.app.model.CloudUserCredentials.table.c.name==credName ).first(): cred_error = "Credentials with that name already exist." else: # Create new user stored credentials - credentials = model.StoredUserCredentials() + credentials = model.CloudUserCredentials() credentials.name = credName credentials.user = user credentials.access_key = accessKey @@ -853,7 +872,7 @@ if not isinstance( id, int ): id = trans.security.decode_id( id ) - stored = trans.sa_session.query( model.StoredUserCredentials ).get( id ) + stored = trans.sa_session.query( model.CloudUserCredentials ).get( id ) if not stored: error( "Credentials not found" ) # Verify ownership @@ -865,13 +884,13 @@ # Looks good return stored -def get_defalt_credentials( trans, check_ownership=True ): +def get_default_credentials( trans, check_ownership=True ): """ Get a StoredUserCredntials from the database by 'default' setting, verifying ownership. """ user = trans.get_user() # Load credentials from database - stored = trans.sa_session.query( model.StoredUserCredentials ) \ + stored = trans.sa_session.query( model.CloudUserCredentials ) \ .filter_by (user=user, defaultCred=True) \ .first() @@ -879,11 +898,11 @@ def get_instance( trans, id, check_ownership=True ): """ - Get a UserInstances from the database by id, verifying ownership. + Get a CloudInstance from the database by id, verifying ownership. """ id = trans.security.decode_id( id ) - live = trans.sa_session.query( model.UserInstances ).get( id ) + live = trans.sa_session.query( model.CloudInstance ).get( id ) if not live: error( "Instance not found" ) # Verify ownership diff -r c28dd8c0a231 -r a620afab791d templates/cloud/configure_cloud.mako --- a/templates/cloud/configure_cloud.mako Wed Aug 26 17:34:58 2009 -0400 +++ b/templates/cloud/configure_cloud.mako Thu Aug 27 17:36:03 2009 -0400 @@ -20,7 +20,7 @@ <h2>Galaxy in the clouds</h2> -%if awsCredentials: +%if cloudCredentials: ## Manage user credentials <ul class="manage-table-actions"> <li> @@ -37,28 +37,28 @@ <th>Default</th> <th></th> </tr> - %for i, awsCredential in enumerate( awsCredentials ): + %for i, cloudCredential in enumerate( cloudCredentials ): <tr> <td> - ${awsCredential.name} + ${cloudCredential.name} <a id="cr-${i}-popup" class="popup-arrow" style="display: none;">▼</a> </td> ## Comment <td>${len(workflow.latest_workflow.steps)}</td> - ##<td>${str(awsCredential.update_time)[:19]}</td> + ##<td>${str(cloudCredential.update_time)[:19]}</td> <td> - ##${str(awsCredential.defaultCred)} + ##${str(cloudCredential.defaultCred)} <% - if awsCredential.defaultCred: + if cloudCredential.defaultCred: context.write('*') %> </td> <td> <div popupmenu="cr-${i}-popup"> - <a class="action-button" href="${h.url_for( action='view', id=trans.security.encode_id(awsCredential.id) )}">View</a> - <a class="action-button" href="${h.url_for( action='rename', id=trans.security.encode_id(awsCredential.id) )}">Rename</a> - <a class="action-button" href="${h.url_for( action='makeDefault', id=trans.security.encode_id(awsCredential.id) )}" target="_parent">Make default</a> - <a class="action-button" confirm="Are you sure you want to delete credentials '${awsCredential.name}'?" href="${h.url_for( action='delete', id=trans.security.encode_id(awsCredential.id) )}">Delete</a> + <a class="action-button" href="${h.url_for( action='view', id=trans.security.encode_id(cloudCredential.id) )}">View</a> + <a class="action-button" href="${h.url_for( action='rename', id=trans.security.encode_id(cloudCredential.id) )}">Rename</a> + <a class="action-button" href="${h.url_for( action='makeDefault', id=trans.security.encode_id(cloudCredential.id) )}" target="_parent">Make default</a> + <a class="action-button" confirm="Are you sure you want to delete credentials '${cloudCredential.name}'?" href="${h.url_for( action='delete', id=trans.security.encode_id(cloudCredential.id) )}">Delete</a> </div> </td> </tr> @@ -97,7 +97,7 @@ ${liveInstance.name} <a id="li-${i}-popup" class="popup-arrow" style="display: none;">▼</a> </td> - <td>${str(liveInstance.name)}</td>  + <td>${str(liveInstance.size)}</td>  <td>${str(liveInstance.state)}</td> <td> ${str(liveInstance.launch_time)[:16]} @@ -161,7 +161,7 @@ <a id="pi-${i}-popup" class="popup-arrow" style="display: none;">▼</a> </td> ## Comment <td>${len(workflow.latest_workflow.steps)}</td> - <td>${str(prevInstance.name)}</td>  + <td>${str(prevInstance.CloudStorage.size)}</td>  <td>${str(prevInstance.state)}</td> <td> <div popupmenu="pi-${i}-popup">

1 0

[hg] galaxy 3057: Cloud UI & DB connected and interfaced.
by Greg Von Kuster 23 Nov '09

23 Nov '09

details: http://www.bx.psu.edu/hg/galaxy/rev/f87f1eed6e0c changeset: 3057:f87f1eed6e0c user: Enis Afgan <afgane(a)gmail.com> date: Fri Aug 28 17:45:40 2009 -0400 description: Cloud UI & DB connected and interfaced. diffstat: lib/galaxy/model/__init__.py | 21 ++- lib/galaxy/model/mapping.py | 33 ++++- lib/galaxy/model/migrate/versions/0014_cloud_tables.py | 48 +++++-- lib/galaxy/web/controllers/cloud.py | 138 ++++++++++++++++------ templates/cloud/configure_cloud.mako | 14 +- 5 files changed, 183 insertions(+), 71 deletions(-) diffs (540 lines): diff -r a620afab791d -r f87f1eed6e0c lib/galaxy/model/__init__.py --- a/lib/galaxy/model/__init__.py Thu Aug 27 17:36:03 2009 -0400 +++ b/lib/galaxy/model/__init__.py Fri Aug 28 17:45:40 2009 -0400 @@ -931,7 +931,18 @@ def __init__( self, galaxy_session, history ): self.galaxy_session = galaxy_session self.history = history - + +class CloudImage( object ): + def __init__( self ): + self.id = None + self.instance_id = None + self.state = None + +class UCI( object ): + def __init__( self ): + self.id = None + self.user = None + class CloudInstance( object ): def __init__( self ): self.id = None @@ -944,13 +955,7 @@ self.public_dns = None self.availability_zone = None -class CloudImage( object ): - def __init__( self ): - self.id = None - self.instance_id = None - self.state = None - -class CloudStorage( object ): +class CloudStore( object ): def __init__( self ): self.id = None self.volume_id = None diff -r a620afab791d -r f87f1eed6e0c lib/galaxy/model/mapping.py --- a/lib/galaxy/model/mapping.py Thu Aug 27 17:36:03 2009 -0400 +++ b/lib/galaxy/model/mapping.py Fri Aug 28 17:45:40 2009 -0400 @@ -394,29 +394,42 @@ Column( "manifest", TEXT ), Column( "state", TEXT ) ) +""" UserConfiguredInstance (UCI) table """ +UCI.table = Table( "uci", metadata, + Column( "id", Integer, primary_key=True ), + Column( "create_time", DateTime, default=now ), + Column( "update_time", DateTime, default=now, onupdate=now ), + Column( "user_id", Integer, ForeignKey( "galaxy_user.id" ), index=True, nullable=False ), + Column( "name", TEXT ), + Column( "state", TEXT ), + Column( "total_size", Integer ), + Column( "launch_time", DateTime ) ) + CloudInstance.table = Table( "cloud_instance", metadata, Column( "id", Integer, primary_key=True ), Column( "create_time", DateTime, default=now ), Column( "update_time", DateTime, default=now, onupdate=now ), Column( "launch_time", DateTime ), + Column( "stop_time", DateTime ), Column( "user_id", Integer, ForeignKey( "galaxy_user.id" ), index=True, nullable=False ), - Column( "name", TEXT ), + Column( "uci_id", Integer, ForeignKey( "uci.id" ), index=True ), Column( "type", TEXT ), Column( "reservation_id", TEXT ), Column( "instance_id", TEXT ), - Column( "mi", TEXT, ForeignKey( "cloud_image.image_id" ), index=True, nullable=False ), + Column( "mi_id", TEXT, ForeignKey( "cloud_image.image_id" ), index=True, nullable=False ), Column( "state", TEXT ), Column( "public_dns", TEXT ), Column( "private_dns", TEXT ), Column( "keypair_name", TEXT ), Column( "availability_zone", TEXT ) ) -CloudStorage.table = Table( "cloud_storage", metadata, +CloudStore.table = Table( "cloud_store", metadata, Column( "id", Integer, primary_key=True ), Column( "create_time", DateTime, default=now ), Column( "update_time", DateTime, default=now, onupdate=now ), Column( "attach_time", DateTime ), Column( "user_id", Integer, ForeignKey( "galaxy_user.id" ), index=True, nullable=False ), + Column( "uci_id", Integer, ForeignKey( "uci.id" ), index=True, nullable=False ), Column( "volume_id", TEXT, nullable=False ), Column( "size", Integer, nullable=False ), Column( "availability_zone", TEXT, nullable=False ), @@ -947,19 +960,25 @@ # ************************************************************ assign_mapper( context, CloudImage, CloudImage.table ) +assign_mapper( context, UCI, UCI.table, + properties=dict( user=relation( User ), + instance=relation( CloudInstance, backref='uci' ), + store=relation( CloudStore, backref='uci' ) + ) ) + assign_mapper( context, CloudInstance, CloudInstance.table, properties=dict( user=relation( User ), image=relation( CloudImage ) ) ) -assign_mapper( context, CloudStorage, CloudStorage.table, +assign_mapper( context, CloudStore, CloudStore.table, properties=dict( user=relation( User ), - instance=relation( CloudInstance, backref='cloud_instance' ) + i=relation( CloudInstance ) ) ) assign_mapper( context, CloudUserCredentials, CloudUserCredentials.table, - properties=dict( user=relation( User) ) - ) + properties=dict( user=relation( User) + ) ) # ^^^^^^^^^^^^^^^ End cloud table mappings ^^^^^^^^^^^^^^^^^^ assign_mapper( context, StoredWorkflow, StoredWorkflow.table, diff -r a620afab791d -r f87f1eed6e0c lib/galaxy/model/migrate/versions/0014_cloud_tables.py --- a/lib/galaxy/model/migrate/versions/0014_cloud_tables.py Thu Aug 27 17:36:03 2009 -0400 +++ b/lib/galaxy/model/migrate/versions/0014_cloud_tables.py Fri Aug 28 17:45:40 2009 -0400 @@ -20,29 +20,41 @@ Column( "manifest", TEXT ), Column( "state", TEXT ) ) +UCI_table = Table( "uci", metadata, + Column( "id", Integer, primary_key=True ), + Column( "create_time", DateTime, default=now ), + Column( "update_time", DateTime, default=now, onupdate=now ), + Column( "user_id", Integer, ForeignKey( "galaxy_user.id" ), index=True, nullable=False ), + Column( "name", TEXT ), + Column( "state", TEXT ), + Column( "total_size", Integer ), + Column( "launch_time", DateTime ) ) + CloudInstance_table = Table( "cloud_instance", metadata, Column( "id", Integer, primary_key=True ), Column( "create_time", DateTime, default=now ), Column( "update_time", DateTime, default=now, onupdate=now ), Column( "launch_time", DateTime ), + Column( "stop_time", DateTime ), Column( "user_id", Integer, ForeignKey( "galaxy_user.id" ), index=True, nullable=False ), - Column( "name", TEXT ), + Column( "uci_id", Integer, ForeignKey( "uci.id" ), index=True ), Column( "type", TEXT ), Column( "reservation_id", TEXT ), Column( "instance_id", TEXT ), - Column( "mi", TEXT, ForeignKey( "cloud_image.image_id" ), index=True, nullable=False ), + Column( "mi_id", TEXT, ForeignKey( "cloud_image.image_id" ), index=True, nullable=False ), Column( "state", TEXT ), Column( "public_dns", TEXT ), Column( "private_dns", TEXT ), Column( "keypair_name", TEXT ), Column( "availability_zone", TEXT ) ) -CloudStorage_table = Table( "cloud_storage", metadata, +CloudStore_table = Table( "cloud_store", metadata, Column( "id", Integer, primary_key=True ), Column( "create_time", DateTime, default=now ), Column( "update_time", DateTime, default=now, onupdate=now ), Column( "attach_time", DateTime ), Column( "user_id", Integer, ForeignKey( "galaxy_user.id" ), index=True, nullable=False ), + Column( "uci_id", Integer, ForeignKey( "uci.id" ), index=True, nullable=False ), Column( "volume_id", TEXT, nullable=False ), Column( "size", Integer, nullable=False ), Column( "availability_zone", TEXT, nullable=False ), @@ -62,20 +74,26 @@ Column( "secret_key", TEXT), Column( "defaultCred", Boolean, default=False) ) + def upgrade(): metadata.reflect() - CloudUserCredentials_table.create() - CloudInstance_table.create() - CloudStorage_table.create() try: CloudImage_table.create() except Exception, e: log.debug( "Creating cloud_image table failed. Table probably exists already." ) - + UCI_table.create() + CloudInstance_table.create() + CloudStore_table.create() + try: + CloudUserCredentials_table.create() + except Exception, e: + log.debug( "Creating cloud_image table failed. Table probably exists already." ) + def downgrade(): metadata.reflect() try: - CloudImage_table.drop() + log.deboug( "Would drop cloud_image table." ) + #CloudImage_table.drop() #Enable before putting final version except Exception, e: log.debug( "Dropping cloud_image table failed: %s" % str( e ) ) @@ -85,16 +103,20 @@ log.debug( "Dropping cloud_instance table failed: %s" % str( e ) ) try: - CloudStorage_table.drop() + CloudStore_table.drop() + except Exception, e: + log.debug( "Dropping cloud_store table failed: %s" % str( e ) ) + + try: + log.deboug( "Would drop cloud_user_credentials table." ) + #CloudUserCredentials_table.drop() #Enable before putting final version except Exception, e: log.debug( "Dropping cloud_user_credentials table failed: %s" % str( e ) ) try: - log.deboug( "Would drop cloud user credentials table." ) - #CloudUserCredentials_table.drop() + UCI_table.drop() except Exception, e: - log.debug( "Dropping cloud_user_credentials table failed: %s" % str( e ) ) - + log.debug( "Dropping UCI table failed: %s" % str( e ) ) diff -r a620afab791d -r f87f1eed6e0c lib/galaxy/web/controllers/cloud.py --- a/lib/galaxy/web/controllers/cloud.py Thu Aug 27 17:36:03 2009 -0400 +++ b/lib/galaxy/web/controllers/cloud.py Fri Aug 28 17:45:40 2009 -0400 @@ -14,6 +14,7 @@ from galaxy.workflow.modules import * from galaxy.model.mapping import desc from galaxy.model.orm import * +from datetime import datetime # Required for Cloud tab import galaxy.eggs @@ -40,15 +41,21 @@ .order_by( desc( model.CloudUserCredentials.c.update_time ) ) \ .all() - prevInstances = trans.sa_session.query( model.CloudInstance ) \ + prevInstances = trans.sa_session.query( model.UCI ) \ .filter_by( user=user, state="available" ) \ - .order_by( desc( model.CloudInstance.c.create_time ) ) \ + .order_by( desc( model.UCI.c.update_time ) ) \ .all() #TODO: diff between live and previous instances liveInstances = trans.sa_session.query( model.CloudInstance ) \ .filter_by( user=user ) \ .filter( or_(model.CloudInstance.c.state=="running", model.CloudInstance.c.state=="pending") ) \ - .order_by( desc( model.CloudInstance.c.create_time ) ) \ + .order_by( desc( model.CloudInstance.c.launch_time ) ) \ + .all() + + liveInstances = trans.sa_session.query( model.UCI ) \ + .filter_by( user=user ) \ + .filter( or_(model.UCI.c.state=="running", model.UCI.c.state=="pending") ) \ + .order_by( desc( model.UCI.c.launch_time ) ) \ .all() return trans.fill_template( "cloud/configure_cloud.mako", @@ -178,15 +185,35 @@ @web.expose @web.require_login( "start Galaxy cloud instance" ) - def start( self, trans, id ): + def start( self, trans, id, size='small' ): """ Start a new cloud resource instance """ - instance = get_instance( trans, id ) + user = trans.get_user() + mi = get_mi( trans, size ) + uci = get_uci( trans, id ) + stores = get_stores( trans, uci ) #TODO: handle list! + instance = model.CloudInstance() + instance.user = user + instance.image = mi + instance.uci = uci + # TODO: get real value from AWS + instance.state = "pending" + uci.state = instance.state + uci.launch_time = datetime.utcnow() + instance.launch_time = datetime.utcnow() + instance.availability_zone = stores.availability_zone + instance.type = size - error( "Starting instance '%s' is not supported yet." % instance.name ) + # Persist + session = trans.sa_session + session.save_or_update( instance ) + session.flush() + #error( "Starting instance '%s' is not supported yet." % uci.name ) + trans.log_event( "User started cloud instance '%s'" % uci.name ) + trans.set_message( "Galaxy instance '%s' started." % uci.name ) return self.list( trans ) @web.expose @@ -195,17 +222,27 @@ """ Stop a cloud resource instance """ - instance = get_instance( trans, id ) + uci = get_uci( trans, id ) + instances = get_instances( trans, uci ) #TODO: handle list! - - error( "Stopping instance '%s' is not supported yet." % instance.name ) + instances.state = 'done' + instances.stop_time = datetime.utcnow() + uci.state = 'available' + uci.launch_time = None + # Persist + session = trans.sa_session + session.save_or_update( uci ) + session.save_or_update( instances ) + session.flush() + trans.log_event( "User stopped cloud instance '%s'" % uci.name ) + trans.set_message( "Galaxy instance '%s' stopped." % uci.name ) return self.list( trans ) @web.expose @web.require_login( "delete Galaxy cloud instance" ) def deleteInstance( self, trans, id ): - instance = get_instance( trans, id ) + instance = get_uci( trans, id ) error( "Deleting instance '%s' is not supported yet." % instance.name ) @@ -215,7 +252,7 @@ @web.expose @web.require_login( "add instance storage" ) def addStorage( self, trans, id ): - instance = get_instance( trans, id ) + instance = get_uci( trans, id ) error( "Adding storage to instance '%s' is not supported yet." % instance.name ) @@ -229,48 +266,42 @@ Configure and add new cloud instance to user's instance pool """ user = trans.get_user() - # TODO: If more images are made available, must add code to choose between those - mi = trans.app.model.CloudImage.filter( - trans.app.model.CloudImage.table.c.id==1).first() - - inst_error = vol_error = None if instanceName: # Create new user configured instance try: if len( instanceName ) > 255: inst_error = "Instance name exceeds maximum allowable length." - elif trans.app.model.CloudInstance.filter( - trans.app.model.CloudInstance.table.c.name==instanceName ).first(): + elif trans.app.model.UCI.filter( + trans.app.model.UCI.table.c.name==instanceName ).first(): inst_error = "An instance with that name already exist." elif int( volSize ) > 1000: vol_error = "Volume size cannot exceed 1000GB. You must specify an integer between 1 and 1000." elif int( volSize ) < 1: vol_error = "Volume size cannot be less than 1GB. You must specify an integer between 1 and 1000." else: - instance = model.CloudInstance() - instance.user = user - instance.name = instanceName - instance.mi = mi.image_id - # TODO: get state from AWS - also, update state on page update - # Currently, valid states include: "available", "running" or "pending" - instance.state = "available" - # Capture storage related information - storage = model.CloudStorage() + # Capture user configured instance information + uci = model.UCI() + uci.name = instanceName + uci.user= user + uci.state = "available" # Valid states include: "available", "running" or "pending" + uci.total_size = volSize # This is OK now because new instance is being created. + # Capture store related information + storage = model.CloudStore() storage.user = user + storage.uci = uci storage.size = volSize - log.debug("******** Size: %s" % storage.size ) # TODO: get correct values from AWS storage.volume_id = "made up" storage.availability_zone = "avail zone" # Persist session = trans.sa_session - session.save_or_update( instance ) + session.save_or_update( uci ) session.save_or_update( storage ) session.flush() # Log and display the management page - trans.log_event( "User configured new cloud resource instance" ) - trans.set_message( "Instance '%s' configured" % instance.name ) + trans.log_event( "User configured new cloud instance" ) + trans.set_message( "New Galaxy instance '%s' configured." % uci.name ) return self.list( trans ) except ValueError: vol_error = "Volume size must be specified as an integer value only, between 1 and 1000." @@ -332,7 +363,7 @@ @web.expose @web.require_login( "use Galaxy cloud" ) def renameInstance( self, trans, id, new_name=None ): - instance = get_instance( trans, id ) + instance = get_uci( trans, id ) if new_name is not None: instance.name = new_name trans.sa_session.flush() @@ -398,7 +429,7 @@ """ View details about running instance """ - instance = get_instance( trans, id ) + instance = get_uci( trans, id ) log.debug ( instance.name ) return trans.fill_template( "cloud/viewInstance.mako", @@ -896,15 +927,15 @@ return stored -def get_instance( trans, id, check_ownership=True ): +def get_uci( trans, id, check_ownership=True ): """ - Get a CloudInstance from the database by id, verifying ownership. + Get a UCI from the database by id, verifying ownership. """ id = trans.security.decode_id( id ) - live = trans.sa_session.query( model.CloudInstance ).get( id ) + live = trans.sa_session.query( model.UCI ).get( id ) if not live: - error( "Instance not found" ) + error( "Galaxy instance not found." ) # Verify ownership user = trans.get_user() if not user: @@ -914,6 +945,39 @@ # Looks good return live +def get_mi( trans, size='small' ): + """ + Get appropriate machine image (mi) based on instance size. + TODO: Dummy method - need to implement logic + For valid sizes, see http://aws.amazon.com/ec2/instance-types/ + """ + return trans.app.model.CloudImage.filter( + trans.app.model.CloudImage.table.c.id==1).first() + +def get_stores( trans, uci ): + """ + Get store objects/tables that are connected to uci object/table + """ + user = trans.get_user() + stores = trans.sa_session.query( model.CloudStore ) \ + .filter_by( user=user, uci_id=uci.id ) \ + .first() + #.all() #TODO: return all but need to edit calling method(s) to handle list + + return stores + +def get_instances( trans, uci ): + """ + Get instance objects/tables that are connected to uci object/table + """ + user = trans.get_user() + instances = trans.sa_session.query( model.CloudInstance ) \ + .filter_by( user=user, uci_id=uci.id ) \ + .first() + #.all() #TODO: return all but need to edit calling method(s) to handle list + + return instances + def attach_ordered_steps( workflow, steps ): ordered_steps = order_workflow_steps( steps ) if ordered_steps: diff -r a620afab791d -r f87f1eed6e0c templates/cloud/configure_cloud.mako --- a/templates/cloud/configure_cloud.mako Thu Aug 27 17:36:03 2009 -0400 +++ b/templates/cloud/configure_cloud.mako Fri Aug 28 17:45:40 2009 -0400 @@ -85,7 +85,7 @@ <colgroup width="35%"></colgroup> <tr class="header"> <th>Live instances</th> - <th>Volume size (GB)</th> + <th>Storage size (GB)</th> <th>State</th> <th>Alive since</th> <th></th> @@ -97,7 +97,7 @@ ${liveInstance.name} <a id="li-${i}-popup" class="popup-arrow" style="display: none;">▼</a> </td> - <td>${str(liveInstance.size)}</td>  + <td>${str(liveInstance.total_size)}</td>  <td>${str(liveInstance.state)}</td> <td> ${str(liveInstance.launch_time)[:16]} @@ -106,7 +106,7 @@ from datetime import timedelta # DB stores all times in GMT, so adjust for difference (4 hours) - adjustedStarttime = liveInstance.launch_time - timedelta(hours=4) + adjustedStarttime = liveInstance.update_time - timedelta(hours=4) # (NOT CURRENTLY USED BLOCK OF CODE) Calculate time difference from now delta = datetime.now() - adjustedStarttime @@ -117,6 +117,8 @@ #context.write( str( liveInstance.launch_time ) ) context.write( ' UTC (' ) context.write( str(h.date.distance_of_time_in_words (liveInstance.launch_time, h.date.datetime.utcnow() ) ) ) + + %>) </td> <td> @@ -144,7 +146,7 @@ <colgroup width="35%"></colgroup> <tr class="header"> <th>Previously configured instances</th> - ##<th>Volume size (GB)</th> + ##<th>Storage size (GB)</th> ##<th>State</th> ##<th>Alive since</th> <th></th> @@ -160,9 +162,9 @@ ${prevInstance.name} <a id="pi-${i}-popup" class="popup-arrow" style="display: none;">▼</a> </td> - ## Comment <td>${len(workflow.latest_workflow.steps)}</td> - <td>${str(prevInstance.CloudStorage.size)}</td>  + <td>${str(prevInstance.total_size)}</td>  <td>${str(prevInstance.state)}</td> + <td>N/A</td> <td> <div popupmenu="pi-${i}-popup"> <a class="action-button" href="${h.url_for( action='start', id=trans.security.encode_id(prevInstance.id) )}">Start</a>

1 0

[hg] galaxy 3055: Changed cloud page to more consistent UI.
by Greg Von Kuster 23 Nov '09

23 Nov '09

details: http://www.bx.psu.edu/hg/galaxy/rev/c28dd8c0a231 changeset: 3055:c28dd8c0a231 user: Enis Afgan <afgane(a)gmail.com> date: Wed Aug 26 17:34:58 2009 -0400 description: Changed cloud page to more consistent UI. diffstat: lib/galaxy/web/controllers/cloud.py | 7 ++- templates/cloud/configure_cloud.mako | 51 ++++++++++++++----------- templates/cloud/viewInstance.mako | 20 ++++++++-- 3 files changed, 48 insertions(+), 30 deletions(-) diffs (186 lines): diff -r 33c00350d29a -r c28dd8c0a231 lib/galaxy/web/controllers/cloud.py --- a/lib/galaxy/web/controllers/cloud.py Wed Aug 26 12:29:54 2009 -0400 +++ b/lib/galaxy/web/controllers/cloud.py Wed Aug 26 17:34:58 2009 -0400 @@ -208,11 +208,11 @@ @web.expose @web.require_login( "delete Galaxy cloud instance" ) - def edit( self, trans, id ): + def addStorage( self, trans, id ): instance = get_instance( trans, id ) - error( "Editing instance '%s' is not supported yet." % instance.name ) + error( "Adding storage to instance '%s' is not supported yet." % instance.name ) return self.list( trans ) @@ -262,7 +262,8 @@ return trans.show_form( web.FormBuilder( web.url_for(), "Configure new instance", submit_text="Add" ) .add_text( "instanceName", "Instance name", value="Unnamed instance", error=inst_error ) - .add_text( "volSize", "Permanent storage size (1GB - 1000GB)", value='', error=vol_error ) ) + .add_text( "volSize", "Permanent storage size (1GB - 1000GB)" + "<br />Note: you will be able to add more storage later", value='', error=vol_error ) ) @web.expose @web.require_login( "add a cloud image" ) diff -r 33c00350d29a -r c28dd8c0a231 templates/cloud/configure_cloud.mako --- a/templates/cloud/configure_cloud.mako Wed Aug 26 12:29:54 2009 -0400 +++ b/templates/cloud/configure_cloud.mako Wed Aug 26 17:34:58 2009 -0400 @@ -79,12 +79,16 @@ </ul> <table class="mange-table colored" border="0" cellspacing="0" cellpadding="0" width="100%"> - <tr class="header"> + <colgroup width="40%"></colgroup> + <colgroup width="15%"></colgroup> + <colgroup width="10%"></colgroup> + <colgroup width="35%"></colgroup> + <tr class="header"> <th>Live instances</th> + <th>Volume size (GB)</th> <th>State</th> <th>Alive since</th> - <th>Action</th> - <th></th> + <th></th> </tr> %if liveInstances: %for i, liveInstance in enumerate( liveInstances ): @@ -93,7 +97,8 @@ ${liveInstance.name} <a id="li-${i}-popup" class="popup-arrow" style="display: none;">▼</a> </td> - <td>${str(liveInstance.state)}</td> + <td>${str(liveInstance.name)}</td>  + <td>${str(liveInstance.state)}</td> <td> ${str(liveInstance.launch_time)[:16]} <% @@ -115,13 +120,10 @@ %>) </td> <td> - <a class="action-button" href="${h.url_for( action='stop', id=trans.security.encode_id(liveInstance.id) )}">Stop</a> - </td> - <td> <div popupmenu="li-${i}-popup"> + <a class="action-button" confirm="Are you sure you want to stop instance '${liveInstance.name}'?" href="${h.url_for( action='stop', id=trans.security.encode_id(liveInstance.id) )}">Stop</a> + <a class="action-button" href="${h.url_for( action='renameInstance', id=trans.security.encode_id(liveInstance.id) )}">Rename</a> <a class="action-button" href="${h.url_for( action='viewInstance', id=trans.security.encode_id(liveInstance.id) )}">View details</a> - <a class="action-button" href="${h.url_for( action='renameInstance', id=trans.security.encode_id(liveInstance.id) )}">Rename</a> - <a class="action-button" confirm="Are you sure you want to stop instance '${liveInstance.name}'?" href="${h.url_for( action='stop', id=trans.security.encode_id(liveInstance.id) )}">Stop</a> </div> </td> </tr> @@ -135,13 +137,20 @@ ## ***************************************************** ## Manage previously configured instances - <p /> <p /> <table class="mange-table colored" border="0" cellspacing="0" cellpadding="0" width="100%"> - <tr class="header"> + <colgroup width="40%"></colgroup> + <colgroup width="15%"></colgroup> + <colgroup width="10%"></colgroup> + <colgroup width="35%"></colgroup> + <tr class="header"> <th>Previously configured instances</th> - <th>Volume size</th> - <th>Action</th> - <th></th> + ##<th>Volume size (GB)</th> + ##<th>State</th> + ##<th>Alive since</th> + <th></th> + <th></th> + <th></th> + <th></th> </tr> %if prevInstances: @@ -152,18 +161,14 @@ <a id="pi-${i}-popup" class="popup-arrow" style="display: none;">▼</a> </td> ## Comment <td>${len(workflow.latest_workflow.steps)}</td> - ## Chnage to show vol size once available - <td>${str(prevInstance.name)}</td> + <td>${str(prevInstance.name)}</td>  + <td>${str(prevInstance.state)}</td> <td> - <a class="action-button" href="${h.url_for( action='start', id=trans.security.encode_id(prevInstance.id) )}">Start</a> - </td> - - <td> <div popupmenu="pi-${i}-popup"> - ##<a class="action-button" href="${h.url_for( action='viewInstance', id=trans.security.encode_id(prevInstance.id) )}">View details</a> + <a class="action-button" href="${h.url_for( action='start', id=trans.security.encode_id(prevInstance.id) )}">Start</a> <a class="action-button" href="${h.url_for( action='renameInstance', id=trans.security.encode_id(prevInstance.id) )}">Rename</a> - <a class="action-button" href="${h.url_for( action='edit', id=trans.security.encode_id(prevInstance.id) )}" target="_parent">Edit</a> - <a class="action-button" confirm="Are you sure you want to delete instance '${prevInstance.name}'?" href="${h.url_for( action='deleteInstance', id=trans.security.encode_id(prevInstance.id) )}">Delete</a> + <a class="action-button" href="${h.url_for( action='addStorage', id=trans.security.encode_id(prevInstance.id) )}" target="_parent">Add storage</a> + <a class="action-button" confirm="Are you sure you want to delete instance '${prevInstance.name}'? This will delete all of your data assocaiated with this instance!" href="${h.url_for( action='deleteInstance', id=trans.security.encode_id(prevInstance.id) )}">Delete</a> </div> </td> </tr> diff -r 33c00350d29a -r c28dd8c0a231 templates/cloud/viewInstance.mako --- a/templates/cloud/viewInstance.mako Wed Aug 26 12:29:54 2009 -0400 +++ b/templates/cloud/viewInstance.mako Wed Aug 26 17:34:58 2009 -0400 @@ -46,14 +46,18 @@ %> ago) </td> </tr> + %if liveInstance.instance_id != None: + <tr> + <td> Instance ID: </td> + <td> ${liveInstance.instance_id} </td> + </tr> + %endif + %if liveInstance.reservation_id != None: <tr> - <td> Instance ID: </td> - <td> ${liveInstance.instance_id} </td> - </tr> - <tr> <td> Reservation ID: </td> <td> ${liveInstance.reservation_id} </td> </tr> + %endif <tr> <td> AMI: </td> <td> ${liveInstance.ami} </td> @@ -66,18 +70,25 @@ <td> Public DNS:</td> <td> ${liveInstance.public_dns} </td> </tr> + %if liveInstance.private_dns != None: <tr> <td> Private DNS:</td> <td> ${liveInstance.private_dns} </td> </tr> + %endif + %if liveInstance.availability_zone != None: <tr> <td> Availabilty zone:</td> <td> ${liveInstance.availability_zone} </td> </tr> + %endif + %if liveInstance.keypair_fingerprint != None: <tr> <td> Keypair fingerprint:</td> <td> ${liveInstance.keypair_fingerprint} </td> </tr> + %endif + %if liveInstance.keypair_material != None: <tr> <td> Keypair private key:</td> <td> @@ -98,6 +109,7 @@ </div> </td> </tr> + %endif </table> %else: There is no live instance under that name.

1 0

[hg] galaxy 3053: Added cloud resource management tables and som...
by Greg Von Kuster 23 Nov '09

23 Nov '09

details: http://www.bx.psu.edu/hg/galaxy/rev/a47ad6a21a5d changeset: 3053:a47ad6a21a5d user: Enis Afgan <afgane(a)gmail.com> date: Tue Aug 25 17:42:34 2009 -0400 description: Added cloud resource management tables and some code. diffstat: lib/galaxy/model/__init__.py | 16 ++ lib/galaxy/model/mapping.py | 40 +++++ lib/galaxy/model/migrate/versions/0014_cloud_tables.py | 68 +++++++++ lib/galaxy/model/migrate/versions/0014_credentials_table.py | 31 ---- lib/galaxy/web/controllers/cloud.py | 88 ++++++++++++- templates/cloud/configure_cloud.mako | 118 +++++++++++++++- 6 files changed, 317 insertions(+), 44 deletions(-) diffs (458 lines): diff -r 6b3f453078c4 -r a47ad6a21a5d lib/galaxy/model/__init__.py --- a/lib/galaxy/model/__init__.py Mon Aug 24 17:46:22 2009 -0400 +++ b/lib/galaxy/model/__init__.py Tue Aug 25 17:42:34 2009 -0400 @@ -932,6 +932,22 @@ self.galaxy_session = galaxy_session self.history = history +class UserInstances( object ): + def __init__( self ): + self.id = None + self.user = None + self.name = None + self.instance_id = None + self.state = None + self.public_dns = None + self.availability_zone = None + +class CloudImages( object ): + def __init__( self ): + self.id = None + self.instance_id = None + self.state = None + class StoredUserCredentials( object ): def __init__( self ): self.id = None diff -r 6b3f453078c4 -r a47ad6a21a5d lib/galaxy/model/mapping.py --- a/lib/galaxy/model/mapping.py Mon Aug 24 17:46:22 2009 -0400 +++ b/lib/galaxy/model/mapping.py Tue Aug 25 17:42:34 2009 -0400 @@ -379,6 +379,37 @@ Column( "session_id", Integer, ForeignKey( "galaxy_session.id" ), index=True ), Column( "history_id", Integer, ForeignKey( "history.id" ), index=True ) ) + + + + +# *************************************************************************** +# *************************** Cloud tables*********************************** +# *************************************************************************** +UserInstances.table = Table( "user_instances", metadata, + Column( "id", Integer, primary_key=True ), + Column( "create_time", DateTime, default=now ), + Column( "launch_time", DateTime, onupdate=now ), + Column( "user_id", Integer, ForeignKey( "galaxy_user.id" ), index=True, nullable=False ), + Column( "name", TEXT ), + Column( "reservation_id", TEXT ), + Column( "instance_id", TEXT ), + Column( "ami", TEXT, ForeignKey( "cloud_images.image_id" ), nullable=False ), + Column( "state", TEXT ), + Column( "public_dns", TEXT ), + Column( "private_dns", TEXT ), + Column( "keypair_fingerprint", TEXT ), + Column( "keypair_material", TEXT ), + Column( "availability_zone", TEXT ) ) + +CloudImages.table = Table( "cloud_images", metadata, + Column( "id", Integer, primary_key=True ), + Column( "create_time", DateTime, default=now ), + Column( "upadte_time", DateTime, default=now, onupdate=now ), + Column( "image_id", TEXT, nullable=False ), + Column( "manifest", TEXT ), + Column( "state", TEXT ) ) + StoredUserCredentials.table = Table( "stored_user_credentials", metadata, Column( "id", Integer, primary_key=True ), Column( "create_time", DateTime, default=now ), @@ -895,9 +926,18 @@ output_step=relation( WorkflowStep, backref="output_connections", cascade="all", primaryjoin=( WorkflowStepConnection.table.c.output_step_id == WorkflowStep.table.c.id ) ) ) ) +# vvvvvvvvvvvvvvvv Start cloud table mappings vvvvvvvvvvvvvvvv +assign_mapper( context, UserInstances, UserInstances.table, + properties=dict( user=relation( User ), + cloud_image=relation( CloudImages ) + ) ) + +assign_mapper( context, CloudImages, CloudImages.table ) + assign_mapper( context, StoredUserCredentials, StoredUserCredentials.table, properties=dict( user=relation( User) ) ) +# ^^^^^^^^^^^^^^^ End cloud table mappings ^^^^^^^^^^^^^^^^^^ assign_mapper( context, StoredWorkflow, StoredWorkflow.table, properties=dict( user=relation( User ), diff -r 6b3f453078c4 -r a47ad6a21a5d lib/galaxy/model/migrate/versions/0014_cloud_tables.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/lib/galaxy/model/migrate/versions/0014_cloud_tables.py Tue Aug 25 17:42:34 2009 -0400 @@ -0,0 +1,68 @@ +from sqlalchemy import * +from migrate import * + +import datetime +now = datetime.datetime.utcnow + +# Need our custom types, but don't import anything else from model +from galaxy.model.custom_types import * + +import logging +log = logging.getLogger( __name__ ) + +metadata = MetaData( migrate_engine ) + +Credentials_table = Table( "stored_user_credentials", metadata, + Column( "id", Integer, primary_key=True ), + Column( "create_time", DateTime, default=now ), + Column( "update_time", DateTime, default=now, onupdate=now ), + Column( "user_id", Integer, ForeignKey( "galaxy_user.id" ), index=True, nullable=False ), + Column( "name", TEXT), + Column( "access_key", TEXT), + Column( "secret_key", TEXT), + Column( "defaultCred", Boolean, default=False ) ) + +UserInstances_table = Table ( "user_instances", metadata, + Column( "id", Integer, primary_key=True ), + Column( "create_time", DateTime, default=now ), + Column( "launch_time", DateTime, onupdate=now ), + Column( "user_id", Integer, ForeignKey( "galaxy_user.id" ), index=True, nullable=False ), + Column( "name", TEXT ), + Column( "reservation_id", TEXT ), + Column( "instance_id", TEXT ), + Column( "ami", TEXT, ForeignKey( "cloud_images.image_id" ), nullable=False ), + Column( "state", TEXT ), + Column( "public_dns", TEXT ), + Column( "private_dns", TEXT ), + Column( "keypair_fingerprint", TEXT ), + Column( "keypair_material", TEXT ), + Column( "availability_zone", TEXT ) ) + +CloudImages_table = Table( "cloud_images", metadata, + Column( "id", Integer, primary_key=True ), + Column( "create_time", DateTime, default=now ), + Column( "upadte_time", DateTime, default=now, onupdate=now ), + Column( "image_id", TEXT, nullable=False ), + Column( "manifest", TEXT ), + Column( "state", TEXT ) ) + +def upgrade(): + metadata.reflect() + Credentials_table.create() + UserInstances_table.create() + try: + CloudImages_table.create() + except Exception, e: + log.debug( "Creating CloudImages table failed. Table probably exists already." ) + +def downgrade(): + metadata.reflect() + try: + Credentials_table.drop() + except Exception, e: + log.debug( "Dropping stored_user_credentials table failed: %s" % str( e ) ) + + try: + UserInstances_table.drop() + except Exception, e: + log.debug( "Dropping user_instances table failed: %s" % str( e ) ) diff -r 6b3f453078c4 -r a47ad6a21a5d lib/galaxy/model/migrate/versions/0014_credentials_table.py --- a/lib/galaxy/model/migrate/versions/0014_credentials_table.py Mon Aug 24 17:46:22 2009 -0400 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,31 +0,0 @@ -from sqlalchemy import * -from migrate import * - -import datetime -now = datetime.datetime.utcnow - -# Need our custom types, but don't import anything else from model -from galaxy.model.custom_types import * - -import logging -log = logging.getLogger( __name__ ) - -metadata = MetaData( migrate_engine ) - -Credentials_table = Table( "stored_user_credentials", metadata, - Column( "id", Integer, primary_key=True ), - Column( "create_time", DateTime, default=now ), - Column( "update_time", DateTime, default=now, onupdate=now ), - Column( "user_id", Integer, ForeignKey( "galaxy_user.id" ), index=True, nullable=False ), - Column( "name", TEXT), - Column( "access_key", TEXT), - Column( "secret_key", TEXT), - Column( "defaultCred", Boolean, default=False ) ) - -def upgrade(): - metadata.reflect() - Credentials_table.create() - -def downgrade(): - metadata.reflect() - Credentials_table.drop() diff -r 6b3f453078c4 -r a47ad6a21a5d lib/galaxy/web/controllers/cloud.py --- a/lib/galaxy/web/controllers/cloud.py Mon Aug 24 17:46:22 2009 -0400 +++ b/lib/galaxy/web/controllers/cloud.py Tue Aug 25 17:42:34 2009 -0400 @@ -36,12 +36,17 @@ Render cloud main page (management of cloud resources) """ user = trans.get_user() - awsCredentials = trans.sa_session.query ( model.StoredUserCredentials ) \ + awsCredentials = trans.sa_session.query( model.StoredUserCredentials ) \ .order_by( desc( model.StoredUserCredentials.c.update_time ) ) \ .all() + prevInstances = trans.sa_session.query( model.UserInstances ) \ + .order_by( desc( model.UserInstances.c.create_time ) ) \ + .all() #TODO: diff between live and previous instances + return trans.fill_template( "cloud/configure_cloud.mako", - awsCredentials = awsCredentials ) + awsCredentials = awsCredentials, + prevInstances = prevInstances ) @web.expose @web.require_login( "use Galaxy cloud" ) @@ -162,7 +167,82 @@ session.flush() # Redirect to load galaxy frames. return trans.response.send_redirect( url_for( controller='workflow' ) ) - + + + @web.expose + @web.require_login( "use Galaxy cloud" ) + def configureNew( self, trans, instanceName='', volSize=''): + """ + Configure and add new cloud instance to user's instance pool + """ + user = trans.get_user() + ami = trans.app.model.CloudImages.filter( + trans.app.model.CloudImages.table.c.id==1).first() + log.debug(ami.image_id) + inst_error = vol_error = None + if instanceName: + # Create new user configured instance + if len( instanceName ) > 255: + inst_error = "Instance name exceeds maximum allowable length." + elif trans.app.model.UserInstances.filter( + trans.app.model.UserInstances.table.c.name==instanceName ).first(): + inst_error = "An instance with that name already exist." + elif int( volSize ) > 1000: + vol_error = "Volume size cannot exceed 1000GB. You must specify an integer between 1 and 1000." + elif int( volSize ) < 1: + vol_error = "Volume size cannot be less than 1GB. You must specify an integer between 1 and 1000." + else: + instance = model.UserInstances() + instance.user_id = user + instance.name = instanceName + instance.ami = ami.image_id + #TODO: include storage volume size code + # Persist + session = trans.sa_session + session.save_or_update( instance ) + session.flush() + # Log and display the management page + trans.log_event( "User configured new cloud resource instance" ) + trans.set_message( "Instance '%s' configured" % credentials.name ) + return self.list( trans ) + + return trans.show_form( + web.FormBuilder( web.url_for(), "Configure new instance", submit_text="Add" ) + .add_text( "instanceName", "Instance name", value="Unnamed instance", error=inst_error ) + .add_text( "volSize", "Permanent storage size (1GB - 1000GB)", value='', error=vol_error ) ) + + @web.expose + @web.require_login( "add a cloud image" ) + #(a)web.require_admin + def addNewImage( self, trans, image_id='', manifest='', state=None ): + error = None + if image_id: + if len( image_id ) > 255: + error = "Image ID name exceeds maximum allowable length." + elif trans.app.model.StoredUserCredentials.filter( + trans.app.model.CloudImages.table.c.image_id==image_id ).first(): + error = "Image with that ID is already registered." + else: + # Create new image + image = model.CloudImages() + image.image_id = image_id + image.manifest = manifest + # Persist + session = trans.sa_session + session.save_or_update( image ) + session.flush() + # Log and display the management page + trans.log_event( "New cloud image added: '%s'" % image.image_id ) + trans.set_message( "Cloud image '%s' added." % image.image_id ) + if state: + image.state= state + return self.list( trans ) + + return trans.show_form( + web.FormBuilder( web.url_for(), "Add new cloud image", submit_text="Add" ) + .add_text( "image_id", "Image ID", value='', error=error ) + .add_text( "manifest", "Manifest", value='', error=error ) ) + @web.expose @web.require_login( "use Galaxy cloud" ) def rename( self, trans, id, new_name=None ): @@ -178,7 +258,7 @@ @web.expose @web.require_login( "add credentials" ) - def add( self, trans, credName='', accessKey='', secretKey='', defaultCred=False ): + def add( self, trans, credName='', accessKey='', secretKey='', defaultCred=True ): """ Add user's AWS credentials stored under name `credName`. """ diff -r 6b3f453078c4 -r a47ad6a21a5d templates/cloud/configure_cloud.mako --- a/templates/cloud/configure_cloud.mako Mon Aug 24 17:46:22 2009 -0400 +++ b/templates/cloud/configure_cloud.mako Tue Aug 25 17:42:34 2009 -0400 @@ -21,6 +21,7 @@ <h2>Galaxy in the clouds</h2> %if awsCredentials: + ## Manage user credentials <ul class="manage-table-actions"> <li> <a class="action-button" href="${h.url_for( action='add' )}"> @@ -45,12 +46,11 @@ ## Comment <td>${len(workflow.latest_workflow.steps)}</td> ##<td>${str(awsCredential.update_time)[:19]}</td> <td> - ${str(awsCredential.defaultCred)} + ##${str(awsCredential.defaultCred)} <% - c=str(awsCredential.defaultCred) + if awsCredential.defaultCred: + context.write('*') %> - - ##${dflt(cred=c)} </td> <td> @@ -64,6 +64,105 @@ </tr> %endfor </table> + + ## ***************************************************** + ## Manage live instances + <p /> + <h2>Manage cloud instances</h2> + <ul class="manage-table-actions"> + <li> + <a class="action-button" href="${h.url_for( action='configureNew' )}"> + <img src="${h.url_for('/static/images/silk/add.png')}" /> + <span>Configure new instance</span> + </a> + </li> + </ul> + + <table class="mange-table colored" border="0" cellspacing="0" cellpadding="0" width="100%"> + <tr class="header"> + <th>Live instances</th> + <th>Alive since</th> + <th></th> + </tr> + %for i, awsCredential in enumerate( awsCredentials ): + <tr> + <td> + ${awsCredential.name} + <a id="wf-${i}-popup" class="popup-arrow" style="display: none;">▼</a> + </td> + <td> + ##${str(awsCredential.update_time)[:19]} + <% + from datetime import datetime + from datetime import timedelta + + # DB stores all times in GMT, so adjust for difference (4 hours) + adjustedStarttime = awsCredential.update_time - timedelta(hours=4) + + # (NOT CURRENTLY USED BLOCK OF CODE) Calculate time difference from now + delta = datetime.now() - adjustedStarttime + #context.write( str(datetime.utcnow() ) ) + #context.write( str(delta) ) + + # This is where current time and since duration is calculated + context.write( str( awsCredential.update_time ) ) + context.write( ' UTC (' ) + context.write( str(h.date.distance_of_time_in_words (awsCredential.update_time, h.date.datetime.utcnow() ) ) ) + %>) + </td> + + <td> + <div popupmenu="wf-${i}-popup"> + <a class="action-button" href="${h.url_for( action='view', id=trans.security.encode_id(awsCredential.id) )}">View</a> + <a class="action-button" href="${h.url_for( action='rename', id=trans.security.encode_id(awsCredential.id) )}">Rename</a> + <a class="action-button" href="${h.url_for( action='makeDefault', id=trans.security.encode_id(awsCredential.id) )}" target="_parent">Make default</a> + <a class="action-button" confirm="Are you sure you want to delete workflow '${awsCredential.name}'?" href="${h.url_for( action='delete', id=trans.security.encode_id(awsCredential.id) )}">Delete</a> + </div> + </td> + </tr> + %endfor + </table> + + ## ***************************************************** + ## Manage previously configured instances + <p /> <p /> + <table class="mange-table colored" border="0" cellspacing="0" cellpadding="0" width="100%"> + <tr class="header"> + <th>Previously configured instances</th> + <th>Action</th> + <th></th> + </tr> + + %if prevInstances: + %for i, prevInstance in enumerate( prevInstances ): + <tr> + <td> + ${prevInstance.name} + <a id="wf-${i}-popup" class="popup-arrow" style="display: none;">▼</a> + </td> + ## Comment <td>${len(workflow.latest_workflow.steps)}</td> + ##<td>${str(awsCredential.update_time)[:19]}</td> + <td> + <a class="action-button" href="${h.url_for( action='start', id=trans.security.encode_id(awsCredential.id) )}">Start</a> + </td> + + <td> + <div popupmenu="wf-${i}-popup"> + <a class="action-button" href="${h.url_for( action='view', id=trans.security.encode_id(awsCredential.id) )}">View</a> + <a class="action-button" href="${h.url_for( action='rename', id=trans.security.encode_id(awsCredential.id) )}">Rename</a> + <a class="action-button" href="${h.url_for( action='makeDefault', id=trans.security.encode_id(awsCredential.id) )}" target="_parent">Make default</a> + <a class="action-button" confirm="Are you sure you want to delete workflow '${awsCredential.name}'?" href="${h.url_for( action='delete', id=trans.security.encode_id(awsCredential.id) )}">Delete</a> + </div> + </td> + </tr> + %endfor + %else: + <tr> + <td>You have no previously configured instances.</td> + </tr> + %endif + </table> + %else: You have no AWS credentials associated with your Galaxy account: <a class="action-button" href="${h.url_for( action='add' )}"> @@ -76,8 +175,9 @@ %endif -<%def name="dflt(cred)"> - %if cred: - default - %endif -</%def> +<p /><br /> +<ul class="manage-table-actions"> + <li> + <a class="action-button" href="${h.url_for( action='addNewImage' )}"><span>Add new image</span></a> + </li> +</ul> \ No newline at end of file

1 0