1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/changeset/a7fec5917853/ changeset: a7fec5917853 user: dannon date: 2012-02-29 17:58:22 summary: Cloud launch basics. TODO: Tons of help text, shift instance dns wait to page level. pkey direct download. affected #: 11 files diff -r a86a94f46c6cad83eabf76aa20fb3537ddda4ee1 -r a7fec5917853c921043be653c87dea3ce781452b lib/galaxy/config.py --- a/lib/galaxy/config.py +++ b/lib/galaxy/config.py @@ -216,11 +216,7 @@ self.tool_runners = [] self.datatypes_config = kwargs.get( 'datatypes_config_file', 'datatypes_conf.xml' ) # Cloud configuration options - self.cloud_controller_instance = string_as_bool( kwargs.get( 'cloud_controller_instance', 'False' ) ) - if self.cloud_controller_instance == True: - self.enable_cloud_execution = string_as_bool( kwargs.get( 'enable_cloud_execution', 'True' ) ) - else: - self.enable_cloud_execution = string_as_bool( kwargs.get( 'enable_cloud_execution', 'False' ) ) + self.enable_cloud_launch = string_as_bool( kwargs.get( 'enable_cloud_launch', False ) ) # Galaxy messaging (AMQP) configuration options self.amqp = {} try: diff -r a86a94f46c6cad83eabf76aa20fb3537ddda4ee1 -r a7fec5917853c921043be653c87dea3ce781452b lib/galaxy/model/__init__.py --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -1671,58 +1671,11 @@ self.galaxy_session = galaxy_session self.history = history -class CloudImage( object ): - def __init__( self ): - self.id = None - self.instance_id = None - self.state = None - class UCI( object ): def __init__( self ): self.id = None self.user = None -class CloudInstance( object ): - def __init__( self ): - self.id = None - self.user = None - self.name = None - self.instance_id = None - self.mi = None - self.state = None - self.public_dns = None - self.availability_zone = None - -class CloudStore( object ): - def __init__( self ): - self.id = None - self.volume_id = None - self.user = None - self.size = None - self.availability_zone = None - -class CloudSnapshot( object ): - def __init__( self ): - self.id = None - self.user = None - self.store_id = None - self.snapshot_id = None - -class CloudProvider( object ): - def __init__( self ): - self.id = None - self.user = None - self.type = None - -class CloudUserCredentials( object ): - def __init__( self ): - self.id = None - self.user = None - self.name = None - self.accessKey = None - self.secretKey = None - self.credentials = [] - class StoredWorkflow( object, APIItem): api_collection_visible_keys = ( 'id', 'name' ) api_element_visible_keys = ( 'id', 'name' ) diff -r a86a94f46c6cad83eabf76aa20fb3537ddda4ee1 -r a7fec5917853c921043be653c87dea3ce781452b lib/galaxy/web/controllers/cloud.py --- /dev/null +++ b/lib/galaxy/web/controllers/cloud.py @@ -0,0 +1,215 @@ +""" +Cloud Controller: handles all cloud interactions. + +Adapted from Brad Chapman and Enis Afgan's BioCloudCentral +BioCloudCentral Source: https://github.com/chapmanb/biocloudcentral + +""" + +import boto +import datetime +import logging +import time +from galaxy import web +from galaxy.web.base.controller import BaseUIController +from boto.ec2.regioninfo import RegionInfo +from boto.exception import EC2ResponseError + +log = logging.getLogger(__name__) + +class CloudController(BaseUIController): + def __init__(self, app): + BaseUIController.__init__(self, app) + + @web.expose + def index(self, trans): + return trans.fill_template("cloud/index.mako") + + @web.expose + def launch_instance(self, trans, cluster_name, password, key_id, secret, instance_type): + ec2_error = None + try: + # Create security group & key pair used when starting an instance + ec2_conn = connect_ec2(key_id, secret) + sg_name = create_cm_security_group(ec2_conn) + kp_name, kp_material = create_key_pair(ec2_conn) + except EC2ResponseError, err: + ec2_error = err.error_message + if ec2_error: + return trans.fill_template("cloud/run.mako", error = ec2_error) + else: + rs = run_instance(ec2_conn=ec2_conn, + user_provided_data={'cluster_name':cluster_name, + 'password':password, + 'access_key':key_id, + 'secret_key':secret, + 'instance_type':instance_type}, + key_name=kp_name, + security_groups=[sg_name]) + if rs: + instance = rs.instances[0] + ct = 0 + while not instance.public_dns_name: + # Can take a second to have public dns name registered. + # DBTODO, push this into a page update, this is not ideal. + instance.update() + ct +=1 + time.sleep(1) + return trans.fill_template("cloud/run.mako", + instance = rs.instances[0], + kp_name = kp_name, + kp_material = kp_material) + else: + return trans.fill_template("cloud/run.mako", + error = "Instance failure, but no specific error was detected. Please check your AWS Console.") + +# ## Cloud interaction methods +def connect_ec2(a_key, s_key): + """ Create and return an EC2 connection object. + """ + # Use variables for forward looking flexibility + # AWS connection values + region_name = 'us-east-1' + region_endpoint = 'ec2.amazonaws.com' + is_secure = True + ec2_port = None + ec2_conn_path = '/' + r = RegionInfo(name=region_name, endpoint=region_endpoint) + ec2_conn = boto.connect_ec2(aws_access_key_id=a_key, + aws_secret_access_key=s_key, + api_version='2011-11-01', # needed for availability zone support + is_secure=is_secure, + region=r, + port=ec2_port, + path=ec2_conn_path) + return ec2_conn + +def create_cm_security_group(ec2_conn, sg_name='CloudMan'): + """ Create a security group with all authorizations required to run CloudMan. + If the group already exists, check its rules and add the missing ones. + Return the name of the created security group. + """ + cmsg = None + # Check if this security group already exists + sgs = ec2_conn.get_all_security_groups() + for sg in sgs: + if sg.name == sg_name: + cmsg = sg + log.debug("Security group '%s' already exists; will add authorizations next." % sg_name) + break + # If it does not exist, create security group + if cmsg is None: + log.debug("Creating Security Group %s" % sg_name) + cmsg = ec2_conn.create_security_group(sg_name, 'A security group for CloudMan') + # Add appropriate authorization rules + # If these rules already exist, nothing will be changed in the SG + ports = (('80', '80'), # Web UI + ('20', '21'), # FTP + ('22', '22'), # ssh + ('30000', '30100'), # FTP transfer + ('42284', '42284')) # CloudMan UI + for port in ports: + try: + if not rule_exists(cmsg.rules, from_port=port[0], to_port=port[1]): + cmsg.authorize(ip_protocol='tcp', from_port=port[0], to_port=port[1], cidr_ip='0.0.0.0/0') + else: + log.debug("Rule (%s:%s) already exists in the SG" % (port[0], port[1])) + except EC2ResponseError, e: + log.error("A problem with security group authorizations: %s" % e) + # Add rule that allows communication between instances in the same SG + g_rule_exists = False # Flag to indicate if group rule already exists + for rule in cmsg.rules: + for grant in rule.grants: + if grant.name == cmsg.name: + g_rule_exists = True + log.debug("Group rule already exists in the SG") + if g_rule_exists: + break + if g_rule_exists is False: + try: + cmsg.authorize(src_group=cmsg) + except EC2ResponseError, e: + log.error("A problem w/ security group authorization: %s" % e) + log.info("Done configuring '%s' security group" % cmsg.name) + return cmsg.name + +def rule_exists(rules, from_port, to_port, ip_protocol='tcp', cidr_ip='0.0.0.0/0'): + """ A convenience method to check if an authorization rule in a security + group exists. + """ + for rule in rules: + if rule.ip_protocol == ip_protocol and rule.from_port == from_port and \ + rule.to_port == to_port and cidr_ip in [ip.cidr_ip for ip in rule.grants]: + return True + return False + +def create_key_pair(ec2_conn, key_name='cloudman_key_pair'): + """ Create a key pair with the provided name. + Return the name of the key or None if there was an error creating the key. + """ + kp = None + # Check if a key pair under the given name already exists. If it does not, + # create it, else return. + kps = ec2_conn.get_all_key_pairs() + for akp in kps: + if akp.name == key_name: + log.debug("Key pair '%s' already exists; not creating it again." % key_name) + return akp.name, None + try: + kp = ec2_conn.create_key_pair(key_name) + except EC2ResponseError, e: + log.error("Problem creating key pair '%s': %s" % (key_name, e)) + return None, None + return kp.name, kp.material + +def run_instance(ec2_conn, user_provided_data, image_id='ami-da58aab3', + kernel_id=None, ramdisk_id=None, key_name='cloudman_key_pair', + security_groups=['CloudMan']): + """ Start an instance. If instance start was OK, return the ResultSet object + else return None. + """ + rs = None + instance_type = user_provided_data['instance_type'] + # Remove 'instance_type' key from the dict before creating user data + del user_provided_data['instance_type'] + placement = _find_placement(ec2_conn, instance_type) + ud = "\n".join(['%s: %s' % (key, value) for key, value in user_provided_data.iteritems() if key != 'kp_material']) + try: + rs = ec2_conn.run_instances(image_id=image_id, + instance_type=instance_type, + key_name=key_name, + security_groups=security_groups, + user_data=ud, + kernel_id=kernel_id, + ramdisk_id=ramdisk_id, + placement=placement) + except EC2ResponseError, e: + log.error("Problem starting an instance: %s" % e) + if rs: + try: + log.info("Started an instance with ID %s" % rs.instances[0].id) + except Exception, e: + log.error("Problem with the started instance object: %s" % e) + else: + log.warning("Problem starting an instance?") + return rs + +def _find_placement(ec2_conn, instance_type): + """Find a region zone that supports our requested instance type. + + We need to check spot prices in the potential availability zones + for support before deciding on a region: + + http://blog.piefox.com/2011/07/ec2-availability-zones-and-instance.html + """ + base = ec2_conn.region.name + yesterday = datetime.datetime.now() - datetime.timedelta(1) + for loc_choice in ["b", "a", "c", "d"]: + cur_loc = "{base}{ext}".format(base=base, ext=loc_choice) + if len(ec2_conn.get_spot_price_history(instance_type=instance_type, + end_time=yesterday.isoformat(), + availability_zone=cur_loc)) > 0: + return cur_loc + log.error("Did not find availabilty zone in {0} for {1}".format(base, instance_type)) + return None + diff -r a86a94f46c6cad83eabf76aa20fb3537ddda4ee1 -r a7fec5917853c921043be653c87dea3ce781452b lib/galaxy/web/controllers/root.py --- a/lib/galaxy/web/controllers/root.py +++ b/lib/galaxy/web/controllers/root.py @@ -481,7 +481,15 @@ trans.sa_session.flush() return trans.show_message( "<p>Secondary dataset has been made primary.</p>", refresh_frames=['history'] ) except: - return trans.show_error_message( "<p>Failed to make secondary dataset primary.</p>" ) + return trans.show_error_message( "<p>Failed to make secondary dataset primary.</p>" ) + + @web.expose + def bucket_proxy( self, trans, bucket=None, **kwd): + if bucket: + trans.response.set_content_type( 'text/xml' ) + b_list_xml = urllib.urlopen('http://s3.amazonaws.com/%s/' % bucket) + return b_list_xml.read() + raise Exception("You must specify a bucket") # ---- Debug methods ---------------------------------------------------- @@ -496,7 +504,7 @@ if isinstance( kwd[k], FieldStorage ): rval += "-> %s" % kwd[k].file.read() return rval - + @web.expose def generate_error( self, trans ): raise Exception( "Fake error!" ) diff -r a86a94f46c6cad83eabf76aa20fb3537ddda4ee1 -r a7fec5917853c921043be653c87dea3ce781452b templates/cloud/index.mako --- /dev/null +++ b/templates/cloud/index.mako @@ -0,0 +1,94 @@ +<%inherit file="/webapps/galaxy/base_panels.mako"/> + +<%def name="init()"> +<% + self.has_left_panel=False + self.has_right_panel=False + self.active_view="shared" + self.message_box_visible=False +%> +</%def> + +<%def name="stylesheets()"> + ${parent.stylesheets()} + ${h.css( "autocomplete_tagging" )} + <style type="text/css"> + #new_history_p{ + line-height:2.5em; + margin:0em 0em .5em 0em; + } + #new_history_cbx{ + margin-right:.5em; + } + #new_history_input{ + display:none; + line-height:1em; + } + #ec_button_container{ + float:right; + } + div.toolForm{ + margin-top: 10px; + margin-bottom: 10px; + } + div.toolFormTitle{ + cursor:pointer; + } + .title_ul_text{ + text-decoration:underline; + } + .step-annotation { + margin-top: 0.25em; + font-weight: normal; + font-size: 97%; + } + .workflow-annotation { + margin-bottom: 1em; + } + </style> +</%def> + + + + +<%def name="center_panel()"> + <div style="overflow: auto; height: 100%;"> + <div class="page-container" style="padding: 10px;"> + <h2>Launch a Galaxy Cloud Instance</h2> + <div class="toolForm"> + <form action="cloud/launch_instance" method="post"> + <div class="form-row"> + <label for="id_cluster_name">Cluster Name</label> + <input type="text" size="80" name="cluster_name" id="id_cluster_name"/><br/> + </div> + <div class="form-row"> + <label for="id_password">Password</label> + <input type="password" size="40" name="password" id="id_password"/><br/> + </div> + <div class="form-row"> + <label for="id_key_id">Key ID</label> + <input type="text" size="40" name="key_id" id="id_key_id"/><br/> + </div> + <div class="form-row"> + <label for="id_secret">Secret Key</label> + <input type="text" size="120" name="secret" id="id_secret"/><br/> + </div> + <div class="form-row"> + <label for="id_instance_type">Instance Type</label> + <select name="instance_type" id="id_instance_type"> + <option value="m1.large">Large</option> + <option value="t1.micro">Micro</option> + <option value="m1.xlarge">Extra Large</option> + <option value="m2.4xlarge">High-Memory Quadruple Extra Large</option> + </select> + </div> + <div class="form-row"> + <p>Requesting the instance may take a moment, please be patient. Do not refresh your browser or navigate away from the page</p> + <input type="submit" value="Submit" id="id_submit"/> + </div> + </form> + </div> + </div> + </div> +</%def> + diff -r a86a94f46c6cad83eabf76aa20fb3537ddda4ee1 -r a7fec5917853c921043be653c87dea3ce781452b templates/cloud/run.mako --- /dev/null +++ b/templates/cloud/run.mako @@ -0,0 +1,41 @@ +<%inherit file="/webapps/galaxy/base_panels.mako"/> + +<%def name="init()"> +<% + self.has_left_panel=False + self.has_right_panel=False + self.active_view="shared" + self.message_box_visible=False +%> +</%def> + + +<%def name="center_panel()"> + <div style="overflow: auto; height: 100%;"> + <div class="page-container" style="padding: 10px;"> + <h2>Launching a Galaxy Cloud Instance</h2> +%if error: + <p>${error}</p> +%elif instance: + %if kp_material: + <h3>Very Important Key Pair Information</h3> + <p>A new key pair named '${kp_name}' has been created in your AWS + account and will be used to access this instance via ssh. It is + <strong>very important</strong> that you save the following private key + as it is not saved on this Galaxy instance and will be permanently lost + once you leave this page. To do this, save the following key block as + a plain text file named '${kp_name}'.</p> + <pre>${kp_material}</pre> + %endif + <p>The instance '${instance.id} has been successfully launched using the + '${instance.image_id}' AMI.<br/> Access it at <a + href="http://${instance.public_dns_name}">http://${instance.public_dns_name}</a></p> + <p>SSH access is available using your private key '${kp_name}'.</p> +%else: + <p> Unknown failure, no instance. Please refer to your AWS console at <a + href="https://console.aws.amazon.com">https://console.aws.amazon.com</a></p> +%endif + </div> + </div> +</%def> + diff -r a86a94f46c6cad83eabf76aa20fb3537ddda4ee1 -r a7fec5917853c921043be653c87dea3ce781452b templates/root/index.mako --- a/templates/root/index.mako +++ b/templates/root/index.mako @@ -184,14 +184,9 @@ <%def name="init()"><% - if trans.app.config.cloud_controller_instance: - self.has_left_panel=False - self.has_right_panel=False - self.active_view="cloud" - else: - self.has_left_panel=True - self.has_right_panel=True - self.active_view="analysis" + self.has_left_panel = True + self.has_right_panel = True + self.active_view = "analysis" %> %if trans.app.config.require_login and not trans.user: <script type="text/javascript"> @@ -228,8 +223,6 @@ center_url = h.url_for( controller='workflow', action='run', id=workflow_id ) elif m_c is not None: center_url = h.url_for( controller=m_c, action=m_a ) - elif trans.app.config.cloud_controller_instance: - center_url = h.url_for( controller='cloud', action='list' ) else: center_url = h.url_for( '/static/welcome.html' ) %> diff -r a86a94f46c6cad83eabf76aa20fb3537ddda4ee1 -r a7fec5917853c921043be653c87dea3ce781452b templates/webapps/galaxy/base_panels.mako --- a/templates/webapps/galaxy/base_panels.mako +++ b/templates/webapps/galaxy/base_panels.mako @@ -105,6 +105,16 @@ %> %endif + ## Cloud menu. + %if app.config.get_bool( 'enable_cloud_control', False ): + <% + menu_options = [ + [_('New Cloud Cluster'), h.url_for( controller='/cloud', action='index' ) ], + ] + tab( "cloud", _("Cloud"), h.url_for( controller='/cloud', action='index'), menu_options=menu_options ) + %> + %endif + ## Admin tab. ${tab( "admin", "Admin", h.url_for( controller='/admin', action='index' ), extra_class="admin-only", visible=( trans.user and app.config.is_admin_user( trans.user ) ) )} diff -r a86a94f46c6cad83eabf76aa20fb3537ddda4ee1 -r a7fec5917853c921043be653c87dea3ce781452b templates/workflow/display.mako --- a/templates/workflow/display.mako +++ b/templates/workflow/display.mako @@ -61,12 +61,11 @@ </div></%def> - <%def name="render_item_links( workflow )"> %if workflow.importable: - <a + <a href="${h.url_for( controller='/workflow', action='imp', id=trans.security.encode_id(workflow.id) )}" - class="icon-button import" + class="icon-button import" ## Needed to overwide initial width so that link is floated left appropriately. style="width: 100%" title="Import workflow">Import workflow</a> diff -r a86a94f46c6cad83eabf76aa20fb3537ddda4ee1 -r a7fec5917853c921043be653c87dea3ce781452b templates/workflow/run.mako --- a/templates/workflow/run.mako +++ b/templates/workflow/run.mako @@ -252,7 +252,15 @@ if not enable_unique_defaults: del already_used[:] %> - ${param.get_html_field( t, value, other_values ).get_html( str(step.id) + "|" + prefix )} + %if step.type == None: + ##Input Dataset Step, wrap for multiinput. + <span class='multiinput_wrap'> + ${param.get_html_field( t, value, other_values ).get_html( str(step.id) + "|" + prefix )} + </span> + %else: + ${param.get_html_field( t, value, other_values ).get_html( str(step.id) + "|" + prefix )} + %endif + <input type="hidden" name="${step.id}|__force_update__${prefix}${param.name}" value="true" /> %endif %elif isinstance( value, RuntimeValue ) or ( str(step.id) + '|__runtime__' + prefix + param.name ) in incoming: diff -r a86a94f46c6cad83eabf76aa20fb3537ddda4ee1 -r a7fec5917853c921043be653c87dea3ce781452b universe_wsgi.ini.sample --- a/universe_wsgi.ini.sample +++ b/universe_wsgi.ini.sample @@ -251,6 +251,10 @@ # Note that this requires java > 1.4 for executing yuicompressor.jar #pack_scripts = False +# Enable Cloud Launch + +#enable_cloud_launch = False + # -- Advanced proxy features # For help on configuring the Advanced proxy features, see: Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.