1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/changeset/93a04e500eb4/ changeset: 93a04e500eb4 user: dannon date: 2012-09-12 18:51:56 summary: Boto bumped to 2.5.2, async cloudlaunch update with downloadable keys and basic cluster picker. affected #: 3 files diff -r e2cb958b3f7596744bb08febc858fa25139fd1ac -r 93a04e500eb464b7dd7c0dba565b60661307e94b eggs.ini --- a/eggs.ini +++ b/eggs.ini @@ -33,7 +33,7 @@ [eggs:noplatform] amqplib = 0.6.1 Beaker = 1.4 -boto = 2.2.2 +boto = 2.5.2 decorator = 3.1.2 docutils = 0.7 drmaa = 0.4b3 diff -r e2cb958b3f7596744bb08febc858fa25139fd1ac -r 93a04e500eb464b7dd7c0dba565b60661307e94b lib/galaxy/web/controllers/cloudlaunch.py --- a/lib/galaxy/web/controllers/cloudlaunch.py +++ b/lib/galaxy/web/controllers/cloudlaunch.py @@ -8,38 +8,77 @@ import datetime import logging +import os +import tempfile import time +import boto +import pkg_resources from galaxy import eggs -import pkg_resources pkg_resources.require('boto') -import boto from galaxy import web from galaxy.web.base.controller import BaseUIController +from galaxy.util.json import to_json_string from boto.ec2.regioninfo import RegionInfo from boto.exception import EC2ResponseError +from boto.s3.connection import OrdinaryCallingFormat, S3Connection + log = logging.getLogger(__name__) +PKEY_PREFIX = 'gxy_pkey' +DEFAULT_KEYPAIR = 'deleteme_keypair' +DEFAULT_AMI = 'ami-da58aab3' + class CloudController(BaseUIController): + def __init__(self, app): BaseUIController.__init__(self, app) @web.expose def index(self, trans, share_string=None): - return trans.fill_template("cloud/index.mako", share_string=share_string) + return trans.fill_template("cloud/index.mako", default_keypair = DEFAULT_KEYPAIR, share_string=share_string) @web.expose - def launch_instance(self, trans, cluster_name, password, key_id, secret, instance_type, share_string): + def get_account_info(self, trans, key_id, secret, **kwargs): + """ + Get EC2 Account Info + """ + #Keypairs + account_info = {} + try: + ec2_conn = connect_ec2(key_id, secret) + kps = ec2_conn.get_all_key_pairs() + except EC2ResponseError, e: + log.error("Problem starting an instance: %s\n%s" % (e, e.body)) + account_info['keypairs'] = [akp.name for akp in kps] + #Existing Clusters + s3_conn = S3Connection(key_id, secret, calling_format=OrdinaryCallingFormat()) + buckets = s3_conn.get_all_buckets() + clusters = [] + for bucket in buckets: + pd = bucket.get_key('persistent_data.yaml') + if pd: + # This is a cloudman bucket. + # We need to get persistent data, and the cluster name. + for key in bucket.list(): + if key.name.endswith('.clusterName'): + clusters.append({'name':key.name.split('.clusterName')[0], 'persistent_data': pd.get_contents_as_string()}) + account_info['clusters'] = clusters + return to_json_string(account_info) + + @web.expose + def launch_instance(self, trans, cluster_name, password, key_id, secret, instance_type, share_string, keypair, **kwargs): ec2_error = None try: # Create security group & key pair used when starting an instance ec2_conn = connect_ec2(key_id, secret) sg_name = create_cm_security_group(ec2_conn) - kp_name, kp_material = create_key_pair(ec2_conn) + kp_name, kp_material = create_key_pair(ec2_conn, key_name=keypair) except EC2ResponseError, err: ec2_error = err.error_message if ec2_error: - return trans.fill_template("cloud/run.mako", error = ec2_error) + #return trans.fill_template("cloud/run.mako", error = ec2_error) + return {'errors':[ec2_error]} else: user_provided_data={'cluster_name':cluster_name, 'access_key':key_id, @@ -62,13 +101,41 @@ instance.update() ct +=1 time.sleep(1) - return trans.fill_template("cloud/run.mako", - instance = rs.instances[0], - kp_name = kp_name, - kp_material = kp_material) + if kp_material: + #We have created a keypair. Save to tempfile for one time retrieval. + (fd, fname) = tempfile.mkstemp(prefix=PKEY_PREFIX, dir=trans.app.config.new_file_path) + f = os.fdopen(fd, 'wt') + f.write(kp_material) + f.close() + kp_material_tag = fname[fname.rfind(PKEY_PREFIX) + len(PKEY_PREFIX):] + else: + kp_material_tag = None + return to_json_string({ + 'cluster_name': cluster_name, + 'instance_id': rs.instances[0].id, + 'image_id': rs.instances[0].image_id, + 'public_dns_name': rs.instances[0].public_dns_name, + 'kp_name': kp_name, + 'kp_material_tag':kp_material_tag + }) else: - return trans.fill_template("cloud/run.mako", - error = "Instance failure, but no specific error was detected. Please check your AWS Console.") + return {'errors':["Instance failure, but no specific error was detected. Please check your AWS Console."]} + + @web.expose + def get_pkey(self, trans, kp_material_tag=None): + if kp_material_tag: + expected_path = os.path.join(trans.app.config.new_file_path, PKEY_PREFIX + kp_material_tag) + if os.path.exists(expected_path): + f = open(expected_path) + kp_material = f.read() + f.close() + trans.response.headers['Content-Length'] = int( os.stat( expected_path ).st_size ) + trans.response.set_content_type( "application/octet-stream" ) #force octet-stream so Safari doesn't append mime extensions to filename + trans.response.headers["Content-Disposition"] = 'attachment; filename="%s.pem"' % DEFAULT_KEYPAIR + os.remove(expected_path) + return kp_material + trans.response.status = 400 + return "Invalid identifier" # ## Cloud interaction methods def connect_ec2(a_key, s_key): @@ -150,7 +217,7 @@ return True return False -def create_key_pair(ec2_conn, key_name='cloudman_key_pair'): +def create_key_pair(ec2_conn, key_name=DEFAULT_KEYPAIR): """ Create a key pair with the provided name. Return the name of the key or None if there was an error creating the key. """ @@ -169,8 +236,8 @@ return None, None return kp.name, kp.material -def run_instance(ec2_conn, user_provided_data, image_id='ami-da58aab3', - kernel_id=None, ramdisk_id=None, key_name='cloudman_key_pair', +def run_instance(ec2_conn, user_provided_data, image_id=DEFAULT_AMI, + kernel_id=None, ramdisk_id=None, key_name=DEFAULT_KEYPAIR, security_groups=['CloudMan']): """ Start an instance. If instance start was OK, return the ResultSet object else return None. diff -r e2cb958b3f7596744bb08febc858fa25139fd1ac -r 93a04e500eb464b7dd7c0dba565b60661307e94b templates/cloud/index.mako --- a/templates/cloud/index.mako +++ b/templates/cloud/index.mako @@ -27,6 +27,9 @@ #ec_button_container{ float:right; } + #hidden_options{ + display:none; + } div.toolForm{ margin-top: 10px; margin-bottom: 10px; @@ -45,57 +48,215 @@ .workflow-annotation { margin-bottom: 1em; } + #loading_indicator{ + position:fixed; + top:40px; + } </style></%def> +<%def name="javascripts()"> + ${parent.javascripts()} + <script type="text/javascript"> + var ACCOUNT_URL = "${h.url_for( controller='/cloudlaunch', action='get_account_info')}"; + var PKEY_DL_URL = "${h.url_for( controller='/cloudlaunch', action='get_pkey')}"; + $(document).ready(function(){ + $('#id_existing_instance').change(function(){ + var ei_name = $(this).val(); + if (ei_name === "New Cluster"){ + //For new instances, need to see the cluster name field. + $('#id_cluster_name').val("New Cluster") + $('#cluster_name_wrapper').show('fast'); + }else{ + //Hide the Cluster Name field, but set the value + $('#id_cluster_name').val($(this).val()); + $('#cluster_name_wrapper').hide('fast'); + } + }); + //When id_secret and id_key are complete, submit to get_account_info + $("#id_secret, #id_key_id").bind("change paste keyup", function(){ + secret_el = $("#id_secret"); + key_el = $("#id_key_id"); + if (secret_el.val().length === 40 && key_el.val().length === 20){ + //Submit these to get_account_info, unhide fields, and update as appropriate + $.getJSON(ACCOUNT_URL, + {key_id: key_el.val(),secret:secret_el.val()}, + function(result){ + var kplist = $("#id_keypair"); + var clusterlist = $("#id_existing_instance"); + kplist.find('option').remove(); + clusterlist.find('option').remove(); + //Update fields with appropriate elements + if (_.size(result.clusters) > 0){ + clusterlist.append($('<option/>').val('New Cluster').text('New Cluster')); + _.each(result.clusters, function(cluster, index){ + clusterlist.append($('<option/>').val(cluster.name).text(cluster.name)); + }); + $('#existing_instance_wrapper').show(); + } + if (!_.include(result.keypairs, '${default_keypair}')){ + kplist.append($('<option/>').val('${default_keypair}').text('Create New - ${default_keypair}')); + } + _.each(result.keypairs, function(keypair, index){ + kplist.append($('<option/>').val(keypair).text(keypair)); + }); + $('#hidden_options').show('fast'); + }); + } + }); + $('#loading_indicator').ajaxStart(function(){ + $(this).show('fast'); + }).ajaxStop(function(){ + $(this).hide('fast'); + }); + $('form').ajaxForm({ + type: 'POST', + dataType: 'json', + beforeSubmit: function(data){ + //Hide the form, show pending box with spinner. + $('#launchFormContainer').hide('fast'); + $('#responsePanel').show('fast'); + }, + success: function(data){ + //Success Message, link to key download if required, link to server itself. + $('#launchPending').hide('fast'); + //Check for success/error. + if (data.error){ + //Apologize profusely. + $("launchPending").hide(); + $("#launchError").show(); + }else{ + //Set appropriate fields (dns, key, ami) and then display. + if(data.kp_material_tag){ + var kp_download_link = $('<a/>').attr('href', PKEY_DL_URL + '?kp_material_tag=' + data.kp_material_tag) + .attr('target','_blank') + .text("Download your key now"); + $('#keypairInfo').append(kp_download_link); + $('#keypairInfo').show(); + } + $('.kp_name').text(data.kp_name); + $('#instance_id').text(data.instance_id); + $('#image_id').text(data.image_id); + $('#instance_link').html($('<a/>') + .attr('href', 'http://' + data.public_dns_name + '/cloud') + .attr('target','_blank') + .text(data.public_dns_name + '/cloud')); + $('#instance_dns').text(data.public_dns_name); + $('#launchSuccess').show('fast'); + } + } + }); + }); + </script> +</%def><%def name="center_panel()"><div style="overflow: auto; height: 100%;"><div class="page-container" style="padding: 10px;"> + <div id="loading_indicator"></div><h2>Launch a Galaxy Cloud Instance</h2> - <div class="toolForm"> - <form action="${h.url_for( controller='cloudlaunch', action='launch_instance' )}" method="post"> - <div class="form-row"> - <label for="id_cluster_name">Cluster Name</label> - <input type="text" size="40" name="cluster_name" id="id_cluster_name"/><br/> - </div> - <div class="form-row"> - <label for="id_password">Password</label> - <input type="password" size="40" name="password" id="id_password"/><br/> - </div> + <div id="launchFormContainer" class="toolForm"> + <form id="cloudlaunch_form" action="${h.url_for( controller='/cloudlaunch', action='launch_instance')}" method="post"> + + <p>To launch a Galaxy Cloud Cluster, enter your AWS Secret Key ID, and Secret Key. Galaxy will use these to present appropriate +options for launching your cluster.</p> + <div class="form-row"><label for="id_key_id">Key ID</label> - <input type="text" size="40" name="key_id" id="id_key_id"/><br/> + <input type="text" size="30" maxlength="20" name="key_id" id="id_key_id" value=""/><br/> + <div class="toolParamHelp"> + This is the text string that uniquely identifies your account, found in the <a +href="https://portal.aws.amazon.com/gp/aws/securityCredentials">Security Credentials section of the AWS +Console</a>. + </div></div> + <div class="form-row"><label for="id_secret">Secret Key</label> - <input type="password" size="120" name="secret" id="id_secret"/><br/> + <input type="text" size="50" maxlength="40" name="secret" id="id_secret" value=""/><br/> + <div class="toolParamHelp"> + This is your AWS Secret Key, also found in the <a href="https://portal.aws.amazon.com/gp/aws/securityCredentials">Security +Credentials section of the AWS Console</a>. </div></div> - %if share_string: - <input type='hidden' name='share_string' value='${share_string}'/> - %else: - <div class="form-row"> - <label for="id_share_string">Instance Share String (optional)</label> - <input type="text" size="120" name="share_string" id="id_share_string"/><br/> + + <div id="hidden_options"> + <div id='existing_instance_wrapper' style="display:none;" class="form-row"> + <label for="id_existing_instance">Instances in your account</label> + <select name="existing_instance" id="id_existing_instance"> + </select> + </div> + <div id='cluster_name_wrapper' class="form-row"> + <label for="id_cluster_name">Cluster Name</label> + <input type="text" size="40" class="text-and-autocomplete-select" name="cluster_name" id="id_cluster_name"/><br/> + <div class="toolParamHelp"> + This is the name for your cluster. You'll use this when you want to restart. + </div> + </div> + + <div class="form-row"> + <label for="id_password">Cluster Password</label> + <input type="password" size="40" name="password" id="id_password"/><br/> + </div> + + <div class="form-row"> + <label for="id_keypair">Key Pair</label> + <select name="keypair" id="id_keypair"> + <option name="Create" value="cloudman_keypair">cloudman_keypair</option> + </select> + </div> + + %if share_string: + <input type='hidden' name='share_string' value='${share_string}'/> + %else: + <div class="form-row"> + <label for="id_share_string">Instance Share String (optional)</label> + <input type="text" size="120" name="share_string" id="id_share_string"/><br/> + </div> + %endif + <div class="form-row"> + <label for="id_instance_type">Instance Type</label> + <select name="instance_type" id="id_instance_type"> + <option value="m1.large">Large</option> + <option value="m1.xlarge">Extra Large</option> + <option value="m2.4xlarge">High-Memory Quadruple Extra Large</option> + </select> + </div> + <div class="form-row"> + <p>Requesting the instance may take a moment, please be patient. Do not refresh your browser or navigate away from the page</p> + <input type="submit" value="Submit" id="id_submit"/> + </div></div> - %endif - <div class="form-row"> - <label for="id_instance_type">Instance Type</label> - <select name="instance_type" id="id_instance_type"> - <option value="m1.large">Large</option> - <option value="t1.micro">Micro</option> - <option value="m1.xlarge">Extra Large</option> - <option value="m2.4xlarge">High-Memory Quadruple Extra Large</option> - </select> - </div> - <div class="form-row"> - <p>Requesting the instance may take a moment, please be patient. Do not refresh your browser or navigate away from the page</p> - <input type="submit" value="Submit" id="id_submit"/> - </div> + <div class="form-row"> + <div id="loading_indicator" style="position:relative;left:10px;right:0px"></div> + </div></form></div> + <div id="responsePanel" class="toolForm" style="display:none;"> + <div id="launchPending">Launch Pending, please be patient.</div> + <div id="launchError" style="display:none;">ERROR</div> + <div id="launchSuccess" style="display:none;"> + <div id="keypairInfo" style="display:none;margin-bottom:20px;"> + <h3>Very Important Key Pair Information</h3> + <p>A new key pair named <strong><span class="kp_name">kp_name</span></strong> has been created in your AWS + account and will be used to access this instance via ssh. It is + <strong>very important</strong> that you save the following private key + as it is not saved on this Galaxy instance and will be permanently lost if not saved. Additionally, this link will + only allow a single download, after which the key is removed from the Galaxy server permanently.<br/> + </div> + <div> + <h3>Access Information</h3> + <ul> + <li>Your instance '<span id="instance_id">undefined</span>' has been successfully launched using the + '<span id="image_id">undefined</span>' AMI.</li> + <li>While it may take a few moments to boot, you will be able to access the cloud control + panel at <span id="instance_link">undefined.</span>.</li> + <li>SSH access is also available using your private key. From the terminal, you would execute something like:</br> `ssh -i <span class="kp_name">undefined</span>.pem ubuntu@<span +id="instance_dns">undefined</span>`</li> + </ul> + </div> + </div></div></div></%def> Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.