June 2010 - galaxy-commits - lists.galaxyproject.org

galaxy-dist commit 47411bb09df4: Remove the display application definitions for the bx application.
by commits-noreply＠bitbucket.org 29 Jun '10

29 Jun '10

<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.1//EN" "http://www.w3.org/TR/xhtml11/DTD/xhtml11.dtd"> <html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en"> <head> <title> 404 - Not Found — bitbucket.org </title> <meta http-equiv="Content-Type" content="text/html; charset=utf-8" /> <meta name="description" content="Mercurial hosting - we're here to serve." /> <meta name="keywords" content="mercurial,hg,hosting,bitbucket," /> <link rel="stylesheet" type="text/css" href="http://bitbucket-assets.s3.amazonaws.com/css/layout.css" /> <link rel="stylesheet" type="text/css" href="http://bitbucket-assets.s3.amazonaws.com/css/screen.css" /> <link rel="stylesheet" type="text/css" href="http://bitbucket-assets.s3.amazonaws.com/css/print.css" media="print" /> <link rel="search" type="application/opensearchdescription+xml" href="/opensearch.xml" title="Bitbucket" /> <link rel="icon" href="http://bitbucket-assets.s3.amazonaws.com/img/logo_new.png" type="image/png"/> <script type="text/javascript">var MEDIA_URL = "http://bitbucket-assets.s3.amazonaws.com/"</script> <script type="text/javascript" src="http://bitbucket-assets.s3.amazonaws.com/js/lib/bundle.020510May.js"></script> <script type="text/javascript"> $(document).ready(function() { Dropdown.init(); $(".tooltip").tipsy({gravity:'s'}); }); </script> <noscript> <style type="text/css"> .dropdown-container-text .dropdown { position: static !important; } </style> </noscript>  </head> <body class=""> <div id="main-wrapper"> <div id="header-wrapper"> <div id="header"> <a href="/"><img src="http://bitbucket-assets.s3.amazonaws.com/img/logo_myriad.png" alt="Bitbucket" id="header-wrapper-logo" /></a> <div id="header-nav"> <ul class="right"> <li><a href="/">Home</a></li> <li><a href="/plans"><b>Plans & Signup</b></a></li> <li><a href="/repo/all">Repositories</a></li> <li><a href="/news">News</a></li> <li><a href="/help">Help</a></li> <li><a href="/account/signin/">Sign in</a></li> </ul> </div> </div> </div> <div id="content-wrapper"> <div class="container"> <h2>404 - Not Found</h2> <p>We're sorry, but we couldn't find the page you were looking for.</p> <p>You can perhaps find what you're looking for by doing a search in the list of <a href="/repo/all/">public repositories</a>.</p> <p>If you feel that this is an error on our part, please <a href="mailto:support@bitbucket.org">send us an email</a> with details about the error and what you were doing when it occurred.</p> <p>-- The Bitbucket team</p> </div> <div class="cb"></div> </div> <div class="cb footer-placeholder"></div> </div> <div id="footer-wrapper"> <div id="footer"> <a href="/site/terms/">TOS</a> | <a href="/site/privacy/">Privacy Policy</a> | <a href="http://blog.bitbucket.org/">Blog</a> | <a href="http://bitbucket.org/jespern/bitbucket/issues/new/">Report Bug</a> | <a href="http://groups.google.com/group/bitbucket-users">Discuss</a> | <a href="http://avantlumiere.com/">© 2008-2010</a> | We run <small><b> <a href="http://www.djangoproject.com/">Django 1.2.1</a> / <a href="http://bitbucket.org/jespern/django-piston/">Piston 0.2.3rc1</a> / <a href="http://www.selenic.com/mercurial/">Hg 1.3.1</a> / <a href="http://www.python.org">Python 2.5.2</a> / r3056| fe02 </b></small> </div> </div> <script type="text/javascript"> var _gaq = _gaq || []; _gaq.push(['_setAccount', 'UA-2456069-3'], ['_trackPageview']); var _gaq = _gaq || []; _gaq.push(['atl._setAccount', 'UA-6032469-33'], ['atl._trackPageview']); (function() { var ga = document.createElement('script'); ga.src = ('https:' == document.location.protocol ? 'https://ssl' : 'http://www') + '.google-analytics.com/ga.js'; ga.setAttribute('async', 'true'); document.documentElement.firstChild.appendChild(ga); })(); </script> </body> </html>

1 0

galaxy-dist commit 067a8649dae7: Remove code and drop database tables related to the old Galaxy Cloud functionality.
by commits-noreply＠bitbucket.org 29 Jun '10

29 Jun '10

# HG changeset patch -- Bitbucket.org # Project galaxy-dist # URL http://bitbucket.org/galaxy/galaxy-dist/overview # User Enis Afgan <afgane(a)gmail.com> # Date 1277306301 14400 # Node ID 067a8649dae79ab973ff3821b62be89e21a0117a # Parent 752cb3e325374381bdd0db6b9163b12ba88b7151 Remove code and drop database tables related to the old Galaxy Cloud functionality. --- a/templates/cloud/edit_image.mako +++ /dev/null @@ -1,92 +0,0 @@ -<% _=n_ %> -<%inherit file="/base.mako"/> -<%def name="title()">Edit machine image</%def> - -<%def name="javascripts()"> -${parent.javascripts()} -<script type="text/javascript"> -$(function(){ - $("input:text:first").focus(); -}) -</script> -</%def> - -%if header: - ${header} -%endif - -%if image: - -<div class="form"> - <div class="form-title">Edit image</div> - <div class="form-body"> - <form name="edit_image" action="${h.url_for( action='edit_image', id=trans.security.encode_id(image.id), edited="true" )}" method="post" > - <% - cls = "form-row" - if error.has_key('provider_error'): - cls += " form-row-error" - %> - <div class="${cls}"> - <label>Provider type:</label> - <div class="form-row-input"> - ${image.provider_type} - </div> - %if error.has_key('provider_error'): - <div class="form-row-error-message">${error['provider_error']}</div> - %endif - <div style="clear: both"></div> - </div> - <% - cls = "form-row" - if error.has_key('id_error'): - cls += " form-row-error" - %> - <div class="${cls}"> - <label>Machine Image ID (AMI or EMI):</label> - <div class="form-row-input"> - <input type="text" name="image_id" value="${image.image_id}" size="40"> - </div> - %if error.has_key('id_error'): - <div class="form-row-error-message">${error['id_error']}</div> - %endif - <div style="clear: both"></div> - </div> - <% - cls = "form-row" - if error.has_key('manifest_error'): - cls += " form-row-error" - %> - <div class="${cls}"> - <label>Manifest:</label> - <div class="form-row-input"> - <input type="text" name="manifest" value="${image.manifest}" size="40"> - </div> - %if error.has_key('manifest_error'): - <div class="form-row-error-message">${error['manifest_error']}</div> - %endif - <div style="clear: both"></div> - </div> - <% - cls = "form-row" - if error.has_key('arch_error'): - cls += " form-row-error" - %> - <div class="${cls}"> - <label>Architecture:</label> - <div class="form-row-input"> - <input type="text" name="architecture" value="${image.architecture}" size="40"> - </div> - %if error.has_key('arch_error'): - <div class="form-row-error-message">${error['arch_error']}</div> - %endif - <div style="clear: both"></div> - </div> - - <div class="form-row"><input type="submit" value="Save"></div> - </form> - </div> -</div> - -%else: - Specified machine image could not be found. -%endif --- a/templates/cloud/view_provider.mako +++ /dev/null @@ -1,126 +0,0 @@ -<%inherit file="/base.mako"/> - -<%def name="title()">Cloud provider</%def> - -<h2>Cloud provider details</h2> - -<ul class="manage-table-actions"> - <li> - <a class="action-button" href="${h.url_for( action='list' )}"> - <img src="${h.url_for('/static/images/silk/resultset_previous.png')}" /> - <span>Return to cloud management console</span> - </a> - </li> -</ul> - -%if provider: - ${view_provider( provider )} -%else: - There is no cloud provider under that name. -%endif - - - - -<%def name="view_provider( provider )"> - <table class="mange-table colored" border="0" cellspacing="0" cellpadding="0" width="100%"> - <tr> - <td> Cloud provider name: </td> - <td> - ${provider.name} - <a id="cp-popup" class="popup-arrow" style="display: none;">▼</a> - </td> - <td> - <div popupmenu="cp-popup"> - <a class="action-button" href="${h.url_for( action='edit_provider', id=trans.security.encode_id(provider.id) )}">Edit</a> - <a class="action-button" confirm="Are you sure you want to delete cloud provider '${provider.name}'?" href="${h.url_for( action='delete_provider', id=trans.security.encode_id(provider.id) )}">Delete</a> - </div> - </td> - </tr> - <tr> - <td> Last updated: </td> - <td> ${str(provider.update_time)[:16]} - <% - context.write( ' UTC (' ) - context.write( str(h.date.distance_of_time_in_words (provider.update_time, h.date.datetime.utcnow() ) ) ) - %> ago) - </td> - </tr> - <tr> - <td> Cloud provider type: </td> - <td> ${str(provider.type)[:16]}</td> - </tr> - %if provider.region_connection != None: - <tr> - <td> Region connection: </td> - <td> ${provider.region_connection} </td> - </tr> - %endif - %if provider.region_name != None: - <tr> - <td> Region name: </td> - <td> ${provider.region_name} </td> - </tr> - %endif - %if provider.region_endpoint != None: - <tr> - <td> Region endpoint: </td> - <td> ${provider.region_endpoint} </td> - </tr> - %endif - %if provider.is_secure != None: - <tr> - <td> Is secure: </td> - <td> ${provider.is_secure} </td> - </tr> - %endif - %if provider.host != None: - <tr> - <td> Host: </td> - <td> ${provider.host} </td> - </tr> - %endif - %if provider.port != None: - <tr> - <td> Port: </td> - <td> ${provider.port} </td> - </tr> - %endif - %if provider.proxy != None: - <tr> - <td> Proxy: </td> - <td> ${provider.proxy} </td> - </tr> - %endif - %if provider.proxy_port != None: - <tr> - <td> Proxy port: </td> - <td> ${provider.proxy_port} </td> - </tr> - %endif - %if provider.proxy_pass != None: - <tr> - <td> Proxy pass: </td> - <td> ${provider.proxy_pass} </td> - </tr> - %endif - %if provider.debug != None: - <tr> - <td> Debug: </td> - <td> ${provider.debug} </td> - </tr> - %endif - %if provider.https_connection_factory != None: - <tr> - <td> HTTPS connection factory: </td> - <td> ${provider.https_connection_factory} </td> - </tr> - %endif - %if provider.path != None: - <tr> - <td> Path: </td> - <td> ${provider.path} </td> - </tr> - %endif - </table> -</%def> --- a/templates/cloud/edit_credentials.mako +++ /dev/null @@ -1,91 +0,0 @@ -<% _=n_ %> -<%inherit file="/base.mako"/> -<%def name="title()">Add credentials</%def> - -<%def name="javascripts()"> -${parent.javascripts()} -<script type="text/javascript"> -$(function(){ - $("input:text:first").focus(); -}) -</script> -</%def> - -%if header: - ${header} -%endif - -%if credential: - -<div class="form"> - <div class="form-title">Edit credentials</div> - <div class="form-body"> - <form name="edit_credentials" action="${h.url_for( action='edit_credentials', id=trans.security.encode_id(credential.id), edited="true" )}" method="post" > - - <% - cls = "form-row" - if error.has_key('cred_error'): - cls += " form-row-error" - %> - <div class="${cls}"> - <label>Credentials name:</label> - <div class="form-row-input"> - <input type="text" name="credName" value="${credential.name}" size="40"> - </div> - %if error.has_key('cred_error'): - <div class="form-row-error-message">${error['cred_error']}</div> - %endif - <div style="clear: both"></div> - </div> - - - <% - cls = "form-row" - if error.has_key('provider_error'): - cls += " form-row-error" - %> - <div class="${cls}"> - <label>Cloud provider name (type):</label> - <div class="form-row-input">${credential.provider.name} (${credential.provider.type})</div> - <div style="clear: both"></div> - </div> - <% - cls = "form-row" - if error.has_key('access_key_error'): - cls += " form-row-error" - %> - <div class="${cls}"> - <label>Access key:</label> - <div class="form-row-input"> - <input type="text" name="accessKey" value="${credential.access_key}" size="40"> - </div> - %if error.has_key('access_key_error'): - <div class="form-row-error-message">${error['access_key_error']}</div> - %endif - <div style="clear: both"></div> - </div> - - <% - cls = "form-row" - if error.has_key('secret_key_error'): - cls += " form-row-error" - %> - <div class="${cls}"> - <label>Secret key:</label> - <div class="form-row-input"> - <input type="password" name="secretKey" value="${credential.secret_key}" size="40"> - </div> - %if error.has_key('secret_key_error'): - <div class="form-row-error-message">${error['secret_key_error']}</div> - %endif - <div style="clear: both"></div> - </div> - - <div class="form-row"><input type="submit" value="Edit"></div> - </form> - </div> -</div> - -%else: - Specified credentials could not be found. -%endif --- a/templates/cloud/edit_provider.mako +++ /dev/null @@ -1,261 +0,0 @@ -<% _=n_ %> -<%inherit file="/base.mako"/> -<%def name="title()">Edit provider</%def> - -<%def name="javascripts()"> -${parent.javascripts()} -<script type="text/javascript"> -$(function(){ - - $("input:text:first").focus(); - - $("#type").change(function() { - if ($(this).val() == 'ec2') { - clear(); - $("#autofill").attr( 'disabled', true ); - $("#autofill").attr( 'checked', false ); - $("#name").val( "EC2" ); - $("#region_name").val( "us-east-1" ); - $("#region_endpoint").val( "us-east-1.ec2.amazonaws.com" ); - $("#is_secure").val("1"); - $("#debug").val(""); - $("#path").val("/"); - } - else if ($(this).val() == 'eucalyptus') { - clear(); - $("#autofill").attr( 'disabled', false ); - } - }); -}) - - -function af(){ - - if ( $("#autofill_epc").attr('checked') ) { - $("#region_name").val("eucalyptus"); - $("#region_endpoint").val("mayhem9.cs.ucsb.edu"); - $("#is_secure").val("0"); - $("#port").val("8773"); - $("#path").val("/services/Eucalyptus"); - } - else if ( $("#autofill_ec2").attr('checked') ) { - $("#region_name").val( "us-east-1" ); - $("#region_endpoint").val( "us-east-1.ec2.amazonaws.com" ); - $("#is_secure").val("1"); - $("#debug").val(""); - $("#path").val("/"); - } -} - -function clear() { - //$("#name").val(""); - $("#region_name").val(""); - $("#region_endpoint").val(""); - $("#is_secure").val(""); - $("#port").val(""); - $("#proxy").val(""); - $("#proxy_port").val(""); - $("#proxy_user").val(""); - $("#proxy_pass").val(""); - $("#debug").val(""); - $("#https_connection_factory").val(""); - $("#path").val(""); - -} - -</script> -</%def> - -%if header: - ${header} -%endif - -%if provider: - <div class="form"> - <div class="form-title">Edit cloud provider</div> - <div class="form-body"> - <form name="edit_provider_form" action="${h.url_for( action='edit_provider', id=trans.security.encode_id(provider.id), edited="true" )}" method="post" > - <% - cls = "form-row" - if error.has_key('type_error'): - cls += " form-row-error" - %> - <div class="${cls}"> - <label>Provider type:</label> - <div class="form-row-input">${provider.type} - %if provider.type == 'eucalyptus': - <p><input type="checkbox" id="autofill_epc" onclick="javascript:af()"> - auto fill using Eucalyptus Public Cloud values - </p></div> - %elif provider.type == 'ec2': - <p><input type="checkbox" id="autofill_ec2" onclick="javascript:af()"> - auto fill for Amazon EC2 (us-east-1 region) - </p></div> - %endif - <div style="clear: both"></div> - </div> - - <% - cls = "form-row" - if error.has_key('name_error'): - cls += " form-row-error" - %> - <div class="${cls}"> - <label>Provider name:</label> - <div class="form-row-input"> - <input type="text" id="name" name="name" value="${provider.name}" size="40"> - </div> - %if error.has_key('name_error'): - <div class="form-row-error-message">${error['name_error']}</div> - %endif - <div style="clear: both"></div> - </div> - - <% - cls = "form-row" - %> - <div class="${cls}"> - <label>Region name:</label> - <div id="region_selection" class="form-row-input"> - <input type="text" name="region_name" id="region_name" value="${provider.region_name}" size="40"> - </div> - <div style="clear: both"></div> - </div> - - <% - cls = "form-row" - %> - <div class="${cls}"> - <label>Region endpoint:</label> - <div class="form-row-input"> - <input type="text" name="region_endpoint" id="region_endpoint" value="${provider.region_endpoint}" size="40"> - </div> - <div style="clear: both"></div> - </div> - - <% - cls = "form-row" - if error.has_key('is_secure_error'): - cls += " form-row-error" - %> - <div class="${cls}"> - <label>Is secure ('O' for False or '1' for True):</label> - <div class="form-row-input"> - %if provider.is_secure == True: - <input type="text" name="is_secure" id="is_secure" value="1" size="40"> - %else: - <input type="text" name="is_secure" id="is_secure" value="0" size="40"> - %endif - </div> - %if error.has_key('is_secure_error'): - <div class="form-row-error-message">${error['is_secure_error']}; you entered: '${provider.is_secure}'</div> - %endif - <div style="clear: both"></div> - </div> - - <% - cls = "form-row" - %> - <div class="${cls}"> - <label>Host:</label> - <div class="form-row-input"> - <input type="text" name="host" value="${provider.host}" size="40"> - </div> - <div style="clear: both"></div> - </div> - - <% - cls = "form-row" - %> - <div class="${cls}"> - <label>Port:</label> - <div class="form-row-input"> - <input type="text" name="port" id="port" value="${provider.port}" size="40"> - </div> - <div style="clear: both"></div> - </div> - - <% - cls = "form-row" - %> - <div class="${cls}"> - <label>Proxy:</label> - <div class="form-row-input"> - <input type="text" name="proxy" value="${provider.proxy}" size="40"> - </div> - <div style="clear: both"></div> - </div> - - <% - cls = "form-row" - %> - <div class="${cls}"> - <label>Proxy port:</label> - <div class="form-row-input"> - <input type="text" name="proxy_port" value="${provider.proxy_port}" size="40"> - </div> - <div style="clear: both"></div> - </div> - - <% - cls = "form-row" - %> - <div class="${cls}"> - <label>Proxy user:</label> - <div class="form-row-input"> - <input type="text" name="proxy_user" value="${provider.proxy_user}" size="40"> - </div> - <div style="clear: both"></div> - </div> - - <% - cls = "form-row" - %> - <div class="${cls}"> - <label>Proxy pass:</label> - <div class="form-row-input"> - <input type="text" name="proxy_pass" value="${provider.proxy_pass}" size="40"> - </div> - <div style="clear: both"></div> - </div> - - <% - cls = "form-row" - %> - <div class="${cls}"> - <label>Debug:</label> - <div class="form-row-input"> - <input type="text" name="debug" value="${provider.debug}" size="40"> - </div> - <div style="clear: both"></div> - </div> - - <% - cls = "form-row" - %> - <div class="${cls}"> - <label>HTTPS connection factory:</label> - <div class="form-row-input"> - <input type="text" name="https_connection_factory" value="${provider.https_connection_factory}" size="40"> - </div> - <div style="clear: both"></div> - </div> - - <% - cls = "form-row" - %> - <div class="${cls}"> - <label>Path:</label> - <div class="form-row-input"> - <input type="text" name="path" id="path" value="${provider.path}" size="40"> - </div> - <div style="clear: both"></div> - </div> - - <div class="form-row"><input type="submit" value="Save"></div> - - </form> - - </div> - </div> -%endif --- a/templates/cloud/view_usage.mako +++ /dev/null @@ -1,117 +0,0 @@ -<%inherit file="/base.mako"/> - -<%def name="title()">Cloud home</%def> - -%if message: -<% - try: - messagetype - except: - messagetype = "done" -%> - - - -<p /> -<div class="${messagetype}message"> - ${message} -</div> -%endif - -%if prevInstances: - <h2>Usage report for instance ${prevInstances[0].uci.name}</h2> -%else: - <h2>Selected instance has no record of being used.</h2> -%endif - -<ul class="manage-table-actions"> - <li> - <a class="action-button" href="${h.url_for( action='list' )}"> - <img src="${h.url_for('/static/images/silk/resultset_previous.png')}" /> - <span>Return to cloud management console</span> - </a> - </li> -</ul> - -%if prevInstances: - <table class="mange-table colored" border="0" cellspacing="0" cellpadding="0" width="100%"> - <colgroup width="2%"></colgroup> - <colgroup width="16%"></colgroup> - <colgroup width="16%"></colgroup> - <colgroup width="10%"></colgroup> - <colgroup width="5%"></colgroup> - <tr class="header"> - <th>#</th> - <th>Launch time</th> - <th>Termination time</th> - <th>Time alive</th> - <th>Type</th> - <th></th> - </tr> - <% - total_hours = 0 - %> - %for i, prevInstance in enumerate( prevInstances ): - <tr> - <td>${i+1}</td> - <td> - %if prevInstance.launch_time: - ${str(prevInstance.launch_time)[:16]} UCT - %else: - N/A - %endif - </td> - <td> - %if prevInstance.stop_time: - ${str(prevInstance.stop_time)[:16]} UCT - %else: - N/A - %endif - </td> - <td> - <% - # This is where current time and since duration is calculated - if prevInstance.launch_time is None or prevInstance.stop_time is None: - context.write( 'N/A' ) - else: - context.write( str(h.date.distance_of_time_in_words (prevInstance.launch_time, prevInstance.stop_time ) ) ) - time_delta = prevInstance.stop_time - prevInstance.launch_time - total_hours += time_delta.seconds / 3600 - if time_delta.seconds != 0: - total_hours += 1 - - %> - </td> - <td>${prevInstance.type}</td> - </tr> - %endfor - </table> - <br/>Total number of hours instance was alive: ${total_hours} <br /> - Note that these are just best effort estimates - true usage should be obtained from respective cloud provider. <br /> - <%namespace name="view_cred" file="view_credentials.mako" /> - - <div id="hide_cred_details"> - This instance uses credentials: - <a onclick="document.getElementById('show_cred_details').style.display = 'block'; - document.getElementById('hide_cred_details').style.display = 'none'; return 0" - href="javascript:void(0)"> - ${prevInstances[0].uci.credentials.name} - </a> - </div> - <div id="show_cred_details" style="DISPLAY: none"> - This instance uses credentials: - <a onclick="document.getElementById('hide_cred_details').style.display = 'block'; - document.getElementById('show_cred_details').style.display = 'none'; return 0;" - href="javascript:void(0)"> - ${prevInstances[0].uci.credentials.name} - </a> - ${view_cred.view_cred( prevInstances[0].uci.credentials ) } - </div> - - -%endif - - - - - --- a/templates/cloud/view_credentials.mako +++ /dev/null @@ -1,157 +0,0 @@ -<%inherit file="/base.mako"/> - -<%def name="title()">Cloud credentials</%def> - -<h2>Credentials details</h2> - -<ul class="manage-table-actions"> - <li> - <a class="action-button" href="${h.url_for( action='list' )}"> - <img src="${h.url_for('/static/images/silk/resultset_previous.png')}" /> - <span>Return to cloud management console</span> - </a> - </li> -</ul> - -%if credDetails: - ${view_cred( credDetails )} -%else: - There are no credentials under that name. -%endif - - - - -<%def name="view_cred( credDetails )"> - <table class="mange-table colored" border="0" cellspacing="0" cellpadding="0" width="100%"> - <tr> - <td> Credentials name: </td> - <td> - ${credDetails.name} - <a id="wf-popup" class="popup-arrow" style="display: none;">▼</a> - </td> - <td> - <div popupmenu="wf-popup"> - <a class="action-button" href="${h.url_for( action='edit_credentials', id=trans.security.encode_id(credDetails.id) )}">Edit</a> - <a class="action-button" confirm="Are you sure you want to delete credentials '${credDetails.name}'?" href="${h.url_for( action='delete_credentials', id=trans.security.encode_id(credDetails.id) )}">Delete</a> - </div> - </td> - </tr> - <tr> - <td> Last updated: </td> - <td> ${str(credDetails.update_time)[:16]} - <% - context.write( ' UTC (' ) - context.write( str(h.date.distance_of_time_in_words (credDetails.update_time, h.date.datetime.utcnow() ) ) ) - %> ago) - </td> - </tr> - <tr> - <td> Cloud provider type: </td> - <td> ${str(credDetails.provider.type)}</td> - </tr> - <tr> - <td> Cloud provider name: </td> - <td> ${str(credDetails.provider.name)}</td> - </tr> - <tr> - <td> Access key: </td> - <td> - ${credDetails.access_key} - </td> - </tr> - <tr> - <td> Secret key: </td> - <td> - <div id="shortComment2"> - <a onclick="document.getElementById('fullComment2').style.display = 'block'; - document.getElementById('shortComment2').style.display = 'none'; return 0" - href="javascript:void(0)"> - + Show - </a> - </div> - <div id="fullComment2" style="DISPLAY: none"> - <a onclick="document.getElementById('shortComment2').style.display = 'block'; - document.getElementById('fullComment2').style.display = 'none'; return 0;" - href="javascript:void(0)"> - - Hide - </a><br /> - <nobr>${credDetails.secret_key}</nobr><br/> - </div> - </td> - </tr> - <tr><td id="addl"><b>Additional cloud provider information (if available):</b></td></tr> - %if credDetails.provider.region_connection != None: - <tr> - <td> Region connection: </td> - <td> ${credDetails.provider.region_connection} </td> - </tr> - %endif - %if credDetails.provider.region_name != None: - <tr> - <td> Region name: </td> - <td> ${credDetails.provider.region_name} </td> - </tr> - %endif - %if credDetails.provider.region_endpoint != None: - <tr> - <td> Region endpoint: </td> - <td> ${credDetails.provider.region_endpoint} </td> - </tr> - %endif - %if credDetails.provider.is_secure != None: - <tr> - <td> Is secure: </td> - <td> ${credDetails.provider.is_secure} </td> - </tr> - %endif - %if credDetails.provider.host != None: - <tr> - <td> Host: </td> - <td> ${credDetails.provider.host} </td> - </tr> - %endif - %if credDetails.provider.port != None: - <tr> - <td> Port: </td> - <td> ${credDetails.provider.port} </td> - </tr> - %endif - %if credDetails.provider.proxy != None: - <tr> - <td> Proxy: </td> - <td> ${credDetails.provider.proxy} </td> - </tr> - %endif - %if credDetails.provider.proxy_port != None: - <tr> - <td> Proxy port: </td> - <td> ${credDetails.provider.proxy_port} </td> - </tr> - %endif - %if credDetails.provider.proxy_pass != None: - <tr> - <td> Proxy pass: </td> - <td> ${credDetails.provider.proxy_pass} </td> - </tr> - %endif - %if credDetails.provider.debug != None: - <tr> - <td> Debug: </td> - <td> ${credDetails.provider.debug} </td> - </tr> - %endif - %if credDetails.provider.https_connection_factory != None: - <tr> - <td> HTTPS connection factory: </td> - <td> ${credDetails.provider.https_connection_factory} </td> - </tr> - %endif - %if credDetails.provider.path != None: - <tr> - <td> Path: </td> - <td> ${credDetails.provider.path} </td> - </tr> - %endif - </table> -</%def> --- a/lib/galaxy/cloud/providers/eucalyptus.py +++ /dev/null @@ -1,1039 +0,0 @@ -import subprocess, threading, os, errno, time, datetime -from Queue import Queue, Empty -from datetime import datetime - -from galaxy import model # Database interaction class -from galaxy.model import mapping -from galaxy.datatypes.data import nice_size -from galaxy.util.bunch import Bunch -from galaxy.cloud import UCIwrapper -from Queue import Queue -from sqlalchemy import or_, and_ - -import galaxy.eggs -galaxy.eggs.require("boto") -from boto.ec2.connection import EC2Connection -from boto.ec2.regioninfo import RegionInfo -import boto.exception -import boto - -import logging -log = logging.getLogger( __name__ ) - -uci_states = Bunch( - NEW_UCI = "newUCI", - NEW = "new", - CREATING = "creating", - DELETING_UCI = "deletingUCI", - DELETING = "deleting", - SUBMITTED_UCI = "submittedUCI", - SUBMITTED = "submitted", - SHUTTING_DOWN_UCI = "shutting-downUCI", - SHUTTING_DOWN = "shutting-down", - ADD_STORAGE_UCI = "add-storageUCI", - ADD_STORAGE = "add-storage", - AVAILABLE = "available", - RUNNING = "running", - PENDING = "pending", - ERROR = "error", - DELETED = "deleted", - SNAPSHOT_UCI = "snapshotUCI", - SNAPSHOT = "snapshot" -) - -instance_states = Bunch( - TERMINATED = "terminated", - SUBMITTED = "submitted", - RUNNING = "running", - ADDING = "adding-storage", - PENDING = "pending", - SHUTTING_DOWN = "shutting-down", - ERROR = "error" -) - -store_status = Bunch( - WAITING = "waiting", - IN_USE = "in-use", - ADDING = "adding", - CREATING = "creating", - DELETED = 'deleted', - ERROR = "error" -) - -snapshot_status = Bunch( - SUBMITTED = 'submitted', - PENDING = 'pending', - COMPLETED = 'completed', - DELETE = 'delete', - DELETED= 'deleted', - ERROR = "error" -) - -class EucalyptusCloudProvider( object ): - """ - Eucalyptus-based cloud provider implementation for managing instances. - """ - STOP_SIGNAL = object() - def __init__( self, app ): - self.type = "eucalyptus" # cloud provider type (e.g., ec2, eucalyptus, opennebula) - self.zone = "epc" - self.queue = Queue() - self.sa_session = app.model.context - - self.threads = [] - nworkers = 5 - log.info( "Starting eucalyptus cloud controller workers..." ) - for i in range( nworkers ): - worker = threading.Thread( target=self.run_next ) - worker.start() - self.threads.append( worker ) - log.debug( "%d eucalyptus cloud workers ready", nworkers ) - - def shutdown( self ): - """Attempts to gracefully shut down the monitor thread""" - log.info( "sending stop signal to worker threads in eucalyptus cloud manager" ) - for i in range( len( self.threads ) ): - self.queue.put( self.STOP_SIGNAL ) - log.info( "eucalyptus cloud manager stopped" ) - - def put( self, uci_wrapper ): - """ - Add uci_wrapper object to the end of the request queue to be handled by - this cloud provider. - """ - state = uci_wrapper.get_uci_state() - uci_wrapper.change_state( state.split('U')[0] ) # remove 'UCI' from end of state description (i.e., mark as accepted and ready for processing) - self.queue.put( uci_wrapper ) - - def run_next( self ): - """Process next request, waiting until one is available if necessary.""" - cnt = 0 - while 1: - uci_wrapper = self.queue.get() - uci_state = uci_wrapper.get_uci_state() - if uci_state is self.STOP_SIGNAL: - return - try: - if uci_state==uci_states.NEW: - self.create_uci( uci_wrapper ) - elif uci_state==uci_states.DELETING: - self.delete_uci( uci_wrapper ) - elif uci_state==uci_states.SUBMITTED: - self.start_uci( uci_wrapper ) - #self.dummy_start_uci( uci_wrapper ) - elif uci_state==uci_states.SHUTTING_DOWN: - self.stop_uci( uci_wrapper ) - elif uci_state==uci_states.SNAPSHOT: - self.snapshot_uci( uci_wrapper ) - elif uci_state==uci_states.ADD_STORAGE: - self.add_storage_to_uci( uci_wrapper ) - except: - log.exception( "Uncaught exception executing cloud request." ) - cnt += 1 - - def get_connection( self, uci_wrapper ): - """ - Establishes cloud connection using user's credentials associated with given UCI - """ - log.debug( 'Establishing %s cloud connection.' % self.type ) - provider = uci_wrapper.get_provider() - try: - region = RegionInfo( None, provider.region_name, provider.region_endpoint ) - except Exception, ex: - err = "Selecting region with cloud provider failed: " + str( ex ) - log.error( err ) - uci_wrapper.set_error( err, True ) - return None - try: - conn = EC2Connection( aws_access_key_id=uci_wrapper.get_access_key(), - aws_secret_access_key=uci_wrapper.get_secret_key(), - is_secure=provider.is_secure, - port=provider.port, - region=region, - path=provider.path ) - except boto.exception.EC2ResponseError, e: - err = "Establishing connection with cloud failed: " + str( e ) - log.error( err ) - uci_wrapper.set_error( err, True ) - return None - - return conn - - def check_key_pair( self, uci_wrapper, conn ): - """ - Check if a key pair associated with this UCI exists on cloud provider. - If yes, return key pair name; otherwise, generate a key pair with the cloud - provider and, again, return key pair name. - Key pair name for given UCI is generated from UCI's name and suffix '_kp' - """ - kp = None - kp_name = uci_wrapper.get_name().replace(' ','_') + "_kp" - log.debug( "Checking user's key pair: '%s'" % kp_name ) - try: - kp = conn.get_key_pair( kp_name ) - uci_kp_name = uci_wrapper.get_key_pair_name() - uci_material = uci_wrapper.get_key_pair_material() - if kp != None: - if kp.name != uci_kp_name or uci_material == None: - # key pair exists on the cloud but not in local database, so re-generate it (i.e., delete and then create) - try: - conn.delete_key_pair( kp_name ) - kp = self.create_key_pair( conn, kp_name ) - uci_wrapper.set_key_pair( kp.name, kp.material ) - except boto.exception.EC2ResponseError, e: - err = "EC2 response error while deleting key pair: " + str( e ) - log.error( err ) - uci_wrapper.set_error( err, True ) - else: - try: - kp = self.create_key_pair( conn, kp_name ) - uci_wrapper.set_key_pair( kp.name, kp.material ) - except boto.exception.EC2ResponseError, e: - err = "EC2 response error while creating key pair: " + str( e ) - log.error( err ) - uci_wrapper.set_error( err, True ) - except Exception, ex: - err = "Exception while creating key pair: " + str( ex ) - log.error( err ) - uci_wrapper.set_error( err, True ) - except boto.exception.EC2ResponseError, e: # No keypair under this name exists so create it - if e.code == 'InvalidKeyPair.NotFound': - log.info( "No keypair found, creating keypair '%s'" % kp_name ) - kp = self.create_key_pair( conn, kp_name ) - uci_wrapper.set_key_pair( kp.name, kp.material ) - else: - err = "EC2 response error while retrieving key pair: " + str( e ) - log.error( err ) - uci_wrapper.set_error( err, True ) - - if kp != None: - return kp.name - else: - return None - - def create_key_pair( self, conn, kp_name ): - """ Initiate creation of key pair under kp_name by current cloud provider. """ - try: - return conn.create_key_pair( kp_name ) - except boto.exception.EC2ResponseError, e: - return None - - def get_mi_id( self, uci_wrapper, i_index ): - """ - Get appropriate machine image (mi) ID based on instance type. - """ - i_type = uci_wrapper.get_instance_type( i_index ) - if i_type=='m1.small' or i_type=='c1.medium': - arch = 'i386' - else: - arch = 'x86_64' - - mi = self.sa_session.query( model.CloudImage ).filter_by( deleted=False, provider_type=self.type, architecture=arch ).first() - if mi: - return mi.image_id - else: - err = "Machine image could not be retrieved" - log.error( "%s for UCI '%s'." % (err, uci_wrapper.get_name() ) ) - uci_wrapper.set_error( err+". Contact site administrator to ensure needed machine image is registered.", True ) - return None - - def create_uci( self, uci_wrapper ): - """ - Create User Configured Instance (UCI) - i.e., create storage volume on cloud provider - and register relevant information in local Galaxy database. - """ - conn = self.get_connection( uci_wrapper ) - - # Because only 1 storage volume may be created at UCI config time, index of this storage volume in local Galaxy DB w.r.t - # current UCI is 0; therefore, it can be referenced in following code - log.info( "Creating volume in zone '%s'..." % uci_wrapper.get_uci_availability_zone() ) - if uci_wrapper.get_uci_availability_zone()=='': - log.info( "Availability zone for UCI (i.e., storage volume) was not selected, using default zone: %s" % self.zone ) - uci_wrapper.set_store_availability_zone( self.zone ) - -# log.debug( "Creating volume; using command: conn.create_volume( %s, '%s', snapshot=None )" % ( uci_wrapper.get_store_size( 0 ), uci_wrapper.get_uci_availability_zone() )) -# vol = conn.create_volume( uci_wrapper.get_store_size( 0 ), uci_wrapper.get_uci_availability_zone(), snapshot=None ) -# uci_wrapper.set_store_volume_id( 0, vol.id ) - store = uci_wrapper.get_all_stores_in_status( store_status.ADDING )[0] # Because at UCI creation time only 1 storage volume can be created, reference it directly - - log.info( "Creating storage volume in zone '%s' of size '%s'..." % ( uci_wrapper.get_uci_availability_zone(), store.size ) ) - # Because only 1 storage volume may be created at UCI config time, index of this storage volume in local Galaxy DB w.r.t - # current UCI is 0, so reference it in following methods - vol = conn.create_volume( store.size, uci_wrapper.get_uci_availability_zone(), snapshot=None ) - uci_wrapper.set_store_volume_id( store.id, vol.id ) - - # Retrieve created volume again to get updated status - try: - vl = conn.get_all_volumes( [vol.id] ) - except boto.exception.EC2ResponseError, e: - err = "EC2 response error while retrieving (i.e., updating status) of just created storage volume '" + vol.id + "': " + str( e ) - log.error( err ) - uci_wrapper.set_store_status( vol.id, uci_states.ERROR ) - uci_wrapper.set_error( err, True ) - return - except Exception, ex: - err = "Error while retrieving (i.e., updating status) of just created storage volume '" + vol.id + "': " + str( ex ) - log.error( err ) - uci_wrapper.set_error( err, True ) - return - - if len( vl ) > 0: - # EPC does not allow creation of storage volumes (it deletes one as soon as it is created, so manually set uci_state here) - if vl[0].status == store_status.DELETED: - uci_wrapper.change_state( uci_state=uci_states.AVAILABLE ) - else: - uci_wrapper.change_state( uci_state=vl[0].status ) - uci_wrapper.set_store_status( vol.id, vl[0].status ) - else: - err = "Volume '" + vol.id +"' not found by EC2 after being created." - log.error( err ) - uci_wrapper.set_store_status( vol.id, uci_states.ERROR ) - uci_wrapper.set_error( err, True ) - - def delete_uci( self, uci_wrapper ): - """ - Delete UCI - i.e., delete all storage volumes associated with this UCI. - NOTE that this implies deletion of any and all data associated - with this UCI from the cloud. All data will be deleted. - Information in local Galaxy database is marked as deleted but not actually removed - from the database. - """ - conn = self.get_connection( uci_wrapper ) - vl = [] # volume list - count = 0 # counter for checking if all volumes assoc. w/ UCI were deleted - - # Get all volumes assoc. w/ UCI, delete them from cloud as well as in local DB - vl = uci_wrapper.get_all_stores() - deletedList = [] - failedList = [] - for v in vl: - log.debug( "Deleting volume with id='%s'" % v.volume_id ) - try: - if conn.delete_volume( v.volume_id ): - deletedList.append( v.volume_id ) - v.deleted = True - self.sa_session.add( v ) - self.sa_session.flush() - count += 1 - else: - failedList.append( v.volume_id ) - except boto.exception.EC2ResponseError, e: - err = "EC2 response error while deleting storage volume '" + v.volume_id + "': " + str( e ) - log.error( err ) - uci_wrapper.set_store_error( err, store_id = v.volume_id ) - uci_wrapper.set_error( err, True ) - - # Delete UCI if all of associated - if count == len( vl ): - uci_wrapper.set_deleted() - else: - err = "Deleting following volume(s) failed: "+ str( failedList )+". However, these volumes were successfully deleted: " \ - + str( deletedList ) +". MANUAL intervention and processing needed." - log.error( err ) - uci_wrapper.set_error( err, True ) - - def snapshot_uci( self, uci_wrapper ): - """ - Initiate creation of a snapshot by cloud provider for all storage volumes - associated with this UCI. - """ - if uci_wrapper.get_uci_state() != uci_states.ERROR: - conn = self.get_connection( uci_wrapper ) - - snapshots = uci_wrapper.get_snapshots( status = snapshot_status.SUBMITTED ) - for snapshot in snapshots: - log.debug( "Snapshot DB id: '%s', volume id: '%s'" % ( snapshot.id, snapshot.store.volume_id ) ) - try: - snap = conn.create_snapshot( volume_id=snapshot.store.volume_id ) - snap_id = str( snap ).split(':')[1] - uci_wrapper.set_snapshot_id( snapshot.id, snap_id ) - sh = conn.get_all_snapshots( snap_id ) # get updated status - uci_wrapper.set_snapshot_status( status=sh[0].status, snap_id=snap_id ) - except boto.exception.EC2ResponseError, e: - err = "Cloud provider response error while creating snapshot: " + str( e ) - log.error( err ) - uci_wrapper.set_snapshot_error( error=err, snap_index=snapshot.id, set_status=True ) - uci_wrapper.set_error( err, True ) - return - except Exception, ex: - err = "Error while creating snapshot: " + str( ex ) - log.error( err ) - uci_wrapper.set_snapshot_error( error=err, snap_index=snapshot.id, set_status=True ) - uci_wrapper.set_error( err, True ) - return - - uci_wrapper.change_state( uci_state=uci_states.AVAILABLE ) - -# if uci_wrapper.get_uci_state() != uci_states.ERROR: -# -# snapshots = uci_wrapper.get_snapshots( status = 'submitted' ) -# for snapshot in snapshots: -# uci_wrapper.set_snapshot_id( snapshot.id, None, 'euca_error' ) -# -# log.debug( "Eucalyptus snapshot attempted by user for UCI '%s'" % uci_wrapper.get_name() ) -# uci_wrapper.set_error( "Eucalyptus does not support creation of snapshots at this moment. No snapshot or other changes were performed. \ -# Feel free to resent state of this instance and use it normally.", True ) - - - def add_storage_to_uci( self, uci_wrapper ): - """ Adds more storage to specified UCI """ - uci_wrapper.set_error( "Adding storage to eucalyptus-based clouds is not yet supported.", True ) - - def dummy_start_uci( self, uci_wrapper ): - - uci = uci_wrapper.get_uci() - log.debug( "Would be starting instance '%s'" % uci.name ) -# uci_wrapper.change_state( uci_states.SUBMITTED_UCI ) -# log.debug( "Set UCI state to SUBMITTED_UCI" ) - log.debug( "Sleeping a bit... (%s)" % uci.name ) - time.sleep(10) - log.debug( "Woke up! (%s)" % uci.name ) - - def start_uci( self, uci_wrapper ): - """ - Start instance(s) of given UCI on the cloud. - """ - if uci_wrapper.get_uci_state() != uci_states.ERROR: - conn = self.get_connection( uci_wrapper ) - self.check_key_pair( uci_wrapper, conn ) - if uci_wrapper.get_key_pair_name() == None: - err = "Key pair not found" - log.error( "%s for UCI '%s'." % ( err, uci_wrapper.get_name() ) ) - uci_wrapper.set_error( err + ". Try resetting the state and starting the instance again.", True ) - return - - i_indexes = uci_wrapper.get_instances_indexes( state=instance_states.SUBMITTED ) # Get indexes of i_indexes associated with this UCI that are in 'submitted' state - log.debug( "Starting instances with IDs: '%s' associated with UCI '%s' " % ( i_indexes, uci_wrapper.get_name(), ) ) - if len( i_indexes ) > 0: - for i_index in i_indexes: - # Get machine image for current instance - mi_id = self.get_mi_id( uci_wrapper, i_index ) - log.debug( "mi_id: %s, uci_wrapper.get_key_pair_name(): %s" % ( mi_id, uci_wrapper.get_key_pair_name() ) ) - uci_wrapper.set_mi( i_index, mi_id ) - - if uci_wrapper.get_uci_state() != uci_states.ERROR: - # Start an instance - log.debug( "Starting UCI instance '%s'" % uci_wrapper.get_name() ) - log.debug( "Using following command: conn.run_instances( image_id='%s', key_name='%s', instance_type='%s' )" - % ( mi_id, uci_wrapper.get_key_pair_name(), uci_wrapper.get_instance_type( i_index ) ) ) - reservation = None - try: - reservation = conn.run_instances( image_id=mi_id, - key_name=uci_wrapper.get_key_pair_name(), - instance_type=uci_wrapper.get_instance_type( i_index ) ) - except boto.exception.EC2ResponseError, e: - err = "EC2 response error when starting UCI '"+ uci_wrapper.get_name() +"': " + str( e ) - log.error( err ) - uci_wrapper.set_error( err, True ) - except Exception, ex: - err = "Error when starting UCI '" + uci_wrapper.get_name() + "': " + str( ex ) - log.error( err ) - uci_wrapper.set_error( err, True ) - # Record newly available instance data into local Galaxy database - if reservation: - l_time = datetime.utcnow() -# uci_wrapper.set_instance_launch_time( self.format_time( reservation.instances[0].launch_time ), i_index=i_index ) - uci_wrapper.set_instance_launch_time( l_time, i_index=i_index ) - if not uci_wrapper.uci_launch_time_set(): - uci_wrapper.set_uci_launch_time( l_time ) - try: - uci_wrapper.set_reservation_id( i_index, str( reservation ).split(":")[1] ) - # TODO: if more than a single instance will be started through single reservation, change this reference from element [0] - i_id = str( reservation.instances[0]).split(":")[1] - uci_wrapper.set_instance_id( i_index, i_id ) - s = reservation.instances[0].state - uci_wrapper.change_state( s, i_id, s ) - vol_id = uci_wrapper.get_store_volume_id( store_id=0 ) # TODO: Once more that one vol/UCI is allowed, update this! - uci_wrapper.set_store_status( vol_id, store_status.WAITING ) - log.debug( "Instance of UCI '%s' started, current state: '%s'" % ( uci_wrapper.get_name(), uci_wrapper.get_uci_state() ) ) - except boto.exception.EC2ResponseError, e: - err = "EC2 response error when retrieving instance information for UCI '" + uci_wrapper.get_name() + "': " + str( e ) - log.error( err ) - uci_wrapper.set_error( err, True ) - else: - log.error( "UCI '%s' is in 'error' state, starting instance was aborted." % uci_wrapper.get_name() ) - else: - err = "No instances in state '"+ instance_states.SUBMITTED +"' found for UCI '" + uci_wrapper.get_name() + \ - "'. Nothing to start." - log.error( err ) - uci_wrapper.set_error( err, True ) - else: - log.error( "UCI '%s' is in 'error' state, starting instance was aborted." % uci_wrapper.get_name() ) - - def stop_uci( self, uci_wrapper): - """ - Stop all cloud instances associated with given UCI. - """ - conn = self.get_connection( uci_wrapper ) - - # Get all instances associated with given UCI - il = uci_wrapper.get_instances_ids() # instance list - # Process list of instances and remove any references to empty instance id's - for i in il: - if i is None: - il.remove( i ) - log.debug( 'List of instances being terminated: %s' % il ) - rl = conn.get_all_instances( il ) # Reservation list associated with given instances - - # Initiate shutdown of all instances under given UCI - cnt = 0 - stopped = [] - not_stopped = [] - for r in rl: - for inst in r.instances: - log.debug( "Sending stop signal to instance '%s' associated with reservation '%s' (UCI: %s)." % ( inst, r, uci_wrapper.get_name() ) ) - try: - inst.stop() - uci_wrapper.set_stop_time( datetime.utcnow(), i_id=inst.id ) - uci_wrapper.change_state( instance_id=inst.id, i_state=inst.update() ) - stopped.append( inst ) - except boto.exception.EC2ResponseError, e: - not_stopped.append( inst ) - err = "EC2 response error when stopping instance '" + inst.instance_id + "': " + str( e ) - log.error( err ) - uci_wrapper.set_error( err, True ) - - uci_wrapper.reset_uci_launch_time() - log.debug( "Termination was initiated for all instances of UCI '%s'." % uci_wrapper.get_name() ) - -# dbInstances = get_instances( trans, uci ) #TODO: handle list! -# -# # Get actual cloud instance object -# cloudInstance = get_cloud_instance( conn, dbInstances.instance_id ) -# -# # TODO: Detach persistent storage volume(s) from instance and update volume data in local database -# stores = get_stores( trans, uci ) -# for i, store in enumerate( stores ): -# log.debug( "Detaching volume '%s' to instance '%s'." % ( store.volume_id, dbInstances.instance_id ) ) -# mntDevice = store.device -# volStat = None -## Detaching volume does not work with Eucalyptus Public Cloud, so comment it out -## try: -## volStat = conn.detach_volume( store.volume_id, dbInstances.instance_id, mntDevice ) -## except: -## log.debug ( 'Error detaching volume; still going to try and stop instance %s.' % dbInstances.instance_id ) -# store.attach_time = None -# store.device = None -# store.inst.instance_id = None -# store.status = volStat -# log.debug ( '***** volume status: %s' % volStat ) -# -# # Stop the instance and update status in local database -# cloudInstance.stop() -# dbInstances.stop_time = datetime.utcnow() -# while cloudInstance.state != 'terminated': -# log.debug( "Stopping instance %s state; current state: %s" % ( str( cloudInstance ).split(":")[1], cloudInstance.state ) ) -# time.sleep(3) -# cloudInstance.update() -# dbInstances.state = cloudInstance.state -# -# # Reset relevant UCI fields -# uci.state = 'available' -# uci.launch_time = None -# -# # Persist -# session = trans.sa_session -## session.save_or_update( stores ) -# session.save_or_update( dbInstances ) # TODO: Is this going to work w/ multiple instances stored in dbInstances variable? -# session.save_or_update( uci ) -# session.flush() -# trans.log_event( "User stopped cloud instance '%s'" % uci.name ) -# trans.set_message( "Galaxy instance '%s' stopped." % uci.name ) - - def update( self ): - """ - Run status update on all instances that are in 'running', 'pending', or 'shutting-down' state. - Run status update on all storage volumes whose status is 'in-use', 'creating', or 'None'. - Run status update on all snapshots whose status is 'pending' or 'delete' - Run status update on any zombie UCIs, i.e., UCI's that is in 'submitted' state for an - extended period of time. - - Reason behind this method is to sync state of local DB and real-world resources - """ - log.debug( "Running general status update for %s UCIs..." % self.type ) - # Update instances - instances = self.sa_session.query( model.CloudInstance ) \ - .filter( or_( model.CloudInstance.table.c.state==instance_states.RUNNING, - model.CloudInstance.table.c.state==instance_states.PENDING, - model.CloudInstance.table.c.state==instance_states.SHUTTING_DOWN ) ) \ - .all() - for inst in instances: - if self.type == inst.uci.credentials.provider.type: - log.debug( "[%s] Running general status update on instance '%s'" % ( inst.uci.credentials.provider.type, inst.instance_id ) ) - self.update_instance( inst ) - - # Update storage volume(s) - stores = self.sa_session.query( model.CloudStore ) \ - .filter( or_( model.CloudStore.table.c.status==store_status.IN_USE, - model.CloudStore.table.c.status==store_status.CREATING, - model.CloudStore.table.c.status==store_status.WAITING, - model.CloudStore.table.c.status==None ) ) \ - .all() - for store in stores: - if self.type == store.uci.credentials.provider.type: # and store.volume_id != None: - log.debug( "[%s] Running general status update on store with local database ID: '%s'" % ( store.uci.credentials.provider.type, store.id ) ) - self.update_store( store ) - - # Update pending snapshots or delete ones marked for deletion - snapshots = self.sa_session.query( model.CloudSnapshot ) \ - .filter( or_( model.CloudSnapshot.table.c.status == snapshot_status.PENDING, model.CloudSnapshot.table.c.status == snapshot_status.DELETE ) ) \ - .all() - for snapshot in snapshots: - if self.type == snapshot.uci.credentials.provider.type and snapshot.status == snapshot_status.PENDING: - log.debug( "[%s] Running general status update on snapshot '%s'" % ( snapshot.uci.credentials.provider.type, snapshot.snapshot_id ) ) - self.update_snapshot( snapshot ) - elif self.type == snapshot.uci.credentials.provider.type and snapshot.status == snapshot_status.DELETE: - log.debug( "[%s] Initiating deletion of snapshot '%s'" % ( snapshot.uci.credentials.provider.type, snapshot.snapshot_id ) ) - self.delete_snapshot( snapshot ) - - # Attempt at updating any zombie UCIs (i.e., instances that have been in SUBMITTED state for longer than expected - see below for exact time) - zombies = self.sa_session.query( model.UCI ).filter_by( state=uci_states.SUBMITTED ).all() - for zombie in zombies: - log.debug( "zombie UCI: %s" % zombie.name ) - z_instances = self.sa_session.query( model.CloudInstance ) \ - .filter( or_( model.CloudInstance.table.c.state != instance_states.TERMINATED, - model.CloudInstance.table.c.state == None ) ) \ - .all() - for z_inst in z_instances: - if self.type == z_inst.uci.credentials.provider.type: -# log.debug( "z_inst.id: '%s', state: '%s'" % ( z_inst.id, z_inst.state ) ) - td = datetime.utcnow() - z_inst.update_time -# log.debug( "z_inst.id: %s, time delta is %s sec" % ( z_inst.id, td.seconds ) ) - if td.seconds > 180: # if instance has been in SUBMITTED state for more than 3 minutes - log.debug( "[%s](td=%s) Running zombie repair update on instance with DB id '%s'" % ( z_inst.uci.credentials.provider.type, td.seconds, z_inst.id ) ) - self.process_zombie( z_inst ) - - def update_instance( self, inst ): - """ - Update information in local database for given instance as it is obtained from cloud provider. - Along with updating information about given instance, information about the UCI controlling - this instance is also updated. - """ - # Get credentials associated wit this instance - uci_id = inst.uci_id - uci = self.sa_session.query( model.UCI ).get( uci_id ) - self.sa_session.refresh( uci ) - conn = self.get_connection_from_uci( uci ) - - # Get reservations handle for given instance - try: - rl= conn.get_all_instances( [inst.instance_id] ) - except boto.exception.EC2ResponseError, e: - err = "Retrieving instance(s) from cloud failed for UCI '"+ uci.name +"' during general status update: " + str( e ) - log.error( err ) - uci.error = err - uci.state = uci_states.ERROR - self.sa_session.add( uci ) - self.sa_session.flush() - return None - - # Because references to reservations are deleted shortly after instances have been terminated, getting an empty list as a response to a query - # typically means the instance has successfully shut down but the check was not performed in short enough amount of time. Until an alternative solution - # is found, below code sets state of given UCI to 'error' to indicate to the user something out of ordinary happened. - if len( rl ) == 0: - err = "Instance ID '"+inst.instance_id+"' was not found by the cloud provider. Instance might have crashed or otherwise been terminated."+ \ - "Manual check is recommended." - log.error( err ) - inst.error = err - uci.error = err - inst.state = instance_states.TERMINATED - uci.state = uci_states.ERROR - uci.launch_time = None - self.sa_session.add( inst ) - self.sa_session.add( uci ) - self.sa_session.flush() - # Update instance status in local DB with info from cloud provider - for r in rl: - for i, cInst in enumerate( r.instances ): - try: - s = cInst.update() - log.debug( "Checking state of cloud instance '%s' associated with UCI '%s' and reservation '%s'. State='%s'" % ( cInst, uci.name, r, s ) ) - if s != inst.state: - inst.state = s - self.sa_session.add( inst ) - self.sa_session.flush() - # After instance has shut down, ensure UCI is marked as 'available' - if s == instance_states.TERMINATED and uci.state != uci_states.ERROR: - uci.state = uci_states.AVAILABLE - uci.launch_time = None - self.sa_session.add( uci ) - self.sa_session.flush() - # Making sure state of UCI is updated. Once multiple instances become associated with single UCI, this will need to be changed. - if s != uci.state and s != instance_states.TERMINATED: - uci.state = s - self.sa_session.add( uci ) - self.sa_session.flush() - if cInst.public_dns_name != inst.public_dns: - inst.public_dns = cInst.public_dns_name - self.sa_session.add( inst ) - self.sa_session.flush() - if cInst.private_dns_name != inst.private_dns: - inst.private_dns = cInst.private_dns_name - self.sa_session.add( inst ) - self.sa_session.flush() - except boto.exception.EC2ResponseError, e: - err = "Updating instance status from cloud failed for UCI '"+ uci.name + "' during general status update: " + str( e ) - log.error( err ) - uci.error = err - uci.state = uci_states.ERROR - self.sa_session.add( uci ) - self.sa_session.flush() - return None - - def update_store( self, store ): - """ - Update information in local database for given storage volume as it is obtained from cloud provider. - Along with updating information about given storage volume, information about the UCI controlling - this storage volume is also updated. - """ - # Get credentials associated wit this store - uci_id = store.uci_id - uci = self.sa_session.query( model.UCI ).get( uci_id ) - self.sa_session.refresh( uci ) - conn = self.get_connection_from_uci( uci ) - - if store.volume_id != None: - # Get reservations handle for given store - try: - log.debug( "Updating storage volume command: vl = conn.get_all_volumes( [%s] )" % store.volume_id ) - vl = conn.get_all_volumes( [store.volume_id] ) - except boto.exception.EC2ResponseError, e: - err = "Retrieving volume(s) from cloud failed for UCI '"+ uci.name + "' during general status update: " + str( e ) - log.error( err ) - uci.error = err - uci.state = uci_states.ERROR - self.sa_session.add( uci ) - self.sa_session.flush() - return None - - # Update store status in local DB with info from cloud provider - if len(vl) > 0: - try: - log.debug( "Storage volume '%s' current status: '%s'" % (store.volume_id, vl[0].status ) ) - if store.status != vl[0].status: - # In case something failed during creation of UCI but actual storage volume was created and yet - # UCI state remained as 'new', try to remedy this by updating UCI state here - if ( store.status == None ) and ( store.volume_id != None ): - uci.state = vl[0].status - self.sa_session.add( uci ) - self.sa_session.flush() - # If UCI was marked in state 'CREATING', update its status to reflect new status - elif ( uci.state == uci_states.CREATING ): - # Because Eucalyptus Public Cloud (EPC) deletes volumes immediately after they are created, artificially - # set status of given UCI to 'available' based on storage volume's availability zone (i.e., it's residing - # in EPC as opposed to some other Eucalyptus based cloud that allows creation of storage volumes. - if store.availability_zone == 'epc': - uci.state = uci_states.AVAILABLE - else: - uci.state = vl[0].status - - self.sa_session.add( uci ) - self.sa_session.flush() - - store.status = vl[0].status - self.sa_session.add( store ) - self.sa_session.flush() - if store.inst != None: - if store.inst.instance_id != vl[0].instance_id: - store.inst.instance_id = vl[0].instance_id - self.sa_session.add( store ) - self.sa_session.flush() - if store.attach_time != vl[0].attach_time: - store.attach_time = vl[0].attach_time - self.sa_session.add( store ) - self.sa_session.flush() - if store.device != vl[0].device: - store.device = vl[0].device - self.sa_session.add( store ) - self.sa_session.flush() - except boto.exception.EC2ResponseError, e: - err = "Updating status of volume(s) from cloud failed for UCI '"+ uci.name + "' during general status update: " + str( e ) - log.error( err ) - uci.error = err - uci.state = uci_states.ERROR - self.sa_session.add( uci ) - self.sa_session.flush() - return None - else: - err = "No storage volumes returned by cloud provider on general update" - log.error( "%s for UCI '%s'" % ( err, uci.name ) ) - store.status = store_status.ERROR - store.error = err - uci.error = err - uci.state = uci_states.ERROR - self.sa_session.add( uci ) - self.sa_session.add( store ) - self.sa_session.flush() - else: - err = "Missing storage volume ID in local database on general update. Manual check is needed to check " \ - "if storage volume was actually created by cloud provider." - log.error( "%s (for UCI '%s')" % ( err, uci.name ) ) - store.status = store_status.ERROR - store.error = err - uci.error = err - uci.state = uci_states.ERROR - self.sa_session.add( uci ) - self.sa_session.add( store ) - self.sa_session.flush() - - def update_snapshot( self, snapshot ): - """ - Update information in local database for given snapshot as it is obtained from cloud provider. - Along with updating information about given snapshot, information about the UCI controlling - this snapshot is also updated. - """ - # Get credentials associated wit this store - uci_id = snapshot.uci_id - uci = self.sa_session.query( model.UCI ).get( uci_id ) - self.sa_session.refresh( uci ) - conn = self.get_connection_from_uci( uci ) - - try: - log.debug( "Updating status of snapshot '%s'" % snapshot.snapshot_id ) - snap = conn.get_all_snapshots( [snapshot.snapshot_id] ) - if len( snap ) > 0: - log.debug( "Snapshot '%s' status: %s" % ( snapshot.snapshot_id, snap[0].status ) ) - snapshot.status = snap[0].status - self.sa_session.add( snapshot ) - self.sa_session.flush() - else: - err = "No snapshots returned by EC2 on general update" - log.error( "%s for UCI '%s'" % ( err, uci.name ) ) - snapshot.status = snapshot_status.ERROR - snapshot.error = err - uci.error = err - uci.state = uci_states.ERROR - self.sa_session.add( uci ) - self.sa_session.add( snapshot ) - self.sa_session.flush() - except boto.exception.EC2ResponseError, e: - err = "EC2 response error while updating snapshot status: " + str( e ) - log.error( err ) - snapshot.status = snapshot_status.ERROR - snapshot.error = err - uci.error = err - uci.state = uci_states.ERROR - self.sa_session.add( uci ) - self.sa_session.add( snapshot ) - self.sa_session.flush() - except Exception, ex: - err = "Error while updating snapshot status: " + str( ex ) - log.error( err ) - snapshot.status = snapshot_status.ERROR - snapshot.error = err - uci.error = err - uci.state = uci_states.ERROR - self.sa_session.add( uci ) - self.sa_session.add( snapshot ) - self.sa_session.flush() - - def delete_snapshot( self, snapshot ): - """ - Initiate deletion of given snapshot from cloud provider. - """ - if snapshot.status == snapshot_status.DELETE: - # Get credentials associated wit this store - uci_id = snapshot.uci_id - uci = self.sa_session.query( model.UCI ).get( uci_id ) - self.sa_session.refresh( uci ) - conn = self.get_connection_from_uci( uci ) - - try: - log.debug( "Deleting snapshot '%s'" % snapshot.snapshot_id ) - snap = conn.delete_snapshot( snapshot.snapshot_id ) - if snap == True: - snapshot.deleted = True - snapshot.status = snapshot_status.DELETED - self.sa_session.add( snapshot ) - self.sa_session.flush() - return snap - except boto.exception.EC2ResponseError, e: - err = "EC2 response error while deleting snapshot: " + str( e ) - log.error( err ) - snapshot.status = snapshot_status.ERROR - snapshot.error = err - uci.error = err - uci.state = uci_states.ERROR - self.sa_session.add( uci ) - self.sa_session.add( snapshot ) - self.sa_session.flush() - except Exception, ex: - err = "Error while deleting snapshot: " + str( ex ) - log.error( err ) - snapshot.status = snapshot_status.ERROR - snapshot.error = err - uci.error = err - uci.state = uci_states.ERROR - self.sa_session.add( uci ) - self.sa_session.add( snapshot ) - self.sa_session.flush() - else: - err = "Cannot delete snapshot '"+snapshot.snapshot_id+"' because its status is '"+snapshot.status+"'. Only snapshots with '" + \ - snapshot_status.COMPLETED+"' status can be deleted." - log.error( err ) - snapshot.error = err - self.sa_session.add( snapshot ) - self.sa_session.flush() - - def process_zombie( self, inst ): - """ - Attempt at discovering if starting a cloud instance was successful but local database was not updated - accordingly or if something else failed and instance was never started. Currently, no automatic - repairs are being attempted; instead, appropriate error messages are set. - """ - uci_id = inst.uci_id - uci = self.sa_session.query( model.UCI ).get( uci_id ) - self.sa_session.refresh( uci ) - - # Check if any instance-specific information was written to local DB; if 'yes', set instance and UCI's error message - # suggesting manual check. - if inst.launch_time != None or inst.reservation_id != None or inst.instance_id != None: - # Try to recover state - this is best-case effort, so if something does not work immediately, not - # recovery steps are attempted. Recovery is based on hope that instance_id is available in local DB; if not, - # report as error. - # Fields attempting to be recovered are: reservation_id, instance status, and launch_time - if inst.instance_id != None: - conn = self.get_connection_from_uci( uci ) - rl = conn.get_all_instances( [inst.instance_id] ) # reservation list - # Update local DB with relevant data from instance - if inst.reservation_id == None: - try: - inst.reservation_id = str(rl[0]).split(":")[1] - except: # something failed, so skip - pass - - try: - state = rl[0].instances[0].update() - inst.state = state - uci.state = state - self.sa_session.add( inst ) - self.sa_session.add( uci ) - self.sa_session.flush() - except: # something failed, so skip - pass - - if inst.launch_time == None: - try: - launch_time = self.format_time( rl[0].instances[0].launch_time ) - inst.launch_time = launch_time - self.sa_session.add( inst ) - self.sa_session.flush() - if inst.uci.launch_time == None: - uci.launch_time = launch_time - self.sa_session.add( uci ) - self.sa_session.flush() - except: # something failed, so skip - pass - else: - err = "Starting a machine instance (DB id: '"+str(inst.id)+"') associated with this UCI '" + str(inst.uci.name) + \ - "' seems to have failed. Because it appears that cloud instance might have gotten started, manual check is recommended." - inst.error = err - inst.state = instance_states.ERROR - inst.uci.error = err - inst.uci.state = uci_states.ERROR - log.error( err ) - self.sa_session.add( inst ) - self.sa_session.add( uci ) - self.sa_session.flush() - - else: #Instance most likely never got processed, so set error message suggesting user to try starting instance again. - err = "Starting a machine instance (DB id: '"+str(inst.id)+"') associated with this UCI '" + str(inst.uci.name) + \ - "' seems to have failed. Because it appears that cloud instance never got started, it should be safe to reset state and try " \ - "starting the instance again." - inst.error = err - inst.state = instance_states.ERROR - uci.error = err - uci.state = uci_states.ERROR - log.error( err ) - self.sa_session.add( inst ) - self.sa_session.add( uci ) - self.sa_session.flush() -# uw = UCIwrapper( inst.uci ) -# log.debug( "Try automatically re-submitting UCI '%s'." % uw.get_name() ) - - def get_connection_from_uci( self, uci ): - """ - Establish and return connection to cloud provider. Information needed to do so is obtained - directly from uci database object. - """ - log.debug( 'Establishing %s cloud connection' % self.type ) - a_key = uci.credentials.access_key - s_key = uci.credentials.secret_key - # Get connection - try: - region = RegionInfo( None, uci.credentials.provider.region_name, uci.credentials.provider.region_endpoint ) -# log.debug( "[%s] Using following command to connect to cloud provider: " -# "conn = EC2Connection( aws_access_key_id=%s, " -# "aws_secret_access_key=%s, " -# "port=%s, " -# "is_secure=%s, " -# "region=region, " -# "path=%s )" % ( self.type, a_key, s_key, uci.credentials.provider.is_secure, uci.credentials.provider.port, uci.credentials.provider.path ) ) - conn = EC2Connection( aws_access_key_id=a_key, - aws_secret_access_key=s_key, - is_secure=uci.credentials.provider.is_secure, - port=uci.credentials.provider.port, - region=region, - path=uci.credentials.provider.path ) - except boto.exception.EC2ResponseError, e: - err = "Establishing connection with cloud failed: " + str( e ) - log.error( err ) - uci.error = err - uci.state = uci_states.ERROR - self.sa_session.add( uci ) - self.sa_session.flush() - return None - - return conn - -# def updateUCI( self, uci ): -# """ -# Runs a global status update on all storage volumes and all instances that are -# associated with specified UCI -# """ -# conn = self.get_connection( uci ) -# -# # Update status of storage volumes -# vl = model.CloudStore.filter( model.CloudInstance.table.c.uci_id == uci.id ).all() -# vols = [] -# for v in vl: -# vols.append( v.volume_id ) -# try: -# volumes = conn.get_all_volumes( vols ) -# for i, v in enumerate( volumes ): -# uci.store[i].inst.instance_id = v.instance_id -# uci.store[i].status = v.status -# uci.store[i].device = v.device -# uci.store[i].flush() -# except: -# log.debug( "Error updating status of volume(s) associated with UCI '%s'. Status was not updated." % uci.name ) -# pass -# -# # Update status of instances -# il = model.CloudInstance.filter_by( uci_id=uci.id ).filter( model.CloudInstance.table.c.state != 'terminated' ).all() -# instanceList = [] -# for i in il: -# instanceList.append( i.instance_id ) -# log.debug( 'instanceList: %s' % instanceList ) -# try: -# reservations = conn.get_all_instances( instanceList ) -# for i, r in enumerate( reservations ): -# uci.instance[i].state = r.instances[0].update() -# log.debug('updating instance %s; status: %s' % ( uci.instance[i].instance_id, uci.instance[i].state ) ) -# uci.state = uci.instance[i].state -# uci.instance[i].public_dns = r.instances[0].dns_name -# uci.instance[i].private_dns = r.instances[0].private_dns_name -# uci.instance[i].flush() -# uci.flush() -# except: -# log.debug( "Error updating status of instances associated with UCI '%s'. Instance status was not updated." % uci.name ) -# pass - - # --------- Helper methods ------------ - - def format_time( self, time ): - dict = {'T':' ', 'Z':''} - for i, j in dict.iteritems(): - time = time.replace(i, j) - return time - --- a/templates/cloud/add_credentials.mako +++ /dev/null @@ -1,110 +0,0 @@ -<% _=n_ %> -<%inherit file="/base.mako"/> -<%def name="title()">Add credentials</%def> - -<%def name="javascripts()"> -${parent.javascripts()} -<script type="text/javascript"> -$(function(){ - $("input:text:first").focus(); -}) -</script> -</%def> - -%if header: - ${header} -%endif - -%if providers: - -<div class="form"> - <div class="form-title">Add credentials</div> - <div class="form-body"> - <form name="add_credentials" action="${h.url_for( action='add_credentials' )}" method="post" > - - <% - cls = "form-row" - if error.has_key('cred_error'): - cls += " form-row-error" - %> - <div class="${cls}"> - <label>Credentials name:</label> - <div class="form-row-input"> - <input type="text" name="credName" value="${credName}" size="40"> - </div> - %if error.has_key('cred_error'): - <div class="form-row-error-message">${error['cred_error']}</div> - %endif - <div style="clear: both"></div> - </div> - - - <% - cls = "form-row" - if error.has_key('provider_error'): - cls += " form-row-error" - %> - <div class="${cls}"> - <label>Cloud provider name:</label> - <div class="form-row-input"> - <select name="providerName" style="width:40em"> - <option value="">Select Provider...</option> - %for provider in providers: - <option value="${provider.name}">${provider.name}</option> - %endfor - </select> - <br/>or <a href="${h.url_for( action='add_provider' )}"> - <span>register additional cloud provider</span></a> - </div> - %if error.has_key('provider_error'): - <div class="form-row-error-message">${error['provider_error']}</div> - %endif - <div style="clear: both"></div> - </div> - - <% - cls = "form-row" - if error.has_key('access_key_error'): - cls += " form-row-error" - %> - <div class="${cls}"> - <label>Access key:</label> - <div class="form-row-input"> - <input type="text" name="accessKey" value="${accessKey}" size="40"> - </div> - %if error.has_key('access_key_error'): - <div class="form-row-error-message">${error['access_key_error']}</div> - %endif - <div style="clear: both"></div> - </div> - - - <% - cls = "form-row" - if error.has_key('secret_key_error'): - cls += " form-row-error" - %> - <div class="${cls}"> - <label>Secret key:</label> - <div class="form-row-input"> - <input type="password" name="secretKey" value="${secretKey}" size="40"> - </div> - %if error.has_key('secret_key_error'): - <div class="form-row-error-message">${error['secret_key_error']}</div> - %endif - <div style="clear: both"></div> - </div> - - <div class="form-row"><input type="submit" value="Add"></div> - </form> - </div> -</div> - -%else: - In order to add credentials, desired cloud provider needs to be registered first.<p/> - Register <a href="${h.url_for( action='add_ec2' )}"> - <span>Amazon EC2 (us-east-1 region) automatically</span></a> - or add - <a href="${h.url_for( action='add_provider' )}"> - <span>custom cloud provider</span></a>. -%endif --- a/templates/cloud/list_images.mako +++ /dev/null @@ -1,91 +0,0 @@ -<%inherit file="/base.mako"/> - -<%def name="title()">Cloud home</%def> - -%if message: -<% - try: - messagetype - except: - messagetype = "done" -%> - -<p /> -<div class="${messagetype}message"> - ${message} -</div> -%endif - -<h2>List of registered machine images:</h2> -<ul class="manage-table-actions"> - <li> - <a class="action-button" href="${h.url_for( controller='cloud', action='add_new_image' )}" target="galaxy_main"> - <img src="${h.url_for('/static/images/silk/add.png')}" /> - <span>Add machine image</span> - </a> - </li> -</ul> - -%if images: - <table class="mange-table colored" border="0" cellspacing="0" cellpadding="0" width="100%"> - <colgroup width="2%"></colgroup> - <colgroup width="10%"></colgroup> - <colgroup width="13%"></colgroup> - <colgroup width="55%"></colgroup> - <colgroup width="10%"></colgroup> - <colgroup width="5%"></colgroup> - <colgroup width="5%"></colgroup> - <tr class="header"> - <th>#</th> - <th>Provider type</th> - <th>Machime image ID</th> - <th>Manifest</th> - <th>Architecture</th> - <th>Edit</th> - <th>Delete</th> - <th></th> - </tr> - %for i, image in enumerate( images ): - <tr> - <td>${i+1}</td> - <td> - %if image.provider_type: - ${image.provider_type} - %else: - N/A - %endif - </td> - <td> - %if image.image_id: - ${image.image_id} - %else: - N/A - %endif - </td> - <td> - %if image.manifest: - ${image.manifest} - %else: - N/A - %endif - </td> - <td> - %if image.architecture: - ${image.architecture} - %else: - N/A - %endif - </td> - <td> - <a href="${h.url_for( controller='cloud', action='edit_image', image_id=image.image_id, manifest=image.manifest, id=trans.security.encode_id(image.id) )}">e</a> - </td> - <td> - <a confirm="Are you sure you want to delete machine image '${image.image_id}'? Note that this may result in users' UCI's not to work any more!" - href="${h.url_for( controller='cloud', action='delete_image', id=trans.security.encode_id(image.id) )}">x</a> - </td> - </tr> - %endfor - </table> -%else: - <h3>There are no registered machine images.</h3><br /> -%endif --- a/templates/cloud/configure_uci.mako +++ /dev/null @@ -1,116 +0,0 @@ -<% _=n_ %> -<%inherit file="/base.mako"/> -<%def name="title()">Configure new UCI</%def> - -<%def name="javascripts()"> -${parent.javascripts()} -<script type="text/javascript"> - -var providers_zones = ${h.to_json_string(providersToZones)}; - -$(function(){ - $("input:text:first").focus(); - - $("#credName").change(function() { - var zones = providers_zones[ $(this).val() ]; - var zones_select = $("#zones"); - - zones_select.children().remove(); - - for (var i in zones) { - var zone = zones[i]; - var new_option = $('<option value="' + zone + '">' + zone + '</option>'); - new_option.appendTo(zones_select); - } - - }); -}) -</script> -</%def> - -%if header: - ${header} -%endif - -<div class="form"> - <div class="form-title">Configure new Galaxy instance</div> - <div class="form-body"> - <form name="Configure new UCI" action="${h.url_for( action='configure_new_uci' )}" method="post" > - - <% - cls = "form-row" - if error.has_key('inst_error'): - cls += " form-row-error" - %> - <div class="${cls}"> - <label>Instance name:</label> - <div class="form-row-input"> - <input type="text" name="instanceName" value="${instanceName}" size="40"> - </div> - %if error.has_key('inst_error'): - <div class="form-row-error-message">${error['inst_error']}</div> - %endif - <div style="clear: both"></div> - </div> - - <% - cls = "form-row" - if error.has_key('cred_error'): - cls += " form-row-error" - %> - <div class="${cls}"> - <label>Name of registered credentials to use:</label> - <div class="form-row-input"> - <select id="credName" name="credName" style="width:40em"> - <option value="">Select Credential...</option> - % for cred in credName: - <option value="${cred.name}">${cred.name}</option> - %endfor - </select> - </div> - %if error.has_key('cred_error'): - <div class="form-row-error-message">${error['cred_error']}</div> - %endif - <div style="clear: both"></div> - </div> - - - <% - cls = "form-row" - if error.has_key('vol_error'): - cls += " form-row-error" - %> - <div class="${cls}"> - <label>Permanent storage size (1-1000GB):<br/>(Note: you will be able to add more storage later)</label> - <div class="form-row-input"> - <input type="text" name="volSize" value="${volSize}" size="40"> - </div> - %if error.has_key('vol_error'): - <div class="form-row-error-message">${error['vol_error']}</div> - %endif - <div style="clear: both"></div> - </div> - - <% - cls = "form-row" - if error.has_key('zone_error'): - cls += " form-row-error" - %> - <div class="${cls}"> - <label>Create storage in zone:</label> - <div class="form-row-input"> - <select id="zones" name="zone" style="width:40em"> - </select> - </div> - %if error.has_key('zone_error'): - <div class="form-row-error-message">${error['zone_error']}</div> - %endif - <div style="clear: both"></div> - </div> - - - <div class="form-row"><input type="submit" value="Add"></div> - - </form> - </div> -</div> --- a/templates/cloud/add_provider.mako +++ /dev/null @@ -1,252 +0,0 @@ -<% _=n_ %> -<%inherit file="/base.mako"/> -<%def name="title()">Add provider</%def> - -<%def name="javascripts()"> -${parent.javascripts()} -<script type="text/javascript"> -$(function(){ - - $("input:text:first").focus(); - - $("#type").change(function() { - if ($(this).val() == 'ec2') { - clear(); - $("#autofill").attr( 'disabled', true ); - $("#autofill").attr( 'checked', false ); - $("#name").val( "EC2" ); - $("#region_name").val( "us-east-1" ); - $("#region_endpoint").val( "us-east-1.ec2.amazonaws.com" ); - $("#is_secure").val("1"); - $("#debug").val(""); - $("#path").val("/"); - } - else if ($(this).val() == 'eucalyptus') { - clear(); - $("#autofill").attr( 'disabled', false ); - } - }); -}) - - -function af(){ - - if ( $("#autofill").attr('checked') ) { - $("#name").val("Eucalyptus Public Cloud"); - $("#region_name").val("eucalyptus"); - $("#region_endpoint").val("mayhem9.cs.ucsb.edu"); - $("#is_secure").val("0"); - $("#port").val("8773"); - $("#path").val("/services/Eucalyptus"); - } -} - -function clear() { - $("#name").val(""); - $("#region_name").val(""); - $("#region_endpoint").val(""); - $("#is_secure").val(""); - $("#port").val(""); - $("#proxy").val(""); - $("#proxy_port").val(""); - $("#proxy_user").val(""); - $("#proxy_pass").val(""); - $("#debug").val(""); - $("#https_connection_factory").val(""); - $("#path").val(""); - -} - -</script> -</%def> - -%if header: - ${header} -%endif - -<div class="form"> - <div class="form-title">Add cloud provider</div> - <div class="form-body"> - <form name="add_provider_form" action="${h.url_for( action='add_provider' )}" method="post" > - <% - cls = "form-row" - if error.has_key('type_error'): - cls += " form-row-error" - %> - <div class="${cls}"> - <label>Provider type:</label> - <div class="form-row-input"> - <select id="type" name="type" style="width:40em"> - <option value="">Select Provider...</option> - <option value="eucalyptus">Eucalyptus</option> - <option value="ec2">Amazon EC2</option> - </select> - <br/> - <input type="checkbox" id="autofill" onclick="javascript:af()" disabled="true"> - auto fill using Eucalyptus Public Cloud values - </div> - %if error.has_key('type_error'): - <div class="form-row-error-message">${error['type_error']}</div> - %endif - <div style="clear: both"></div> - </div> - - <% - cls = "form-row" - if error.has_key('name_error'): - cls += " form-row-error" - %> - <div class="${cls}"> - <label>Provider name:</label> - <div class="form-row-input"> - <input type="text" id="name" name="name" value="${name}" size="40"> - </div> - %if error.has_key('name_error'): - <div class="form-row-error-message">${error['name_error']}</div> - %endif - <div style="clear: both"></div> - </div> - - <% - cls = "form-row" - %> - <div class="${cls}"> - <label>Region name:</label> - <div id="region_selection" class="form-row-input"> - <input type="text" name="region_name" id="region_name" value="${region_name}" size="40"> - </div> - <div style="clear: both"></div> - </div> - - <% - cls = "form-row" - %> - <div class="${cls}"> - <label>Region endpoint:</label> - <div class="form-row-input"> - <input type="text" name="region_endpoint" id="region_endpoint" value="${region_endpoint}" size="40"> - </div> - <div style="clear: both"></div> - </div> - - <% - cls = "form-row" - if error.has_key('is_secure_error'): - cls += " form-row-error" - %> - <div class="${cls}"> - <label>Is secure ('O' for False or '1' for True):</label> - <div class="form-row-input"> - <input type="text" name="is_secure" id="is_secure" value="${is_secure}" size="40"> - </div> - %if error.has_key('is_secure_error'): - <div class="form-row-error-message">${error['is_secure_error']}; you entered: '${is_secure}'</div> - %endif - <div style="clear: both"></div> - </div> - - <% - cls = "form-row" - %> - <div class="${cls}"> - <label>Host:</label> - <div class="form-row-input"> - <input type="text" name="host" value="${host}" size="40"> - </div> - <div style="clear: both"></div> - </div> - - <% - cls = "form-row" - %> - <div class="${cls}"> - <label>Port:</label> - <div class="form-row-input"> - <input type="text" name="port" id="port" value="${port}" size="40"> - </div> - <div style="clear: both"></div> - </div> - - <% - cls = "form-row" - %> - <div class="${cls}"> - <label>Proxy:</label> - <div class="form-row-input"> - <input type="text" name="proxy" value="${proxy}" size="40"> - </div> - <div style="clear: both"></div> - </div> - - <% - cls = "form-row" - %> - <div class="${cls}"> - <label>Proxy port:</label> - <div class="form-row-input"> - <input type="text" name="proxy_port" value="${proxy_port}" size="40"> - </div> - <div style="clear: both"></div> - </div> - - <% - cls = "form-row" - %> - <div class="${cls}"> - <label>Proxy user:</label> - <div class="form-row-input"> - <input type="text" name="proxy_user" value="${proxy_user}" size="40"> - </div> - <div style="clear: both"></div> - </div> - - <% - cls = "form-row" - %> - <div class="${cls}"> - <label>Proxy pass:</label> - <div class="form-row-input"> - <input type="text" name="proxy_pass" value="${proxy_pass}" size="40"> - </div> - <div style="clear: both"></div> - </div> - - <% - cls = "form-row" - %> - <div class="${cls}"> - <label>Debug:</label> - <div class="form-row-input"> - <input type="text" name="debug" value="${debug}" size="40"> - </div> - <div style="clear: both"></div> - </div> - - <% - cls = "form-row" - %> - <div class="${cls}"> - <label>HTTPS connection factory:</label> - <div class="form-row-input"> - <input type="text" name="https_connection_factory" value="${https_connection_factory}" size="40"> - </div> - <div style="clear: both"></div> - </div> - - <% - cls = "form-row" - %> - <div class="${cls}"> - <label>Path:</label> - <div class="form-row-input"> - <input type="text" name="path" id="path" value="${path}" size="40"> - </div> - <div style="clear: both"></div> - </div> - - <div class="form-row"><input type="submit" value="Add"></div> - - </form> - - </div> -</div> --- a/templates/cloud/index.mako +++ /dev/null @@ -1,16 +0,0 @@ -<%inherit file="/webapps/galaxy/base_panels.mako"/> - -<%def name="init()"> -<% - self.has_left_panel=False - self.has_right_panel=False - self.active_view="cloud" - self.message_box_visible=False -%> -</%def> - -<%def name="center_panel()"> - - <iframe name="galaxy_main" id="galaxy_main" frameborder="0" style="position: absolute; width: 100%; height: 100%;" src="${h.url_for( controller="cloud", action="list" )}"></iframe> - -</%def> --- a/templates/cloud/add_image.mako +++ /dev/null @@ -1,98 +0,0 @@ -<% _=n_ %> -<%inherit file="/base.mako"/> -<%def name="title()">Add machine image</%def> - -<%def name="javascripts()"> -${parent.javascripts()} -<script type="text/javascript"> -$(function(){ - //$("input:text:first").focus(); -}) -</script> -</%def> - -%if header: - ${header} -%endif - -<div class="form"> - <div class="form-title">Add machine image</div> - <div class="form-body"> - <form name="add_image" action="${h.url_for( action='add_new_image' )}" method="post" > - <% - cls = "form-row" - if error.has_key('provider_error'): - cls += " form-row-error" - %> - <div class="${cls}"> - <label>Cloud provider type:</label> - <div class="form-row-input"> - <select name="provider_type" style="width:40em"> - <option value="">Select Provider Type...</option> - <option value="eucalyptus">Eucalyptus</option> - <option value="ec2">Amazon EC2</option> - </select> - </div> - %if error.has_key('provider_error'): - <div class="form-row-error-message">${error['provider_error']}</div> - %endif - <div style="clear: both"></div> - </div> - - <% - cls = "form-row" - if error.has_key('id_error'): - cls += " form-row-error" - %> - <div class="${cls}"> - <label>Machine Image ID (AMI or EMI):</label> - <div class="form-row-input"> - <input type="text" name="image_id" value="${image_id}" size="40"> - </div> - %if error.has_key('id_error'): - <div class="form-row-error-message">${error['id_error']}</div> - %endif - <div style="clear: both"></div> - </div> - - <% - cls = "form-row" - if error.has_key('manifest_error'): - cls += " form-row-error" - %> - <div class="${cls}"> - <label>Manifest:</label> - <div class="form-row-input"> - <input type="text" name="manifest" value="${manifest}" size="40"> - </div> - %if error.has_key('manifest_error'): - <div class="form-row-error-message">${error['manifest_error']}</div> - %endif - <div style="clear: both"></div> - </div> - - - <% - cls = "form-row" - if error.has_key('arch_error'): - cls += " form-row-error" - %> - <div class="${cls}"> - <label>Image architecture:</label> - <div class="form-row-input"> - <select name="architecture" style="width:40em"> - <option value="">Select Architecture Type...</option> - <option value="i386">i386 (32 bit)</option> - <option value="x86_64">x86_64 (64 bit)</option> - </select> - </div> - %if error.has_key('arch_error'): - <div class="form-row-error-message">${error['arch_error']}</div> - %endif - <div style="clear: both"></div> - </div> - - <div class="form-row"><input type="submit" value="Add"></div> - </form> - </div> -</div> --- a/lib/galaxy/cloud/providers/ec2.py +++ /dev/null @@ -1,1232 +0,0 @@ -import subprocess, threading, os, errno, time, datetime, stat -from Queue import Queue, Empty -from datetime import datetime - -from galaxy import model # Database interaction class -from galaxy.model import mapping -from galaxy.datatypes.data import nice_size -from galaxy.util.bunch import Bunch -from galaxy.cloud import UCIwrapper -from Queue import Queue -from sqlalchemy import or_, and_ - -import galaxy.eggs -galaxy.eggs.require("boto") -from boto.ec2.connection import EC2Connection -from boto.ec2.regioninfo import RegionInfo -import boto.exception -import boto - -import logging -log = logging.getLogger( __name__ ) - -uci_states = Bunch( - NEW_UCI = "newUCI", - NEW = "new", - CREATING = "creating", - DELETING_UCI = "deletingUCI", - DELETING = "deleting", - SUBMITTED_UCI = "submittedUCI", - SUBMITTED = "submitted", - SHUTTING_DOWN_UCI = "shutting-downUCI", - SHUTTING_DOWN = "shutting-down", - ADD_STORAGE_UCI = "add-storageUCI", - ADD_STORAGE = "add-storage", - AVAILABLE = "available", - RUNNING = "running", - PENDING = "pending", - ERROR = "error", - DELETED = "deleted", - SNAPSHOT_UCI = "snapshotUCI", - SNAPSHOT = "snapshot" -) - -instance_states = Bunch( - TERMINATED = "terminated", - SUBMITTED = "submitted", - RUNNING = "running", - ADDING = "adding-storage", - PENDING = "pending", - SHUTTING_DOWN = "shutting-down", - ERROR = "error" -) - -store_status = Bunch( - WAITING = "waiting", - IN_USE = "in-use", - ADDING = "adding", - CREATING = "creating", - DELETED = 'deleted', - ERROR = "error" -) - -snapshot_status = Bunch( - SUBMITTED = 'submitted', - PENDING = 'pending', - COMPLETED = 'completed', - DELETE = 'delete', - DELETED= 'deleted', - ERROR = "error" -) - -class EC2CloudProvider( object ): - """ - Amazon EC2-based cloud provider implementation for managing instances. - """ - STOP_SIGNAL = object() - def __init__( self, app ): - self.type = "ec2" # cloud provider type (e.g., ec2, eucalyptus, opennebula) - self.zone = "us-east-1a" - self.security_group = "galaxyWeb" - self.queue = Queue() - self.sa_session = app.model.context - - self.threads = [] - nworkers = 5 - log.info( "Starting EC2 cloud controller workers..." ) - for i in range( nworkers ): - worker = threading.Thread( target=self.run_next ) - worker.start() - self.threads.append( worker ) - log.debug( "%d EC2 cloud workers ready", nworkers ) - - def shutdown( self ): - """Attempts to gracefully shut down the monitor thread""" - log.info( "sending stop signal to worker threads in EC2 cloud manager" ) - for i in range( len( self.threads ) ): - self.queue.put( self.STOP_SIGNAL ) - log.info( "EC2 cloud manager stopped" ) - - def put( self, uci_wrapper ): - """ - Add uci_wrapper object to the end of the request queue to be handled by - this cloud provider. - """ - state = uci_wrapper.get_uci_state() - uci_wrapper.change_state( state.split('U')[0] ) # remove 'UCI' from end of state description (i.e., mark as accepted and ready for processing) - self.queue.put( uci_wrapper ) - - def run_next( self ): - """Process next request, waiting until one is available if necessary.""" - cnt = 0 - while 1: - - uci_wrapper = self.queue.get() - uci_state = uci_wrapper.get_uci_state() - if uci_state is self.STOP_SIGNAL: - return - try: - if uci_state==uci_states.NEW: - self.create_uci( uci_wrapper ) - elif uci_state==uci_states.DELETING: - self.delete_uci( uci_wrapper ) - elif uci_state==uci_states.SUBMITTED: - self.start_uci( uci_wrapper ) - elif uci_state==uci_states.SHUTTING_DOWN: - self.stop_uci( uci_wrapper ) - elif uci_state==uci_states.SNAPSHOT: - self.snapshot_uci( uci_wrapper ) - elif uci_state==uci_states.ADD_STORAGE: - self.add_storage_to_uci( uci_wrapper ) - #self.dummy_start_uci( uci_wrapper ) - except: - log.exception( "Uncaught exception executing cloud request." ) - cnt += 1 - - def get_connection( self, uci_wrapper ): - """ - Establishes cloud connection using user's credentials associated with given UCI - """ - log.debug( 'Establishing %s cloud connection.' % self.type ) - provider = uci_wrapper.get_provider() - try: - region = RegionInfo( None, provider.region_name, provider.region_endpoint ) - except Exception, ex: - err = "Selecting region with cloud provider failed: " + str( ex ) - log.error( err ) - uci_wrapper.set_error( err, True ) - return None - try: - conn = EC2Connection( aws_access_key_id=uci_wrapper.get_access_key(), - aws_secret_access_key=uci_wrapper.get_secret_key(), - is_secure=provider.is_secure, - region=region, - path=provider.path ) - except boto.exception.EC2ResponseError, e: - err = "Establishing connection with cloud failed: " + str( e ) - log.error( err ) - uci_wrapper.set_error( err, True ) - return None - - return conn - - def check_key_pair( self, uci_wrapper, conn ): - """ - Check if a key pair associated with this UCI exists on cloud provider. - If yes, return key pair name; otherwise, generate a key pair with the cloud - provider and, again, return key pair name. - Key pair name for given UCI is generated from UCI's name and suffix '_kp' - """ - kp = None - kp_name = uci_wrapper.get_name().replace(' ','_') + "_kp" - log.debug( "Checking user's key pair: '%s'" % kp_name ) - try: - kp = conn.get_key_pair( kp_name ) - uci_kp_name = uci_wrapper.get_key_pair_name() - uci_material = uci_wrapper.get_key_pair_material() - if kp != None: - if kp.name != uci_kp_name or uci_material == None: - # key pair exists on the cloud but not in local database, so re-generate it (i.e., delete and then create) - try: - conn.delete_key_pair( kp_name ) - kp = self.create_key_pair( conn, kp_name ) - uci_wrapper.set_key_pair( kp.name, kp.material ) - except boto.exception.EC2ResponseError, e: - err = "EC2 response error while deleting key pair: " + str( e ) - log.error( err ) - uci_wrapper.set_error( err, True ) - else: - try: - kp = self.create_key_pair( conn, kp_name ) - uci_wrapper.set_key_pair( kp.name, kp.material ) - except boto.exception.EC2ResponseError, e: - err = "EC2 response error while creating key pair: " + str( e ) - log.error( err ) - uci_wrapper.set_error( err, True ) - except Exception, ex: - err = "Exception while creating key pair: " + str( ex ) - log.error( err ) - uci_wrapper.set_error( err, True ) - except boto.exception.EC2ResponseError, e: # No keypair under this name exists so create it - if e.code == 'InvalidKeyPair.NotFound': - log.info( "No keypair found, creating keypair '%s'" % kp_name ) - kp = self.create_key_pair( conn, kp_name ) - uci_wrapper.set_key_pair( kp.name, kp.material ) - else: - err = "EC2 response error while retrieving key pair: " + str( e ) - log.error( err ) - uci_wrapper.set_error( err, True ) - - if kp != None: - return kp.name - else: - return None - - def create_key_pair( self, conn, kp_name ): - """ Initiate creation of key pair under kp_name by current cloud provider. """ - try: - return conn.create_key_pair( kp_name ) - except boto.exception.EC2ResponseError, e: - return None - - def get_mi_id( self, uci_wrapper, i_index ): - """ - Get appropriate machine image (mi) based on instance size. - """ - i_type = uci_wrapper.get_instance_type( i_index ) - if i_type=='m1.small' or i_type=='c1.medium': - arch = 'i386' - else: - arch = 'x86_64' - - mi = self.sa_session.query( model.CloudImage ).filter_by( deleted=False, provider_type=self.type, architecture=arch ).first() - if mi: - return mi.image_id - else: - err = "Machine image could not be retrieved" - log.error( "%s for UCI '%s'." % (err, uci_wrapper.get_name() ) ) - uci_wrapper.set_error( err+". Contact site administrator to ensure needed machine image is registered.", True ) - return None - - def create_uci( self, uci_wrapper ): - """ - Create User Configured Instance (UCI) - i.e., create storage volume on cloud provider - and register relevant information in local Galaxy database. - """ - conn = self.get_connection( uci_wrapper ) - if uci_wrapper.get_uci_availability_zone()=='': - log.info( "Availability zone for UCI (i.e., storage volume) was not selected, using default zone: %s" % self.zone ) - uci_wrapper.set_store_availability_zone( self.zone ) - - store = uci_wrapper.get_all_stores_in_status( store_status.ADDING )[0] # Because at UCI creation time only 1 storage volume can be created, reference it directly - - log.info( "Creating storage volume in zone '%s' of size '%s'..." % ( uci_wrapper.get_uci_availability_zone(), store.size ) ) - # Because only 1 storage volume may be created at UCI config time, index of this storage volume in local Galaxy DB w.r.t - # current UCI is 0, so reference it in following methods - vol = conn.create_volume( store.size, uci_wrapper.get_uci_availability_zone(), snapshot=None ) - uci_wrapper.set_store_volume_id( store.id, vol.id ) - - # Retrieve created volume again to get updated status - try: - vl = conn.get_all_volumes( [vol.id] ) - except boto.exception.EC2ResponseError, e: - err = "EC2 response error while retrieving (i.e., updating status) of just created storage volume '" + vol.id + "': " + str( e ) - log.error( err ) - uci_wrapper.set_store_status( vol.id, uci_states.ERROR ) - uci_wrapper.set_error( err, True ) - return - except Exception, ex: - err = "Error while retrieving (i.e., updating status) of just created storage volume '" + vol.id + "': " + str( ex ) - log.error( err ) - uci_wrapper.set_error( err, True ) - return - - if len( vl ) > 0: - uci_wrapper.change_state( uci_state=vl[0].status ) - uci_wrapper.set_store_status( vol.id, vl[0].status ) - else: - err = "Volume '" + vol.id +"' not found by EC2 after being created." - log.error( err ) - uci_wrapper.set_store_status( vol.id, uci_states.ERROR ) - uci_wrapper.set_error( err, True ) - - def delete_uci( self, uci_wrapper ): - """ - Delete UCI - i.e., delete all storage volumes associated with this UCI. - NOTE that this implies deletion of any and all data associated - with this UCI from the cloud. All data will be deleted. - Information in local Galaxy database is marked as deleted but not actually removed - from the database. - """ - conn = self.get_connection( uci_wrapper ) - vl = [] # volume list - count = 0 # counter for checking if all volumes assoc. w/ UCI were deleted - - # Get all volumes assoc. w/ UCI, delete them from cloud as well as in local DB - vl = uci_wrapper.get_all_stores() - deletedList = [] - failedList = [] - for v in vl: - log.debug( "Deleting volume with id='%s'" % v.volume_id ) - try: - if conn.delete_volume( v.volume_id ): - deletedList.append( v.volume_id ) - v.deleted = True - v.status = store_status.DELETED - self.sa_session.add( v ) - self.sa_session.flush() - count += 1 - else: - failedList.append( v.volume_id ) - except boto.exception.EC2ResponseError, e: - err = "EC2 response error while deleting storage volume '" + v.volume_id + "': " + str( e ) - log.error( err ) - uci_wrapper.set_store_error( err, store_id = v.volume_id ) - uci_wrapper.set_error( err, True ) - - # Delete UCI if all of associated - if count == len( vl ): - uci_wrapper.set_deleted() - else: - if uci_wrapper.get_uci_state != uci_states.ERROR: - err = "Deleting following volume(s) failed: " + str( failedList ) + ". However, these volumes were successfully deleted: " \ - + str( deletedList ) + ". MANUAL intervention and processing needed." - log.error( err ) - uci_wrapper.set_error( err, True ) - - def snapshot_uci( self, uci_wrapper ): - """ - Initiate creation of a snapshot by cloud provider for all storage volumes - associated with this UCI. - """ - if uci_wrapper.get_uci_state() != uci_states.ERROR: - conn = self.get_connection( uci_wrapper ) - - snapshots = uci_wrapper.get_snapshots( status = snapshot_status.SUBMITTED ) - for snapshot in snapshots: - log.debug( "Snapshot DB id: '%s', volume id: '%s'" % ( snapshot.id, snapshot.store.volume_id ) ) - try: - snap = conn.create_snapshot( volume_id=snapshot.store.volume_id ) - snap_id = str( snap ).split(':')[1] - uci_wrapper.set_snapshot_id( snapshot.id, snap_id ) - sh = conn.get_all_snapshots( snap_id ) # get updated status - uci_wrapper.set_snapshot_status( status=sh[0].status, snap_id=snap_id ) - except boto.exception.EC2ResponseError, e: - err = "EC2 response error while creating snapshot: " + str( e ) - log.error( err ) - uci_wrapper.set_snapshot_error( error=err, snap_index=snapshot.id, set_status=True ) - uci_wrapper.set_error( err, True ) - return - except Exception, ex: - err = "Error while creating snapshot: " + str( ex ) - log.error( err ) - uci_wrapper.set_snapshot_error( error=err, snap_index=snapshot.id, set_status=True ) - uci_wrapper.set_error( err, True ) - return - - uci_wrapper.change_state( uci_state=uci_states.AVAILABLE ) - - def add_storage_to_uci( self, uci_wrapper ): - """ - Add an additional storage volume to specified UCI by creating the storage volume - on cloud provider, attaching it to currently running instance and adding it to - 'galaxyData' zpool on remote instance. - """ - conn = self.get_connection( uci_wrapper ) - - stores = uci_wrapper.get_all_stores_in_status( store_status.ADDING ) - for store in stores: - vol_size = store.size - availability_zone = uci_wrapper.get_uci_availability_zone() - log.info( "Adding storage volume to UCI '%s' in zone '%s' of size '%s'..." % ( uci_wrapper.get_name(), availability_zone, vol_size ) ) - - try: - vol = conn.create_volume( vol_size, availability_zone, snapshot=None ) - uci_wrapper.set_store_volume_id( store.id, vol.id ) - uci_wrapper.set_store_availability_zone( availability_zone, vol.id ) - log.debug( "New storage volume created: '%s'" % vol.id ) - except boto.exception.EC2ResponseError, e: - err = "EC2 response error while creating storage volume: " + str( e ) - log.error( err ) - uci_wrapper.set_store_error( err, store_id=vol.id ) - uci_wrapper.set_error( err, True ) - return - except Exception, ex: - err = "Error while creating storage volume: " + str( ex ) - log.error( err ) - uci_wrapper.set_error( err, True ) - return - - # Retrieve created volume again to get updated status - try: - vl = conn.get_all_volumes( [vol.id] ) - except boto.exception.EC2ResponseError, e: - err = "EC2 response error while retrieving (i.e., updating status) of just created storage volume '" + vol.id + "': " + str( e ) - log.error( err ) - uci_wrapper.set_store_error( err, store_id=vol.id ) - uci_wrapper.set_error( err, True ) - return - except Exception, ex: - err = "Error while retrieving (i.e., updating status) of just created storage volume '" + vol.id + "': " + str( ex ) - log.error( err ) - uci_wrapper.set_error( err, True ) - return - - # Wait for a while to ensure volume was created - if len( vl ) > 0: - vol_status = vl[0].status # Bc. only single vol is queried, reference it as 0th list element - for i in range( 30 ): - if vol_status != "available": - log.debug( "(%s) Updating volume status; current status: '%s'" % (i, vol_status ) ) - uci_wrapper.change_state( uci_state=vol_status ) - time.sleep(5) - vol_status = vl[0].status - if vol_status == "available": - log.debug( "(%s) New volume status '%s', continuing with file system adjustment." % (i, vol_status ) ) - uci_wrapper.set_store_status( vl[0].id, vol_status ) - break - if i is 29: - err = "Error while creating volume '"+vl[0].id+"'; stuck in state '"+vol_status+"'; deleting volume." - conn.delete_volume( vl[0].id ) - log.error( err ) - uci_wrapper.set_error( err, True ) - uci_wrapper.set_store_error( err, store_id=vol.id ) - conn.delete_volume( vl[0].id ) - uci_wrapper.set_store_deleted( vl[0].id ) - return - else: - err = "Volume '" + vol.id +"' not found by EC2 after being created." - log.error( err ) - uci_wrapper.set_store_error( err, store_id=vol.id ) - uci_wrapper.set_error( err, True ) - return - - # Get private key for given instance - pk = uci_wrapper.get_key_pair_material() - if pk == None: #If pk does not exist, create it - self.check_key_pair( uci_wrapper, conn ) - pk = uci_wrapper.get_key_pair_material() - - # Get working directory for this UCI and store pk into a file - wd = uci_wrapper.get_uci_working_directory() - if not os.path.exists( wd ): - os.mkdir( wd ) - pk_file_path = os.path.join( wd, "pk" ) - - if pk != None: - # Save private key to a file - pk_file = open( pk_file_path, "w" ) - pk_file.write( pk ) - pk_file.close() - else: - err = "ERROR: Private key not available for this UCI." - log.error( err ) - uci_wrapper.set_store_error( err, store_id=vol.id ) - uci_wrapper.set_error( err, True ) - return - - if os.path.exists( pk_file_path ): - # Change permissions of the file - this is required by later used ssh - os.chmod( pk_file_path, stat.S_IRUSR | stat.S_IWUSR ) - - # Get # of storage volumes associated with this UCI to know as which device to connect new volume to the instance - device_num = len( uci_wrapper.get_all_stores_in_status( store_status.IN_USE ) ) + 5 # First device num is 5, so all subsequent ones should follow - - # Get instance that the new storage volume is to be attached to. Although a list is returned, - # only 1 instance can be in 'adding-storage' state (because, for now, only 1 instance is assoc. with - # each UCI) and volume can be attached to only to it - il = uci_wrapper.get_instaces_in_state( instance_states.ADDING ) - if len( il ) > 0: - # Attach new volume to the instance - log.debug( "Attaching new storage volume '%s' to UCI '%s' as device '%s'" % - ( vol.id, uci_wrapper.get_name(), device_num ) ) - try: - vol_status = conn.attach_volume( vol.id, il[0].instance_id, device_num ) - except boto.exception.EC2ResponseError, e: - err = "Attaching just created storage volume '" + vol.id + "'to instance '" + \ - il[0].instance_id + "' as device '" + str( device_num ) + "' failed: " + str( e ) - log.error( err ) - uci_wrapper.set_store_error( err, store_id=vol.id ) - uci_wrapper.set_error( err, True ) - return - # For a while, keep checking attachment status of the new volume - for i in range(30): - log.debug( "Checking attachment status of new volume '%s': '%s'" % ( vol.id, vol_status ) ) - if vol_status == 'attached': - uci_wrapper.set_store_status( vol.id, vol_status ) - uci_wrapper.set_store_device( vol.id, device_num ) - break - if i == 29: - err = "Storage volume '" + vol.id + "' failed to attach to instance '" + il[0].instance_id + \ - "'. Manual check needed." - log.error( err ) - uci_wrapper.set_store_error( err, store_id=vol.id ) - uci_wrapper.set_error( err, False ) - return - - time.sleep(4) - vol_list = conn.get_all_volumes( [vol.id] ) - for v in vol_list: - vol_status = v.attachment_state() - - # Once storage volume is attached, add it to the zpool by issuing system level command - cmd = 'ssh -o StrictHostKeyChecking=no -i '+ pk_file_path +' root@'+il[0].public_dns+' "zpool add galaxyData c7d' + str( device_num )+'"' - log.debug( "Adding new storage volume to zpool cmd: %s" % cmd ) - stdout = os.system( cmd ) - if stdout != 0: - err = "Adding newly created storage volume to zpool on instance '" + il[0].instance_id + \ - "' failed. Error code: " + str( stdout ) - log.error( err ) - uci_wrapper.set_store_error( err, store_id=vol.id ) - uci_wrapper.set_error( err, False ) - return - else: - err = "No instance(s) found in 'adding-storage' state. New disk not added to UCI's zpool." - log.error( err ) - uci_wrapper.set_store_error( err, store_id=vol.id ) - uci_wrapper.set_error( err, True ) - return - - # Update UCI's total storage size - uci_wrapper.set_uci_total_size( uci_wrapper.get_uci_total_size() + vol.size ) - # Reset UCI's and instance's state - uci_wrapper.change_state( uci_state=uci_states.RUNNING, instance_id=il[0].instance_id, i_state=instance_states.RUNNING ) - log.debug( "Successfully added storage volume '%s' to UCI '%s'." % ( vol.id, uci_wrapper.get_name() ) ) - - def dummy_start_uci( self, uci_wrapper ): - - uci = uci_wrapper.get_uci() - log.debug( "Dummy start UCI '%s'" % uci.name ) - - -# uci_wrapper.change_state( uci_state.PENDING ) -# log.debug( "Sleeping a bit... (%s)" % uci.name ) -# time.sleep(20) -# log.debug( "Woke up! (%s)" % uci.name ) - - def start_uci( self, uci_wrapper ): - """ - Start instance(s) of given UCI on the cloud. - """ - if uci_wrapper.get_uci_state() != uci_states.ERROR: - conn = self.get_connection( uci_wrapper ) - self.check_key_pair( uci_wrapper, conn ) - if uci_wrapper.get_key_pair_name() == None: - err = "Key pair not found" - log.error( "%s for UCI '%s'." % ( err, uci_wrapper.get_name() ) ) - uci_wrapper.set_error( err + ". Try resetting the state and starting the instance again.", True ) - return - - i_indexes = uci_wrapper.get_instances_indexes( state=instance_states.SUBMITTED ) # Get indexes of i_indexes associated with this UCI that are in 'submitted' state - log.debug( "Starting instances with IDs: '%s' associated with UCI '%s' " % ( i_indexes, uci_wrapper.get_name(), ) ) - if len( i_indexes ) > 0: - for i_index in i_indexes: - # Get machine image for current instance - mi_id = self.get_mi_id( uci_wrapper, i_index ) - log.debug( "mi_id: %s, uci_wrapper.get_key_pair_name(): %s" % ( mi_id, uci_wrapper.get_key_pair_name() ) ) - uci_wrapper.set_mi( i_index, mi_id ) - - if mi_id != None: - # Check if galaxy security group exists (and create it if it does not) - log.debug( "Setting up '%s' security group." % self.security_group ) - try: - conn.get_all_security_groups( [self.security_group] ) # security groups - except boto.exception.EC2ResponseError, e: - if e.code == 'InvalidGroup.NotFound': - log.info( "No security group found, creating security group '%s'" % self.security_group ) - try: - gSecurityGroup = conn.create_security_group(self.security_group, 'Security group for Galaxy.') - gSecurityGroup.authorize( 'tcp', 80, 80, '0.0.0.0/0' ) # Open HTTP port - gSecurityGroup.authorize( 'tcp', 22, 22, '0.0.0.0/0' ) # Open SSH port - except boto.exception.EC2ResponseError, ee: - err = "EC2 response error while creating security group: " + str( ee ) - log.error( err ) - uci_wrapper.set_error( err, True ) - else: - err = "EC2 response error while retrieving security group: " + str( e ) - log.error( err ) - uci_wrapper.set_error( err, True ) - - - if uci_wrapper.get_uci_state() != uci_states.ERROR: - # Start an instance - log.debug( "Starting instance for UCI '%s'" % uci_wrapper.get_name() ) - #TODO: Once multiple volumes can be attached to a single instance, update 'userdata' composition - # Compose user data; for storage volumes, separate multiple volumes with a colon (:) ensuring that - # the last volume in the list is not followed by a colon. - stores = uci_wrapper.get_all_stores() - volume_ids = "" - if len( stores ) > 0: - for i, store in enumerate( stores ): - volume_ids += store.volume_id - if i < len( stores )-1: - volume_ids += ":" - else: - err = "No storage volumes found that are associated with UCI '%s'" + uci_wrapper.get_name() - log.error( err ) - uci_wrapper.set_error( err, True ) - return - userdata = volume_ids+"|"+uci_wrapper.get_access_key()+"|"+uci_wrapper.get_secret_key() - log.debug( "Using following command: conn.run_instances( image_id='%s', key_name='%s', security_groups=['%s'], user_data=[OMITTED], instance_type='%s', placement='%s' )" - % ( mi_id, uci_wrapper.get_key_pair_name(), self.security_group, uci_wrapper.get_instance_type( i_index ), uci_wrapper.get_uci_availability_zone() ) ) - # Start an instance - reservation = None - try: - reservation = conn.run_instances( image_id=mi_id, - key_name=uci_wrapper.get_key_pair_name(), - security_groups=[self.security_group], - user_data=userdata, - instance_type=uci_wrapper.get_instance_type( i_index ), - placement=uci_wrapper.get_uci_availability_zone() ) - except boto.exception.EC2ResponseError, e: - err = "EC2 response error when starting UCI '"+ uci_wrapper.get_name() +"': " + str( e ) - log.error( err ) - uci_wrapper.set_error( err, True ) - except Exception, ex: - err = "Error when starting UCI '" + uci_wrapper.get_name() + "': " + str( ex ) - log.error( err ) - uci_wrapper.set_error( err, True ) - # Record newly available instance data into local Galaxy database - if reservation: - l_time = datetime.utcnow() - # uci_wrapper.set_instance_launch_time( self.format_time( reservation.instances[0].launch_time ), i_index=i_index ) - uci_wrapper.set_instance_launch_time( l_time, i_index=i_index ) - if not uci_wrapper.uci_launch_time_set(): - uci_wrapper.set_uci_launch_time( l_time ) - try: - uci_wrapper.set_reservation_id( i_index, str( reservation ).split(":")[1] ) - # TODO: if more than a single instance will be started through single reservation, change this reference to element [0] - i_id = str( reservation.instances[0]).split(":")[1] - uci_wrapper.set_instance_id( i_index, i_id ) - s = reservation.instances[0].state - uci_wrapper.change_state( s, i_id, s ) - uci_wrapper.set_security_group_name( self.security_group, i_id=i_id ) - vol_id = uci_wrapper.get_store_volume_id( store_id=0 ) # TODO: Once more that one vol/UCI is allowed, update this! - # Following line is pointless bc. general update updates status of volume to 'available' - # before it actually connects to starting instance... This has been dealt w/ in general update method - #uci_wrapper.set_store_status( vol_id, store_status.WAITING ) - uci_wrapper.set_store_instance( vol_id, i_id ) - log.debug( "Instance of UCI '%s' started, current state: '%s'" % ( uci_wrapper.get_name(), uci_wrapper.get_uci_state() ) ) - except boto.exception.EC2ResponseError, e: - err = "EC2 response error when retrieving instance information for UCI '" + uci_wrapper.get_name() + "': " + str( e ) - log.error( err ) - uci_wrapper.set_error( err, True ) - else: - log.error( "UCI '%s' is in 'error' state, starting instance was aborted." % uci_wrapper.get_name() ) - else: - err = "No instances in state '"+ instance_states.SUBMITTED +"' found for UCI '" + uci_wrapper.get_name() + \ - "'. Nothing to start." - log.error( err ) - uci_wrapper.set_error( err, True ) - else: - log.error( "UCI '%s' is in 'error' state, starting instance was aborted." % uci_wrapper.get_name() ) - - def stop_uci( self, uci_wrapper): - """ - Stop all of cloud instances associated with given UCI. - """ - conn = self.get_connection( uci_wrapper ) - - # Get all instances associated with given UCI - il = uci_wrapper.get_instances_ids() # instance list - # Process list of instances and remove any references to empty instance id's - for i in il: - if i is None: - il.remove( i ) - log.debug( 'List of instances being terminated: %s' % il ) - rl = conn.get_all_instances( il ) # Reservation list associated with given instances - - # Initiate shutdown of all instances under given UCI - cnt = 0 - stopped = [] - not_stopped = [] - for r in rl: - for inst in r.instances: - log.debug( "Sending stop signal to instance '%s' associated with reservation '%s'." % ( inst, r ) ) - try: - inst.stop() - uci_wrapper.set_stop_time( datetime.utcnow(), i_id=inst.id ) - uci_wrapper.change_state( instance_id=inst.id, i_state=inst.update() ) - stopped.append( inst ) - except boto.exception.EC2ResponseError, e: - not_stopped.append( inst ) - err = "EC2 response error when stopping instance '" + inst.instance_id + "': " + str(e) - log.error( err ) - uci_wrapper.set_error( err, True ) - - uci_wrapper.reset_uci_launch_time() - log.debug( "Termination was initiated for all instances of UCI '%s'." % uci_wrapper.get_name() ) - - -# dbInstances = get_instances( trans, uci ) #TODO: handle list! -# -# # Get actual cloud instance object -# cloudInstance = get_cloud_instance( conn, dbInstances.instance_id ) -# -# # TODO: Detach persistent storage volume(s) from instance and update volume data in local database -# stores = get_stores( trans, uci ) -# for i, store in enumerate( stores ): -# log.debug( "Detaching volume '%s' to instance '%s'." % ( store.volume_id, dbInstances.instance_id ) ) -# mntDevice = store.device -# volStat = None -## Detaching volume does not work with Eucalyptus Public Cloud, so comment it out -## try: -## volStat = conn.detach_volume( store.volume_id, dbInstances.instance_id, mntDevice ) -## except: -## log.debug ( 'Error detaching volume; still going to try and stop instance %s.' % dbInstances.instance_id ) -# store.attach_time = None -# store.device = None -# store.i_id = None -# store.status = volStat -# log.debug ( '***** volume status: %s' % volStat ) -# -# -# # Stop the instance and update status in local database -# cloudInstance.stop() -# dbInstances.stop_time = datetime.utcnow() -# while cloudInstance.state != 'terminated': -# log.debug( "Stopping instance %s state; current state: %s" % ( str( cloudInstance ).split(":")[1], cloudInstance.state ) ) -# time.sleep(3) -# cloudInstance.update() -# dbInstances.state = cloudInstance.state -# -# # Reset relevant UCI fields -# uci.state = 'available' -# uci.launch_time = None -# -# # Persist -# session = trans.sa_session -## session.save_or_update( stores ) -# session.save_or_update( dbInstances ) # TODO: Is this going to work w/ multiple instances stored in dbInstances variable? -# session.save_or_update( uci ) -# session.flush() -# trans.log_event( "User stopped cloud instance '%s'" % uci.name ) -# trans.set_message( "Galaxy instance '%s' stopped." % uci.name ) - - def update( self ): - """ - Run status update on all instances that are in 'running', 'pending', or 'shutting-down' state. - Run status update on all storage volumes whose status is 'in-use', 'creating', or 'None'. - Run status update on all snapshots whose status is 'pending' or 'delete' - Run status update on any zombie UCIs, i.e., UCI's that is in 'submitted' state for an - extended period of time. - - Reason behind this method is to sync state of local DB and real-world resources - """ - log.debug( "Running general status update for %s UCIs..." % self.type ) - # Update instances - instances = self.sa_session.query( model.CloudInstance ) \ - .filter( or_( model.CloudInstance.table.c.state==instance_states.RUNNING, - model.CloudInstance.table.c.state==instance_states.PENDING, - model.CloudInstance.table.c.state==instance_states.SHUTTING_DOWN ) ) \ - .all() - for inst in instances: - if self.type == inst.uci.credentials.provider.type: - log.debug( "[%s] Running general status update on instance '%s'" - % ( inst.uci.credentials.provider.type, inst.instance_id ) ) - self.update_instance( inst ) - # Update storage volume(s) associated with current instance - stores = self.sa_session.query( model.CloudStore ) \ - .filter_by( uci_id=inst.uci_id, deleted=False ) \ - .all() - for store in stores: - if self.type == store.uci.credentials.provider.type: # and store.volume_id != None: - log.debug( "[%s] Running general status update on store with local database ID: '%s'" - % ( store.uci.credentials.provider.type, store.id ) ) - self.update_store( store ) - - # Update storage volume(s) - stores = self.sa_session.query( model.CloudStore ) \ - .filter( or_( model.CloudStore.table.c.status==store_status.CREATING, -# model.CloudStore.table.c.status==store_status.IN_USE, -# model.CloudStore.table.c.status==store_status.WAITING, - model.CloudStore.table.c.status==None ) ) \ - .all() - for store in stores: - if self.type == store.uci.credentials.provider.type: # and store.volume_id != None: - log.debug( "[%s] Running general status update on store with local database ID: '%s'" - % ( store.uci.credentials.provider.type, store.id ) ) - self.update_store( store ) -# else: -# log.error( "[%s] There exists an entry for UCI (%s) storage volume without an ID. Storage volume might have been created with " -# "cloud provider though. Manual check is recommended." % ( store.uci.credentials.provider.type, store.uci.name ) ) -# store.uci.error = "There exists an entry in local database for a storage volume without an ID. Storage volume might have been created " \ -# "with cloud provider though. Manual check is recommended. After understanding what happened, local database entry for given " \ -# "storage volume should be updated." -# store.status = store_status.ERROR -# store.uci.state = uci_states.ERROR -# store.uci.flush() -# store.flush() - - # Update pending snapshots or delete ones marked for deletion - snapshots = self.sa_session.query( model.CloudSnapshot ) \ - .filter( or_( model.CloudSnapshot.table.c.status == snapshot_status.PENDING, model.CloudSnapshot.table.c.status == snapshot_status.DELETE ) ) \ - .all() - for snapshot in snapshots: - if self.type == snapshot.uci.credentials.provider.type and snapshot.status == snapshot_status.PENDING: - log.debug( "[%s] Running general status update on snapshot '%s'" % ( snapshot.uci.credentials.provider.type, snapshot.snapshot_id ) ) - self.update_snapshot( snapshot ) - elif self.type == snapshot.uci.credentials.provider.type and snapshot.status == snapshot_status.DELETE: - log.debug( "[%s] Initiating deletion of snapshot '%s'" % ( snapshot.uci.credentials.provider.type, snapshot.snapshot_id ) ) - self.delete_snapshot( snapshot ) - - # Attempt at updating any zombie UCIs (i.e., instances that have been in SUBMITTED state for longer than expected - see below for exact time) - zombies = self.sa_session.query( model.UCI ).filter_by( state=uci_states.SUBMITTED ).all() - for zombie in zombies: - z_instances = self.sa_session.query( model.CloudInstance ) \ - .filter_by( uci_id=zombie.id ) \ - .filter( or_( model.CloudInstance.table.c.state != instance_states.TERMINATED, - model.CloudInstance.table.c.state == None ) ) \ - .all() - for z_inst in z_instances: - if self.type == z_inst.uci.credentials.provider.type: -# log.debug( "z_inst.id: '%s', state: '%s'" % ( z_inst.id, z_inst.state ) ) - td = datetime.utcnow() - z_inst.update_time - if td.seconds > 180: # if instance has been in SUBMITTED state for more than 3 minutes - log.debug( "[%s] Running zombie repair update on instance with DB id '%s'" % ( z_inst.uci.credentials.provider.type, z_inst.id ) ) - self.process_zombie( z_inst ) - - def update_instance( self, inst ): - """ - Update information in local database for given instance as it is obtained from cloud provider. - Along with updating information about given instance, information about the UCI controlling - this instance is also updated. - """ - # Get credentials associated wit this instance - uci_id = inst.uci_id - uci = self.sa_session.query( model.UCI ).get( uci_id ) - self.sa_session.refresh( uci ) - conn = self.get_connection_from_uci( uci ) - - # Get reservations handle for given instance - try: - rl= conn.get_all_instances( [inst.instance_id] ) - except boto.exception.EC2ResponseError, e: - err = "Retrieving instance(s) from cloud failed for UCI '"+ uci.name +"' during general status update: " + str( e ) - log.error( err ) - uci.error = err - uci.state = uci_states.ERROR - self.sa_session.add( uci ) - self.sa_session.flush() - return None - - # Because references to reservations are deleted shortly after instances have been terminated, getting an empty list as a response to a query - # typically means the instance has successfully shut down but the check was not performed in short enough amount of time. Until an alternative solution - # is found, below code sets state of given UCI to 'error' to indicate to the user something out of ordinary happened. - if len( rl ) == 0: - err = "Instance ID '"+inst.instance_id+"' was not found by the cloud provider. Instance might have crashed or otherwise been terminated."+ \ - "Manual check is recommended." - log.error( err ) - inst.error = err - uci.error = err - inst.state = instance_states.TERMINATED - uci.state = uci_states.ERROR - uci.launch_time = None - self.sa_session.add( inst ) - self.sa_session.add( uci ) - self.sa_session.flush() - # Update instance status in local DB with info from cloud provider - for r in rl: - for i, cInst in enumerate( r.instances ): - try: - s = cInst.update() - log.debug( "Checking state of cloud instance '%s' associated with UCI '%s' " \ - "and reservation '%s'. State='%s'" % ( cInst, uci.name, r, s ) ) - if s != inst.state: - inst.state = s - self.sa_session.add( inst ) - self.sa_session.flush() - # After instance has shut down, ensure UCI is marked as 'available' - if s == instance_states.TERMINATED and uci.state != uci_states.ERROR: - uci.state = uci_states.AVAILABLE - uci.launch_time = None - self.sa_session.add( uci ) - self.sa_session.flush() - # Making sure state of UCI is updated. Once multiple instances become associated with single UCI, this will need to be changed. - if s != uci.state and s != instance_states.TERMINATED: - uci.state = s - self.sa_session.add( uci ) - self.sa_session.flush() - if cInst.public_dns_name != inst.public_dns: - inst.public_dns = cInst.public_dns_name - self.sa_session.add( inst ) - self.sa_session.flush() - if cInst.private_dns_name != inst.private_dns: - inst.private_dns = cInst.private_dns_name - self.sa_session.add( inst ) - self.sa_session.flush() - except boto.exception.EC2ResponseError, e: - err = "Updating instance status from cloud failed for UCI '"+ uci.name + \ - "' during general status update: " + str( e ) - log.error( err ) - uci.error = err - uci.state = uci_states.ERROR - self.sa_session.add( uci ) - self.sa_session.flush() - return None - - def update_store( self, store ): - """ - Update information in local database for given storage volume as it is obtained from cloud provider. - Along with updating information about given storage volume, information about the UCI controlling - this storage volume is also updated. - """ - # Get credentials associated wit this store - uci_id = store.uci_id - uci = self.sa_session.query( model.UCI ).get( uci_id ) - self.sa_session.refresh( uci ) - conn = self.get_connection_from_uci( uci ) - - # Get reservations handle for given store - try: - log.debug( "Retrieving reference to storage volume '%s' during update..." % store.volume_id ) - vl = conn.get_all_volumes( [store.volume_id] ) - except boto.exception.EC2ResponseError, e: - err = "Retrieving volume(s) from cloud failed for UCI '"+ uci.name + "' during general status update: " + str( e ) - log.error( err ) - uci.error = err - uci.state = uci_states.ERROR - self.sa_session.add( uci ) - self.sa_session.flush() - return None - - # Update store status in local DB with info from cloud provider - if len(vl) > 0: - try: - log.debug( "General status update for storage volume '%s'; current status: '%s'" % (store.volume_id, vl[0].status ) ) - if store.status != vl[0].status: - # In case something failed during creation of UCI but actual storage volume was created and yet - # UCI state remained as 'new', try to remedy this by updating UCI state here - if ( store.status == None ) and ( store.volume_id != None ): - uci.state = vl[0].status - self.sa_session.add( uci ) - self.sa_session.flush() - # If UCI was marked in state 'CREATING', update its status to reflect new status - elif ( uci.state == uci_states.CREATING ): - uci.state = vl[0].status - self.sa_session.add( uci ) - self.sa_session.flush() - - store.status = vl[0].status - self.sa_session.add( store ) - self.sa_session.flush() - # Boto does not seem to be reporting these values although fields exist so comment them out... -# log.debug( "vl[0].instance_id: '%s'" % vl[0].instance_id ) -# if store.inst != None: -# if store.inst.instance_id != vl[0].instance_id: -# store.inst.instance_id = vl[0].instance_id -# self.sa_session.add( store ) -# self.sa_session.flush() -# log.debug( "vl[0].attach_time: '%s'" % vl[0].attach_time ) -# if store.attach_time != vl[0].attach_time: -# store.attach_time = vl[0].attach_time -# self.sa_session.add( store ) -# self.sa_session.flush() -## log.debug( "vl[0].device: '%s'" % vl[0].device ) -# if store.device != vl[0].device: -# store.device = vl[0].device -# self.sa_session.add( store ) -# self.sa_session.flush() - except boto.exception.EC2ResponseError, e: - err = "Updating status of volume(s) from cloud failed for UCI '"+ uci.name + \ - "' during general status update: " + str( e ) - log.error( err ) - uci.error = err - uci.state = uci_states.ERROR - self.sa_session.add( uci ) - self.sa_session.flush() - return None - else: - err = "No storage volumes returned by cloud provider on general update for volume with id: " + store.volume_id - log.error( "%s for UCI '%s'" % ( err, uci.name ) ) - store.status = store_status.ERROR - store.error = err - uci.error = err - uci.state = uci_states.ERROR - self.sa_session.add( uci ) - self.sa_session.add( store ) - self.sa_session.flush() - - def update_snapshot( self, snapshot ): - """ - Update information in local database for given snapshot as it is obtained from cloud provider. - Along with updating information about given snapshot, information about the UCI controlling - this snapshot is also updated. - """ - # Get credentials associated wit this store - uci_id = snapshot.uci_id - uci = self.sa_session.query( model.UCI ).get( uci_id ) - self.sa_session.refresh( uci ) - conn = self.get_connection_from_uci( uci ) - - try: - log.debug( "Updating status of snapshot '%s'" % snapshot.snapshot_id ) - snap = conn.get_all_snapshots( [snapshot.snapshot_id] ) - if len( snap ) > 0: - log.debug( "Snapshot '%s' status: %s" % ( snapshot.snapshot_id, snap[0].status ) ) - snapshot.status = snap[0].status - self.sa_session.add( snapshot ) - self.sa_session.flush() - else: - err = "No snapshots returned by EC2 on general update" - log.error( "%s for UCI '%s'" % ( err, uci.name ) ) - snapshot.status = snapshot_status.ERROR - snapshot.error = err - uci.error = err - uci.state = uci_states.ERROR - self.sa_session.add( uci ) - self.sa_session.add( snapshot ) - self.sa_session.flush() - except boto.exception.EC2ResponseError, e: - err = "EC2 response error while updating snapshot status: " + str( e ) - log.error( err ) - snapshot.status = snapshot_status.ERROR - snapshot.error = err - uci.error = err - uci.state = uci_states.ERROR - self.sa_session.add( uci ) - self.sa_session.add( snapshot ) - self.sa_session.flush() - except Exception, ex: - err = "Error while updating snapshot status: " + str( ex ) - log.error( err ) - snapshot.status = snapshot_status.ERROR - snapshot.error = err - uci.error = err - uci.state = uci_states.ERROR - self.sa_session.add( uci ) - self.sa_session.add( snapshot ) - self.sa_session.flush() - - def delete_snapshot( self, snapshot ): - """ - Initiate deletion of given snapshot from cloud provider. - """ - if snapshot.status == snapshot_status.DELETE: - # Get credentials associated wit this store - uci_id = snapshot.uci_id - uci = self.sa_session.query( model.UCI ).get( uci_id ) - self.sa_session.refresh( uci ) - conn = self.get_connection_from_uci( uci ) - - try: - log.debug( "Deleting snapshot '%s'" % snapshot.snapshot_id ) - snap = conn.delete_snapshot( snapshot.snapshot_id ) - if snap == True: - snapshot.deleted = True - snapshot.status = snapshot_status.DELETED - self.sa_session.add( snapshot ) - self.sa_session.flush() - return snap - except boto.exception.EC2ResponseError, e: - err = "EC2 response error while deleting snapshot: " + str( e ) - log.error( err ) - snapshot.status = snapshot_status.ERROR - snapshot.error = err - uci.error = err - uci.state = uci_states.ERROR - self.sa_session.add( uci ) - self.sa_session.add( snapshot ) - self.sa_session.flush() - except Exception, ex: - err = "Error while deleting snapshot: " + str( ex ) - log.error( err ) - snapshot.status = snapshot_status.ERROR - snapshot.error = err - uci.error = err - uci.state = uci_states.ERROR - self.sa_session.add( uci ) - self.sa_session.add( snapshot ) - self.sa_session.flush() - else: - err = "Cannot delete snapshot '"+snapshot.snapshot_id+"' because its status is '"+snapshot.status+"'. Only snapshots with '" + \ - snapshot_status.COMPLETED+"' status can be deleted." - log.error( err ) - snapshot.error = err - self.sa_session.add( snapshot ) - self.sa_session.flush() - - def process_zombie( self, inst ): - """ - Attempt at discovering if starting a cloud instance was successful but local database was not updated - accordingly or if something else failed and instance was never started. Currently, no automatic - repairs are being attempted; instead, appropriate error messages are set. - """ - uci_id = inst.uci_id - uci = self.sa_session.query( model.UCI ).get( uci_id ) - self.sa_session.refresh( uci ) - - # Check if any instance-specific information was written to local DB; if 'yes', set instance and UCI's error message - # suggesting manual check. - if inst.launch_time != None or inst.reservation_id != None or inst.instance_id != None: - # Try to recover state - this is best-case effort, so if something does not work immediately, not - # recovery steps are attempted. Recovery is based on hope that instance_id is available in local DB; if not, - # report as error. - # Fields attempting to be recovered are: reservation_id, instance status, and launch_time - if inst.instance_id != None: - conn = self.get_connection_from_uci( uci ) - rl = conn.get_all_instances( [inst.instance_id] ) # reservation list - # Update local DB with relevant data from instance - if inst.reservation_id == None: - try: - inst.reservation_id = str(rl[0]).split(":")[1] - except: # something failed, so skip - pass - - try: - state = rl[0].instances[0].update() - inst.state = state - uci.state = state - self.sa_session.add( inst ) - self.sa_session.add( uci ) - self.sa_session.flush() - except: # something failed, so skip - pass - - if inst.launch_time == None: - try: - launch_time = self.format_time( rl[0].instances[0].launch_time ) - inst.launch_time = launch_time - self.sa_session.add( inst ) - self.sa_session.flush() - if inst.uci.launch_time == None: - uci.launch_time = launch_time - self.sa_session.add( uci ) - self.sa_session.flush() - except: # something failed, so skip - pass - else: - err = "Starting a machine instance (DB id: '"+str(inst.id)+"') associated with this UCI '" + str(inst.uci.name) + \ - "' seems to have failed. Because it appears that cloud instance might have gotten started, manual check is recommended." - inst.error = err - inst.state = instance_states.ERROR - inst.uci.error = err - inst.uci.state = uci_states.ERROR - log.error( err ) - self.sa_session.add( inst ) - self.sa_session.add( uci ) - self.sa_session.flush() - - else: #Instance most likely never got processed, so set error message suggesting user to try starting instance again. - err = "Starting a machine instance (DB id: '"+str(inst.id)+"') associated with this UCI '" + str(inst.uci.name) + \ - "' seems to have failed. Because it appears that cloud instance never got started, it should be safe to reset state and try " \ - "starting the instance again." - inst.error = err - inst.state = instance_states.ERROR - uci.error = err - uci.state = uci_states.ERROR - log.error( err ) - self.sa_session.add( inst ) - self.sa_session.add( uci ) - self.sa_session.flush() -# uw = UCIwrapper( inst.uci ) -# log.debug( "Try automatically re-submitting UCI '%s'." % uw.get_name() ) - - def get_connection_from_uci( self, uci ): - """ - Establish and return connection to cloud provider. Information needed to do so is obtained - directly from uci database object. - """ - log.debug( 'Establishing %s cloud connection' % self.type ) - a_key = uci.credentials.access_key - s_key = uci.credentials.secret_key - # Get connection - try: - region = RegionInfo( None, uci.credentials.provider.region_name, uci.credentials.provider.region_endpoint ) - conn = EC2Connection( aws_access_key_id=a_key, - aws_secret_access_key=s_key, - is_secure=uci.credentials.provider.is_secure, - region=region, - path=uci.credentials.provider.path ) - except boto.exception.EC2ResponseError, e: - err = "Establishing connection with cloud failed: " + str( e ) - log.error( err ) - uci.error = err - uci.state = uci_states.ERROR - self.sa_session.add( uci ) - self.sa_session.flush() - return None - - return conn - -# def updateUCI( self, uci ): -# """ -# Runs a global status update on all storage volumes and all instances that are -# associated with specified UCI -# """ -# conn = self.get_connection( uci ) -# -# # Update status of storage volumes -# vl = model.CloudStore.filter( model.CloudInstance.table.c.uci_id == uci.id ).all() -# vols = [] -# for v in vl: -# vols.append( v.volume_id ) -# try: -# volumes = conn.get_all_volumes( vols ) -# for i, v in enumerate( volumes ): -# uci.store[i].i_id = v.instance_id -# uci.store[i].status = v.status -# uci.store[i].device = v.device -# uci.store[i].flush() -# except: -# log.debug( "Error updating status of volume(s) associated with UCI '%s'. Status was not updated." % uci.name ) -# pass -# -# # Update status of instances -# il = model.CloudInstance.filter_by( uci_id=uci.id ).filter( model.CloudInstance.table.c.state != 'terminated' ).all() -# instanceList = [] -# for i in il: -# instanceList.append( i.instance_id ) -# log.debug( 'instanceList: %s' % instanceList ) -# try: -# reservations = conn.get_all_instances( instanceList ) -# for i, r in enumerate( reservations ): -# uci.instance[i].state = r.instances[0].update() -# log.debug('updating instance %s; status: %s' % ( uci.instance[i].instance_id, uci.instance[i].state ) ) -# uci.state = uci.instance[i].state -# uci.instance[i].public_dns = r.instances[0].dns_name -# uci.instance[i].private_dns = r.instances[0].private_dns_name -# uci.instance[i].flush() -# uci.flush() -# except: -# log.debug( "Error updating status of instances associated with UCI '%s'. Instance status was not updated." % uci.name ) -# pass - - # --------- Helper methods ------------ - - def format_time( self, time ): - dict = {'T':' ', 'Z':''} - for i, j in dict.iteritems(): - time = time.replace(i, j) - return time - --- a/templates/cloud/view_instance.mako +++ /dev/null @@ -1,142 +0,0 @@ -<%inherit file="/base.mako"/> -<%def name="title()">Live instance details</%def> - -<% - # Because of the one-to-many relationship between liveInstance (i.e., UCI) and actual instances, need to know - # which one is currently active. Because only one instance of UCI can be alive at any point in time, simply - # select the most recent one. - # TODO: Once individual UCI's will be able to start more than one instance, this will need to be fixed - #i_id = len(liveInstance.instance) - 1 -%> - -<h2>Live instance details</h2> - -%if liveInstance: - <ul class="manage-table-actions"> - <li> - <a class="action-button" href="${h.url_for( action='list' )}"> - <img src="${h.url_for('/static/images/silk/resultset_previous.png')}" /> - <span>Return to cloud management console</span> - </a> - </li> - </ul> - - <table class="mange-table colored" border="0" cellspacing="0" cellpadding="0" width="100%"> - <tr> - <td> Instance name: </td> - <td> - ${liveInstance.uci.name} - <a id="li-popup" class="popup-arrow" style="display: none;">▼</a> - </td> - <td> - <div popupmenu="li-popup"> - <a class="action-button" href="${h.url_for( action='rename_uci', id=trans.security.encode_id(liveInstance.uci.id) )}">Rename</a> - <a class="action-button" confirm="Are you sure you want to stop instance '${liveInstance.uci.name}'?" href="${h.url_for( action='stop', id=trans.security.encode_id(liveInstance.uci.id) )}">Stop</a> - </div> - </td> - </tr> - <tr> - <td> Date created: </td> - <td> ${str(liveInstance.uci.create_time)[:16]} - <% - context.write( ' UTC (' ) - context.write( str(h.date.distance_of_time_in_words (liveInstance.uci.create_time, h.date.datetime.utcnow() ) ) ) - %> ago) - </td> - </tr> - <tr> - <td> Alive since: </td> - <td> ${str(liveInstance.launch_time)[:16]} - <% - context.write( ' UTC (' ) - context.write( str(h.date.distance_of_time_in_words (liveInstance.launch_time, h.date.datetime.utcnow() ) ) ) - %> ago) - </td> - </tr> - %if liveInstance.instance_id != None: - <tr> - <td> Instance ID: </td> - <td> ${liveInstance.instance_id} </td> - </tr> - %endif - %if liveInstance.reservation_id != None: - <tr> - <td> Reservation ID: </td> - <td> ${liveInstance.reservation_id} </td> - </tr> - %endif - <tr> - <td> AMI: </td> - <td> ${liveInstance.image.image_id} </td> - </tr> - <tr> - <td> State:</td> - <td> ${liveInstance.state} </td> - </tr> - <tr> - <td> Type:</td> - <td> ${liveInstance.type} </td> - </tr> - <tr> - <td> Storage size:</td> - <td> ${liveInstance.uci.total_size} </td> - </tr> - %if liveInstance.public_dns != None and liveInstance.public_dns != '': - <tr> - <td> Public DNS:</td> - <% - lnk="http://"+str(liveInstance.public_dns) - %> - <td><a href="${lnk}" target="_blank">${liveInstance.public_dns}</a></td> - </tr> - %endif - %if liveInstance.private_dns != None and liveInstance.private_dns != '': - <tr> - <td> Private DNS:</td> - <td> ${liveInstance.private_dns}</td> - </tr> - %endif - %if liveInstance.security_group != None: - <tr> - <td> Security group zone:</td> - <td> ${liveInstance.security_group} </td> - </tr> - %endif - %if liveInstance.availability_zone != None: - <tr> - <td> Availabilty zone:</td> - <td> ${liveInstance.availability_zone} </td> - </tr> - %endif - %if liveInstance.uci.key_pair_name != None: - <tr> - <td> Keypair file name:</td> - <td> ${liveInstance.uci.key_pair_name} </td> - </tr> - %endif - %if liveInstance.uci.key_pair_material != None: - <tr> - <td> Keypair material:</td> - <td> - <div id="short"> - <a onclick="document.getElementById('full').style.display = 'block'; - document.getElementById('short').style.display = 'none'; return 0" - href="javascript:void(0)"> - + Show - </a> - </div> - <div id="full" style="DISPLAY: none"> - <a onclick="document.getElementById('short').style.display = 'block'; - document.getElementById('full').style.display = 'none'; return 0;" - href="javascript:void(0)"> - - Hide</a> - ${liveInstance.uci.key_pair_material}<br/> - </div> - </td> - </tr> - %endif - - </table> -%else: - There is no live instance under that name. -%endif --- a/templates/webapps/galaxy/admin/index.mako +++ b/templates/webapps/galaxy/admin/index.mako @@ -118,14 +118,6 @@ <div class="toolTitle"><a href="${h.url_for( controller='requests_admin', action='list')}" target="galaxy_main">Manage requests</a></div></div></div> - <div class="toolSectionTitle"> - <span>Cloud</span> - </div> - <div class="toolSectionBody"> - <div class="toolSectionBg"> - <div class="toolTitle"><a href="${h.url_for( controller='cloud', action='list_machine_images' )}" target="galaxy_main">Manage machine images</a></div> - </div> - </div></div></div></div> --- a/lib/galaxy/web/controllers/cloud.py +++ /dev/null @@ -1,1276 +0,0 @@ -from galaxy.web.base.controller import * - -import pkg_resources -pkg_resources.require( "simplejson" ) -import simplejson -import urllib2 - -from galaxy.tools.parameters import * -from galaxy.tools import DefaultToolState -from galaxy.tools.parameters.grouping import Repeat, Conditional -from galaxy.datatypes.data import Data -from galaxy.util.odict import odict -from galaxy.util.bunch import Bunch -from galaxy.util.topsort import topsort, topsort_levels, CycleError -from galaxy.model.mapping import desc -from galaxy.model.orm import * -from datetime import datetime, timedelta - -pkg_resources.require( "WebHelpers" ) -from webhelpers import * - -# Required for Cloud tab -import galaxy.eggs -galaxy.eggs.require("boto") -from boto.ec2.connection import EC2Connection -from boto.ec2.regioninfo import RegionInfo -from galaxy.cloud import CloudManager -import boto.exception -import boto - -import logging -log = logging.getLogger( __name__ ) - -uci_states = Bunch( - NEW_UCI = "newUCI", - NEW = "new", - CREATING = "creating", - DELETING_UCI = "deletingUCI", - DELETING = "deleting", - SUBMITTED_UCI = "submittedUCI", - SUBMITTED = "submitted", - SHUTTING_DOWN_UCI = "shutting-downUCI", - SHUTTING_DOWN = "shutting-down", - ADD_STORAGE_UCI = "add-storageUCI", - ADD_STORAGE = "add-storage", - AVAILABLE = "available", - RUNNING = "running", - PENDING = "pending", - ERROR = "error", - DELETED = "deleted", - SNAPSHOT_UCI = "snapshotUCI", - SNAPSHOT = "snapshot" -) - -instance_states = Bunch( - TERMINATED = "terminated", - SUBMITTED = "submitted", - RUNNING = "running", - ADDING = "adding-storage", - PENDING = "pending", - SHUTTING_DOWN = "shutting-down", - ERROR = "error" -) - -store_status = Bunch( - WAITING = "waiting", - IN_USE = "in-use", - ADDING = "adding", - CREATING = "creating", - DELETED = 'deleted', - ERROR = "error" -) - -snapshot_status = Bunch( - SUBMITTED = 'submitted', - PENDING = 'pending', - COMPLETED = 'completed', - DELETE = 'delete', - DELETED= 'deleted', - ERROR = "error" -) - -class CloudController( BaseController ): - - @web.expose - def index( self, trans ): - return trans.fill_template( "cloud/index.mako" ) - - @web.expose - @web.require_login( "use Galaxy cloud" ) - def list( self, trans ): - """ - Render cloud main page (management of cloud resources) - """ - user = trans.get_user() - - cloudCredentials = trans.sa_session.query( model.CloudUserCredentials ) \ - .filter_by( user=user ) \ - .filter( model.CloudUserCredentials.table.c.deleted != True ) \ - .order_by( model.CloudUserCredentials.table.c.name ) \ - .all() - - cloudProviders = trans.sa_session.query( model.CloudProvider ) \ - .filter_by( user=user ) \ - .filter( model.CloudProvider.table.c.deleted != True ) \ - .order_by( model.CloudProvider.table.c.name ) \ - .all() - - liveInstances = trans.sa_session.query( model.UCI ) \ - .filter_by( user=user ) \ - .filter( or_( model.UCI.table.c.state==uci_states.RUNNING, - model.UCI.table.c.state==uci_states.PENDING, - model.UCI.table.c.state==uci_states.SUBMITTED, - model.UCI.table.c.state==uci_states.SUBMITTED_UCI, - model.UCI.table.c.state==uci_states.SHUTTING_DOWN, - model.UCI.table.c.state==uci_states.SHUTTING_DOWN_UCI, - model.UCI.table.c.state==uci_states.ADD_STORAGE, - model.UCI.table.c.state==uci_states.ADD_STORAGE_UCI ) ) \ - .order_by( desc( model.UCI.table.c.update_time ) ) \ - .all() - - prevInstances = trans.sa_session.query( model.UCI ) \ - .filter_by( user=user, deleted=False ) \ - .filter( or_( model.UCI.table.c.state==uci_states.AVAILABLE, - model.UCI.table.c.state==uci_states.NEW, - model.UCI.table.c.state==uci_states.NEW_UCI, - model.UCI.table.c.state==uci_states.CREATING, - model.UCI.table.c.state==uci_states.ERROR, - model.UCI.table.c.state==uci_states.DELETED, - model.UCI.table.c.state==uci_states.DELETING, - model.UCI.table.c.state==uci_states.DELETING_UCI, - model.UCI.table.c.state==uci_states.SNAPSHOT, - model.UCI.table.c.state==uci_states.SNAPSHOT_UCI ) ) \ - .order_by( desc( model.UCI.table.c.update_time ) ) \ - .all() - - # Check after update there are instances in pending state; if so, display message - pendingInstances = trans.sa_session.query( model.UCI ) \ - .filter_by( user=user ) \ - .filter( or_( model.UCI.table.c.state==uci_states.PENDING, - model.UCI.table.c.state==uci_states.SUBMITTED, - model.UCI.table.c.state==uci_states.SUBMITTED_UCI ) ) \ - .all() - if pendingInstances: - trans.set_message( "Galaxy instance started. NOTE: Please wait about 5 minutes for the instance to " - "start up. A button to connect to the instance will appear alongside " - "instance description once cloud instance of Galaxy is ready." ) - -# log.debug( "provider.is_secure: '%s'" % trans.sa_session.query( model.CloudProvider).filter_by(id=1).first().is_secure ) -# trans.sa_session.query( model.CloudProvider).filter_by(id=1).first().is_secure=False -# trans.sa_session.flush() -# log.debug( "provider.is_secure: '%s'" % trans.sa_session.query( model.CloudProvider).filter_by(id=1).first().is_secure ) - -# log.debug( "image: '%s'" % model.CloudImage.is_secure ) - - return trans.fill_template( "cloud/configure_cloud.mako", - cloudCredentials = cloudCredentials, - liveInstances = liveInstances, - prevInstances = prevInstances, - cloudProviders = cloudProviders ) - - # ----- UCI methods ----- - - @web.expose - @web.require_login( "use Galaxy cloud" ) - def configure_new_uci( self, trans, instanceName='', credName='', volSize='', zone='' ): - """ - Configure and add new cloud instance to user's instance pool - """ - inst_error = vol_error = cred_error = None - error = {} - user = trans.get_user() - - if instanceName: - # Check if volume size is entered as an integer - try: - volSize = int( volSize ) - except ValueError: - error['vol_error'] = "Volume size must be integer value between 1 and 1000." - - # Create new user configured instance - try: - if trans.sa_session.query( model.UCI ) \ - .filter_by (user=user, deleted=False, name=instanceName ) \ - .first(): - error['inst_error'] = "An instance with that name already exist." - elif instanceName=='' or len( instanceName ) > 255: - error['inst_error'] = "Instance name must be between 1 and 255 characters long." - elif credName=='': - error['cred_error'] = "You must select credentials." - elif volSize == '': - error['vol_error'] = "You must specify volume size as an integer value between 1 and 1000." - elif ( int( volSize ) < 1 ) or ( int( volSize ) > 1000 ): - error['vol_error'] = "Volume size must be integer value between 1 and 1000." - elif zone=='': - error['zone_error'] = "You must select a zone where this UCI will be registered." - else: - # Capture user configured instance information - uci = model.UCI() - uci.name = instanceName - creds = trans.sa_session.query( model.CloudUserCredentials ) \ - .filter( model.CloudUserCredentials.table.c.name==credName ).first() - uci.credentials = creds - uci.user= user - uci.total_size = volSize # This is OK now because new instance is being created and only one storage volume can be created at UCI creation time - uci.state = uci_states.NEW_UCI - - storage = model.CloudStore() - storage.user = user - storage.uci = uci - storage.size = volSize - # If '(any)' zone was selected, just choose the first one that's available - if zone == "(any)": - zones = None - conn = get_connection( trans, creds ) - if conn != None: - try: - zones = conn.get_all_zones() - if len( zones ) > 0: - zone = str( zones[0] ).split(':')[1] - except boto.exception.EC2ResponseError, e: - log.error( "Retrieving zones for credentials '%s' failed: %s" % ( storedCred.name, e ) ) - providersToZones[storedCred.name] = [ "Retrieving zones failed: " + str( e ) ] - storage.availability_zone = zone - storage.status = store_status.ADDING - # Persist - session = trans.sa_session - session.add( uci ) - session.add( storage ) - session.flush() - # Log and display the management page - trans.log_event( "User configured new cloud instance: '%s'" % instanceName ) - trans.set_message( "New Galaxy instance '%s' configured. Once instance status shows 'available' you will be able to start the instance." % instanceName ) - return self.list( trans ) - except AttributeError, ae: - inst_error = "No registered cloud images. You must contact administrator to add some before proceeding." - log.debug("AttributeError when registering new UCI '%s': %s " % ( instanceName, str( ae ) ) ) - else: - storedCreds = trans.sa_session.query( model.CloudUserCredentials ).filter_by( user=user, deleted=False ).all() - if len( storedCreds ) == 0: - return trans.show_error_message( "You must register credentials before configuring a Galaxy cloud instance." ) - # Create dict mapping of cloud-providers-to-zones available by those providers - providersToZones = {} - for storedCred in storedCreds: - zones = None - conn = get_connection( trans, storedCred ) - if conn != None: - avail_zones = [] - try: - zones = conn.get_all_zones() - if len( zones ) > 0: - avail_zones.append( "(any)" ) - for z in zones: - z = str( z ).split(':')[1] - avail_zones.append( z ) - providersToZones[storedCred.name] = avail_zones - except boto.exception.EC2ResponseError, e: - log.error( "Retrieving zones for credentials '%s' failed: %s" % ( storedCred.name, e ) ) - providersToZones[storedCred.name] = [ "Retrieving zones failed: " + str( e ) ] - else: - providersToZones[storedCred.name] = ['Connection with cloud provider could not be established.'] - - # Hard-coded solution - # if storedCred.provider.storedCred.provider.region_name == 'us-east-1': - # ec2_zones = ['us-east-1a', 'us-east-1b', 'us-east-1c', 'us-east-1d'] - # providersToZones[storedCred.name] = ec2_zones - # elif storedCred.provider.region_name == 'eu-west-1': - # ec2_zones = ['eu-west-1a', 'eu-west-1b'] - # providersToZones[storedCred.name] = ec2_zones - # elif storedCred.provider.type == 'eucalyptus': - # providersToZones[storedCred.name] = ['epc'] - # else: - # providersToZones[storedCred.name] = ['Unknown provider zone'] - - return trans.fill_template( "cloud/configure_uci.mako", - instanceName = instanceName, - credName = storedCreds, - volSize = volSize, - zone = zone, - error = error, - providersToZones = providersToZones ) - - @web.expose - @web.require_login( "start Galaxy cloud instance" ) - def start( self, trans, id, type='m1.small' ): - """ - Start a new cloud resource instance - """ - user = trans.get_user() - uci = get_uci( trans, id ) - stores = get_stores( trans, uci ) - # Ensure instance is available and then store relevant data - # into DB to initiate instance startup by cloud manager - if ( len(stores) is not 0 ) and ( uci.state == uci_states.AVAILABLE ): - instance = model.CloudInstance() - instance.user = user - instance.uci = uci - instance.state = instance_states.SUBMITTED - instance.availability_zone = stores[0].availability_zone # Bc. all EBS volumes need to be in the same avail. zone, just check 1st - instance.type = type - uci.state = uci_states.SUBMITTED_UCI - # Persist - session = trans.sa_session - session.add( instance ) - session.add( uci ) - session.flush() - # Log - trans.log_event ("User initiated starting of UCI '%s'." % uci.name ) - trans.set_message( "Galaxy instance started. NOTE: Please wait about 5 minutes for the instance to " - "start up. A button to connect to the instance will appear alongside " - "instance description once cloud instance of Galaxy is ready." ) - return self.list( trans ) - - if len(stores) == 0: - error( "This instance does not have any storage volumes associated it and thus cannot be started." ) - else: - error( "Cannot start instance that is in state '%s'." % uci.state ) - return self.list( trans ) - - @web.expose - @web.require_login( "stop Galaxy cloud instance" ) - def stop( self, trans, id ): - """ - Stop a cloud UCI instance. - """ - uci = get_uci( trans, id ) - if ( uci.state != uci_states.DELETING ) and \ - ( uci.state != uci_states.DELETING_UCI ) and \ - ( uci.state != uci_states.ERROR ) and \ - ( uci.state != uci_states.SHUTTING_DOWN_UCI ) and \ - ( uci.state != uci_states.SHUTTING_DOWN ) and \ - ( uci.state != uci_states.ADD_STORAGE_UCI ) and \ - ( uci.state != uci_states.ADD_STORAGE ) and \ - ( uci.state != uci_states.AVAILABLE ): - uci.state = uci_states.SHUTTING_DOWN_UCI - session = trans.sa_session - session.add( uci ) - session.flush() - trans.log_event( "User stopped cloud instance '%s' (id: %s)" % ( uci.name, uci.id ) ) - trans.set_message( "Stopping of Galaxy instance '%s' initiated." % uci.name ) - - return self.list( trans ) - - trans.show_error_message( "Cannot stop instance that is in state '%s'." % uci.state ) - return self.list( trans ) - - @web.expose - @web.require_login( "use Galaxy cloud" ) - def set_uci_state( self, trans, id, state='available', clear_error=True ): - """ - Sets state of UCI to given state, optionally resets error field, and resets UCI's launch time field to 'None'. - """ - uci = get_uci( trans, id ) - uci.state = state - if clear_error: - uci.error = None - uci.launch_time = None - trans.sa_session.flush() - trans.set_message( "Instance '%s' state reset." % uci.name ) - return self.list( trans ) - - @web.expose - @web.require_login( "view instance details" ) - def view_uci_details( self, trans, id=None ): - """ - View details about running instance - """ - uci = get_uci( trans, id ) - instances = get_instances( trans, uci ) # TODO: Handle list (will probably need to be done in mako template) - - return trans.fill_template( "cloud/view_instance.mako", - liveInstance = instances ) - - @web.expose - @web.require_login( "use Galaxy cloud" ) - def rename_uci( self, trans, id, new_name=None ): - instance = get_uci( trans, id ) - if new_name is not None: - if len(new_name) > 255: - error( "Instance name must be less than 255 characters long." ) - user = trans.get_user() - name_exists = trans.sa_session.query( model.UCI ) \ - .filter_by( user=user, name=new_name ) \ - .first() - if name_exists: - error( "Specified name ('%s') is already used by an existing instance. Please choose an alternative name." % new_name ) - - # Update name in local DB - instance.name = new_name - trans.sa_session.flush() - trans.set_message( "Instance renamed to '%s'." % new_name ) - return self.list( trans ) - else: - return trans.show_form( - web.FormBuilder( url_for( id=trans.security.encode_id(instance.id) ), "Rename instance", submit_text="Rename" ) - .add_text( "new_name", "Instance name", value=instance.name ) ) - - @web.expose - @web.require_login( "use Galaxy cloud" ) - def uci_usage_report( self, trans, id ): - user = trans.get_user() - id = trans.security.decode_id( id ) - - prevInstances = trans.sa_session.query( model.CloudInstance ) \ - .filter_by( user=user, state=instance_states.TERMINATED, uci_id=id ) \ - .order_by( desc( model.CloudInstance.table.c.update_time ) ) \ - .all() - - return trans.fill_template( "cloud/view_usage.mako", prevInstances = prevInstances ) - - @web.expose - @web.require_login( "delete user configured Galaxy cloud instance" ) - def delete_uci( self, trans, id ): - """ - Deletes User Configured Instance (UCI) from the cloud and local database. NOTE that this implies deletion of - any and all storage associated with this UCI! - """ - uci = get_uci( trans, id ) - - if ( uci.state != uci_states.DELETING_UCI ) and ( uci.state != uci_states.DELETING ) and ( uci.state != uci_states.ERROR ): - name = uci.name - uci.state = uci_states.DELETING_UCI - session = trans.sa_session - session.add( uci ) - session.flush() - trans.log_event( "User marked cloud instance '%s' for deletion." % name ) - trans.set_message( "Galaxy instance '%s' marked for deletion." % name ) - return self.list( trans ) - - if uci.state != uci_states.ERROR: - trans.set_message( "Cannot delete instance in state ERROR." ) - else: - trans.set_message( "Instance '%s' is already marked for deletion." % uci.name ) - return self.list( trans ) - - # ----- Snapshot methods ----- - - @web.expose - @web.require_login( "use Galaxy cloud" ) - def create_snapshot( self, trans, id ): - user = trans.get_user() - id = trans.security.decode_id( id ) - uci = get_uci( trans, id ) - - stores = trans.sa_session.query( model.CloudStore ) \ - .filter_by( user=user, deleted=False, uci_id=id ) \ - .all() - - if ( len( stores ) > 0 ) and ( uci.state == uci_states.AVAILABLE ): - for store in stores: - snapshot = model.CloudSnapshot() - snapshot.user = user - snapshot.uci = uci - snapshot.store = store - snapshot.status = snapshot_status.SUBMITTED - uci.state = uci_states.SNAPSHOT_UCI - # Persist - session = trans.sa_session - session.add( snapshot ) - session.add( uci ) - session.flush() - elif len( stores ) == 0: - error( "No storage volumes found that are associated with this instance." ) - else: - error( "Snapshot can be created only for an instance that is in 'available' state." ) - - # Log and display the management page - trans.log_event( "User initiated creation of new snapshot." ) - trans.set_message( "Creation of new snapshot initiated. " ) - return self.list( trans ) - - @web.expose - @web.require_login( "use Galaxy cloud" ) - def view_snapshots( self, trans, id=None ): - """ - View details about any snapshots associated with given UCI - """ - user = trans.get_user() - id = trans.security.decode_id( id ) - - snaps = trans.sa_session.query( model.CloudSnapshot ) \ - .filter_by( user=user, uci_id=id, deleted=False ) \ - .order_by( desc( model.CloudSnapshot.table.c.update_time ) ) \ - .all() - - return trans.fill_template( "cloud/view_snapshots.mako", - snaps = snaps ) - - @web.expose - @web.require_login( "use Galaxy cloud" ) - def delete_snapshot( self, trans, uci_id=None, snap_id=None ): - """ - Initiates deletion of a snapshot - """ - user = trans.get_user() - snap_id = trans.security.decode_id( snap_id ) - # Set snapshot as 'ready for deletion' to be picked up by general updater - snap = trans.sa_session.query( model.CloudSnapshot ).get( snap_id ) - - if snap.status == snapshot_status.COMPLETED: - snap.status = snapshot_status.DELETE - trans.sa_session.add( snap ) - trans.sa_session.flush() - trans.set_message( "Snapshot '%s' is marked for deletion. Once the deletion is complete, it will no longer be visible in this list. " - "Please note that this process may take up to a minute." % snap.snapshot_id ) - else: - error( "Only snapshots in state 'completed' can be deleted. See the cloud provider directly " - "if you believe the snapshot is available and can be deleted." ) - - # Display new list of snapshots - uci_id = trans.security.decode_id( uci_id ) - snaps = trans.sa_session.query( model.CloudSnapshot ) \ - .filter_by( user=user, uci_id=uci_id, deleted=False ) \ - .order_by( desc( model.CloudSnapshot.table.c.update_time ) ) \ - .all() - - return trans.fill_template( "cloud/view_snapshots.mako", - snaps = snaps ) - - # ----- Storage methods ----- - - @web.expose - @web.require_login( "add instance storage" ) - def add_storage( self, trans, id, vol_size=None ): - error = None - uci = get_uci( trans, id ) - stores = get_stores_in_status( trans, uci, store_status.IN_USE ) - - # Start adding of storage making sure given UCI is running and that at least one - # storage volume is attached to it (this is needed to by cloud controller to know - # as which device to attach the new storage volume) - if uci.state == uci_states.RUNNING and len( stores ) > 0: - if vol_size is not None: - try: - vol_size = int( vol_size ) - except ValueError: - error = "Volume size must be integer value between 1 and 1000." - - if not error: - user = trans.get_user() - - storage = model.CloudStore() - storage.user = user - storage.uci = uci - storage.size = vol_size - storage.status = store_status.ADDING - - # Set state of instance - NOTE that this code will only work (with code in cloud controller) - # for scenario where a UCI is associated with *1* compute instance!!! - instances = get_instances( trans, uci ) - instances.state = instance_states.ADDING - - uci.state = uci_states.ADD_STORAGE_UCI - # Persist - session = trans.sa_session - session.add( instances ) - session.add( storage ) - session.add( uci ) - session.flush() - # Log and display the management page - trans.log_event( "User added storage volume to UCI: '%s'" % uci.name ) - trans.set_message( "Adding of storage to instance '%s' initiated." % uci.name ) - return self.list( trans ) - else: - error( "Storage can only be added to instances that are in state 'RUNNING' with existing " \ - "storage volume(s) already attached." ) - - return trans.show_form( - web.FormBuilder( url_for( id=trans.security.encode_id(uci.id) ), "Add storage to an instance", submit_text="Add" ) - .add_text( "vol_size", "Storage size (1-1000 GB)", value='', error=error ) ) - - # ----- Image methods ----- - @web.expose - @web.require_admin - def add_new_image( self, trans, provider_type='', image_id='', manifest='', architecture='', state=None ): - #id_error = arch_error = provider_error = manifest_error = None - error = {} - if provider_type or image_id or manifest or architecture: - if provider_type=='': - error['provider_error'] = "You must select cloud provider type for this machine image." - elif image_id=='' or len( image_id ) > 255: - error['id_error'] = "Image ID must be between 1 and 255 characters long." - elif trans.sa_session.query( model.CloudImage ) \ - .filter_by( deleted=False ) \ - .filter( model.CloudImage.table.c.image_id == image_id ) \ - .first(): - error['id_error'] = "Image with ID '" + image_id + "' is already registered. \ - Please choose another ID." - elif architecture=='': - error['arch_error'] = "You must select architecture type for this machine image." - else: - # Create new image - image = model.CloudImage() - image.provider_type = provider_type - image.image_id = image_id - image.manifest = manifest - image.architecture = architecture - # Persist - session = trans.sa_session - session.add( image ) - session.flush() - # Log and display the management page - trans.log_event( "New cloud image added: '%s'" % image.image_id ) - trans.set_message( "Cloud image '%s' added." % image.image_id ) - if state: - image.state = state - images = trans.sa_session.query( model.CloudImage ).all() - return trans.fill_template( '/cloud/list_images.mako', images=images ) - - return trans.fill_template( "cloud/add_image.mako", - provider_type = provider_type, - image_id = image_id, - manifest = manifest, - architecture = architecture, - error = error ) -# return trans.show_form( -# web.FormBuilder( web.url_for(), "Add new cloud image", submit_text="Add" ) -# .add_text( "provider_type", "Provider type", value='ec2 or eucalyptus', error=provider_error ) -# .add_text( "image_id", "Machine Image ID (AMI or EMI)", value='', error=id_error ) -# .add_text( "manifest", "Manifest", value='', error=manifest_error ) -# .add_text( "architecture", "Architecture", value='i386 or x86_64', error=arch_error ) ) - - @web.expose - @web.require_login( "use Galaxy cloud" ) - def list_machine_images( self, trans ): - images = trans.sa_session.query( model.CloudImage ).filter_by( deleted=False ).all() - return trans.fill_template( '/cloud/list_images.mako', images=images ) - - @web.expose - @web.require_admin - def delete_image( self, trans, id=None ): - if not isinstance( id, int ): - id = trans.security.decode_id( id ) - - image = trans.sa_session.query( model.CloudImage ).get( id ) - image.deleted = True - trans.sa_session.add( image ) - trans.sa_session.flush() - return self.list_machine_images( trans ) - - @web.expose - @web.require_admin - def edit_image( self, trans, provider_type='', image_id='', manifest='', architecture='', id='', edited=False ): - error = {} - if not isinstance( id, int ): - id = trans.security.decode_id( id ) - - if not edited: - image = trans.sa_session.query( model.CloudImage ).get( id ) - return trans.fill_template( "cloud/edit_image.mako", - image = image, - error = error - ) - else: - image = trans.sa_session.query( model.CloudImage ).get( id ) - if image_id=='' or len( image_id ) > 255: - error['id_error'] = "Image ID must be between 1 and 255 characters in length." - elif trans.sa_session.query( model.CloudImage ) \ - .filter_by( deleted=False ) \ - .filter( and_( model.CloudImage.table.c.id != image.id, model.CloudImage.table.c.image_id==image_id ) ) \ - .first(): - error['id_error'] = "Image with ID '" + image_id + "' already exist. Please choose an alternative name." - elif architecture=='' or len( architecture ) > 255: - error['arch_error'] = "Architecture type must be between 1 and 255 characters long." - if error: - return trans.fill_template( "cloud/edit_image.mako", - image = image, - error = error - ) - else: - image.image_id = image_id - image.manifest = manifest - image.architecture = architecture - # Persist - session = trans.sa_session - session.add( image ) - session.flush() - # Log and display the management page - trans.set_message( "Machine image '%s' edited." % image.image_id ) - return self.list_machine_images( trans ) - - # ----- Credentials methods ----- - - @web.expose - @web.require_login( "add credentials" ) - def add_credentials( self, trans, credName='', accessKey='', secretKey='', providerName='' ): - """ - Add user's cloud credentials stored under name `credName`. - """ - user = trans.get_user() - error = {} - - if credName or providerName or accessKey or secretKey: - if credName=='' or len( credName ) > 255: - error['cred_error'] = "Credentials name must be between 1 and 255 characters in length." - elif trans.sa_session.query( model.CloudUserCredentials ) \ - .filter_by( user=user, deleted=False ) \ - .filter( model.CloudUserCredentials.table.c.name == credName ) \ - .first(): - error['cred_error'] = "Credentials with that name already exist." - elif providerName=='': - error['provider_error'] = "You must select cloud provider associated with these credentials." - elif accessKey=='' or len( accessKey ) > 255: - error['access_key_error'] = "Access key must be between 1 and 255 characters long." - elif secretKey=='' or len( secretKey ) > 255: - error['secret_key_error'] = "Secret key must be between 1 and 255 characters long." - else: - # Create new user stored credentials - credentials = model.CloudUserCredentials() - credentials.name = credName - credentials.user = user - credentials.access_key = accessKey - credentials.secret_key = secretKey - provider = get_provider( trans, providerName ) - credentials.provider = provider - # Persist - session = trans.sa_session - session.add( credentials ) - session.flush() - # Log and display the management page - trans.log_event( "User added new credentials" ) - trans.set_message( "Credential '%s' created" % credentials.name ) - return self.list( trans ) - - providers = trans.sa_session.query( model.CloudProvider ).filter_by( user=user ).all() - return trans.fill_template( "cloud/add_credentials.mako", - credName = credName, - providerName = providerName, - accessKey = accessKey, - secretKey = secretKey, - error = error, - providers = providers - ) - - @web.expose - @web.require_login( "use Galaxy cloud" ) - def edit_credentials( self, trans, id, credName=None, accessKey=None, secretKey=None, edited=False ): - error = {} - if not edited: - credentials = get_stored_credentials( trans, id ) - return trans.fill_template( "cloud/edit_credentials.mako", - credential = credentials, - error = error - ) - else: - user = trans.get_user() - credentials = get_stored_credentials( trans, id ) - if credName=='' or len( credName ) > 255: - error['cred_error'] = "Credentials name must be between 1 and 255 characters in length." - elif trans.sa_session.query( model.CloudUserCredentials ) \ - .filter_by( user=user ) \ - .filter( and_( model.CloudUserCredentials.table.c.id != credentials.id, model.CloudUserCredentials.table.c.name==credName ) ) \ - .first(): - error['cred_error'] = "Credentials with name '" + credName + "' already exist. Please choose an alternative name." - elif accessKey=='' or len( accessKey ) > 255: - error['access_key_error'] = "Access key must be between 1 and 255 characters long." - elif secretKey=='' or len( secretKey ) > 255: - error['secret_key_error'] = "Secret key must be between 1 and 255 characters long." - - if error: - return trans.fill_template( "cloud/edit_credentials.mako", - credential = credentials, - error = error - ) - else: - # Edit user stored credentials - credentials.name = credName - credentials.access_key = accessKey - credentials.secret_key = secretKey - # Persist - session = trans.sa_session - session.add( credentials ) - session.flush() - # Log and display the management page - trans.set_message( "Credential '%s' edited." % credentials.name ) - return self.list( trans ) - - @web.expose - @web.require_login( "view credentials" ) - def view_credentials( self, trans, id=None ): - """ - View details for user credentials - """ - # Load credentials from database - stored = get_stored_credentials( trans, id ) - - return trans.fill_template( "cloud/view_credentials.mako", - credDetails = stored ) - - @web.expose - @web.require_login( "test cloud credentials" ) - def test_cred( self, trans, id=None ): - """ - Tests credentials provided by user with selected cloud provider - """ - - @web.expose - @web.require_login( "delete credentials" ) - def delete_credentials( self, trans, id=None ): - """ - Delete user's cloud credentials checking that no registered instances are tied to given credentials. - """ - # Load credentials from database - user = trans.get_user() - stored = get_stored_credentials( trans, id ) - # Check if there are any UCIs that depend on these credentials - UCI = None - UCI = trans.sa_session.query( model.UCI ) \ - .filter_by( user=user, credentials_id=stored.id, deleted=False ) \ - .first() - - if UCI == None: - # Delete and save - stored.deleted = True - trans.sa_session.add( stored ) - trans.sa_session.flush() - # Display the management page - trans.set_message( "Credentials '%s' deleted." % stored.name ) - return self.list( trans ) - else: - error( "Existing instance(s) depend on credentials '%s'. You must delete those instances before being able \ - to delete these credentials." % stored.name ) - return self.list( trans ) - - # ----- Provider methods ----- - - @web.expose - @web.require_login( "add provider" ) - def add_provider( self, trans, name='', type='', region_name='', region_endpoint='', is_secure='', host='', port='', proxy='', proxy_port='', - proxy_user='', proxy_pass='', debug='', https_connection_factory='', path='' ): - user = trans.get_user() - error = {} - - if region_name or region_endpoint or name or is_secure or port or proxy or debug or path: - try: - is_secure = int(is_secure) - except ValueError: - error['is_secure_error'] = "Field 'is secure' can only take on an integer value '0' or '1'" - - if trans.sa_session.query( model.CloudProvider ) \ - .filter_by (user=user, name=name) \ - .filter( model.CloudProvider.table.c.deleted != True ) \ - .first(): - error['name_error'] = "A provider with that name already exist." - elif name=='' or len( name ) > 255: - error['name_error'] = "Provider name must be between 1 and 255 characters long." - elif type=='': - error['type_error'] = "Provider type must be selected." - elif not (is_secure == 0 or is_secure == 1): - error['is_secure_error'] = "Field 'is secure' can only take on an integer value '0' or '1'" - else: - provider = model.CloudProvider() - provider.user = user - provider.type = type - provider.name = name - if region_name: - provider.region_name = region_name - else: - provider.region_name = None - - if region_endpoint: - provider.region_endpoint = region_endpoint - else: - provider.region_endpoint = None - - if is_secure==0: - provider.is_secure = False - else: - provider.is_secure = True - - if host: - provider.host = host - else: - provider.host = None - - if port: - provider.port = port - else: - provider.port = None - - if proxy: - provider.proxy = proxy - else: - provider.proxy = None - - if proxy_port: - provider.proxy_port = proxy_port - else: - provider.proxy_port = None - - if proxy_user: - provider.proxy_user = proxy_user - else: - provider.proxy_user = None - - if proxy_pass: - provider.proxy_pass = proxy_pass - else: - provider.proxy_pass = None - - if debug: - provider.debug = debug - else: - provider.debug = None - - if https_connection_factory: - provider.https_connection_factory = https_connection_factory - else: - provider.https_connection_factory = None - - provider.path = path - # Persist - session = trans.sa_session - session.add( provider ) - session.flush() - # Log and display the management page - trans.log_event( "User configured new cloud provider: '%s'" % name ) - trans.set_message( "New cloud provider '%s' added." % name ) - return self.list( trans ) - - return trans.fill_template( "cloud/add_provider.mako", - name = name, - type = type, - region_name = region_name, - region_endpoint = region_endpoint, - is_secure = is_secure, - host = host, - port = port, - proxy = proxy, - proxy_port = proxy_port, - proxy_user = proxy_user, - proxy_pass = proxy_pass, - debug = debug, - https_connection_factory = https_connection_factory, - path = path, - error = error - ) - - @web.expose - @web.require_login( "add Amazon EC2 provider" ) - def add_ec2( self, trans ): - """ Default provider setup for Amazon's EC2. """ - self.add_provider( trans, name='Amazon EC2', type='ec2', region_name='us-east-1', region_endpoint='us-east-1.ec2.amazonaws.com', is_secure=1, path='/' ) - return self.add( trans ) - - @web.expose - @web.require_login( "use Galaxy cloud" ) - def view_provider( self, trans, id=None ): - """ - View details about given cloud provider - """ - # Load credentials from database - provider = get_provider_by_id( trans, id ) - - return trans.fill_template( "cloud/view_provider.mako", - provider = provider ) - - @web.expose - @web.require_login( "use Galaxy cloud" ) - def edit_provider( self, trans, id, name='', type='', region_name='', region_endpoint='', is_secure='', host='', port='', proxy='', proxy_port='', - proxy_user='', proxy_pass='', debug='', https_connection_factory='', path='', edited=False ): - error = {} - if edited == False: - provider = get_provider_by_id( trans, id ) - return trans.fill_template( "cloud/edit_provider.mako", - provider = provider, - error = error - ) - else: - user = trans.get_user() - provider = get_provider_by_id( trans, id ) - - try: - is_secure = int(is_secure) - except ValueError: - error['is_secure_error'] = "Field 'is secure' can only take on an integer value '0' or '1'" - - if name=='' or len( name ) > 255: - error['name_error'] = "Cloud provider name must be between 1 and 255 characters in length." - elif trans.sa_session.query( model.CloudProvider ) \ - .filter_by( user=user ) \ - .filter( and_( model.CloudProvider.table.c.id != provider.id, model.CloudProvider.table.c.name == name ) ) \ - .first(): - error['name_error'] = "Cloud provider with name '" + name + "' already exist. Please choose an alternative name." - elif not ( is_secure == 0 or is_secure == 1): - error['is_secure_error'] = "Field 'is secure' can only take on an integer value '0' or '1'" - - if error: - return trans.fill_template( "cloud/edit_provider.mako", - provider = provider, - error = error - ) - else: - provider.name = name - if region_name and region_name != 'None': - provider.region_name = region_name - else: - provider.region_name = None - - if region_endpoint and region_endpoint != 'None': - provider.region_endpoint = region_endpoint - else: - provider.region_endpoint = None - - if is_secure==0: - provider.is_secure = False - else: - provider.is_secure = True - - if host and host != 'None': - provider.host = host - else: - provider.host = None - - if port and port != 'None': - provider.port = port - else: - provider.port = None - - if proxy and proxy != 'None': - provider.proxy = proxy - else: - provider.proxy = None - - if proxy_port and proxy_port != 'None': - provider.proxy_port = proxy_port - else: - provider.proxy_port = None - - if proxy_user and proxy_user != 'None': - provider.proxy_user = proxy_user - else: - provider.proxy_user = None - - if proxy_pass and proxy_pass != 'None': - provider.proxy_pass = proxy_pass - else: - provider.proxy_pass = None - - if debug and debug != 'None': - provider.debug = debug - else: - provider.debug = None - - if https_connection_factory and https_connection_factory != 'None': - provider.https_connection_factory = https_connection_factory - else: - provider.https_connection_factory = None - - if path and path != 'None': - provider.path = path - else: - provider.path = None - # Persist - session = trans.sa_session - session.add( provider ) - session.flush() - # Log and display the management page - trans.log_event( "User edited cloud provider: '%s'" % name ) - trans.set_message( "Cloud provider '%s' edited." % name ) - return self.list( trans ) - - @web.expose - @web.require_login( "delete credentials" ) - def delete_provider( self, trans, id=None ): - """ - Delete use-registered cloud provider checking that no registered credentials are tied to given provider. - """ - # Load provider from database - user = trans.get_user() - provider = get_provider_by_id( trans, id ) - creds = trans.sa_session.query( model.CloudUserCredentials ) \ - .filter_by( user=user, provider_id=provider.id ) \ - .filter( model.CloudUserCredentials.table.c.deleted != True ) \ - .all() - - if len( creds ) == 0: - # Delete and save - #sess = trans.sa_session - provider.deleted = True - trans.sa_session.add( provider ) - trans.sa_session.flush() - # Display the management page - trans.set_message( "Cloud provider '%s' deleted." % provider.name ) - return self.list( trans ) - - error( "Existing credentails depend on cloud provider '%s'. You must delete those credentials before being able \ - to delete this cloud provider." % provider.name ) - return self.list( trans ) - - # ----- AJAX methods ----- - - @web.json - def json_update( self, trans ): - user = trans.get_user() - UCIs = trans.sa_session.query( model.UCI ).filter_by( user=user, deleted=False ).all() - insd = {} # instance name-state dict - for uci in UCIs: - dict = {} - dict['id'] = uci.id - dict['state'] = uci.state - dict['total_size'] = uci.total_size - if uci.error != None: - dict['error'] = str( uci.error ) - else: - dict['error'] = None - if uci.launch_time != None: - dict['launch_time'] = str( uci.launch_time ) - dict['time_ago'] = str( date.distance_of_time_in_words( uci.launch_time, date.datetime.utcnow() ) ) - else: - dict['launch_time'] = None - dict['time_ago'] = None - insd[uci.name] = dict - return insd - - @web.json - def link_update( self, trans, uci_id=0 ): - ild = {} # instance-link-dict - dict = {} - dict['uci_id'] = uci_id - try: - user = trans.get_user() - # TODO: This query can assumes only one instance under given UCI can be running (i.e., started). - inst = trans.sa_session.query( model.CloudInstance ).filter_by( user=user, uci_id=uci_id, state=uci_states.RUNNING ).first() - urllib2.urlopen( "http://" + inst.public_dns ) - dict['public_dns'] = inst.public_dns - dict['inst_id'] = inst.id - ild['data'] = dict - return ild - except urllib2.URLError: - dict['public_dns'] = False - ild['data'] = dict - return ild - -## ---- Utility methods ------------------------------------------------------- - -def get_provider( trans, name ): - user = trans.get_user() - return trans.sa_session.query( model.CloudProvider ) \ - .filter_by (user=user, name=name) \ - .first() - -def get_provider_by_id( trans, id, check_ownership=True ): - # Check if 'id' is in int (i.e., it was called from this program) or - # it was passed from the web (in which case decode it) - if not isinstance( id, int ): - id = trans.security.decode_id( id ) - - stored = trans.sa_session.query( model.CloudProvider ).get( id ) - if not stored: - error( "Cloud provider not found" ) - # Verify ownership - user = trans.get_user() - if not user: - error( "Must be logged in to use the cloud." ) - if check_ownership and not( stored.user == user ): - error( "Cloud provider '%s' is not registered by current user." % stored.name ) - # Looks good - return stored - -def get_stored_credentials( trans, id, check_ownership=True ): - """ - Get StoredUserCredentials from the database by id, verifying ownership. - """ - # Check if 'id' is in int (i.e., it was called from this program) or - # it was passed from the web (in which case decode it) - if not isinstance( id, int ): - id = trans.security.decode_id( id ) - - stored = trans.sa_session.query( model.CloudUserCredentials ).get( id ) - if not stored: - error( "Credentials not found" ) - # Verify ownership - user = trans.get_user() - if not user: - error( "Must be logged in to use the cloud." ) - if check_ownership and not( stored.user == user ): - error( "Credentials are not owned by current user." ) - # Looks good - return stored - -def get_uci( trans, id, check_ownership=True ): - """ - Get a UCI object from the database by id, verifying ownership. - """ - # Check if 'id' is in int (i.e., it was called from this program) or - # it was passed from the web (in which case decode it) - if not isinstance( id, int ): - id = trans.security.decode_id( id ) - - live = trans.sa_session.query( model.UCI ).get( id ) - if not live: - error( "Galaxy instance not found." ) - # Verify ownership - user = trans.get_user() - if not user: - error( "Must be logged in to use the cloud." ) - if check_ownership and not( live.user == user ): - error( "Instance is not owned by current user." ) - # Looks good - return live - -def get_stores( trans, uci ): - """ - Get stores objects that are associated with given uci and are not in 'error' status - """ - user = trans.get_user() - stores = trans.sa_session.query( model.CloudStore ) \ - .filter_by( user=user, uci_id=uci.id, deleted=False ) \ - .filter( model.CloudStore.table.c.status != store_status.ERROR ) \ - .all() - - return stores - -def get_stores_in_status( trans, uci, status ): - """ - Get stores objects that are associated with given uci and are not have given status - """ - user = trans.get_user() - stores = trans.sa_session.query( model.CloudStore ) \ - .filter_by( user=user, uci_id=uci.id, status=status ) \ - .all() - - return stores - -def get_instances( trans, uci ): - """ - Get objects of instances that are pending or running and are connected to the given uci object - """ - user = trans.get_user() - instances = trans.sa_session.query( model.CloudInstance ) \ - .filter_by( user=user, uci_id=uci.id ) \ - .filter( or_(model.CloudInstance.table.c.state==instance_states.RUNNING, model.CloudInstance.table.c.state==instance_states.PENDING ) ) \ - .first() - #.all() #TODO: return all but need to edit calling method(s) to handle list - - return instances - -def get_instances_in_state( trans, uci, state ): - """ - Get objects of instances that are in specified state and are connected to the given uci object - """ - user = trans.get_user() - instances = trans.sa_session.query( model.CloudInstance ) \ - .filter_by( user=user, uci_id=uci.id, state=state ) \ - .all() - - return instances - -def get_connection( trans, creds ): - """ - Establishes cloud connection using user's credentials - """ - log.debug( 'Establishing cloud connection.' ) -# user = trans.get_user() -# creds = trans.sa_session.query( model.CloudUserCredentials ) \ -# .filter_by( user=user, name=credName ) \ -# .first() - #.filter( model.CloudUserCredentials.table.c.deleted != True ) \ MOVE TO LINE ABOVE ONCE DELETE COLUMS ARE IMPLEMENTED - - if creds: - a_key = creds.access_key - s_key = creds.secret_key - try: - euca_region = RegionInfo( None, creds.provider.region_name, creds.provider.region_endpoint ) - conn = EC2Connection( aws_access_key_id=a_key, - aws_secret_access_key=s_key, - is_secure=creds.provider.is_secure, - port=creds.provider.port, - region=euca_region, - path=creds.provider.path ) - except boto.exception.EC2ResponseError, e: - log.error( "Establishing connection with cloud failed: %s" % str(e) ) - return None - - return conn --- /dev/null +++ b/lib/galaxy/model/migrate/versions/0050_drop_cloud_tables.py @@ -0,0 +1,147 @@ +from sqlalchemy import * +from migrate import * + +import datetime +now = datetime.datetime.utcnow + +# Need our custom types, but don't import anything else from model +from galaxy.model.custom_types import * + +import logging +log = logging.getLogger( __name__ ) + +metadata = MetaData( migrate_engine ) + +def display_migration_details(): + print + print "========================================" + print "This script drops tables that were associated with the old Galaxy Cloud functionality." + print "========================================" + +CloudImage_table = Table( "cloud_image", metadata, + Column( "id", Integer, primary_key=True ), + Column( "create_time", DateTime, default=now ), + Column( "update_time", DateTime, default=now, onupdate=now ), + Column( "provider_type", TEXT ), + Column( "image_id", TEXT, nullable=False ), + Column( "manifest", TEXT ), + Column( "state", TEXT ), + Column( "architecture", TEXT ), + Column( "deleted", Boolean, default=False ) ) + +""" UserConfiguredInstance (UCI) table """ +UCI_table = Table( "cloud_uci", metadata, + Column( "id", Integer, primary_key=True ), + Column( "create_time", DateTime, default=now ), + Column( "update_time", DateTime, default=now, onupdate=now ), + Column( "user_id", Integer, ForeignKey( "galaxy_user.id" ), index=True, nullable=False ), + Column( "credentials_id", Integer, ForeignKey( "cloud_user_credentials.id" ), index=True ), + Column( "key_pair_name", TEXT ), + Column( "key_pair_material", TEXT ), + Column( "name", TEXT ), + Column( "state", TEXT ), + Column( "error", TEXT ), + Column( "total_size", Integer ), + Column( "launch_time", DateTime ), + Column( "deleted", Boolean, default=False ) ) + +CloudInstance_table = Table( "cloud_instance", metadata, + Column( "id", Integer, primary_key=True ), + Column( "create_time", DateTime, default=now ), + Column( "update_time", DateTime, default=now, onupdate=now ), + Column( "launch_time", DateTime ), + Column( "stop_time", DateTime ), + Column( "user_id", Integer, ForeignKey( "galaxy_user.id" ), index=True, nullable=False ), + Column( "uci_id", Integer, ForeignKey( "cloud_uci.id" ), index=True ), + Column( "type", TEXT ), + Column( "reservation_id", TEXT ), + Column( "instance_id", TEXT ), + Column( "mi_id", Integer, ForeignKey( "cloud_image.id" ), index=True ), + Column( "state", TEXT ), + Column( "error", TEXT ), + Column( "public_dns", TEXT ), + Column( "private_dns", TEXT ), + Column( "security_group", TEXT ), + Column( "availability_zone", TEXT ) ) + +CloudStore_table = Table( "cloud_store", metadata, + Column( "id", Integer, primary_key=True ), + Column( "create_time", DateTime, default=now ), + Column( "update_time", DateTime, default=now, onupdate=now ), + Column( "attach_time", DateTime ), + Column( "user_id", Integer, ForeignKey( "galaxy_user.id" ), index=True, nullable=False ), + Column( "uci_id", Integer, ForeignKey( "cloud_uci.id" ), index=True, nullable=False ), + Column( "volume_id", TEXT ), + Column( "size", Integer, nullable=False ), + Column( "availability_zone", TEXT ), + Column( "inst_id", Integer, ForeignKey( "cloud_instance.id" ) ), + Column( "status", TEXT ), + Column( "device", TEXT ), + Column( "space_consumed", Integer ), + Column( "error", TEXT ), + Column( "deleted", Boolean, default=False ) ) + +CloudSnapshot_table = Table( "cloud_snapshot", metadata, + Column( "id", Integer, primary_key=True ), + Column( "create_time", DateTime, default=now ), + Column( "update_time", DateTime, default=now, onupdate=now ), + Column( "user_id", Integer, ForeignKey( "galaxy_user.id" ), index=True, nullable=False ), + Column( "uci_id", Integer, ForeignKey( "cloud_uci.id" ), index=True ), + Column( "store_id", Integer, ForeignKey( "cloud_store.id" ), index=True, nullable=False ), + Column( "snapshot_id", TEXT ), + Column( "status", TEXT ), + Column( "description", TEXT ), + Column( "error", TEXT ), + Column( "deleted", Boolean, default=False ) ) + +CloudUserCredentials_table = Table( "cloud_user_credentials", metadata, + Column( "id", Integer, primary_key=True ), + Column( "create_time", DateTime, default=now ), + Column( "update_time", DateTime, default=now, onupdate=now ), + Column( "user_id", Integer, ForeignKey( "galaxy_user.id" ), index=True, nullable=False ), + Column( "provider_id", Integer, ForeignKey( "cloud_provider.id" ), index=True, nullable=False ), + Column( "name", TEXT ), + Column( "access_key", TEXT ), + Column( "secret_key", TEXT ), + Column( "deleted", Boolean, default=False ) ) + +CloudProvider_table = Table( "cloud_provider", metadata, + Column( "id", Integer, primary_key=True ), + Column( "create_time", DateTime, default=now ), + Column( "update_time", DateTime, default=now, onupdate=now ), + Column( "user_id", Integer, ForeignKey( "galaxy_user.id" ), index=True, nullable=False ), + Column( "type", TEXT, nullable=False ), + Column( "name", TEXT ), + Column( "region_connection", TEXT ), + Column( "region_name", TEXT ), + Column( "region_endpoint", TEXT ), + Column( "is_secure", Boolean ), + Column( "host", TEXT ), + Column( "port", Integer ), + Column( "proxy", TEXT ), + Column( "proxy_port", TEXT ), + Column( "proxy_user", TEXT ), + Column( "proxy_pass", TEXT ), + Column( "debug", Integer ), + Column( "https_connection_factory", TEXT ), + Column( "path", TEXT ), + Column( "deleted", Boolean, default=False ) ) + +def upgrade(): + display_migration_details() + # Load existing tables + metadata.reflect() + try: + CloudProvider_table.drop() + CloudUserCredentials_table.drop() + CloudImage_table.drop() + UCI_table.drop() + CloudInstance_table.drop() + CloudStore_table.drop() + CloudSnapshot_table.drop() + except Exception, e: + log.debug( "Dropping cloud tables failed: %s" % str( e ) ) + + +def downgrade(): + pass --- a/lib/galaxy/model/mapping.py +++ b/lib/galaxy/model/mapping.py @@ -409,117 +409,6 @@ GalaxySessionToHistoryAssociation.table Column( "session_id", Integer, ForeignKey( "galaxy_session.id" ), index=True ), Column( "history_id", Integer, ForeignKey( "history.id" ), index=True ) ) -# *************************** Start cloud tables*********************************** -CloudImage.table = Table( "cloud_image", metadata, - Column( "id", Integer, primary_key=True ), - Column( "create_time", DateTime, default=now ), - Column( "update_time", DateTime, default=now, onupdate=now ), - Column( "provider_type", TEXT ), - Column( "image_id", TEXT, nullable=False ), - Column( "manifest", TEXT ), - Column( "state", TEXT ), - Column( "architecture", TEXT ), - Column( "deleted", Boolean, default=False ) ) - -""" UserConfiguredInstance (UCI) table """ -UCI.table = Table( "cloud_uci", metadata, - Column( "id", Integer, primary_key=True ), - Column( "create_time", DateTime, default=now ), - Column( "update_time", DateTime, default=now, onupdate=now ), - Column( "user_id", Integer, ForeignKey( "galaxy_user.id" ), index=True, nullable=False ), - Column( "credentials_id", Integer, ForeignKey( "cloud_user_credentials.id" ), index=True ), - Column( "key_pair_name", TEXT ), - Column( "key_pair_material", TEXT ), - Column( "name", TEXT ), - Column( "state", TEXT ), - Column( "error", TEXT ), - Column( "total_size", Integer ), - Column( "launch_time", DateTime ), - Column( "deleted", Boolean, default=False ) ) - -CloudInstance.table = Table( "cloud_instance", metadata, - Column( "id", Integer, primary_key=True ), - Column( "create_time", DateTime, default=now ), - Column( "update_time", DateTime, default=now, onupdate=now ), - Column( "launch_time", DateTime ), - Column( "stop_time", DateTime ), - Column( "user_id", Integer, ForeignKey( "galaxy_user.id" ), index=True, nullable=False ), - Column( "uci_id", Integer, ForeignKey( "cloud_uci.id" ), index=True ), - Column( "type", TEXT ), - Column( "reservation_id", TEXT ), - Column( "instance_id", TEXT ), - Column( "mi_id", Integer, ForeignKey( "cloud_image.id" ), index=True ), - Column( "state", TEXT ), - Column( "error", TEXT ), - Column( "public_dns", TEXT ), - Column( "private_dns", TEXT ), - Column( "security_group", TEXT ), - Column( "availability_zone", TEXT ) ) - -CloudStore.table = Table( "cloud_store", metadata, - Column( "id", Integer, primary_key=True ), - Column( "create_time", DateTime, default=now ), - Column( "update_time", DateTime, default=now, onupdate=now ), - Column( "attach_time", DateTime ), - Column( "user_id", Integer, ForeignKey( "galaxy_user.id" ), index=True, nullable=False ), - Column( "uci_id", Integer, ForeignKey( "cloud_uci.id" ), index=True, nullable=False ), - Column( "volume_id", TEXT ), - Column( "size", Integer, nullable=False ), - Column( "availability_zone", TEXT ), - Column( "inst_id", Integer, ForeignKey( "cloud_instance.id" ) ), - Column( "status", TEXT ), - Column( "device", TEXT ), - Column( "space_consumed", Integer ), - Column( "error", TEXT ), - Column( "deleted", Boolean, default=False ) ) - -CloudSnapshot.table = Table( "cloud_snapshot", metadata, - Column( "id", Integer, primary_key=True ), - Column( "create_time", DateTime, default=now ), - Column( "update_time", DateTime, default=now, onupdate=now ), - Column( "user_id", Integer, ForeignKey( "galaxy_user.id" ), index=True, nullable=False ), - Column( "uci_id", Integer, ForeignKey( "cloud_uci.id" ), index=True ), - Column( "store_id", Integer, ForeignKey( "cloud_store.id" ), index=True, nullable=False ), - Column( "snapshot_id", TEXT ), - Column( "status", TEXT ), - Column( "description", TEXT ), - Column( "error", TEXT ), - Column( "deleted", Boolean, default=False ) ) - -CloudUserCredentials.table = Table( "cloud_user_credentials", metadata, - Column( "id", Integer, primary_key=True ), - Column( "create_time", DateTime, default=now ), - Column( "update_time", DateTime, default=now, onupdate=now ), - Column( "user_id", Integer, ForeignKey( "galaxy_user.id" ), index=True, nullable=False ), - Column( "provider_id", Integer, ForeignKey( "cloud_provider.id" ), index=True, nullable=False ), - Column( "name", TEXT ), - Column( "access_key", TEXT ), - Column( "secret_key", TEXT ), - Column( "deleted", Boolean, default=False ) ) - -CloudProvider.table = Table( "cloud_provider", metadata, - Column( "id", Integer, primary_key=True ), - Column( "create_time", DateTime, default=now ), - Column( "update_time", DateTime, default=now, onupdate=now ), - Column( "user_id", Integer, ForeignKey( "galaxy_user.id" ), index=True, nullable=False ), - Column( "type", TEXT, nullable=False ), - Column( "name", TEXT ), - Column( "region_connection", TEXT ), - Column( "region_name", TEXT ), - Column( "region_endpoint", TEXT ), - Column( "is_secure", Boolean ), - Column( "host", TEXT ), - Column( "port", Integer ), - Column( "proxy", TEXT ), - Column( "proxy_port", TEXT ), - Column( "proxy_user", TEXT ), - Column( "proxy_pass", TEXT ), - Column( "debug", Integer ), - Column( "https_connection_factory", TEXT ), - Column( "path", TEXT ), - Column( "deleted", Boolean, default=False ) ) -# *************************** End cloud tables*********************************** - StoredWorkflow.table = Table( "stored_workflow", metadata, Column( "id", Integer, primary_key=True ), Column( "create_time", DateTime, default=now ), @@ -1284,42 +1173,6 @@ assign_mapper( context, GalaxySessionToH HistoryDatasetAssociation.mapper.add_property( "creating_job_associations", relation( JobToOutputDatasetAssociation ) ) -# vvvvvvvvvvvvvvvv Start cloud table mappings vvvvvvvvvvvvvvvv -assign_mapper( context, CloudImage, CloudImage.table ) - -assign_mapper( context, UCI, UCI.table, - properties=dict( user=relation( User ), - credentials=relation( CloudUserCredentials ), - instance=relation( CloudInstance, backref='uci' ), - store=relation( CloudStore, backref='uci', cascade='all, delete-orphan' ), - snapshot=relation( CloudSnapshot, backref='uci' ) - ) ) - -assign_mapper( context, CloudInstance, CloudInstance.table, - properties=dict( user=relation( User ), - image=relation( CloudImage ) - ) ) - -assign_mapper( context, CloudStore, CloudStore.table, - properties=dict( user=relation( User ), - inst=relation( CloudInstance ), - snapshot=relation( CloudSnapshot, backref="store" ) - ) ) - -assign_mapper( context, CloudSnapshot, CloudSnapshot.table, - properties=dict( user=relation( User ) - ) ) - -assign_mapper( context, CloudProvider, CloudProvider.table, - properties=dict( user=relation( User ) - ) ) - -assign_mapper( context, CloudUserCredentials, CloudUserCredentials.table, - properties=dict( user=relation( User), - provider=relation( CloudProvider ) - ) ) -# ^^^^^^^^^^^^^^^ End cloud table mappings ^^^^^^^^^^^^^^^^^^ - assign_mapper( context, Workflow, Workflow.table, properties=dict( steps=relation( WorkflowStep, backref='workflow', order_by=asc(WorkflowStep.table.c.order_index), --- a/lib/galaxy/app.py +++ b/lib/galaxy/app.py @@ -73,8 +73,6 @@ class UniverseApplication( object ): # FIXME: These are exposed directly for backward compatibility self.job_queue = self.job_manager.job_queue self.job_stop_queue = self.job_manager.job_stop_queue - # Start the cloud manager - self.cloud_manager = cloud.CloudManager( self ) # Track Store ## self.track_store = store.TrackStoreManager( self.config.track_store_path ) --- a/templates/cloud/view_snapshots.mako +++ /dev/null @@ -1,90 +0,0 @@ -<%inherit file="/base.mako"/> - -<%def name="title()">Snapshots</%def> - -%if message: -<% - try: - messagetype - except: - messagetype = "done" -%> - - - -<p /> -<div class="${messagetype}message"> - ${message} -</div> -%endif - -%if snaps: - <h2>Snapshots for instance ${snaps[0].uci.name}</h2> -%else: - <h2>Selected instance has no recorded or associated snapshots.</h2> -%endif - -<ul class="manage-table-actions"> - <li> - <a class="action-button" href="${h.url_for( action='list' )}"> - <img src="${h.url_for('/static/images/silk/resultset_previous.png')}" /> - <span>Return to cloud management console</span> - </a> - </li> -</ul> - -%if snaps: - <table class="mange-table colored" border="0" cellspacing="0" cellpadding="0" width="100%"> - <colgroup width="2%"></colgroup> - <colgroup width="16%"></colgroup> - <colgroup width="16%"></colgroup> - <colgroup width="10%"></colgroup> - <colgroup width="5%"></colgroup> - <tr class="header"> - <th>#</th> - <th>Create time</th> - <th>Snapshot ID</th> - <th>Status</th> - <th>Delete?</th> - <th></th> - </tr> - <% - total_hours = 0 - %> - %for i, snap in enumerate( snaps ): - <tr> - <td>${i+1}</td> - <td> - %if snap.create_time: - ${str(snap.create_time)[:16]} UCT - %else: - N/A - %endif - </td> - <td> - %if snap.snapshot_id: - ${snap.snapshot_id} - %else: - N/A - %endif - </td> - <td> - %if snap.status: - ${snap.status} - %else: - N/A - %endif - </td> - <td> - <a confirm="Are you sure you want to delete snapshot '${snap.snapshot_id}'?" - href="${h.url_for( controller='cloud', action='delete_snapshot', uci_id=trans.security.encode_id(snap.uci.id), snap_id=trans.security.encode_id(snap.id) )}">x</a> - </td> - </tr> - %endfor - </table> -%endif - - - - - --- a/eggs.ini +++ b/eggs.ini @@ -54,7 +54,6 @@ wsgiref = 0.1.2 Babel = 0.9.4 wchartype = 0.1 Whoosh = 0.3.18 -boto = 1.8d ; extra version information [tags] --- a/templates/cloud/configure_cloud.mako +++ /dev/null @@ -1,360 +0,0 @@ -<%inherit file="/base.mako"/> - -<%def name="title()">Cloud home</%def> - -%if message: -<% - try: - messagetype - except: - messagetype = "done" -%> - -<div class="${messagetype}message"> - ${message} -</div> -%endif - -<%def name="javascripts()"> -${parent.javascripts()} -${h.js( "jquery" )} - -<script type="text/javascript"> -function trim19(str){ - var str = str.replace(/^\s\s*/, ''), - ws = /\s/, - i = str.length; - while (ws.test(str.charAt(--i))); - return str.slice(0, i + 1); -} - - function update_state() { - $.getJSON( "${h.url_for( action='json_update' )}", {}, function ( data ) { - for (var i in data) { - var elem = '#' + data[i].id; - // Because of different list managing 'live' vs. 'available' instances, reload url on various state changes - old_state = $(elem + "-state").text(); - prev_old_state = trim19( $(elem + "-state-p").text() ); - new_state = data[i].state; - error_msg = data[i].error; - //console.log( "old_state[%d] = %s", i, old_state ); - //console.log( "prev_old_state[%d] = %s", i, prev_old_state ); - //console.log( "new_state[%d] = %s", i, new_state ); - //console.log( "error_msg[%d] = %s", i, error_msg ); - if ( ( old_state=='pending' && new_state=='running' ) || ( old_state=='shutting-down' && new_state=='available' ) || \ - ( old_state=='running' && new_state=='available' ) || ( old_state=='running' && new_state=='error' ) || \ - ( old_state=='pending' && new_state=='available' ) || ( old_state=='submitted' && new_state=='available' ) || \ - ( prev_old_state.match('creating') && new_state=='available' ) ) { - var url = "${h.url_for( controller='cloud', action='list')}"; - location.replace( url ); - } - else if ( ( ( old_state != 'error' && old_state != '' ) && new_state == 'error' ) || ( !prev_old_state.match('error') && new_state == 'error' ) ) { - var url = "${h.url_for( controller='cloud', action='list')}"; - location.replace( url ); - } - - if ( new_state=='shutting-down' || new_state=='shutting-downUCI' ) { - $(elem + "-link").text( "" ); - } - - // Check if Galaxy website is accessible on given instance; if so, provide link. Otherwise, wait more. - else if ( ( $(elem+"-link").text().match('starting') || $(elem+"-link").text()=='' ) && new_state=='running' ) { - //console.log ( 'elem.text: ' + $(elem+"-link").text() ); - $.getJSON( "${h.url_for( action='link_update' )}", { uci_id: data[i].id }, function ( data ) { - for (var i in data) { - var dns = data[i].public_dns; - var uci = '#' + data[i].uci_id; - if( !dns ) { - $(uci+"-link").text( 'Galaxy starting...' ); - // http://stackoverflow.com/questions/275931/how-do-you-make-an-element-flash-… - //$(uci+"-link").stop().animate({ fontSize: "14px" }, 1000).animate({ fontSize: "12px" }, 1000); - } - else { - $(uci+"-link").html( '<div align="right"><a class="action-button" href="http://'+dns+'" target="_blank">' + - '<span>Access Galaxy</span>'+ - '<img src="' + "${h.url_for( '/static/images/silk/resultset_next.png' )}" + '" /></div>' ); - //$(uci+"-link").stop().animate({ fontSize: "14px" }, 1000).animate({ fontSize: "12px" }, 1000); - } - } - }); - } - - // Update 'size', 'state' and 'time alive' fields - $(elem + "-size").text( data[i].total_size ); - $(elem + "-state").text( data[i].state ); - if ( new_state != 'error' ) { // Because 'error' state is handled as a JS link, don't include it in update - $(elem + "-state-p").text( data[i].state ); - } - - if (data[i].launch_time) { - $(elem + "-launch_time").text( data[i].launch_time.substring(0, 16 ) + " UTC (" + data[i].time_ago + ")" ); - } - else { - $(elem + "-launch_time").text( "N/A" ); - } - } - }); - setTimeout("update_state()", 15000); - } - - $(function() { - update_state(); - }); - -</script> -</%def> - -<h2>Galaxy in the clouds</h2> - -%if cloudProviders: - ## Manage user-registered cloud providers - <h3>Your registered cloud providers</h3> - <ul class="manage-table-actions"> - <li> - <a class="action-button" href="${h.url_for( action='add_provider' )}"> - <img src="${h.url_for('/static/images/silk/add.png')}" /> - <span>Add provider</span> - </a> - </li> - </ul> - - <table class="mange-table colored" border="0" cellspacing="0" cellpadding="0" width="100%"> - <tr class="header"> - <th>Provider name</th> - <th>Provider type</th> - <th></th> - </tr> - %for i, cloudProvder in enumerate( cloudProviders ): - <tr> - <td> - ${cloudProvder.name} - <a id="cp-${i}-popup" class="popup-arrow" style="display: none;">▼</a> - </td> - <td> - ${cloudProvder.type} - </td> - <td> - <div popupmenu="cp-${i}-popup"> - <a class="action-button" href="${h.url_for( action='view_provider', id=trans.security.encode_id(cloudProvder.id) )}">View</a> - <a class="action-button" href="${h.url_for( action='edit_provider', id=trans.security.encode_id(cloudProvder.id) )}">Edit</a> - <a class="action-button" confirm="Are you sure you want to delete cloud provider '${cloudProvder.name}'?" href="${h.url_for( action='delete_provider', id=trans.security.encode_id(cloudProvder.id) )}">Delete</a> - </div> - </td> - </tr> - %endfor - </table> - - - ## ***************************************************** - ## Manage user credentials - <h3>Your registered cloud credentials</h3> - - %if cloudCredentials: - <ul class="manage-table-actions"> - <li> - <a class="action-button" href="${h.url_for( action='add_credentials' )}"> - <img src="${h.url_for('/static/images/silk/add.png')}" /> - <span>Add credentials</span> - </a> - </li> - </ul> - <table class="mange-table colored" border="0" cellspacing="0" cellpadding="0" width="100%"> - <tr class="header"> - <th>Credentials name</th> - <th>Provider name (type)</th> - <th></th> - </tr> - - %for i, cloudCredential in enumerate( cloudCredentials ): - <tr> - <td> - ${cloudCredential.name} - <a id="cr-${i}-popup" class="popup-arrow" style="display: none;">▼</a> - </td> - <td> - ${cloudCredential.provider.name} - (${cloudCredential.provider.type}) - </td> - <td> - <div popupmenu="cr-${i}-popup"> - <a class="action-button" href="${h.url_for( action='view_credentials', id=trans.security.encode_id(cloudCredential.id) )}">View</a> - <a class="action-button" href="${h.url_for( action='edit_credentials', id=trans.security.encode_id(cloudCredential.id) )}">Edit</a> - <a class="action-button" confirm="Are you sure you want to delete credentials '${cloudCredential.name}'?" href="${h.url_for( action='delete_credentials', id=trans.security.encode_id(cloudCredential.id) )}">Delete</a> - </div> - </td> - </tr> - %endfor - </table> - - ## ***************************************************** - ## Manage live instances - <p /> - <h3>Manage your cloud instances</h3> - <ul class="manage-table-actions"> - <li> - <a class="action-button" href="${h.url_for( action='configure_new_uci' )}"> - <img src="${h.url_for('/static/images/silk/add.png')}" /> - <span>Configure new instance</span> - </a> - </li> - </ul> - - <table class="mange-table colored" border="0" cellspacing="0" cellpadding="0" width="100%"> - <colgroup width="40%"></colgroup> - <colgroup width="15%"></colgroup> - <colgroup width="10%"></colgroup> - <colgroup width="25%"></colgroup> - <colgroup width="10%"></colgroup> - <tr class="header"> - <th>Live instance name (credentials)</th> - <th>Storage size (GB)</th> - <th>State</th> - <th>Alive since</th> - <th></th> - <th></th> - </tr> - %if liveInstances: - %for i, liveInstance in enumerate( liveInstances ): - <tr> - <td> - ${liveInstance.name} (${liveInstance.credentials.name}) - <a id="li-${i}-popup" class="popup-arrow" style="display: none;">▼</a> - </td> - <td id="${ liveInstance.id }-size">${str(liveInstance.total_size)}</td> - <td id="${ liveInstance.id }-state">${str(liveInstance.state)}</td> - <td id="${ liveInstance.id }-launch_time"> - ##${str(liveInstance.launch_time)[:16]} - <% - #from datetime import datetime - #from datetime import timedelta - - # DB stores all times in GMT, so adjust for difference (4 hours) - #adjustedStarttime = liveInstance.update_time - timedelta(hours=4) - - #delta = datetime.now() - adjustedStarttime - #context.write( str(datetime.utcnow() ) ) - #context.write( str(delta) ) - - # This is where current time and since duration is calculated - if liveInstance.launch_time is None: - context.write( 'N/A' ) - else: - context.write( str( liveInstance.launch_time )[:16] ) - context.write( ' UTC (' ) - context.write( str(h.date.distance_of_time_in_words (liveInstance.launch_time, h.date.datetime.utcnow() ) ) ) - context.write( ')' ) - %> - </td> - ## Handled by JavaScript function - <td id="${ liveInstance.id }-link"></td> - <td> - <div popupmenu="li-${i}-popup"> - <a class="action-button" confirm="Are you sure you want to stop instance '${liveInstance.name}'?" href="${h.url_for( action='stop', id=trans.security.encode_id(liveInstance.id) )}">Stop</a> - <a class="action-button" href="${h.url_for( action='rename_uci', id=trans.security.encode_id(liveInstance.id) )}">Rename</a> - <a class="action-button" href="${h.url_for( action='view_uci_details', id=trans.security.encode_id(liveInstance.id) )}">View details</a> - <a class="action-button" href="${h.url_for( action='add_storage', id=trans.security.encode_id(liveInstance.id) )}">Add storage</a> - <a class="action-button" href="${h.url_for( action='uci_usage_report', id=trans.security.encode_id(liveInstance.id) )}">Usage report</a> - </div> - </td> - </tr> - %endfor - %else: - <tr> - <td>Currently, you have no live instances.</td> - </tr> - %endif - </table> - <p /> - ## ***************************************************** - ## Manage previously configured instances - ## <table class="mange-table noHR" border="0" cellspacing="0" cellpadding="0" width="100%"> - <table class="mange-table colored" border="0" cellspacing="0" cellpadding="0" width="100%"> - <colgroup width="40%"></colgroup> - <colgroup width="15%"></colgroup> - <colgroup width="10%"></colgroup> - <colgroup width="25%"></colgroup> - <colgroup width="10%"></colgroup> - <tr class="header"> - <th>Configured instance name (credentials)</th> - <th>Storage size (GB)</th> - <th>State</th> - <th></th> - <th></th> - </tr> - - %if prevInstances: - %for i, prevInstance in enumerate( prevInstances ): - <tr> - <td> - ${prevInstance.name} (${prevInstance.credentials.name}) - <a id="pi-${i}-popup" class="popup-arrow" style="display: none;">▼</a> - </td> - <td>${str(prevInstance.total_size)}</td> - <td id="${ prevInstance.id }-state-p"> - <%state = str(prevInstance.state)%> - %if state =='error': - <div id="${prevInstance.name}-short"> - <a onclick="document.getElementById('${prevInstance.name}-full').style.display = 'block'; - document.getElementById('${prevInstance.name}-short').style.display = 'none'; return 0" - href="javascript:void(0)"> - error - </a> - </div> - <div id="${prevInstance.name}-full" style="DISPLAY: none"> - <a onclick="document.getElementById('${prevInstance.name}-short').style.display = 'block'; - document.getElementById('${prevInstance.name}-full').style.display = 'none'; return 0;" - href="javascript:void(0)"> - error:</a><br /> - ${str(prevInstance.error)} - <p /> - <div style="font-size:10px;"> - <a href="${h.url_for( action='set_uci_state', id=trans.security.encode_id(prevInstance.id), state='available' )}">reset state</a> - </div> - </div> - %else: - ${str(prevInstance.state)} - %endif - </td> - <td> - <div popupmenu="pi-${i}-popup"> - <a class="action-button" href="${h.url_for( action='uci_usage_report', id=trans.security.encode_id(prevInstance.id) )}">Usage report</a> - <a class="action-button" href="${h.url_for( action='rename_uci', id=trans.security.encode_id(prevInstance.id) )}">Rename</a> - <a class="action-button" href="${h.url_for( action='start', id=trans.security.encode_id(prevInstance.id), type='m1.small' )}"> Start m1.small</a> - <a class="action-button" href="${h.url_for( action='start', id=trans.security.encode_id(prevInstance.id), type='c1.medium' )}"> Start c1.medium</a> - <a class="action-button" href="${h.url_for( action='start', id=trans.security.encode_id(prevInstance.id), type='m1.large' )}"> Start m1.large</a> - <a class="action-button" href="${h.url_for( action='start', id=trans.security.encode_id(prevInstance.id), type='m1.xlarge' )}"> Start m1.xlarge</a> - <a class="action-button" href="${h.url_for( action='start', id=trans.security.encode_id(prevInstance.id), type='c1.xlarge' )}"> Start c1.xlarge</a> - <a class="action-button" href="${h.url_for( action='start', id=trans.security.encode_id(prevInstance.id), type='m2.2xlarge' )}"> Start m2.2xlarge</a> - <a class="action-button" href="${h.url_for( action='start', id=trans.security.encode_id(prevInstance.id), type='m2.4xlarge' )}"> Start m2.4xlarge</a> - <a class="action-button" href="${h.url_for( action='create_snapshot', id=trans.security.encode_id(prevInstance.id) )}">Create snapshot</a> - <a class="action-button" href="${h.url_for( action='view_snapshots', id=trans.security.encode_id(prevInstance.id) )}">View snapshots</a> - <a class="action-button" confirm="Are you sure you want to delete instance '${prevInstance.name}'? This will delete all of your data assocaiated with this instance!" href="${h.url_for( action='delete_uci', id=trans.security.encode_id(prevInstance.id) )}">Delete</a> - </div> - </td> - </tr> - %endfor - %else: - <tr> - <td>You have no previously configured instances (or they are all currently alive).</td> - </tr> - %endif - </table> - - %else: - You have no credentials associated with your Galaxy account: - <a class="action-button" href="${h.url_for( action='add_credentials' )}"> - <img src="${h.url_for('/static/images/silk/add.png')}" /> - <span>add credentials</span> - </a> - or - <a href="http://aws.amazon.com/" target="_blank"> - open AWS account with Amazon</a>. - %endif - -%else: - You have no cloud providers registered with your Galaxy account: - <a class="action-button" href="${h.url_for( action='add_provider' )}"> - <img src="${h.url_for('/static/images/silk/add.png')}" /> - <span>add provider now</span> - </a> -%endif --- a/universe_wsgi.ini.sample +++ b/universe_wsgi.ini.sample @@ -201,11 +201,6 @@ use_interactive = True # Should default dataset access permissions be private for new users; default is False (datasets are public) new_user_dataset_access_role_default_private = False -# ---- Cloud Management -------------------------------------------------------- - -# Uncomment following line to enable cloud management mode (only leaves the Cloud, Help and User tabs on) -# cloud_controller_instance = True - # ---- Job Execution -------------------------------------------------------- # If running multiple Galaxy processes, one can be designated as the job --- a/lib/galaxy/cloud/__init__.py +++ /dev/null @@ -1,734 +0,0 @@ -import logging, threading, sys, os, time, subprocess, string, tempfile, re, traceback, shutil - -from galaxy import util, model, config -from galaxy.model import mapping -from galaxy.model.orm import lazyload -from galaxy.datatypes.tabular import * -from galaxy.datatypes.interval import * -from galaxy.datatypes import metadata -from galaxy.util.bunch import Bunch -from sqlalchemy import or_ - -import pkg_resources -pkg_resources.require( "PasteDeploy" ) - -from paste.deploy.converters import asbool - -from Queue import Queue, Empty - -log = logging.getLogger( __name__ ) - -uci_states = Bunch( - NEW_UCI = "newUCI", - NEW = "new", - CREATING = "creating", - DELETING_UCI = "deletingUCI", - DELETING = "deleting", - DELETED = "deleted", - SUBMITTED_UCI = "submittedUCI", - SUBMITTED = "submitted", - SHUTTING_DOWN_UCI = "shutting-downUCI", - SHUTTING_DOWN = "shutting-down", - ADD_STORAGE_UCI = "add-storageUCI", - ADD_STORAGE = "add-storage", - AVAILABLE = "available", - RUNNING = "running", - PENDING = "pending", - ERROR = "error", - SNAPSHOT_UCI = "snapshotUCI", - SNAPSHOT = "snapshot" -) -instance_states = Bunch( - TERMINATED = "terminated", - SUBMITTED = "submitted", - RUNNING = "running", - ADDING = "adding-storage", - PENDING = "pending", - SHUTTING_DOWN = "shutting-down", - ERROR = "error" -) - -store_status = Bunch( - WAITING = "waiting", - IN_USE = "in-use", - ADDING = "adding", - CREATING = "creating", - DELETED = 'deleted', - ERROR = "error" -) - -snapshot_status = Bunch( - SUBMITTED = 'submitted', - PENDING = 'pending', - COMPLETED = 'completed', - DELETE = 'delete', - DELETED= 'deleted', - ERROR = "error" -) - -class CloudManager( object ): - """ - Highest level interface to cloud management. - """ - def __init__( self, app ): - self.app = app - self.sa_session = app.model.context - if self.app.config.enable_cloud_execution == True: - # The dispatcher manager for underlying cloud instances - implements and contacts individual cloud providers - self.provider = CloudProvider( app ) - # Monitor for updating status of cloud instances - self.cloud_monitor = CloudMonitor( self.app, self.provider ) - else: - self.job_queue = self.job_stop_queue = NoopCloudMonitor() - - def shutdown( self ): - self.cloud_monitor.shutdown() - -class Sleeper( object ): - """ - Provides a 'sleep' method that sleeps for a number of seconds *unless* - the notify method is called (from a different thread). - """ - def __init__( self ): - self.condition = threading.Condition() - def sleep( self, seconds ): - self.condition.acquire() - self.condition.wait( seconds ) - self.condition.release() - def wake( self ): - self.condition.acquire() - self.condition.notify() - self.condition.release() - -class CloudMonitor( object ): - """ - Cloud manager, waits for user to instantiate a cloud instance and then invokes a - CloudProvider. - """ - STOP_SIGNAL = object() - def __init__( self, app, provider ): - """Start the cloud manager""" - self.app = app - # Keep track of the pid that started the cloud manager, only it - # has valid threads - self.parent_pid = os.getpid() - self.sa_session = app.model.context - - # Contains requests that are waiting (only use from monitor thread) - self.waiting = [] - - # Helper for interruptable sleep - self.sleeper = Sleeper() - self.running = True - self.provider = provider - self.monitor_thread = threading.Thread( target=self.__monitor ) - self.monitor_thread.start() - log.info( "Cloud manager started" ) - - def __monitor( self ): - """ - Daemon that continuously monitors cloud instance requests as well as state - of running instances. - """ - # HACK: Delay until after forking, we need a way to do post fork notification!!! - time.sleep( 10 ) - - cnt = 0 # Run global update only periodically so keep counter variable - while self.running: - try: - self.__monitor_step() - if cnt%30 == 0: # Run global update every 30 iterations (1 minute) - self.provider.update() - cnt = 0 - except: - log.exception( "Exception in cloud manager monitor_step" ) - # Sleep - cnt += 1 - self.sleeper.sleep( 2 ) - - def __monitor_step( self ): - """ - Called repeatedly by `monitor` to process cloud instance requests. - TODO: Update following description to match the code - Gets any new cloud instance requests from the database, then iterates - over all new and waiting jobs to check the state of the jobs each - depends on. If the job has dependencies that have not finished, it - it goes to the waiting queue. If the job has dependencies with errors, - it is marked as having errors and removed from the queue. Otherwise, - the job is dispatched. - """ - model = self.app.model - new_requests = [] - - for r in self.sa_session.query( model.UCI ) \ - .filter( or_( model.UCI.table.c.state==uci_states.NEW_UCI, - model.UCI.table.c.state==uci_states.SUBMITTED_UCI, - model.UCI.table.c.state==uci_states.SHUTTING_DOWN_UCI, - model.UCI.table.c.state==uci_states.DELETING_UCI, - model.UCI.table.c.state==uci_states.SNAPSHOT_UCI, - model.UCI.table.c.state==uci_states.ADD_STORAGE_UCI ) ) \ - .all(): - uci_wrapper = UCIwrapper( r, self.app ) - new_requests.append( uci_wrapper ) - - for uci_wrapper in new_requests: - self.sa_session.expunge_all() - self.put( uci_wrapper ) - - def put( self, uci_wrapper ): - """Add a request to the queue.""" - self.provider.put( uci_wrapper ) - self.sleeper.wake() - - def shutdown( self ): - """Attempts to gracefully shut down the worker thread""" - if self.parent_pid != os.getpid(): - # We're not the real queue, do nothing - return - else: - log.info( "Sending stop signal to worker thread" ) - self.running = False - self.sleeper.wake() - log.info( "cloud manager stopped" ) - self.dispatcher.shutdown() - -class UCIwrapper( object ): - """ - Wraps 'model.UCI' with convenience methods for state management - """ - def __init__( self, uci, app ): - self.uci_id = uci.id - self.app = app - self.sa_session = self.app.model.context - base_directory = os.path.join( self.app.config.job_working_directory, "cloud" ) - self.working_directory = os.path.join( base_directory, str( self.uci_id ) ) -# log.debug( "Cloud controller working directory for UCI DB ID '%s': '%s'" % ( self.uci_id, self.working_directory ) ) - if not os.path.exists( base_directory ): - os.mkdir( base_directory ) - - - # --------- Setter methods ----------------- - - def change_state( self, uci_state=None, instance_id=None, i_state=None ): - """ - Sets state for UCI and/or UCI's instance with instance_id as provided by cloud provider and stored in local - Galaxy database. - Need to provide either: (1) state for the UCI, or (2) instance_id and it's state, or (3) all arguments. - """ -# log.debug( "Changing state - new uci_state: %s, instance_id: %s, i_state: %s" % ( uci_state, instance_id, i_state ) ) - if uci_state is not None: - uci = self.sa_session.query( model.UCI ).get( self.uci_id ) - self.sa_session.refresh( uci ) - uci.state = uci_state - self.sa_session.flush() - if ( instance_id is not None ) and ( i_state is not None ): - instance = self.sa_session.query( model.CloudInstance ).filter_by( uci_id=self.uci_id, instance_id=instance_id).first() - instance.state = i_state - self.sa_session.add( instance ) - self.sa_session.flush() - - def set_mi( self, i_index, mi_id ): - """ - Sets Machine Image (MI), e.g., 'ami-66fa190f', for UCI's instance with given index as it - is stored in local Galaxy database. - """ - mi = self.sa_session.query( model.CloudImage ).filter( model.CloudImage.table.c.image_id==mi_id ).first() - instance = self.sa_session.query( model.CloudInstance ).get( i_index ) - instance.image = mi - self.sa_session.add( instance ) - self.sa_session.flush() - - def set_key_pair( self, key_name, key_material=None ): - """ - Sets key pair value for current UCI. - """ - uci = self.sa_session.query( model.UCI ).get( self.uci_id ) - self.sa_session.refresh( uci ) - uci.key_pair_name = key_name - if key_material is not None: - uci.key_pair_material = key_material - self.sa_session.flush() - - def set_instance_launch_time( self, launch_time, i_index=None, i_id=None ): - """ - Stores launch time in local database for instance with specified index - i_index (as it is stored in local - Galaxy database) or with specified instance ID - i_id (as obtained from the cloud provider AND stored - in local Galaxy Database). Either 'i_index' or 'i_id' needs to be provided. - """ - if i_index != None: - instance = self.sa_session.query( model.CloudInstance ).get( i_index ) - elif i_id != None: - instance = self.sa_session.query( model.CloudInstance ).filter_by( uci_id=self.uci_id, instance_id=i_id).first() - else: - return None - - instance.launch_time = launch_time - self.sa_session.add( instance ) - self.sa_session.flush() - - def set_uci_launch_time( self, launch_time ): - uci = self.sa_session.query( model.UCI ).get( self.uci_id ) - self.sa_session.refresh( uci ) - uci.launch_time = launch_time - self.sa_session.add( uci ) - self.sa_session.flush() - - def set_stop_time( self, stop_time, i_index=None, i_id=None ): - """ - Stores stop time in local database for instance with specified index - i_index (as it is stored in local - Galaxy database) or with specified instance ID - i_id (as obtained from the cloud provider AND stored - in local Galaxy Database). Either 'i_index' or 'i_id' needs to be provided. - """ - if i_index != None: - instance = self.sa_session.query( model.CloudInstance ).get( i_index ) - elif i_id != None: - instance = self.sa_session.query( model.CloudInstance ).filter_by( uci_id=self.uci_id, instance_id=i_id).first() - else: - return None - - instance.stop_time = stop_time - self.sa_session.add( instance ) - self.sa_session.flush() - - def set_security_group_name( self, security_group_name, i_index=None, i_id=None ): - """ - Stores security group name in local database for instance with specified index - i_index (as it is stored in local - Galaxy database) or with specified instance ID - i_id (as obtained from the cloud provider AND stored - in local Galaxy Database). Either 'i_index' or 'i_id' needs to be provided. - """ - if i_index != None: - instance = self.sa_session.query( model.CloudInstance ).get( i_index ) - elif i_id != None: - instance = self.sa_session.query( model.CloudInstance ).filter_by( uci_id=self.uci_id, instance_id=i_id).first() - else: - return None - - instance.security_group = security_group_name - self.sa_session.add( instance ) - self.sa_session.flush() - - def set_reservation_id( self, i_index, reservation_id ): - instance = self.sa_session.query( model.CloudInstance ).get( i_index ) - instance.reservation_id = reservation_id - self.sa_session.add( instance ) - self.sa_session.flush() - - def set_instance_id( self, i_index, instance_id ): - """ - i_index refers to UCI's instance ID as stored in local database - instance_id refers to real-world, cloud resource ID (e.g., 'i-78hd823a') - """ - instance = self.sa_session.query( model.CloudInstance ).get( i_index ) - instance.instance_id = instance_id - self.sa_session.add( instance ) - self.sa_session.flush() - -# def set_public_dns( self, instance_id, public_dns ): -# uci = self.sa_session.query( model.UCI ).get( self.uci_id ) -# self.sa_session.refresh( uci ) -# uci.instance[instance_id].public_dns = public_dns -# uci.instance[instance_id].flush() -# -# def set_private_dns( self, instance_id, private_dns ): -# uci = self.sa_session.query( model.UCI ).get( self.uci_id ) -# self.sa_session.refresh( uci ) -# uci.instance[instance_id].private_dns = private_dns -# uci.instance[instance_id].flush() - - def reset_uci_launch_time( self ): - uci = self.sa_session.query( model.UCI ).get( self.uci_id ) - self.sa_session.refresh( uci ) - uci.launch_time = None - self.sa_session.add( uci ) - self.sa_session.flush() - - def set_error( self, error, set_state=False ): - """ - Sets error field of given UCI in local Galaxy database as well as any instances associated with - this UCI whose state is 'None' or 'SUBMITTED'. If set_state is set to 'true', - method also sets state of give UCI and corresponding instances to 'error' - """ - uci = self.sa_session.query( model.UCI ).get( self.uci_id ) - self.sa_session.refresh( uci ) - uci.error = error - if set_state: - uci.state = uci_states.ERROR - # Process all instances associated with this UCI - instances = self.sa_session.query( model.CloudInstance ) \ - .filter_by( uci=uci ) \ - .filter( or_( model.CloudInstance.table.c.state==None, model.CloudInstance.table.c.state==instance_states.SUBMITTED ) ) \ - .all() - for i in instances: - i.error = error - i.state = instance_states.ERROR - self.sa_session.add( i ) - self.sa_session.flush() - - self.sa_session.add( uci ) - self.sa_session.flush() - - def set_deleted( self ): - uci = self.sa_session.query( model.UCI ).get( self.uci_id ) - self.sa_session.refresh( uci ) - uci.state = uci_states.DELETED # for bookkeeping reasons, mark as deleted but don't actually delete. - uci.deleted = True - self.sa_session.add( uci ) - self.sa_session.flush() - -# def set_store_device( self, store_id, device ): -# uci = self.sa_session.query( model.UCI ).get( self.uci_id ) -# self.sa_session.refresh( uci ) -# uci.store[store_id].device = device -# uci.store[store_id].flush() - - def set_uci_total_size( self, total_size ): - uci = self.sa_session.query( model.UCI ).get( self.uci_id ) - self.sa_session.refresh( uci ) - uci.total_size = total_size - self.sa_session.add( uci ) - self.sa_session.flush() - - def set_store_error( self, error, store_index=None, store_id=None ): - if store_index != None: - store = self.sa_session.query( model.CloudStore ).get( store_index ) - elif store_id != None: - store = self.sa_session.query( model.CloudStore ).filter_by( volume_id = store_id ).first() - else: - return None - - store.error = error - store.status = store_status.ERROR - self.sa_session.add( store ) - self.sa_session.flush() - - def set_store_status( self, vol_id, status ): - vol = self.sa_session.query( model.CloudStore ).filter( model.CloudStore.table.c.volume_id == vol_id ).first() - vol.status = status - self.sa_session.add( vol ) - self.sa_session.flush() - - def set_store_availability_zone( self, availability_zone, vol_id=None ): - """ - Sets availability zone of storage volumes for either ALL volumes associated with current - UCI or for the volume whose volume ID (e.g., 'vol-39F80512') is provided as argument. - """ - if vol_id is not None: - vol = self.sa_session.query( model.CloudStore ).filter( model.CloudStore.table.c.volume_id == vol_id ).all() - else: - vol = self.sa_session.query( model.CloudStore ).filter( model.CloudStore.table.c.uci_id == self.uci_id ).all() - - for v in vol: - v.availability_zone = availability_zone - self.sa_session.add( v ) - self.sa_session.flush() - - def set_store_volume_id( self, store_index, volume_id ): - """ - Given store index as it is stored in local database, set volume ID as it is registered - on the cloud provider (e.g., vol-39890501) - """ - - if store_index != None: - store = self.sa_session.query( model.CloudStore ).get( store_index ) - store.volume_id = volume_id - self.sa_session.add( store ) - self.sa_session.flush() - else: - return None - -# uci = self.sa_session.query( model.UCI ).get( self.uci_id ) -# self.sa_session.refresh( uci ) -# uci.store[store_index].volume_id = volume_id -# #uci.store[store_index].flush() -# self.sa_session.add( uci ) -# self.sa_session.flush() - - def set_store_instance( self, vol_id, instance_id ): - """ - Stores instance ID that given store volume is attached to. Store volume ID should - be given in following format: 'vol-78943248' - """ - vol = self.sa_session.query( model.CloudStore ).filter( model.CloudStore.table.c.volume_id == vol_id ).first() - inst = self.sa_session.query( model.CloudInstance ).filter_by( instance_id=instance_id ).first() - vol.inst = inst - self.sa_session.add( vol ) - self.sa_session.flush() - - def set_store_device( self, vol_id, device ): - """ - Stores instance ID that given store volume is attached to. Store volume ID should - be given in following format: 'vol-78943248' - """ - vol = self.sa_session.query( model.CloudStore ).filter( model.CloudStore.table.c.volume_id == vol_id ).first() - vol.device = str( device ) - self.sa_session.add( vol ) - self.sa_session.flush() - - def set_store_deleted( self, vol_id, status=None ): - """ - Set storage volume as deleted in local database. Optionally, set the volume status too. - """ - vol = self.sa_session.query( model.CloudStore ).filter( model.CloudStore.table.c.volume_id == vol_id ).first() - vol.deleted = True - if status != None: - vol.status = status - self.sa_session.add( vol ) - self.sa_session.flush() - - def set_snapshot_id( self, snap_index, id ): - snap = model.CloudSnapshot.get( snap_index ) - - snap.snapshot_id = id - self.sa_session.add( snap ) - self.sa_session.flush() - - def set_snapshot_status( self, status, snap_index=None, snap_id=None ): - if snap_index != None: - snap = self.sa_session.query( model.CloudSnapshot ).get( snap_index ) - elif snap_id != None: - snap = self.sa_session.query( model.CloudSnapshot ).filter_by( snapshot_id = snap_id).first() - else: - return - snap.status = status - self.sa_session.add( snap ) - self.sa_session.flush() - - def set_snapshot_error( self, error, snap_index=None, snap_id=None, set_status=False ): - if snap_index != None: - snap = self.sa_session.query( model.CloudSnapshot ).get( snap_index ) - elif snap_id != None: - snap = self.sa_session.query( model.CloudSnapshot ).filter_by( snapshot_id = snap_id).first() - else: - return - snap.error = error - if set_status: - snap.status = snapshot_status.ERROR - - self.sa_session.add( snap ) - self.sa_session.flush() - - # --------- Getter methods ----------------- - - def get_provider_type( self ): - """ Returns type of cloud provider associated with given UCI. """ - uci = self.sa_session.query( model.UCI ).get( self.uci_id ) - self.sa_session.refresh( uci ) - return uci.credentials.provider.type - - def get_provider( self ): - """ Returns database object of cloud provider associated with credentials of given UCI. """ - uci = self.sa_session.query( model.UCI ).get( self.uci_id ) - self.sa_session.refresh( uci ) - return uci.credentials.provider - - def get_instance_type( self, i_index ): - instance = self.sa_session.query( model.CloudInstance ).get( i_index ) - self.sa_session.refresh( instance ) - return instance.type - - def get_uci_state( self ): - uci = self.sa_session.query( model.UCI ).get( self.uci_id ) - self.sa_session.refresh( uci ) - return uci.state - - def get_instances_indexes( self, state=None ): - """ - Returns indexes of instances associated with given UCI as they are stored in local Galaxy database and - whose state corresponds to passed argument. Returned values enable indexing instances from local Galaxy database. - """ - uci = self.sa_session.query( model.UCI ).get( self.uci_id ) - self.sa_session.refresh( uci ) - instances = self.sa_session.query( model.CloudInstance ) \ - .filter_by( uci=uci ) \ - .filter( model.CloudInstance.table.c.state==state ) \ - .all() - il = [] - for i in instances: - il.append( i.id ) - - return il - - def get_instance_state( self, instance_id ): - uci = self.sa_session.query( model.UCI ).get( self.uci_id ) - self.sa_session.refresh( uci ) - return uci.instance[instance_id].state - - def get_instaces_in_state( self, state ): - """ Get database objects of all instances associated with this UCI in given state. """ - return self.sa_session.query( model.CloudInstance ) \ - .filter_by( uci_id=self.uci_id, state = state ) \ - .all() - - def get_instances_ids( self ): - """ - Returns list IDs of all instances' associated with this UCI that are not in 'terminated' or - 'error' but the state is defined (i.e., state is not None) - (e.g., return value: ['i-402906D2', 'i-q0290dsD2'] ). - """ - il = self.sa_session.query( model.CloudInstance ) \ - .filter_by( uci_id=self.uci_id ) \ - .filter( or_( model.CloudInstance.table.c.state != 'terminated', - model.CloudInstance.table.c.state != 'error', - model.CloudInstance.table.c.state != None ) ) \ - .all() - instanceList = [] - for i in il: - instanceList.append( i.instance_id ) - return instanceList - - def get_name( self ): - uci = self.sa_session.query( model.UCI ).get( self.uci_id ) - self.sa_session.refresh( uci ) - return uci.name - - def get_key_pair_name( self ): - """ - Returns keypair name associated with given UCI. - """ - uci = self.sa_session.query( model.UCI ).get( self.uci_id ) - self.sa_session.refresh( uci ) - return uci.key_pair_name - - def get_key_pair_material( self ): - """ - Returns keypair material (i.e., private key) associated with given UCI. - """ - uci = self.sa_session.query( model.UCI ).get( self.uci_id ) - self.sa_session.refresh( uci ) - return uci.key_pair_material - - def get_security_group_name( self, i_index=None, i_id=None ): - """ - Given EITHER instance index as it is stored in local Galaxy database OR instance ID as it is - obtained from cloud provider and stored in local Galaxy database, return security group name associated - with given instance. - """ - if i_index != None: - instance = self.sa_session.query( model.CloudInstance ).get( i_index ) - return instance.security_group - elif i_id != None: - instance = self.sa_session.query( model.CloudInstance ).filter_by( uci_id=self.uci_id, instance_id=i_id).first() - return instance.security_group - - def get_access_key( self ): - uci = self.sa_session.query( model.UCI ).get( self.uci_id ) - self.sa_session.refresh( uci ) - return uci.credentials.access_key - - def get_secret_key( self ): - uci = self.sa_session.query( model.UCI ).get( self.uci_id ) - self.sa_session.refresh( uci ) - return uci.credentials.secret_key - - def get_mi_id( self, instance_id=0 ): - uci = self.sa_session.query( model.UCI ).get( self.uci_id ) - self.sa_session.refresh( uci ) - return uci.instance[instance_id].image.image_id - - def get_public_dns( self, instance_id=0 ): - uci = self.sa_session.query( model.UCI ).get( self.uci_id ) - self.sa_session.refresh( uci ) - return uci.instance[instance_id].public_dns - - def get_private_dns( self, instance_id=0 ): - uci = self.sa_session.query( model.UCI ).get( self.uci_id ) - self.sa_session.refresh( uci ) - return uci.instance[instance_id].private_dns - - def get_uci_availability_zone( self ): - """ - Returns UCI's availability zone. - Because all of storage volumes associated with a given UCI must be in the same - availability zone, availability of a UCI is determined by availability zone of - any one storage volume. - """ - uci = self.sa_session.query( model.UCI ).get( self.uci_id ) - self.sa_session.refresh( uci ) - return uci.store[0].availability_zone - - def get_uci_total_size( self ): - uci = self.sa_session.query( model.UCI ).get( self.uci_id ) - self.sa_session.refresh( uci ) - return uci.total_size - - def get_store_size( self, store_id=0 ): - uci = self.sa_session.query( model.UCI ).get( self.uci_id ) - self.sa_session.refresh( uci ) - return uci.store[store_id].size - - def get_store_volume_id( self, store_id=0 ): - """ - Given store ID associated with this UCI, get volume ID as it is registered - on the cloud provider (e.g., 'vol-39890501') - """ - uci = self.sa_session.query( model.UCI ).get( self.uci_id ) - self.sa_session.refresh( uci ) - return uci.store[store_id].volume_id - - def get_all_stores_in_status( self, status ): - """ - Return database objects of all stores associated with this UCI that have their - status set to value passed as parameter. - """ - return self.sa_session.query( model.CloudStore ).filter_by( deleted=False, uci_id=self.uci_id, status=status ).all() - - def get_all_stores( self ): - """ Returns all storage volumes' database objects associated with this UCI that have not been marked as 'deleted'. """ - return self.sa_session.query( model.CloudStore ) \ - .filter_by( deleted=False, uci_id=self.uci_id ) \ - .all() - - def get_snapshots( self, status=None ): - """ Returns database objects for all snapshots associated with this UCI and in given status.""" - return self.sa_session.query( model.CloudSnapshot ).filter_by( uci_id=self.uci_id, status=status ).all() - - def get_uci( self ): - """ Returns database object for given UCI. """ - uci = self.sa_session.query( model.UCI ).get( self.uci_id ) - self.sa_session.refresh( uci ) - return uci - - def get_uci_working_directory( self ): - return self.working_directory - - def uci_launch_time_set( self ): - uci = self.sa_session.query( model.UCI ).get( self.uci_id ) - self.sa_session.refresh( uci ) - return uci.launch_time - -class CloudProvider( object ): - def __init__( self, app ): - import providers.eucalyptus - import providers.ec2 - - self.app = app - self.cloud_provider = {} - self.cloud_provider["eucalyptus"] = providers.eucalyptus.EucalyptusCloudProvider( app ) - self.cloud_provider["ec2"] = providers.ec2.EC2CloudProvider( app ) - - def put( self, uci_wrapper ): - """ Put given request for UCI manipulation into provider's request queue.""" - self.cloud_provider[uci_wrapper.get_provider_type()].put( uci_wrapper ) - - def update( self ): - """ - Runs a global status update across all providers for all UCIs in state other than 'terminated' and 'available'. - Reason behind this method is to sync state of local DB and real world resources. - """ - for provider in self.cloud_provider.keys(): -# log.debug( "Running global update for provider: '%s'" % provider ) - self.cloud_provider[provider].update() - - def shutdown( self ): - for runner in self.cloud_provider.itervalues(): - runner.shutdown() - -class NoopCloudMonitor( object ): - """ - Implements the CloudMonitor interface but does nothing - """ - def put( self, *args ): - return - def shutdown( self ): - return -

1 0

galaxy-dist commit 040be2b94b8c: Added logging to EmailAction. Should gracefully fail if email is not available.
by commits-noreply＠bitbucket.org 29 Jun '10

29 Jun '10

# HG changeset patch -- Bitbucket.org # Project galaxy-dist # URL http://bitbucket.org/galaxy/galaxy-dist/overview # User Dannon Baker <dannon.baker(a)emory.edu> # Date 1277252544 14400 # Node ID 040be2b94b8ce845278d5ac19139a4bdae8fb088 # Parent beb30aadccd75bfdec6b06d4afcd704c2c16dbc3 Added logging to EmailAction. Should gracefully fail if email is not available. --- a/lib/galaxy/jobs/actions/post.py +++ b/lib/galaxy/jobs/actions/post.py @@ -67,7 +67,7 @@ class EmailAction(DefaultJobAction): def execute(cls, trans, action, job): smtp_server = trans.app.config.smtp_server if smtp_server is None: - return trans.show_error_message( "Mail is not configured for this galaxy instance, workflow action aborted." ) + log.error("Mail is not configured for this galaxy instance. Workflow action aborted.") # Build the email message msg = MIMEText( "Your job '%s' at Galaxy instance %s is complete as of %s." % (job.history.name, trans.request.host, job.update_time)) msg[ 'To' ] = job.user.email @@ -78,9 +78,8 @@ class EmailAction(DefaultJobAction): s.connect( smtp_server ) s.sendmail( frm, [ to ], msg.as_string() ) s.close() - return trans.show_ok_message( "Your error report has been sent" ) except Exception, e: - return trans.show_error_message( "An error occurred sending the report by email: %s" % str( e ) ) + log.error("EmailAction PJA Failed, exception: %s" % e) @classmethod def get_config_form(cls, trans):

1 0

galaxy-dist commit 45e1d5636fff: Fixed typo in jobs.actions.post
by commits-noreply＠bitbucket.org 29 Jun '10

29 Jun '10

# HG changeset patch -- Bitbucket.org # Project galaxy-dist # URL http://bitbucket.org/galaxy/galaxy-dist/overview # User Dannon Baker <dannon.baker(a)emory.edu> # Date 1277253455 14400 # Node ID 45e1d5636fffd284ea6cda2f14318b364370e985 # Parent a579674852103600ecd8e8b2549e76abc3917791 Fixed typo in jobs.actions.post --- a/lib/galaxy/jobs/actions/post.py +++ b/lib/galaxy/jobs/actions/post.py @@ -82,7 +82,7 @@ class EmailAction(DefaultJobAction): s.close() except Exception, e: log.error("EmailAction PJA Failed, exception: %s" % e) -L + @classmethod def get_config_form(cls, trans): form = """

1 0

galaxy-dist commit a57967485210: EmailAction now comes from 'galaxy-noreply@<host>'.
by commits-noreply＠bitbucket.org 29 Jun '10

29 Jun '10

# HG changeset patch -- Bitbucket.org # Project galaxy-dist # URL http://bitbucket.org/galaxy/galaxy-dist/overview # User Dannon Baker <dannon.baker(a)emory.edu> # Date 1277253193 14400 # Node ID a579674852103600ecd8e8b2549e76abc3917791 # Parent 040be2b94b8ce845278d5ac19139a4bdae8fb088 EmailAction now comes from 'galaxy-noreply@<host>'. --- a/lib/galaxy/jobs/actions/post.py +++ b/lib/galaxy/jobs/actions/post.py @@ -69,9 +69,11 @@ class EmailAction(DefaultJobAction): if smtp_server is None: log.error("Mail is not configured for this galaxy instance. Workflow action aborted.") # Build the email message + frm = 'galaxy-noreply@%s' % trans.request.host + to = job.user.email msg = MIMEText( "Your job '%s' at Galaxy instance %s is complete as of %s." % (job.history.name, trans.request.host, job.update_time)) - msg[ 'To' ] = job.user.email - msg[ 'From' ] = job.user.email + msg[ 'To' ] = to + msg[ 'From' ] = frm msg[ 'Subject' ] = "Galaxy workflow step notification '%s'" try: s = smtplib.SMTP() @@ -80,7 +82,7 @@ class EmailAction(DefaultJobAction): s.close() except Exception, e: log.error("EmailAction PJA Failed, exception: %s" % e) - +L @classmethod def get_config_form(cls, trans): form = """

1 0

galaxy-dist commit 752cb3e32537: Improvements to GOPS intersect: (a) preserve metadata for interval inputs; (b) allow arbitrary mix of interval and GFF inputs; and (c) functional tests updated to test new functionality.
by commits-noreply＠bitbucket.org 29 Jun '10

29 Jun '10

# HG changeset patch -- Bitbucket.org # Project galaxy-dist # URL http://bitbucket.org/galaxy/galaxy-dist/overview # User jeremy goecks <jeremy.goecks(a)emory.edu> # Date 1277306218 14400 # Node ID 752cb3e325374381bdd0db6b9163b12ba88b7151 # Parent 03eb69c6e92ea5472b7627bd9990327f7546a8a7 Improvements to GOPS intersect: (a) preserve metadata for interval inputs; (b) allow arbitrary mix of interval and GFF inputs; and (c) functional tests updated to test new functionality. --- a/tools/new_operations/intersect.xml +++ b/tools/new_operations/intersect.xml @@ -1,131 +1,110 @@ <tool id="gops_intersect_1" name="Intersect"><description>the intervals of two queries</description><command interpreter="python">gops_intersect.py - #if $inputs.type == "Interval": - $inputs.interval_input1 $inputs.interval_input2 $output - -1 ${inputs.interval_input1.metadata.chromCol},${inputs.interval_input1.metadata.startCol},${inputs.interval_input1.metadata.endCol},${inputs.interval_input1.metadata.strandCol} - -2 ${inputs.interval_input2.metadata.chromCol},${inputs.interval_input2.metadata.startCol},${inputs.interval_input2.metadata.endCol},${inputs.interval_input2.metadata.strandCol} - $inputs.interval_returntype - #else - $inputs.gff_input1 $inputs.gff_input2 $output - ## TODO: can we use metadata like above to set these columns rather than hardcode them? - -1 1,4,5,7 - -2 1,4,5,7 - --gff - $inputs.gff_returntype + $input1 $input2 $output + + #if isinstance( $input1.datatype, $__app__.datatypes_registry.get_datatype_by_extension('gff').__class__): + -1 1,4,5,7 --gff1 + #else: + -1 ${input1.metadata.chromCol},${input1.metadata.startCol},${input1.metadata.endCol},${input1.metadata.strandCol} #end if - -m $min + + #if isinstance( $input2.datatype, $__app__.datatypes_registry.get_datatype_by_extension('gff').__class__): + -2 1,4,5,7 --gff2 + #else: + -2 ${input2.metadata.chromCol},${input2.metadata.startCol},${input2.metadata.endCol},${input2.metadata.strandCol} + #end if + + -m $min $returntype </command><inputs> - <conditional name="inputs"> - <param name="type" type="select" label="File Format to Use"> - <option value="Interval">Interval</option> - <option value="GFF">GFF</option> - </param> - <when value="Interval"> - <param name="interval_returntype" type="select" label="Return" help="(see figure below)"> - <option value="">Overlapping Intervals</option> - <option value="-p">Overlapping pieces of Intervals</option> - </param> - <param format="interval" name="interval_input1" type="data" help="First query"> - <label>of</label> - </param> - <param format="interval" name="interval_input2" type="data" help="Second query"> - <label>that intersect</label> - </param> - </when> - <when value="GFF"> - <param name="gff_returntype" type="select" label="Return" help="(see figure below)"> - <option value="">Overlapping Intervals</option> - <option value="-p">Overlapping pieces of Intervals</option> - </param> - <param format="gff" name="gff_input1" type="data" help="First query"> - <label>of</label> - </param> - <param format="gff" name="gff_input2" type="data" help="Second query"> - <label>that intersect</label> - </param> - </when> - </conditional> + <param name="returntype" type="select" label="Return" help="(see figure below)"> + <option value="">Overlapping Intervals</option> + <option value="-p">Overlapping pieces of Intervals</option> + </param> + <param format="interval,gff" name="input1" type="data" help="First query"> + <label>of</label> + </param> + <param format="interval,gff" name="input2" type="data" help="Second query"> + <label>that intersect</label> + </param><param name="min" size="4" type="integer" value="1" help="(bp)"><label>for at least</label></param></inputs><outputs> - <data format="input" name="output"> - #if inputs.type == "Interval": - metadata_source="inputs.interval_input1" - #else: - metadata_source="inputs.gff_input1" - #end if - </data> + <data format="input" name="output" metadata_source="input1"/></outputs><code file="operation_filter.py"/><tests><test><param name="type" value="Interval"/> - <param name="interval_input1" value="1.bed" /> - <param name="interval_input2" value="2.bed" /> + <param name="input1" value="1.bed" /> + <param name="input2" value="2.bed" /><param name="min" value="1" /> - <param name="interval_returntype" value="" /> + <param name="returntype" value="" /><output name="output" file="gops_intersect_out.bed" /></test><test><param name="type" value="Interval"/> - <param name="interval_input1" value="1.bed" /> - <param name="interval_input2" value="2_mod.bed" ftype="interval"/> + <param name="input1" value="1.bed" /> + <param name="input2" value="2_mod.bed" ftype="interval"/><param name="min" value="1" /> - <param name="interval_returntype" value="" /> + <param name="returntype" value="" /><output name="output" file="gops_intersect_diffCols.bed" /></test><test> - <param name="type" value="Interval"/> - <param name="interval_input1" value="1.bed" /> - <param name="interval_input2" value="2_mod.bed" ftype="interval"/> + <param name="input1" value="1.bed" /> + <param name="input2" value="2_mod.bed" ftype="interval"/><param name="min" value="1" /> - <param name="interval_returntype" value="Overlapping pieces of Intervals" /> + <param name="returntype" value="Overlapping pieces of Intervals" /><output name="output" file="gops_intersect_p_diffCols.bed" /></test><test> - <param name="type" value="Interval"/> - <param name="interval_input1" value="1.bed" /> - <param name="interval_input2" value="2.bed" /> + <param name="input1" value="1.bed" /> + <param name="input2" value="2.bed" /><param name="min" value="10" /> - <param name="interval_returntype" value="Overlapping pieces of Intervals" /> + <param name="returntype" value="Overlapping pieces of Intervals" /><output name="output" file="gops_intersect_p_out.bed" /></test><test> - <param name="type" value="Interval"/> - <param name="interval_input1" value="gops_bigint.interval" ftype="interval" /> - <param name="interval_input2" value="gops_bigint2.interval" ftype="interval" /> + <param name="input1" value="gops_bigint.interval" ftype="interval" /> + <param name="input2" value="gops_bigint2.interval" ftype="interval" /><param name="min" value="1" /> - <param name="interval_returntype" value="" /> + <param name="returntype" value="" /><output name="output" file="gops_intersect_bigint_out.interval" /></test><test> - <param name="type" value="Interval"/> - <param name="interval_input1" value="gops_bigint2.interval" ftype="interval" /> - <param name="interval_input2" value="gops_bigint.interval" ftype="interval" /> + <param name="input1" value="gops_bigint2.interval" ftype="interval" /> + <param name="input2" value="gops_bigint.interval" ftype="interval" /><param name="min" value="1" /> - <param name="interval_returntype" value="" /> + <param name="returntype" value="" /><output name="output" file="gops_intersect_bigint_out.interval" /></test><test> - <param name="type" value="Interval"/> - <param name="interval_input1" value="12.bed" ftype="bed" /> - <param name="interval_input2" value="1.bed" ftype="bed" /> + <param name="input1" value="12.bed" ftype="bed" /> + <param name="input2" value="1.bed" ftype="bed" /><param name="min" value="1" /> - <param name="interval_returntype" value="" /> + <param name="returntype" value="" /><output name="output" file="gops_intersect_no_strand_out.bed" /></test> + <test> - <param name="type" value="GFF"/> - <param name="gff_input1" value="gops_subtract_in1.gff" /> - <param name="gff_input2" value="gops_subtract_in2.gff" /> + <param name="input1" value="gops_subtract_in1.gff" /> + <param name="input2" value="gops_subtract_in2.gff" /><param name="min" value="1" /> - <param name="gff_returntype" value="" /> + <param name="returntype" value="" /><output name="output" file="gops_intersect_out2.gff" /></test> +  + <test> + <param name="input1" value="gops_subtract_in1.gff" /> + <param name="input2" value="gops_subtract_in2.bed" /> + <param name="min" value="1" /> + <param name="returntype" value="" /> + <output name="output" file="gops_intersect_out2.gff" /> + </test> + </tests><help> --- a/tools/new_operations/subtract.xml +++ b/tools/new_operations/subtract.xml @@ -3,7 +3,6 @@ <command interpreter="python">gops_subtract.py $input1 $input2 $output - ##if $input1.ext in ['gff','gtf','gff3']: #if isinstance( $input1.datatype, $__app__.datatypes_registry.get_datatype_by_extension('gff').__class__): -1 1,4,5,7 --gff1 #else: --- a/tools/new_operations/gops_intersect.py +++ b/tools/new_operations/gops_intersect.py @@ -1,13 +1,14 @@ #!/usr/bin/env python """ -Find regions of first bed file that overlap regions in a second bed file +Find regions of first interval/GFF file that overlap regions in a second interval/GFF file usage: %prog bed_file_1 bed_file_2 out_file -1, --cols1=N,N,N,N: Columns for start, end, strand in first file -2, --cols2=N,N,N,N: Columns for start, end, strand in second file -m, --mincols=N: Require this much overlap (default 1bp) -p, --pieces: just print pieces of second set (after padding) - -G, --gff: inputs are GFF format, meaning start and end coordinates are 1-based, closed interval + -G, --gff1: input 1 is GFF format, meaning start and end coordinates are 1-based, closed interval + -H, --gff2: input 2 is GFF format, meaning start and end coordinates are 1-based, closed interval """ from galaxy import eggs import pkg_resources @@ -34,24 +35,29 @@ def main(): chr_col_2, start_col_2, end_col_2, strand_col_2 = parse_cols_arg( options.cols2 ) if options.mincols: mincols = int( options.mincols ) pieces = bool( options.pieces ) - gff_format = bool( options.gff ) + in1_gff_format = bool( options.gff1 ) + in2_gff_format = bool( options.gff2 ) in_fname, in2_fname, out_fname = args except: doc_optparse.exception() - # Set reader to handle either GFF or default format. - if gff_format: - reader_wrapper = GFFReaderWrapper + # Set readers to handle either GFF or default format. + if in1_gff_format: + in1_reader_wrapper = GFFReaderWrapper else: - reader_wrapper = NiceReaderWrapper + in1_reader_wrapper = NiceReaderWrapper + if in2_gff_format: + in2_reader_wrapper = GFFReaderWrapper + else: + in2_reader_wrapper = NiceReaderWrapper - g1 = reader_wrapper( fileinput.FileInput( in_fname ), + g1 = in1_reader_wrapper( fileinput.FileInput( in_fname ), chrom_col=chr_col_1, start_col=start_col_1, end_col=end_col_1, strand_col=strand_col_1, fix_strand=True ) - g2 = reader_wrapper( fileinput.FileInput( in2_fname ), + g2 = in2_reader_wrapper( fileinput.FileInput( in2_fname ), chrom_col=chr_col_2, start_col=start_col_2, end_col=end_col_2, @@ -63,7 +69,7 @@ def main(): try: for line in intersect( [g1,g2], pieces=pieces, mincols=mincols ): if type( line ) == GenomicInterval: - if gff_format: + if in1_gff_format: line = convert_to_gff_coordinates( line ) out_file.write( "%s\n" % "\t".join( line.fields ) ) else:

1 0

galaxy-dist commit 03eb69c6e92e: Improvements to GOPS subtract: (a) preserve metadata for interval inputs; (b) allow arbitrary mix of interval and GFF inputs; and (c) functional tests updated to test new functionality.
by commits-noreply＠bitbucket.org 29 Jun '10

29 Jun '10

# HG changeset patch -- Bitbucket.org # Project galaxy-dist # URL http://bitbucket.org/galaxy/galaxy-dist/overview # User jeremy goecks <jeremy.goecks(a)emory.edu> # Date 1277304197 14400 # Node ID 03eb69c6e92ea5472b7627bd9990327f7546a8a7 # Parent 45e1d5636fffd284ea6cda2f14318b364370e985 Improvements to GOPS subtract: (a) preserve metadata for interval inputs; (b) allow arbitrary mix of interval and GFF inputs; and (c) functional tests updated to test new functionality. --- a/tools/new_operations/gops_subtract.py +++ b/tools/new_operations/gops_subtract.py @@ -8,7 +8,8 @@ usage: %prog bed_file_1 bed_file_2 out_f -2, --cols2=N,N,N,N: Columns for start, end, strand in second file -m, --mincols=N: Require this much overlap (default 1bp) -p, --pieces: just print pieces of second set (after padding) - -G, --gff: inputs are GFF format, meaning start and end coordinates are 1-based, closed interval + -G, --gff1: input 1 is GFF format, meaning start and end coordinates are 1-based, closed interval + -H, --gff2: input 2 is GFF format, meaning start and end coordinates are 1-based, closed interval """ from galaxy import eggs import pkg_resources @@ -35,24 +36,29 @@ def main(): chr_col_2, start_col_2, end_col_2, strand_col_2 = parse_cols_arg( options.cols2 ) if options.mincols: mincols = int( options.mincols ) pieces = bool( options.pieces ) - gff_format = bool( options.gff ) + in1_gff_format = bool( options.gff1 ) + in2_gff_format = bool( options.gff2 ) in_fname, in2_fname, out_fname = args except: doc_optparse.exception() - # Set reader to handle either GFF or default format. - if gff_format: - reader_wrapper = GFFReaderWrapper + # Set readers to handle either GFF or default format. + if in1_gff_format: + in1_reader_wrapper = GFFReaderWrapper else: - reader_wrapper = NiceReaderWrapper + in1_reader_wrapper = NiceReaderWrapper + if in2_gff_format: + in2_reader_wrapper = GFFReaderWrapper + else: + in2_reader_wrapper = NiceReaderWrapper - g1 = reader_wrapper( fileinput.FileInput( in_fname ), + g1 = in1_reader_wrapper( fileinput.FileInput( in_fname ), chrom_col=chr_col_1, start_col=start_col_1, end_col=end_col_1, strand_col=strand_col_1, fix_strand=True ) - g2 = reader_wrapper( fileinput.FileInput( in2_fname ), + g2 = in2_reader_wrapper( fileinput.FileInput( in2_fname ), chrom_col=chr_col_2, start_col=start_col_2, end_col=end_col_2, @@ -64,7 +70,7 @@ def main(): try: for line in subtract( [g1,g2], pieces=pieces, mincols=mincols ): if type( line ) is GenomicInterval: - if gff_format: + if in1_gff_format: line = convert_to_gff_coordinates( line ) out_file.write( "%s\n" % "\t".join( line.fields ) ) else: --- a/tools/new_operations/operation_filter.py +++ b/tools/new_operations/operation_filter.py @@ -28,13 +28,17 @@ def validate_input( trans, error_map, pa # check meta data try: param = param_values[name] - startCol = int( param.metadata.startCol ) - endCol = int( param.metadata.endCol ) - chromCol = int( param.metadata.chromCol ) - if param.metadata.strandCol is not None: - strandCol = int ( param.metadata.strandCol ) - else: - strandCol = 0 + if isinstance( param.datatype, trans.app.datatypes_registry.get_datatype_by_extension( 'gff' ).__class__ ): + # TODO: currently cannot validate GFF inputs b/c they are not derived from interval. + pass + else: # Validate interval datatype. + startCol = int( param.metadata.startCol ) + endCol = int( param.metadata.endCol ) + chromCol = int( param.metadata.chromCol ) + if param.metadata.strandCol is not None: + strandCol = int ( param.metadata.strandCol ) + else: + strandCol = 0 except: error_msg = "The attributes of this dataset are not properly set. " + \ "Click the pencil icon in the history item to set the chrom, start, end and strand columns." --- /dev/null +++ b/test-data/gops_subtract_in2.bed @@ -0,0 +1,500 @@ +chr13 3633323 3651020 transcript 0 + +chr13 3633323 3633421 exon 0 + +chr13 3641310 3641498 exon 0 + +chr13 3642720 3642871 exon 0 + +chr13 3644161 3644296 exon 0 + +chr13 3648564 3648756 exon 0 + +chr13 3649425 3651020 exon 0 + +chr13 3802138 3803564 transcript 0 - +chr13 3802138 3803564 exon 0 - +chr13 3881808 3892824 transcript 0 - +chr13 3881808 3883719 exon 0 - +chr13 3884048 3884235 exon 0 - +chr13 3885293 3885464 exon 0 - +chr13 3885783 3886041 exon 0 - +chr13 3886616 3886693 exon 0 - +chr13 3886816 3886913 exon 0 - +chr13 3887223 3887286 exon 0 - +chr13 3887600 3887768 exon 0 - +chr13 3888116 3888224 exon 0 - +chr13 3892538 3892824 exon 0 - +chr13 4131872 4149877 transcript 0 - +chr13 4131872 4132082 exon 0 - +chr13 4134448 4134531 exon 0 - +chr13 4135847 4136013 exon 0 - +chr13 4136379 4136489 exon 0 - +chr13 4141372 4141495 exon 0 - +chr13 4142463 4142541 exon 0 - +chr13 4143535 4143652 exon 0 - +chr13 4144451 4144619 exon 0 - +chr13 4149760 4149877 exon 0 - +chr13 4232985 4247605 transcript 0 + +chr13 4232985 4233100 exon 0 + +chr13 4235334 4235502 exon 0 + +chr13 4236281 4236398 exon 0 + +chr13 4237639 4237717 exon 0 + +chr13 4238213 4238336 exon 0 + +chr13 4241797 4241907 exon 0 + +chr13 4242172 4242338 exon 0 + +chr13 4246048 4246131 exon 0 + +chr13 4247323 4247605 exon 0 + +chr13 4247980 4249023 transcript 0 + +chr13 4247980 4249023 exon 0 + +chr13 4591735 4608410 transcript 0 - +chr13 4591735 4592544 exon 0 - +chr13 4592807 4592890 exon 0 - +chr13 4594316 4594400 exon 0 - +chr13 4594905 4594978 exon 0 - +chr13 4596730 4596828 exon 0 - +chr13 4598015 4598138 exon 0 - +chr13 4600463 4600541 exon 0 - +chr13 4601926 4602043 exon 0 - +chr13 4606636 4606804 exon 0 - +chr13 4608330 4608410 exon 0 - +chr13 5860734 5869639 transcript 0 + +chr13 5860734 5861088 exon 0 + +chr13 5864013 5864590 exon 0 + +chr13 5865884 5866008 exon 0 + +chr13 5866477 5869639 exon 0 + +chr13 3537320 3565507 transcript 0 + +chr13 3537320 3537589 exon 0 + +chr13 3548108 3548216 exon 0 + +chr13 3550234 3550334 exon 0 + +chr13 3553657 3553792 exon 0 + +chr13 3555559 3555758 exon 0 + +chr13 3556171 3556303 exon 0 + +chr13 3559237 3559337 exon 0 + +chr13 3561111 3561283 exon 0 + +chr13 3563785 3563930 exon 0 + +chr13 3564104 3564159 exon 0 + +chr13 3564241 3565507 exon 0 + +chr13 3565280 3610354 transcript 0 - +chr13 3565280 3565913 exon 0 - +chr13 3566163 3566278 exon 0 - +chr13 3566681 3566863 exon 0 - +chr13 3567997 3568103 exon 0 - +chr13 3568733 3568887 exon 0 - +chr13 3569557 3569683 exon 0 - +chr13 3572732 3576446 exon 0 - +chr13 3580997 3581439 exon 0 - +chr13 3583745 3584619 exon 0 - +chr13 3587543 3587780 exon 0 - +chr13 3589186 3589894 exon 0 - +chr13 3593376 3593439 exon 0 - +chr13 3593538 3593611 exon 0 - +chr13 3594742 3594846 exon 0 - +chr13 3596022 3596123 exon 0 - +chr13 3598897 3598997 exon 0 - +chr13 3599082 3599102 exon 0 - +chr13 3599184 3599308 exon 0 - +chr13 3599437 3599580 exon 0 - +chr13 3610035 3610354 exon 0 - +chr13 8202154 8759554 transcript 0 + +chr13 8202154 8202566 exon 0 + +chr13 8558349 8558436 exon 0 + +chr13 8568912 8569820 exon 0 + +chr13 8671650 8671765 exon 0 + +chr13 8696865 8697034 exon 0 + +chr13 8700825 8700977 exon 0 + +chr13 8712845 8713014 exon 0 + +chr13 8731035 8731217 exon 0 + +chr13 8751818 8751997 exon 0 + +chr13 8756471 8759554 exon 0 + +chr13 8884851 8891641 transcript 0 + +chr13 8884851 8885242 exon 0 + +chr13 8885977 8886150 exon 0 + +chr13 8886730 8886823 exon 0 + +chr13 8887168 8887299 exon 0 + +chr13 8889563 8891641 exon 0 + +chr13 8802213 8870288 transcript 0 - +chr13 8802213 8805192 exon 0 - +chr13 8819108 8819223 exon 0 - +chr13 8819656 8819791 exon 0 - +chr13 8834542 8834684 exon 0 - +chr13 8836028 8836269 exon 0 - +chr13 8841975 8842052 exon 0 - +chr13 8844032 8844077 exon 0 - +chr13 8846860 8846932 exon 0 - +chr13 8848779 8848915 exon 0 - +chr13 8852915 8852980 exon 0 - +chr13 8853245 8853341 exon 0 - +chr13 8856077 8856174 exon 0 - +chr13 8860333 8860511 exon 0 - +chr13 8870115 8870288 exon 0 - +chr13 8849724 8870288 transcript 0 - +chr13 8849724 8852980 exon 0 - +chr13 8853245 8853341 exon 0 - +chr13 8856077 8856174 exon 0 - +chr13 8860333 8860511 exon 0 - +chr13 8870115 8870288 exon 0 - +chr13 8971723 8995258 transcript 0 - +chr13 8971723 8972520 exon 0 - +chr13 8973409 8973553 exon 0 - +chr13 8974190 8974256 exon 0 - +chr13 8976481 8976679 exon 0 - +chr13 8977773 8977874 exon 0 - +chr13 8978674 8978726 exon 0 - +chr13 8979147 8979225 exon 0 - +chr13 8984454 8984565 exon 0 - +chr13 8984883 8984973 exon 0 - +chr13 8986494 8986560 exon 0 - +chr13 8987001 8987193 exon 0 - +chr13 8988298 8988391 exon 0 - +chr13 8989943 8990044 exon 0 - +chr13 8990960 8991097 exon 0 - +chr13 8991177 8991281 exon 0 - +chr13 8991890 8992061 exon 0 - +chr13 8995194 8995258 exon 0 - +chr13 6547402 6579395 transcript 0 + +chr13 6547402 6547475 exon 0 + +chr13 6548825 6548928 exon 0 + +chr13 6551879 6551986 exon 0 + +chr13 6552427 6552579 exon 0 + +chr13 6554599 6554714 exon 0 + +chr13 6554797 6554894 exon 0 + +chr13 6555787 6555948 exon 0 + +chr13 6557007 6557134 exon 0 + +chr13 6557401 6557490 exon 0 + +chr13 6558600 6558729 exon 0 + +chr13 6559275 6559389 exon 0 + +chr13 6559847 6559944 exon 0 + +chr13 6561198 6561333 exon 0 + +chr13 6562621 6562760 exon 0 + +chr13 6564236 6564353 exon 0 + +chr13 6566595 6566728 exon 0 + +chr13 6567909 6568030 exon 0 + +chr13 6568144 6568221 exon 0 + +chr13 6568445 6568611 exon 0 + +chr13 6569847 6569948 exon 0 + +chr13 6572286 6572407 exon 0 + +chr13 6573605 6573677 exon 0 + +chr13 6574262 6574375 exon 0 + +chr13 6576635 6576761 exon 0 + +chr13 6577686 6577832 exon 0 + +chr13 6578663 6578766 exon 0 + +chr13 6578895 6579395 exon 0 + +chr13 6579119 6647970 transcript 0 - +chr13 6579119 6580838 exon 0 - +chr13 6581648 6581751 exon 0 - +chr13 6583846 6583946 exon 0 - +chr13 6585725 6585837 exon 0 - +chr13 6586297 6586359 exon 0 - +chr13 6587734 6587899 exon 0 - +chr13 6597104 6597257 exon 0 - +chr13 6597984 6598072 exon 0 - +chr13 6599841 6599912 exon 0 - +chr13 6601958 6602105 exon 0 - +chr13 6602324 6602394 exon 0 - +chr13 6602644 6602709 exon 0 - +chr13 6604201 6604327 exon 0 - +chr13 6604881 6604974 exon 0 - +chr13 6613949 6614045 exon 0 - +chr13 6618420 6618529 exon 0 - +chr13 6618765 6618810 exon 0 - +chr13 6620131 6620297 exon 0 - +chr13 6621152 6621342 exon 0 - +chr13 6624381 6624459 exon 0 - +chr13 6635170 6635244 exon 0 - +chr13 6647816 6647970 exon 0 - +chr13 9093150 9172336 transcript 0 + +chr13 9093150 9093426 exon 0 + +chr13 9121353 9121472 exon 0 + +chr13 9123148 9123208 exon 0 + +chr13 9136024 9136172 exon 0 + +chr13 9136400 9136547 exon 0 + +chr13 9142999 9143078 exon 0 + +chr13 9144627 9144764 exon 0 + +chr13 9146646 9146750 exon 0 + +chr13 9149071 9149182 exon 0 + +chr13 9150110 9150164 exon 0 + +chr13 9150260 9150470 exon 0 + +chr13 9157368 9157475 exon 0 + +chr13 9157796 9158048 exon 0 + +chr13 9163920 9163966 exon 0 + +chr13 9165562 9165727 exon 0 + +chr13 9167923 9168048 exon 0 + +chr13 9168143 9168252 exon 0 + +chr13 9169897 9172336 exon 0 + +chr13 9275771 9668171 transcript 0 + +chr13 9275771 9276312 exon 0 + +chr13 9492350 9492422 exon 0 + +chr13 9505824 9505935 exon 0 + +chr13 9532497 9532623 exon 0 + +chr13 9535901 9536111 exon 0 + +chr13 9549429 9549564 exon 0 + +chr13 9550991 9551111 exon 0 + +chr13 9552519 9552717 exon 0 + +chr13 9559916 9560008 exon 0 + +chr13 9562359 9562470 exon 0 + +chr13 9567076 9567200 exon 0 + +chr13 9567546 9567656 exon 0 + +chr13 9567836 9567939 exon 0 + +chr13 9570295 9570360 exon 0 + +chr13 9574379 9574473 exon 0 + +chr13 9574607 9574727 exon 0 + +chr13 9576049 9576164 exon 0 + +chr13 9583250 9583267 exon 0 + +chr13 9591989 9592004 exon 0 + +chr13 9600631 9600742 exon 0 + +chr13 9603734 9603871 exon 0 + +chr13 9605568 9605777 exon 0 + +chr13 9608189 9608304 exon 0 + +chr13 9609942 9610144 exon 0 + +chr13 9613592 9613702 exon 0 + +chr13 9614990 9615071 exon 0 + +chr13 9621124 9621248 exon 0 + +chr13 9621827 9621949 exon 0 + +chr13 9623002 9623114 exon 0 + +chr13 9623203 9623313 exon 0 + +chr13 9627218 9627349 exon 0 + +chr13 9633967 9634136 exon 0 + +chr13 9636340 9636511 exon 0 + +chr13 9646224 9646286 exon 0 + +chr13 9653848 9653906 exon 0 + +chr13 9655882 9655957 exon 0 + +chr13 9658490 9658665 exon 0 + +chr13 9661382 9661506 exon 0 + +chr13 9665023 9668171 exon 0 + +chr13 9684081 9764454 transcript 0 - +chr13 9684081 9686230 exon 0 - +chr13 9688308 9688494 exon 0 - +chr13 9688697 9688970 exon 0 - +chr13 9689691 9689760 exon 0 - +chr13 9690148 9690356 exon 0 - +chr13 9692675 9692794 exon 0 - +chr13 9693514 9693592 exon 0 - +chr13 9694374 9694430 exon 0 - +chr13 9694984 9695072 exon 0 - +chr13 9696889 9696982 exon 0 - +chr13 9697873 9697951 exon 0 - +chr13 9720018 9720178 exon 0 - +chr13 9734577 9734712 exon 0 - +chr13 9764300 9764454 exon 0 - +chr13 9875858 10360049 transcript 0 - +chr13 9875858 9878263 exon 0 - +chr13 10027698 10027745 exon 0 - +chr13 10121472 10121535 exon 0 - +chr13 10223890 10224003 exon 0 - +chr13 10359509 10360049 exon 0 - +chr13 11645369 12199212 transcript 0 - +chr13 11645369 11646878 exon 0 - +chr13 11647697 11647749 exon 0 - +chr13 11648864 11648965 exon 0 - +chr13 11650313 11650378 exon 0 - +chr13 11652790 11652947 exon 0 - +chr13 11659114 11659249 exon 0 - +chr13 11660738 11660885 exon 0 - +chr13 11664512 11664573 exon 0 - +chr13 11670046 11670180 exon 0 - +chr13 11676402 11676445 exon 0 - +chr13 11677674 11677805 exon 0 - +chr13 11679482 11679704 exon 0 - +chr13 11680408 11680495 exon 0 - +chr13 11683491 11683639 exon 0 - +chr13 11685359 11685424 exon 0 - +chr13 11686948 11688246 exon 0 - +chr13 11689337 11689419 exon 0 - +chr13 11690772 11690877 exon 0 - +chr13 11694025 11694154 exon 0 - +chr13 11695951 11696040 exon 0 - +chr13 11697422 11697503 exon 0 - +chr13 11710873 11710947 exon 0 - +chr13 11711736 11711813 exon 0 - +chr13 11713581 11713661 exon 0 - +chr13 11715168 11715268 exon 0 - +chr13 11727791 11727845 exon 0 - +chr13 11730698 11730750 exon 0 - +chr13 11732540 11732644 exon 0 - +chr13 11736070 11736167 exon 0 - +chr13 11738759 11738872 exon 0 - +chr13 11739968 11740004 exon 0 - +chr13 11741996 11742131 exon 0 - +chr13 11746632 11746692 exon 0 - +chr13 11747884 11748055 exon 0 - +chr13 11749299 11749392 exon 0 - +chr13 11751216 11751304 exon 0 - +chr13 11752367 11752608 exon 0 - +chr13 11754332 11754653 exon 0 - +chr13 11758147 11758278 exon 0 - +chr13 11759345 11759427 exon 0 - +chr13 11761079 11761318 exon 0 - +chr13 11762195 11762256 exon 0 - +chr13 11768464 11768514 exon 0 - +chr13 11773232 11773354 exon 0 - +chr13 11776728 11776793 exon 0 - +chr13 11779209 11779325 exon 0 - +chr13 11780156 11780280 exon 0 - +chr13 11782558 11782634 exon 0 - +chr13 11784905 11784983 exon 0 - +chr13 11792514 11792652 exon 0 - +chr13 11793045 11793135 exon 0 - +chr13 11795763 11795842 exon 0 - +chr13 11797901 11798065 exon 0 - +chr13 11798848 11798989 exon 0 - +chr13 11800048 11800139 exon 0 - +chr13 11800288 11800509 exon 0 - +chr13 11802183 11802353 exon 0 - +chr13 11804401 11804522 exon 0 - +chr13 11806783 11806889 exon 0 - +chr13 11809207 11809394 exon 0 - +chr13 11810636 11810772 exon 0 - +chr13 11814017 11814121 exon 0 - +chr13 11814997 11815130 exon 0 - +chr13 11816529 11816644 exon 0 - +chr13 11819123 11819397 exon 0 - +chr13 11822609 11822753 exon 0 - +chr13 11823912 11824018 exon 0 - +chr13 11827737 11827938 exon 0 - +chr13 11829988 11830787 exon 0 - +chr13 11834120 11834347 exon 0 - +chr13 11837444 11837531 exon 0 - +chr13 11837863 11838026 exon 0 - +chr13 11839754 11839915 exon 0 - +chr13 11841702 11841817 exon 0 - +chr13 11842955 11843308 exon 0 - +chr13 11844462 11844671 exon 0 - +chr13 11851937 11852112 exon 0 - +chr13 11853493 11853702 exon 0 - +chr13 11862118 11862266 exon 0 - +chr13 11864686 11864846 exon 0 - +chr13 11871451 11871535 exon 0 - +chr13 11877304 11877408 exon 0 - +chr13 11882525 11882630 exon 0 - +chr13 11884935 11885152 exon 0 - +chr13 11886808 11887001 exon 0 - +chr13 11891948 11892190 exon 0 - +chr13 11893404 11893538 exon 0 - +chr13 11894073 11894192 exon 0 - +chr13 11903149 11903245 exon 0 - +chr13 11916540 11916676 exon 0 - +chr13 11919800 11919984 exon 0 - +chr13 11921823 11921945 exon 0 - +chr13 11926190 11926355 exon 0 - +chr13 11943324 11943481 exon 0 - +chr13 11945382 11945457 exon 0 - +chr13 11960383 11960480 exon 0 - +chr13 11961383 11961483 exon 0 - +chr13 11971741 11971854 exon 0 - +chr13 11975311 11975390 exon 0 - +chr13 11977730 11977805 exon 0 - +chr13 11983100 11983115 exon 0 - +chr13 11995484 11995505 exon 0 - +chr13 12010577 12010682 exon 0 - +chr13 12038149 12038269 exon 0 - +chr13 12198665 12199212 exon 0 - +chr13 12279085 12350267 transcript 0 - +chr13 12279085 12279358 exon 0 - +chr13 12280307 12280420 exon 0 - +chr13 12281613 12281806 exon 0 - +chr13 12282470 12282671 exon 0 - +chr13 12285880 12286077 exon 0 - +chr13 12287504 12287660 exon 0 - +chr13 12290179 12290255 exon 0 - +chr13 12291312 12291411 exon 0 - +chr13 12292463 12292545 exon 0 - +chr13 12297009 12297130 exon 0 - +chr13 12297556 12297624 exon 0 - +chr13 12304654 12304755 exon 0 - +chr13 12304937 12305045 exon 0 - +chr13 12307679 12307832 exon 0 - +chr13 12309076 12309166 exon 0 - +chr13 12313688 12313829 exon 0 - +chr13 12314421 12314538 exon 0 - +chr13 12317728 12317908 exon 0 - +chr13 12319950 12320136 exon 0 - +chr13 12323211 12323352 exon 0 - +chr13 12325168 12325281 exon 0 - +chr13 12327723 12327803 exon 0 - +chr13 12330247 12330315 exon 0 - +chr13 12331915 12331977 exon 0 - +chr13 12333831 12333932 exon 0 - +chr13 12336201 12336296 exon 0 - +chr13 12336821 12336884 exon 0 - +chr13 12339585 12339692 exon 0 - +chr13 12340159 12340252 exon 0 - +chr13 12342111 12342181 exon 0 - +chr13 12342868 12342958 exon 0 - +chr13 12346030 12346245 exon 0 - +chr13 12350128 12350267 exon 0 - +chr13 12361693 12432999 transcript 0 - +chr13 12361693 12361919 exon 0 - +chr13 12363044 12363203 exon 0 - +chr13 12364965 12365031 exon 0 - +chr13 12367344 12367491 exon 0 - +chr13 12368629 12368809 exon 0 - +chr13 12369665 12369800 exon 0 - +chr13 12371060 12371243 exon 0 - +chr13 12372695 12372836 exon 0 - +chr13 12374781 12374890 exon 0 - +chr13 12380773 12380924 exon 0 - +chr13 12382940 12383088 exon 0 - +chr13 12383926 12384157 exon 0 - +chr13 12386576 12386669 exon 0 - +chr13 12388741 12388827 exon 0 - +chr13 12393201 12393283 exon 0 - +chr13 12396580 12396659 exon 0 - +chr13 12398540 12398628 exon 0 - +chr13 12401166 12401253 exon 0 - +chr13 12401861 12401981 exon 0 - +chr13 12403113 12403228 exon 0 - +chr13 12432641 12432999 exon 0 - +chr13 12487641 12531160 transcript 0 + +chr13 12487641 12487737 exon 0 + +chr13 12488006 12488180 exon 0 + +chr13 12488688 12488905 exon 0 + +chr13 12491169 12491311 exon 0 + +chr13 12493316 12493418 exon 0 + +chr13 12493716 12493857 exon 0 + +chr13 12494980 12495192 exon 0 + +chr13 12495411 12495545 exon 0 + +chr13 12497154 12497257 exon 0 + +chr13 12498289 12498400 exon 0 + +chr13 12498791 12498909 exon 0 + +chr13 12499783 12499891 exon 0 + +chr13 12500920 12501016 exon 0 + +chr13 12501475 12501564 exon 0 + +chr13 12502662 12502874 exon 0 + +chr13 12503486 12503608 exon 0 + +chr13 12504311 12504503 exon 0 + +chr13 12505491 12505675 exon 0 + +chr13 12505778 12505915 exon 0 + +chr13 12506639 12506832 exon 0 + +chr13 12507683 12507853 exon 0 + +chr13 12508217 12508376 exon 0 + +chr13 12509726 12509969 exon 0 + +chr13 12510364 12510496 exon 0 + +chr13 12513308 12513412 exon 0 + +chr13 12513544 12513688 exon 0 + +chr13 12514307 12514426 exon 0 + +chr13 12514507 12514629 exon 0 + +chr13 12515900 12516029 exon 0 + +chr13 12516886 12517117 exon 0 + +chr13 12518420 12518547 exon 0 + +chr13 12519028 12519189 exon 0 + +chr13 12522033 12522080 exon 0 + +chr13 12522407 12522531 exon 0 + +chr13 12523177 12523318 exon 0 + +chr13 12523946 12524239 exon 0 + +chr13 12524895 12525046 exon 0 + +chr13 12525878 12526037 exon 0 + +chr13 12526183 12526318 exon 0 + +chr13 12526453 12526558 exon 0 + +chr13 12526640 12526809 exon 0 + +chr13 12527313 12527468 exon 0 + +chr13 12527936 12528095 exon 0 + +chr13 12530100 12530209 exon 0 + +chr13 12530855 12531160 exon 0 + +chr13 12531685 12553757 transcript 0 - +chr13 12531685 12533261 exon 0 - +chr13 12539413 12539579 exon 0 - +chr13 12540685 12540774 exon 0 - +chr13 12543667 12543694 exon 0 - +chr13 12544988 12545045 exon 0 - +chr13 12545551 12545671 exon 0 - +chr13 12547001 12547212 exon 0 - +chr13 12548610 12548699 exon 0 - +chr13 12551440 12551583 exon 0 - +chr13 12553693 12553757 exon 0 - +chr13 12569174 12612715 transcript 0 - +chr13 12569174 12570841 exon 0 - +chr13 12575866 12575912 exon 0 - --- a/tools/new_operations/subtract.xml +++ b/tools/new_operations/subtract.xml @@ -1,46 +1,33 @@ <tool id="gops_subtract_1" name="Subtract"><description>the intervals of two queries</description><command interpreter="python">gops_subtract.py - #if $inputs.type == "Interval": - $inputs.interval_input1 $inputs.interval_input2 $output - -1 ${inputs.interval_input1.metadata.chromCol},${inputs.interval_input1.metadata.startCol},${inputs.interval_input1.metadata.endCol},${inputs.interval_input1.metadata.strandCol} - -2 ${inputs.interval_input2.metadata.chromCol},${inputs.interval_input2.metadata.startCol},${inputs.interval_input2.metadata.endCol},${inputs.interval_input2.metadata.strandCol} - #else - $inputs.gff_input1 $inputs.gff_input2 $output - ## TODO: can we use metadata like above to set these columns rather than hardcode them? - -1 1,4,5,7 - -2 1,4,5,7 - --gff + $input1 $input2 $output + + ##if $input1.ext in ['gff','gtf','gff3']: + #if isinstance( $input1.datatype, $__app__.datatypes_registry.get_datatype_by_extension('gff').__class__): + -1 1,4,5,7 --gff1 + #else: + -1 ${input1.metadata.chromCol},${input1.metadata.startCol},${input1.metadata.endCol},${input1.metadata.strandCol} #end if + + #if isinstance( $input2.datatype, $__app__.datatypes_registry.get_datatype_by_extension('gff').__class__): + -2 1,4,5,7 --gff2 + #else: + -2 ${input2.metadata.chromCol},${input2.metadata.startCol},${input2.metadata.endCol},${input2.metadata.strandCol} + #end if + -m $min $returntype </command><inputs> - <conditional name="inputs"> - <param name="type" type="select" label="File Format to Use"> - <option value="Interval">Interval</option> - <option value="GFF">GFF</option> - </param> - <when value="Interval"> - <param format="interval" name="interval_input2" type="data" help="Second query"> - <label>Subtract</label> - </param> - - <param format="interval" name="interval_input1" type="data" help="First query"> - <label>from</label> - </param> - </when> - <when value="GFF"> - <param format="gff" name="gff_input2" type="data" help="Second query"> - <label>Subtract</label> - </param> - - <param format="gff" name="gff_input1" type="data" help="First query"> - <label>from</label> - </param> - </when> - </conditional> + <param format="interval,gff" name="input2" type="data" help="Second query"> + <label>Subtract</label> + </param> -<param name="returntype" type="select" label="Return" help="of the first query (see figure below)"> + <param format="interval,gff" name="input1" type="data" help="First query"> + <label>from</label> + </param> + + <param name="returntype" type="select" label="Return" help="of the first query (see figure below)"><option value="">Intervals with no overlap</option><option value="-p">Non-overlapping pieces of intervals</option></param> @@ -48,59 +35,57 @@ <param name="min" size="4" type="integer" value="1" help="(bp)"><label>where minimal overlap is</label></param> - </inputs><outputs> - <data format="input" name="output"> - #if inputs.type == "Interval": - metadata_source="inputs.interval_input1" - #else: - metadata_source="inputs.gff_input1" - #end if - </data> + <data format="input" name="output" metadata_source="input1"/></outputs><code file="operation_filter.py"/><tests><test> - <param name="type" value="Interval"/> - <param name="interval_input1" value="1.bed" /> - <param name="interval_input2" value="2.bed" /> + <param name="input1" value="1.bed" /> + <param name="input2" value="2.bed" /><param name="min" value="1" /><param name="returntype" value="" /><output name="output" file="gops-subtract.dat" /></test><test> - <param name="type" value="Interval"/> - <param name="interval_input1" value="1.bed" /> - <param name="interval_input2" value="2_mod.bed" ftype="interval"/> + <param name="input1" value="1.bed" /> + <param name="input2" value="2_mod.bed" ftype="interval"/><param name="min" value="1" /><param name="returntype" value="" /><output name="output" file="gops_subtract_diffCols.dat" /></test><test> - <param name="type" value="Interval"/> - <param name="interval_input1" value="gops_subtract_bigint.bed" /> - <param name="interval_input2" value="2.bed" /> + <param name="input1" value="gops_subtract_bigint.bed" /> + <param name="input2" value="2.bed" /><param name="min" value="1" /><param name="returntype" value="" /><output name="output" file="gops-subtract.dat" /></test><test> - <param name="type" value="Interval"/> - <param name="interval_input1" value="1.bed" /> - <param name="interval_input2" value="2.bed" /> + <param name="input1" value="1.bed" /> + <param name="input2" value="2.bed" /><param name="min" value="10" /><param name="returntype" value="Non-overlapping pieces of intervals" /><output name="output" file="gops-subtract-p.dat" /></test> + <test> - <param name="type" value="GFF"/> - <param name="gff_input1" value="gops_subtract_in1.gff" /> - <param name="gff_input2" value="gops_subtract_in2.gff" /> + <param name="input1" value="gops_subtract_in1.gff" /> + <param name="input2" value="gops_subtract_in2.gff" /><param name="min" value="1" /><param name="returntype" value="" /> - <output name="output" file="gops_subtract_out1.gff" /> + <output name="output" file="gops_subtract_out1.gff" /></test> +  + <test> + <param name="input1" value="gops_subtract_in1.gff" /> + <param name="input2" value="gops_subtract_in2.bed" /> + <param name="min" value="1" /> + <param name="returntype" value="" /> + <output name="output" file="gops_subtract_out1.gff" /> + </test> + </tests><help>

1 0

galaxy-dist commit 764ecc0cf3fd: Refactor code that builds recent tools list to make it easier to understand and more efficient.
by commits-noreply＠bitbucket.org 29 Jun '10

29 Jun '10

# HG changeset patch -- Bitbucket.org # Project galaxy-dist # URL http://bitbucket.org/galaxy/galaxy-dist/overview # User jeremy goecks <jeremy.goecks(a)emory.edu> # Date 1277168327 14400 # Node ID 764ecc0cf3fdc4885de777f26e1ae7c06a85acbf # Parent 7d3a9212b15e82327c2af7c3222995f358532d60 Refactor code that builds recent tools list to make it easier to understand and more efficient. --- a/lib/galaxy/web/controllers/root.py +++ b/lib/galaxy/web/controllers/root.py @@ -38,8 +38,9 @@ class RootController( BaseController, Us filter( self.app.model.Job.user==trans.user ). \ order_by( self.app.model.Job.create_time.desc() ): tool_id = row[0] - if tool_id in toolbox.tools_by_id: - recent_tools.append( toolbox.tools_by_id[tool_id] ) + a_tool = toolbox.tools_by_id.get( tool_id, None ) + if a_tool and a_tool not in recent_tools: + recent_tools.append( a_tool ) ## TODO: make number of recently used tools a user preference. if len ( recent_tools ) == 5: break

1 0

galaxy-dist commit 5e75bad5a780: Merge.
by commits-noreply＠bitbucket.org 29 Jun '10

29 Jun '10

# HG changeset patch -- Bitbucket.org # Project galaxy-dist # URL http://bitbucket.org/galaxy/galaxy-dist/overview # User Dannon Baker <dannon.baker(a)emory.edu> # Date 1277239092 14400 # Node ID 5e75bad5a780acb7c2975bcc375216c1abb5d148 # Parent f2f31fe4cf08f9500a886184b5c5f15fa4184e4d # Parent 464734ed09ad4b36afc3409b5d975732b1c6eb21 Merge.

1 0

galaxy-dist commit 4b4385fbf529: Fix bug in metadata setting for intersect tool.
by commits-noreply＠bitbucket.org 29 Jun '10

29 Jun '10

# HG changeset patch -- Bitbucket.org # Project galaxy-dist # URL http://bitbucket.org/galaxy/galaxy-dist/overview # User jeremy goecks <jeremy.goecks(a)emory.edu> # Date 1277222908 14400 # Node ID 4b4385fbf52982ee3bf6394c5cb269e7bf3ab912 # Parent 764ecc0cf3fdc4885de777f26e1ae7c06a85acbf Fix bug in metadata setting for intersect tool. --- a/tools/new_operations/intersect.xml +++ b/tools/new_operations/intersect.xml @@ -54,7 +54,7 @@ <outputs><data format="input" name="output"> #if inputs.type == "Interval": - metadata_source="inputs.bed_input1" + metadata_source="inputs.interval_input1" #else: metadata_source="inputs.gff_input1" #end if

1 0