- galaxy-commits - lists.galaxyproject.org

galaxy-dist commit 067a8649dae7: Remove code and drop database tables related to the old Galaxy Cloud functionality.
by commits-noreply＠bitbucket.org 29 Jun '10

29 Jun '10

# HG changeset patch -- Bitbucket.org # Project galaxy-dist # URL http://bitbucket.org/galaxy/galaxy-dist/overview # User Enis Afgan <afgane(a)gmail.com> # Date 1277306301 14400 # Node ID 067a8649dae79ab973ff3821b62be89e21a0117a # Parent 752cb3e325374381bdd0db6b9163b12ba88b7151 Remove code and drop database tables related to the old Galaxy Cloud functionality. --- a/templates/cloud/edit_image.mako +++ /dev/null @@ -1,92 +0,0 @@ -<% _=n_ %> -<%inherit file="/base.mako"/> -<%def name="title()">Edit machine image</%def> - -<%def name="javascripts()"> -${parent.javascripts()} -<script type="text/javascript"> -$(function(){ - $("input:text:first").focus(); -}) -</script> -</%def> - -%if header: - ${header} -%endif - -%if image: - -<div class="form"> - <div class="form-title">Edit image</div> - <div class="form-body"> - <form name="edit_image" action="${h.url_for( action='edit_image', id=trans.security.encode_id(image.id), edited="true" )}" method="post" > - <% - cls = "form-row" - if error.has_key('provider_error'): - cls += " form-row-error" - %> - <div class="${cls}"> - <label>Provider type:</label> - <div class="form-row-input"> - ${image.provider_type} - </div> - %if error.has_key('provider_error'): - <div class="form-row-error-message">${error['provider_error']}</div> - %endif - <div style="clear: both"></div> - </div> - <% - cls = "form-row" - if error.has_key('id_error'): - cls += " form-row-error" - %> - <div class="${cls}"> - <label>Machine Image ID (AMI or EMI):</label> - <div class="form-row-input"> - <input type="text" name="image_id" value="${image.image_id}" size="40"> - </div> - %if error.has_key('id_error'): - <div class="form-row-error-message">${error['id_error']}</div> - %endif - <div style="clear: both"></div> - </div> - <% - cls = "form-row" - if error.has_key('manifest_error'): - cls += " form-row-error" - %> - <div class="${cls}"> - <label>Manifest:</label> - <div class="form-row-input"> - <input type="text" name="manifest" value="${image.manifest}" size="40"> - </div> - %if error.has_key('manifest_error'): - <div class="form-row-error-message">${error['manifest_error']}</div> - %endif - <div style="clear: both"></div> - </div> - <% - cls = "form-row" - if error.has_key('arch_error'): - cls += " form-row-error" - %> - <div class="${cls}"> - <label>Architecture:</label> - <div class="form-row-input"> - <input type="text" name="architecture" value="${image.architecture}" size="40"> - </div> - %if error.has_key('arch_error'): - <div class="form-row-error-message">${error['arch_error']}</div> - %endif - <div style="clear: both"></div> - </div> - - <div class="form-row"><input type="submit" value="Save"></div> - </form> - </div> -</div> - -%else: - Specified machine image could not be found. -%endif --- a/templates/cloud/view_provider.mako +++ /dev/null @@ -1,126 +0,0 @@ -<%inherit file="/base.mako"/> - -<%def name="title()">Cloud provider</%def> - -<h2>Cloud provider details</h2> - -<ul class="manage-table-actions"> - <li> - <a class="action-button" href="${h.url_for( action='list' )}"> - <img src="${h.url_for('/static/images/silk/resultset_previous.png')}" /> - <span>Return to cloud management console</span> - </a> - </li> -</ul> - -%if provider: - ${view_provider( provider )} -%else: - There is no cloud provider under that name. -%endif - - - - -<%def name="view_provider( provider )"> - <table class="mange-table colored" border="0" cellspacing="0" cellpadding="0" width="100%"> - <tr> - <td> Cloud provider name: </td> - <td> - ${provider.name} - <a id="cp-popup" class="popup-arrow" style="display: none;">▼</a> - </td> - <td> - <div popupmenu="cp-popup"> - <a class="action-button" href="${h.url_for( action='edit_provider', id=trans.security.encode_id(provider.id) )}">Edit</a> - <a class="action-button" confirm="Are you sure you want to delete cloud provider '${provider.name}'?" href="${h.url_for( action='delete_provider', id=trans.security.encode_id(provider.id) )}">Delete</a> - </div> - </td> - </tr> - <tr> - <td> Last updated: </td> - <td> ${str(provider.update_time)[:16]} - <% - context.write( ' UTC (' ) - context.write( str(h.date.distance_of_time_in_words (provider.update_time, h.date.datetime.utcnow() ) ) ) - %> ago) - </td> - </tr> - <tr> - <td> Cloud provider type: </td> - <td> ${str(provider.type)[:16]}</td> - </tr> - %if provider.region_connection != None: - <tr> - <td> Region connection: </td> - <td> ${provider.region_connection} </td> - </tr> - %endif - %if provider.region_name != None: - <tr> - <td> Region name: </td> - <td> ${provider.region_name} </td> - </tr> - %endif - %if provider.region_endpoint != None: - <tr> - <td> Region endpoint: </td> - <td> ${provider.region_endpoint} </td> - </tr> - %endif - %if provider.is_secure != None: - <tr> - <td> Is secure: </td> - <td> ${provider.is_secure} </td> - </tr> - %endif - %if provider.host != None: - <tr> - <td> Host: </td> - <td> ${provider.host} </td> - </tr> - %endif - %if provider.port != None: - <tr> - <td> Port: </td> - <td> ${provider.port} </td> - </tr> - %endif - %if provider.proxy != None: - <tr> - <td> Proxy: </td> - <td> ${provider.proxy} </td> - </tr> - %endif - %if provider.proxy_port != None: - <tr> - <td> Proxy port: </td> - <td> ${provider.proxy_port} </td> - </tr> - %endif - %if provider.proxy_pass != None: - <tr> - <td> Proxy pass: </td> - <td> ${provider.proxy_pass} </td> - </tr> - %endif - %if provider.debug != None: - <tr> - <td> Debug: </td> - <td> ${provider.debug} </td> - </tr> - %endif - %if provider.https_connection_factory != None: - <tr> - <td> HTTPS connection factory: </td> - <td> ${provider.https_connection_factory} </td> - </tr> - %endif - %if provider.path != None: - <tr> - <td> Path: </td> - <td> ${provider.path} </td> - </tr> - %endif - </table> -</%def> --- a/templates/cloud/edit_credentials.mako +++ /dev/null @@ -1,91 +0,0 @@ -<% _=n_ %> -<%inherit file="/base.mako"/> -<%def name="title()">Add credentials</%def> - -<%def name="javascripts()"> -${parent.javascripts()} -<script type="text/javascript"> -$(function(){ - $("input:text:first").focus(); -}) -</script> -</%def> - -%if header: - ${header} -%endif - -%if credential: - -<div class="form"> - <div class="form-title">Edit credentials</div> - <div class="form-body"> - <form name="edit_credentials" action="${h.url_for( action='edit_credentials', id=trans.security.encode_id(credential.id), edited="true" )}" method="post" > - - <% - cls = "form-row" - if error.has_key('cred_error'): - cls += " form-row-error" - %> - <div class="${cls}"> - <label>Credentials name:</label> - <div class="form-row-input"> - <input type="text" name="credName" value="${credential.name}" size="40"> - </div> - %if error.has_key('cred_error'): - <div class="form-row-error-message">${error['cred_error']}</div> - %endif - <div style="clear: both"></div> - </div> - - - <% - cls = "form-row" - if error.has_key('provider_error'): - cls += " form-row-error" - %> - <div class="${cls}"> - <label>Cloud provider name (type):</label> - <div class="form-row-input">${credential.provider.name} (${credential.provider.type})</div> - <div style="clear: both"></div> - </div> - <% - cls = "form-row" - if error.has_key('access_key_error'): - cls += " form-row-error" - %> - <div class="${cls}"> - <label>Access key:</label> - <div class="form-row-input"> - <input type="text" name="accessKey" value="${credential.access_key}" size="40"> - </div> - %if error.has_key('access_key_error'): - <div class="form-row-error-message">${error['access_key_error']}</div> - %endif - <div style="clear: both"></div> - </div> - - <% - cls = "form-row" - if error.has_key('secret_key_error'): - cls += " form-row-error" - %> - <div class="${cls}"> - <label>Secret key:</label> - <div class="form-row-input"> - <input type="password" name="secretKey" value="${credential.secret_key}" size="40"> - </div> - %if error.has_key('secret_key_error'): - <div class="form-row-error-message">${error['secret_key_error']}</div> - %endif - <div style="clear: both"></div> - </div> - - <div class="form-row"><input type="submit" value="Edit"></div> - </form> - </div> -</div> - -%else: - Specified credentials could not be found. -%endif --- a/templates/cloud/edit_provider.mako +++ /dev/null @@ -1,261 +0,0 @@ -<% _=n_ %> -<%inherit file="/base.mako"/> -<%def name="title()">Edit provider</%def> - -<%def name="javascripts()"> -${parent.javascripts()} -<script type="text/javascript"> -$(function(){ - - $("input:text:first").focus(); - - $("#type").change(function() { - if ($(this).val() == 'ec2') { - clear(); - $("#autofill").attr( 'disabled', true ); - $("#autofill").attr( 'checked', false ); - $("#name").val( "EC2" ); - $("#region_name").val( "us-east-1" ); - $("#region_endpoint").val( "us-east-1.ec2.amazonaws.com" ); - $("#is_secure").val("1"); - $("#debug").val(""); - $("#path").val("/"); - } - else if ($(this).val() == 'eucalyptus') { - clear(); - $("#autofill").attr( 'disabled', false ); - } - }); -}) - - -function af(){ - - if ( $("#autofill_epc").attr('checked') ) { - $("#region_name").val("eucalyptus"); - $("#region_endpoint").val("mayhem9.cs.ucsb.edu"); - $("#is_secure").val("0"); - $("#port").val("8773"); - $("#path").val("/services/Eucalyptus"); - } - else if ( $("#autofill_ec2").attr('checked') ) { - $("#region_name").val( "us-east-1" ); - $("#region_endpoint").val( "us-east-1.ec2.amazonaws.com" ); - $("#is_secure").val("1"); - $("#debug").val(""); - $("#path").val("/"); - } -} - -function clear() { - //$("#name").val(""); - $("#region_name").val(""); - $("#region_endpoint").val(""); - $("#is_secure").val(""); - $("#port").val(""); - $("#proxy").val(""); - $("#proxy_port").val(""); - $("#proxy_user").val(""); - $("#proxy_pass").val(""); - $("#debug").val(""); - $("#https_connection_factory").val(""); - $("#path").val(""); - -} - -</script> -</%def> - -%if header: - ${header} -%endif - -%if provider: - <div class="form"> - <div class="form-title">Edit cloud provider</div> - <div class="form-body"> - <form name="edit_provider_form" action="${h.url_for( action='edit_provider', id=trans.security.encode_id(provider.id), edited="true" )}" method="post" > - <% - cls = "form-row" - if error.has_key('type_error'): - cls += " form-row-error" - %> - <div class="${cls}"> - <label>Provider type:</label> - <div class="form-row-input">${provider.type} - %if provider.type == 'eucalyptus': - <p><input type="checkbox" id="autofill_epc" onclick="javascript:af()"> - auto fill using Eucalyptus Public Cloud values - </p></div> - %elif provider.type == 'ec2': - <p><input type="checkbox" id="autofill_ec2" onclick="javascript:af()"> - auto fill for Amazon EC2 (us-east-1 region) - </p></div> - %endif - <div style="clear: both"></div> - </div> - - <% - cls = "form-row" - if error.has_key('name_error'): - cls += " form-row-error" - %> - <div class="${cls}"> - <label>Provider name:</label> - <div class="form-row-input"> - <input type="text" id="name" name="name" value="${provider.name}" size="40"> - </div> - %if error.has_key('name_error'): - <div class="form-row-error-message">${error['name_error']}</div> - %endif - <div style="clear: both"></div> - </div> - - <% - cls = "form-row" - %> - <div class="${cls}"> - <label>Region name:</label> - <div id="region_selection" class="form-row-input"> - <input type="text" name="region_name" id="region_name" value="${provider.region_name}" size="40"> - </div> - <div style="clear: both"></div> - </div> - - <% - cls = "form-row" - %> - <div class="${cls}"> - <label>Region endpoint:</label> - <div class="form-row-input"> - <input type="text" name="region_endpoint" id="region_endpoint" value="${provider.region_endpoint}" size="40"> - </div> - <div style="clear: both"></div> - </div> - - <% - cls = "form-row" - if error.has_key('is_secure_error'): - cls += " form-row-error" - %> - <div class="${cls}"> - <label>Is secure ('O' for False or '1' for True):</label> - <div class="form-row-input"> - %if provider.is_secure == True: - <input type="text" name="is_secure" id="is_secure" value="1" size="40"> - %else: - <input type="text" name="is_secure" id="is_secure" value="0" size="40"> - %endif - </div> - %if error.has_key('is_secure_error'): - <div class="form-row-error-message">${error['is_secure_error']}; you entered: '${provider.is_secure}'</div> - %endif - <div style="clear: both"></div> - </div> - - <% - cls = "form-row" - %> - <div class="${cls}"> - <label>Host:</label> - <div class="form-row-input"> - <input type="text" name="host" value="${provider.host}" size="40"> - </div> - <div style="clear: both"></div> - </div> - - <% - cls = "form-row" - %> - <div class="${cls}"> - <label>Port:</label> - <div class="form-row-input"> - <input type="text" name="port" id="port" value="${provider.port}" size="40"> - </div> - <div style="clear: both"></div> - </div> - - <% - cls = "form-row" - %> - <div class="${cls}"> - <label>Proxy:</label> - <div class="form-row-input"> - <input type="text" name="proxy" value="${provider.proxy}" size="40"> - </div> - <div style="clear: both"></div> - </div> - - <% - cls = "form-row" - %> - <div class="${cls}"> - <label>Proxy port:</label> - <div class="form-row-input"> - <input type="text" name="proxy_port" value="${provider.proxy_port}" size="40"> - </div> - <div style="clear: both"></div> - </div> - - <% - cls = "form-row" - %> - <div class="${cls}"> - <label>Proxy user:</label> - <div class="form-row-input"> - <input type="text" name="proxy_user" value="${provider.proxy_user}" size="40"> - </div> - <div style="clear: both"></div> - </div> - - <% - cls = "form-row" - %> - <div class="${cls}"> - <label>Proxy pass:</label> - <div class="form-row-input"> - <input type="text" name="proxy_pass" value="${provider.proxy_pass}" size="40"> - </div> - <div style="clear: both"></div> - </div> - - <% - cls = "form-row" - %> - <div class="${cls}"> - <label>Debug:</label> - <div class="form-row-input"> - <input type="text" name="debug" value="${provider.debug}" size="40"> - </div> - <div style="clear: both"></div> - </div> - - <% - cls = "form-row" - %> - <div class="${cls}"> - <label>HTTPS connection factory:</label> - <div class="form-row-input"> - <input type="text" name="https_connection_factory" value="${provider.https_connection_factory}" size="40"> - </div> - <div style="clear: both"></div> - </div> - - <% - cls = "form-row" - %> - <div class="${cls}"> - <label>Path:</label> - <div class="form-row-input"> - <input type="text" name="path" id="path" value="${provider.path}" size="40"> - </div> - <div style="clear: both"></div> - </div> - - <div class="form-row"><input type="submit" value="Save"></div> - - </form> - - </div> - </div> -%endif --- a/templates/cloud/view_usage.mako +++ /dev/null @@ -1,117 +0,0 @@ -<%inherit file="/base.mako"/> - -<%def name="title()">Cloud home</%def> - -%if message: -<% - try: - messagetype - except: - messagetype = "done" -%> - - - -<p /> -<div class="${messagetype}message"> - ${message} -</div> -%endif - -%if prevInstances: - <h2>Usage report for instance ${prevInstances[0].uci.name}</h2> -%else: - <h2>Selected instance has no record of being used.</h2> -%endif - -<ul class="manage-table-actions"> - <li> - <a class="action-button" href="${h.url_for( action='list' )}"> - <img src="${h.url_for('/static/images/silk/resultset_previous.png')}" /> - <span>Return to cloud management console</span> - </a> - </li> -</ul> - -%if prevInstances: - <table class="mange-table colored" border="0" cellspacing="0" cellpadding="0" width="100%"> - <colgroup width="2%"></colgroup> - <colgroup width="16%"></colgroup> - <colgroup width="16%"></colgroup> - <colgroup width="10%"></colgroup> - <colgroup width="5%"></colgroup> - <tr class="header"> - <th>#</th> - <th>Launch time</th> - <th>Termination time</th> - <th>Time alive</th> - <th>Type</th> - <th></th> - </tr> - <% - total_hours = 0 - %> - %for i, prevInstance in enumerate( prevInstances ): - <tr> - <td>${i+1}</td> - <td> - %if prevInstance.launch_time: - ${str(prevInstance.launch_time)[:16]} UCT - %else: - N/A - %endif - </td> - <td> - %if prevInstance.stop_time: - ${str(prevInstance.stop_time)[:16]} UCT - %else: - N/A - %endif - </td> - <td> - <% - # This is where current time and since duration is calculated - if prevInstance.launch_time is None or prevInstance.stop_time is None: - context.write( 'N/A' ) - else: - context.write( str(h.date.distance_of_time_in_words (prevInstance.launch_time, prevInstance.stop_time ) ) ) - time_delta = prevInstance.stop_time - prevInstance.launch_time - total_hours += time_delta.seconds / 3600 - if time_delta.seconds != 0: - total_hours += 1 - - %> - </td> - <td>${prevInstance.type}</td> - </tr> - %endfor - </table> - <br/>Total number of hours instance was alive: ${total_hours} <br /> - Note that these are just best effort estimates - true usage should be obtained from respective cloud provider. <br /> - <%namespace name="view_cred" file="view_credentials.mako" /> - - <div id="hide_cred_details"> - This instance uses credentials: - <a onclick="document.getElementById('show_cred_details').style.display = 'block'; - document.getElementById('hide_cred_details').style.display = 'none'; return 0" - href="javascript:void(0)"> - ${prevInstances[0].uci.credentials.name} - </a> - </div> - <div id="show_cred_details" style="DISPLAY: none"> - This instance uses credentials: - <a onclick="document.getElementById('hide_cred_details').style.display = 'block'; - document.getElementById('show_cred_details').style.display = 'none'; return 0;" - href="javascript:void(0)"> - ${prevInstances[0].uci.credentials.name} - </a> - ${view_cred.view_cred( prevInstances[0].uci.credentials ) } - </div> - - -%endif - - - - - --- a/templates/cloud/view_credentials.mako +++ /dev/null @@ -1,157 +0,0 @@ -<%inherit file="/base.mako"/> - -<%def name="title()">Cloud credentials</%def> - -<h2>Credentials details</h2> - -<ul class="manage-table-actions"> - <li> - <a class="action-button" href="${h.url_for( action='list' )}"> - <img src="${h.url_for('/static/images/silk/resultset_previous.png')}" /> - <span>Return to cloud management console</span> - </a> - </li> -</ul> - -%if credDetails: - ${view_cred( credDetails )} -%else: - There are no credentials under that name. -%endif - - - - -<%def name="view_cred( credDetails )"> - <table class="mange-table colored" border="0" cellspacing="0" cellpadding="0" width="100%"> - <tr> - <td> Credentials name: </td> - <td> - ${credDetails.name} - <a id="wf-popup" class="popup-arrow" style="display: none;">▼</a> - </td> - <td> - <div popupmenu="wf-popup"> - <a class="action-button" href="${h.url_for( action='edit_credentials', id=trans.security.encode_id(credDetails.id) )}">Edit</a> - <a class="action-button" confirm="Are you sure you want to delete credentials '${credDetails.name}'?" href="${h.url_for( action='delete_credentials', id=trans.security.encode_id(credDetails.id) )}">Delete</a> - </div> - </td> - </tr> - <tr> - <td> Last updated: </td> - <td> ${str(credDetails.update_time)[:16]} - <% - context.write( ' UTC (' ) - context.write( str(h.date.distance_of_time_in_words (credDetails.update_time, h.date.datetime.utcnow() ) ) ) - %> ago) - </td> - </tr> - <tr> - <td> Cloud provider type: </td> - <td> ${str(credDetails.provider.type)}</td> - </tr> - <tr> - <td> Cloud provider name: </td> - <td> ${str(credDetails.provider.name)}</td> - </tr> - <tr> - <td> Access key: </td> - <td> - ${credDetails.access_key} - </td> - </tr> - <tr> - <td> Secret key: </td> - <td> - <div id="shortComment2"> - <a onclick="document.getElementById('fullComment2').style.display = 'block'; - document.getElementById('shortComment2').style.display = 'none'; return 0" - href="javascript:void(0)"> - + Show - </a> - </div> - <div id="fullComment2" style="DISPLAY: none"> - <a onclick="document.getElementById('shortComment2').style.display = 'block'; - document.getElementById('fullComment2').style.display = 'none'; return 0;" - href="javascript:void(0)"> - - Hide - </a><br /> - <nobr>${credDetails.secret_key}</nobr><br/> - </div> - </td> - </tr> - <tr><td id="addl"><b>Additional cloud provider information (if available):</b></td></tr> - %if credDetails.provider.region_connection != None: - <tr> - <td> Region connection: </td> - <td> ${credDetails.provider.region_connection} </td> - </tr> - %endif - %if credDetails.provider.region_name != None: - <tr> - <td> Region name: </td> - <td> ${credDetails.provider.region_name} </td> - </tr> - %endif - %if credDetails.provider.region_endpoint != None: - <tr> - <td> Region endpoint: </td> - <td> ${credDetails.provider.region_endpoint} </td> - </tr> - %endif - %if credDetails.provider.is_secure != None: - <tr> - <td> Is secure: </td> - <td> ${credDetails.provider.is_secure} </td> - </tr> - %endif - %if credDetails.provider.host != None: - <tr> - <td> Host: </td> - <td> ${credDetails.provider.host} </td> - </tr> - %endif - %if credDetails.provider.port != None: - <tr> - <td> Port: </td> - <td> ${credDetails.provider.port} </td> - </tr> - %endif - %if credDetails.provider.proxy != None: - <tr> - <td> Proxy: </td> - <td> ${credDetails.provider.proxy} </td> - </tr> - %endif - %if credDetails.provider.proxy_port != None: - <tr> - <td> Proxy port: </td> - <td> ${credDetails.provider.proxy_port} </td> - </tr> - %endif - %if credDetails.provider.proxy_pass != None: - <tr> - <td> Proxy pass: </td> - <td> ${credDetails.provider.proxy_pass} </td> - </tr> - %endif - %if credDetails.provider.debug != None: - <tr> - <td> Debug: </td> - <td> ${credDetails.provider.debug} </td> - </tr> - %endif - %if credDetails.provider.https_connection_factory != None: - <tr> - <td> HTTPS connection factory: </td> - <td> ${credDetails.provider.https_connection_factory} </td> - </tr> - %endif - %if credDetails.provider.path != None: - <tr> - <td> Path: </td> - <td> ${credDetails.provider.path} </td> - </tr> - %endif - </table> -</%def> --- a/lib/galaxy/cloud/providers/eucalyptus.py +++ /dev/null @@ -1,1039 +0,0 @@ -import subprocess, threading, os, errno, time, datetime -from Queue import Queue, Empty -from datetime import datetime - -from galaxy import model # Database interaction class -from galaxy.model import mapping -from galaxy.datatypes.data import nice_size -from galaxy.util.bunch import Bunch -from galaxy.cloud import UCIwrapper -from Queue import Queue -from sqlalchemy import or_, and_ - -import galaxy.eggs -galaxy.eggs.require("boto") -from boto.ec2.connection import EC2Connection -from boto.ec2.regioninfo import RegionInfo -import boto.exception -import boto - -import logging -log = logging.getLogger( __name__ ) - -uci_states = Bunch( - NEW_UCI = "newUCI", - NEW = "new", - CREATING = "creating", - DELETING_UCI = "deletingUCI", - DELETING = "deleting", - SUBMITTED_UCI = "submittedUCI", - SUBMITTED = "submitted", - SHUTTING_DOWN_UCI = "shutting-downUCI", - SHUTTING_DOWN = "shutting-down", - ADD_STORAGE_UCI = "add-storageUCI", - ADD_STORAGE = "add-storage", - AVAILABLE = "available", - RUNNING = "running", - PENDING = "pending", - ERROR = "error", - DELETED = "deleted", - SNAPSHOT_UCI = "snapshotUCI", - SNAPSHOT = "snapshot" -) - -instance_states = Bunch( - TERMINATED = "terminated", - SUBMITTED = "submitted", - RUNNING = "running", - ADDING = "adding-storage", - PENDING = "pending", - SHUTTING_DOWN = "shutting-down", - ERROR = "error" -) - -store_status = Bunch( - WAITING = "waiting", - IN_USE = "in-use", - ADDING = "adding", - CREATING = "creating", - DELETED = 'deleted', - ERROR = "error" -) - -snapshot_status = Bunch( - SUBMITTED = 'submitted', - PENDING = 'pending', - COMPLETED = 'completed', - DELETE = 'delete', - DELETED= 'deleted', - ERROR = "error" -) - -class EucalyptusCloudProvider( object ): - """ - Eucalyptus-based cloud provider implementation for managing instances. - """ - STOP_SIGNAL = object() - def __init__( self, app ): - self.type = "eucalyptus" # cloud provider type (e.g., ec2, eucalyptus, opennebula) - self.zone = "epc" - self.queue = Queue() - self.sa_session = app.model.context - - self.threads = [] - nworkers = 5 - log.info( "Starting eucalyptus cloud controller workers..." ) - for i in range( nworkers ): - worker = threading.Thread( target=self.run_next ) - worker.start() - self.threads.append( worker ) - log.debug( "%d eucalyptus cloud workers ready", nworkers ) - - def shutdown( self ): - """Attempts to gracefully shut down the monitor thread""" - log.info( "sending stop signal to worker threads in eucalyptus cloud manager" ) - for i in range( len( self.threads ) ): - self.queue.put( self.STOP_SIGNAL ) - log.info( "eucalyptus cloud manager stopped" ) - - def put( self, uci_wrapper ): - """ - Add uci_wrapper object to the end of the request queue to be handled by - this cloud provider. - """ - state = uci_wrapper.get_uci_state() - uci_wrapper.change_state( state.split('U')[0] ) # remove 'UCI' from end of state description (i.e., mark as accepted and ready for processing) - self.queue.put( uci_wrapper ) - - def run_next( self ): - """Process next request, waiting until one is available if necessary.""" - cnt = 0 - while 1: - uci_wrapper = self.queue.get() - uci_state = uci_wrapper.get_uci_state() - if uci_state is self.STOP_SIGNAL: - return - try: - if uci_state==uci_states.NEW: - self.create_uci( uci_wrapper ) - elif uci_state==uci_states.DELETING: - self.delete_uci( uci_wrapper ) - elif uci_state==uci_states.SUBMITTED: - self.start_uci( uci_wrapper ) - #self.dummy_start_uci( uci_wrapper ) - elif uci_state==uci_states.SHUTTING_DOWN: - self.stop_uci( uci_wrapper ) - elif uci_state==uci_states.SNAPSHOT: - self.snapshot_uci( uci_wrapper ) - elif uci_state==uci_states.ADD_STORAGE: - self.add_storage_to_uci( uci_wrapper ) - except: - log.exception( "Uncaught exception executing cloud request." ) - cnt += 1 - - def get_connection( self, uci_wrapper ): - """ - Establishes cloud connection using user's credentials associated with given UCI - """ - log.debug( 'Establishing %s cloud connection.' % self.type ) - provider = uci_wrapper.get_provider() - try: - region = RegionInfo( None, provider.region_name, provider.region_endpoint ) - except Exception, ex: - err = "Selecting region with cloud provider failed: " + str( ex ) - log.error( err ) - uci_wrapper.set_error( err, True ) - return None - try: - conn = EC2Connection( aws_access_key_id=uci_wrapper.get_access_key(), - aws_secret_access_key=uci_wrapper.get_secret_key(), - is_secure=provider.is_secure, - port=provider.port, - region=region, - path=provider.path ) - except boto.exception.EC2ResponseError, e: - err = "Establishing connection with cloud failed: " + str( e ) - log.error( err ) - uci_wrapper.set_error( err, True ) - return None - - return conn - - def check_key_pair( self, uci_wrapper, conn ): - """ - Check if a key pair associated with this UCI exists on cloud provider. - If yes, return key pair name; otherwise, generate a key pair with the cloud - provider and, again, return key pair name. - Key pair name for given UCI is generated from UCI's name and suffix '_kp' - """ - kp = None - kp_name = uci_wrapper.get_name().replace(' ','_') + "_kp" - log.debug( "Checking user's key pair: '%s'" % kp_name ) - try: - kp = conn.get_key_pair( kp_name ) - uci_kp_name = uci_wrapper.get_key_pair_name() - uci_material = uci_wrapper.get_key_pair_material() - if kp != None: - if kp.name != uci_kp_name or uci_material == None: - # key pair exists on the cloud but not in local database, so re-generate it (i.e., delete and then create) - try: - conn.delete_key_pair( kp_name ) - kp = self.create_key_pair( conn, kp_name ) - uci_wrapper.set_key_pair( kp.name, kp.material ) - except boto.exception.EC2ResponseError, e: - err = "EC2 response error while deleting key pair: " + str( e ) - log.error( err ) - uci_wrapper.set_error( err, True ) - else: - try: - kp = self.create_key_pair( conn, kp_name ) - uci_wrapper.set_key_pair( kp.name, kp.material ) - except boto.exception.EC2ResponseError, e: - err = "EC2 response error while creating key pair: " + str( e ) - log.error( err ) - uci_wrapper.set_error( err, True ) - except Exception, ex: - err = "Exception while creating key pair: " + str( ex ) - log.error( err ) - uci_wrapper.set_error( err, True ) - except boto.exception.EC2ResponseError, e: # No keypair under this name exists so create it - if e.code == 'InvalidKeyPair.NotFound': - log.info( "No keypair found, creating keypair '%s'" % kp_name ) - kp = self.create_key_pair( conn, kp_name ) - uci_wrapper.set_key_pair( kp.name, kp.material ) - else: - err = "EC2 response error while retrieving key pair: " + str( e ) - log.error( err ) - uci_wrapper.set_error( err, True ) - - if kp != None: - return kp.name - else: - return None - - def create_key_pair( self, conn, kp_name ): - """ Initiate creation of key pair under kp_name by current cloud provider. """ - try: - return conn.create_key_pair( kp_name ) - except boto.exception.EC2ResponseError, e: - return None - - def get_mi_id( self, uci_wrapper, i_index ): - """ - Get appropriate machine image (mi) ID based on instance type. - """ - i_type = uci_wrapper.get_instance_type( i_index ) - if i_type=='m1.small' or i_type=='c1.medium': - arch = 'i386' - else: - arch = 'x86_64' - - mi = self.sa_session.query( model.CloudImage ).filter_by( deleted=False, provider_type=self.type, architecture=arch ).first() - if mi: - return mi.image_id - else: - err = "Machine image could not be retrieved" - log.error( "%s for UCI '%s'." % (err, uci_wrapper.get_name() ) ) - uci_wrapper.set_error( err+". Contact site administrator to ensure needed machine image is registered.", True ) - return None - - def create_uci( self, uci_wrapper ): - """ - Create User Configured Instance (UCI) - i.e., create storage volume on cloud provider - and register relevant information in local Galaxy database. - """ - conn = self.get_connection( uci_wrapper ) - - # Because only 1 storage volume may be created at UCI config time, index of this storage volume in local Galaxy DB w.r.t - # current UCI is 0; therefore, it can be referenced in following code - log.info( "Creating volume in zone '%s'..." % uci_wrapper.get_uci_availability_zone() ) - if uci_wrapper.get_uci_availability_zone()=='': - log.info( "Availability zone for UCI (i.e., storage volume) was not selected, using default zone: %s" % self.zone ) - uci_wrapper.set_store_availability_zone( self.zone ) - -# log.debug( "Creating volume; using command: conn.create_volume( %s, '%s', snapshot=None )" % ( uci_wrapper.get_store_size( 0 ), uci_wrapper.get_uci_availability_zone() )) -# vol = conn.create_volume( uci_wrapper.get_store_size( 0 ), uci_wrapper.get_uci_availability_zone(), snapshot=None ) -# uci_wrapper.set_store_volume_id( 0, vol.id ) - store = uci_wrapper.get_all_stores_in_status( store_status.ADDING )[0] # Because at UCI creation time only 1 storage volume can be created, reference it directly - - log.info( "Creating storage volume in zone '%s' of size '%s'..." % ( uci_wrapper.get_uci_availability_zone(), store.size ) ) - # Because only 1 storage volume may be created at UCI config time, index of this storage volume in local Galaxy DB w.r.t - # current UCI is 0, so reference it in following methods - vol = conn.create_volume( store.size, uci_wrapper.get_uci_availability_zone(), snapshot=None ) - uci_wrapper.set_store_volume_id( store.id, vol.id ) - - # Retrieve created volume again to get updated status - try: - vl = conn.get_all_volumes( [vol.id] ) - except boto.exception.EC2ResponseError, e: - err = "EC2 response error while retrieving (i.e., updating status) of just created storage volume '" + vol.id + "': " + str( e ) - log.error( err ) - uci_wrapper.set_store_status( vol.id, uci_states.ERROR ) - uci_wrapper.set_error( err, True ) - return - except Exception, ex: - err = "Error while retrieving (i.e., updating status) of just created storage volume '" + vol.id + "': " + str( ex ) - log.error( err ) - uci_wrapper.set_error( err, True ) - return - - if len( vl ) > 0: - # EPC does not allow creation of storage volumes (it deletes one as soon as it is created, so manually set uci_state here) - if vl[0].status == store_status.DELETED: - uci_wrapper.change_state( uci_state=uci_states.AVAILABLE ) - else: - uci_wrapper.change_state( uci_state=vl[0].status ) - uci_wrapper.set_store_status( vol.id, vl[0].status ) - else: - err = "Volume '" + vol.id +"' not found by EC2 after being created." - log.error( err ) - uci_wrapper.set_store_status( vol.id, uci_states.ERROR ) - uci_wrapper.set_error( err, True ) - - def delete_uci( self, uci_wrapper ): - """ - Delete UCI - i.e., delete all storage volumes associated with this UCI. - NOTE that this implies deletion of any and all data associated - with this UCI from the cloud. All data will be deleted. - Information in local Galaxy database is marked as deleted but not actually removed - from the database. - """ - conn = self.get_connection( uci_wrapper ) - vl = [] # volume list - count = 0 # counter for checking if all volumes assoc. w/ UCI were deleted - - # Get all volumes assoc. w/ UCI, delete them from cloud as well as in local DB - vl = uci_wrapper.get_all_stores() - deletedList = [] - failedList = [] - for v in vl: - log.debug( "Deleting volume with id='%s'" % v.volume_id ) - try: - if conn.delete_volume( v.volume_id ): - deletedList.append( v.volume_id ) - v.deleted = True - self.sa_session.add( v ) - self.sa_session.flush() - count += 1 - else: - failedList.append( v.volume_id ) - except boto.exception.EC2ResponseError, e: - err = "EC2 response error while deleting storage volume '" + v.volume_id + "': " + str( e ) - log.error( err ) - uci_wrapper.set_store_error( err, store_id = v.volume_id ) - uci_wrapper.set_error( err, True ) - - # Delete UCI if all of associated - if count == len( vl ): - uci_wrapper.set_deleted() - else: - err = "Deleting following volume(s) failed: "+ str( failedList )+". However, these volumes were successfully deleted: " \ - + str( deletedList ) +". MANUAL intervention and processing needed." - log.error( err ) - uci_wrapper.set_error( err, True ) - - def snapshot_uci( self, uci_wrapper ): - """ - Initiate creation of a snapshot by cloud provider for all storage volumes - associated with this UCI. - """ - if uci_wrapper.get_uci_state() != uci_states.ERROR: - conn = self.get_connection( uci_wrapper ) - - snapshots = uci_wrapper.get_snapshots( status = snapshot_status.SUBMITTED ) - for snapshot in snapshots: - log.debug( "Snapshot DB id: '%s', volume id: '%s'" % ( snapshot.id, snapshot.store.volume_id ) ) - try: - snap = conn.create_snapshot( volume_id=snapshot.store.volume_id ) - snap_id = str( snap ).split(':')[1] - uci_wrapper.set_snapshot_id( snapshot.id, snap_id ) - sh = conn.get_all_snapshots( snap_id ) # get updated status - uci_wrapper.set_snapshot_status( status=sh[0].status, snap_id=snap_id ) - except boto.exception.EC2ResponseError, e: - err = "Cloud provider response error while creating snapshot: " + str( e ) - log.error( err ) - uci_wrapper.set_snapshot_error( error=err, snap_index=snapshot.id, set_status=True ) - uci_wrapper.set_error( err, True ) - return - except Exception, ex: - err = "Error while creating snapshot: " + str( ex ) - log.error( err ) - uci_wrapper.set_snapshot_error( error=err, snap_index=snapshot.id, set_status=True ) - uci_wrapper.set_error( err, True ) - return - - uci_wrapper.change_state( uci_state=uci_states.AVAILABLE ) - -# if uci_wrapper.get_uci_state() != uci_states.ERROR: -# -# snapshots = uci_wrapper.get_snapshots( status = 'submitted' ) -# for snapshot in snapshots: -# uci_wrapper.set_snapshot_id( snapshot.id, None, 'euca_error' ) -# -# log.debug( "Eucalyptus snapshot attempted by user for UCI '%s'" % uci_wrapper.get_name() ) -# uci_wrapper.set_error( "Eucalyptus does not support creation of snapshots at this moment. No snapshot or other changes were performed. \ -# Feel free to resent state of this instance and use it normally.", True ) - - - def add_storage_to_uci( self, uci_wrapper ): - """ Adds more storage to specified UCI """ - uci_wrapper.set_error( "Adding storage to eucalyptus-based clouds is not yet supported.", True ) - - def dummy_start_uci( self, uci_wrapper ): - - uci = uci_wrapper.get_uci() - log.debug( "Would be starting instance '%s'" % uci.name ) -# uci_wrapper.change_state( uci_states.SUBMITTED_UCI ) -# log.debug( "Set UCI state to SUBMITTED_UCI" ) - log.debug( "Sleeping a bit... (%s)" % uci.name ) - time.sleep(10) - log.debug( "Woke up! (%s)" % uci.name ) - - def start_uci( self, uci_wrapper ): - """ - Start instance(s) of given UCI on the cloud. - """ - if uci_wrapper.get_uci_state() != uci_states.ERROR: - conn = self.get_connection( uci_wrapper ) - self.check_key_pair( uci_wrapper, conn ) - if uci_wrapper.get_key_pair_name() == None: - err = "Key pair not found" - log.error( "%s for UCI '%s'." % ( err, uci_wrapper.get_name() ) ) - uci_wrapper.set_error( err + ". Try resetting the state and starting the instance again.", True ) - return - - i_indexes = uci_wrapper.get_instances_indexes( state=instance_states.SUBMITTED ) # Get indexes of i_indexes associated with this UCI that are in 'submitted' state - log.debug( "Starting instances with IDs: '%s' associated with UCI '%s' " % ( i_indexes, uci_wrapper.get_name(), ) ) - if len( i_indexes ) > 0: - for i_index in i_indexes: - # Get machine image for current instance - mi_id = self.get_mi_id( uci_wrapper, i_index ) - log.debug( "mi_id: %s, uci_wrapper.get_key_pair_name(): %s" % ( mi_id, uci_wrapper.get_key_pair_name() ) ) - uci_wrapper.set_mi( i_index, mi_id ) - - if uci_wrapper.get_uci_state() != uci_states.ERROR: - # Start an instance - log.debug( "Starting UCI instance '%s'" % uci_wrapper.get_name() ) - log.debug( "Using following command: conn.run_instances( image_id='%s', key_name='%s', instance_type='%s' )" - % ( mi_id, uci_wrapper.get_key_pair_name(), uci_wrapper.get_instance_type( i_index ) ) ) - reservation = None - try: - reservation = conn.run_instances( image_id=mi_id, - key_name=uci_wrapper.get_key_pair_name(), - instance_type=uci_wrapper.get_instance_type( i_index ) ) - except boto.exception.EC2ResponseError, e: - err = "EC2 response error when starting UCI '"+ uci_wrapper.get_name() +"': " + str( e ) - log.error( err ) - uci_wrapper.set_error( err, True ) - except Exception, ex: - err = "Error when starting UCI '" + uci_wrapper.get_name() + "': " + str( ex ) - log.error( err ) - uci_wrapper.set_error( err, True ) - # Record newly available instance data into local Galaxy database - if reservation: - l_time = datetime.utcnow() -# uci_wrapper.set_instance_launch_time( self.format_time( reservation.instances[0].launch_time ), i_index=i_index ) - uci_wrapper.set_instance_launch_time( l_time, i_index=i_index ) - if not uci_wrapper.uci_launch_time_set(): - uci_wrapper.set_uci_launch_time( l_time ) - try: - uci_wrapper.set_reservation_id( i_index, str( reservation ).split(":")[1] ) - # TODO: if more than a single instance will be started through single reservation, change this reference from element [0] - i_id = str( reservation.instances[0]).split(":")[1] - uci_wrapper.set_instance_id( i_index, i_id ) - s = reservation.instances[0].state - uci_wrapper.change_state( s, i_id, s ) - vol_id = uci_wrapper.get_store_volume_id( store_id=0 ) # TODO: Once more that one vol/UCI is allowed, update this! - uci_wrapper.set_store_status( vol_id, store_status.WAITING ) - log.debug( "Instance of UCI '%s' started, current state: '%s'" % ( uci_wrapper.get_name(), uci_wrapper.get_uci_state() ) ) - except boto.exception.EC2ResponseError, e: - err = "EC2 response error when retrieving instance information for UCI '" + uci_wrapper.get_name() + "': " + str( e ) - log.error( err ) - uci_wrapper.set_error( err, True ) - else: - log.error( "UCI '%s' is in 'error' state, starting instance was aborted." % uci_wrapper.get_name() ) - else: - err = "No instances in state '"+ instance_states.SUBMITTED +"' found for UCI '" + uci_wrapper.get_name() + \ - "'. Nothing to start." - log.error( err ) - uci_wrapper.set_error( err, True ) - else: - log.error( "UCI '%s' is in 'error' state, starting instance was aborted." % uci_wrapper.get_name() ) - - def stop_uci( self, uci_wrapper): - """ - Stop all cloud instances associated with given UCI. - """ - conn = self.get_connection( uci_wrapper ) - - # Get all instances associated with given UCI - il = uci_wrapper.get_instances_ids() # instance list - # Process list of instances and remove any references to empty instance id's - for i in il: - if i is None: - il.remove( i ) - log.debug( 'List of instances being terminated: %s' % il ) - rl = conn.get_all_instances( il ) # Reservation list associated with given instances - - # Initiate shutdown of all instances under given UCI - cnt = 0 - stopped = [] - not_stopped = [] - for r in rl: - for inst in r.instances: - log.debug( "Sending stop signal to instance '%s' associated with reservation '%s' (UCI: %s)." % ( inst, r, uci_wrapper.get_name() ) ) - try: - inst.stop() - uci_wrapper.set_stop_time( datetime.utcnow(), i_id=inst.id ) - uci_wrapper.change_state( instance_id=inst.id, i_state=inst.update() ) - stopped.append( inst ) - except boto.exception.EC2ResponseError, e: - not_stopped.append( inst ) - err = "EC2 response error when stopping instance '" + inst.instance_id + "': " + str( e ) - log.error( err ) - uci_wrapper.set_error( err, True ) - - uci_wrapper.reset_uci_launch_time() - log.debug( "Termination was initiated for all instances of UCI '%s'." % uci_wrapper.get_name() ) - -# dbInstances = get_instances( trans, uci ) #TODO: handle list! -# -# # Get actual cloud instance object -# cloudInstance = get_cloud_instance( conn, dbInstances.instance_id ) -# -# # TODO: Detach persistent storage volume(s) from instance and update volume data in local database -# stores = get_stores( trans, uci ) -# for i, store in enumerate( stores ): -# log.debug( "Detaching volume '%s' to instance '%s'." % ( store.volume_id, dbInstances.instance_id ) ) -# mntDevice = store.device -# volStat = None -## Detaching volume does not work with Eucalyptus Public Cloud, so comment it out -## try: -## volStat = conn.detach_volume( store.volume_id, dbInstances.instance_id, mntDevice ) -## except: -## log.debug ( 'Error detaching volume; still going to try and stop instance %s.' % dbInstances.instance_id ) -# store.attach_time = None -# store.device = None -# store.inst.instance_id = None -# store.status = volStat -# log.debug ( '***** volume status: %s' % volStat ) -# -# # Stop the instance and update status in local database -# cloudInstance.stop() -# dbInstances.stop_time = datetime.utcnow() -# while cloudInstance.state != 'terminated': -# log.debug( "Stopping instance %s state; current state: %s" % ( str( cloudInstance ).split(":")[1], cloudInstance.state ) ) -# time.sleep(3) -# cloudInstance.update() -# dbInstances.state = cloudInstance.state -# -# # Reset relevant UCI fields -# uci.state = 'available' -# uci.launch_time = None -# -# # Persist -# session = trans.sa_session -## session.save_or_update( stores ) -# session.save_or_update( dbInstances ) # TODO: Is this going to work w/ multiple instances stored in dbInstances variable? -# session.save_or_update( uci ) -# session.flush() -# trans.log_event( "User stopped cloud instance '%s'" % uci.name ) -# trans.set_message( "Galaxy instance '%s' stopped." % uci.name ) - - def update( self ): - """ - Run status update on all instances that are in 'running', 'pending', or 'shutting-down' state. - Run status update on all storage volumes whose status is 'in-use', 'creating', or 'None'. - Run status update on all snapshots whose status is 'pending' or 'delete' - Run status update on any zombie UCIs, i.e., UCI's that is in 'submitted' state for an - extended period of time. - - Reason behind this method is to sync state of local DB and real-world resources - """ - log.debug( "Running general status update for %s UCIs..." % self.type ) - # Update instances - instances = self.sa_session.query( model.CloudInstance ) \ - .filter( or_( model.CloudInstance.table.c.state==instance_states.RUNNING, - model.CloudInstance.table.c.state==instance_states.PENDING, - model.CloudInstance.table.c.state==instance_states.SHUTTING_DOWN ) ) \ - .all() - for inst in instances: - if self.type == inst.uci.credentials.provider.type: - log.debug( "[%s] Running general status update on instance '%s'" % ( inst.uci.credentials.provider.type, inst.instance_id ) ) - self.update_instance( inst ) - - # Update storage volume(s) - stores = self.sa_session.query( model.CloudStore ) \ - .filter( or_( model.CloudStore.table.c.status==store_status.IN_USE, - model.CloudStore.table.c.status==store_status.CREATING, - model.CloudStore.table.c.status==store_status.WAITING, - model.CloudStore.table.c.status==None ) ) \ - .all() - for store in stores: - if self.type == store.uci.credentials.provider.type: # and store.volume_id != None: - log.debug( "[%s] Running general status update on store with local database ID: '%s'" % ( store.uci.credentials.provider.type, store.id ) ) - self.update_store( store ) - - # Update pending snapshots or delete ones marked for deletion - snapshots = self.sa_session.query( model.CloudSnapshot ) \ - .filter( or_( model.CloudSnapshot.table.c.status == snapshot_status.PENDING, model.CloudSnapshot.table.c.status == snapshot_status.DELETE ) ) \ - .all() - for snapshot in snapshots: - if self.type == snapshot.uci.credentials.provider.type and snapshot.status == snapshot_status.PENDING: - log.debug( "[%s] Running general status update on snapshot '%s'" % ( snapshot.uci.credentials.provider.type, snapshot.snapshot_id ) ) - self.update_snapshot( snapshot ) - elif self.type == snapshot.uci.credentials.provider.type and snapshot.status == snapshot_status.DELETE: - log.debug( "[%s] Initiating deletion of snapshot '%s'" % ( snapshot.uci.credentials.provider.type, snapshot.snapshot_id ) ) - self.delete_snapshot( snapshot ) - - # Attempt at updating any zombie UCIs (i.e., instances that have been in SUBMITTED state for longer than expected - see below for exact time) - zombies = self.sa_session.query( model.UCI ).filter_by( state=uci_states.SUBMITTED ).all() - for zombie in zombies: - log.debug( "zombie UCI: %s" % zombie.name ) - z_instances = self.sa_session.query( model.CloudInstance ) \ - .filter( or_( model.CloudInstance.table.c.state != instance_states.TERMINATED, - model.CloudInstance.table.c.state == None ) ) \ - .all() - for z_inst in z_instances: - if self.type == z_inst.uci.credentials.provider.type: -# log.debug( "z_inst.id: '%s', state: '%s'" % ( z_inst.id, z_inst.state ) ) - td = datetime.utcnow() - z_inst.update_time -# log.debug( "z_inst.id: %s, time delta is %s sec" % ( z_inst.id, td.seconds ) ) - if td.seconds > 180: # if instance has been in SUBMITTED state for more than 3 minutes - log.debug( "[%s](td=%s) Running zombie repair update on instance with DB id '%s'" % ( z_inst.uci.credentials.provider.type, td.seconds, z_inst.id ) ) - self.process_zombie( z_inst ) - - def update_instance( self, inst ): - """ - Update information in local database for given instance as it is obtained from cloud provider. - Along with updating information about given instance, information about the UCI controlling - this instance is also updated. - """ - # Get credentials associated wit this instance - uci_id = inst.uci_id - uci = self.sa_session.query( model.UCI ).get( uci_id ) - self.sa_session.refresh( uci ) - conn = self.get_connection_from_uci( uci ) - - # Get reservations handle for given instance - try: - rl= conn.get_all_instances( [inst.instance_id] ) - except boto.exception.EC2ResponseError, e: - err = "Retrieving instance(s) from cloud failed for UCI '"+ uci.name +"' during general status update: " + str( e ) - log.error( err ) - uci.error = err - uci.state = uci_states.ERROR - self.sa_session.add( uci ) - self.sa_session.flush() - return None - - # Because references to reservations are deleted shortly after instances have been terminated, getting an empty list as a response to a query - # typically means the instance has successfully shut down but the check was not performed in short enough amount of time. Until an alternative solution - # is found, below code sets state of given UCI to 'error' to indicate to the user something out of ordinary happened. - if len( rl ) == 0: - err = "Instance ID '"+inst.instance_id+"' was not found by the cloud provider. Instance might have crashed or otherwise been terminated."+ \ - "Manual check is recommended." - log.error( err ) - inst.error = err - uci.error = err - inst.state = instance_states.TERMINATED - uci.state = uci_states.ERROR - uci.launch_time = None - self.sa_session.add( inst ) - self.sa_session.add( uci ) - self.sa_session.flush() - # Update instance status in local DB with info from cloud provider - for r in rl: - for i, cInst in enumerate( r.instances ): - try: - s = cInst.update() - log.debug( "Checking state of cloud instance '%s' associated with UCI '%s' and reservation '%s'. State='%s'" % ( cInst, uci.name, r, s ) ) - if s != inst.state: - inst.state = s - self.sa_session.add( inst ) - self.sa_session.flush() - # After instance has shut down, ensure UCI is marked as 'available' - if s == instance_states.TERMINATED and uci.state != uci_states.ERROR: - uci.state = uci_states.AVAILABLE - uci.launch_time = None - self.sa_session.add( uci ) - self.sa_session.flush() - # Making sure state of UCI is updated. Once multiple instances become associated with single UCI, this will need to be changed. - if s != uci.state and s != instance_states.TERMINATED: - uci.state = s - self.sa_session.add( uci ) - self.sa_session.flush() - if cInst.public_dns_name != inst.public_dns: - inst.public_dns = cInst.public_dns_name - self.sa_session.add( inst ) - self.sa_session.flush() - if cInst.private_dns_name != inst.private_dns: - inst.private_dns = cInst.private_dns_name - self.sa_session.add( inst ) - self.sa_session.flush() - except boto.exception.EC2ResponseError, e: - err = "Updating instance status from cloud failed for UCI '"+ uci.name + "' during general status update: " + str( e ) - log.error( err ) - uci.error = err - uci.state = uci_states.ERROR - self.sa_session.add( uci ) - self.sa_session.flush() - return None - - def update_store( self, store ): - """ - Update information in local database for given storage volume as it is obtained from cloud provider. - Along with updating information about given storage volume, information about the UCI controlling - this storage volume is also updated. - """ - # Get credentials associated wit this store - uci_id = store.uci_id - uci = self.sa_session.query( model.UCI ).get( uci_id ) - self.sa_session.refresh( uci ) - conn = self.get_connection_from_uci( uci ) - - if store.volume_id != None: - # Get reservations handle for given store - try: - log.debug( "Updating storage volume command: vl = conn.get_all_volumes( [%s] )" % store.volume_id ) - vl = conn.get_all_volumes( [store.volume_id] ) - except boto.exception.EC2ResponseError, e: - err = "Retrieving volume(s) from cloud failed for UCI '"+ uci.name + "' during general status update: " + str( e ) - log.error( err ) - uci.error = err - uci.state = uci_states.ERROR - self.sa_session.add( uci ) - self.sa_session.flush() - return None - - # Update store status in local DB with info from cloud provider - if len(vl) > 0: - try: - log.debug( "Storage volume '%s' current status: '%s'" % (store.volume_id, vl[0].status ) ) - if store.status != vl[0].status: - # In case something failed during creation of UCI but actual storage volume was created and yet - # UCI state remained as 'new', try to remedy this by updating UCI state here - if ( store.status == None ) and ( store.volume_id != None ): - uci.state = vl[0].status - self.sa_session.add( uci ) - self.sa_session.flush() - # If UCI was marked in state 'CREATING', update its status to reflect new status - elif ( uci.state == uci_states.CREATING ): - # Because Eucalyptus Public Cloud (EPC) deletes volumes immediately after they are created, artificially - # set status of given UCI to 'available' based on storage volume's availability zone (i.e., it's residing - # in EPC as opposed to some other Eucalyptus based cloud that allows creation of storage volumes. - if store.availability_zone == 'epc': - uci.state = uci_states.AVAILABLE - else: - uci.state = vl[0].status - - self.sa_session.add( uci ) - self.sa_session.flush() - - store.status = vl[0].status - self.sa_session.add( store ) - self.sa_session.flush() - if store.inst != None: - if store.inst.instance_id != vl[0].instance_id: - store.inst.instance_id = vl[0].instance_id - self.sa_session.add( store ) - self.sa_session.flush() - if store.attach_time != vl[0].attach_time: - store.attach_time = vl[0].attach_time - self.sa_session.add( store ) - self.sa_session.flush() - if store.device != vl[0].device: - store.device = vl[0].device - self.sa_session.add( store ) - self.sa_session.flush() - except boto.exception.EC2ResponseError, e: - err = "Updating status of volume(s) from cloud failed for UCI '"+ uci.name + "' during general status update: " + str( e ) - log.error( err ) - uci.error = err - uci.state = uci_states.ERROR - self.sa_session.add( uci ) - self.sa_session.flush() - return None - else: - err = "No storage volumes returned by cloud provider on general update" - log.error( "%s for UCI '%s'" % ( err, uci.name ) ) - store.status = store_status.ERROR - store.error = err - uci.error = err - uci.state = uci_states.ERROR - self.sa_session.add( uci ) - self.sa_session.add( store ) - self.sa_session.flush() - else: - err = "Missing storage volume ID in local database on general update. Manual check is needed to check " \ - "if storage volume was actually created by cloud provider." - log.error( "%s (for UCI '%s')" % ( err, uci.name ) ) - store.status = store_status.ERROR - store.error = err - uci.error = err - uci.state = uci_states.ERROR - self.sa_session.add( uci ) - self.sa_session.add( store ) - self.sa_session.flush() - - def update_snapshot( self, snapshot ): - """ - Update information in local database for given snapshot as it is obtained from cloud provider. - Along with updating information about given snapshot, information about the UCI controlling - this snapshot is also updated. - """ - # Get credentials associated wit this store - uci_id = snapshot.uci_id - uci = self.sa_session.query( model.UCI ).get( uci_id ) - self.sa_session.refresh( uci ) - conn = self.get_connection_from_uci( uci ) - - try: - log.debug( "Updating status of snapshot '%s'" % snapshot.snapshot_id ) - snap = conn.get_all_snapshots( [snapshot.snapshot_id] ) - if len( snap ) > 0: - log.debug( "Snapshot '%s' status: %s" % ( snapshot.snapshot_id, snap[0].status ) ) - snapshot.status = snap[0].status - self.sa_session.add( snapshot ) - self.sa_session.flush() - else: - err = "No snapshots returned by EC2 on general update" - log.error( "%s for UCI '%s'" % ( err, uci.name ) ) - snapshot.status = snapshot_status.ERROR - snapshot.error = err - uci.error = err - uci.state = uci_states.ERROR - self.sa_session.add( uci ) - self.sa_session.add( snapshot ) - self.sa_session.flush() - except boto.exception.EC2ResponseError, e: - err = "EC2 response error while updating snapshot status: " + str( e ) - log.error( err ) - snapshot.status = snapshot_status.ERROR - snapshot.error = err - uci.error = err - uci.state = uci_states.ERROR - self.sa_session.add( uci ) - self.sa_session.add( snapshot ) - self.sa_session.flush() - except Exception, ex: - err = "Error while updating snapshot status: " + str( ex ) - log.error( err ) - snapshot.status = snapshot_status.ERROR - snapshot.error = err - uci.error = err - uci.state = uci_states.ERROR - self.sa_session.add( uci ) - self.sa_session.add( snapshot ) - self.sa_session.flush() - - def delete_snapshot( self, snapshot ): - """ - Initiate deletion of given snapshot from cloud provider. - """ - if snapshot.status == snapshot_status.DELETE: - # Get credentials associated wit this store - uci_id = snapshot.uci_id - uci = self.sa_session.query( model.UCI ).get( uci_id ) - self.sa_session.refresh( uci ) - conn = self.get_connection_from_uci( uci ) - - try: - log.debug( "Deleting snapshot '%s'" % snapshot.snapshot_id ) - snap = conn.delete_snapshot( snapshot.snapshot_id ) - if snap == True: - snapshot.deleted = True - snapshot.status = snapshot_status.DELETED - self.sa_session.add( snapshot ) - self.sa_session.flush() - return snap - except boto.exception.EC2ResponseError, e: - err = "EC2 response error while deleting snapshot: " + str( e ) - log.error( err ) - snapshot.status = snapshot_status.ERROR - snapshot.error = err - uci.error = err - uci.state = uci_states.ERROR - self.sa_session.add( uci ) - self.sa_session.add( snapshot ) - self.sa_session.flush() - except Exception, ex: - err = "Error while deleting snapshot: " + str( ex ) - log.error( err ) - snapshot.status = snapshot_status.ERROR - snapshot.error = err - uci.error = err - uci.state = uci_states.ERROR - self.sa_session.add( uci ) - self.sa_session.add( snapshot ) - self.sa_session.flush() - else: - err = "Cannot delete snapshot '"+snapshot.snapshot_id+"' because its status is '"+snapshot.status+"'. Only snapshots with '" + \ - snapshot_status.COMPLETED+"' status can be deleted." - log.error( err ) - snapshot.error = err - self.sa_session.add( snapshot ) - self.sa_session.flush() - - def process_zombie( self, inst ): - """ - Attempt at discovering if starting a cloud instance was successful but local database was not updated - accordingly or if something else failed and instance was never started. Currently, no automatic - repairs are being attempted; instead, appropriate error messages are set. - """ - uci_id = inst.uci_id - uci = self.sa_session.query( model.UCI ).get( uci_id ) - self.sa_session.refresh( uci ) - - # Check if any instance-specific information was written to local DB; if 'yes', set instance and UCI's error message - # suggesting manual check. - if inst.launch_time != None or inst.reservation_id != None or inst.instance_id != None: - # Try to recover state - this is best-case effort, so if something does not work immediately, not - # recovery steps are attempted. Recovery is based on hope that instance_id is available in local DB; if not, - # report as error. - # Fields attempting to be recovered are: reservation_id, instance status, and launch_time - if inst.instance_id != None: - conn = self.get_connection_from_uci( uci ) - rl = conn.get_all_instances( [inst.instance_id] ) # reservation list - # Update local DB with relevant data from instance - if inst.reservation_id == None: - try: - inst.reservation_id = str(rl[0]).split(":")[1] - except: # something failed, so skip - pass - - try: - state = rl[0].instances[0].update() - inst.state = state - uci.state = state - self.sa_session.add( inst ) - self.sa_session.add( uci ) - self.sa_session.flush() - except: # something failed, so skip - pass - - if inst.launch_time == None: - try: - launch_time = self.format_time( rl[0].instances[0].launch_time ) - inst.launch_time = launch_time - self.sa_session.add( inst ) - self.sa_session.flush() - if inst.uci.launch_time == None: - uci.launch_time = launch_time - self.sa_session.add( uci ) - self.sa_session.flush() - except: # something failed, so skip - pass - else: - err = "Starting a machine instance (DB id: '"+str(inst.id)+"') associated with this UCI '" + str(inst.uci.name) + \ - "' seems to have failed. Because it appears that cloud instance might have gotten started, manual check is recommended." - inst.error = err - inst.state = instance_states.ERROR - inst.uci.error = err - inst.uci.state = uci_states.ERROR - log.error( err ) - self.sa_session.add( inst ) - self.sa_session.add( uci ) - self.sa_session.flush() - - else: #Instance most likely never got processed, so set error message suggesting user to try starting instance again. - err = "Starting a machine instance (DB id: '"+str(inst.id)+"') associated with this UCI '" + str(inst.uci.name) + \ - "' seems to have failed. Because it appears that cloud instance never got started, it should be safe to reset state and try " \ - "starting the instance again." - inst.error = err - inst.state = instance_states.ERROR - uci.error = err - uci.state = uci_states.ERROR - log.error( err ) - self.sa_session.add( inst ) - self.sa_session.add( uci ) - self.sa_session.flush() -# uw = UCIwrapper( inst.uci ) -# log.debug( "Try automatically re-submitting UCI '%s'." % uw.get_name() ) - - def get_connection_from_uci( self, uci ): - """ - Establish and return connection to cloud provider. Information needed to do so is obtained - directly from uci database object. - """ - log.debug( 'Establishing %s cloud connection' % self.type ) - a_key = uci.credentials.access_key - s_key = uci.credentials.secret_key - # Get connection - try: - region = RegionInfo( None, uci.credentials.provider.region_name, uci.credentials.provider.region_endpoint ) -# log.debug( "[%s] Using following command to connect to cloud provider: " -# "conn = EC2Connection( aws_access_key_id=%s, " -# "aws_secret_access_key=%s, " -# "port=%s, " -# "is_secure=%s, " -# "region=region, " -# "path=%s )" % ( self.type, a_key, s_key, uci.credentials.provider.is_secure, uci.credentials.provider.port, uci.credentials.provider.path ) ) - conn = EC2Connection( aws_access_key_id=a_key, - aws_secret_access_key=s_key, - is_secure=uci.credentials.provider.is_secure, - port=uci.credentials.provider.port, - region=region, - path=uci.credentials.provider.path ) - except boto.exception.EC2ResponseError, e: - err = "Establishing connection with cloud failed: " + str( e ) - log.error( err ) - uci.error = err - uci.state = uci_states.ERROR - self.sa_session.add( uci ) - self.sa_session.flush() - return None - - return conn - -# def updateUCI( self, uci ): -# """ -# Runs a global status update on all storage volumes and all instances that are -# associated with specified UCI -# """ -# conn = self.get_connection( uci ) -# -# # Update status of storage volumes -# vl = model.CloudStore.filter( model.CloudInstance.table.c.uci_id == uci.id ).all() -# vols = [] -# for v in vl: -# vols.append( v.volume_id ) -# try: -# volumes = conn.get_all_volumes( vols ) -# for i, v in enumerate( volumes ): -# uci.store[i].inst.instance_id = v.instance_id -# uci.store[i].status = v.status -# uci.store[i].device = v.device -# uci.store[i].flush() -# except: -# log.debug( "Error updating status of volume(s) associated with UCI '%s'. Status was not updated." % uci.name ) -# pass -# -# # Update status of instances -# il = model.CloudInstance.filter_by( uci_id=uci.id ).filter( model.CloudInstance.table.c.state != 'terminated' ).all() -# instanceList = [] -# for i in il: -# instanceList.append( i.instance_id ) -# log.debug( 'instanceList: %s' % instanceList ) -# try: -# reservations = conn.get_all_instances( instanceList ) -# for i, r in enumerate( reservations ): -# uci.instance[i].state = r.instances[0].update() -# log.debug('updating instance %s; status: %s' % ( uci.instance[i].instance_id, uci.instance[i].state ) ) -# uci.state = uci.instance[i].state -# uci.instance[i].public_dns = r.instances[0].dns_name -# uci.instance[i].private_dns = r.instances[0].private_dns_name -# uci.instance[i].flush() -# uci.flush() -# except: -# log.debug( "Error updating status of instances associated with UCI '%s'. Instance status was not updated." % uci.name ) -# pass - - # --------- Helper methods ------------ - - def format_time( self, time ): - dict = {'T':' ', 'Z':''} - for i, j in dict.iteritems(): - time = time.replace(i, j) - return time - --- a/templates/cloud/add_credentials.mako +++ /dev/null @@ -1,110 +0,0 @@ -<% _=n_ %> -<%inherit file="/base.mako"/> -<%def name="title()">Add credentials</%def> - -<%def name="javascripts()"> -${parent.javascripts()} -<script type="text/javascript"> -$(function(){ - $("input:text:first").focus(); -}) -</script> -</%def> - -%if header: - ${header} -%endif - -%if providers: - -<div class="form"> - <div class="form-title">Add credentials</div> - <div class="form-body"> - <form name="add_credentials" action="${h.url_for( action='add_credentials' )}" method="post" > - - <% - cls = "form-row" - if error.has_key('cred_error'): - cls += " form-row-error" - %> - <div class="${cls}"> - <label>Credentials name:</label> - <div class="form-row-input"> - <input type="text" name="credName" value="${credName}" size="40"> - </div> - %if error.has_key('cred_error'): - <div class="form-row-error-message">${error['cred_error']}</div> - %endif - <div style="clear: both"></div> - </div> - - - <% - cls = "form-row" - if error.has_key('provider_error'): - cls += " form-row-error" - %> - <div class="${cls}"> - <label>Cloud provider name:</label> - <div class="form-row-input"> - <select name="providerName" style="width:40em"> - <option value="">Select Provider...</option> - %for provider in providers: - <option value="${provider.name}">${provider.name}</option> - %endfor - </select> - <br/>or <a href="${h.url_for( action='add_provider' )}"> - <span>register additional cloud provider</span></a> - </div> - %if error.has_key('provider_error'): - <div class="form-row-error-message">${error['provider_error']}</div> - %endif - <div style="clear: both"></div> - </div> - - <% - cls = "form-row" - if error.has_key('access_key_error'): - cls += " form-row-error" - %> - <div class="${cls}"> - <label>Access key:</label> - <div class="form-row-input"> - <input type="text" name="accessKey" value="${accessKey}" size="40"> - </div> - %if error.has_key('access_key_error'): - <div class="form-row-error-message">${error['access_key_error']}</div> - %endif - <div style="clear: both"></div> - </div> - - - <% - cls = "form-row" - if error.has_key('secret_key_error'): - cls += " form-row-error" - %> - <div class="${cls}"> - <label>Secret key:</label> - <div class="form-row-input"> - <input type="password" name="secretKey" value="${secretKey}" size="40"> - </div> - %if error.has_key('secret_key_error'): - <div class="form-row-error-message">${error['secret_key_error']}</div> - %endif - <div style="clear: both"></div> - </div> - - <div class="form-row"><input type="submit" value="Add"></div> - </form> - </div> -</div> - -%else: - In order to add credentials, desired cloud provider needs to be registered first.<p/> - Register <a href="${h.url_for( action='add_ec2' )}"> - <span>Amazon EC2 (us-east-1 region) automatically</span></a> - or add - <a href="${h.url_for( action='add_provider' )}"> - <span>custom cloud provider</span></a>. -%endif --- a/templates/cloud/list_images.mako +++ /dev/null @@ -1,91 +0,0 @@ -<%inherit file="/base.mako"/> - -<%def name="title()">Cloud home</%def> - -%if message: -<% - try: - messagetype - except: - messagetype = "done" -%> - -<p /> -<div class="${messagetype}message"> - ${message} -</div> -%endif - -<h2>List of registered machine images:</h2> -<ul class="manage-table-actions"> - <li> - <a class="action-button" href="${h.url_for( controller='cloud', action='add_new_image' )}" target="galaxy_main"> - <img src="${h.url_for('/static/images/silk/add.png')}" /> - <span>Add machine image</span> - </a> - </li> -</ul> - -%if images: - <table class="mange-table colored" border="0" cellspacing="0" cellpadding="0" width="100%"> - <colgroup width="2%"></colgroup> - <colgroup width="10%"></colgroup> - <colgroup width="13%"></colgroup> - <colgroup width="55%"></colgroup> - <colgroup width="10%"></colgroup> - <colgroup width="5%"></colgroup> - <colgroup width="5%"></colgroup> - <tr class="header"> - <th>#</th> - <th>Provider type</th> - <th>Machime image ID</th> - <th>Manifest</th> - <th>Architecture</th> - <th>Edit</th> - <th>Delete</th> - <th></th> - </tr> - %for i, image in enumerate( images ): - <tr> - <td>${i+1}</td> - <td> - %if image.provider_type: - ${image.provider_type} - %else: - N/A - %endif - </td> - <td> - %if image.image_id: - ${image.image_id} - %else: - N/A - %endif - </td> - <td> - %if image.manifest: - ${image.manifest} - %else: - N/A - %endif - </td> - <td> - %if image.architecture: - ${image.architecture} - %else: - N/A - %endif - </td> - <td> - <a href="${h.url_for( controller='cloud', action='edit_image', image_id=image.image_id, manifest=image.manifest, id=trans.security.encode_id(image.id) )}">e</a> - </td> - <td> - <a confirm="Are you sure you want to delete machine image '${image.image_id}'? Note that this may result in users' UCI's not to work any more!" - href="${h.url_for( controller='cloud', action='delete_image', id=trans.security.encode_id(image.id) )}">x</a> - </td> - </tr> - %endfor - </table> -%else: - <h3>There are no registered machine images.</h3><br /> -%endif --- a/templates/cloud/configure_uci.mako +++ /dev/null @@ -1,116 +0,0 @@ -<% _=n_ %> -<%inherit file="/base.mako"/> -<%def name="title()">Configure new UCI</%def> - -<%def name="javascripts()"> -${parent.javascripts()} -<script type="text/javascript"> - -var providers_zones = ${h.to_json_string(providersToZones)}; - -$(function(){ - $("input:text:first").focus(); - - $("#credName").change(function() { - var zones = providers_zones[ $(this).val() ]; - var zones_select = $("#zones"); - - zones_select.children().remove(); - - for (var i in zones) { - var zone = zones[i]; - var new_option = $('<option value="' + zone + '">' + zone + '</option>'); - new_option.appendTo(zones_select); - } - - }); -}) -</script> -</%def> - -%if header: - ${header} -%endif - -<div class="form"> - <div class="form-title">Configure new Galaxy instance</div> - <div class="form-body"> - <form name="Configure new UCI" action="${h.url_for( action='configure_new_uci' )}" method="post" > - - <% - cls = "form-row" - if error.has_key('inst_error'): - cls += " form-row-error" - %> - <div class="${cls}"> - <label>Instance name:</label> - <div class="form-row-input"> - <input type="text" name="instanceName" value="${instanceName}" size="40"> - </div> - %if error.has_key('inst_error'): - <div class="form-row-error-message">${error['inst_error']}</div> - %endif - <div style="clear: both"></div> - </div> - - <% - cls = "form-row" - if error.has_key('cred_error'): - cls += " form-row-error" - %> - <div class="${cls}"> - <label>Name of registered credentials to use:</label> - <div class="form-row-input"> - <select id="credName" name="credName" style="width:40em"> - <option value="">Select Credential...</option> - % for cred in credName: - <option value="${cred.name}">${cred.name}</option> - %endfor - </select> - </div> - %if error.has_key('cred_error'): - <div class="form-row-error-message">${error['cred_error']}</div> - %endif - <div style="clear: both"></div> - </div> - - - <% - cls = "form-row" - if error.has_key('vol_error'): - cls += " form-row-error" - %> - <div class="${cls}"> - <label>Permanent storage size (1-1000GB):<br/>(Note: you will be able to add more storage later)</label> - <div class="form-row-input"> - <input type="text" name="volSize" value="${volSize}" size="40"> - </div> - %if error.has_key('vol_error'): - <div class="form-row-error-message">${error['vol_error']}</div> - %endif - <div style="clear: both"></div> - </div> - - <% - cls = "form-row" - if error.has_key('zone_error'): - cls += " form-row-error" - %> - <div class="${cls}"> - <label>Create storage in zone:</label> - <div class="form-row-input"> - <select id="zones" name="zone" style="width:40em"> - </select> - </div> - %if error.has_key('zone_error'): - <div class="form-row-error-message">${error['zone_error']}</div> - %endif - <div style="clear: both"></div> - </div> - - - <div class="form-row"><input type="submit" value="Add"></div> - - </form> - </div> -</div> --- a/templates/cloud/add_provider.mako +++ /dev/null @@ -1,252 +0,0 @@ -<% _=n_ %> -<%inherit file="/base.mako"/> -<%def name="title()">Add provider</%def> - -<%def name="javascripts()"> -${parent.javascripts()} -<script type="text/javascript"> -$(function(){ - - $("input:text:first").focus(); - - $("#type").change(function() { - if ($(this).val() == 'ec2') { - clear(); - $("#autofill").attr( 'disabled', true ); - $("#autofill").attr( 'checked', false ); - $("#name").val( "EC2" ); - $("#region_name").val( "us-east-1" ); - $("#region_endpoint").val( "us-east-1.ec2.amazonaws.com" ); - $("#is_secure").val("1"); - $("#debug").val(""); - $("#path").val("/"); - } - else if ($(this).val() == 'eucalyptus') { - clear(); - $("#autofill").attr( 'disabled', false ); - } - }); -}) - - -function af(){ - - if ( $("#autofill").attr('checked') ) { - $("#name").val("Eucalyptus Public Cloud"); - $("#region_name").val("eucalyptus"); - $("#region_endpoint").val("mayhem9.cs.ucsb.edu"); - $("#is_secure").val("0"); - $("#port").val("8773"); - $("#path").val("/services/Eucalyptus"); - } -} - -function clear() { - $("#name").val(""); - $("#region_name").val(""); - $("#region_endpoint").val(""); - $("#is_secure").val(""); - $("#port").val(""); - $("#proxy").val(""); - $("#proxy_port").val(""); - $("#proxy_user").val(""); - $("#proxy_pass").val(""); - $("#debug").val(""); - $("#https_connection_factory").val(""); - $("#path").val(""); - -} - -</script> -</%def> - -%if header: - ${header} -%endif - -<div class="form"> - <div class="form-title">Add cloud provider</div> - <div class="form-body"> - <form name="add_provider_form" action="${h.url_for( action='add_provider' )}" method="post" > - <% - cls = "form-row" - if error.has_key('type_error'): - cls += " form-row-error" - %> - <div class="${cls}"> - <label>Provider type:</label> - <div class="form-row-input"> - <select id="type" name="type" style="width:40em"> - <option value="">Select Provider...</option> - <option value="eucalyptus">Eucalyptus</option> - <option value="ec2">Amazon EC2</option> - </select> - <br/> - <input type="checkbox" id="autofill" onclick="javascript:af()" disabled="true"> - auto fill using Eucalyptus Public Cloud values - </div> - %if error.has_key('type_error'): - <div class="form-row-error-message">${error['type_error']}</div> - %endif - <div style="clear: both"></div> - </div> - - <% - cls = "form-row" - if error.has_key('name_error'): - cls += " form-row-error" - %> - <div class="${cls}"> - <label>Provider name:</label> - <div class="form-row-input"> - <input type="text" id="name" name="name" value="${name}" size="40"> - </div> - %if error.has_key('name_error'): - <div class="form-row-error-message">${error['name_error']}</div> - %endif - <div style="clear: both"></div> - </div> - - <% - cls = "form-row" - %> - <div class="${cls}"> - <label>Region name:</label> - <div id="region_selection" class="form-row-input"> - <input type="text" name="region_name" id="region_name" value="${region_name}" size="40"> - </div> - <div style="clear: both"></div> - </div> - - <% - cls = "form-row" - %> - <div class="${cls}"> - <label>Region endpoint:</label> - <div class="form-row-input"> - <input type="text" name="region_endpoint" id="region_endpoint" value="${region_endpoint}" size="40"> - </div> - <div style="clear: both"></div> - </div> - - <% - cls = "form-row" - if error.has_key('is_secure_error'): - cls += " form-row-error" - %> - <div class="${cls}"> - <label>Is secure ('O' for False or '1' for True):</label> - <div class="form-row-input"> - <input type="text" name="is_secure" id="is_secure" value="${is_secure}" size="40"> - </div> - %if error.has_key('is_secure_error'): - <div class="form-row-error-message">${error['is_secure_error']}; you entered: '${is_secure}'</div> - %endif - <div style="clear: both"></div> - </div> - - <% - cls = "form-row" - %> - <div class="${cls}"> - <label>Host:</label> - <div class="form-row-input"> - <input type="text" name="host" value="${host}" size="40"> - </div> - <div style="clear: both"></div> - </div> - - <% - cls = "form-row" - %> - <div class="${cls}"> - <label>Port:</label> - <div class="form-row-input"> - <input type="text" name="port" id="port" value="${port}" size="40"> - </div> - <div style="clear: both"></div> - </div> - - <% - cls = "form-row" - %> - <div class="${cls}"> - <label>Proxy:</label> - <div class="form-row-input"> - <input type="text" name="proxy" value="${proxy}" size="40"> - </div> - <div style="clear: both"></div> - </div> - - <% - cls = "form-row" - %> - <div class="${cls}"> - <label>Proxy port:</label> - <div class="form-row-input"> - <input type="text" name="proxy_port" value="${proxy_port}" size="40"> - </div> - <div style="clear: both"></div> - </div> - - <% - cls = "form-row" - %> - <div class="${cls}"> - <label>Proxy user:</label> - <div class="form-row-input"> - <input type="text" name="proxy_user" value="${proxy_user}" size="40"> - </div> - <div style="clear: both"></div> - </div> - - <% - cls = "form-row" - %> - <div class="${cls}"> - <label>Proxy pass:</label> - <div class="form-row-input"> - <input type="text" name="proxy_pass" value="${proxy_pass}" size="40"> - </div> - <div style="clear: both"></div> - </div> - - <% - cls = "form-row" - %> - <div class="${cls}"> - <label>Debug:</label> - <div class="form-row-input"> - <input type="text" name="debug" value="${debug}" size="40"> - </div> - <div style="clear: both"></div> - </div> - - <% - cls = "form-row" - %> - <div class="${cls}"> - <label>HTTPS connection factory:</label> - <div class="form-row-input"> - <input type="text" name="https_connection_factory" value="${https_connection_factory}" size="40"> - </div> - <div style="clear: both"></div> - </div> - - <% - cls = "form-row" - %> - <div class="${cls}"> - <label>Path:</label> - <div class="form-row-input"> - <input type="text" name="path" id="path" value="${path}" size="40"> - </div> - <div style="clear: both"></div> - </div> - - <div class="form-row"><input type="submit" value="Add"></div> - - </form> - - </div> -</div> --- a/templates/cloud/index.mako +++ /dev/null @@ -1,16 +0,0 @@ -<%inherit file="/webapps/galaxy/base_panels.mako"/> - -<%def name="init()"> -<% - self.has_left_panel=False - self.has_right_panel=False - self.active_view="cloud" - self.message_box_visible=False -%> -</%def> - -<%def name="center_panel()"> - - <iframe name="galaxy_main" id="galaxy_main" frameborder="0" style="position: absolute; width: 100%; height: 100%;" src="${h.url_for( controller="cloud", action="list" )}"></iframe> - -</%def> --- a/templates/cloud/add_image.mako +++ /dev/null @@ -1,98 +0,0 @@ -<% _=n_ %> -<%inherit file="/base.mako"/> -<%def name="title()">Add machine image</%def> - -<%def name="javascripts()"> -${parent.javascripts()} -<script type="text/javascript"> -$(function(){ - //$("input:text:first").focus(); -}) -</script> -</%def> - -%if header: - ${header} -%endif - -<div class="form"> - <div class="form-title">Add machine image</div> - <div class="form-body"> - <form name="add_image" action="${h.url_for( action='add_new_image' )}" method="post" > - <% - cls = "form-row" - if error.has_key('provider_error'): - cls += " form-row-error" - %> - <div class="${cls}"> - <label>Cloud provider type:</label> - <div class="form-row-input"> - <select name="provider_type" style="width:40em"> - <option value="">Select Provider Type...</option> - <option value="eucalyptus">Eucalyptus</option> - <option value="ec2">Amazon EC2</option> - </select> - </div> - %if error.has_key('provider_error'): - <div class="form-row-error-message">${error['provider_error']}</div> - %endif - <div style="clear: both"></div> - </div> - - <% - cls = "form-row" - if error.has_key('id_error'): - cls += " form-row-error" - %> - <div class="${cls}"> - <label>Machine Image ID (AMI or EMI):</label> - <div class="form-row-input"> - <input type="text" name="image_id" value="${image_id}" size="40"> - </div> - %if error.has_key('id_error'): - <div class="form-row-error-message">${error['id_error']}</div> - %endif - <div style="clear: both"></div> - </div> - - <% - cls = "form-row" - if error.has_key('manifest_error'): - cls += " form-row-error" - %> - <div class="${cls}"> - <label>Manifest:</label> - <div class="form-row-input"> - <input type="text" name="manifest" value="${manifest}" size="40"> - </div> - %if error.has_key('manifest_error'): - <div class="form-row-error-message">${error['manifest_error']}</div> - %endif - <div style="clear: both"></div> - </div> - - - <% - cls = "form-row" - if error.has_key('arch_error'): - cls += " form-row-error" - %> - <div class="${cls}"> - <label>Image architecture:</label> - <div class="form-row-input"> - <select name="architecture" style="width:40em"> - <option value="">Select Architecture Type...</option> - <option value="i386">i386 (32 bit)</option> - <option value="x86_64">x86_64 (64 bit)</option> - </select> - </div> - %if error.has_key('arch_error'): - <div class="form-row-error-message">${error['arch_error']}</div> - %endif - <div style="clear: both"></div> - </div> - - <div class="form-row"><input type="submit" value="Add"></div> - </form> - </div> -</div> --- a/lib/galaxy/cloud/providers/ec2.py +++ /dev/null @@ -1,1232 +0,0 @@ -import subprocess, threading, os, errno, time, datetime, stat -from Queue import Queue, Empty -from datetime import datetime - -from galaxy import model # Database interaction class -from galaxy.model import mapping -from galaxy.datatypes.data import nice_size -from galaxy.util.bunch import Bunch -from galaxy.cloud import UCIwrapper -from Queue import Queue -from sqlalchemy import or_, and_ - -import galaxy.eggs -galaxy.eggs.require("boto") -from boto.ec2.connection import EC2Connection -from boto.ec2.regioninfo import RegionInfo -import boto.exception -import boto - -import logging -log = logging.getLogger( __name__ ) - -uci_states = Bunch( - NEW_UCI = "newUCI", - NEW = "new", - CREATING = "creating", - DELETING_UCI = "deletingUCI", - DELETING = "deleting", - SUBMITTED_UCI = "submittedUCI", - SUBMITTED = "submitted", - SHUTTING_DOWN_UCI = "shutting-downUCI", - SHUTTING_DOWN = "shutting-down", - ADD_STORAGE_UCI = "add-storageUCI", - ADD_STORAGE = "add-storage", - AVAILABLE = "available", - RUNNING = "running", - PENDING = "pending", - ERROR = "error", - DELETED = "deleted", - SNAPSHOT_UCI = "snapshotUCI", - SNAPSHOT = "snapshot" -) - -instance_states = Bunch( - TERMINATED = "terminated", - SUBMITTED = "submitted", - RUNNING = "running", - ADDING = "adding-storage", - PENDING = "pending", - SHUTTING_DOWN = "shutting-down", - ERROR = "error" -) - -store_status = Bunch( - WAITING = "waiting", - IN_USE = "in-use", - ADDING = "adding", - CREATING = "creating", - DELETED = 'deleted', - ERROR = "error" -) - -snapshot_status = Bunch( - SUBMITTED = 'submitted', - PENDING = 'pending', - COMPLETED = 'completed', - DELETE = 'delete', - DELETED= 'deleted', - ERROR = "error" -) - -class EC2CloudProvider( object ): - """ - Amazon EC2-based cloud provider implementation for managing instances. - """ - STOP_SIGNAL = object() - def __init__( self, app ): - self.type = "ec2" # cloud provider type (e.g., ec2, eucalyptus, opennebula) - self.zone = "us-east-1a" - self.security_group = "galaxyWeb" - self.queue = Queue() - self.sa_session = app.model.context - - self.threads = [] - nworkers = 5 - log.info( "Starting EC2 cloud controller workers..." ) - for i in range( nworkers ): - worker = threading.Thread( target=self.run_next ) - worker.start() - self.threads.append( worker ) - log.debug( "%d EC2 cloud workers ready", nworkers ) - - def shutdown( self ): - """Attempts to gracefully shut down the monitor thread""" - log.info( "sending stop signal to worker threads in EC2 cloud manager" ) - for i in range( len( self.threads ) ): - self.queue.put( self.STOP_SIGNAL ) - log.info( "EC2 cloud manager stopped" ) - - def put( self, uci_wrapper ): - """ - Add uci_wrapper object to the end of the request queue to be handled by - this cloud provider. - """ - state = uci_wrapper.get_uci_state() - uci_wrapper.change_state( state.split('U')[0] ) # remove 'UCI' from end of state description (i.e., mark as accepted and ready for processing) - self.queue.put( uci_wrapper ) - - def run_next( self ): - """Process next request, waiting until one is available if necessary.""" - cnt = 0 - while 1: - - uci_wrapper = self.queue.get() - uci_state = uci_wrapper.get_uci_state() - if uci_state is self.STOP_SIGNAL: - return - try: - if uci_state==uci_states.NEW: - self.create_uci( uci_wrapper ) - elif uci_state==uci_states.DELETING: - self.delete_uci( uci_wrapper ) - elif uci_state==uci_states.SUBMITTED: - self.start_uci( uci_wrapper ) - elif uci_state==uci_states.SHUTTING_DOWN: - self.stop_uci( uci_wrapper ) - elif uci_state==uci_states.SNAPSHOT: - self.snapshot_uci( uci_wrapper ) - elif uci_state==uci_states.ADD_STORAGE: - self.add_storage_to_uci( uci_wrapper ) - #self.dummy_start_uci( uci_wrapper ) - except: - log.exception( "Uncaught exception executing cloud request." ) - cnt += 1 - - def get_connection( self, uci_wrapper ): - """ - Establishes cloud connection using user's credentials associated with given UCI - """ - log.debug( 'Establishing %s cloud connection.' % self.type ) - provider = uci_wrapper.get_provider() - try: - region = RegionInfo( None, provider.region_name, provider.region_endpoint ) - except Exception, ex: - err = "Selecting region with cloud provider failed: " + str( ex ) - log.error( err ) - uci_wrapper.set_error( err, True ) - return None - try: - conn = EC2Connection( aws_access_key_id=uci_wrapper.get_access_key(), - aws_secret_access_key=uci_wrapper.get_secret_key(), - is_secure=provider.is_secure, - region=region, - path=provider.path ) - except boto.exception.EC2ResponseError, e: - err = "Establishing connection with cloud failed: " + str( e ) - log.error( err ) - uci_wrapper.set_error( err, True ) - return None - - return conn - - def check_key_pair( self, uci_wrapper, conn ): - """ - Check if a key pair associated with this UCI exists on cloud provider. - If yes, return key pair name; otherwise, generate a key pair with the cloud - provider and, again, return key pair name. - Key pair name for given UCI is generated from UCI's name and suffix '_kp' - """ - kp = None - kp_name = uci_wrapper.get_name().replace(' ','_') + "_kp" - log.debug( "Checking user's key pair: '%s'" % kp_name ) - try: - kp = conn.get_key_pair( kp_name ) - uci_kp_name = uci_wrapper.get_key_pair_name() - uci_material = uci_wrapper.get_key_pair_material() - if kp != None: - if kp.name != uci_kp_name or uci_material == None: - # key pair exists on the cloud but not in local database, so re-generate it (i.e., delete and then create) - try: - conn.delete_key_pair( kp_name ) - kp = self.create_key_pair( conn, kp_name ) - uci_wrapper.set_key_pair( kp.name, kp.material ) - except boto.exception.EC2ResponseError, e: - err = "EC2 response error while deleting key pair: " + str( e ) - log.error( err ) - uci_wrapper.set_error( err, True ) - else: - try: - kp = self.create_key_pair( conn, kp_name ) - uci_wrapper.set_key_pair( kp.name, kp.material ) - except boto.exception.EC2ResponseError, e: - err = "EC2 response error while creating key pair: " + str( e ) - log.error( err ) - uci_wrapper.set_error( err, True ) - except Exception, ex: - err = "Exception while creating key pair: " + str( ex ) - log.error( err ) - uci_wrapper.set_error( err, True ) - except boto.exception.EC2ResponseError, e: # No keypair under this name exists so create it - if e.code == 'InvalidKeyPair.NotFound': - log.info( "No keypair found, creating keypair '%s'" % kp_name ) - kp = self.create_key_pair( conn, kp_name ) - uci_wrapper.set_key_pair( kp.name, kp.material ) - else: - err = "EC2 response error while retrieving key pair: " + str( e ) - log.error( err ) - uci_wrapper.set_error( err, True ) - - if kp != None: - return kp.name - else: - return None - - def create_key_pair( self, conn, kp_name ): - """ Initiate creation of key pair under kp_name by current cloud provider. """ - try: - return conn.create_key_pair( kp_name ) - except boto.exception.EC2ResponseError, e: - return None - - def get_mi_id( self, uci_wrapper, i_index ): - """ - Get appropriate machine image (mi) based on instance size. - """ - i_type = uci_wrapper.get_instance_type( i_index ) - if i_type=='m1.small' or i_type=='c1.medium': - arch = 'i386' - else: - arch = 'x86_64' - - mi = self.sa_session.query( model.CloudImage ).filter_by( deleted=False, provider_type=self.type, architecture=arch ).first() - if mi: - return mi.image_id - else: - err = "Machine image could not be retrieved" - log.error( "%s for UCI '%s'." % (err, uci_wrapper.get_name() ) ) - uci_wrapper.set_error( err+". Contact site administrator to ensure needed machine image is registered.", True ) - return None - - def create_uci( self, uci_wrapper ): - """ - Create User Configured Instance (UCI) - i.e., create storage volume on cloud provider - and register relevant information in local Galaxy database. - """ - conn = self.get_connection( uci_wrapper ) - if uci_wrapper.get_uci_availability_zone()=='': - log.info( "Availability zone for UCI (i.e., storage volume) was not selected, using default zone: %s" % self.zone ) - uci_wrapper.set_store_availability_zone( self.zone ) - - store = uci_wrapper.get_all_stores_in_status( store_status.ADDING )[0] # Because at UCI creation time only 1 storage volume can be created, reference it directly - - log.info( "Creating storage volume in zone '%s' of size '%s'..." % ( uci_wrapper.get_uci_availability_zone(), store.size ) ) - # Because only 1 storage volume may be created at UCI config time, index of this storage volume in local Galaxy DB w.r.t - # current UCI is 0, so reference it in following methods - vol = conn.create_volume( store.size, uci_wrapper.get_uci_availability_zone(), snapshot=None ) - uci_wrapper.set_store_volume_id( store.id, vol.id ) - - # Retrieve created volume again to get updated status - try: - vl = conn.get_all_volumes( [vol.id] ) - except boto.exception.EC2ResponseError, e: - err = "EC2 response error while retrieving (i.e., updating status) of just created storage volume '" + vol.id + "': " + str( e ) - log.error( err ) - uci_wrapper.set_store_status( vol.id, uci_states.ERROR ) - uci_wrapper.set_error( err, True ) - return - except Exception, ex: - err = "Error while retrieving (i.e., updating status) of just created storage volume '" + vol.id + "': " + str( ex ) - log.error( err ) - uci_wrapper.set_error( err, True ) - return - - if len( vl ) > 0: - uci_wrapper.change_state( uci_state=vl[0].status ) - uci_wrapper.set_store_status( vol.id, vl[0].status ) - else: - err = "Volume '" + vol.id +"' not found by EC2 after being created." - log.error( err ) - uci_wrapper.set_store_status( vol.id, uci_states.ERROR ) - uci_wrapper.set_error( err, True ) - - def delete_uci( self, uci_wrapper ): - """ - Delete UCI - i.e., delete all storage volumes associated with this UCI. - NOTE that this implies deletion of any and all data associated - with this UCI from the cloud. All data will be deleted. - Information in local Galaxy database is marked as deleted but not actually removed - from the database. - """ - conn = self.get_connection( uci_wrapper ) - vl = [] # volume list - count = 0 # counter for checking if all volumes assoc. w/ UCI were deleted - - # Get all volumes assoc. w/ UCI, delete them from cloud as well as in local DB - vl = uci_wrapper.get_all_stores() - deletedList = [] - failedList = [] - for v in vl: - log.debug( "Deleting volume with id='%s'" % v.volume_id ) - try: - if conn.delete_volume( v.volume_id ): - deletedList.append( v.volume_id ) - v.deleted = True - v.status = store_status.DELETED - self.sa_session.add( v ) - self.sa_session.flush() - count += 1 - else: - failedList.append( v.volume_id ) - except boto.exception.EC2ResponseError, e: - err = "EC2 response error while deleting storage volume '" + v.volume_id + "': " + str( e ) - log.error( err ) - uci_wrapper.set_store_error( err, store_id = v.volume_id ) - uci_wrapper.set_error( err, True ) - - # Delete UCI if all of associated - if count == len( vl ): - uci_wrapper.set_deleted() - else: - if uci_wrapper.get_uci_state != uci_states.ERROR: - err = "Deleting following volume(s) failed: " + str( failedList ) + ". However, these volumes were successfully deleted: " \ - + str( deletedList ) + ". MANUAL intervention and processing needed." - log.error( err ) - uci_wrapper.set_error( err, True ) - - def snapshot_uci( self, uci_wrapper ): - """ - Initiate creation of a snapshot by cloud provider for all storage volumes - associated with this UCI. - """ - if uci_wrapper.get_uci_state() != uci_states.ERROR: - conn = self.get_connection( uci_wrapper ) - - snapshots = uci_wrapper.get_snapshots( status = snapshot_status.SUBMITTED ) - for snapshot in snapshots: - log.debug( "Snapshot DB id: '%s', volume id: '%s'" % ( snapshot.id, snapshot.store.volume_id ) ) - try: - snap = conn.create_snapshot( volume_id=snapshot.store.volume_id ) - snap_id = str( snap ).split(':')[1] - uci_wrapper.set_snapshot_id( snapshot.id, snap_id ) - sh = conn.get_all_snapshots( snap_id ) # get updated status - uci_wrapper.set_snapshot_status( status=sh[0].status, snap_id=snap_id ) - except boto.exception.EC2ResponseError, e: - err = "EC2 response error while creating snapshot: " + str( e ) - log.error( err ) - uci_wrapper.set_snapshot_error( error=err, snap_index=snapshot.id, set_status=True ) - uci_wrapper.set_error( err, True ) - return - except Exception, ex: - err = "Error while creating snapshot: " + str( ex ) - log.error( err ) - uci_wrapper.set_snapshot_error( error=err, snap_index=snapshot.id, set_status=True ) - uci_wrapper.set_error( err, True ) - return - - uci_wrapper.change_state( uci_state=uci_states.AVAILABLE ) - - def add_storage_to_uci( self, uci_wrapper ): - """ - Add an additional storage volume to specified UCI by creating the storage volume - on cloud provider, attaching it to currently running instance and adding it to - 'galaxyData' zpool on remote instance. - """ - conn = self.get_connection( uci_wrapper ) - - stores = uci_wrapper.get_all_stores_in_status( store_status.ADDING ) - for store in stores: - vol_size = store.size - availability_zone = uci_wrapper.get_uci_availability_zone() - log.info( "Adding storage volume to UCI '%s' in zone '%s' of size '%s'..." % ( uci_wrapper.get_name(), availability_zone, vol_size ) ) - - try: - vol = conn.create_volume( vol_size, availability_zone, snapshot=None ) - uci_wrapper.set_store_volume_id( store.id, vol.id ) - uci_wrapper.set_store_availability_zone( availability_zone, vol.id ) - log.debug( "New storage volume created: '%s'" % vol.id ) - except boto.exception.EC2ResponseError, e: - err = "EC2 response error while creating storage volume: " + str( e ) - log.error( err ) - uci_wrapper.set_store_error( err, store_id=vol.id ) - uci_wrapper.set_error( err, True ) - return - except Exception, ex: - err = "Error while creating storage volume: " + str( ex ) - log.error( err ) - uci_wrapper.set_error( err, True ) - return - - # Retrieve created volume again to get updated status - try: - vl = conn.get_all_volumes( [vol.id] ) - except boto.exception.EC2ResponseError, e: - err = "EC2 response error while retrieving (i.e., updating status) of just created storage volume '" + vol.id + "': " + str( e ) - log.error( err ) - uci_wrapper.set_store_error( err, store_id=vol.id ) - uci_wrapper.set_error( err, True ) - return - except Exception, ex: - err = "Error while retrieving (i.e., updating status) of just created storage volume '" + vol.id + "': " + str( ex ) - log.error( err ) - uci_wrapper.set_error( err, True ) - return - - # Wait for a while to ensure volume was created - if len( vl ) > 0: - vol_status = vl[0].status # Bc. only single vol is queried, reference it as 0th list element - for i in range( 30 ): - if vol_status != "available": - log.debug( "(%s) Updating volume status; current status: '%s'" % (i, vol_status ) ) - uci_wrapper.change_state( uci_state=vol_status ) - time.sleep(5) - vol_status = vl[0].status - if vol_status == "available": - log.debug( "(%s) New volume status '%s', continuing with file system adjustment." % (i, vol_status ) ) - uci_wrapper.set_store_status( vl[0].id, vol_status ) - break - if i is 29: - err = "Error while creating volume '"+vl[0].id+"'; stuck in state '"+vol_status+"'; deleting volume." - conn.delete_volume( vl[0].id ) - log.error( err ) - uci_wrapper.set_error( err, True ) - uci_wrapper.set_store_error( err, store_id=vol.id ) - conn.delete_volume( vl[0].id ) - uci_wrapper.set_store_deleted( vl[0].id ) - return - else: - err = "Volume '" + vol.id +"' not found by EC2 after being created." - log.error( err ) - uci_wrapper.set_store_error( err, store_id=vol.id ) - uci_wrapper.set_error( err, True ) - return - - # Get private key for given instance - pk = uci_wrapper.get_key_pair_material() - if pk == None: #If pk does not exist, create it - self.check_key_pair( uci_wrapper, conn ) - pk = uci_wrapper.get_key_pair_material() - - # Get working directory for this UCI and store pk into a file - wd = uci_wrapper.get_uci_working_directory() - if not os.path.exists( wd ): - os.mkdir( wd ) - pk_file_path = os.path.join( wd, "pk" ) - - if pk != None: - # Save private key to a file - pk_file = open( pk_file_path, "w" ) - pk_file.write( pk ) - pk_file.close() - else: - err = "ERROR: Private key not available for this UCI." - log.error( err ) - uci_wrapper.set_store_error( err, store_id=vol.id ) - uci_wrapper.set_error( err, True ) - return - - if os.path.exists( pk_file_path ): - # Change permissions of the file - this is required by later used ssh - os.chmod( pk_file_path, stat.S_IRUSR | stat.S_IWUSR ) - - # Get # of storage volumes associated with this UCI to know as which device to connect new volume to the instance - device_num = len( uci_wrapper.get_all_stores_in_status( store_status.IN_USE ) ) + 5 # First device num is 5, so all subsequent ones should follow - - # Get instance that the new storage volume is to be attached to. Although a list is returned, - # only 1 instance can be in 'adding-storage' state (because, for now, only 1 instance is assoc. with - # each UCI) and volume can be attached to only to it - il = uci_wrapper.get_instaces_in_state( instance_states.ADDING ) - if len( il ) > 0: - # Attach new volume to the instance - log.debug( "Attaching new storage volume '%s' to UCI '%s' as device '%s'" % - ( vol.id, uci_wrapper.get_name(), device_num ) ) - try: - vol_status = conn.attach_volume( vol.id, il[0].instance_id, device_num ) - except boto.exception.EC2ResponseError, e: - err = "Attaching just created storage volume '" + vol.id + "'to instance '" + \ - il[0].instance_id + "' as device '" + str( device_num ) + "' failed: " + str( e ) - log.error( err ) - uci_wrapper.set_store_error( err, store_id=vol.id ) - uci_wrapper.set_error( err, True ) - return - # For a while, keep checking attachment status of the new volume - for i in range(30): - log.debug( "Checking attachment status of new volume '%s': '%s'" % ( vol.id, vol_status ) ) - if vol_status == 'attached': - uci_wrapper.set_store_status( vol.id, vol_status ) - uci_wrapper.set_store_device( vol.id, device_num ) - break - if i == 29: - err = "Storage volume '" + vol.id + "' failed to attach to instance '" + il[0].instance_id + \ - "'. Manual check needed." - log.error( err ) - uci_wrapper.set_store_error( err, store_id=vol.id ) - uci_wrapper.set_error( err, False ) - return - - time.sleep(4) - vol_list = conn.get_all_volumes( [vol.id] ) - for v in vol_list: - vol_status = v.attachment_state() - - # Once storage volume is attached, add it to the zpool by issuing system level command - cmd = 'ssh -o StrictHostKeyChecking=no -i '+ pk_file_path +' root@'+il[0].public_dns+' "zpool add galaxyData c7d' + str( device_num )+'"' - log.debug( "Adding new storage volume to zpool cmd: %s" % cmd ) - stdout = os.system( cmd ) - if stdout != 0: - err = "Adding newly created storage volume to zpool on instance '" + il[0].instance_id + \ - "' failed. Error code: " + str( stdout ) - log.error( err ) - uci_wrapper.set_store_error( err, store_id=vol.id ) - uci_wrapper.set_error( err, False ) - return - else: - err = "No instance(s) found in 'adding-storage' state. New disk not added to UCI's zpool." - log.error( err ) - uci_wrapper.set_store_error( err, store_id=vol.id ) - uci_wrapper.set_error( err, True ) - return - - # Update UCI's total storage size - uci_wrapper.set_uci_total_size( uci_wrapper.get_uci_total_size() + vol.size ) - # Reset UCI's and instance's state - uci_wrapper.change_state( uci_state=uci_states.RUNNING, instance_id=il[0].instance_id, i_state=instance_states.RUNNING ) - log.debug( "Successfully added storage volume '%s' to UCI '%s'." % ( vol.id, uci_wrapper.get_name() ) ) - - def dummy_start_uci( self, uci_wrapper ): - - uci = uci_wrapper.get_uci() - log.debug( "Dummy start UCI '%s'" % uci.name ) - - -# uci_wrapper.change_state( uci_state.PENDING ) -# log.debug( "Sleeping a bit... (%s)" % uci.name ) -# time.sleep(20) -# log.debug( "Woke up! (%s)" % uci.name ) - - def start_uci( self, uci_wrapper ): - """ - Start instance(s) of given UCI on the cloud. - """ - if uci_wrapper.get_uci_state() != uci_states.ERROR: - conn = self.get_connection( uci_wrapper ) - self.check_key_pair( uci_wrapper, conn ) - if uci_wrapper.get_key_pair_name() == None: - err = "Key pair not found" - log.error( "%s for UCI '%s'." % ( err, uci_wrapper.get_name() ) ) - uci_wrapper.set_error( err + ". Try resetting the state and starting the instance again.", True ) - return - - i_indexes = uci_wrapper.get_instances_indexes( state=instance_states.SUBMITTED ) # Get indexes of i_indexes associated with this UCI that are in 'submitted' state - log.debug( "Starting instances with IDs: '%s' associated with UCI '%s' " % ( i_indexes, uci_wrapper.get_name(), ) ) - if len( i_indexes ) > 0: - for i_index in i_indexes: - # Get machine image for current instance - mi_id = self.get_mi_id( uci_wrapper, i_index ) - log.debug( "mi_id: %s, uci_wrapper.get_key_pair_name(): %s" % ( mi_id, uci_wrapper.get_key_pair_name() ) ) - uci_wrapper.set_mi( i_index, mi_id ) - - if mi_id != None: - # Check if galaxy security group exists (and create it if it does not) - log.debug( "Setting up '%s' security group." % self.security_group ) - try: - conn.get_all_security_groups( [self.security_group] ) # security groups - except boto.exception.EC2ResponseError, e: - if e.code == 'InvalidGroup.NotFound': - log.info( "No security group found, creating security group '%s'" % self.security_group ) - try: - gSecurityGroup = conn.create_security_group(self.security_group, 'Security group for Galaxy.') - gSecurityGroup.authorize( 'tcp', 80, 80, '0.0.0.0/0' ) # Open HTTP port - gSecurityGroup.authorize( 'tcp', 22, 22, '0.0.0.0/0' ) # Open SSH port - except boto.exception.EC2ResponseError, ee: - err = "EC2 response error while creating security group: " + str( ee ) - log.error( err ) - uci_wrapper.set_error( err, True ) - else: - err = "EC2 response error while retrieving security group: " + str( e ) - log.error( err ) - uci_wrapper.set_error( err, True ) - - - if uci_wrapper.get_uci_state() != uci_states.ERROR: - # Start an instance - log.debug( "Starting instance for UCI '%s'" % uci_wrapper.get_name() ) - #TODO: Once multiple volumes can be attached to a single instance, update 'userdata' composition - # Compose user data; for storage volumes, separate multiple volumes with a colon (:) ensuring that - # the last volume in the list is not followed by a colon. - stores = uci_wrapper.get_all_stores() - volume_ids = "" - if len( stores ) > 0: - for i, store in enumerate( stores ): - volume_ids += store.volume_id - if i < len( stores )-1: - volume_ids += ":" - else: - err = "No storage volumes found that are associated with UCI '%s'" + uci_wrapper.get_name() - log.error( err ) - uci_wrapper.set_error( err, True ) - return - userdata = volume_ids+"|"+uci_wrapper.get_access_key()+"|"+uci_wrapper.get_secret_key() - log.debug( "Using following command: conn.run_instances( image_id='%s', key_name='%s', security_groups=['%s'], user_data=[OMITTED], instance_type='%s', placement='%s' )" - % ( mi_id, uci_wrapper.get_key_pair_name(), self.security_group, uci_wrapper.get_instance_type( i_index ), uci_wrapper.get_uci_availability_zone() ) ) - # Start an instance - reservation = None - try: - reservation = conn.run_instances( image_id=mi_id, - key_name=uci_wrapper.get_key_pair_name(), - security_groups=[self.security_group], - user_data=userdata, - instance_type=uci_wrapper.get_instance_type( i_index ), - placement=uci_wrapper.get_uci_availability_zone() ) - except boto.exception.EC2ResponseError, e: - err = "EC2 response error when starting UCI '"+ uci_wrapper.get_name() +"': " + str( e ) - log.error( err ) - uci_wrapper.set_error( err, True ) - except Exception, ex: - err = "Error when starting UCI '" + uci_wrapper.get_name() + "': " + str( ex ) - log.error( err ) - uci_wrapper.set_error( err, True ) - # Record newly available instance data into local Galaxy database - if reservation: - l_time = datetime.utcnow() - # uci_wrapper.set_instance_launch_time( self.format_time( reservation.instances[0].launch_time ), i_index=i_index ) - uci_wrapper.set_instance_launch_time( l_time, i_index=i_index ) - if not uci_wrapper.uci_launch_time_set(): - uci_wrapper.set_uci_launch_time( l_time ) - try: - uci_wrapper.set_reservation_id( i_index, str( reservation ).split(":")[1] ) - # TODO: if more than a single instance will be started through single reservation, change this reference to element [0] - i_id = str( reservation.instances[0]).split(":")[1] - uci_wrapper.set_instance_id( i_index, i_id ) - s = reservation.instances[0].state - uci_wrapper.change_state( s, i_id, s ) - uci_wrapper.set_security_group_name( self.security_group, i_id=i_id ) - vol_id = uci_wrapper.get_store_volume_id( store_id=0 ) # TODO: Once more that one vol/UCI is allowed, update this! - # Following line is pointless bc. general update updates status of volume to 'available' - # before it actually connects to starting instance... This has been dealt w/ in general update method - #uci_wrapper.set_store_status( vol_id, store_status.WAITING ) - uci_wrapper.set_store_instance( vol_id, i_id ) - log.debug( "Instance of UCI '%s' started, current state: '%s'" % ( uci_wrapper.get_name(), uci_wrapper.get_uci_state() ) ) - except boto.exception.EC2ResponseError, e: - err = "EC2 response error when retrieving instance information for UCI '" + uci_wrapper.get_name() + "': " + str( e ) - log.error( err ) - uci_wrapper.set_error( err, True ) - else: - log.error( "UCI '%s' is in 'error' state, starting instance was aborted." % uci_wrapper.get_name() ) - else: - err = "No instances in state '"+ instance_states.SUBMITTED +"' found for UCI '" + uci_wrapper.get_name() + \ - "'. Nothing to start." - log.error( err ) - uci_wrapper.set_error( err, True ) - else: - log.error( "UCI '%s' is in 'error' state, starting instance was aborted." % uci_wrapper.get_name() ) - - def stop_uci( self, uci_wrapper): - """ - Stop all of cloud instances associated with given UCI. - """ - conn = self.get_connection( uci_wrapper ) - - # Get all instances associated with given UCI - il = uci_wrapper.get_instances_ids() # instance list - # Process list of instances and remove any references to empty instance id's - for i in il: - if i is None: - il.remove( i ) - log.debug( 'List of instances being terminated: %s' % il ) - rl = conn.get_all_instances( il ) # Reservation list associated with given instances - - # Initiate shutdown of all instances under given UCI - cnt = 0 - stopped = [] - not_stopped = [] - for r in rl: - for inst in r.instances: - log.debug( "Sending stop signal to instance '%s' associated with reservation '%s'." % ( inst, r ) ) - try: - inst.stop() - uci_wrapper.set_stop_time( datetime.utcnow(), i_id=inst.id ) - uci_wrapper.change_state( instance_id=inst.id, i_state=inst.update() ) - stopped.append( inst ) - except boto.exception.EC2ResponseError, e: - not_stopped.append( inst ) - err = "EC2 response error when stopping instance '" + inst.instance_id + "': " + str(e) - log.error( err ) - uci_wrapper.set_error( err, True ) - - uci_wrapper.reset_uci_launch_time() - log.debug( "Termination was initiated for all instances of UCI '%s'." % uci_wrapper.get_name() ) - - -# dbInstances = get_instances( trans, uci ) #TODO: handle list! -# -# # Get actual cloud instance object -# cloudInstance = get_cloud_instance( conn, dbInstances.instance_id ) -# -# # TODO: Detach persistent storage volume(s) from instance and update volume data in local database -# stores = get_stores( trans, uci ) -# for i, store in enumerate( stores ): -# log.debug( "Detaching volume '%s' to instance '%s'." % ( store.volume_id, dbInstances.instance_id ) ) -# mntDevice = store.device -# volStat = None -## Detaching volume does not work with Eucalyptus Public Cloud, so comment it out -## try: -## volStat = conn.detach_volume( store.volume_id, dbInstances.instance_id, mntDevice ) -## except: -## log.debug ( 'Error detaching volume; still going to try and stop instance %s.' % dbInstances.instance_id ) -# store.attach_time = None -# store.device = None -# store.i_id = None -# store.status = volStat -# log.debug ( '***** volume status: %s' % volStat ) -# -# -# # Stop the instance and update status in local database -# cloudInstance.stop() -# dbInstances.stop_time = datetime.utcnow() -# while cloudInstance.state != 'terminated': -# log.debug( "Stopping instance %s state; current state: %s" % ( str( cloudInstance ).split(":")[1], cloudInstance.state ) ) -# time.sleep(3) -# cloudInstance.update() -# dbInstances.state = cloudInstance.state -# -# # Reset relevant UCI fields -# uci.state = 'available' -# uci.launch_time = None -# -# # Persist -# session = trans.sa_session -## session.save_or_update( stores ) -# session.save_or_update( dbInstances ) # TODO: Is this going to work w/ multiple instances stored in dbInstances variable? -# session.save_or_update( uci ) -# session.flush() -# trans.log_event( "User stopped cloud instance '%s'" % uci.name ) -# trans.set_message( "Galaxy instance '%s' stopped." % uci.name ) - - def update( self ): - """ - Run status update on all instances that are in 'running', 'pending', or 'shutting-down' state. - Run status update on all storage volumes whose status is 'in-use', 'creating', or 'None'. - Run status update on all snapshots whose status is 'pending' or 'delete' - Run status update on any zombie UCIs, i.e., UCI's that is in 'submitted' state for an - extended period of time. - - Reason behind this method is to sync state of local DB and real-world resources - """ - log.debug( "Running general status update for %s UCIs..." % self.type ) - # Update instances - instances = self.sa_session.query( model.CloudInstance ) \ - .filter( or_( model.CloudInstance.table.c.state==instance_states.RUNNING, - model.CloudInstance.table.c.state==instance_states.PENDING, - model.CloudInstance.table.c.state==instance_states.SHUTTING_DOWN ) ) \ - .all() - for inst in instances: - if self.type == inst.uci.credentials.provider.type: - log.debug( "[%s] Running general status update on instance '%s'" - % ( inst.uci.credentials.provider.type, inst.instance_id ) ) - self.update_instance( inst ) - # Update storage volume(s) associated with current instance - stores = self.sa_session.query( model.CloudStore ) \ - .filter_by( uci_id=inst.uci_id, deleted=False ) \ - .all() - for store in stores: - if self.type == store.uci.credentials.provider.type: # and store.volume_id != None: - log.debug( "[%s] Running general status update on store with local database ID: '%s'" - % ( store.uci.credentials.provider.type, store.id ) ) - self.update_store( store ) - - # Update storage volume(s) - stores = self.sa_session.query( model.CloudStore ) \ - .filter( or_( model.CloudStore.table.c.status==store_status.CREATING, -# model.CloudStore.table.c.status==store_status.IN_USE, -# model.CloudStore.table.c.status==store_status.WAITING, - model.CloudStore.table.c.status==None ) ) \ - .all() - for store in stores: - if self.type == store.uci.credentials.provider.type: # and store.volume_id != None: - log.debug( "[%s] Running general status update on store with local database ID: '%s'" - % ( store.uci.credentials.provider.type, store.id ) ) - self.update_store( store ) -# else: -# log.error( "[%s] There exists an entry for UCI (%s) storage volume without an ID. Storage volume might have been created with " -# "cloud provider though. Manual check is recommended." % ( store.uci.credentials.provider.type, store.uci.name ) ) -# store.uci.error = "There exists an entry in local database for a storage volume without an ID. Storage volume might have been created " \ -# "with cloud provider though. Manual check is recommended. After understanding what happened, local database entry for given " \ -# "storage volume should be updated." -# store.status = store_status.ERROR -# store.uci.state = uci_states.ERROR -# store.uci.flush() -# store.flush() - - # Update pending snapshots or delete ones marked for deletion - snapshots = self.sa_session.query( model.CloudSnapshot ) \ - .filter( or_( model.CloudSnapshot.table.c.status == snapshot_status.PENDING, model.CloudSnapshot.table.c.status == snapshot_status.DELETE ) ) \ - .all() - for snapshot in snapshots: - if self.type == snapshot.uci.credentials.provider.type and snapshot.status == snapshot_status.PENDING: - log.debug( "[%s] Running general status update on snapshot '%s'" % ( snapshot.uci.credentials.provider.type, snapshot.snapshot_id ) ) - self.update_snapshot( snapshot ) - elif self.type == snapshot.uci.credentials.provider.type and snapshot.status == snapshot_status.DELETE: - log.debug( "[%s] Initiating deletion of snapshot '%s'" % ( snapshot.uci.credentials.provider.type, snapshot.snapshot_id ) ) - self.delete_snapshot( snapshot ) - - # Attempt at updating any zombie UCIs (i.e., instances that have been in SUBMITTED state for longer than expected - see below for exact time) - zombies = self.sa_session.query( model.UCI ).filter_by( state=uci_states.SUBMITTED ).all() - for zombie in zombies: - z_instances = self.sa_session.query( model.CloudInstance ) \ - .filter_by( uci_id=zombie.id ) \ - .filter( or_( model.CloudInstance.table.c.state != instance_states.TERMINATED, - model.CloudInstance.table.c.state == None ) ) \ - .all() - for z_inst in z_instances: - if self.type == z_inst.uci.credentials.provider.type: -# log.debug( "z_inst.id: '%s', state: '%s'" % ( z_inst.id, z_inst.state ) ) - td = datetime.utcnow() - z_inst.update_time - if td.seconds > 180: # if instance has been in SUBMITTED state for more than 3 minutes - log.debug( "[%s] Running zombie repair update on instance with DB id '%s'" % ( z_inst.uci.credentials.provider.type, z_inst.id ) ) - self.process_zombie( z_inst ) - - def update_instance( self, inst ): - """ - Update information in local database for given instance as it is obtained from cloud provider. - Along with updating information about given instance, information about the UCI controlling - this instance is also updated. - """ - # Get credentials associated wit this instance - uci_id = inst.uci_id - uci = self.sa_session.query( model.UCI ).get( uci_id ) - self.sa_session.refresh( uci ) - conn = self.get_connection_from_uci( uci ) - - # Get reservations handle for given instance - try: - rl= conn.get_all_instances( [inst.instance_id] ) - except boto.exception.EC2ResponseError, e: - err = "Retrieving instance(s) from cloud failed for UCI '"+ uci.name +"' during general status update: " + str( e ) - log.error( err ) - uci.error = err - uci.state = uci_states.ERROR - self.sa_session.add( uci ) - self.sa_session.flush() - return None - - # Because references to reservations are deleted shortly after instances have been terminated, getting an empty list as a response to a query - # typically means the instance has successfully shut down but the check was not performed in short enough amount of time. Until an alternative solution - # is found, below code sets state of given UCI to 'error' to indicate to the user something out of ordinary happened. - if len( rl ) == 0: - err = "Instance ID '"+inst.instance_id+"' was not found by the cloud provider. Instance might have crashed or otherwise been terminated."+ \ - "Manual check is recommended." - log.error( err ) - inst.error = err - uci.error = err - inst.state = instance_states.TERMINATED - uci.state = uci_states.ERROR - uci.launch_time = None - self.sa_session.add( inst ) - self.sa_session.add( uci ) - self.sa_session.flush() - # Update instance status in local DB with info from cloud provider - for r in rl: - for i, cInst in enumerate( r.instances ): - try: - s = cInst.update() - log.debug( "Checking state of cloud instance '%s' associated with UCI '%s' " \ - "and reservation '%s'. State='%s'" % ( cInst, uci.name, r, s ) ) - if s != inst.state: - inst.state = s - self.sa_session.add( inst ) - self.sa_session.flush() - # After instance has shut down, ensure UCI is marked as 'available' - if s == instance_states.TERMINATED and uci.state != uci_states.ERROR: - uci.state = uci_states.AVAILABLE - uci.launch_time = None - self.sa_session.add( uci ) - self.sa_session.flush() - # Making sure state of UCI is updated. Once multiple instances become associated with single UCI, this will need to be changed. - if s != uci.state and s != instance_states.TERMINATED: - uci.state = s - self.sa_session.add( uci ) - self.sa_session.flush() - if cInst.public_dns_name != inst.public_dns: - inst.public_dns = cInst.public_dns_name - self.sa_session.add( inst ) - self.sa_session.flush() - if cInst.private_dns_name != inst.private_dns: - inst.private_dns = cInst.private_dns_name - self.sa_session.add( inst ) - self.sa_session.flush() - except boto.exception.EC2ResponseError, e: - err = "Updating instance status from cloud failed for UCI '"+ uci.name + \ - "' during general status update: " + str( e ) - log.error( err ) - uci.error = err - uci.state = uci_states.ERROR - self.sa_session.add( uci ) - self.sa_session.flush() - return None - - def update_store( self, store ): - """ - Update information in local database for given storage volume as it is obtained from cloud provider. - Along with updating information about given storage volume, information about the UCI controlling - this storage volume is also updated. - """ - # Get credentials associated wit this store - uci_id = store.uci_id - uci = self.sa_session.query( model.UCI ).get( uci_id ) - self.sa_session.refresh( uci ) - conn = self.get_connection_from_uci( uci ) - - # Get reservations handle for given store - try: - log.debug( "Retrieving reference to storage volume '%s' during update..." % store.volume_id ) - vl = conn.get_all_volumes( [store.volume_id] ) - except boto.exception.EC2ResponseError, e: - err = "Retrieving volume(s) from cloud failed for UCI '"+ uci.name + "' during general status update: " + str( e ) - log.error( err ) - uci.error = err - uci.state = uci_states.ERROR - self.sa_session.add( uci ) - self.sa_session.flush() - return None - - # Update store status in local DB with info from cloud provider - if len(vl) > 0: - try: - log.debug( "General status update for storage volume '%s'; current status: '%s'" % (store.volume_id, vl[0].status ) ) - if store.status != vl[0].status: - # In case something failed during creation of UCI but actual storage volume was created and yet - # UCI state remained as 'new', try to remedy this by updating UCI state here - if ( store.status == None ) and ( store.volume_id != None ): - uci.state = vl[0].status - self.sa_session.add( uci ) - self.sa_session.flush() - # If UCI was marked in state 'CREATING', update its status to reflect new status - elif ( uci.state == uci_states.CREATING ): - uci.state = vl[0].status - self.sa_session.add( uci ) - self.sa_session.flush() - - store.status = vl[0].status - self.sa_session.add( store ) - self.sa_session.flush() - # Boto does not seem to be reporting these values although fields exist so comment them out... -# log.debug( "vl[0].instance_id: '%s'" % vl[0].instance_id ) -# if store.inst != None: -# if store.inst.instance_id != vl[0].instance_id: -# store.inst.instance_id = vl[0].instance_id -# self.sa_session.add( store ) -# self.sa_session.flush() -# log.debug( "vl[0].attach_time: '%s'" % vl[0].attach_time ) -# if store.attach_time != vl[0].attach_time: -# store.attach_time = vl[0].attach_time -# self.sa_session.add( store ) -# self.sa_session.flush() -## log.debug( "vl[0].device: '%s'" % vl[0].device ) -# if store.device != vl[0].device: -# store.device = vl[0].device -# self.sa_session.add( store ) -# self.sa_session.flush() - except boto.exception.EC2ResponseError, e: - err = "Updating status of volume(s) from cloud failed for UCI '"+ uci.name + \ - "' during general status update: " + str( e ) - log.error( err ) - uci.error = err - uci.state = uci_states.ERROR - self.sa_session.add( uci ) - self.sa_session.flush() - return None - else: - err = "No storage volumes returned by cloud provider on general update for volume with id: " + store.volume_id - log.error( "%s for UCI '%s'" % ( err, uci.name ) ) - store.status = store_status.ERROR - store.error = err - uci.error = err - uci.state = uci_states.ERROR - self.sa_session.add( uci ) - self.sa_session.add( store ) - self.sa_session.flush() - - def update_snapshot( self, snapshot ): - """ - Update information in local database for given snapshot as it is obtained from cloud provider. - Along with updating information about given snapshot, information about the UCI controlling - this snapshot is also updated. - """ - # Get credentials associated wit this store - uci_id = snapshot.uci_id - uci = self.sa_session.query( model.UCI ).get( uci_id ) - self.sa_session.refresh( uci ) - conn = self.get_connection_from_uci( uci ) - - try: - log.debug( "Updating status of snapshot '%s'" % snapshot.snapshot_id ) - snap = conn.get_all_snapshots( [snapshot.snapshot_id] ) - if len( snap ) > 0: - log.debug( "Snapshot '%s' status: %s" % ( snapshot.snapshot_id, snap[0].status ) ) - snapshot.status = snap[0].status - self.sa_session.add( snapshot ) - self.sa_session.flush() - else: - err = "No snapshots returned by EC2 on general update" - log.error( "%s for UCI '%s'" % ( err, uci.name ) ) - snapshot.status = snapshot_status.ERROR - snapshot.error = err - uci.error = err - uci.state = uci_states.ERROR - self.sa_session.add( uci ) - self.sa_session.add( snapshot ) - self.sa_session.flush() - except boto.exception.EC2ResponseError, e: - err = "EC2 response error while updating snapshot status: " + str( e ) - log.error( err ) - snapshot.status = snapshot_status.ERROR - snapshot.error = err - uci.error = err - uci.state = uci_states.ERROR - self.sa_session.add( uci ) - self.sa_session.add( snapshot ) - self.sa_session.flush() - except Exception, ex: - err = "Error while updating snapshot status: " + str( ex ) - log.error( err ) - snapshot.status = snapshot_status.ERROR - snapshot.error = err - uci.error = err - uci.state = uci_states.ERROR - self.sa_session.add( uci ) - self.sa_session.add( snapshot ) - self.sa_session.flush() - - def delete_snapshot( self, snapshot ): - """ - Initiate deletion of given snapshot from cloud provider. - """ - if snapshot.status == snapshot_status.DELETE: - # Get credentials associated wit this store - uci_id = snapshot.uci_id - uci = self.sa_session.query( model.UCI ).get( uci_id ) - self.sa_session.refresh( uci ) - conn = self.get_connection_from_uci( uci ) - - try: - log.debug( "Deleting snapshot '%s'" % snapshot.snapshot_id ) - snap = conn.delete_snapshot( snapshot.snapshot_id ) - if snap == True: - snapshot.deleted = True - snapshot.status = snapshot_status.DELETED - self.sa_session.add( snapshot ) - self.sa_session.flush() - return snap - except boto.exception.EC2ResponseError, e: - err = "EC2 response error while deleting snapshot: " + str( e ) - log.error( err ) - snapshot.status = snapshot_status.ERROR - snapshot.error = err - uci.error = err - uci.state = uci_states.ERROR - self.sa_session.add( uci ) - self.sa_session.add( snapshot ) - self.sa_session.flush() - except Exception, ex: - err = "Error while deleting snapshot: " + str( ex ) - log.error( err ) - snapshot.status = snapshot_status.ERROR - snapshot.error = err - uci.error = err - uci.state = uci_states.ERROR - self.sa_session.add( uci ) - self.sa_session.add( snapshot ) - self.sa_session.flush() - else: - err = "Cannot delete snapshot '"+snapshot.snapshot_id+"' because its status is '"+snapshot.status+"'. Only snapshots with '" + \ - snapshot_status.COMPLETED+"' status can be deleted." - log.error( err ) - snapshot.error = err - self.sa_session.add( snapshot ) - self.sa_session.flush() - - def process_zombie( self, inst ): - """ - Attempt at discovering if starting a cloud instance was successful but local database was not updated - accordingly or if something else failed and instance was never started. Currently, no automatic - repairs are being attempted; instead, appropriate error messages are set. - """ - uci_id = inst.uci_id - uci = self.sa_session.query( model.UCI ).get( uci_id ) - self.sa_session.refresh( uci ) - - # Check if any instance-specific information was written to local DB; if 'yes', set instance and UCI's error message - # suggesting manual check. - if inst.launch_time != None or inst.reservation_id != None or inst.instance_id != None: - # Try to recover state - this is best-case effort, so if something does not work immediately, not - # recovery steps are attempted. Recovery is based on hope that instance_id is available in local DB; if not, - # report as error. - # Fields attempting to be recovered are: reservation_id, instance status, and launch_time - if inst.instance_id != None: - conn = self.get_connection_from_uci( uci ) - rl = conn.get_all_instances( [inst.instance_id] ) # reservation list - # Update local DB with relevant data from instance - if inst.reservation_id == None: - try: - inst.reservation_id = str(rl[0]).split(":")[1] - except: # something failed, so skip - pass - - try: - state = rl[0].instances[0].update() - inst.state = state - uci.state = state - self.sa_session.add( inst ) - self.sa_session.add( uci ) - self.sa_session.flush() - except: # something failed, so skip - pass - - if inst.launch_time == None: - try: - launch_time = self.format_time( rl[0].instances[0].launch_time ) - inst.launch_time = launch_time - self.sa_session.add( inst ) - self.sa_session.flush() - if inst.uci.launch_time == None: - uci.launch_time = launch_time - self.sa_session.add( uci ) - self.sa_session.flush() - except: # something failed, so skip - pass - else: - err = "Starting a machine instance (DB id: '"+str(inst.id)+"') associated with this UCI '" + str(inst.uci.name) + \ - "' seems to have failed. Because it appears that cloud instance might have gotten started, manual check is recommended." - inst.error = err - inst.state = instance_states.ERROR - inst.uci.error = err - inst.uci.state = uci_states.ERROR - log.error( err ) - self.sa_session.add( inst ) - self.sa_session.add( uci ) - self.sa_session.flush() - - else: #Instance most likely never got processed, so set error message suggesting user to try starting instance again. - err = "Starting a machine instance (DB id: '"+str(inst.id)+"') associated with this UCI '" + str(inst.uci.name) + \ - "' seems to have failed. Because it appears that cloud instance never got started, it should be safe to reset state and try " \ - "starting the instance again." - inst.error = err - inst.state = instance_states.ERROR - uci.error = err - uci.state = uci_states.ERROR - log.error( err ) - self.sa_session.add( inst ) - self.sa_session.add( uci ) - self.sa_session.flush() -# uw = UCIwrapper( inst.uci ) -# log.debug( "Try automatically re-submitting UCI '%s'." % uw.get_name() ) - - def get_connection_from_uci( self, uci ): - """ - Establish and return connection to cloud provider. Information needed to do so is obtained - directly from uci database object. - """ - log.debug( 'Establishing %s cloud connection' % self.type ) - a_key = uci.credentials.access_key - s_key = uci.credentials.secret_key - # Get connection - try: - region = RegionInfo( None, uci.credentials.provider.region_name, uci.credentials.provider.region_endpoint ) - conn = EC2Connection( aws_access_key_id=a_key, - aws_secret_access_key=s_key, - is_secure=uci.credentials.provider.is_secure, - region=region, - path=uci.credentials.provider.path ) - except boto.exception.EC2ResponseError, e: - err = "Establishing connection with cloud failed: " + str( e ) - log.error( err ) - uci.error = err - uci.state = uci_states.ERROR - self.sa_session.add( uci ) - self.sa_session.flush() - return None - - return conn - -# def updateUCI( self, uci ): -# """ -# Runs a global status update on all storage volumes and all instances that are -# associated with specified UCI -# """ -# conn = self.get_connection( uci ) -# -# # Update status of storage volumes -# vl = model.CloudStore.filter( model.CloudInstance.table.c.uci_id == uci.id ).all() -# vols = [] -# for v in vl: -# vols.append( v.volume_id ) -# try: -# volumes = conn.get_all_volumes( vols ) -# for i, v in enumerate( volumes ): -# uci.store[i].i_id = v.instance_id -# uci.store[i].status = v.status -# uci.store[i].device = v.device -# uci.store[i].flush() -# except: -# log.debug( "Error updating status of volume(s) associated with UCI '%s'. Status was not updated." % uci.name ) -# pass -# -# # Update status of instances -# il = model.CloudInstance.filter_by( uci_id=uci.id ).filter( model.CloudInstance.table.c.state != 'terminated' ).all() -# instanceList = [] -# for i in il: -# instanceList.append( i.instance_id ) -# log.debug( 'instanceList: %s' % instanceList ) -# try: -# reservations = conn.get_all_instances( instanceList ) -# for i, r in enumerate( reservations ): -# uci.instance[i].state = r.instances[0].update() -# log.debug('updating instance %s; status: %s' % ( uci.instance[i].instance_id, uci.instance[i].state ) ) -# uci.state = uci.instance[i].state -# uci.instance[i].public_dns = r.instances[0].dns_name -# uci.instance[i].private_dns = r.instances[0].private_dns_name -# uci.instance[i].flush() -# uci.flush() -# except: -# log.debug( "Error updating status of instances associated with UCI '%s'. Instance status was not updated." % uci.name ) -# pass - - # --------- Helper methods ------------ - - def format_time( self, time ): - dict = {'T':' ', 'Z':''} - for i, j in dict.iteritems(): - time = time.replace(i, j) - return time - --- a/templates/cloud/view_instance.mako +++ /dev/null @@ -1,142 +0,0 @@ -<%inherit file="/base.mako"/> -<%def name="title()">Live instance details</%def> - -<% - # Because of the one-to-many relationship between liveInstance (i.e., UCI) and actual instances, need to know - # which one is currently active. Because only one instance of UCI can be alive at any point in time, simply - # select the most recent one. - # TODO: Once individual UCI's will be able to start more than one instance, this will need to be fixed - #i_id = len(liveInstance.instance) - 1 -%> - -<h2>Live instance details</h2> - -%if liveInstance: - <ul class="manage-table-actions"> - <li> - <a class="action-button" href="${h.url_for( action='list' )}"> - <img src="${h.url_for('/static/images/silk/resultset_previous.png')}" /> - <span>Return to cloud management console</span> - </a> - </li> - </ul> - - <table class="mange-table colored" border="0" cellspacing="0" cellpadding="0" width="100%"> - <tr> - <td> Instance name: </td> - <td> - ${liveInstance.uci.name} - <a id="li-popup" class="popup-arrow" style="display: none;">▼</a> - </td> - <td> - <div popupmenu="li-popup"> - <a class="action-button" href="${h.url_for( action='rename_uci', id=trans.security.encode_id(liveInstance.uci.id) )}">Rename</a> - <a class="action-button" confirm="Are you sure you want to stop instance '${liveInstance.uci.name}'?" href="${h.url_for( action='stop', id=trans.security.encode_id(liveInstance.uci.id) )}">Stop</a> - </div> - </td> - </tr> - <tr> - <td> Date created: </td> - <td> ${str(liveInstance.uci.create_time)[:16]} - <% - context.write( ' UTC (' ) - context.write( str(h.date.distance_of_time_in_words (liveInstance.uci.create_time, h.date.datetime.utcnow() ) ) ) - %> ago) - </td> - </tr> - <tr> - <td> Alive since: </td> - <td> ${str(liveInstance.launch_time)[:16]} - <% - context.write( ' UTC (' ) - context.write( str(h.date.distance_of_time_in_words (liveInstance.launch_time, h.date.datetime.utcnow() ) ) ) - %> ago) - </td> - </tr> - %if liveInstance.instance_id != None: - <tr> - <td> Instance ID: </td> - <td> ${liveInstance.instance_id} </td> - </tr> - %endif - %if liveInstance.reservation_id != None: - <tr> - <td> Reservation ID: </td> - <td> ${liveInstance.reservation_id} </td> - </tr> - %endif - <tr> - <td> AMI: </td> - <td> ${liveInstance.image.image_id} </td> - </tr> - <tr> - <td> State:</td> - <td> ${liveInstance.state} </td> - </tr> - <tr> - <td> Type:</td> - <td> ${liveInstance.type} </td> - </tr> - <tr> - <td> Storage size:</td> - <td> ${liveInstance.uci.total_size} </td> - </tr> - %if liveInstance.public_dns != None and liveInstance.public_dns != '': - <tr> - <td> Public DNS:</td> - <% - lnk="http://"+str(liveInstance.public_dns) - %> - <td><a href="${lnk}" target="_blank">${liveInstance.public_dns}</a></td> - </tr> - %endif - %if liveInstance.private_dns != None and liveInstance.private_dns != '': - <tr> - <td> Private DNS:</td> - <td> ${liveInstance.private_dns}</td> - </tr> - %endif - %if liveInstance.security_group != None: - <tr> - <td> Security group zone:</td> - <td> ${liveInstance.security_group} </td> - </tr> - %endif - %if liveInstance.availability_zone != None: - <tr> - <td> Availabilty zone:</td> - <td> ${liveInstance.availability_zone} </td> - </tr> - %endif - %if liveInstance.uci.key_pair_name != None: - <tr> - <td> Keypair file name:</td> - <td> ${liveInstance.uci.key_pair_name} </td> - </tr> - %endif - %if liveInstance.uci.key_pair_material != None: - <tr> - <td> Keypair material:</td> - <td> - <div id="short"> - <a onclick="document.getElementById('full').style.display = 'block'; - document.getElementById('short').style.display = 'none'; return 0" - href="javascript:void(0)"> - + Show - </a> - </div> - <div id="full" style="DISPLAY: none"> - <a onclick="document.getElementById('short').style.display = 'block'; - document.getElementById('full').style.display = 'none'; return 0;" - href="javascript:void(0)"> - - Hide</a> - ${liveInstance.uci.key_pair_material}<br/> - </div> - </td> - </tr> - %endif - - </table> -%else: - There is no live instance under that name. -%endif --- a/templates/webapps/galaxy/admin/index.mako +++ b/templates/webapps/galaxy/admin/index.mako @@ -118,14 +118,6 @@ <div class="toolTitle"><a href="${h.url_for( controller='requests_admin', action='list')}" target="galaxy_main">Manage requests</a></div></div></div> - <div class="toolSectionTitle"> - <span>Cloud</span> - </div> - <div class="toolSectionBody"> - <div class="toolSectionBg"> - <div class="toolTitle"><a href="${h.url_for( controller='cloud', action='list_machine_images' )}" target="galaxy_main">Manage machine images</a></div> - </div> - </div></div></div></div> --- a/lib/galaxy/web/controllers/cloud.py +++ /dev/null @@ -1,1276 +0,0 @@ -from galaxy.web.base.controller import * - -import pkg_resources -pkg_resources.require( "simplejson" ) -import simplejson -import urllib2 - -from galaxy.tools.parameters import * -from galaxy.tools import DefaultToolState -from galaxy.tools.parameters.grouping import Repeat, Conditional -from galaxy.datatypes.data import Data -from galaxy.util.odict import odict -from galaxy.util.bunch import Bunch -from galaxy.util.topsort import topsort, topsort_levels, CycleError -from galaxy.model.mapping import desc -from galaxy.model.orm import * -from datetime import datetime, timedelta - -pkg_resources.require( "WebHelpers" ) -from webhelpers import * - -# Required for Cloud tab -import galaxy.eggs -galaxy.eggs.require("boto") -from boto.ec2.connection import EC2Connection -from boto.ec2.regioninfo import RegionInfo -from galaxy.cloud import CloudManager -import boto.exception -import boto - -import logging -log = logging.getLogger( __name__ ) - -uci_states = Bunch( - NEW_UCI = "newUCI", - NEW = "new", - CREATING = "creating", - DELETING_UCI = "deletingUCI", - DELETING = "deleting", - SUBMITTED_UCI = "submittedUCI", - SUBMITTED = "submitted", - SHUTTING_DOWN_UCI = "shutting-downUCI", - SHUTTING_DOWN = "shutting-down", - ADD_STORAGE_UCI = "add-storageUCI", - ADD_STORAGE = "add-storage", - AVAILABLE = "available", - RUNNING = "running", - PENDING = "pending", - ERROR = "error", - DELETED = "deleted", - SNAPSHOT_UCI = "snapshotUCI", - SNAPSHOT = "snapshot" -) - -instance_states = Bunch( - TERMINATED = "terminated", - SUBMITTED = "submitted", - RUNNING = "running", - ADDING = "adding-storage", - PENDING = "pending", - SHUTTING_DOWN = "shutting-down", - ERROR = "error" -) - -store_status = Bunch( - WAITING = "waiting", - IN_USE = "in-use", - ADDING = "adding", - CREATING = "creating", - DELETED = 'deleted', - ERROR = "error" -) - -snapshot_status = Bunch( - SUBMITTED = 'submitted', - PENDING = 'pending', - COMPLETED = 'completed', - DELETE = 'delete', - DELETED= 'deleted', - ERROR = "error" -) - -class CloudController( BaseController ): - - @web.expose - def index( self, trans ): - return trans.fill_template( "cloud/index.mako" ) - - @web.expose - @web.require_login( "use Galaxy cloud" ) - def list( self, trans ): - """ - Render cloud main page (management of cloud resources) - """ - user = trans.get_user() - - cloudCredentials = trans.sa_session.query( model.CloudUserCredentials ) \ - .filter_by( user=user ) \ - .filter( model.CloudUserCredentials.table.c.deleted != True ) \ - .order_by( model.CloudUserCredentials.table.c.name ) \ - .all() - - cloudProviders = trans.sa_session.query( model.CloudProvider ) \ - .filter_by( user=user ) \ - .filter( model.CloudProvider.table.c.deleted != True ) \ - .order_by( model.CloudProvider.table.c.name ) \ - .all() - - liveInstances = trans.sa_session.query( model.UCI ) \ - .filter_by( user=user ) \ - .filter( or_( model.UCI.table.c.state==uci_states.RUNNING, - model.UCI.table.c.state==uci_states.PENDING, - model.UCI.table.c.state==uci_states.SUBMITTED, - model.UCI.table.c.state==uci_states.SUBMITTED_UCI, - model.UCI.table.c.state==uci_states.SHUTTING_DOWN, - model.UCI.table.c.state==uci_states.SHUTTING_DOWN_UCI, - model.UCI.table.c.state==uci_states.ADD_STORAGE, - model.UCI.table.c.state==uci_states.ADD_STORAGE_UCI ) ) \ - .order_by( desc( model.UCI.table.c.update_time ) ) \ - .all() - - prevInstances = trans.sa_session.query( model.UCI ) \ - .filter_by( user=user, deleted=False ) \ - .filter( or_( model.UCI.table.c.state==uci_states.AVAILABLE, - model.UCI.table.c.state==uci_states.NEW, - model.UCI.table.c.state==uci_states.NEW_UCI, - model.UCI.table.c.state==uci_states.CREATING, - model.UCI.table.c.state==uci_states.ERROR, - model.UCI.table.c.state==uci_states.DELETED, - model.UCI.table.c.state==uci_states.DELETING, - model.UCI.table.c.state==uci_states.DELETING_UCI, - model.UCI.table.c.state==uci_states.SNAPSHOT, - model.UCI.table.c.state==uci_states.SNAPSHOT_UCI ) ) \ - .order_by( desc( model.UCI.table.c.update_time ) ) \ - .all() - - # Check after update there are instances in pending state; if so, display message - pendingInstances = trans.sa_session.query( model.UCI ) \ - .filter_by( user=user ) \ - .filter( or_( model.UCI.table.c.state==uci_states.PENDING, - model.UCI.table.c.state==uci_states.SUBMITTED, - model.UCI.table.c.state==uci_states.SUBMITTED_UCI ) ) \ - .all() - if pendingInstances: - trans.set_message( "Galaxy instance started. NOTE: Please wait about 5 minutes for the instance to " - "start up. A button to connect to the instance will appear alongside " - "instance description once cloud instance of Galaxy is ready." ) - -# log.debug( "provider.is_secure: '%s'" % trans.sa_session.query( model.CloudProvider).filter_by(id=1).first().is_secure ) -# trans.sa_session.query( model.CloudProvider).filter_by(id=1).first().is_secure=False -# trans.sa_session.flush() -# log.debug( "provider.is_secure: '%s'" % trans.sa_session.query( model.CloudProvider).filter_by(id=1).first().is_secure ) - -# log.debug( "image: '%s'" % model.CloudImage.is_secure ) - - return trans.fill_template( "cloud/configure_cloud.mako", - cloudCredentials = cloudCredentials, - liveInstances = liveInstances, - prevInstances = prevInstances, - cloudProviders = cloudProviders ) - - # ----- UCI methods ----- - - @web.expose - @web.require_login( "use Galaxy cloud" ) - def configure_new_uci( self, trans, instanceName='', credName='', volSize='', zone='' ): - """ - Configure and add new cloud instance to user's instance pool - """ - inst_error = vol_error = cred_error = None - error = {} - user = trans.get_user() - - if instanceName: - # Check if volume size is entered as an integer - try: - volSize = int( volSize ) - except ValueError: - error['vol_error'] = "Volume size must be integer value between 1 and 1000." - - # Create new user configured instance - try: - if trans.sa_session.query( model.UCI ) \ - .filter_by (user=user, deleted=False, name=instanceName ) \ - .first(): - error['inst_error'] = "An instance with that name already exist." - elif instanceName=='' or len( instanceName ) > 255: - error['inst_error'] = "Instance name must be between 1 and 255 characters long." - elif credName=='': - error['cred_error'] = "You must select credentials." - elif volSize == '': - error['vol_error'] = "You must specify volume size as an integer value between 1 and 1000." - elif ( int( volSize ) < 1 ) or ( int( volSize ) > 1000 ): - error['vol_error'] = "Volume size must be integer value between 1 and 1000." - elif zone=='': - error['zone_error'] = "You must select a zone where this UCI will be registered." - else: - # Capture user configured instance information - uci = model.UCI() - uci.name = instanceName - creds = trans.sa_session.query( model.CloudUserCredentials ) \ - .filter( model.CloudUserCredentials.table.c.name==credName ).first() - uci.credentials = creds - uci.user= user - uci.total_size = volSize # This is OK now because new instance is being created and only one storage volume can be created at UCI creation time - uci.state = uci_states.NEW_UCI - - storage = model.CloudStore() - storage.user = user - storage.uci = uci - storage.size = volSize - # If '(any)' zone was selected, just choose the first one that's available - if zone == "(any)": - zones = None - conn = get_connection( trans, creds ) - if conn != None: - try: - zones = conn.get_all_zones() - if len( zones ) > 0: - zone = str( zones[0] ).split(':')[1] - except boto.exception.EC2ResponseError, e: - log.error( "Retrieving zones for credentials '%s' failed: %s" % ( storedCred.name, e ) ) - providersToZones[storedCred.name] = [ "Retrieving zones failed: " + str( e ) ] - storage.availability_zone = zone - storage.status = store_status.ADDING - # Persist - session = trans.sa_session - session.add( uci ) - session.add( storage ) - session.flush() - # Log and display the management page - trans.log_event( "User configured new cloud instance: '%s'" % instanceName ) - trans.set_message( "New Galaxy instance '%s' configured. Once instance status shows 'available' you will be able to start the instance." % instanceName ) - return self.list( trans ) - except AttributeError, ae: - inst_error = "No registered cloud images. You must contact administrator to add some before proceeding." - log.debug("AttributeError when registering new UCI '%s': %s " % ( instanceName, str( ae ) ) ) - else: - storedCreds = trans.sa_session.query( model.CloudUserCredentials ).filter_by( user=user, deleted=False ).all() - if len( storedCreds ) == 0: - return trans.show_error_message( "You must register credentials before configuring a Galaxy cloud instance." ) - # Create dict mapping of cloud-providers-to-zones available by those providers - providersToZones = {} - for storedCred in storedCreds: - zones = None - conn = get_connection( trans, storedCred ) - if conn != None: - avail_zones = [] - try: - zones = conn.get_all_zones() - if len( zones ) > 0: - avail_zones.append( "(any)" ) - for z in zones: - z = str( z ).split(':')[1] - avail_zones.append( z ) - providersToZones[storedCred.name] = avail_zones - except boto.exception.EC2ResponseError, e: - log.error( "Retrieving zones for credentials '%s' failed: %s" % ( storedCred.name, e ) ) - providersToZones[storedCred.name] = [ "Retrieving zones failed: " + str( e ) ] - else: - providersToZones[storedCred.name] = ['Connection with cloud provider could not be established.'] - - # Hard-coded solution - # if storedCred.provider.storedCred.provider.region_name == 'us-east-1': - # ec2_zones = ['us-east-1a', 'us-east-1b', 'us-east-1c', 'us-east-1d'] - # providersToZones[storedCred.name] = ec2_zones - # elif storedCred.provider.region_name == 'eu-west-1': - # ec2_zones = ['eu-west-1a', 'eu-west-1b'] - # providersToZones[storedCred.name] = ec2_zones - # elif storedCred.provider.type == 'eucalyptus': - # providersToZones[storedCred.name] = ['epc'] - # else: - # providersToZones[storedCred.name] = ['Unknown provider zone'] - - return trans.fill_template( "cloud/configure_uci.mako", - instanceName = instanceName, - credName = storedCreds, - volSize = volSize, - zone = zone, - error = error, - providersToZones = providersToZones ) - - @web.expose - @web.require_login( "start Galaxy cloud instance" ) - def start( self, trans, id, type='m1.small' ): - """ - Start a new cloud resource instance - """ - user = trans.get_user() - uci = get_uci( trans, id ) - stores = get_stores( trans, uci ) - # Ensure instance is available and then store relevant data - # into DB to initiate instance startup by cloud manager - if ( len(stores) is not 0 ) and ( uci.state == uci_states.AVAILABLE ): - instance = model.CloudInstance() - instance.user = user - instance.uci = uci - instance.state = instance_states.SUBMITTED - instance.availability_zone = stores[0].availability_zone # Bc. all EBS volumes need to be in the same avail. zone, just check 1st - instance.type = type - uci.state = uci_states.SUBMITTED_UCI - # Persist - session = trans.sa_session - session.add( instance ) - session.add( uci ) - session.flush() - # Log - trans.log_event ("User initiated starting of UCI '%s'." % uci.name ) - trans.set_message( "Galaxy instance started. NOTE: Please wait about 5 minutes for the instance to " - "start up. A button to connect to the instance will appear alongside " - "instance description once cloud instance of Galaxy is ready." ) - return self.list( trans ) - - if len(stores) == 0: - error( "This instance does not have any storage volumes associated it and thus cannot be started." ) - else: - error( "Cannot start instance that is in state '%s'." % uci.state ) - return self.list( trans ) - - @web.expose - @web.require_login( "stop Galaxy cloud instance" ) - def stop( self, trans, id ): - """ - Stop a cloud UCI instance. - """ - uci = get_uci( trans, id ) - if ( uci.state != uci_states.DELETING ) and \ - ( uci.state != uci_states.DELETING_UCI ) and \ - ( uci.state != uci_states.ERROR ) and \ - ( uci.state != uci_states.SHUTTING_DOWN_UCI ) and \ - ( uci.state != uci_states.SHUTTING_DOWN ) and \ - ( uci.state != uci_states.ADD_STORAGE_UCI ) and \ - ( uci.state != uci_states.ADD_STORAGE ) and \ - ( uci.state != uci_states.AVAILABLE ): - uci.state = uci_states.SHUTTING_DOWN_UCI - session = trans.sa_session - session.add( uci ) - session.flush() - trans.log_event( "User stopped cloud instance '%s' (id: %s)" % ( uci.name, uci.id ) ) - trans.set_message( "Stopping of Galaxy instance '%s' initiated." % uci.name ) - - return self.list( trans ) - - trans.show_error_message( "Cannot stop instance that is in state '%s'." % uci.state ) - return self.list( trans ) - - @web.expose - @web.require_login( "use Galaxy cloud" ) - def set_uci_state( self, trans, id, state='available', clear_error=True ): - """ - Sets state of UCI to given state, optionally resets error field, and resets UCI's launch time field to 'None'. - """ - uci = get_uci( trans, id ) - uci.state = state - if clear_error: - uci.error = None - uci.launch_time = None - trans.sa_session.flush() - trans.set_message( "Instance '%s' state reset." % uci.name ) - return self.list( trans ) - - @web.expose - @web.require_login( "view instance details" ) - def view_uci_details( self, trans, id=None ): - """ - View details about running instance - """ - uci = get_uci( trans, id ) - instances = get_instances( trans, uci ) # TODO: Handle list (will probably need to be done in mako template) - - return trans.fill_template( "cloud/view_instance.mako", - liveInstance = instances ) - - @web.expose - @web.require_login( "use Galaxy cloud" ) - def rename_uci( self, trans, id, new_name=None ): - instance = get_uci( trans, id ) - if new_name is not None: - if len(new_name) > 255: - error( "Instance name must be less than 255 characters long." ) - user = trans.get_user() - name_exists = trans.sa_session.query( model.UCI ) \ - .filter_by( user=user, name=new_name ) \ - .first() - if name_exists: - error( "Specified name ('%s') is already used by an existing instance. Please choose an alternative name." % new_name ) - - # Update name in local DB - instance.name = new_name - trans.sa_session.flush() - trans.set_message( "Instance renamed to '%s'." % new_name ) - return self.list( trans ) - else: - return trans.show_form( - web.FormBuilder( url_for( id=trans.security.encode_id(instance.id) ), "Rename instance", submit_text="Rename" ) - .add_text( "new_name", "Instance name", value=instance.name ) ) - - @web.expose - @web.require_login( "use Galaxy cloud" ) - def uci_usage_report( self, trans, id ): - user = trans.get_user() - id = trans.security.decode_id( id ) - - prevInstances = trans.sa_session.query( model.CloudInstance ) \ - .filter_by( user=user, state=instance_states.TERMINATED, uci_id=id ) \ - .order_by( desc( model.CloudInstance.table.c.update_time ) ) \ - .all() - - return trans.fill_template( "cloud/view_usage.mako", prevInstances = prevInstances ) - - @web.expose - @web.require_login( "delete user configured Galaxy cloud instance" ) - def delete_uci( self, trans, id ): - """ - Deletes User Configured Instance (UCI) from the cloud and local database. NOTE that this implies deletion of - any and all storage associated with this UCI! - """ - uci = get_uci( trans, id ) - - if ( uci.state != uci_states.DELETING_UCI ) and ( uci.state != uci_states.DELETING ) and ( uci.state != uci_states.ERROR ): - name = uci.name - uci.state = uci_states.DELETING_UCI - session = trans.sa_session - session.add( uci ) - session.flush() - trans.log_event( "User marked cloud instance '%s' for deletion." % name ) - trans.set_message( "Galaxy instance '%s' marked for deletion." % name ) - return self.list( trans ) - - if uci.state != uci_states.ERROR: - trans.set_message( "Cannot delete instance in state ERROR." ) - else: - trans.set_message( "Instance '%s' is already marked for deletion." % uci.name ) - return self.list( trans ) - - # ----- Snapshot methods ----- - - @web.expose - @web.require_login( "use Galaxy cloud" ) - def create_snapshot( self, trans, id ): - user = trans.get_user() - id = trans.security.decode_id( id ) - uci = get_uci( trans, id ) - - stores = trans.sa_session.query( model.CloudStore ) \ - .filter_by( user=user, deleted=False, uci_id=id ) \ - .all() - - if ( len( stores ) > 0 ) and ( uci.state == uci_states.AVAILABLE ): - for store in stores: - snapshot = model.CloudSnapshot() - snapshot.user = user - snapshot.uci = uci - snapshot.store = store - snapshot.status = snapshot_status.SUBMITTED - uci.state = uci_states.SNAPSHOT_UCI - # Persist - session = trans.sa_session - session.add( snapshot ) - session.add( uci ) - session.flush() - elif len( stores ) == 0: - error( "No storage volumes found that are associated with this instance." ) - else: - error( "Snapshot can be created only for an instance that is in 'available' state." ) - - # Log and display the management page - trans.log_event( "User initiated creation of new snapshot." ) - trans.set_message( "Creation of new snapshot initiated. " ) - return self.list( trans ) - - @web.expose - @web.require_login( "use Galaxy cloud" ) - def view_snapshots( self, trans, id=None ): - """ - View details about any snapshots associated with given UCI - """ - user = trans.get_user() - id = trans.security.decode_id( id ) - - snaps = trans.sa_session.query( model.CloudSnapshot ) \ - .filter_by( user=user, uci_id=id, deleted=False ) \ - .order_by( desc( model.CloudSnapshot.table.c.update_time ) ) \ - .all() - - return trans.fill_template( "cloud/view_snapshots.mako", - snaps = snaps ) - - @web.expose - @web.require_login( "use Galaxy cloud" ) - def delete_snapshot( self, trans, uci_id=None, snap_id=None ): - """ - Initiates deletion of a snapshot - """ - user = trans.get_user() - snap_id = trans.security.decode_id( snap_id ) - # Set snapshot as 'ready for deletion' to be picked up by general updater - snap = trans.sa_session.query( model.CloudSnapshot ).get( snap_id ) - - if snap.status == snapshot_status.COMPLETED: - snap.status = snapshot_status.DELETE - trans.sa_session.add( snap ) - trans.sa_session.flush() - trans.set_message( "Snapshot '%s' is marked for deletion. Once the deletion is complete, it will no longer be visible in this list. " - "Please note that this process may take up to a minute." % snap.snapshot_id ) - else: - error( "Only snapshots in state 'completed' can be deleted. See the cloud provider directly " - "if you believe the snapshot is available and can be deleted." ) - - # Display new list of snapshots - uci_id = trans.security.decode_id( uci_id ) - snaps = trans.sa_session.query( model.CloudSnapshot ) \ - .filter_by( user=user, uci_id=uci_id, deleted=False ) \ - .order_by( desc( model.CloudSnapshot.table.c.update_time ) ) \ - .all() - - return trans.fill_template( "cloud/view_snapshots.mako", - snaps = snaps ) - - # ----- Storage methods ----- - - @web.expose - @web.require_login( "add instance storage" ) - def add_storage( self, trans, id, vol_size=None ): - error = None - uci = get_uci( trans, id ) - stores = get_stores_in_status( trans, uci, store_status.IN_USE ) - - # Start adding of storage making sure given UCI is running and that at least one - # storage volume is attached to it (this is needed to by cloud controller to know - # as which device to attach the new storage volume) - if uci.state == uci_states.RUNNING and len( stores ) > 0: - if vol_size is not None: - try: - vol_size = int( vol_size ) - except ValueError: - error = "Volume size must be integer value between 1 and 1000." - - if not error: - user = trans.get_user() - - storage = model.CloudStore() - storage.user = user - storage.uci = uci - storage.size = vol_size - storage.status = store_status.ADDING - - # Set state of instance - NOTE that this code will only work (with code in cloud controller) - # for scenario where a UCI is associated with *1* compute instance!!! - instances = get_instances( trans, uci ) - instances.state = instance_states.ADDING - - uci.state = uci_states.ADD_STORAGE_UCI - # Persist - session = trans.sa_session - session.add( instances ) - session.add( storage ) - session.add( uci ) - session.flush() - # Log and display the management page - trans.log_event( "User added storage volume to UCI: '%s'" % uci.name ) - trans.set_message( "Adding of storage to instance '%s' initiated." % uci.name ) - return self.list( trans ) - else: - error( "Storage can only be added to instances that are in state 'RUNNING' with existing " \ - "storage volume(s) already attached." ) - - return trans.show_form( - web.FormBuilder( url_for( id=trans.security.encode_id(uci.id) ), "Add storage to an instance", submit_text="Add" ) - .add_text( "vol_size", "Storage size (1-1000 GB)", value='', error=error ) ) - - # ----- Image methods ----- - @web.expose - @web.require_admin - def add_new_image( self, trans, provider_type='', image_id='', manifest='', architecture='', state=None ): - #id_error = arch_error = provider_error = manifest_error = None - error = {} - if provider_type or image_id or manifest or architecture: - if provider_type=='': - error['provider_error'] = "You must select cloud provider type for this machine image." - elif image_id=='' or len( image_id ) > 255: - error['id_error'] = "Image ID must be between 1 and 255 characters long." - elif trans.sa_session.query( model.CloudImage ) \ - .filter_by( deleted=False ) \ - .filter( model.CloudImage.table.c.image_id == image_id ) \ - .first(): - error['id_error'] = "Image with ID '" + image_id + "' is already registered. \ - Please choose another ID." - elif architecture=='': - error['arch_error'] = "You must select architecture type for this machine image." - else: - # Create new image - image = model.CloudImage() - image.provider_type = provider_type - image.image_id = image_id - image.manifest = manifest - image.architecture = architecture - # Persist - session = trans.sa_session - session.add( image ) - session.flush() - # Log and display the management page - trans.log_event( "New cloud image added: '%s'" % image.image_id ) - trans.set_message( "Cloud image '%s' added." % image.image_id ) - if state: - image.state = state - images = trans.sa_session.query( model.CloudImage ).all() - return trans.fill_template( '/cloud/list_images.mako', images=images ) - - return trans.fill_template( "cloud/add_image.mako", - provider_type = provider_type, - image_id = image_id, - manifest = manifest, - architecture = architecture, - error = error ) -# return trans.show_form( -# web.FormBuilder( web.url_for(), "Add new cloud image", submit_text="Add" ) -# .add_text( "provider_type", "Provider type", value='ec2 or eucalyptus', error=provider_error ) -# .add_text( "image_id", "Machine Image ID (AMI or EMI)", value='', error=id_error ) -# .add_text( "manifest", "Manifest", value='', error=manifest_error ) -# .add_text( "architecture", "Architecture", value='i386 or x86_64', error=arch_error ) ) - - @web.expose - @web.require_login( "use Galaxy cloud" ) - def list_machine_images( self, trans ): - images = trans.sa_session.query( model.CloudImage ).filter_by( deleted=False ).all() - return trans.fill_template( '/cloud/list_images.mako', images=images ) - - @web.expose - @web.require_admin - def delete_image( self, trans, id=None ): - if not isinstance( id, int ): - id = trans.security.decode_id( id ) - - image = trans.sa_session.query( model.CloudImage ).get( id ) - image.deleted = True - trans.sa_session.add( image ) - trans.sa_session.flush() - return self.list_machine_images( trans ) - - @web.expose - @web.require_admin - def edit_image( self, trans, provider_type='', image_id='', manifest='', architecture='', id='', edited=False ): - error = {} - if not isinstance( id, int ): - id = trans.security.decode_id( id ) - - if not edited: - image = trans.sa_session.query( model.CloudImage ).get( id ) - return trans.fill_template( "cloud/edit_image.mako", - image = image, - error = error - ) - else: - image = trans.sa_session.query( model.CloudImage ).get( id ) - if image_id=='' or len( image_id ) > 255: - error['id_error'] = "Image ID must be between 1 and 255 characters in length." - elif trans.sa_session.query( model.CloudImage ) \ - .filter_by( deleted=False ) \ - .filter( and_( model.CloudImage.table.c.id != image.id, model.CloudImage.table.c.image_id==image_id ) ) \ - .first(): - error['id_error'] = "Image with ID '" + image_id + "' already exist. Please choose an alternative name." - elif architecture=='' or len( architecture ) > 255: - error['arch_error'] = "Architecture type must be between 1 and 255 characters long." - if error: - return trans.fill_template( "cloud/edit_image.mako", - image = image, - error = error - ) - else: - image.image_id = image_id - image.manifest = manifest - image.architecture = architecture - # Persist - session = trans.sa_session - session.add( image ) - session.flush() - # Log and display the management page - trans.set_message( "Machine image '%s' edited." % image.image_id ) - return self.list_machine_images( trans ) - - # ----- Credentials methods ----- - - @web.expose - @web.require_login( "add credentials" ) - def add_credentials( self, trans, credName='', accessKey='', secretKey='', providerName='' ): - """ - Add user's cloud credentials stored under name `credName`. - """ - user = trans.get_user() - error = {} - - if credName or providerName or accessKey or secretKey: - if credName=='' or len( credName ) > 255: - error['cred_error'] = "Credentials name must be between 1 and 255 characters in length." - elif trans.sa_session.query( model.CloudUserCredentials ) \ - .filter_by( user=user, deleted=False ) \ - .filter( model.CloudUserCredentials.table.c.name == credName ) \ - .first(): - error['cred_error'] = "Credentials with that name already exist." - elif providerName=='': - error['provider_error'] = "You must select cloud provider associated with these credentials." - elif accessKey=='' or len( accessKey ) > 255: - error['access_key_error'] = "Access key must be between 1 and 255 characters long." - elif secretKey=='' or len( secretKey ) > 255: - error['secret_key_error'] = "Secret key must be between 1 and 255 characters long." - else: - # Create new user stored credentials - credentials = model.CloudUserCredentials() - credentials.name = credName - credentials.user = user - credentials.access_key = accessKey - credentials.secret_key = secretKey - provider = get_provider( trans, providerName ) - credentials.provider = provider - # Persist - session = trans.sa_session - session.add( credentials ) - session.flush() - # Log and display the management page - trans.log_event( "User added new credentials" ) - trans.set_message( "Credential '%s' created" % credentials.name ) - return self.list( trans ) - - providers = trans.sa_session.query( model.CloudProvider ).filter_by( user=user ).all() - return trans.fill_template( "cloud/add_credentials.mako", - credName = credName, - providerName = providerName, - accessKey = accessKey, - secretKey = secretKey, - error = error, - providers = providers - ) - - @web.expose - @web.require_login( "use Galaxy cloud" ) - def edit_credentials( self, trans, id, credName=None, accessKey=None, secretKey=None, edited=False ): - error = {} - if not edited: - credentials = get_stored_credentials( trans, id ) - return trans.fill_template( "cloud/edit_credentials.mako", - credential = credentials, - error = error - ) - else: - user = trans.get_user() - credentials = get_stored_credentials( trans, id ) - if credName=='' or len( credName ) > 255: - error['cred_error'] = "Credentials name must be between 1 and 255 characters in length." - elif trans.sa_session.query( model.CloudUserCredentials ) \ - .filter_by( user=user ) \ - .filter( and_( model.CloudUserCredentials.table.c.id != credentials.id, model.CloudUserCredentials.table.c.name==credName ) ) \ - .first(): - error['cred_error'] = "Credentials with name '" + credName + "' already exist. Please choose an alternative name." - elif accessKey=='' or len( accessKey ) > 255: - error['access_key_error'] = "Access key must be between 1 and 255 characters long." - elif secretKey=='' or len( secretKey ) > 255: - error['secret_key_error'] = "Secret key must be between 1 and 255 characters long." - - if error: - return trans.fill_template( "cloud/edit_credentials.mako", - credential = credentials, - error = error - ) - else: - # Edit user stored credentials - credentials.name = credName - credentials.access_key = accessKey - credentials.secret_key = secretKey - # Persist - session = trans.sa_session - session.add( credentials ) - session.flush() - # Log and display the management page - trans.set_message( "Credential '%s' edited." % credentials.name ) - return self.list( trans ) - - @web.expose - @web.require_login( "view credentials" ) - def view_credentials( self, trans, id=None ): - """ - View details for user credentials - """ - # Load credentials from database - stored = get_stored_credentials( trans, id ) - - return trans.fill_template( "cloud/view_credentials.mako", - credDetails = stored ) - - @web.expose - @web.require_login( "test cloud credentials" ) - def test_cred( self, trans, id=None ): - """ - Tests credentials provided by user with selected cloud provider - """ - - @web.expose - @web.require_login( "delete credentials" ) - def delete_credentials( self, trans, id=None ): - """ - Delete user's cloud credentials checking that no registered instances are tied to given credentials. - """ - # Load credentials from database - user = trans.get_user() - stored = get_stored_credentials( trans, id ) - # Check if there are any UCIs that depend on these credentials - UCI = None - UCI = trans.sa_session.query( model.UCI ) \ - .filter_by( user=user, credentials_id=stored.id, deleted=False ) \ - .first() - - if UCI == None: - # Delete and save - stored.deleted = True - trans.sa_session.add( stored ) - trans.sa_session.flush() - # Display the management page - trans.set_message( "Credentials '%s' deleted." % stored.name ) - return self.list( trans ) - else: - error( "Existing instance(s) depend on credentials '%s'. You must delete those instances before being able \ - to delete these credentials." % stored.name ) - return self.list( trans ) - - # ----- Provider methods ----- - - @web.expose - @web.require_login( "add provider" ) - def add_provider( self, trans, name='', type='', region_name='', region_endpoint='', is_secure='', host='', port='', proxy='', proxy_port='', - proxy_user='', proxy_pass='', debug='', https_connection_factory='', path='' ): - user = trans.get_user() - error = {} - - if region_name or region_endpoint or name or is_secure or port or proxy or debug or path: - try: - is_secure = int(is_secure) - except ValueError: - error['is_secure_error'] = "Field 'is secure' can only take on an integer value '0' or '1'" - - if trans.sa_session.query( model.CloudProvider ) \ - .filter_by (user=user, name=name) \ - .filter( model.CloudProvider.table.c.deleted != True ) \ - .first(): - error['name_error'] = "A provider with that name already exist." - elif name=='' or len( name ) > 255: - error['name_error'] = "Provider name must be between 1 and 255 characters long." - elif type=='': - error['type_error'] = "Provider type must be selected." - elif not (is_secure == 0 or is_secure == 1): - error['is_secure_error'] = "Field 'is secure' can only take on an integer value '0' or '1'" - else: - provider = model.CloudProvider() - provider.user = user - provider.type = type - provider.name = name - if region_name: - provider.region_name = region_name - else: - provider.region_name = None - - if region_endpoint: - provider.region_endpoint = region_endpoint - else: - provider.region_endpoint = None - - if is_secure==0: - provider.is_secure = False - else: - provider.is_secure = True - - if host: - provider.host = host - else: - provider.host = None - - if port: - provider.port = port - else: - provider.port = None - - if proxy: - provider.proxy = proxy - else: - provider.proxy = None - - if proxy_port: - provider.proxy_port = proxy_port - else: - provider.proxy_port = None - - if proxy_user: - provider.proxy_user = proxy_user - else: - provider.proxy_user = None - - if proxy_pass: - provider.proxy_pass = proxy_pass - else: - provider.proxy_pass = None - - if debug: - provider.debug = debug - else: - provider.debug = None - - if https_connection_factory: - provider.https_connection_factory = https_connection_factory - else: - provider.https_connection_factory = None - - provider.path = path - # Persist - session = trans.sa_session - session.add( provider ) - session.flush() - # Log and display the management page - trans.log_event( "User configured new cloud provider: '%s'" % name ) - trans.set_message( "New cloud provider '%s' added." % name ) - return self.list( trans ) - - return trans.fill_template( "cloud/add_provider.mako", - name = name, - type = type, - region_name = region_name, - region_endpoint = region_endpoint, - is_secure = is_secure, - host = host, - port = port, - proxy = proxy, - proxy_port = proxy_port, - proxy_user = proxy_user, - proxy_pass = proxy_pass, - debug = debug, - https_connection_factory = https_connection_factory, - path = path, - error = error - ) - - @web.expose - @web.require_login( "add Amazon EC2 provider" ) - def add_ec2( self, trans ): - """ Default provider setup for Amazon's EC2. """ - self.add_provider( trans, name='Amazon EC2', type='ec2', region_name='us-east-1', region_endpoint='us-east-1.ec2.amazonaws.com', is_secure=1, path='/' ) - return self.add( trans ) - - @web.expose - @web.require_login( "use Galaxy cloud" ) - def view_provider( self, trans, id=None ): - """ - View details about given cloud provider - """ - # Load credentials from database - provider = get_provider_by_id( trans, id ) - - return trans.fill_template( "cloud/view_provider.mako", - provider = provider ) - - @web.expose - @web.require_login( "use Galaxy cloud" ) - def edit_provider( self, trans, id, name='', type='', region_name='', region_endpoint='', is_secure='', host='', port='', proxy='', proxy_port='', - proxy_user='', proxy_pass='', debug='', https_connection_factory='', path='', edited=False ): - error = {} - if edited == False: - provider = get_provider_by_id( trans, id ) - return trans.fill_template( "cloud/edit_provider.mako", - provider = provider, - error = error - ) - else: - user = trans.get_user() - provider = get_provider_by_id( trans, id ) - - try: - is_secure = int(is_secure) - except ValueError: - error['is_secure_error'] = "Field 'is secure' can only take on an integer value '0' or '1'" - - if name=='' or len( name ) > 255: - error['name_error'] = "Cloud provider name must be between 1 and 255 characters in length." - elif trans.sa_session.query( model.CloudProvider ) \ - .filter_by( user=user ) \ - .filter( and_( model.CloudProvider.table.c.id != provider.id, model.CloudProvider.table.c.name == name ) ) \ - .first(): - error['name_error'] = "Cloud provider with name '" + name + "' already exist. Please choose an alternative name." - elif not ( is_secure == 0 or is_secure == 1): - error['is_secure_error'] = "Field 'is secure' can only take on an integer value '0' or '1'" - - if error: - return trans.fill_template( "cloud/edit_provider.mako", - provider = provider, - error = error - ) - else: - provider.name = name - if region_name and region_name != 'None': - provider.region_name = region_name - else: - provider.region_name = None - - if region_endpoint and region_endpoint != 'None': - provider.region_endpoint = region_endpoint - else: - provider.region_endpoint = None - - if is_secure==0: - provider.is_secure = False - else: - provider.is_secure = True - - if host and host != 'None': - provider.host = host - else: - provider.host = None - - if port and port != 'None': - provider.port = port - else: - provider.port = None - - if proxy and proxy != 'None': - provider.proxy = proxy - else: - provider.proxy = None - - if proxy_port and proxy_port != 'None': - provider.proxy_port = proxy_port - else: - provider.proxy_port = None - - if proxy_user and proxy_user != 'None': - provider.proxy_user = proxy_user - else: - provider.proxy_user = None - - if proxy_pass and proxy_pass != 'None': - provider.proxy_pass = proxy_pass - else: - provider.proxy_pass = None - - if debug and debug != 'None': - provider.debug = debug - else: - provider.debug = None - - if https_connection_factory and https_connection_factory != 'None': - provider.https_connection_factory = https_connection_factory - else: - provider.https_connection_factory = None - - if path and path != 'None': - provider.path = path - else: - provider.path = None - # Persist - session = trans.sa_session - session.add( provider ) - session.flush() - # Log and display the management page - trans.log_event( "User edited cloud provider: '%s'" % name ) - trans.set_message( "Cloud provider '%s' edited." % name ) - return self.list( trans ) - - @web.expose - @web.require_login( "delete credentials" ) - def delete_provider( self, trans, id=None ): - """ - Delete use-registered cloud provider checking that no registered credentials are tied to given provider. - """ - # Load provider from database - user = trans.get_user() - provider = get_provider_by_id( trans, id ) - creds = trans.sa_session.query( model.CloudUserCredentials ) \ - .filter_by( user=user, provider_id=provider.id ) \ - .filter( model.CloudUserCredentials.table.c.deleted != True ) \ - .all() - - if len( creds ) == 0: - # Delete and save - #sess = trans.sa_session - provider.deleted = True - trans.sa_session.add( provider ) - trans.sa_session.flush() - # Display the management page - trans.set_message( "Cloud provider '%s' deleted." % provider.name ) - return self.list( trans ) - - error( "Existing credentails depend on cloud provider '%s'. You must delete those credentials before being able \ - to delete this cloud provider." % provider.name ) - return self.list( trans ) - - # ----- AJAX methods ----- - - @web.json - def json_update( self, trans ): - user = trans.get_user() - UCIs = trans.sa_session.query( model.UCI ).filter_by( user=user, deleted=False ).all() - insd = {} # instance name-state dict - for uci in UCIs: - dict = {} - dict['id'] = uci.id - dict['state'] = uci.state - dict['total_size'] = uci.total_size - if uci.error != None: - dict['error'] = str( uci.error ) - else: - dict['error'] = None - if uci.launch_time != None: - dict['launch_time'] = str( uci.launch_time ) - dict['time_ago'] = str( date.distance_of_time_in_words( uci.launch_time, date.datetime.utcnow() ) ) - else: - dict['launch_time'] = None - dict['time_ago'] = None - insd[uci.name] = dict - return insd - - @web.json - def link_update( self, trans, uci_id=0 ): - ild = {} # instance-link-dict - dict = {} - dict['uci_id'] = uci_id - try: - user = trans.get_user() - # TODO: This query can assumes only one instance under given UCI can be running (i.e., started). - inst = trans.sa_session.query( model.CloudInstance ).filter_by( user=user, uci_id=uci_id, state=uci_states.RUNNING ).first() - urllib2.urlopen( "http://" + inst.public_dns ) - dict['public_dns'] = inst.public_dns - dict['inst_id'] = inst.id - ild['data'] = dict - return ild - except urllib2.URLError: - dict['public_dns'] = False - ild['data'] = dict - return ild - -## ---- Utility methods ------------------------------------------------------- - -def get_provider( trans, name ): - user = trans.get_user() - return trans.sa_session.query( model.CloudProvider ) \ - .filter_by (user=user, name=name) \ - .first() - -def get_provider_by_id( trans, id, check_ownership=True ): - # Check if 'id' is in int (i.e., it was called from this program) or - # it was passed from the web (in which case decode it) - if not isinstance( id, int ): - id = trans.security.decode_id( id ) - - stored = trans.sa_session.query( model.CloudProvider ).get( id ) - if not stored: - error( "Cloud provider not found" ) - # Verify ownership - user = trans.get_user() - if not user: - error( "Must be logged in to use the cloud." ) - if check_ownership and not( stored.user == user ): - error( "Cloud provider '%s' is not registered by current user." % stored.name ) - # Looks good - return stored - -def get_stored_credentials( trans, id, check_ownership=True ): - """ - Get StoredUserCredentials from the database by id, verifying ownership. - """ - # Check if 'id' is in int (i.e., it was called from this program) or - # it was passed from the web (in which case decode it) - if not isinstance( id, int ): - id = trans.security.decode_id( id ) - - stored = trans.sa_session.query( model.CloudUserCredentials ).get( id ) - if not stored: - error( "Credentials not found" ) - # Verify ownership - user = trans.get_user() - if not user: - error( "Must be logged in to use the cloud." ) - if check_ownership and not( stored.user == user ): - error( "Credentials are not owned by current user." ) - # Looks good - return stored - -def get_uci( trans, id, check_ownership=True ): - """ - Get a UCI object from the database by id, verifying ownership. - """ - # Check if 'id' is in int (i.e., it was called from this program) or - # it was passed from the web (in which case decode it) - if not isinstance( id, int ): - id = trans.security.decode_id( id ) - - live = trans.sa_session.query( model.UCI ).get( id ) - if not live: - error( "Galaxy instance not found." ) - # Verify ownership - user = trans.get_user() - if not user: - error( "Must be logged in to use the cloud." ) - if check_ownership and not( live.user == user ): - error( "Instance is not owned by current user." ) - # Looks good - return live - -def get_stores( trans, uci ): - """ - Get stores objects that are associated with given uci and are not in 'error' status - """ - user = trans.get_user() - stores = trans.sa_session.query( model.CloudStore ) \ - .filter_by( user=user, uci_id=uci.id, deleted=False ) \ - .filter( model.CloudStore.table.c.status != store_status.ERROR ) \ - .all() - - return stores - -def get_stores_in_status( trans, uci, status ): - """ - Get stores objects that are associated with given uci and are not have given status - """ - user = trans.get_user() - stores = trans.sa_session.query( model.CloudStore ) \ - .filter_by( user=user, uci_id=uci.id, status=status ) \ - .all() - - return stores - -def get_instances( trans, uci ): - """ - Get objects of instances that are pending or running and are connected to the given uci object - """ - user = trans.get_user() - instances = trans.sa_session.query( model.CloudInstance ) \ - .filter_by( user=user, uci_id=uci.id ) \ - .filter( or_(model.CloudInstance.table.c.state==instance_states.RUNNING, model.CloudInstance.table.c.state==instance_states.PENDING ) ) \ - .first() - #.all() #TODO: return all but need to edit calling method(s) to handle list - - return instances - -def get_instances_in_state( trans, uci, state ): - """ - Get objects of instances that are in specified state and are connected to the given uci object - """ - user = trans.get_user() - instances = trans.sa_session.query( model.CloudInstance ) \ - .filter_by( user=user, uci_id=uci.id, state=state ) \ - .all() - - return instances - -def get_connection( trans, creds ): - """ - Establishes cloud connection using user's credentials - """ - log.debug( 'Establishing cloud connection.' ) -# user = trans.get_user() -# creds = trans.sa_session.query( model.CloudUserCredentials ) \ -# .filter_by( user=user, name=credName ) \ -# .first() - #.filter( model.CloudUserCredentials.table.c.deleted != True ) \ MOVE TO LINE ABOVE ONCE DELETE COLUMS ARE IMPLEMENTED - - if creds: - a_key = creds.access_key - s_key = creds.secret_key - try: - euca_region = RegionInfo( None, creds.provider.region_name, creds.provider.region_endpoint ) - conn = EC2Connection( aws_access_key_id=a_key, - aws_secret_access_key=s_key, - is_secure=creds.provider.is_secure, - port=creds.provider.port, - region=euca_region, - path=creds.provider.path ) - except boto.exception.EC2ResponseError, e: - log.error( "Establishing connection with cloud failed: %s" % str(e) ) - return None - - return conn --- /dev/null +++ b/lib/galaxy/model/migrate/versions/0050_drop_cloud_tables.py @@ -0,0 +1,147 @@ +from sqlalchemy import * +from migrate import * + +import datetime +now = datetime.datetime.utcnow + +# Need our custom types, but don't import anything else from model +from galaxy.model.custom_types import * + +import logging +log = logging.getLogger( __name__ ) + +metadata = MetaData( migrate_engine ) + +def display_migration_details(): + print + print "========================================" + print "This script drops tables that were associated with the old Galaxy Cloud functionality." + print "========================================" + +CloudImage_table = Table( "cloud_image", metadata, + Column( "id", Integer, primary_key=True ), + Column( "create_time", DateTime, default=now ), + Column( "update_time", DateTime, default=now, onupdate=now ), + Column( "provider_type", TEXT ), + Column( "image_id", TEXT, nullable=False ), + Column( "manifest", TEXT ), + Column( "state", TEXT ), + Column( "architecture", TEXT ), + Column( "deleted", Boolean, default=False ) ) + +""" UserConfiguredInstance (UCI) table """ +UCI_table = Table( "cloud_uci", metadata, + Column( "id", Integer, primary_key=True ), + Column( "create_time", DateTime, default=now ), + Column( "update_time", DateTime, default=now, onupdate=now ), + Column( "user_id", Integer, ForeignKey( "galaxy_user.id" ), index=True, nullable=False ), + Column( "credentials_id", Integer, ForeignKey( "cloud_user_credentials.id" ), index=True ), + Column( "key_pair_name", TEXT ), + Column( "key_pair_material", TEXT ), + Column( "name", TEXT ), + Column( "state", TEXT ), + Column( "error", TEXT ), + Column( "total_size", Integer ), + Column( "launch_time", DateTime ), + Column( "deleted", Boolean, default=False ) ) + +CloudInstance_table = Table( "cloud_instance", metadata, + Column( "id", Integer, primary_key=True ), + Column( "create_time", DateTime, default=now ), + Column( "update_time", DateTime, default=now, onupdate=now ), + Column( "launch_time", DateTime ), + Column( "stop_time", DateTime ), + Column( "user_id", Integer, ForeignKey( "galaxy_user.id" ), index=True, nullable=False ), + Column( "uci_id", Integer, ForeignKey( "cloud_uci.id" ), index=True ), + Column( "type", TEXT ), + Column( "reservation_id", TEXT ), + Column( "instance_id", TEXT ), + Column( "mi_id", Integer, ForeignKey( "cloud_image.id" ), index=True ), + Column( "state", TEXT ), + Column( "error", TEXT ), + Column( "public_dns", TEXT ), + Column( "private_dns", TEXT ), + Column( "security_group", TEXT ), + Column( "availability_zone", TEXT ) ) + +CloudStore_table = Table( "cloud_store", metadata, + Column( "id", Integer, primary_key=True ), + Column( "create_time", DateTime, default=now ), + Column( "update_time", DateTime, default=now, onupdate=now ), + Column( "attach_time", DateTime ), + Column( "user_id", Integer, ForeignKey( "galaxy_user.id" ), index=True, nullable=False ), + Column( "uci_id", Integer, ForeignKey( "cloud_uci.id" ), index=True, nullable=False ), + Column( "volume_id", TEXT ), + Column( "size", Integer, nullable=False ), + Column( "availability_zone", TEXT ), + Column( "inst_id", Integer, ForeignKey( "cloud_instance.id" ) ), + Column( "status", TEXT ), + Column( "device", TEXT ), + Column( "space_consumed", Integer ), + Column( "error", TEXT ), + Column( "deleted", Boolean, default=False ) ) + +CloudSnapshot_table = Table( "cloud_snapshot", metadata, + Column( "id", Integer, primary_key=True ), + Column( "create_time", DateTime, default=now ), + Column( "update_time", DateTime, default=now, onupdate=now ), + Column( "user_id", Integer, ForeignKey( "galaxy_user.id" ), index=True, nullable=False ), + Column( "uci_id", Integer, ForeignKey( "cloud_uci.id" ), index=True ), + Column( "store_id", Integer, ForeignKey( "cloud_store.id" ), index=True, nullable=False ), + Column( "snapshot_id", TEXT ), + Column( "status", TEXT ), + Column( "description", TEXT ), + Column( "error", TEXT ), + Column( "deleted", Boolean, default=False ) ) + +CloudUserCredentials_table = Table( "cloud_user_credentials", metadata, + Column( "id", Integer, primary_key=True ), + Column( "create_time", DateTime, default=now ), + Column( "update_time", DateTime, default=now, onupdate=now ), + Column( "user_id", Integer, ForeignKey( "galaxy_user.id" ), index=True, nullable=False ), + Column( "provider_id", Integer, ForeignKey( "cloud_provider.id" ), index=True, nullable=False ), + Column( "name", TEXT ), + Column( "access_key", TEXT ), + Column( "secret_key", TEXT ), + Column( "deleted", Boolean, default=False ) ) + +CloudProvider_table = Table( "cloud_provider", metadata, + Column( "id", Integer, primary_key=True ), + Column( "create_time", DateTime, default=now ), + Column( "update_time", DateTime, default=now, onupdate=now ), + Column( "user_id", Integer, ForeignKey( "galaxy_user.id" ), index=True, nullable=False ), + Column( "type", TEXT, nullable=False ), + Column( "name", TEXT ), + Column( "region_connection", TEXT ), + Column( "region_name", TEXT ), + Column( "region_endpoint", TEXT ), + Column( "is_secure", Boolean ), + Column( "host", TEXT ), + Column( "port", Integer ), + Column( "proxy", TEXT ), + Column( "proxy_port", TEXT ), + Column( "proxy_user", TEXT ), + Column( "proxy_pass", TEXT ), + Column( "debug", Integer ), + Column( "https_connection_factory", TEXT ), + Column( "path", TEXT ), + Column( "deleted", Boolean, default=False ) ) + +def upgrade(): + display_migration_details() + # Load existing tables + metadata.reflect() + try: + CloudProvider_table.drop() + CloudUserCredentials_table.drop() + CloudImage_table.drop() + UCI_table.drop() + CloudInstance_table.drop() + CloudStore_table.drop() + CloudSnapshot_table.drop() + except Exception, e: + log.debug( "Dropping cloud tables failed: %s" % str( e ) ) + + +def downgrade(): + pass --- a/lib/galaxy/model/mapping.py +++ b/lib/galaxy/model/mapping.py @@ -409,117 +409,6 @@ GalaxySessionToHistoryAssociation.table Column( "session_id", Integer, ForeignKey( "galaxy_session.id" ), index=True ), Column( "history_id", Integer, ForeignKey( "history.id" ), index=True ) ) -# *************************** Start cloud tables*********************************** -CloudImage.table = Table( "cloud_image", metadata, - Column( "id", Integer, primary_key=True ), - Column( "create_time", DateTime, default=now ), - Column( "update_time", DateTime, default=now, onupdate=now ), - Column( "provider_type", TEXT ), - Column( "image_id", TEXT, nullable=False ), - Column( "manifest", TEXT ), - Column( "state", TEXT ), - Column( "architecture", TEXT ), - Column( "deleted", Boolean, default=False ) ) - -""" UserConfiguredInstance (UCI) table """ -UCI.table = Table( "cloud_uci", metadata, - Column( "id", Integer, primary_key=True ), - Column( "create_time", DateTime, default=now ), - Column( "update_time", DateTime, default=now, onupdate=now ), - Column( "user_id", Integer, ForeignKey( "galaxy_user.id" ), index=True, nullable=False ), - Column( "credentials_id", Integer, ForeignKey( "cloud_user_credentials.id" ), index=True ), - Column( "key_pair_name", TEXT ), - Column( "key_pair_material", TEXT ), - Column( "name", TEXT ), - Column( "state", TEXT ), - Column( "error", TEXT ), - Column( "total_size", Integer ), - Column( "launch_time", DateTime ), - Column( "deleted", Boolean, default=False ) ) - -CloudInstance.table = Table( "cloud_instance", metadata, - Column( "id", Integer, primary_key=True ), - Column( "create_time", DateTime, default=now ), - Column( "update_time", DateTime, default=now, onupdate=now ), - Column( "launch_time", DateTime ), - Column( "stop_time", DateTime ), - Column( "user_id", Integer, ForeignKey( "galaxy_user.id" ), index=True, nullable=False ), - Column( "uci_id", Integer, ForeignKey( "cloud_uci.id" ), index=True ), - Column( "type", TEXT ), - Column( "reservation_id", TEXT ), - Column( "instance_id", TEXT ), - Column( "mi_id", Integer, ForeignKey( "cloud_image.id" ), index=True ), - Column( "state", TEXT ), - Column( "error", TEXT ), - Column( "public_dns", TEXT ), - Column( "private_dns", TEXT ), - Column( "security_group", TEXT ), - Column( "availability_zone", TEXT ) ) - -CloudStore.table = Table( "cloud_store", metadata, - Column( "id", Integer, primary_key=True ), - Column( "create_time", DateTime, default=now ), - Column( "update_time", DateTime, default=now, onupdate=now ), - Column( "attach_time", DateTime ), - Column( "user_id", Integer, ForeignKey( "galaxy_user.id" ), index=True, nullable=False ), - Column( "uci_id", Integer, ForeignKey( "cloud_uci.id" ), index=True, nullable=False ), - Column( "volume_id", TEXT ), - Column( "size", Integer, nullable=False ), - Column( "availability_zone", TEXT ), - Column( "inst_id", Integer, ForeignKey( "cloud_instance.id" ) ), - Column( "status", TEXT ), - Column( "device", TEXT ), - Column( "space_consumed", Integer ), - Column( "error", TEXT ), - Column( "deleted", Boolean, default=False ) ) - -CloudSnapshot.table = Table( "cloud_snapshot", metadata, - Column( "id", Integer, primary_key=True ), - Column( "create_time", DateTime, default=now ), - Column( "update_time", DateTime, default=now, onupdate=now ), - Column( "user_id", Integer, ForeignKey( "galaxy_user.id" ), index=True, nullable=False ), - Column( "uci_id", Integer, ForeignKey( "cloud_uci.id" ), index=True ), - Column( "store_id", Integer, ForeignKey( "cloud_store.id" ), index=True, nullable=False ), - Column( "snapshot_id", TEXT ), - Column( "status", TEXT ), - Column( "description", TEXT ), - Column( "error", TEXT ), - Column( "deleted", Boolean, default=False ) ) - -CloudUserCredentials.table = Table( "cloud_user_credentials", metadata, - Column( "id", Integer, primary_key=True ), - Column( "create_time", DateTime, default=now ), - Column( "update_time", DateTime, default=now, onupdate=now ), - Column( "user_id", Integer, ForeignKey( "galaxy_user.id" ), index=True, nullable=False ), - Column( "provider_id", Integer, ForeignKey( "cloud_provider.id" ), index=True, nullable=False ), - Column( "name", TEXT ), - Column( "access_key", TEXT ), - Column( "secret_key", TEXT ), - Column( "deleted", Boolean, default=False ) ) - -CloudProvider.table = Table( "cloud_provider", metadata, - Column( "id", Integer, primary_key=True ), - Column( "create_time", DateTime, default=now ), - Column( "update_time", DateTime, default=now, onupdate=now ), - Column( "user_id", Integer, ForeignKey( "galaxy_user.id" ), index=True, nullable=False ), - Column( "type", TEXT, nullable=False ), - Column( "name", TEXT ), - Column( "region_connection", TEXT ), - Column( "region_name", TEXT ), - Column( "region_endpoint", TEXT ), - Column( "is_secure", Boolean ), - Column( "host", TEXT ), - Column( "port", Integer ), - Column( "proxy", TEXT ), - Column( "proxy_port", TEXT ), - Column( "proxy_user", TEXT ), - Column( "proxy_pass", TEXT ), - Column( "debug", Integer ), - Column( "https_connection_factory", TEXT ), - Column( "path", TEXT ), - Column( "deleted", Boolean, default=False ) ) -# *************************** End cloud tables*********************************** - StoredWorkflow.table = Table( "stored_workflow", metadata, Column( "id", Integer, primary_key=True ), Column( "create_time", DateTime, default=now ), @@ -1284,42 +1173,6 @@ assign_mapper( context, GalaxySessionToH HistoryDatasetAssociation.mapper.add_property( "creating_job_associations", relation( JobToOutputDatasetAssociation ) ) -# vvvvvvvvvvvvvvvv Start cloud table mappings vvvvvvvvvvvvvvvv -assign_mapper( context, CloudImage, CloudImage.table ) - -assign_mapper( context, UCI, UCI.table, - properties=dict( user=relation( User ), - credentials=relation( CloudUserCredentials ), - instance=relation( CloudInstance, backref='uci' ), - store=relation( CloudStore, backref='uci', cascade='all, delete-orphan' ), - snapshot=relation( CloudSnapshot, backref='uci' ) - ) ) - -assign_mapper( context, CloudInstance, CloudInstance.table, - properties=dict( user=relation( User ), - image=relation( CloudImage ) - ) ) - -assign_mapper( context, CloudStore, CloudStore.table, - properties=dict( user=relation( User ), - inst=relation( CloudInstance ), - snapshot=relation( CloudSnapshot, backref="store" ) - ) ) - -assign_mapper( context, CloudSnapshot, CloudSnapshot.table, - properties=dict( user=relation( User ) - ) ) - -assign_mapper( context, CloudProvider, CloudProvider.table, - properties=dict( user=relation( User ) - ) ) - -assign_mapper( context, CloudUserCredentials, CloudUserCredentials.table, - properties=dict( user=relation( User), - provider=relation( CloudProvider ) - ) ) -# ^^^^^^^^^^^^^^^ End cloud table mappings ^^^^^^^^^^^^^^^^^^ - assign_mapper( context, Workflow, Workflow.table, properties=dict( steps=relation( WorkflowStep, backref='workflow', order_by=asc(WorkflowStep.table.c.order_index), --- a/lib/galaxy/app.py +++ b/lib/galaxy/app.py @@ -73,8 +73,6 @@ class UniverseApplication( object ): # FIXME: These are exposed directly for backward compatibility self.job_queue = self.job_manager.job_queue self.job_stop_queue = self.job_manager.job_stop_queue - # Start the cloud manager - self.cloud_manager = cloud.CloudManager( self ) # Track Store ## self.track_store = store.TrackStoreManager( self.config.track_store_path ) --- a/templates/cloud/view_snapshots.mako +++ /dev/null @@ -1,90 +0,0 @@ -<%inherit file="/base.mako"/> - -<%def name="title()">Snapshots</%def> - -%if message: -<% - try: - messagetype - except: - messagetype = "done" -%> - - - -<p /> -<div class="${messagetype}message"> - ${message} -</div> -%endif - -%if snaps: - <h2>Snapshots for instance ${snaps[0].uci.name}</h2> -%else: - <h2>Selected instance has no recorded or associated snapshots.</h2> -%endif - -<ul class="manage-table-actions"> - <li> - <a class="action-button" href="${h.url_for( action='list' )}"> - <img src="${h.url_for('/static/images/silk/resultset_previous.png')}" /> - <span>Return to cloud management console</span> - </a> - </li> -</ul> - -%if snaps: - <table class="mange-table colored" border="0" cellspacing="0" cellpadding="0" width="100%"> - <colgroup width="2%"></colgroup> - <colgroup width="16%"></colgroup> - <colgroup width="16%"></colgroup> - <colgroup width="10%"></colgroup> - <colgroup width="5%"></colgroup> - <tr class="header"> - <th>#</th> - <th>Create time</th> - <th>Snapshot ID</th> - <th>Status</th> - <th>Delete?</th> - <th></th> - </tr> - <% - total_hours = 0 - %> - %for i, snap in enumerate( snaps ): - <tr> - <td>${i+1}</td> - <td> - %if snap.create_time: - ${str(snap.create_time)[:16]} UCT - %else: - N/A - %endif - </td> - <td> - %if snap.snapshot_id: - ${snap.snapshot_id} - %else: - N/A - %endif - </td> - <td> - %if snap.status: - ${snap.status} - %else: - N/A - %endif - </td> - <td> - <a confirm="Are you sure you want to delete snapshot '${snap.snapshot_id}'?" - href="${h.url_for( controller='cloud', action='delete_snapshot', uci_id=trans.security.encode_id(snap.uci.id), snap_id=trans.security.encode_id(snap.id) )}">x</a> - </td> - </tr> - %endfor - </table> -%endif - - - - - --- a/eggs.ini +++ b/eggs.ini @@ -54,7 +54,6 @@ wsgiref = 0.1.2 Babel = 0.9.4 wchartype = 0.1 Whoosh = 0.3.18 -boto = 1.8d ; extra version information [tags] --- a/templates/cloud/configure_cloud.mako +++ /dev/null @@ -1,360 +0,0 @@ -<%inherit file="/base.mako"/> - -<%def name="title()">Cloud home</%def> - -%if message: -<% - try: - messagetype - except: - messagetype = "done" -%> - -<div class="${messagetype}message"> - ${message} -</div> -%endif - -<%def name="javascripts()"> -${parent.javascripts()} -${h.js( "jquery" )} - -<script type="text/javascript"> -function trim19(str){ - var str = str.replace(/^\s\s*/, ''), - ws = /\s/, - i = str.length; - while (ws.test(str.charAt(--i))); - return str.slice(0, i + 1); -} - - function update_state() { - $.getJSON( "${h.url_for( action='json_update' )}", {}, function ( data ) { - for (var i in data) { - var elem = '#' + data[i].id; - // Because of different list managing 'live' vs. 'available' instances, reload url on various state changes - old_state = $(elem + "-state").text(); - prev_old_state = trim19( $(elem + "-state-p").text() ); - new_state = data[i].state; - error_msg = data[i].error; - //console.log( "old_state[%d] = %s", i, old_state ); - //console.log( "prev_old_state[%d] = %s", i, prev_old_state ); - //console.log( "new_state[%d] = %s", i, new_state ); - //console.log( "error_msg[%d] = %s", i, error_msg ); - if ( ( old_state=='pending' && new_state=='running' ) || ( old_state=='shutting-down' && new_state=='available' ) || \ - ( old_state=='running' && new_state=='available' ) || ( old_state=='running' && new_state=='error' ) || \ - ( old_state=='pending' && new_state=='available' ) || ( old_state=='submitted' && new_state=='available' ) || \ - ( prev_old_state.match('creating') && new_state=='available' ) ) { - var url = "${h.url_for( controller='cloud', action='list')}"; - location.replace( url ); - } - else if ( ( ( old_state != 'error' && old_state != '' ) && new_state == 'error' ) || ( !prev_old_state.match('error') && new_state == 'error' ) ) { - var url = "${h.url_for( controller='cloud', action='list')}"; - location.replace( url ); - } - - if ( new_state=='shutting-down' || new_state=='shutting-downUCI' ) { - $(elem + "-link").text( "" ); - } - - // Check if Galaxy website is accessible on given instance; if so, provide link. Otherwise, wait more. - else if ( ( $(elem+"-link").text().match('starting') || $(elem+"-link").text()=='' ) && new_state=='running' ) { - //console.log ( 'elem.text: ' + $(elem+"-link").text() ); - $.getJSON( "${h.url_for( action='link_update' )}", { uci_id: data[i].id }, function ( data ) { - for (var i in data) { - var dns = data[i].public_dns; - var uci = '#' + data[i].uci_id; - if( !dns ) { - $(uci+"-link").text( 'Galaxy starting...' ); - // http://stackoverflow.com/questions/275931/how-do-you-make-an-element-flash-… - //$(uci+"-link").stop().animate({ fontSize: "14px" }, 1000).animate({ fontSize: "12px" }, 1000); - } - else { - $(uci+"-link").html( '<div align="right"><a class="action-button" href="http://'+dns+'" target="_blank">' + - '<span>Access Galaxy</span>'+ - '<img src="' + "${h.url_for( '/static/images/silk/resultset_next.png' )}" + '" /></div>' ); - //$(uci+"-link").stop().animate({ fontSize: "14px" }, 1000).animate({ fontSize: "12px" }, 1000); - } - } - }); - } - - // Update 'size', 'state' and 'time alive' fields - $(elem + "-size").text( data[i].total_size ); - $(elem + "-state").text( data[i].state ); - if ( new_state != 'error' ) { // Because 'error' state is handled as a JS link, don't include it in update - $(elem + "-state-p").text( data[i].state ); - } - - if (data[i].launch_time) { - $(elem + "-launch_time").text( data[i].launch_time.substring(0, 16 ) + " UTC (" + data[i].time_ago + ")" ); - } - else { - $(elem + "-launch_time").text( "N/A" ); - } - } - }); - setTimeout("update_state()", 15000); - } - - $(function() { - update_state(); - }); - -</script> -</%def> - -<h2>Galaxy in the clouds</h2> - -%if cloudProviders: - ## Manage user-registered cloud providers - <h3>Your registered cloud providers</h3> - <ul class="manage-table-actions"> - <li> - <a class="action-button" href="${h.url_for( action='add_provider' )}"> - <img src="${h.url_for('/static/images/silk/add.png')}" /> - <span>Add provider</span> - </a> - </li> - </ul> - - <table class="mange-table colored" border="0" cellspacing="0" cellpadding="0" width="100%"> - <tr class="header"> - <th>Provider name</th> - <th>Provider type</th> - <th></th> - </tr> - %for i, cloudProvder in enumerate( cloudProviders ): - <tr> - <td> - ${cloudProvder.name} - <a id="cp-${i}-popup" class="popup-arrow" style="display: none;">▼</a> - </td> - <td> - ${cloudProvder.type} - </td> - <td> - <div popupmenu="cp-${i}-popup"> - <a class="action-button" href="${h.url_for( action='view_provider', id=trans.security.encode_id(cloudProvder.id) )}">View</a> - <a class="action-button" href="${h.url_for( action='edit_provider', id=trans.security.encode_id(cloudProvder.id) )}">Edit</a> - <a class="action-button" confirm="Are you sure you want to delete cloud provider '${cloudProvder.name}'?" href="${h.url_for( action='delete_provider', id=trans.security.encode_id(cloudProvder.id) )}">Delete</a> - </div> - </td> - </tr> - %endfor - </table> - - - ## ***************************************************** - ## Manage user credentials - <h3>Your registered cloud credentials</h3> - - %if cloudCredentials: - <ul class="manage-table-actions"> - <li> - <a class="action-button" href="${h.url_for( action='add_credentials' )}"> - <img src="${h.url_for('/static/images/silk/add.png')}" /> - <span>Add credentials</span> - </a> - </li> - </ul> - <table class="mange-table colored" border="0" cellspacing="0" cellpadding="0" width="100%"> - <tr class="header"> - <th>Credentials name</th> - <th>Provider name (type)</th> - <th></th> - </tr> - - %for i, cloudCredential in enumerate( cloudCredentials ): - <tr> - <td> - ${cloudCredential.name} - <a id="cr-${i}-popup" class="popup-arrow" style="display: none;">▼</a> - </td> - <td> - ${cloudCredential.provider.name} - (${cloudCredential.provider.type}) - </td> - <td> - <div popupmenu="cr-${i}-popup"> - <a class="action-button" href="${h.url_for( action='view_credentials', id=trans.security.encode_id(cloudCredential.id) )}">View</a> - <a class="action-button" href="${h.url_for( action='edit_credentials', id=trans.security.encode_id(cloudCredential.id) )}">Edit</a> - <a class="action-button" confirm="Are you sure you want to delete credentials '${cloudCredential.name}'?" href="${h.url_for( action='delete_credentials', id=trans.security.encode_id(cloudCredential.id) )}">Delete</a> - </div> - </td> - </tr> - %endfor - </table> - - ## ***************************************************** - ## Manage live instances - <p /> - <h3>Manage your cloud instances</h3> - <ul class="manage-table-actions"> - <li> - <a class="action-button" href="${h.url_for( action='configure_new_uci' )}"> - <img src="${h.url_for('/static/images/silk/add.png')}" /> - <span>Configure new instance</span> - </a> - </li> - </ul> - - <table class="mange-table colored" border="0" cellspacing="0" cellpadding="0" width="100%"> - <colgroup width="40%"></colgroup> - <colgroup width="15%"></colgroup> - <colgroup width="10%"></colgroup> - <colgroup width="25%"></colgroup> - <colgroup width="10%"></colgroup> - <tr class="header"> - <th>Live instance name (credentials)</th> - <th>Storage size (GB)</th> - <th>State</th> - <th>Alive since</th> - <th></th> - <th></th> - </tr> - %if liveInstances: - %for i, liveInstance in enumerate( liveInstances ): - <tr> - <td> - ${liveInstance.name} (${liveInstance.credentials.name}) - <a id="li-${i}-popup" class="popup-arrow" style="display: none;">▼</a> - </td> - <td id="${ liveInstance.id }-size">${str(liveInstance.total_size)}</td> - <td id="${ liveInstance.id }-state">${str(liveInstance.state)}</td> - <td id="${ liveInstance.id }-launch_time"> - ##${str(liveInstance.launch_time)[:16]} - <% - #from datetime import datetime - #from datetime import timedelta - - # DB stores all times in GMT, so adjust for difference (4 hours) - #adjustedStarttime = liveInstance.update_time - timedelta(hours=4) - - #delta = datetime.now() - adjustedStarttime - #context.write( str(datetime.utcnow() ) ) - #context.write( str(delta) ) - - # This is where current time and since duration is calculated - if liveInstance.launch_time is None: - context.write( 'N/A' ) - else: - context.write( str( liveInstance.launch_time )[:16] ) - context.write( ' UTC (' ) - context.write( str(h.date.distance_of_time_in_words (liveInstance.launch_time, h.date.datetime.utcnow() ) ) ) - context.write( ')' ) - %> - </td> - ## Handled by JavaScript function - <td id="${ liveInstance.id }-link"></td> - <td> - <div popupmenu="li-${i}-popup"> - <a class="action-button" confirm="Are you sure you want to stop instance '${liveInstance.name}'?" href="${h.url_for( action='stop', id=trans.security.encode_id(liveInstance.id) )}">Stop</a> - <a class="action-button" href="${h.url_for( action='rename_uci', id=trans.security.encode_id(liveInstance.id) )}">Rename</a> - <a class="action-button" href="${h.url_for( action='view_uci_details', id=trans.security.encode_id(liveInstance.id) )}">View details</a> - <a class="action-button" href="${h.url_for( action='add_storage', id=trans.security.encode_id(liveInstance.id) )}">Add storage</a> - <a class="action-button" href="${h.url_for( action='uci_usage_report', id=trans.security.encode_id(liveInstance.id) )}">Usage report</a> - </div> - </td> - </tr> - %endfor - %else: - <tr> - <td>Currently, you have no live instances.</td> - </tr> - %endif - </table> - <p /> - ## ***************************************************** - ## Manage previously configured instances - ## <table class="mange-table noHR" border="0" cellspacing="0" cellpadding="0" width="100%"> - <table class="mange-table colored" border="0" cellspacing="0" cellpadding="0" width="100%"> - <colgroup width="40%"></colgroup> - <colgroup width="15%"></colgroup> - <colgroup width="10%"></colgroup> - <colgroup width="25%"></colgroup> - <colgroup width="10%"></colgroup> - <tr class="header"> - <th>Configured instance name (credentials)</th> - <th>Storage size (GB)</th> - <th>State</th> - <th></th> - <th></th> - </tr> - - %if prevInstances: - %for i, prevInstance in enumerate( prevInstances ): - <tr> - <td> - ${prevInstance.name} (${prevInstance.credentials.name}) - <a id="pi-${i}-popup" class="popup-arrow" style="display: none;">▼</a> - </td> - <td>${str(prevInstance.total_size)}</td> - <td id="${ prevInstance.id }-state-p"> - <%state = str(prevInstance.state)%> - %if state =='error': - <div id="${prevInstance.name}-short"> - <a onclick="document.getElementById('${prevInstance.name}-full').style.display = 'block'; - document.getElementById('${prevInstance.name}-short').style.display = 'none'; return 0" - href="javascript:void(0)"> - error - </a> - </div> - <div id="${prevInstance.name}-full" style="DISPLAY: none"> - <a onclick="document.getElementById('${prevInstance.name}-short').style.display = 'block'; - document.getElementById('${prevInstance.name}-full').style.display = 'none'; return 0;" - href="javascript:void(0)"> - error:</a><br /> - ${str(prevInstance.error)} - <p /> - <div style="font-size:10px;"> - <a href="${h.url_for( action='set_uci_state', id=trans.security.encode_id(prevInstance.id), state='available' )}">reset state</a> - </div> - </div> - %else: - ${str(prevInstance.state)} - %endif - </td> - <td> - <div popupmenu="pi-${i}-popup"> - <a class="action-button" href="${h.url_for( action='uci_usage_report', id=trans.security.encode_id(prevInstance.id) )}">Usage report</a> - <a class="action-button" href="${h.url_for( action='rename_uci', id=trans.security.encode_id(prevInstance.id) )}">Rename</a> - <a class="action-button" href="${h.url_for( action='start', id=trans.security.encode_id(prevInstance.id), type='m1.small' )}"> Start m1.small</a> - <a class="action-button" href="${h.url_for( action='start', id=trans.security.encode_id(prevInstance.id), type='c1.medium' )}"> Start c1.medium</a> - <a class="action-button" href="${h.url_for( action='start', id=trans.security.encode_id(prevInstance.id), type='m1.large' )}"> Start m1.large</a> - <a class="action-button" href="${h.url_for( action='start', id=trans.security.encode_id(prevInstance.id), type='m1.xlarge' )}"> Start m1.xlarge</a> - <a class="action-button" href="${h.url_for( action='start', id=trans.security.encode_id(prevInstance.id), type='c1.xlarge' )}"> Start c1.xlarge</a> - <a class="action-button" href="${h.url_for( action='start', id=trans.security.encode_id(prevInstance.id), type='m2.2xlarge' )}"> Start m2.2xlarge</a> - <a class="action-button" href="${h.url_for( action='start', id=trans.security.encode_id(prevInstance.id), type='m2.4xlarge' )}"> Start m2.4xlarge</a> - <a class="action-button" href="${h.url_for( action='create_snapshot', id=trans.security.encode_id(prevInstance.id) )}">Create snapshot</a> - <a class="action-button" href="${h.url_for( action='view_snapshots', id=trans.security.encode_id(prevInstance.id) )}">View snapshots</a> - <a class="action-button" confirm="Are you sure you want to delete instance '${prevInstance.name}'? This will delete all of your data assocaiated with this instance!" href="${h.url_for( action='delete_uci', id=trans.security.encode_id(prevInstance.id) )}">Delete</a> - </div> - </td> - </tr> - %endfor - %else: - <tr> - <td>You have no previously configured instances (or they are all currently alive).</td> - </tr> - %endif - </table> - - %else: - You have no credentials associated with your Galaxy account: - <a class="action-button" href="${h.url_for( action='add_credentials' )}"> - <img src="${h.url_for('/static/images/silk/add.png')}" /> - <span>add credentials</span> - </a> - or - <a href="http://aws.amazon.com/" target="_blank"> - open AWS account with Amazon</a>. - %endif - -%else: - You have no cloud providers registered with your Galaxy account: - <a class="action-button" href="${h.url_for( action='add_provider' )}"> - <img src="${h.url_for('/static/images/silk/add.png')}" /> - <span>add provider now</span> - </a> -%endif --- a/universe_wsgi.ini.sample +++ b/universe_wsgi.ini.sample @@ -201,11 +201,6 @@ use_interactive = True # Should default dataset access permissions be private for new users; default is False (datasets are public) new_user_dataset_access_role_default_private = False -# ---- Cloud Management -------------------------------------------------------- - -# Uncomment following line to enable cloud management mode (only leaves the Cloud, Help and User tabs on) -# cloud_controller_instance = True - # ---- Job Execution -------------------------------------------------------- # If running multiple Galaxy processes, one can be designated as the job --- a/lib/galaxy/cloud/__init__.py +++ /dev/null @@ -1,734 +0,0 @@ -import logging, threading, sys, os, time, subprocess, string, tempfile, re, traceback, shutil - -from galaxy import util, model, config -from galaxy.model import mapping -from galaxy.model.orm import lazyload -from galaxy.datatypes.tabular import * -from galaxy.datatypes.interval import * -from galaxy.datatypes import metadata -from galaxy.util.bunch import Bunch -from sqlalchemy import or_ - -import pkg_resources -pkg_resources.require( "PasteDeploy" ) - -from paste.deploy.converters import asbool - -from Queue import Queue, Empty - -log = logging.getLogger( __name__ ) - -uci_states = Bunch( - NEW_UCI = "newUCI", - NEW = "new", - CREATING = "creating", - DELETING_UCI = "deletingUCI", - DELETING = "deleting", - DELETED = "deleted", - SUBMITTED_UCI = "submittedUCI", - SUBMITTED = "submitted", - SHUTTING_DOWN_UCI = "shutting-downUCI", - SHUTTING_DOWN = "shutting-down", - ADD_STORAGE_UCI = "add-storageUCI", - ADD_STORAGE = "add-storage", - AVAILABLE = "available", - RUNNING = "running", - PENDING = "pending", - ERROR = "error", - SNAPSHOT_UCI = "snapshotUCI", - SNAPSHOT = "snapshot" -) -instance_states = Bunch( - TERMINATED = "terminated", - SUBMITTED = "submitted", - RUNNING = "running", - ADDING = "adding-storage", - PENDING = "pending", - SHUTTING_DOWN = "shutting-down", - ERROR = "error" -) - -store_status = Bunch( - WAITING = "waiting", - IN_USE = "in-use", - ADDING = "adding", - CREATING = "creating", - DELETED = 'deleted', - ERROR = "error" -) - -snapshot_status = Bunch( - SUBMITTED = 'submitted', - PENDING = 'pending', - COMPLETED = 'completed', - DELETE = 'delete', - DELETED= 'deleted', - ERROR = "error" -) - -class CloudManager( object ): - """ - Highest level interface to cloud management. - """ - def __init__( self, app ): - self.app = app - self.sa_session = app.model.context - if self.app.config.enable_cloud_execution == True: - # The dispatcher manager for underlying cloud instances - implements and contacts individual cloud providers - self.provider = CloudProvider( app ) - # Monitor for updating status of cloud instances - self.cloud_monitor = CloudMonitor( self.app, self.provider ) - else: - self.job_queue = self.job_stop_queue = NoopCloudMonitor() - - def shutdown( self ): - self.cloud_monitor.shutdown() - -class Sleeper( object ): - """ - Provides a 'sleep' method that sleeps for a number of seconds *unless* - the notify method is called (from a different thread). - """ - def __init__( self ): - self.condition = threading.Condition() - def sleep( self, seconds ): - self.condition.acquire() - self.condition.wait( seconds ) - self.condition.release() - def wake( self ): - self.condition.acquire() - self.condition.notify() - self.condition.release() - -class CloudMonitor( object ): - """ - Cloud manager, waits for user to instantiate a cloud instance and then invokes a - CloudProvider. - """ - STOP_SIGNAL = object() - def __init__( self, app, provider ): - """Start the cloud manager""" - self.app = app - # Keep track of the pid that started the cloud manager, only it - # has valid threads - self.parent_pid = os.getpid() - self.sa_session = app.model.context - - # Contains requests that are waiting (only use from monitor thread) - self.waiting = [] - - # Helper for interruptable sleep - self.sleeper = Sleeper() - self.running = True - self.provider = provider - self.monitor_thread = threading.Thread( target=self.__monitor ) - self.monitor_thread.start() - log.info( "Cloud manager started" ) - - def __monitor( self ): - """ - Daemon that continuously monitors cloud instance requests as well as state - of running instances. - """ - # HACK: Delay until after forking, we need a way to do post fork notification!!! - time.sleep( 10 ) - - cnt = 0 # Run global update only periodically so keep counter variable - while self.running: - try: - self.__monitor_step() - if cnt%30 == 0: # Run global update every 30 iterations (1 minute) - self.provider.update() - cnt = 0 - except: - log.exception( "Exception in cloud manager monitor_step" ) - # Sleep - cnt += 1 - self.sleeper.sleep( 2 ) - - def __monitor_step( self ): - """ - Called repeatedly by `monitor` to process cloud instance requests. - TODO: Update following description to match the code - Gets any new cloud instance requests from the database, then iterates - over all new and waiting jobs to check the state of the jobs each - depends on. If the job has dependencies that have not finished, it - it goes to the waiting queue. If the job has dependencies with errors, - it is marked as having errors and removed from the queue. Otherwise, - the job is dispatched. - """ - model = self.app.model - new_requests = [] - - for r in self.sa_session.query( model.UCI ) \ - .filter( or_( model.UCI.table.c.state==uci_states.NEW_UCI, - model.UCI.table.c.state==uci_states.SUBMITTED_UCI, - model.UCI.table.c.state==uci_states.SHUTTING_DOWN_UCI, - model.UCI.table.c.state==uci_states.DELETING_UCI, - model.UCI.table.c.state==uci_states.SNAPSHOT_UCI, - model.UCI.table.c.state==uci_states.ADD_STORAGE_UCI ) ) \ - .all(): - uci_wrapper = UCIwrapper( r, self.app ) - new_requests.append( uci_wrapper ) - - for uci_wrapper in new_requests: - self.sa_session.expunge_all() - self.put( uci_wrapper ) - - def put( self, uci_wrapper ): - """Add a request to the queue.""" - self.provider.put( uci_wrapper ) - self.sleeper.wake() - - def shutdown( self ): - """Attempts to gracefully shut down the worker thread""" - if self.parent_pid != os.getpid(): - # We're not the real queue, do nothing - return - else: - log.info( "Sending stop signal to worker thread" ) - self.running = False - self.sleeper.wake() - log.info( "cloud manager stopped" ) - self.dispatcher.shutdown() - -class UCIwrapper( object ): - """ - Wraps 'model.UCI' with convenience methods for state management - """ - def __init__( self, uci, app ): - self.uci_id = uci.id - self.app = app - self.sa_session = self.app.model.context - base_directory = os.path.join( self.app.config.job_working_directory, "cloud" ) - self.working_directory = os.path.join( base_directory, str( self.uci_id ) ) -# log.debug( "Cloud controller working directory for UCI DB ID '%s': '%s'" % ( self.uci_id, self.working_directory ) ) - if not os.path.exists( base_directory ): - os.mkdir( base_directory ) - - - # --------- Setter methods ----------------- - - def change_state( self, uci_state=None, instance_id=None, i_state=None ): - """ - Sets state for UCI and/or UCI's instance with instance_id as provided by cloud provider and stored in local - Galaxy database. - Need to provide either: (1) state for the UCI, or (2) instance_id and it's state, or (3) all arguments. - """ -# log.debug( "Changing state - new uci_state: %s, instance_id: %s, i_state: %s" % ( uci_state, instance_id, i_state ) ) - if uci_state is not None: - uci = self.sa_session.query( model.UCI ).get( self.uci_id ) - self.sa_session.refresh( uci ) - uci.state = uci_state - self.sa_session.flush() - if ( instance_id is not None ) and ( i_state is not None ): - instance = self.sa_session.query( model.CloudInstance ).filter_by( uci_id=self.uci_id, instance_id=instance_id).first() - instance.state = i_state - self.sa_session.add( instance ) - self.sa_session.flush() - - def set_mi( self, i_index, mi_id ): - """ - Sets Machine Image (MI), e.g., 'ami-66fa190f', for UCI's instance with given index as it - is stored in local Galaxy database. - """ - mi = self.sa_session.query( model.CloudImage ).filter( model.CloudImage.table.c.image_id==mi_id ).first() - instance = self.sa_session.query( model.CloudInstance ).get( i_index ) - instance.image = mi - self.sa_session.add( instance ) - self.sa_session.flush() - - def set_key_pair( self, key_name, key_material=None ): - """ - Sets key pair value for current UCI. - """ - uci = self.sa_session.query( model.UCI ).get( self.uci_id ) - self.sa_session.refresh( uci ) - uci.key_pair_name = key_name - if key_material is not None: - uci.key_pair_material = key_material - self.sa_session.flush() - - def set_instance_launch_time( self, launch_time, i_index=None, i_id=None ): - """ - Stores launch time in local database for instance with specified index - i_index (as it is stored in local - Galaxy database) or with specified instance ID - i_id (as obtained from the cloud provider AND stored - in local Galaxy Database). Either 'i_index' or 'i_id' needs to be provided. - """ - if i_index != None: - instance = self.sa_session.query( model.CloudInstance ).get( i_index ) - elif i_id != None: - instance = self.sa_session.query( model.CloudInstance ).filter_by( uci_id=self.uci_id, instance_id=i_id).first() - else: - return None - - instance.launch_time = launch_time - self.sa_session.add( instance ) - self.sa_session.flush() - - def set_uci_launch_time( self, launch_time ): - uci = self.sa_session.query( model.UCI ).get( self.uci_id ) - self.sa_session.refresh( uci ) - uci.launch_time = launch_time - self.sa_session.add( uci ) - self.sa_session.flush() - - def set_stop_time( self, stop_time, i_index=None, i_id=None ): - """ - Stores stop time in local database for instance with specified index - i_index (as it is stored in local - Galaxy database) or with specified instance ID - i_id (as obtained from the cloud provider AND stored - in local Galaxy Database). Either 'i_index' or 'i_id' needs to be provided. - """ - if i_index != None: - instance = self.sa_session.query( model.CloudInstance ).get( i_index ) - elif i_id != None: - instance = self.sa_session.query( model.CloudInstance ).filter_by( uci_id=self.uci_id, instance_id=i_id).first() - else: - return None - - instance.stop_time = stop_time - self.sa_session.add( instance ) - self.sa_session.flush() - - def set_security_group_name( self, security_group_name, i_index=None, i_id=None ): - """ - Stores security group name in local database for instance with specified index - i_index (as it is stored in local - Galaxy database) or with specified instance ID - i_id (as obtained from the cloud provider AND stored - in local Galaxy Database). Either 'i_index' or 'i_id' needs to be provided. - """ - if i_index != None: - instance = self.sa_session.query( model.CloudInstance ).get( i_index ) - elif i_id != None: - instance = self.sa_session.query( model.CloudInstance ).filter_by( uci_id=self.uci_id, instance_id=i_id).first() - else: - return None - - instance.security_group = security_group_name - self.sa_session.add( instance ) - self.sa_session.flush() - - def set_reservation_id( self, i_index, reservation_id ): - instance = self.sa_session.query( model.CloudInstance ).get( i_index ) - instance.reservation_id = reservation_id - self.sa_session.add( instance ) - self.sa_session.flush() - - def set_instance_id( self, i_index, instance_id ): - """ - i_index refers to UCI's instance ID as stored in local database - instance_id refers to real-world, cloud resource ID (e.g., 'i-78hd823a') - """ - instance = self.sa_session.query( model.CloudInstance ).get( i_index ) - instance.instance_id = instance_id - self.sa_session.add( instance ) - self.sa_session.flush() - -# def set_public_dns( self, instance_id, public_dns ): -# uci = self.sa_session.query( model.UCI ).get( self.uci_id ) -# self.sa_session.refresh( uci ) -# uci.instance[instance_id].public_dns = public_dns -# uci.instance[instance_id].flush() -# -# def set_private_dns( self, instance_id, private_dns ): -# uci = self.sa_session.query( model.UCI ).get( self.uci_id ) -# self.sa_session.refresh( uci ) -# uci.instance[instance_id].private_dns = private_dns -# uci.instance[instance_id].flush() - - def reset_uci_launch_time( self ): - uci = self.sa_session.query( model.UCI ).get( self.uci_id ) - self.sa_session.refresh( uci ) - uci.launch_time = None - self.sa_session.add( uci ) - self.sa_session.flush() - - def set_error( self, error, set_state=False ): - """ - Sets error field of given UCI in local Galaxy database as well as any instances associated with - this UCI whose state is 'None' or 'SUBMITTED'. If set_state is set to 'true', - method also sets state of give UCI and corresponding instances to 'error' - """ - uci = self.sa_session.query( model.UCI ).get( self.uci_id ) - self.sa_session.refresh( uci ) - uci.error = error - if set_state: - uci.state = uci_states.ERROR - # Process all instances associated with this UCI - instances = self.sa_session.query( model.CloudInstance ) \ - .filter_by( uci=uci ) \ - .filter( or_( model.CloudInstance.table.c.state==None, model.CloudInstance.table.c.state==instance_states.SUBMITTED ) ) \ - .all() - for i in instances: - i.error = error - i.state = instance_states.ERROR - self.sa_session.add( i ) - self.sa_session.flush() - - self.sa_session.add( uci ) - self.sa_session.flush() - - def set_deleted( self ): - uci = self.sa_session.query( model.UCI ).get( self.uci_id ) - self.sa_session.refresh( uci ) - uci.state = uci_states.DELETED # for bookkeeping reasons, mark as deleted but don't actually delete. - uci.deleted = True - self.sa_session.add( uci ) - self.sa_session.flush() - -# def set_store_device( self, store_id, device ): -# uci = self.sa_session.query( model.UCI ).get( self.uci_id ) -# self.sa_session.refresh( uci ) -# uci.store[store_id].device = device -# uci.store[store_id].flush() - - def set_uci_total_size( self, total_size ): - uci = self.sa_session.query( model.UCI ).get( self.uci_id ) - self.sa_session.refresh( uci ) - uci.total_size = total_size - self.sa_session.add( uci ) - self.sa_session.flush() - - def set_store_error( self, error, store_index=None, store_id=None ): - if store_index != None: - store = self.sa_session.query( model.CloudStore ).get( store_index ) - elif store_id != None: - store = self.sa_session.query( model.CloudStore ).filter_by( volume_id = store_id ).first() - else: - return None - - store.error = error - store.status = store_status.ERROR - self.sa_session.add( store ) - self.sa_session.flush() - - def set_store_status( self, vol_id, status ): - vol = self.sa_session.query( model.CloudStore ).filter( model.CloudStore.table.c.volume_id == vol_id ).first() - vol.status = status - self.sa_session.add( vol ) - self.sa_session.flush() - - def set_store_availability_zone( self, availability_zone, vol_id=None ): - """ - Sets availability zone of storage volumes for either ALL volumes associated with current - UCI or for the volume whose volume ID (e.g., 'vol-39F80512') is provided as argument. - """ - if vol_id is not None: - vol = self.sa_session.query( model.CloudStore ).filter( model.CloudStore.table.c.volume_id == vol_id ).all() - else: - vol = self.sa_session.query( model.CloudStore ).filter( model.CloudStore.table.c.uci_id == self.uci_id ).all() - - for v in vol: - v.availability_zone = availability_zone - self.sa_session.add( v ) - self.sa_session.flush() - - def set_store_volume_id( self, store_index, volume_id ): - """ - Given store index as it is stored in local database, set volume ID as it is registered - on the cloud provider (e.g., vol-39890501) - """ - - if store_index != None: - store = self.sa_session.query( model.CloudStore ).get( store_index ) - store.volume_id = volume_id - self.sa_session.add( store ) - self.sa_session.flush() - else: - return None - -# uci = self.sa_session.query( model.UCI ).get( self.uci_id ) -# self.sa_session.refresh( uci ) -# uci.store[store_index].volume_id = volume_id -# #uci.store[store_index].flush() -# self.sa_session.add( uci ) -# self.sa_session.flush() - - def set_store_instance( self, vol_id, instance_id ): - """ - Stores instance ID that given store volume is attached to. Store volume ID should - be given in following format: 'vol-78943248' - """ - vol = self.sa_session.query( model.CloudStore ).filter( model.CloudStore.table.c.volume_id == vol_id ).first() - inst = self.sa_session.query( model.CloudInstance ).filter_by( instance_id=instance_id ).first() - vol.inst = inst - self.sa_session.add( vol ) - self.sa_session.flush() - - def set_store_device( self, vol_id, device ): - """ - Stores instance ID that given store volume is attached to. Store volume ID should - be given in following format: 'vol-78943248' - """ - vol = self.sa_session.query( model.CloudStore ).filter( model.CloudStore.table.c.volume_id == vol_id ).first() - vol.device = str( device ) - self.sa_session.add( vol ) - self.sa_session.flush() - - def set_store_deleted( self, vol_id, status=None ): - """ - Set storage volume as deleted in local database. Optionally, set the volume status too. - """ - vol = self.sa_session.query( model.CloudStore ).filter( model.CloudStore.table.c.volume_id == vol_id ).first() - vol.deleted = True - if status != None: - vol.status = status - self.sa_session.add( vol ) - self.sa_session.flush() - - def set_snapshot_id( self, snap_index, id ): - snap = model.CloudSnapshot.get( snap_index ) - - snap.snapshot_id = id - self.sa_session.add( snap ) - self.sa_session.flush() - - def set_snapshot_status( self, status, snap_index=None, snap_id=None ): - if snap_index != None: - snap = self.sa_session.query( model.CloudSnapshot ).get( snap_index ) - elif snap_id != None: - snap = self.sa_session.query( model.CloudSnapshot ).filter_by( snapshot_id = snap_id).first() - else: - return - snap.status = status - self.sa_session.add( snap ) - self.sa_session.flush() - - def set_snapshot_error( self, error, snap_index=None, snap_id=None, set_status=False ): - if snap_index != None: - snap = self.sa_session.query( model.CloudSnapshot ).get( snap_index ) - elif snap_id != None: - snap = self.sa_session.query( model.CloudSnapshot ).filter_by( snapshot_id = snap_id).first() - else: - return - snap.error = error - if set_status: - snap.status = snapshot_status.ERROR - - self.sa_session.add( snap ) - self.sa_session.flush() - - # --------- Getter methods ----------------- - - def get_provider_type( self ): - """ Returns type of cloud provider associated with given UCI. """ - uci = self.sa_session.query( model.UCI ).get( self.uci_id ) - self.sa_session.refresh( uci ) - return uci.credentials.provider.type - - def get_provider( self ): - """ Returns database object of cloud provider associated with credentials of given UCI. """ - uci = self.sa_session.query( model.UCI ).get( self.uci_id ) - self.sa_session.refresh( uci ) - return uci.credentials.provider - - def get_instance_type( self, i_index ): - instance = self.sa_session.query( model.CloudInstance ).get( i_index ) - self.sa_session.refresh( instance ) - return instance.type - - def get_uci_state( self ): - uci = self.sa_session.query( model.UCI ).get( self.uci_id ) - self.sa_session.refresh( uci ) - return uci.state - - def get_instances_indexes( self, state=None ): - """ - Returns indexes of instances associated with given UCI as they are stored in local Galaxy database and - whose state corresponds to passed argument. Returned values enable indexing instances from local Galaxy database. - """ - uci = self.sa_session.query( model.UCI ).get( self.uci_id ) - self.sa_session.refresh( uci ) - instances = self.sa_session.query( model.CloudInstance ) \ - .filter_by( uci=uci ) \ - .filter( model.CloudInstance.table.c.state==state ) \ - .all() - il = [] - for i in instances: - il.append( i.id ) - - return il - - def get_instance_state( self, instance_id ): - uci = self.sa_session.query( model.UCI ).get( self.uci_id ) - self.sa_session.refresh( uci ) - return uci.instance[instance_id].state - - def get_instaces_in_state( self, state ): - """ Get database objects of all instances associated with this UCI in given state. """ - return self.sa_session.query( model.CloudInstance ) \ - .filter_by( uci_id=self.uci_id, state = state ) \ - .all() - - def get_instances_ids( self ): - """ - Returns list IDs of all instances' associated with this UCI that are not in 'terminated' or - 'error' but the state is defined (i.e., state is not None) - (e.g., return value: ['i-402906D2', 'i-q0290dsD2'] ). - """ - il = self.sa_session.query( model.CloudInstance ) \ - .filter_by( uci_id=self.uci_id ) \ - .filter( or_( model.CloudInstance.table.c.state != 'terminated', - model.CloudInstance.table.c.state != 'error', - model.CloudInstance.table.c.state != None ) ) \ - .all() - instanceList = [] - for i in il: - instanceList.append( i.instance_id ) - return instanceList - - def get_name( self ): - uci = self.sa_session.query( model.UCI ).get( self.uci_id ) - self.sa_session.refresh( uci ) - return uci.name - - def get_key_pair_name( self ): - """ - Returns keypair name associated with given UCI. - """ - uci = self.sa_session.query( model.UCI ).get( self.uci_id ) - self.sa_session.refresh( uci ) - return uci.key_pair_name - - def get_key_pair_material( self ): - """ - Returns keypair material (i.e., private key) associated with given UCI. - """ - uci = self.sa_session.query( model.UCI ).get( self.uci_id ) - self.sa_session.refresh( uci ) - return uci.key_pair_material - - def get_security_group_name( self, i_index=None, i_id=None ): - """ - Given EITHER instance index as it is stored in local Galaxy database OR instance ID as it is - obtained from cloud provider and stored in local Galaxy database, return security group name associated - with given instance. - """ - if i_index != None: - instance = self.sa_session.query( model.CloudInstance ).get( i_index ) - return instance.security_group - elif i_id != None: - instance = self.sa_session.query( model.CloudInstance ).filter_by( uci_id=self.uci_id, instance_id=i_id).first() - return instance.security_group - - def get_access_key( self ): - uci = self.sa_session.query( model.UCI ).get( self.uci_id ) - self.sa_session.refresh( uci ) - return uci.credentials.access_key - - def get_secret_key( self ): - uci = self.sa_session.query( model.UCI ).get( self.uci_id ) - self.sa_session.refresh( uci ) - return uci.credentials.secret_key - - def get_mi_id( self, instance_id=0 ): - uci = self.sa_session.query( model.UCI ).get( self.uci_id ) - self.sa_session.refresh( uci ) - return uci.instance[instance_id].image.image_id - - def get_public_dns( self, instance_id=0 ): - uci = self.sa_session.query( model.UCI ).get( self.uci_id ) - self.sa_session.refresh( uci ) - return uci.instance[instance_id].public_dns - - def get_private_dns( self, instance_id=0 ): - uci = self.sa_session.query( model.UCI ).get( self.uci_id ) - self.sa_session.refresh( uci ) - return uci.instance[instance_id].private_dns - - def get_uci_availability_zone( self ): - """ - Returns UCI's availability zone. - Because all of storage volumes associated with a given UCI must be in the same - availability zone, availability of a UCI is determined by availability zone of - any one storage volume. - """ - uci = self.sa_session.query( model.UCI ).get( self.uci_id ) - self.sa_session.refresh( uci ) - return uci.store[0].availability_zone - - def get_uci_total_size( self ): - uci = self.sa_session.query( model.UCI ).get( self.uci_id ) - self.sa_session.refresh( uci ) - return uci.total_size - - def get_store_size( self, store_id=0 ): - uci = self.sa_session.query( model.UCI ).get( self.uci_id ) - self.sa_session.refresh( uci ) - return uci.store[store_id].size - - def get_store_volume_id( self, store_id=0 ): - """ - Given store ID associated with this UCI, get volume ID as it is registered - on the cloud provider (e.g., 'vol-39890501') - """ - uci = self.sa_session.query( model.UCI ).get( self.uci_id ) - self.sa_session.refresh( uci ) - return uci.store[store_id].volume_id - - def get_all_stores_in_status( self, status ): - """ - Return database objects of all stores associated with this UCI that have their - status set to value passed as parameter. - """ - return self.sa_session.query( model.CloudStore ).filter_by( deleted=False, uci_id=self.uci_id, status=status ).all() - - def get_all_stores( self ): - """ Returns all storage volumes' database objects associated with this UCI that have not been marked as 'deleted'. """ - return self.sa_session.query( model.CloudStore ) \ - .filter_by( deleted=False, uci_id=self.uci_id ) \ - .all() - - def get_snapshots( self, status=None ): - """ Returns database objects for all snapshots associated with this UCI and in given status.""" - return self.sa_session.query( model.CloudSnapshot ).filter_by( uci_id=self.uci_id, status=status ).all() - - def get_uci( self ): - """ Returns database object for given UCI. """ - uci = self.sa_session.query( model.UCI ).get( self.uci_id ) - self.sa_session.refresh( uci ) - return uci - - def get_uci_working_directory( self ): - return self.working_directory - - def uci_launch_time_set( self ): - uci = self.sa_session.query( model.UCI ).get( self.uci_id ) - self.sa_session.refresh( uci ) - return uci.launch_time - -class CloudProvider( object ): - def __init__( self, app ): - import providers.eucalyptus - import providers.ec2 - - self.app = app - self.cloud_provider = {} - self.cloud_provider["eucalyptus"] = providers.eucalyptus.EucalyptusCloudProvider( app ) - self.cloud_provider["ec2"] = providers.ec2.EC2CloudProvider( app ) - - def put( self, uci_wrapper ): - """ Put given request for UCI manipulation into provider's request queue.""" - self.cloud_provider[uci_wrapper.get_provider_type()].put( uci_wrapper ) - - def update( self ): - """ - Runs a global status update across all providers for all UCIs in state other than 'terminated' and 'available'. - Reason behind this method is to sync state of local DB and real world resources. - """ - for provider in self.cloud_provider.keys(): -# log.debug( "Running global update for provider: '%s'" % provider ) - self.cloud_provider[provider].update() - - def shutdown( self ): - for runner in self.cloud_provider.itervalues(): - runner.shutdown() - -class NoopCloudMonitor( object ): - """ - Implements the CloudMonitor interface but does nothing - """ - def put( self, *args ): - return - def shutdown( self ): - return -

1 0

galaxy-dist commit 03eb69c6e92e: Improvements to GOPS subtract: (a) preserve metadata for interval inputs; (b) allow arbitrary mix of interval and GFF inputs; and (c) functional tests updated to test new functionality.
by commits-noreply＠bitbucket.org 29 Jun '10

29 Jun '10

# HG changeset patch -- Bitbucket.org # Project galaxy-dist # URL http://bitbucket.org/galaxy/galaxy-dist/overview # User jeremy goecks <jeremy.goecks(a)emory.edu> # Date 1277304197 14400 # Node ID 03eb69c6e92ea5472b7627bd9990327f7546a8a7 # Parent 45e1d5636fffd284ea6cda2f14318b364370e985 Improvements to GOPS subtract: (a) preserve metadata for interval inputs; (b) allow arbitrary mix of interval and GFF inputs; and (c) functional tests updated to test new functionality. --- a/tools/new_operations/gops_subtract.py +++ b/tools/new_operations/gops_subtract.py @@ -8,7 +8,8 @@ usage: %prog bed_file_1 bed_file_2 out_f -2, --cols2=N,N,N,N: Columns for start, end, strand in second file -m, --mincols=N: Require this much overlap (default 1bp) -p, --pieces: just print pieces of second set (after padding) - -G, --gff: inputs are GFF format, meaning start and end coordinates are 1-based, closed interval + -G, --gff1: input 1 is GFF format, meaning start and end coordinates are 1-based, closed interval + -H, --gff2: input 2 is GFF format, meaning start and end coordinates are 1-based, closed interval """ from galaxy import eggs import pkg_resources @@ -35,24 +36,29 @@ def main(): chr_col_2, start_col_2, end_col_2, strand_col_2 = parse_cols_arg( options.cols2 ) if options.mincols: mincols = int( options.mincols ) pieces = bool( options.pieces ) - gff_format = bool( options.gff ) + in1_gff_format = bool( options.gff1 ) + in2_gff_format = bool( options.gff2 ) in_fname, in2_fname, out_fname = args except: doc_optparse.exception() - # Set reader to handle either GFF or default format. - if gff_format: - reader_wrapper = GFFReaderWrapper + # Set readers to handle either GFF or default format. + if in1_gff_format: + in1_reader_wrapper = GFFReaderWrapper else: - reader_wrapper = NiceReaderWrapper + in1_reader_wrapper = NiceReaderWrapper + if in2_gff_format: + in2_reader_wrapper = GFFReaderWrapper + else: + in2_reader_wrapper = NiceReaderWrapper - g1 = reader_wrapper( fileinput.FileInput( in_fname ), + g1 = in1_reader_wrapper( fileinput.FileInput( in_fname ), chrom_col=chr_col_1, start_col=start_col_1, end_col=end_col_1, strand_col=strand_col_1, fix_strand=True ) - g2 = reader_wrapper( fileinput.FileInput( in2_fname ), + g2 = in2_reader_wrapper( fileinput.FileInput( in2_fname ), chrom_col=chr_col_2, start_col=start_col_2, end_col=end_col_2, @@ -64,7 +70,7 @@ def main(): try: for line in subtract( [g1,g2], pieces=pieces, mincols=mincols ): if type( line ) is GenomicInterval: - if gff_format: + if in1_gff_format: line = convert_to_gff_coordinates( line ) out_file.write( "%s\n" % "\t".join( line.fields ) ) else: --- a/tools/new_operations/operation_filter.py +++ b/tools/new_operations/operation_filter.py @@ -28,13 +28,17 @@ def validate_input( trans, error_map, pa # check meta data try: param = param_values[name] - startCol = int( param.metadata.startCol ) - endCol = int( param.metadata.endCol ) - chromCol = int( param.metadata.chromCol ) - if param.metadata.strandCol is not None: - strandCol = int ( param.metadata.strandCol ) - else: - strandCol = 0 + if isinstance( param.datatype, trans.app.datatypes_registry.get_datatype_by_extension( 'gff' ).__class__ ): + # TODO: currently cannot validate GFF inputs b/c they are not derived from interval. + pass + else: # Validate interval datatype. + startCol = int( param.metadata.startCol ) + endCol = int( param.metadata.endCol ) + chromCol = int( param.metadata.chromCol ) + if param.metadata.strandCol is not None: + strandCol = int ( param.metadata.strandCol ) + else: + strandCol = 0 except: error_msg = "The attributes of this dataset are not properly set. " + \ "Click the pencil icon in the history item to set the chrom, start, end and strand columns." --- /dev/null +++ b/test-data/gops_subtract_in2.bed @@ -0,0 +1,500 @@ +chr13 3633323 3651020 transcript 0 + +chr13 3633323 3633421 exon 0 + +chr13 3641310 3641498 exon 0 + +chr13 3642720 3642871 exon 0 + +chr13 3644161 3644296 exon 0 + +chr13 3648564 3648756 exon 0 + +chr13 3649425 3651020 exon 0 + +chr13 3802138 3803564 transcript 0 - +chr13 3802138 3803564 exon 0 - +chr13 3881808 3892824 transcript 0 - +chr13 3881808 3883719 exon 0 - +chr13 3884048 3884235 exon 0 - +chr13 3885293 3885464 exon 0 - +chr13 3885783 3886041 exon 0 - +chr13 3886616 3886693 exon 0 - +chr13 3886816 3886913 exon 0 - +chr13 3887223 3887286 exon 0 - +chr13 3887600 3887768 exon 0 - +chr13 3888116 3888224 exon 0 - +chr13 3892538 3892824 exon 0 - +chr13 4131872 4149877 transcript 0 - +chr13 4131872 4132082 exon 0 - +chr13 4134448 4134531 exon 0 - +chr13 4135847 4136013 exon 0 - +chr13 4136379 4136489 exon 0 - +chr13 4141372 4141495 exon 0 - +chr13 4142463 4142541 exon 0 - +chr13 4143535 4143652 exon 0 - +chr13 4144451 4144619 exon 0 - +chr13 4149760 4149877 exon 0 - +chr13 4232985 4247605 transcript 0 + +chr13 4232985 4233100 exon 0 + +chr13 4235334 4235502 exon 0 + +chr13 4236281 4236398 exon 0 + +chr13 4237639 4237717 exon 0 + +chr13 4238213 4238336 exon 0 + +chr13 4241797 4241907 exon 0 + +chr13 4242172 4242338 exon 0 + +chr13 4246048 4246131 exon 0 + +chr13 4247323 4247605 exon 0 + +chr13 4247980 4249023 transcript 0 + +chr13 4247980 4249023 exon 0 + +chr13 4591735 4608410 transcript 0 - +chr13 4591735 4592544 exon 0 - +chr13 4592807 4592890 exon 0 - +chr13 4594316 4594400 exon 0 - +chr13 4594905 4594978 exon 0 - +chr13 4596730 4596828 exon 0 - +chr13 4598015 4598138 exon 0 - +chr13 4600463 4600541 exon 0 - +chr13 4601926 4602043 exon 0 - +chr13 4606636 4606804 exon 0 - +chr13 4608330 4608410 exon 0 - +chr13 5860734 5869639 transcript 0 + +chr13 5860734 5861088 exon 0 + +chr13 5864013 5864590 exon 0 + +chr13 5865884 5866008 exon 0 + +chr13 5866477 5869639 exon 0 + +chr13 3537320 3565507 transcript 0 + +chr13 3537320 3537589 exon 0 + +chr13 3548108 3548216 exon 0 + +chr13 3550234 3550334 exon 0 + +chr13 3553657 3553792 exon 0 + +chr13 3555559 3555758 exon 0 + +chr13 3556171 3556303 exon 0 + +chr13 3559237 3559337 exon 0 + +chr13 3561111 3561283 exon 0 + +chr13 3563785 3563930 exon 0 + +chr13 3564104 3564159 exon 0 + +chr13 3564241 3565507 exon 0 + +chr13 3565280 3610354 transcript 0 - +chr13 3565280 3565913 exon 0 - +chr13 3566163 3566278 exon 0 - +chr13 3566681 3566863 exon 0 - +chr13 3567997 3568103 exon 0 - +chr13 3568733 3568887 exon 0 - +chr13 3569557 3569683 exon 0 - +chr13 3572732 3576446 exon 0 - +chr13 3580997 3581439 exon 0 - +chr13 3583745 3584619 exon 0 - +chr13 3587543 3587780 exon 0 - +chr13 3589186 3589894 exon 0 - +chr13 3593376 3593439 exon 0 - +chr13 3593538 3593611 exon 0 - +chr13 3594742 3594846 exon 0 - +chr13 3596022 3596123 exon 0 - +chr13 3598897 3598997 exon 0 - +chr13 3599082 3599102 exon 0 - +chr13 3599184 3599308 exon 0 - +chr13 3599437 3599580 exon 0 - +chr13 3610035 3610354 exon 0 - +chr13 8202154 8759554 transcript 0 + +chr13 8202154 8202566 exon 0 + +chr13 8558349 8558436 exon 0 + +chr13 8568912 8569820 exon 0 + +chr13 8671650 8671765 exon 0 + +chr13 8696865 8697034 exon 0 + +chr13 8700825 8700977 exon 0 + +chr13 8712845 8713014 exon 0 + +chr13 8731035 8731217 exon 0 + +chr13 8751818 8751997 exon 0 + +chr13 8756471 8759554 exon 0 + +chr13 8884851 8891641 transcript 0 + +chr13 8884851 8885242 exon 0 + +chr13 8885977 8886150 exon 0 + +chr13 8886730 8886823 exon 0 + +chr13 8887168 8887299 exon 0 + +chr13 8889563 8891641 exon 0 + +chr13 8802213 8870288 transcript 0 - +chr13 8802213 8805192 exon 0 - +chr13 8819108 8819223 exon 0 - +chr13 8819656 8819791 exon 0 - +chr13 8834542 8834684 exon 0 - +chr13 8836028 8836269 exon 0 - +chr13 8841975 8842052 exon 0 - +chr13 8844032 8844077 exon 0 - +chr13 8846860 8846932 exon 0 - +chr13 8848779 8848915 exon 0 - +chr13 8852915 8852980 exon 0 - +chr13 8853245 8853341 exon 0 - +chr13 8856077 8856174 exon 0 - +chr13 8860333 8860511 exon 0 - +chr13 8870115 8870288 exon 0 - +chr13 8849724 8870288 transcript 0 - +chr13 8849724 8852980 exon 0 - +chr13 8853245 8853341 exon 0 - +chr13 8856077 8856174 exon 0 - +chr13 8860333 8860511 exon 0 - +chr13 8870115 8870288 exon 0 - +chr13 8971723 8995258 transcript 0 - +chr13 8971723 8972520 exon 0 - +chr13 8973409 8973553 exon 0 - +chr13 8974190 8974256 exon 0 - +chr13 8976481 8976679 exon 0 - +chr13 8977773 8977874 exon 0 - +chr13 8978674 8978726 exon 0 - +chr13 8979147 8979225 exon 0 - +chr13 8984454 8984565 exon 0 - +chr13 8984883 8984973 exon 0 - +chr13 8986494 8986560 exon 0 - +chr13 8987001 8987193 exon 0 - +chr13 8988298 8988391 exon 0 - +chr13 8989943 8990044 exon 0 - +chr13 8990960 8991097 exon 0 - +chr13 8991177 8991281 exon 0 - +chr13 8991890 8992061 exon 0 - +chr13 8995194 8995258 exon 0 - +chr13 6547402 6579395 transcript 0 + +chr13 6547402 6547475 exon 0 + +chr13 6548825 6548928 exon 0 + +chr13 6551879 6551986 exon 0 + +chr13 6552427 6552579 exon 0 + +chr13 6554599 6554714 exon 0 + +chr13 6554797 6554894 exon 0 + +chr13 6555787 6555948 exon 0 + +chr13 6557007 6557134 exon 0 + +chr13 6557401 6557490 exon 0 + +chr13 6558600 6558729 exon 0 + +chr13 6559275 6559389 exon 0 + +chr13 6559847 6559944 exon 0 + +chr13 6561198 6561333 exon 0 + +chr13 6562621 6562760 exon 0 + +chr13 6564236 6564353 exon 0 + +chr13 6566595 6566728 exon 0 + +chr13 6567909 6568030 exon 0 + +chr13 6568144 6568221 exon 0 + +chr13 6568445 6568611 exon 0 + +chr13 6569847 6569948 exon 0 + +chr13 6572286 6572407 exon 0 + +chr13 6573605 6573677 exon 0 + +chr13 6574262 6574375 exon 0 + +chr13 6576635 6576761 exon 0 + +chr13 6577686 6577832 exon 0 + +chr13 6578663 6578766 exon 0 + +chr13 6578895 6579395 exon 0 + +chr13 6579119 6647970 transcript 0 - +chr13 6579119 6580838 exon 0 - +chr13 6581648 6581751 exon 0 - +chr13 6583846 6583946 exon 0 - +chr13 6585725 6585837 exon 0 - +chr13 6586297 6586359 exon 0 - +chr13 6587734 6587899 exon 0 - +chr13 6597104 6597257 exon 0 - +chr13 6597984 6598072 exon 0 - +chr13 6599841 6599912 exon 0 - +chr13 6601958 6602105 exon 0 - +chr13 6602324 6602394 exon 0 - +chr13 6602644 6602709 exon 0 - +chr13 6604201 6604327 exon 0 - +chr13 6604881 6604974 exon 0 - +chr13 6613949 6614045 exon 0 - +chr13 6618420 6618529 exon 0 - +chr13 6618765 6618810 exon 0 - +chr13 6620131 6620297 exon 0 - +chr13 6621152 6621342 exon 0 - +chr13 6624381 6624459 exon 0 - +chr13 6635170 6635244 exon 0 - +chr13 6647816 6647970 exon 0 - +chr13 9093150 9172336 transcript 0 + +chr13 9093150 9093426 exon 0 + +chr13 9121353 9121472 exon 0 + +chr13 9123148 9123208 exon 0 + +chr13 9136024 9136172 exon 0 + +chr13 9136400 9136547 exon 0 + +chr13 9142999 9143078 exon 0 + +chr13 9144627 9144764 exon 0 + +chr13 9146646 9146750 exon 0 + +chr13 9149071 9149182 exon 0 + +chr13 9150110 9150164 exon 0 + +chr13 9150260 9150470 exon 0 + +chr13 9157368 9157475 exon 0 + +chr13 9157796 9158048 exon 0 + +chr13 9163920 9163966 exon 0 + +chr13 9165562 9165727 exon 0 + +chr13 9167923 9168048 exon 0 + +chr13 9168143 9168252 exon 0 + +chr13 9169897 9172336 exon 0 + +chr13 9275771 9668171 transcript 0 + +chr13 9275771 9276312 exon 0 + +chr13 9492350 9492422 exon 0 + +chr13 9505824 9505935 exon 0 + +chr13 9532497 9532623 exon 0 + +chr13 9535901 9536111 exon 0 + +chr13 9549429 9549564 exon 0 + +chr13 9550991 9551111 exon 0 + +chr13 9552519 9552717 exon 0 + +chr13 9559916 9560008 exon 0 + +chr13 9562359 9562470 exon 0 + +chr13 9567076 9567200 exon 0 + +chr13 9567546 9567656 exon 0 + +chr13 9567836 9567939 exon 0 + +chr13 9570295 9570360 exon 0 + +chr13 9574379 9574473 exon 0 + +chr13 9574607 9574727 exon 0 + +chr13 9576049 9576164 exon 0 + +chr13 9583250 9583267 exon 0 + +chr13 9591989 9592004 exon 0 + +chr13 9600631 9600742 exon 0 + +chr13 9603734 9603871 exon 0 + +chr13 9605568 9605777 exon 0 + +chr13 9608189 9608304 exon 0 + +chr13 9609942 9610144 exon 0 + +chr13 9613592 9613702 exon 0 + +chr13 9614990 9615071 exon 0 + +chr13 9621124 9621248 exon 0 + +chr13 9621827 9621949 exon 0 + +chr13 9623002 9623114 exon 0 + +chr13 9623203 9623313 exon 0 + +chr13 9627218 9627349 exon 0 + +chr13 9633967 9634136 exon 0 + +chr13 9636340 9636511 exon 0 + +chr13 9646224 9646286 exon 0 + +chr13 9653848 9653906 exon 0 + +chr13 9655882 9655957 exon 0 + +chr13 9658490 9658665 exon 0 + +chr13 9661382 9661506 exon 0 + +chr13 9665023 9668171 exon 0 + +chr13 9684081 9764454 transcript 0 - +chr13 9684081 9686230 exon 0 - +chr13 9688308 9688494 exon 0 - +chr13 9688697 9688970 exon 0 - +chr13 9689691 9689760 exon 0 - +chr13 9690148 9690356 exon 0 - +chr13 9692675 9692794 exon 0 - +chr13 9693514 9693592 exon 0 - +chr13 9694374 9694430 exon 0 - +chr13 9694984 9695072 exon 0 - +chr13 9696889 9696982 exon 0 - +chr13 9697873 9697951 exon 0 - +chr13 9720018 9720178 exon 0 - +chr13 9734577 9734712 exon 0 - +chr13 9764300 9764454 exon 0 - +chr13 9875858 10360049 transcript 0 - +chr13 9875858 9878263 exon 0 - +chr13 10027698 10027745 exon 0 - +chr13 10121472 10121535 exon 0 - +chr13 10223890 10224003 exon 0 - +chr13 10359509 10360049 exon 0 - +chr13 11645369 12199212 transcript 0 - +chr13 11645369 11646878 exon 0 - +chr13 11647697 11647749 exon 0 - +chr13 11648864 11648965 exon 0 - +chr13 11650313 11650378 exon 0 - +chr13 11652790 11652947 exon 0 - +chr13 11659114 11659249 exon 0 - +chr13 11660738 11660885 exon 0 - +chr13 11664512 11664573 exon 0 - +chr13 11670046 11670180 exon 0 - +chr13 11676402 11676445 exon 0 - +chr13 11677674 11677805 exon 0 - +chr13 11679482 11679704 exon 0 - +chr13 11680408 11680495 exon 0 - +chr13 11683491 11683639 exon 0 - +chr13 11685359 11685424 exon 0 - +chr13 11686948 11688246 exon 0 - +chr13 11689337 11689419 exon 0 - +chr13 11690772 11690877 exon 0 - +chr13 11694025 11694154 exon 0 - +chr13 11695951 11696040 exon 0 - +chr13 11697422 11697503 exon 0 - +chr13 11710873 11710947 exon 0 - +chr13 11711736 11711813 exon 0 - +chr13 11713581 11713661 exon 0 - +chr13 11715168 11715268 exon 0 - +chr13 11727791 11727845 exon 0 - +chr13 11730698 11730750 exon 0 - +chr13 11732540 11732644 exon 0 - +chr13 11736070 11736167 exon 0 - +chr13 11738759 11738872 exon 0 - +chr13 11739968 11740004 exon 0 - +chr13 11741996 11742131 exon 0 - +chr13 11746632 11746692 exon 0 - +chr13 11747884 11748055 exon 0 - +chr13 11749299 11749392 exon 0 - +chr13 11751216 11751304 exon 0 - +chr13 11752367 11752608 exon 0 - +chr13 11754332 11754653 exon 0 - +chr13 11758147 11758278 exon 0 - +chr13 11759345 11759427 exon 0 - +chr13 11761079 11761318 exon 0 - +chr13 11762195 11762256 exon 0 - +chr13 11768464 11768514 exon 0 - +chr13 11773232 11773354 exon 0 - +chr13 11776728 11776793 exon 0 - +chr13 11779209 11779325 exon 0 - +chr13 11780156 11780280 exon 0 - +chr13 11782558 11782634 exon 0 - +chr13 11784905 11784983 exon 0 - +chr13 11792514 11792652 exon 0 - +chr13 11793045 11793135 exon 0 - +chr13 11795763 11795842 exon 0 - +chr13 11797901 11798065 exon 0 - +chr13 11798848 11798989 exon 0 - +chr13 11800048 11800139 exon 0 - +chr13 11800288 11800509 exon 0 - +chr13 11802183 11802353 exon 0 - +chr13 11804401 11804522 exon 0 - +chr13 11806783 11806889 exon 0 - +chr13 11809207 11809394 exon 0 - +chr13 11810636 11810772 exon 0 - +chr13 11814017 11814121 exon 0 - +chr13 11814997 11815130 exon 0 - +chr13 11816529 11816644 exon 0 - +chr13 11819123 11819397 exon 0 - +chr13 11822609 11822753 exon 0 - +chr13 11823912 11824018 exon 0 - +chr13 11827737 11827938 exon 0 - +chr13 11829988 11830787 exon 0 - +chr13 11834120 11834347 exon 0 - +chr13 11837444 11837531 exon 0 - +chr13 11837863 11838026 exon 0 - +chr13 11839754 11839915 exon 0 - +chr13 11841702 11841817 exon 0 - +chr13 11842955 11843308 exon 0 - +chr13 11844462 11844671 exon 0 - +chr13 11851937 11852112 exon 0 - +chr13 11853493 11853702 exon 0 - +chr13 11862118 11862266 exon 0 - +chr13 11864686 11864846 exon 0 - +chr13 11871451 11871535 exon 0 - +chr13 11877304 11877408 exon 0 - +chr13 11882525 11882630 exon 0 - +chr13 11884935 11885152 exon 0 - +chr13 11886808 11887001 exon 0 - +chr13 11891948 11892190 exon 0 - +chr13 11893404 11893538 exon 0 - +chr13 11894073 11894192 exon 0 - +chr13 11903149 11903245 exon 0 - +chr13 11916540 11916676 exon 0 - +chr13 11919800 11919984 exon 0 - +chr13 11921823 11921945 exon 0 - +chr13 11926190 11926355 exon 0 - +chr13 11943324 11943481 exon 0 - +chr13 11945382 11945457 exon 0 - +chr13 11960383 11960480 exon 0 - +chr13 11961383 11961483 exon 0 - +chr13 11971741 11971854 exon 0 - +chr13 11975311 11975390 exon 0 - +chr13 11977730 11977805 exon 0 - +chr13 11983100 11983115 exon 0 - +chr13 11995484 11995505 exon 0 - +chr13 12010577 12010682 exon 0 - +chr13 12038149 12038269 exon 0 - +chr13 12198665 12199212 exon 0 - +chr13 12279085 12350267 transcript 0 - +chr13 12279085 12279358 exon 0 - +chr13 12280307 12280420 exon 0 - +chr13 12281613 12281806 exon 0 - +chr13 12282470 12282671 exon 0 - +chr13 12285880 12286077 exon 0 - +chr13 12287504 12287660 exon 0 - +chr13 12290179 12290255 exon 0 - +chr13 12291312 12291411 exon 0 - +chr13 12292463 12292545 exon 0 - +chr13 12297009 12297130 exon 0 - +chr13 12297556 12297624 exon 0 - +chr13 12304654 12304755 exon 0 - +chr13 12304937 12305045 exon 0 - +chr13 12307679 12307832 exon 0 - +chr13 12309076 12309166 exon 0 - +chr13 12313688 12313829 exon 0 - +chr13 12314421 12314538 exon 0 - +chr13 12317728 12317908 exon 0 - +chr13 12319950 12320136 exon 0 - +chr13 12323211 12323352 exon 0 - +chr13 12325168 12325281 exon 0 - +chr13 12327723 12327803 exon 0 - +chr13 12330247 12330315 exon 0 - +chr13 12331915 12331977 exon 0 - +chr13 12333831 12333932 exon 0 - +chr13 12336201 12336296 exon 0 - +chr13 12336821 12336884 exon 0 - +chr13 12339585 12339692 exon 0 - +chr13 12340159 12340252 exon 0 - +chr13 12342111 12342181 exon 0 - +chr13 12342868 12342958 exon 0 - +chr13 12346030 12346245 exon 0 - +chr13 12350128 12350267 exon 0 - +chr13 12361693 12432999 transcript 0 - +chr13 12361693 12361919 exon 0 - +chr13 12363044 12363203 exon 0 - +chr13 12364965 12365031 exon 0 - +chr13 12367344 12367491 exon 0 - +chr13 12368629 12368809 exon 0 - +chr13 12369665 12369800 exon 0 - +chr13 12371060 12371243 exon 0 - +chr13 12372695 12372836 exon 0 - +chr13 12374781 12374890 exon 0 - +chr13 12380773 12380924 exon 0 - +chr13 12382940 12383088 exon 0 - +chr13 12383926 12384157 exon 0 - +chr13 12386576 12386669 exon 0 - +chr13 12388741 12388827 exon 0 - +chr13 12393201 12393283 exon 0 - +chr13 12396580 12396659 exon 0 - +chr13 12398540 12398628 exon 0 - +chr13 12401166 12401253 exon 0 - +chr13 12401861 12401981 exon 0 - +chr13 12403113 12403228 exon 0 - +chr13 12432641 12432999 exon 0 - +chr13 12487641 12531160 transcript 0 + +chr13 12487641 12487737 exon 0 + +chr13 12488006 12488180 exon 0 + +chr13 12488688 12488905 exon 0 + +chr13 12491169 12491311 exon 0 + +chr13 12493316 12493418 exon 0 + +chr13 12493716 12493857 exon 0 + +chr13 12494980 12495192 exon 0 + +chr13 12495411 12495545 exon 0 + +chr13 12497154 12497257 exon 0 + +chr13 12498289 12498400 exon 0 + +chr13 12498791 12498909 exon 0 + +chr13 12499783 12499891 exon 0 + +chr13 12500920 12501016 exon 0 + +chr13 12501475 12501564 exon 0 + +chr13 12502662 12502874 exon 0 + +chr13 12503486 12503608 exon 0 + +chr13 12504311 12504503 exon 0 + +chr13 12505491 12505675 exon 0 + +chr13 12505778 12505915 exon 0 + +chr13 12506639 12506832 exon 0 + +chr13 12507683 12507853 exon 0 + +chr13 12508217 12508376 exon 0 + +chr13 12509726 12509969 exon 0 + +chr13 12510364 12510496 exon 0 + +chr13 12513308 12513412 exon 0 + +chr13 12513544 12513688 exon 0 + +chr13 12514307 12514426 exon 0 + +chr13 12514507 12514629 exon 0 + +chr13 12515900 12516029 exon 0 + +chr13 12516886 12517117 exon 0 + +chr13 12518420 12518547 exon 0 + +chr13 12519028 12519189 exon 0 + +chr13 12522033 12522080 exon 0 + +chr13 12522407 12522531 exon 0 + +chr13 12523177 12523318 exon 0 + +chr13 12523946 12524239 exon 0 + +chr13 12524895 12525046 exon 0 + +chr13 12525878 12526037 exon 0 + +chr13 12526183 12526318 exon 0 + +chr13 12526453 12526558 exon 0 + +chr13 12526640 12526809 exon 0 + +chr13 12527313 12527468 exon 0 + +chr13 12527936 12528095 exon 0 + +chr13 12530100 12530209 exon 0 + +chr13 12530855 12531160 exon 0 + +chr13 12531685 12553757 transcript 0 - +chr13 12531685 12533261 exon 0 - +chr13 12539413 12539579 exon 0 - +chr13 12540685 12540774 exon 0 - +chr13 12543667 12543694 exon 0 - +chr13 12544988 12545045 exon 0 - +chr13 12545551 12545671 exon 0 - +chr13 12547001 12547212 exon 0 - +chr13 12548610 12548699 exon 0 - +chr13 12551440 12551583 exon 0 - +chr13 12553693 12553757 exon 0 - +chr13 12569174 12612715 transcript 0 - +chr13 12569174 12570841 exon 0 - +chr13 12575866 12575912 exon 0 - --- a/tools/new_operations/subtract.xml +++ b/tools/new_operations/subtract.xml @@ -1,46 +1,33 @@ <tool id="gops_subtract_1" name="Subtract"><description>the intervals of two queries</description><command interpreter="python">gops_subtract.py - #if $inputs.type == "Interval": - $inputs.interval_input1 $inputs.interval_input2 $output - -1 ${inputs.interval_input1.metadata.chromCol},${inputs.interval_input1.metadata.startCol},${inputs.interval_input1.metadata.endCol},${inputs.interval_input1.metadata.strandCol} - -2 ${inputs.interval_input2.metadata.chromCol},${inputs.interval_input2.metadata.startCol},${inputs.interval_input2.metadata.endCol},${inputs.interval_input2.metadata.strandCol} - #else - $inputs.gff_input1 $inputs.gff_input2 $output - ## TODO: can we use metadata like above to set these columns rather than hardcode them? - -1 1,4,5,7 - -2 1,4,5,7 - --gff + $input1 $input2 $output + + ##if $input1.ext in ['gff','gtf','gff3']: + #if isinstance( $input1.datatype, $__app__.datatypes_registry.get_datatype_by_extension('gff').__class__): + -1 1,4,5,7 --gff1 + #else: + -1 ${input1.metadata.chromCol},${input1.metadata.startCol},${input1.metadata.endCol},${input1.metadata.strandCol} #end if + + #if isinstance( $input2.datatype, $__app__.datatypes_registry.get_datatype_by_extension('gff').__class__): + -2 1,4,5,7 --gff2 + #else: + -2 ${input2.metadata.chromCol},${input2.metadata.startCol},${input2.metadata.endCol},${input2.metadata.strandCol} + #end if + -m $min $returntype </command><inputs> - <conditional name="inputs"> - <param name="type" type="select" label="File Format to Use"> - <option value="Interval">Interval</option> - <option value="GFF">GFF</option> - </param> - <when value="Interval"> - <param format="interval" name="interval_input2" type="data" help="Second query"> - <label>Subtract</label> - </param> - - <param format="interval" name="interval_input1" type="data" help="First query"> - <label>from</label> - </param> - </when> - <when value="GFF"> - <param format="gff" name="gff_input2" type="data" help="Second query"> - <label>Subtract</label> - </param> - - <param format="gff" name="gff_input1" type="data" help="First query"> - <label>from</label> - </param> - </when> - </conditional> + <param format="interval,gff" name="input2" type="data" help="Second query"> + <label>Subtract</label> + </param> -<param name="returntype" type="select" label="Return" help="of the first query (see figure below)"> + <param format="interval,gff" name="input1" type="data" help="First query"> + <label>from</label> + </param> + + <param name="returntype" type="select" label="Return" help="of the first query (see figure below)"><option value="">Intervals with no overlap</option><option value="-p">Non-overlapping pieces of intervals</option></param> @@ -48,59 +35,57 @@ <param name="min" size="4" type="integer" value="1" help="(bp)"><label>where minimal overlap is</label></param> - </inputs><outputs> - <data format="input" name="output"> - #if inputs.type == "Interval": - metadata_source="inputs.interval_input1" - #else: - metadata_source="inputs.gff_input1" - #end if - </data> + <data format="input" name="output" metadata_source="input1"/></outputs><code file="operation_filter.py"/><tests><test> - <param name="type" value="Interval"/> - <param name="interval_input1" value="1.bed" /> - <param name="interval_input2" value="2.bed" /> + <param name="input1" value="1.bed" /> + <param name="input2" value="2.bed" /><param name="min" value="1" /><param name="returntype" value="" /><output name="output" file="gops-subtract.dat" /></test><test> - <param name="type" value="Interval"/> - <param name="interval_input1" value="1.bed" /> - <param name="interval_input2" value="2_mod.bed" ftype="interval"/> + <param name="input1" value="1.bed" /> + <param name="input2" value="2_mod.bed" ftype="interval"/><param name="min" value="1" /><param name="returntype" value="" /><output name="output" file="gops_subtract_diffCols.dat" /></test><test> - <param name="type" value="Interval"/> - <param name="interval_input1" value="gops_subtract_bigint.bed" /> - <param name="interval_input2" value="2.bed" /> + <param name="input1" value="gops_subtract_bigint.bed" /> + <param name="input2" value="2.bed" /><param name="min" value="1" /><param name="returntype" value="" /><output name="output" file="gops-subtract.dat" /></test><test> - <param name="type" value="Interval"/> - <param name="interval_input1" value="1.bed" /> - <param name="interval_input2" value="2.bed" /> + <param name="input1" value="1.bed" /> + <param name="input2" value="2.bed" /><param name="min" value="10" /><param name="returntype" value="Non-overlapping pieces of intervals" /><output name="output" file="gops-subtract-p.dat" /></test> + <test> - <param name="type" value="GFF"/> - <param name="gff_input1" value="gops_subtract_in1.gff" /> - <param name="gff_input2" value="gops_subtract_in2.gff" /> + <param name="input1" value="gops_subtract_in1.gff" /> + <param name="input2" value="gops_subtract_in2.gff" /><param name="min" value="1" /><param name="returntype" value="" /> - <output name="output" file="gops_subtract_out1.gff" /> + <output name="output" file="gops_subtract_out1.gff" /></test> +  + <test> + <param name="input1" value="gops_subtract_in1.gff" /> + <param name="input2" value="gops_subtract_in2.bed" /> + <param name="min" value="1" /> + <param name="returntype" value="" /> + <output name="output" file="gops_subtract_out1.gff" /> + </test> + </tests><help>

1 0

galaxy-dist commit 45e1d5636fff: Fixed typo in jobs.actions.post
by commits-noreply＠bitbucket.org 29 Jun '10

29 Jun '10

# HG changeset patch -- Bitbucket.org # Project galaxy-dist # URL http://bitbucket.org/galaxy/galaxy-dist/overview # User Dannon Baker <dannon.baker(a)emory.edu> # Date 1277253455 14400 # Node ID 45e1d5636fffd284ea6cda2f14318b364370e985 # Parent a579674852103600ecd8e8b2549e76abc3917791 Fixed typo in jobs.actions.post --- a/lib/galaxy/jobs/actions/post.py +++ b/lib/galaxy/jobs/actions/post.py @@ -82,7 +82,7 @@ class EmailAction(DefaultJobAction): s.close() except Exception, e: log.error("EmailAction PJA Failed, exception: %s" % e) -L + @classmethod def get_config_form(cls, trans): form = """

1 0

galaxy-dist commit 040be2b94b8c: Added logging to EmailAction. Should gracefully fail if email is not available.
by commits-noreply＠bitbucket.org 29 Jun '10

29 Jun '10

# HG changeset patch -- Bitbucket.org # Project galaxy-dist # URL http://bitbucket.org/galaxy/galaxy-dist/overview # User Dannon Baker <dannon.baker(a)emory.edu> # Date 1277252544 14400 # Node ID 040be2b94b8ce845278d5ac19139a4bdae8fb088 # Parent beb30aadccd75bfdec6b06d4afcd704c2c16dbc3 Added logging to EmailAction. Should gracefully fail if email is not available. --- a/lib/galaxy/jobs/actions/post.py +++ b/lib/galaxy/jobs/actions/post.py @@ -67,7 +67,7 @@ class EmailAction(DefaultJobAction): def execute(cls, trans, action, job): smtp_server = trans.app.config.smtp_server if smtp_server is None: - return trans.show_error_message( "Mail is not configured for this galaxy instance, workflow action aborted." ) + log.error("Mail is not configured for this galaxy instance. Workflow action aborted.") # Build the email message msg = MIMEText( "Your job '%s' at Galaxy instance %s is complete as of %s." % (job.history.name, trans.request.host, job.update_time)) msg[ 'To' ] = job.user.email @@ -78,9 +78,8 @@ class EmailAction(DefaultJobAction): s.connect( smtp_server ) s.sendmail( frm, [ to ], msg.as_string() ) s.close() - return trans.show_ok_message( "Your error report has been sent" ) except Exception, e: - return trans.show_error_message( "An error occurred sending the report by email: %s" % str( e ) ) + log.error("EmailAction PJA Failed, exception: %s" % e) @classmethod def get_config_form(cls, trans):

1 0

galaxy-dist commit 752cb3e32537: Improvements to GOPS intersect: (a) preserve metadata for interval inputs; (b) allow arbitrary mix of interval and GFF inputs; and (c) functional tests updated to test new functionality.
by commits-noreply＠bitbucket.org 29 Jun '10

29 Jun '10

# HG changeset patch -- Bitbucket.org # Project galaxy-dist # URL http://bitbucket.org/galaxy/galaxy-dist/overview # User jeremy goecks <jeremy.goecks(a)emory.edu> # Date 1277306218 14400 # Node ID 752cb3e325374381bdd0db6b9163b12ba88b7151 # Parent 03eb69c6e92ea5472b7627bd9990327f7546a8a7 Improvements to GOPS intersect: (a) preserve metadata for interval inputs; (b) allow arbitrary mix of interval and GFF inputs; and (c) functional tests updated to test new functionality. --- a/tools/new_operations/intersect.xml +++ b/tools/new_operations/intersect.xml @@ -1,131 +1,110 @@ <tool id="gops_intersect_1" name="Intersect"><description>the intervals of two queries</description><command interpreter="python">gops_intersect.py - #if $inputs.type == "Interval": - $inputs.interval_input1 $inputs.interval_input2 $output - -1 ${inputs.interval_input1.metadata.chromCol},${inputs.interval_input1.metadata.startCol},${inputs.interval_input1.metadata.endCol},${inputs.interval_input1.metadata.strandCol} - -2 ${inputs.interval_input2.metadata.chromCol},${inputs.interval_input2.metadata.startCol},${inputs.interval_input2.metadata.endCol},${inputs.interval_input2.metadata.strandCol} - $inputs.interval_returntype - #else - $inputs.gff_input1 $inputs.gff_input2 $output - ## TODO: can we use metadata like above to set these columns rather than hardcode them? - -1 1,4,5,7 - -2 1,4,5,7 - --gff - $inputs.gff_returntype + $input1 $input2 $output + + #if isinstance( $input1.datatype, $__app__.datatypes_registry.get_datatype_by_extension('gff').__class__): + -1 1,4,5,7 --gff1 + #else: + -1 ${input1.metadata.chromCol},${input1.metadata.startCol},${input1.metadata.endCol},${input1.metadata.strandCol} #end if - -m $min + + #if isinstance( $input2.datatype, $__app__.datatypes_registry.get_datatype_by_extension('gff').__class__): + -2 1,4,5,7 --gff2 + #else: + -2 ${input2.metadata.chromCol},${input2.metadata.startCol},${input2.metadata.endCol},${input2.metadata.strandCol} + #end if + + -m $min $returntype </command><inputs> - <conditional name="inputs"> - <param name="type" type="select" label="File Format to Use"> - <option value="Interval">Interval</option> - <option value="GFF">GFF</option> - </param> - <when value="Interval"> - <param name="interval_returntype" type="select" label="Return" help="(see figure below)"> - <option value="">Overlapping Intervals</option> - <option value="-p">Overlapping pieces of Intervals</option> - </param> - <param format="interval" name="interval_input1" type="data" help="First query"> - <label>of</label> - </param> - <param format="interval" name="interval_input2" type="data" help="Second query"> - <label>that intersect</label> - </param> - </when> - <when value="GFF"> - <param name="gff_returntype" type="select" label="Return" help="(see figure below)"> - <option value="">Overlapping Intervals</option> - <option value="-p">Overlapping pieces of Intervals</option> - </param> - <param format="gff" name="gff_input1" type="data" help="First query"> - <label>of</label> - </param> - <param format="gff" name="gff_input2" type="data" help="Second query"> - <label>that intersect</label> - </param> - </when> - </conditional> + <param name="returntype" type="select" label="Return" help="(see figure below)"> + <option value="">Overlapping Intervals</option> + <option value="-p">Overlapping pieces of Intervals</option> + </param> + <param format="interval,gff" name="input1" type="data" help="First query"> + <label>of</label> + </param> + <param format="interval,gff" name="input2" type="data" help="Second query"> + <label>that intersect</label> + </param><param name="min" size="4" type="integer" value="1" help="(bp)"><label>for at least</label></param></inputs><outputs> - <data format="input" name="output"> - #if inputs.type == "Interval": - metadata_source="inputs.interval_input1" - #else: - metadata_source="inputs.gff_input1" - #end if - </data> + <data format="input" name="output" metadata_source="input1"/></outputs><code file="operation_filter.py"/><tests><test><param name="type" value="Interval"/> - <param name="interval_input1" value="1.bed" /> - <param name="interval_input2" value="2.bed" /> + <param name="input1" value="1.bed" /> + <param name="input2" value="2.bed" /><param name="min" value="1" /> - <param name="interval_returntype" value="" /> + <param name="returntype" value="" /><output name="output" file="gops_intersect_out.bed" /></test><test><param name="type" value="Interval"/> - <param name="interval_input1" value="1.bed" /> - <param name="interval_input2" value="2_mod.bed" ftype="interval"/> + <param name="input1" value="1.bed" /> + <param name="input2" value="2_mod.bed" ftype="interval"/><param name="min" value="1" /> - <param name="interval_returntype" value="" /> + <param name="returntype" value="" /><output name="output" file="gops_intersect_diffCols.bed" /></test><test> - <param name="type" value="Interval"/> - <param name="interval_input1" value="1.bed" /> - <param name="interval_input2" value="2_mod.bed" ftype="interval"/> + <param name="input1" value="1.bed" /> + <param name="input2" value="2_mod.bed" ftype="interval"/><param name="min" value="1" /> - <param name="interval_returntype" value="Overlapping pieces of Intervals" /> + <param name="returntype" value="Overlapping pieces of Intervals" /><output name="output" file="gops_intersect_p_diffCols.bed" /></test><test> - <param name="type" value="Interval"/> - <param name="interval_input1" value="1.bed" /> - <param name="interval_input2" value="2.bed" /> + <param name="input1" value="1.bed" /> + <param name="input2" value="2.bed" /><param name="min" value="10" /> - <param name="interval_returntype" value="Overlapping pieces of Intervals" /> + <param name="returntype" value="Overlapping pieces of Intervals" /><output name="output" file="gops_intersect_p_out.bed" /></test><test> - <param name="type" value="Interval"/> - <param name="interval_input1" value="gops_bigint.interval" ftype="interval" /> - <param name="interval_input2" value="gops_bigint2.interval" ftype="interval" /> + <param name="input1" value="gops_bigint.interval" ftype="interval" /> + <param name="input2" value="gops_bigint2.interval" ftype="interval" /><param name="min" value="1" /> - <param name="interval_returntype" value="" /> + <param name="returntype" value="" /><output name="output" file="gops_intersect_bigint_out.interval" /></test><test> - <param name="type" value="Interval"/> - <param name="interval_input1" value="gops_bigint2.interval" ftype="interval" /> - <param name="interval_input2" value="gops_bigint.interval" ftype="interval" /> + <param name="input1" value="gops_bigint2.interval" ftype="interval" /> + <param name="input2" value="gops_bigint.interval" ftype="interval" /><param name="min" value="1" /> - <param name="interval_returntype" value="" /> + <param name="returntype" value="" /><output name="output" file="gops_intersect_bigint_out.interval" /></test><test> - <param name="type" value="Interval"/> - <param name="interval_input1" value="12.bed" ftype="bed" /> - <param name="interval_input2" value="1.bed" ftype="bed" /> + <param name="input1" value="12.bed" ftype="bed" /> + <param name="input2" value="1.bed" ftype="bed" /><param name="min" value="1" /> - <param name="interval_returntype" value="" /> + <param name="returntype" value="" /><output name="output" file="gops_intersect_no_strand_out.bed" /></test> + <test> - <param name="type" value="GFF"/> - <param name="gff_input1" value="gops_subtract_in1.gff" /> - <param name="gff_input2" value="gops_subtract_in2.gff" /> + <param name="input1" value="gops_subtract_in1.gff" /> + <param name="input2" value="gops_subtract_in2.gff" /><param name="min" value="1" /> - <param name="gff_returntype" value="" /> + <param name="returntype" value="" /><output name="output" file="gops_intersect_out2.gff" /></test> +  + <test> + <param name="input1" value="gops_subtract_in1.gff" /> + <param name="input2" value="gops_subtract_in2.bed" /> + <param name="min" value="1" /> + <param name="returntype" value="" /> + <output name="output" file="gops_intersect_out2.gff" /> + </test> + </tests><help> --- a/tools/new_operations/subtract.xml +++ b/tools/new_operations/subtract.xml @@ -3,7 +3,6 @@ <command interpreter="python">gops_subtract.py $input1 $input2 $output - ##if $input1.ext in ['gff','gtf','gff3']: #if isinstance( $input1.datatype, $__app__.datatypes_registry.get_datatype_by_extension('gff').__class__): -1 1,4,5,7 --gff1 #else: --- a/tools/new_operations/gops_intersect.py +++ b/tools/new_operations/gops_intersect.py @@ -1,13 +1,14 @@ #!/usr/bin/env python """ -Find regions of first bed file that overlap regions in a second bed file +Find regions of first interval/GFF file that overlap regions in a second interval/GFF file usage: %prog bed_file_1 bed_file_2 out_file -1, --cols1=N,N,N,N: Columns for start, end, strand in first file -2, --cols2=N,N,N,N: Columns for start, end, strand in second file -m, --mincols=N: Require this much overlap (default 1bp) -p, --pieces: just print pieces of second set (after padding) - -G, --gff: inputs are GFF format, meaning start and end coordinates are 1-based, closed interval + -G, --gff1: input 1 is GFF format, meaning start and end coordinates are 1-based, closed interval + -H, --gff2: input 2 is GFF format, meaning start and end coordinates are 1-based, closed interval """ from galaxy import eggs import pkg_resources @@ -34,24 +35,29 @@ def main(): chr_col_2, start_col_2, end_col_2, strand_col_2 = parse_cols_arg( options.cols2 ) if options.mincols: mincols = int( options.mincols ) pieces = bool( options.pieces ) - gff_format = bool( options.gff ) + in1_gff_format = bool( options.gff1 ) + in2_gff_format = bool( options.gff2 ) in_fname, in2_fname, out_fname = args except: doc_optparse.exception() - # Set reader to handle either GFF or default format. - if gff_format: - reader_wrapper = GFFReaderWrapper + # Set readers to handle either GFF or default format. + if in1_gff_format: + in1_reader_wrapper = GFFReaderWrapper else: - reader_wrapper = NiceReaderWrapper + in1_reader_wrapper = NiceReaderWrapper + if in2_gff_format: + in2_reader_wrapper = GFFReaderWrapper + else: + in2_reader_wrapper = NiceReaderWrapper - g1 = reader_wrapper( fileinput.FileInput( in_fname ), + g1 = in1_reader_wrapper( fileinput.FileInput( in_fname ), chrom_col=chr_col_1, start_col=start_col_1, end_col=end_col_1, strand_col=strand_col_1, fix_strand=True ) - g2 = reader_wrapper( fileinput.FileInput( in2_fname ), + g2 = in2_reader_wrapper( fileinput.FileInput( in2_fname ), chrom_col=chr_col_2, start_col=start_col_2, end_col=end_col_2, @@ -63,7 +69,7 @@ def main(): try: for line in intersect( [g1,g2], pieces=pieces, mincols=mincols ): if type( line ) == GenomicInterval: - if gff_format: + if in1_gff_format: line = convert_to_gff_coordinates( line ) out_file.write( "%s\n" % "\t".join( line.fields ) ) else:

1 0

galaxy-dist commit a57967485210: EmailAction now comes from 'galaxy-noreply@<host>'.
by commits-noreply＠bitbucket.org 29 Jun '10

29 Jun '10

# HG changeset patch -- Bitbucket.org # Project galaxy-dist # URL http://bitbucket.org/galaxy/galaxy-dist/overview # User Dannon Baker <dannon.baker(a)emory.edu> # Date 1277253193 14400 # Node ID a579674852103600ecd8e8b2549e76abc3917791 # Parent 040be2b94b8ce845278d5ac19139a4bdae8fb088 EmailAction now comes from 'galaxy-noreply@<host>'. --- a/lib/galaxy/jobs/actions/post.py +++ b/lib/galaxy/jobs/actions/post.py @@ -69,9 +69,11 @@ class EmailAction(DefaultJobAction): if smtp_server is None: log.error("Mail is not configured for this galaxy instance. Workflow action aborted.") # Build the email message + frm = 'galaxy-noreply@%s' % trans.request.host + to = job.user.email msg = MIMEText( "Your job '%s' at Galaxy instance %s is complete as of %s." % (job.history.name, trans.request.host, job.update_time)) - msg[ 'To' ] = job.user.email - msg[ 'From' ] = job.user.email + msg[ 'To' ] = to + msg[ 'From' ] = frm msg[ 'Subject' ] = "Galaxy workflow step notification '%s'" try: s = smtplib.SMTP() @@ -80,7 +82,7 @@ class EmailAction(DefaultJobAction): s.close() except Exception, e: log.error("EmailAction PJA Failed, exception: %s" % e) - +L @classmethod def get_config_form(cls, trans): form = """

1 0

galaxy-dist commit 764ecc0cf3fd: Refactor code that builds recent tools list to make it easier to understand and more efficient.
by commits-noreply＠bitbucket.org 29 Jun '10

29 Jun '10

# HG changeset patch -- Bitbucket.org # Project galaxy-dist # URL http://bitbucket.org/galaxy/galaxy-dist/overview # User jeremy goecks <jeremy.goecks(a)emory.edu> # Date 1277168327 14400 # Node ID 764ecc0cf3fdc4885de777f26e1ae7c06a85acbf # Parent 7d3a9212b15e82327c2af7c3222995f358532d60 Refactor code that builds recent tools list to make it easier to understand and more efficient. --- a/lib/galaxy/web/controllers/root.py +++ b/lib/galaxy/web/controllers/root.py @@ -38,8 +38,9 @@ class RootController( BaseController, Us filter( self.app.model.Job.user==trans.user ). \ order_by( self.app.model.Job.create_time.desc() ): tool_id = row[0] - if tool_id in toolbox.tools_by_id: - recent_tools.append( toolbox.tools_by_id[tool_id] ) + a_tool = toolbox.tools_by_id.get( tool_id, None ) + if a_tool and a_tool not in recent_tools: + recent_tools.append( a_tool ) ## TODO: make number of recently used tools a user preference. if len ( recent_tools ) == 5: break

1 0

galaxy-dist commit 464734ed09ad: Initial implementation of the Galaxy Web API. Disabled unless
by commits-noreply＠bitbucket.org 29 Jun '10

29 Jun '10

# HG changeset patch -- Bitbucket.org # Project galaxy-dist # URL http://bitbucket.org/galaxy/galaxy-dist/overview # User Nate Coraor <nate(a)bx.psu.edu> # Date 1277226628 14400 # Node ID 464734ed09ad4b36afc3409b5d975732b1c6eb21 # Parent 4b4385fbf52982ee3bf6394c5cb269e7bf3ab912 Initial implementation of the Galaxy Web API. Disabled unless 'enable_api = True' in config file. You should not enable the API on production sites as this code is brand new and may contain serious bugs and security flaws! Implemented: * Display libraries * Display library info * Display library contents * Display library content info * Create library folders * Upload datasets to a library from a server directory or with a path paste. * Basic example scripts in scripts/api/ Framework changes that were made to support this: * API Key interface in User Preferences. * New api_keys database table for storing users' API Keys. * New API-specific route mapper in webapp. * API controllers in galaxy.web.api * Return handling in reused library_common methods. * expose_api decorator for API controller methods validates key and ensures valid JSON format. * UniverseWebTransaction renamed to GalaxyWebTransaction and subclassed for GalaxyWebUITransaction and GalaxyWebAPITransaction. Things that need to be done next: * Documentation! * Refactor reused code from library_common and other controllers into an even-more-generic location and format. The main changes are that the Web UI returns redirects and rendered templates, whereas the API returns various HTTP status codes and JSON. * Implement more functionality. * The request and response format should be considered alpha and are subject to change. They will be standardized as the API matures. Hints to get started can be found in scripts/api/README --- /dev/null +++ b/lib/galaxy/web/api/libraries.py @@ -0,0 +1,62 @@ +""" +API operations on a library. +""" +import logging, os, string, shutil, urllib, re, socket +from cgi import escape, FieldStorage +from galaxy import util, datatypes, jobs, web, util +from galaxy.web.base.controller import * +from galaxy.util.sanitize_html import sanitize_html +from galaxy.model.orm import * + +log = logging.getLogger( __name__ ) + +class LibrariesController( BaseController ): + + @web.expose_api + def index( self, trans, **kwd ): + """ + GET /api/libraries + Displays a collection (list) of libraries. + """ + query = trans.sa_session.query( trans.app.model.Library ).filter( trans.app.model.Library.table.c.deleted == False ) + current_user_role_ids = [ role.id for role in trans.get_current_user_roles() ] + library_access_action = trans.app.security_agent.permitted_actions.LIBRARY_ACCESS.action + restricted_library_ids = [ lp.library_id for lp in trans.sa_session.query( trans.model.LibraryPermissions ) \ + .filter( trans.model.LibraryPermissions.table.c.action == library_access_action ) \ + .distinct() ] + accessible_restricted_library_ids = [ lp.library_id for lp in trans.sa_session.query( trans.model.LibraryPermissions ) \ + .filter( and_( trans.model.LibraryPermissions.table.c.action == library_access_action, + trans.model.LibraryPermissions.table.c.role_id.in_( current_user_role_ids ) ) ) ] + query = query.filter( or_( not_( trans.model.Library.table.c.id.in_( restricted_library_ids ) ), + trans.model.Library.table.c.id.in_( accessible_restricted_library_ids ) ) ) + rval = [] + for library in query: + item = library.get_api_value() + item['url'] = url_for( 'library', id=trans.security.encode_id( library.id ) ) + item['id'] = trans.security.encode_id( item['id'] ) + rval.append( item ) + return rval + + @web.expose_api + def show( self, trans, id, **kwd ): + """ + GET /api/libraries/{encoded_library_id} + Displays information about a library. + """ + library_id = id + params = util.Params( kwd ) + try: + decoded_library_id = trans.security.decode_id( library_id ) + except TypeError: + trans.response.status = 400 + return "Malformed library id ( %s ) specified, unable to decode." % str( library_id ) + try: + library = trans.sa_session.query( trans.app.model.Library ).get( decoded_library_id ) + except: + library = None + if not library or not ( trans.user_is_admin() or trans.app.security_agent.can_access_library( trans.get_current_user_roles(), library ) ): + trans.response.status = 400 + return "Invalid library id ( %s ) specified." % str( library_id ) + item = library.get_api_value( view='element' ) + item['contents_url'] = url_for( 'contents', library_id=library_id ) + return item --- /dev/null +++ b/scripts/api/common.py @@ -0,0 +1,94 @@ +import os, sys, urllib, urllib2 + +new_path = [ os.path.join( os.path.dirname( __file__ ), '..', '..', 'lib' ) ] +new_path.extend( sys.path[1:] ) +sys.path = new_path + +from galaxy import eggs +import pkg_resources + +pkg_resources.require( "simplejson" ) +import simplejson + +def make_url( api_key, url, args=None ): + # Adds the API Key to the URL if it's not already there. + if args is None: + args = [] + argsep = '&' + if '?' not in url: + argsep = '?' + if '?key=' not in url and '&key=' not in url: + args.insert( 0, ( 'key', api_key ) ) + return url + argsep + '&'.join( [ '='.join( t ) for t in args ] ) + +def get( api_key, url ): + # Do the actual GET. + url = make_url( api_key, url ) + return simplejson.loads( urllib2.urlopen( url ).read() ) + +def post( api_key, url, data ): + # Do the actual POST. + url = make_url( api_key, url ) + req = urllib2.Request( url, headers = { 'Content-Type': 'application/json' }, data = simplejson.dumps( data ) ) + return simplejson.loads( urllib2.urlopen( req ).read() ) + +def display( api_key, url ): + # Sends an API GET request and acts as a generic formatter for the JSON response. + try: + r = get( api_key, url ) + except urllib2.HTTPError, e: + print e + print e.read( 1024 ) # Only return the first 1K of errors. + sys.exit( 1 ) + if type( r ) == unicode: + print 'error: %s' % r + return None + elif type( r ) == list: + # Response is a collection as defined in the REST style. + print 'Collection Members' + print '------------------' + for i in r: + # All collection members should have a name and url in the response. + print i.pop( 'url' ) + print ' name: %s' % i.pop( 'name' ) + for k, v in i.items(): + print ' %s: %s' % ( k, v ) + print '' + print '%d elements in collection' % len( r ) + elif type( r ) == dict: + # Response is an element as defined in the REST style. + print 'Member Information' + print '------------------' + for k, v in r.items(): + print '%s: %s' % ( k, v ) + else: + print 'response is unknown type: %s' % type( r ) + +def submit( api_key, url, data ): + # Sends an API POST request and acts as a generic formatter for the JSON response. + # 'data' will become the JSON payload read by Galaxy. + try: + r = post( api_key, url, data ) + except urllib2.HTTPError, e: + print e + print e.read( 1024 ) + sys.exit( 1 ) + print 'Response' + print '--------' + if type( r ) == list: + # Currently the only implemented responses are lists of dicts, because + # submission creates some number of collection elements. + for i in r: + if type( i ) == dict: + if 'url' in i: + print i.pop( 'url' ) + else: + print '----' + if 'name' in i: + print ' name: %s' % i.pop( 'name' ) + for k, v in i.items(): + print ' %s: %s' % ( k, v ) + else: + print i + else: + print r --- a/lib/galaxy/web/framework/__init__.py +++ b/lib/galaxy/web/framework/__init__.py @@ -30,6 +30,7 @@ from babel.support import Translations pkg_resources.require( "SQLAlchemy >= 0.4" ) from sqlalchemy import and_ +from sqlalchemy.orm.exc import NoResultFound pkg_resources.require( "pexpect" ) pkg_resources.require( "amqplib" ) @@ -87,6 +88,43 @@ def require_login( verb="perform this ac return decorator return argcatcher +def expose_api( func ): + def decorator( self, trans, *args, **kwargs ): + def error( environ, start_response ): + start_response( error_status, [('Content-type', 'text/plain')] ) + return error_message + error_status = '403 Forbidden' + if 'key' not in kwargs: + error_message = 'No API key provided with request, please consult the API documentation.' + return error + try: + provided_key = trans.sa_session.query( trans.app.model.APIKeys ).filter( trans.app.model.APIKeys.table.c.key == kwargs['key'] ).one() + except NoResultFound: + error_message = 'Provided API key is not valid.' + return error + newest_key = provided_key.user.api_keys[0] + if newest_key.key != provided_key.key: + error_message = 'Provided API key has expired.' + return error + if trans.request.body: + try: + payload = util.recursively_stringify_dictionary_keys( simplejson.loads( trans.request.body ) ) + kwargs['payload'] = payload + except ValueError: + error_status = '400 Bad Request' + error_message = 'Your request did not appear to be valid JSON, please consult the API documentation' + return error + trans.response.set_content_type( "application/json" ) + trans.set_user( provided_key.user ) + if trans.debug: + return simplejson.dumps( func( self, trans, *args, **kwargs ), indent=4, sort_keys=True ) + else: + return simplejson.dumps( func( self, trans, *args, **kwargs ) ) + if not hasattr(func, '_orig'): + decorator._orig = func + decorator.exposed = True + return decorator + def require_admin( func ): def decorator( self, trans, *args, **kwargs ): admin_users = trans.app.config.get( "admin_users", "" ).split( "," ) @@ -119,7 +157,7 @@ def form( *args, **kwargs ): class WebApplication( base.WebApplication ): def __init__( self, galaxy_app, session_cookie='galaxysession' ): base.WebApplication.__init__( self ) - self.set_transaction_factory( lambda e: UniverseWebTransaction( e, galaxy_app, self, session_cookie ) ) + self.set_transaction_factory( lambda e: self.transaction_chooser( e, galaxy_app, session_cookie ) ) # Mako support self.mako_template_lookup = mako.lookup.TemplateLookup( directories = [ galaxy_app.config.template_path ] , @@ -135,21 +173,21 @@ class WebApplication( base.WebApplicatio if isinstance( body, FormBuilder ): body = trans.show_form( body ) return base.WebApplication.make_body_iterable( self, trans, body ) + def transaction_chooser( self, environ, galaxy_app, session_cookie ): + if 'is_api_request' in environ: + return GalaxyWebAPITransaction( environ, galaxy_app, self ) + else: + return GalaxyWebUITransaction( environ, galaxy_app, self, session_cookie ) -class UniverseWebTransaction( base.DefaultWebTransaction ): +class GalaxyWebTransaction( base.DefaultWebTransaction ): """ - Encapsulates web transaction specific state for the Universe application + Encapsulates web transaction specific state for the Galaxy application (specifically the user's "cookie" session and history) """ - def __init__( self, environ, app, webapp, session_cookie ): + def __init__( self, environ, app, webapp ): self.app = app self.webapp = webapp self.security = webapp.security - # FIXME: the following 3 attributes are not currently used - # Remove them if they are not going to be... - self.__user = NOT_SET - self.__history = NOT_SET - self.__galaxy_session = NOT_SET base.DefaultWebTransaction.__init__( self, environ ) self.setup_i18n() self.sa_session.expunge_all() @@ -158,13 +196,6 @@ class UniverseWebTransaction( base.Defau # that the current history should not be used for parameter values # and such). self.workflow_building_mode = False - # Always have a valid galaxy session - self.__ensure_valid_session( session_cookie ) - # Prevent deleted users from accessing Galaxy - if self.app.config.use_remote_user and self.galaxy_session.user.deleted: - self.response.send_redirect( url_for( '/static/user_disabled.html' ) ) - if self.app.config.require_login: - self.__ensure_logged_in_user( environ ) def setup_i18n( self ): if 'HTTP_ACCEPT_LANGUAGE' in self.environ: # locales looks something like: ['en', 'en-us;q=0.7', 'ja;q=0.3'] @@ -252,12 +283,7 @@ class UniverseWebTransaction( base.Defau tstamp = time.localtime ( time.time() + 3600 * 24 * age ) self.response.cookies[name]['expires'] = time.strftime( '%a, %d-%b-%Y %H:%M:%S GMT', tstamp ) self.response.cookies[name]['version'] = version - #@property - #def galaxy_session( self ): - # if not self.__galaxy_session: - # self.__ensure_valid_session() - # return self.__galaxy_session - def __ensure_valid_session( self, session_cookie ): + def _ensure_valid_session( self, session_cookie ): """ Ensure that a valid Galaxy session exists and is available as trans.session (part of initialization) @@ -344,7 +370,7 @@ class UniverseWebTransaction( base.Defau # If the old session was invalid, get a new history with our new session if invalidate_existing_session: self.new_history() - def __ensure_logged_in_user( self, environ ): + def _ensure_logged_in_user( self, environ ): allowed_paths = ( url_for( controller='root', action='index' ), url_for( controller='root', action='tool_menu' ), @@ -537,15 +563,6 @@ class UniverseWebTransaction( base.Defau self.sa_session.add_all( ( self.galaxy_session, history ) ) self.sa_session.flush() return history - def get_user( self ): - """Return the current user if logged in or None.""" - return self.galaxy_session.user - def set_user( self, user ): - """Set the current user.""" - self.galaxy_session.user = user - self.sa_session.add( self.galaxy_session ) - self.sa_session.flush() - user = property( get_user, set_user ) def get_current_user_roles( self ): user = self.get_user() if user: @@ -735,6 +752,53 @@ class FormInput( object ): self.help = help self.use_label = use_label +class GalaxyWebAPITransaction( GalaxyWebTransaction ): + def __init__( self, environ, app, webapp ): + GalaxyWebTransaction.__init__( self, environ, app, webapp ) + self.__user = None + self._ensure_valid_session( None ) + def _ensure_valid_session( self, session_cookie ): + self.galaxy_session = Bunch() + self.galaxy_session.history = self.galaxy_session.current_history = Bunch() + self.galaxy_session.history.genome_build = None + self.galaxy_session.is_api = True + def get_user( self ): + """Return the current user (the expose_api decorator ensures that it is set).""" + return self.__user + def set_user( self, user ): + """Compatibility method""" + self.__user = user + user = property( get_user, set_user ) + @property + def db_builds( self ): + dbnames = [] + if 'dbkeys' in self.user.preferences: + user_keys = from_json_string( self.user.preferences['dbkeys'] ) + for key, chrom_dict in user_keys.iteritems(): + dbnames.append((key, "%s (%s) [Custom]" % (chrom_dict['name'], key) )) + dbnames.extend( util.dbnames ) + return dbnames + +class GalaxyWebUITransaction( GalaxyWebTransaction ): + def __init__( self, environ, app, webapp, session_cookie ): + GalaxyWebTransaction.__init__( self, environ, app, webapp ) + # Always have a valid galaxy session + self._ensure_valid_session( session_cookie ) + # Prevent deleted users from accessing Galaxy + if self.app.config.use_remote_user and self.galaxy_session.user.deleted: + self.response.send_redirect( url_for( '/static/user_disabled.html' ) ) + if self.app.config.require_login: + self._ensure_logged_in_user( environ ) + def get_user( self ): + """Return the current user if logged in or None.""" + return self.galaxy_session.user + def set_user( self, user ): + """Set the current user.""" + self.galaxy_session.user = user + self.sa_session.add( self.galaxy_session ) + self.sa_session.flush() + user = property( get_user, set_user ) + class SelectInput( FormInput ): """ A select form input. """ def __init__( self, name, label, value=None, options=[], error=None, help=None, use_label=True ): --- a/lib/galaxy/web/buildapp.py +++ b/lib/galaxy/web/buildapp.py @@ -48,6 +48,27 @@ def add_controllers( webapp, app ): if isclass( T ) and T is not BaseController and issubclass( T, BaseController ): webapp.add_controller( name, T( app ) ) +def add_api_controllers( webapp, app ): + from galaxy.web.base.controller import BaseController + from galaxy.web.base.controller import ControllerUnavailable + import galaxy.web.api + controller_dir = galaxy.web.api.__path__[0] + for fname in os.listdir( controller_dir ): + if not( fname.startswith( "_" ) ) and fname.endswith( ".py" ): + name = fname[:-3] + module_name = "galaxy.web.api." + name + try: + module = __import__( module_name ) + except ControllerUnavailable, exc: + log.debug("%s could not be loaded: %s" % (module_name, str(exc))) + continue + for comp in module_name.split( "." )[1:]: + module = getattr( module, comp ) + for key in dir( module ): + T = getattr( module, key ) + if isclass( T ) and T is not BaseController and issubclass( T, BaseController ): + webapp.add_api_controller( name, T( app ) ) + def app_factory( global_conf, **kwargs ): """ Return a wsgi application serving the root object @@ -80,6 +101,11 @@ def app_factory( global_conf, **kwargs ) webapp.add_route( '/u/:username/h/:slug', controller='history', action='display_by_username_and_slug' ) webapp.add_route( '/u/:username/w/:slug', controller='workflow', action='display_by_username_and_slug' ) webapp.add_route( '/u/:username/v/:slug', controller='visualization', action='display_by_username_and_slug' ) + # If enabled, add the web API + if asbool( kwargs.get( 'enable_api', False ) ): + add_api_controllers( webapp, app ) + webapp.api_mapper.resource( 'content', 'contents', path_prefix='/api/libraries/:library_id', parent_resources=dict( member_name='library', collection_name='libraries' ) ) + webapp.api_mapper.resource( 'library', 'libraries', path_prefix='/api' ) webapp.finalize_config() # Wrap the webapp in some useful middleware if kwargs.get( 'middleware', True ): --- /dev/null +++ b/scripts/api/README @@ -0,0 +1,106 @@ +This is not documentation. These are hints and examples to get you started +until the documentation is written. + +Set these options in universe_wsgi.ini and start the server: + +enable_api = True +admin_users = you(a)example.org +library_import_dir = /path/to/some/directory + +In the directory you specified for 'library_import_dir', create some +subdirectories, and put (or symlink) files to import into Galaxy into those +subdirectories. + +In Galaxy, create an account that matches the address you put in 'admin_users', +then browse to that user's preferences and generate a new API Key. Copy the +key to your clipboard. Create a new library (doing this via the API is not yet +implemented). Then take your API Key and use the scripts in scripts/api/ to do +things: + +% ./display.py my_key http://localhost:4096/api/libraries +Collection Members +------------------ +/api/libraries/f3f73e481f432006 + name: api_test + id: f3f73e481f432006 + +% ./display.py my_key http://localhost:4096/api/libraries/f3f73e481f432006 +Member Information +------------------ +synopsys: None +contents_url: /api/libraries/f3f73e481f432006/contents +description: API Test Library +name: api_test + +% ./display.py my_key http://localhost:4096/api/libraries/f3f73e481f432006/contents +Collection Members +------------------ +/api/libraries/f3f73e481f432006/contents/28202595c0d2591f61ddda595d2c3670 + name: / + type: folder + id: 28202595c0d2591f61ddda595d2c3670 + +% ./library_create_folder.py my_key http://localhost:4096/api/libraries/f3f73e481f432006/contents 28202595c0d2591f61ddda595d2c3670 api_test_folder1 'API Test Folder 1' +Response +-------- +/api/libraries/f3f73e481f432006/contents/28202595c0d2591fa4f9089d2303fd89 + name: api_test_folder1 + id: 28202595c0d2591fa4f9089d2303fd89 + +% ./library_upload_from_import_dir.py my_key http://localhost:4096/api/libraries/f3f73e481f432006/contents 28202595c0d2591fa4f9089d2303fd89 bed bed hg19 +Response +-------- +/api/libraries/f3f73e481f432006/contents/e9ef7fdb2db87d7b + name: 2.bed + id: e9ef7fdb2db87d7b +/api/libraries/f3f73e481f432006/contents/3b7f6a31f80a5018 + name: 3.bed + id: 3b7f6a31f80a5018 + +% ./display.py my_key http://localhost:4096/api/libraries/f3f73e481f432006/contents +Collection Members +------------------ +/api/libraries/f3f73e481f432006/contents/28202595c0d2591f61ddda595d2c3670 + name: / + type: folder + id: 28202595c0d2591f61ddda595d2c3670 +/api/libraries/f3f73e481f432006/contents/28202595c0d2591fa4f9089d2303fd89 + name: /api_test_folder1 + type: folder + id: 28202595c0d2591fa4f9089d2303fd89 +/api/libraries/f3f73e481f432006/contents/e9ef7fdb2db87d7b + name: /api_test_folder1/2.bed + type: file + id: e9ef7fdb2db87d7b +/api/libraries/f3f73e481f432006/contents/3b7f6a31f80a5018 + name: /api_test_folder1/3.bed + type: file + id: 3b7f6a31f80a5018 + +% ./display.py my_key http://localhost:4096/api/libraries/f3f73e481f432006/contents/e9ef7fdb2db87… +Member Information +------------------ +misc_blurb: 68 regions +metadata_endCol: 3 +data_type: bed +metadata_columns: 6 +metadata_nameCol: 4 +uploaded_by: nate@... +metadata_strandCol: 6 +name: 2.bed +genome_build: hg19 +metadata_comment_lines: None +metadata_startCol: 2 +metadata_chromCol: 1 +file_size: 4272 +metadata_data_lines: 68 +message: +metadata_dbkey: hg19 +misc_info: uploaded bed file +date_uploaded: 2010-06-22T17:01:51.266119 +metadata_column_types: str, int, int, str, int, str + +Other parameters are valid when uploading, they are the same parameters as are +used in the web form, like 'link_data_only' and etc. + +The request and response format should be considered alpha and are subject to change. --- a/lib/galaxy/web/framework/base.py +++ b/lib/galaxy/web/framework/base.py @@ -45,7 +45,13 @@ class WebApplication( object ): and `__call__` to handle a request (WSGI style). """ self.controllers = dict() + self.api_controllers = dict() self.mapper = routes.Mapper() + # FIXME: The following two options are deprecated and should be + # removed. Consult the Routes documentation. + self.mapper.minimization = True + self.mapper.explicit = False + self.api_mapper = routes.Mapper() self.transaction_factory = DefaultWebTransaction def add_controller( self, controller_name, controller ): """ @@ -56,6 +62,10 @@ class WebApplication( object ): log.debug( "Enabling '%s' controller, class: %s", controller_name, controller.__class__.__name__ ) self.controllers[ controller_name ] = controller + def add_api_controller( self, controller_name, controller ): + log.debug( "Enabling '%s' API controller, class: %s", + controller_name, controller.__class__.__name__ ) + self.api_controllers[ controller_name ] = controller def add_route( self, route, **kwargs ): """ Add a route to match a URL with a method. Accepts all keyword @@ -80,6 +90,7 @@ class WebApplication( object ): """ # Create/compile the regular expressions for route mapping self.mapper.create_regs( self.controllers.keys() ) + self.api_mapper.create_regs( self.api_controllers.keys() ) def __call__( self, environ, start_response ): """ Call interface as specified by WSGI. Wraps the environment in user @@ -88,12 +99,20 @@ class WebApplication( object ): """ # Map url using routes path_info = environ.get( 'PATH_INFO', '' ) - map = self.mapper.match( path_info ) + map = self.mapper.match( path_info, environ ) + if map is None: + environ[ 'is_api_request' ] = True + map = self.api_mapper.match( path_info, environ ) + mapper = self.api_mapper + controllers = self.api_controllers + else: + mapper = self.mapper + controllers = self.controllers if map == None: raise httpexceptions.HTTPNotFound( "No route for " + path_info ) # Setup routes rc = routes.request_config() - rc.mapper = self.mapper + rc.mapper = mapper rc.mapper_dict = map rc.environ = environ # Setup the transaction @@ -101,7 +120,7 @@ class WebApplication( object ): rc.redirect = trans.response.send_redirect # Get the controller class controller_name = map.pop( 'controller', None ) - controller = self.controllers.get( controller_name, None ) + controller = controllers.get( controller_name, None ) if controller_name is None: raise httpexceptions.HTTPNotFound( "No controller for " + path_info ) # Resolve action method on controller @@ -112,7 +131,7 @@ class WebApplication( object ): if method is None: raise httpexceptions.HTTPNotFound( "No action for " + path_info ) # Is the method exposed - if not getattr( method, 'exposed', False ): + if not getattr( method, 'exposed', False ): raise httpexceptions.HTTPNotFound( "Action not exposed for " + path_info ) # Is the method callable if not callable( method ): --- a/lib/galaxy/web/security/__init__.py +++ b/lib/galaxy/web/security/__init__.py @@ -43,6 +43,8 @@ class SecurityHelper( object ): return self.id_cipher.encrypt( s ).encode( 'hex' ) def decode_id( self, obj_id ): return int( self.id_cipher.decrypt( obj_id.decode( 'hex' ) ).lstrip( "!" ) ) + def decode_string_id( self, obj_id ): + return self.id_cipher.decrypt( obj_id.decode( 'hex' ) ).lstrip( "!" ) def encode_guid( self, session_key ): # Session keys are strings # Pad to a multiple of 8 with leading "!" --- a/lib/galaxy/config.py +++ b/lib/galaxy/config.py @@ -39,6 +39,8 @@ class Configuration( object ): self.file_path = resolve_path( kwargs.get( "file_path", "database/files" ), self.root ) self.new_file_path = resolve_path( kwargs.get( "new_file_path", "database/tmp" ), self.root ) self.cookie_path = kwargs.get( "cookie_path", "/" ) + # web API + self.enable_api = string_as_bool( kwargs.get( 'enable_api', False ) ) # dataset Track files self.track_store_path = kwargs.get( "track_store_path", "${extra_files_path}/tracks") self.tool_path = resolve_path( kwargs.get( "tool_path", "tools" ), self.root ) --- /dev/null +++ b/lib/galaxy/model/migrate/versions/0049_api_keys_table.py @@ -0,0 +1,38 @@ +""" +Migration script to add the api_keys table. +""" + +from sqlalchemy import * +from migrate import * +from migrate.changeset import * +from galaxy.model.custom_types import * + +import datetime +now = datetime.datetime.utcnow + +import logging +log = logging.getLogger( __name__ ) + +metadata = MetaData( migrate_engine ) + +APIKeys_table = Table( "api_keys", metadata, + Column( "id", Integer, primary_key=True ), + Column( "create_time", DateTime, default=now ), + Column( "user_id", Integer, ForeignKey( "galaxy_user.id" ), index=True ), + Column( "key", TrimmedString( 32 ), index=True, unique=True ) ) + +def upgrade(): + print __doc__ + metadata.reflect() + try: + APIKeys_table.create() + except Exception, e: + log.debug( "Creating api_keys table failed: %s" % str( e ) ) + +def downgrade(): + # Load existing tables + metadata.reflect() + try: + APIKeys_table.drop() + except Exception, e: + log.debug( "Dropping api_keys table failed: %s" % str( e ) ) --- a/lib/galaxy/tools/actions/upload_common.py +++ b/lib/galaxy/tools/actions/upload_common.py @@ -124,7 +124,7 @@ def new_history_upload( trans, uploaded_ return hda def new_library_upload( trans, cntrller, uploaded_dataset, library_bunch, state=None ): current_user_roles = trans.get_current_user_roles() - if not ( cntrller in [ 'library_admin' ] or trans.app.security_agent.can_add_library_item( current_user_roles, library_bunch.folder ) ): + if not ( ( trans.user_is_admin() and cntrller in [ 'library_admin', 'api' ] ) or trans.app.security_agent.can_add_library_item( current_user_roles, library_bunch.folder ) ): # This doesn't have to be pretty - the only time this should happen is if someone's being malicious. raise Exception( "User is not authorized to add datasets to this library." ) folder = library_bunch.folder --- /dev/null +++ b/lib/galaxy/web/api/contents.py @@ -0,0 +1,154 @@ +""" +API operations on the contents of a library. +""" +import logging, os, string, shutil, urllib, re, socket +from cgi import escape, FieldStorage +from galaxy import util, datatypes, jobs, web, util +from galaxy.web.base.controller import * +from galaxy.util.sanitize_html import sanitize_html +from galaxy.model.orm import * + +log = logging.getLogger( __name__ ) + +class ContentsController( BaseController ): + + @web.expose_api + def index( self, trans, library_id, **kwd ): + """ + GET /api/libraries/{encoded_library_id}/contents + Displays a collection (list) of library contents (files and folders). + """ + rval = [] + current_user_roles = trans.get_current_user_roles() + def traverse( folder ): + admin = trans.user_is_admin() + rval = [] + for subfolder in folder.active_folders: + if not admin: + can_access, folder_ids = trans.app.security_agent.check_folder_contents( trans.user, current_user_roles, subfolder ) + if admin or can_access: + subfolder.api_path = folder.api_path + '/' + subfolder.name + subfolder.api_type = 'folder' + rval.append( subfolder ) + rval.extend( traverse( subfolder ) ) + for ld in folder.datasets: + if not admin: + can_access = trans.app.security_agent.can_access_dataset( current_user_roles, ld.library_dataset_dataset_association.dataset ) + if admin or can_access: + ld.api_path = folder.api_path + '/' + ld.name + ld.api_type = 'file' + rval.append( ld ) + return rval + try: + decoded_library_id = trans.security.decode_id( library_id ) + except TypeError: + trans.response.status = 400 + return "Malformed library id ( %s ) specified, unable to decode." % str( library_id ) + try: + library = trans.sa_session.query( trans.app.model.Library ).get( decoded_library_id ) + except: + library = None + if not library or not ( trans.user_is_admin() or trans.app.security_agent.can_access_library( current_user_roles, library ) ): + trans.response.status = 400 + return "Invalid library id ( %s ) specified." % str( library_id ) + encoded_id = trans.security.encode_id( 'folder.%s' % library.root_folder.id ) + rval.append( dict( id = encoded_id, + type = 'folder', + name = '/', + url = url_for( 'content', library_id=library_id, id=encoded_id ) ) ) + library.root_folder.api_path = '' + for content in traverse( library.root_folder ): + encoded_id = trans.security.encode_id( '%s.%s' % ( content.api_type, content.id ) ) + rval.append( dict( id = encoded_id, + type = content.api_type, + name = content.api_path, + url = url_for( 'content', library_id=library_id, id=encoded_id, ) ) ) + return rval + + @web.expose_api + def show( self, trans, id, library_id, **kwd ): + """ + GET /api/libraries/{encoded_library_id}/contents/{encoded_content_type_and_id} + Displays information about a library content (file or folder). + """ + content_id = id + try: + decoded_type_and_id = trans.security.decode_string_id( content_id ) + content_type, decoded_content_id = decoded_type_and_id.split( '.' ) + except: + trans.response.status = 400 + return "Malformed content id ( %s ) specified, unable to decode." % str( content_id ) + if content_type == 'folder': + model_class = trans.app.model.LibraryFolder + elif content_type == 'file': + model_class = trans.app.model.LibraryDataset + else: + trans.response.status = 400 + return "Invalid type ( %s ) specified." % str( content_type ) + try: + content = trans.sa_session.query( model_class ).get( decoded_content_id ) + except: + content = None + if not content or ( not trans.user_is_admin() and not trans.app.security_agent.can_access_library_item( trans.get_current_user_roles(), content, trans.user ) ): + trans.response.status = 400 + return "Invalid %s id ( %s ) specified." % ( content_type, str( content_id ) ) + return content.get_api_value( view='element' ) + + @web.expose_api + def create( self, trans, library_id, payload, **kwd ): + """ + POST /api/libraries/{encoded_library_id}/contents + Creates a new library content item (file or folder). + """ + create_type = None + if 'create_type' not in payload: + trans.response.status = 400 + return "Missing required 'create_type' parameter. Please consult the API documentation for help." + else: + create_type = payload.pop( 'create_type' ) + if create_type not in ( 'file', 'folder' ): + trans.response.status = 400 + return "Invalid value for 'create_type' parameter ( %s ) specified. Please consult the API documentation for help." % create_type + try: + content_id = str( payload.pop( 'folder_id' ) ) + decoded_type_and_id = trans.security.decode_string_id( content_id ) + parent_type, decoded_parent_id = decoded_type_and_id.split( '.' ) + assert parent_type in ( 'folder', 'file' ) + except: + trans.response.status = 400 + return "Malformed parent id ( %s ) specified, unable to decode." % content_id + # "content" can be either a folder or a file, but the parent of new contents can only be folders. + if parent_type == 'file': + trans.response.status = 400 + try: + # With admins or people who can access the dataset provided as the parent, be descriptive. + dataset = trans.sa_session.query( trans.app.model.LibraryDataset ).get( decoded_parent_id ).library_dataset_dataset_association.dataset + assert trans.user_is_admin() or trans.app.security_agent.can_access_dataset( trans.get_current_user_roles(), dataset ) + return "The parent id ( %s ) points to a file, not a folder." % content_id + except: + # If you can't access the parent we don't want to reveal its existence. + return "Invalid parent folder id ( %s ) specified." % content_id + # The rest of the security happens in the library_common controller. + folder_id = trans.security.encode_id( decoded_parent_id ) + # Now create the desired content object, either file or folder. + if create_type == 'file': + status, output = trans.webapp.controllers['library_common'].upload_library_dataset( trans, 'api', library_id, folder_id, **payload ) + elif create_type == 'folder': + status, output = trans.webapp.controllers['library_common'].create_folder( trans, 'api', folder_id, library_id, **payload ) + if status != 200: + trans.response.status = status + # We don't want to reveal the encoded folder_id since it's invalid + # in the API context. Instead, return the content_id originally + # supplied by the client. + output = output.replace( folder_id, content_id ) + return output + else: + rval = [] + for k, v in output.items(): + if type( v ) == trans.app.model.LibraryDatasetDatasetAssociation: + v = v.library_dataset + encoded_id = trans.security.encode_id( create_type + '.' + str( v.id ) ) + rval.append( dict( id = encoded_id, + name = v.name, + url = url_for( 'content', library_id=library_id, id=encoded_id ) ) ) + return rval --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -829,6 +829,8 @@ class HistoryDatasetAssociationDisplayAt class Library( object ): permitted_actions = get_permitted_actions( filter='LIBRARY' ) + api_collection_visible_keys = ( 'id', 'name' ) + api_element_visible_keys = ( 'name', 'description', 'synopsys' ) def __init__( self, name=None, description=None, synopsis=None, root_folder=None ): self.name = name or "Unnamed library" self.description = description @@ -870,8 +872,21 @@ class Library( object ): if isinstance( name, str ): name = unicode( name, 'utf-8' ) return name + def get_api_value( self, view='collection' ): + rval = {} + try: + visible_keys = self.__getattribute__( 'api_' + view + '_visible_keys' ) + except AttributeError: + raise Exception( 'Unknown API view: %s' % view ) + for key in visible_keys: + try: + rval[key] = self.__getattribute__( key ) + except AttributeError: + rval[key] = None + return rval class LibraryFolder( object ): + api_element_visible_keys = ( 'name', 'description', 'item_count', 'genome_build' ) def __init__( self, name=None, description=None, item_count=0, order_id=None ): self.name = name or "Unnamed folder" self.description = description @@ -961,6 +976,18 @@ class LibraryFolder( object ): if isinstance( name, str ): name = unicode( name, 'utf-8' ) return name + def get_api_value( self, view='collection' ): + rval = {} + try: + visible_keys = self.__getattribute__( 'api_' + view + '_visible_keys' ) + except AttributeError: + raise Exception( 'Unknown API view: %s' % view ) + for key in visible_keys: + try: + rval[key] = self.__getattribute__( key ) + except AttributeError: + rval[key] = None + return rval @property def parent_library( self ): f = self @@ -1011,6 +1038,28 @@ class LibraryDataset( object ): if not purged and self.purged: raise Exception( "Cannot unpurge once purged" ) purged = property( get_purged, set_purged ) + def get_api_value( self, view='collection' ): + # Since this class is a proxy to rather complex attributes we want to + # display in other objects, we can't use the simpler method used by + # other model classes. + ldda = self.library_dataset_dataset_association + rval = dict( name = ldda.name, + uploaded_by = ldda.user.email, + message = ldda.message, + date_uploaded = ldda.create_time.isoformat(), + file_size = int( ldda.get_size() ), + data_type = ldda.ext, + genome_build = ldda.dbkey, + misc_info = ldda.info, + misc_blurb = ldda.blurb ) + for name, spec in ldda.metadata.spec.items(): + val = ldda.metadata.get( name ) + if isinstance( val, MetadataFile ): + val = val.file_name + elif isinstance( val, list ): + val = ', '.join( val ) + rval['metadata_' + name] = val + return rval class LibraryDatasetDatasetAssociation( DatasetInstance ): def __init__( self, @@ -1734,6 +1783,9 @@ class UserAction( object ): self.params = params self.context = context +class APIKeys( object ): + pass + ## ---- Utility methods ------------------------------------------------------- def directory_hash_id( id ): @@ -1748,5 +1800,3 @@ def directory_hash_id( id ): padded = padded[:-3] # Break into chunks of three return [ padded[i*3:(i+1)*3] for i in range( len( padded ) // 3 ) ] - - --- a/lib/galaxy/util/__init__.py +++ b/lib/galaxy/util/__init__.py @@ -417,6 +417,14 @@ def stringify_dictionary_keys( in_dict ) out_dict[ str( key ) ] = value return out_dict +def recursively_stringify_dictionary_keys( d ): + if isinstance(d, dict): + return dict([(k.encode('utf-8'), recursively_stringify_dictionary_keys(v)) for k,v in d.iteritems()]) + elif isinstance(d, list): + return [recursively_stringify_dictionary_keys(x) for x in d] + else: + return d + def mkstemp_ln( src, prefix='mkstemp_ln_' ): """ From tempfile._mkstemp_inner, generate a hard link in the same dir with a --- /dev/null +++ b/scripts/api/library_upload_from_import_dir.py @@ -0,0 +1,19 @@ +#!/usr/bin/python + +import os, sys +sys.path.insert( 0, os.path.dirname( __file__ ) ) +from common import submit + +try: + data = {} + data[ 'folder_id' ] = sys.argv[3] + data[ 'file_type' ] = sys.argv[4] + data[ 'server_dir' ] = sys.argv[5] + data[ 'dbkey' ] = sys.argv[6] + data[ 'upload_option' ] = 'upload_directory' + data[ 'create_type' ] = 'file' +except IndexError: + print 'usage: %s key url folder_id file_type server_dir dbkey' % os.path.basename( sys.argv[0] ) + sys.exit( 1 ) + +submit( sys.argv[1], sys.argv[2], data ) --- a/lib/galaxy/model/mapping.py +++ b/lib/galaxy/model/mapping.py @@ -889,6 +889,12 @@ UserAction.table = Table( "user_action", Column( "context", Unicode( 512 ) ), Column( "params", Unicode( 1024 ) ) ) +APIKeys.table = Table( "api_keys", metadata, + Column( "id", Integer, primary_key=True ), + Column( "create_time", DateTime, default=now ), + Column( "user_id", Integer, ForeignKey( "galaxy_user.id" ), index=True ), + Column( "key", TrimmedString( 32 ), index=True, unique=True ) ) + # With the tables defined we can define the mappers and setup the # relationships between the model objects. @@ -1067,8 +1073,9 @@ assign_mapper( context, User, User.table _preferences=relation( UserPreference, backref="user", collection_class=attribute_mapped_collection('name')), # addresses=relation( UserAddress, # primaryjoin=( User.table.c.id == UserAddress.table.c.user_id ) ) - values=relation( FormValues, - primaryjoin=( User.table.c.form_values_id == FormValues.table.c.id ) ), + values=relation( FormValues, + primaryjoin=( User.table.c.form_values_id == FormValues.table.c.id ) ), + api_keys=relation( APIKeys, backref="user", order_by=desc( APIKeys.table.c.create_time ) ), ) ) # Set up proxy so that this syntax is possible: @@ -1485,6 +1492,9 @@ assign_mapper( context, UserAction, User properties = dict( user=relation( User.mapper ) ) ) +assign_mapper( context, APIKeys, APIKeys.table, + properties = {} ) + def db_next_hid( self ): """ Override __next_hid to generate from the database in a concurrency --- a/lib/galaxy/web/controllers/library_common.py +++ b/lib/galaxy/web/controllers/library_common.py @@ -280,7 +280,7 @@ class LibraryCommon( BaseController ): status = params.get( 'status', 'done' ) show_deleted = util.string_as_bool( params.get( 'show_deleted', False ) ) use_panels = util.string_as_bool( params.get( 'use_panels', False ) ) - is_admin = trans.user_is_admin() and cntrller == 'library_admin' + is_admin = trans.user_is_admin() and cntrller in ( 'library_admin', 'api' ) current_user_roles = trans.get_current_user_roles() try: parent_folder = trans.sa_session.query( trans.app.model.LibraryFolder ).get( trans.security.decode_id( parent_id ) ) @@ -294,6 +294,8 @@ class LibraryCommon( BaseController ): # its parent library, or if they are not able to see the folder's contents. if not parent_folder or ( not is_admin and not trans.app.security_agent.can_access_library_item( current_user_roles, parent_folder, trans.user ) ): message = "Invalid parent folder id ( %s ) specified." % str( parent_id ) + if cntrller == 'api': + return 400, message # This doesn't give away the library's existence since # browse_library will simply punt to browse_libraries if the # user-supplied id is invalid or inaccessible. @@ -309,6 +311,8 @@ class LibraryCommon( BaseController ): if not ( is_admin or trans.app.security_agent.can_add_library_item( current_user_roles, parent_folder ) ): message = "You are not authorized to create a folder in parent folder '%s'." % parent_folder.name # Redirect to the real parent library since we know we have access to it. + if cntrller == 'api': + return 403, message return trans.response.send_redirect( web.url_for( controller='library_common', action='browse_library', cntrller=cntrller, @@ -317,7 +321,7 @@ class LibraryCommon( BaseController ): show_deleted=show_deleted, message=util.sanitize_text( message ), status='error' ) ) - if params.get( 'new_folder_button', False ): + if params.get( 'new_folder_button', False ) or cntrller == 'api': new_folder = trans.app.model.LibraryFolder( name=util.restore_text( params.name ), description=util.restore_text( params.description ) ) # We are associating the last used genome build with folders, so we will always @@ -329,6 +333,9 @@ class LibraryCommon( BaseController ): trans.sa_session.flush() # New folders default to having the same permissions as their parent folder trans.app.security_agent.copy_library_permissions( parent_folder, new_folder ) + # If we're creating in the API, we're done + if cntrller == 'api': + return 200, dict( created=new_folder ) # If we have an inheritable template, redirect to the folder_info page so information # can be filled in immediately. widgets = [] @@ -654,7 +661,7 @@ class LibraryCommon( BaseController ): show_deleted=show_deleted, message=util.sanitize_text( message ), status='error' ) ) - if ( trans.user_is_admin() and cntrller == 'library_admin' ): + if is_admin: # Get all associated hdas and lddas that use the same disk file. associated_hdas = trans.sa_session.query( trans.model.HistoryDatasetAssociation ) \ .filter( and_( trans.model.HistoryDatasetAssociation.deleted == False, @@ -848,7 +855,7 @@ class LibraryCommon( BaseController ): last_used_build = dbkey[0] else: last_used_build = dbkey - is_admin = trans.user_is_admin() and cntrller == 'library_admin' + is_admin = trans.user_is_admin() and cntrller in ( 'library_admin', 'api' ) current_user_roles = trans.get_current_user_roles() if replace_id not in [ None, 'None' ]: try: @@ -859,6 +866,8 @@ class LibraryCommon( BaseController ): # its parent library, or if they are not able to view the dataset itself. if not replace_dataset or ( not is_admin and not trans.app.security_agent.can_access_library_item( current_user_roles, replace_dataset, trans.user ) ): message = "Invalid library dataset id ( %s ) to replace specified." % replace_id + if cntrller == 'api': + return 400, message return trans.response.send_redirect( web.url_for( controller='library_common', action='browse_library', cntrller=cntrller, @@ -870,6 +879,8 @@ class LibraryCommon( BaseController ): # Deny access if the user is not an admin and does not have the LIBRARY_MODIFY permission. if not ( is_admin or trans.app.security_agent.can_modify_library_item( current_user_roles, replace_dataset ) ): message = "You are not authorized to replace library dataset '%s'." % replace_dataset.name + if cntrller == 'api': + return 403, message return trans.response.send_redirect( web.url_for( controller='library_common', action='browse_library', cntrller=cntrller, @@ -896,6 +907,8 @@ class LibraryCommon( BaseController ): # its parent library, or if they are not able to see the folder's contents. if not folder or ( not is_admin and not trans.app.security_agent.can_access_library_item( current_user_roles, folder, trans.user ) ): message = "Invalid parent folder id ( %s ) specified." % str( folder_id ) + if cntrller == 'api': + return 400, message return trans.response.send_redirect( web.url_for( controller='library_common', action='browse_library', cntrller=cntrller, @@ -907,6 +920,8 @@ class LibraryCommon( BaseController ): # Deny access if the user is not an admin and does not have the LIBRARY_ADD permission. if not ( is_admin or trans.app.security_agent.can_add_library_item( current_user_roles, folder ) ): message = "You are not authorized to create a library dataset in parent folder '%s'." % folder.name + if cntrller == 'api': + return 403, message return trans.response.send_redirect( web.url_for( controller='library_common', action='browse_library', cntrller=cntrller, @@ -918,7 +933,7 @@ class LibraryCommon( BaseController ): library = folder.parent_library if folder and last_used_build in [ 'None', None, '?' ]: last_used_build = folder.genome_build - if params.get( 'runtool_btn', False ) or params.get( 'ajax_upload', False ): + if params.get( 'runtool_btn', False ) or params.get( 'ajax_upload', False ) or cntrller == 'api': # Check to see if the user selected roles to associate with the DATASET_ACCESS permission # on the dataset that would cause accessibility issues. roles = params.get( 'roles', False ) @@ -931,7 +946,8 @@ class LibraryCommon( BaseController ): permissions, in_roles, error, message = \ trans.app.security_agent.derive_roles_from_access( trans, library.id, cntrller, library=True, **vars ) if error: - status = 'error' + if cntrller == 'api': + return 400, message trans.response.send_redirect( web.url_for( controller='library_common', action='upload_library_dataset', cntrller=cntrller, @@ -961,6 +977,11 @@ class LibraryCommon( BaseController ): replace_dataset=replace_dataset, **kwd ) if created_outputs_dict: + if cntrller == 'api': + # created_outputs_dict can only ever be a string if cntrller == 'api' + if type( created_outputs_dict ) == str: + return 400, created_outputs_dict + return 200, created_outputs_dict total_added = len( created_outputs_dict.keys() ) ldda_id_list = [ str( v.id ) for k, v in created_outputs_dict.items() ] created_ldda_ids=",".join( ldda_id_list ) @@ -1001,6 +1022,9 @@ class LibraryCommon( BaseController ): created_ldda_ids = '' message = "Upload failed" status='error' + if cntrller == 'api': + return 400, message + response_code = 400 trans.response.send_redirect( web.url_for( controller='library_common', action='browse_library', cntrller=cntrller, @@ -1068,7 +1092,6 @@ class LibraryCommon( BaseController ): # Library-specific params params = util.Params( kwd ) # is this filetoolparam safe? show_deleted = util.string_as_bool( params.get( 'show_deleted', False ) ) - library_bunch = upload_common.handle_library_params( trans, params, folder_id, replace_dataset ) message = util.restore_text( params.get( 'message', '' ) ) status = params.get( 'status', 'done' ) server_dir = util.restore_text( params.get( 'server_dir', '' ) ) @@ -1077,11 +1100,11 @@ class LibraryCommon( BaseController ): else: replace_id = None upload_option = params.get( 'upload_option', 'upload_file' ) - err_redirect = False + response_code = 200 if upload_option == 'upload_directory': if server_dir in [ None, 'None', '' ]: - err_redirect = True - if cntrller == 'library_admin': + response_code = 400 + if cntrller == 'library_admin' or ( cntrller == 'api' and trans.user_is_admin ): import_dir = trans.app.config.library_import_dir import_dir_desc = 'library_import_dir' full_dir = os.path.join( import_dir, server_dir ) @@ -1095,21 +1118,35 @@ class LibraryCommon( BaseController ): if import_dir: message = 'Select a directory' else: + response_code = 403 message = '"%s" is not defined in the Galaxy configuration file' % import_dir_desc - # Proceed with (mostly) regular upload processing - precreated_datasets = upload_common.get_precreated_datasets( trans, tool_params, trans.app.model.LibraryDatasetDatasetAssociation, controller=cntrller ) - if upload_option == 'upload_file': - tool_params = upload_common.persist_uploads( tool_params ) - uploaded_datasets = upload_common.get_uploaded_datasets( trans, cntrller, tool_params, precreated_datasets, dataset_upload_inputs, library_bunch=library_bunch ) - elif upload_option == 'upload_directory': - uploaded_datasets, err_redirect, message = self.get_server_dir_uploaded_datasets( trans, cntrller, params, full_dir, import_dir_desc, library_bunch, err_redirect, message ) elif upload_option == 'upload_paths': - uploaded_datasets, err_redirect, message = self.get_path_paste_uploaded_datasets( trans, cntrller, params, library_bunch, err_redirect, message ) - upload_common.cleanup_unused_precreated_datasets( precreated_datasets ) - if upload_option == 'upload_file' and not uploaded_datasets: - message = 'Select a file, enter a URL or enter text' - err_redirect = True - if err_redirect: + if not trans.app.config.allow_library_path_paste: + response_code = 403 + message = '"allow_library_path_paste" is not defined in the Galaxy configuration file' + # Some error handling should be added to this method. + try: + library_bunch = upload_common.handle_library_params( trans, params, folder_id, replace_dataset ) + except: + response_code = 500 + message = "Unable to parse upload parameters, please report this error." + # Proceed with (mostly) regular upload processing if we're still errorless + if response_code == 200: + precreated_datasets = upload_common.get_precreated_datasets( trans, tool_params, trans.app.model.LibraryDatasetDatasetAssociation, controller=cntrller ) + if upload_option == 'upload_file': + tool_params = upload_common.persist_uploads( tool_params ) + uploaded_datasets = upload_common.get_uploaded_datasets( trans, cntrller, tool_params, precreated_datasets, dataset_upload_inputs, library_bunch=library_bunch ) + elif upload_option == 'upload_directory': + uploaded_datasets, response_code, message = self.get_server_dir_uploaded_datasets( trans, cntrller, params, full_dir, import_dir_desc, library_bunch, response_code, message ) + elif upload_option == 'upload_paths': + uploaded_datasets, response_code, message = self.get_path_paste_uploaded_datasets( trans, cntrller, params, library_bunch, response_code, message ) + upload_common.cleanup_unused_precreated_datasets( precreated_datasets ) + if upload_option == 'upload_file' and not uploaded_datasets: + response_code = 400 + message = 'Select a file, enter a URL or enter text' + if response_code != 200: + if cntrller == 'api': + return ( response_code, message ) trans.response.send_redirect( web.url_for( controller='library_common', action='upload_library_dataset', cntrller=cntrller, @@ -1145,7 +1182,7 @@ class LibraryCommon( BaseController ): trans.sa_session.add_all( ( uploaded_dataset.data, uploaded_dataset.data.dataset ) ) trans.sa_session.flush() return uploaded_dataset - def get_server_dir_uploaded_datasets( self, trans, cntrller, params, full_dir, import_dir_desc, library_bunch, err_redirect, message ): + def get_server_dir_uploaded_datasets( self, trans, cntrller, params, full_dir, import_dir_desc, library_bunch, response_code, message ): files = [] try: for entry in os.listdir( full_dir ): @@ -1173,22 +1210,22 @@ class LibraryCommon( BaseController ): files.append( path ) except Exception, e: message = "Unable to get file list for configured %s, error: %s" % ( import_dir_desc, str( e ) ) - err_redirect = True - return None, err_redirect, message + response_code = 500 + return None, response_code, message if not files: message = "The directory '%s' contains no valid files" % full_dir - err_redirect = True - return None, err_redirect, message + response_code = 400 + return None, response_code, message uploaded_datasets = [] for file in files: name = os.path.basename( file ) uploaded_datasets.append( self.make_library_uploaded_dataset( trans, cntrller, params, name, file, 'server_dir', library_bunch ) ) - return uploaded_datasets, None, None - def get_path_paste_uploaded_datasets( self, trans, cntrller, params, library_bunch, err_redirect, message ): + return uploaded_datasets, 200, None + def get_path_paste_uploaded_datasets( self, trans, cntrller, params, library_bunch, response_code, message ): if params.get( 'filesystem_paths', '' ) == '': message = "No paths entered in the upload form" - err_redirect = True - return None, err_redirect, message + response_code = 400 + return None, response_code, message preserve_dirs = True if params.get( 'dont_preserve_dirs', False ): preserve_dirs = False @@ -1222,8 +1259,8 @@ class LibraryCommon( BaseController ): in_folder ) ) if bad_paths: message = "Invalid paths:<br><ul><li>%s</li></ul>" % "</li><li>".join( bad_paths ) - err_redirect = True - return None, err_redirect, message + response_code = 400 + return None, response_code, message return uploaded_datasets, None, None @web.expose def add_history_datasets_to_library( self, trans, cntrller, library_id, folder_id, hda_ids='', **kwd ): --- a/eggs.ini +++ b/eggs.ini @@ -41,7 +41,7 @@ Paste = 1.6 PasteDeploy = 1.3.3 PasteScript = 1.7.3 pexpect = 2.4 -Routes = 1.11 +Routes = 1.12.3 simplejson = 1.5 SQLAlchemy = 0.5.6 sqlalchemy_migrate = 0.5.4 --- /dev/null +++ b/scripts/api/library_create_folder.py @@ -0,0 +1,20 @@ +#!/usr/bin/python + +import os, sys +sys.path.insert( 0, os.path.dirname( __file__ ) ) +from common import submit + +try: + data = {} + data[ 'folder_id' ] = sys.argv[3] + data[ 'name' ] = sys.argv[4] + data[ 'create_type' ] = 'folder' +except IndexError: + print 'usage: %s key url folder_id name [description]' % os.path.basename( sys.argv[0] ) + sys.exit( 1 ) +try: + data[ 'description' ] = sys.argv[5] +except IndexError: + data[ 'description' ] = '' + +submit( sys.argv[1], sys.argv[2], data ) --- a/lib/galaxy/web/controllers/user.py +++ b/lib/galaxy/web/controllers/user.py @@ -979,3 +979,25 @@ class User( BaseController ): lines_skipped=lines_skipped ) + @web.expose + def new_api_key( self, trans, **kwd ): + params = util.Params( kwd ) + message = util.restore_text( params.get( 'message', '' ) ) + status = params.get( 'status', 'done' ) + admin_view = util.string_as_bool( params.get( 'admin_view', False ) ) + error = '' + user = trans.sa_session.query( trans.app.model.User ).get( int( params.get( 'user_id', None ) ) ) + if params.get( 'new_api_key_button', None ) == 'Generate a new key now': + new_key = trans.app.model.APIKeys() + new_key.user_id = user.id + new_key.key = trans.app.security.get_new_guid() + trans.sa_session.add( new_key ) + trans.sa_session.flush() + message = "Generated a new web API key" + status = "done" + return trans.response.send_redirect( web.url_for( controller='user', + action='show_info', + admin_view=admin_view, + user_id=user.id, + message=message, + status=status ) ) --- a/templates/webapps/galaxy/user/info.mako +++ b/templates/webapps/galaxy/user/info.mako @@ -66,8 +66,9 @@ <p></p> %endif +<p/> + <div class="toolForm"> - <p></p><form name="user_info" id="user_info" action="${h.url_for( controller='user', action='new_address', user_id=user.id, admin_view=admin_view )}" method="post" ><div class="toolFormTitle">User Addresses</div><div class="toolFormBody"> @@ -118,5 +119,37 @@ </div></div></form> - <p></p></div> + +<p/> + +%if trans.app.config.enable_api: + <div class="toolForm"> + <form name="user_info" id="user_info" action="${h.url_for( controller='user', action='new_api_key', user_id=user.id, admin_view=admin_view )}" method="post" > + <div class="toolFormTitle">Web API Key</div> + <div class="toolFormBody"> + <div class="form-row"> + <label>Current API key:</label> + %if user.api_keys: + ${user.api_keys[0].key} + %else: + none set + %endif + </div> + <div class="form-row"> + <input type="submit" name="new_api_key_button" value="Generate a new key now"/> + %if user.api_keys: + (invalidates old key) + %endif + <div class="toolParamHelp" style="clear: both;"> + An API key will allow you to access Galaxy via its web + API (documentation forthcoming). Please note that + <strong>this key acts as an alternate means to access + your account, and should be treated with the same care + as your login password</strong>. + </div> + </div> + </div> + </form> + </div> +%endif --- a/lib/galaxy/web/__init__.py +++ b/lib/galaxy/web/__init__.py @@ -2,6 +2,6 @@ The Galaxy web application. """ -from framework import expose, json, json_pretty, require_login, require_admin, url_for, error, form, FormBuilder +from framework import expose, json, json_pretty, require_login, require_admin, url_for, error, form, FormBuilder, expose_api from framework.base import httpexceptions --- a/universe_wsgi.ini.sample +++ b/universe_wsgi.ini.sample @@ -112,6 +112,9 @@ gbrowse_display_sites = wormbase,tair,mo # Define your GeneTrack servers in tool-data/shared/genetrack/genetrack_sites.txt #genetrack_display_sites = +# Enable the (experimental! beta!) Web API. Documentation forthcoming. +#enable_api = False + # Serving static files (needed if running standalone) static_enabled = True static_cache_time = 360 --- /dev/null +++ b/scripts/api/display.py @@ -0,0 +1,11 @@ +#!/usr/bin/python + +import os, sys +sys.path.insert( 0, os.path.dirname( __file__ ) ) +from common import display + +try: + display( *sys.argv[1:3] ) +except TypeError: + print 'usage: %s key url' % os.path.basename( sys.argv[0] ) + sys.exit( 1 )

1 0

galaxy-dist commit beb30aadccd7: Fix IE crashing on pages with grids
by commits-noreply＠bitbucket.org 29 Jun '10

29 Jun '10

# HG changeset patch -- Bitbucket.org # Project galaxy-dist # URL http://bitbucket.org/galaxy/galaxy-dist/overview # User Kanwei Li <kanwei(a)gmail.com> # Date 1277239345 14400 # Node ID beb30aadccd75bfdec6b06d4afcd704c2c16dbc3 # Parent 5e75bad5a780acb7c2975bcc375216c1abb5d148 Fix IE crashing on pages with grids --- a/templates/grid_base.mako +++ b/templates/grid_base.mako @@ -245,14 +245,13 @@ %endfor // Initialize URL args with filter arguments. - var url_args = ${h.to_json_string( cur_filter_dict )}; + var url_args_init = ${h.to_json_string( cur_filter_dict )}, + url_args = {}; // Place "f-" in front of all filter arguments. - for (arg in url_args) - { - value = url_args[arg]; - delete url_args[arg]; - url_args["f-" + arg] = value; + + for (arg in url_args_init) { + url_args["f-" + arg] = url_args_init[arg]; } // Add sort argument to URL args.

1 0

galaxy-dist commit 4b4385fbf529: Fix bug in metadata setting for intersect tool.
by commits-noreply＠bitbucket.org 29 Jun '10

29 Jun '10

# HG changeset patch -- Bitbucket.org # Project galaxy-dist # URL http://bitbucket.org/galaxy/galaxy-dist/overview # User jeremy goecks <jeremy.goecks(a)emory.edu> # Date 1277222908 14400 # Node ID 4b4385fbf52982ee3bf6394c5cb269e7bf3ab912 # Parent 764ecc0cf3fdc4885de777f26e1ae7c06a85acbf Fix bug in metadata setting for intersect tool. --- a/tools/new_operations/intersect.xml +++ b/tools/new_operations/intersect.xml @@ -54,7 +54,7 @@ <outputs><data format="input" name="output"> #if inputs.type == "Interval": - metadata_source="inputs.bed_input1" + metadata_source="inputs.interval_input1" #else: metadata_source="inputs.gff_input1" #end if

1 0