[galaxy-commits] galaxy-dist commit 067a8649dae7: Remove code and drop database tables related to the old Galaxy Cloud functionality.

# HG changeset patch -- Bitbucket.org # Project galaxy-dist # URL http://bitbucket.org/galaxy/galaxy-dist/overview # User Enis Afgan <afgane@gmail.com> # Date 1277306301 14400 # Node ID 067a8649dae79ab973ff3821b62be89e21a0117a # Parent 752cb3e325374381bdd0db6b9163b12ba88b7151 Remove code and drop database tables related to the old Galaxy Cloud functionality. --- a/templates/cloud/edit_image.mako +++ /dev/null @@ -1,92 +0,0 @@ -<% _=n_ %> -<%inherit file="/base.mako"/> -<%def name="title()">Edit machine image</%def> - -<%def name="javascripts()"> -${parent.javascripts()} -<script type="text/javascript"> -$(function(){ - $("input:text:first").focus(); -}) -</script> -</%def> - -%if header: - ${header} -%endif - -%if image: - -<div class="form"> - <div class="form-title">Edit image</div> - <div class="form-body"> - <form name="edit_image" action="${h.url_for( action='edit_image', id=trans.security.encode_id(image.id), edited="true" )}" method="post" > - <% - cls = "form-row" - if error.has_key('provider_error'): - cls += " form-row-error" - %> - <div class="${cls}"> - <label>Provider type:</label> - <div class="form-row-input"> - ${image.provider_type} - </div> - %if error.has_key('provider_error'): - <div class="form-row-error-message">${error['provider_error']}</div> - %endif - <div style="clear: both"></div> - </div> - <% - cls = "form-row" - if error.has_key('id_error'): - cls += " form-row-error" - %> - <div class="${cls}"> - <label>Machine Image ID (AMI or EMI):</label> - <div class="form-row-input"> - <input type="text" name="image_id" value="${image.image_id}" size="40"> - </div> - %if error.has_key('id_error'): - <div class="form-row-error-message">${error['id_error']}</div> - %endif - <div style="clear: both"></div> - </div> - <% - cls = "form-row" - if error.has_key('manifest_error'): - cls += " form-row-error" - %> - <div class="${cls}"> - <label>Manifest:</label> - <div class="form-row-input"> - <input type="text" name="manifest" value="${image.manifest}" size="40"> - </div> - %if error.has_key('manifest_error'): - <div class="form-row-error-message">${error['manifest_error']}</div> - %endif - <div style="clear: both"></div> - </div> - <% - cls = "form-row" - if error.has_key('arch_error'): - cls += " form-row-error" - %> - <div class="${cls}"> - <label>Architecture:</label> - <div class="form-row-input"> - <input type="text" name="architecture" value="${image.architecture}" size="40"> - </div> - %if error.has_key('arch_error'): - <div class="form-row-error-message">${error['arch_error']}</div> - %endif - <div style="clear: both"></div> - </div> - - <div class="form-row"><input type="submit" value="Save"></div> - </form> - </div> -</div> - -%else: - Specified machine image could not be found. -%endif --- a/templates/cloud/view_provider.mako +++ /dev/null @@ -1,126 +0,0 @@ -<%inherit file="/base.mako"/> - -<%def name="title()">Cloud provider</%def> - -<h2>Cloud provider details</h2> - -<ul class="manage-table-actions"> - <li> - <a class="action-button" href="${h.url_for( action='list' )}"> - <img src="${h.url_for('/static/images/silk/resultset_previous.png')}" /> - <span>Return to cloud management console</span> - </a> - </li> -</ul> - -%if provider: - ${view_provider( provider )} -%else: - There is no cloud provider under that name. -%endif - - - - -<%def name="view_provider( provider )"> - <table class="mange-table colored" border="0" cellspacing="0" cellpadding="0" width="100%"> - <tr> - <td> Cloud provider name: </td> - <td> - ${provider.name} - <a id="cp-popup" class="popup-arrow" style="display: none;">▼</a> - </td> - <td> - <div popupmenu="cp-popup"> - <a class="action-button" href="${h.url_for( action='edit_provider', id=trans.security.encode_id(provider.id) )}">Edit</a> - <a class="action-button" confirm="Are you sure you want to delete cloud provider '${provider.name}'?" href="${h.url_for( action='delete_provider', id=trans.security.encode_id(provider.id) )}">Delete</a> - </div> - </td> - </tr> - <tr> - <td> Last updated: </td> - <td> ${str(provider.update_time)[:16]} - <% - context.write( ' UTC (' ) - context.write( str(h.date.distance_of_time_in_words (provider.update_time, h.date.datetime.utcnow() ) ) ) - %> ago) - </td> - </tr> - <tr> - <td> Cloud provider type: </td> - <td> ${str(provider.type)[:16]}</td> - </tr> - %if provider.region_connection != None: - <tr> - <td> Region connection: </td> - <td> ${provider.region_connection} </td> - </tr> - %endif - %if provider.region_name != None: - <tr> - <td> Region name: </td> - <td> ${provider.region_name} </td> - </tr> - %endif - %if provider.region_endpoint != None: - <tr> - <td> Region endpoint: </td> - <td> ${provider.region_endpoint} </td> - </tr> - %endif - %if provider.is_secure != None: - <tr> - <td> Is secure: </td> - <td> ${provider.is_secure} </td> - </tr> - %endif - %if provider.host != None: - <tr> - <td> Host: </td> - <td> ${provider.host} </td> - </tr> - %endif - %if provider.port != None: - <tr> - <td> Port: </td> - <td> ${provider.port} </td> - </tr> - %endif - %if provider.proxy != None: - <tr> - <td> Proxy: </td> - <td> ${provider.proxy} </td> - </tr> - %endif - %if provider.proxy_port != None: - <tr> - <td> Proxy port: </td> - <td> ${provider.proxy_port} </td> - </tr> - %endif - %if provider.proxy_pass != None: - <tr> - <td> Proxy pass: </td> - <td> ${provider.proxy_pass} </td> - </tr> - %endif - %if provider.debug != None: - <tr> - <td> Debug: </td> - <td> ${provider.debug} </td> - </tr> - %endif - %if provider.https_connection_factory != None: - <tr> - <td> HTTPS connection factory: </td> - <td> ${provider.https_connection_factory} </td> - </tr> - %endif - %if provider.path != None: - <tr> - <td> Path: </td> - <td> ${provider.path} </td> - </tr> - %endif - </table> -</%def> --- a/templates/cloud/edit_credentials.mako +++ /dev/null @@ -1,91 +0,0 @@ -<% _=n_ %> -<%inherit file="/base.mako"/> -<%def name="title()">Add credentials</%def> - -<%def name="javascripts()"> -${parent.javascripts()} -<script type="text/javascript"> -$(function(){ - $("input:text:first").focus(); -}) -</script> -</%def> - -%if header: - ${header} -%endif - -%if credential: - -<div class="form"> - <div class="form-title">Edit credentials</div> - <div class="form-body"> - <form name="edit_credentials" action="${h.url_for( action='edit_credentials', id=trans.security.encode_id(credential.id), edited="true" )}" method="post" > - - <% - cls = "form-row" - if error.has_key('cred_error'): - cls += " form-row-error" - %> - <div class="${cls}"> - <label>Credentials name:</label> - <div class="form-row-input"> - <input type="text" name="credName" value="${credential.name}" size="40"> - </div> - %if error.has_key('cred_error'): - <div class="form-row-error-message">${error['cred_error']}</div> - %endif - <div style="clear: both"></div> - </div> - - - <% - cls = "form-row" - if error.has_key('provider_error'): - cls += " form-row-error" - %> - <div class="${cls}"> - <label>Cloud provider name (type):</label> - <div class="form-row-input">${credential.provider.name} (${credential.provider.type})</div> - <div style="clear: both"></div> - </div> - <% - cls = "form-row" - if error.has_key('access_key_error'): - cls += " form-row-error" - %> - <div class="${cls}"> - <label>Access key:</label> - <div class="form-row-input"> - <input type="text" name="accessKey" value="${credential.access_key}" size="40"> - </div> - %if error.has_key('access_key_error'): - <div class="form-row-error-message">${error['access_key_error']}</div> - %endif - <div style="clear: both"></div> - </div> - - <% - cls = "form-row" - if error.has_key('secret_key_error'): - cls += " form-row-error" - %> - <div class="${cls}"> - <label>Secret key:</label> - <div class="form-row-input"> - <input type="password" name="secretKey" value="${credential.secret_key}" size="40"> - </div> - %if error.has_key('secret_key_error'): - <div class="form-row-error-message">${error['secret_key_error']}</div> - %endif - <div style="clear: both"></div> - </div> - - <div class="form-row"><input type="submit" value="Edit"></div> - </form> - </div> -</div> - -%else: - Specified credentials could not be found. -%endif --- a/templates/cloud/edit_provider.mako +++ /dev/null @@ -1,261 +0,0 @@ -<% _=n_ %> -<%inherit file="/base.mako"/> -<%def name="title()">Edit provider</%def> - -<%def name="javascripts()"> -${parent.javascripts()} -<script type="text/javascript"> -$(function(){ - - $("input:text:first").focus(); - - $("#type").change(function() { - if ($(this).val() == 'ec2') { - clear(); - $("#autofill").attr( 'disabled', true ); - $("#autofill").attr( 'checked', false ); - $("#name").val( "EC2" ); - $("#region_name").val( "us-east-1" ); - $("#region_endpoint").val( "us-east-1.ec2.amazonaws.com" ); - $("#is_secure").val("1"); - $("#debug").val(""); - $("#path").val("/"); - } - else if ($(this).val() == 'eucalyptus') { - clear(); - $("#autofill").attr( 'disabled', false ); - } - }); -}) - - -function af(){ - - if ( $("#autofill_epc").attr('checked') ) { - $("#region_name").val("eucalyptus"); - $("#region_endpoint").val("mayhem9.cs.ucsb.edu"); - $("#is_secure").val("0"); - $("#port").val("8773"); - $("#path").val("/services/Eucalyptus"); - } - else if ( $("#autofill_ec2").attr('checked') ) { - $("#region_name").val( "us-east-1" ); - $("#region_endpoint").val( "us-east-1.ec2.amazonaws.com" ); - $("#is_secure").val("1"); - $("#debug").val(""); - $("#path").val("/"); - } -} - -function clear() { - //$("#name").val(""); - $("#region_name").val(""); - $("#region_endpoint").val(""); - $("#is_secure").val(""); - $("#port").val(""); - $("#proxy").val(""); - $("#proxy_port").val(""); - $("#proxy_user").val(""); - $("#proxy_pass").val(""); - $("#debug").val(""); - $("#https_connection_factory").val(""); - $("#path").val(""); - -} - -</script> -</%def> - -%if header: - ${header} -%endif - -%if provider: - <div class="form"> - <div class="form-title">Edit cloud provider</div> - <div class="form-body"> - <form name="edit_provider_form" action="${h.url_for( action='edit_provider', id=trans.security.encode_id(provider.id), edited="true" )}" method="post" > - <% - cls = "form-row" - if error.has_key('type_error'): - cls += " form-row-error" - %> - <div class="${cls}"> - <label>Provider type:</label> - <div class="form-row-input">${provider.type} - %if provider.type == 'eucalyptus': - <p><input type="checkbox" id="autofill_epc" onclick="javascript:af()"> - auto fill using Eucalyptus Public Cloud values - </p></div> - %elif provider.type == 'ec2': - <p><input type="checkbox" id="autofill_ec2" onclick="javascript:af()"> - auto fill for Amazon EC2 (us-east-1 region) - </p></div> - %endif - <div style="clear: both"></div> - </div> - - <% - cls = "form-row" - if error.has_key('name_error'): - cls += " form-row-error" - %> - <div class="${cls}"> - <label>Provider name:</label> - <div class="form-row-input"> - <input type="text" id="name" name="name" value="${provider.name}" size="40"> - </div> - %if error.has_key('name_error'): - <div class="form-row-error-message">${error['name_error']}</div> - %endif - <div style="clear: both"></div> - </div> - - <% - cls = "form-row" - %> - <div class="${cls}"> - <label>Region name:</label> - <div id="region_selection" class="form-row-input"> - <input type="text" name="region_name" id="region_name" value="${provider.region_name}" size="40"> - </div> - <div style="clear: both"></div> - </div> - - <% - cls = "form-row" - %> - <div class="${cls}"> - <label>Region endpoint:</label> - <div class="form-row-input"> - <input type="text" name="region_endpoint" id="region_endpoint" value="${provider.region_endpoint}" size="40"> - </div> - <div style="clear: both"></div> - </div> - - <% - cls = "form-row" - if error.has_key('is_secure_error'): - cls += " form-row-error" - %> - <div class="${cls}"> - <label>Is secure ('O' for False or '1' for True):</label> - <div class="form-row-input"> - %if provider.is_secure == True: - <input type="text" name="is_secure" id="is_secure" value="1" size="40"> - %else: - <input type="text" name="is_secure" id="is_secure" value="0" size="40"> - %endif - </div> - %if error.has_key('is_secure_error'): - <div class="form-row-error-message">${error['is_secure_error']}; you entered: '${provider.is_secure}'</div> - %endif - <div style="clear: both"></div> - </div> - - <% - cls = "form-row" - %> - <div class="${cls}"> - <label>Host:</label> - <div class="form-row-input"> - <input type="text" name="host" value="${provider.host}" size="40"> - </div> - <div style="clear: both"></div> - </div> - - <% - cls = "form-row" - %> - <div class="${cls}"> - <label>Port:</label> - <div class="form-row-input"> - <input type="text" name="port" id="port" value="${provider.port}" size="40"> - </div> - <div style="clear: both"></div> - </div> - - <% - cls = "form-row" - %> - <div class="${cls}"> - <label>Proxy:</label> - <div class="form-row-input"> - <input type="text" name="proxy" value="${provider.proxy}" size="40"> - </div> - <div style="clear: both"></div> - </div> - - <% - cls = "form-row" - %> - <div class="${cls}"> - <label>Proxy port:</label> - <div class="form-row-input"> - <input type="text" name="proxy_port" value="${provider.proxy_port}" size="40"> - </div> - <div style="clear: both"></div> - </div> - - <% - cls = "form-row" - %> - <div class="${cls}"> - <label>Proxy user:</label> - <div class="form-row-input"> - <input type="text" name="proxy_user" value="${provider.proxy_user}" size="40"> - </div> - <div style="clear: both"></div> - </div> - - <% - cls = "form-row" - %> - <div class="${cls}"> - <label>Proxy pass:</label> - <div class="form-row-input"> - <input type="text" name="proxy_pass" value="${provider.proxy_pass}" size="40"> - </div> - <div style="clear: both"></div> - </div> - - <% - cls = "form-row" - %> - <div class="${cls}"> - <label>Debug:</label> - <div class="form-row-input"> - <input type="text" name="debug" value="${provider.debug}" size="40"> - </div> - <div style="clear: both"></div> - </div> - - <% - cls = "form-row" - %> - <div class="${cls}"> - <label>HTTPS connection factory:</label> - <div class="form-row-input"> - <input type="text" name="https_connection_factory" value="${provider.https_connection_factory}" size="40"> - </div> - <div style="clear: both"></div> - </div> - - <% - cls = "form-row" - %> - <div class="${cls}"> - <label>Path:</label> - <div class="form-row-input"> - <input type="text" name="path" id="path" value="${provider.path}" size="40"> - </div> - <div style="clear: both"></div> - </div> - - <div class="form-row"><input type="submit" value="Save"></div> - - </form> - - </div> - </div> -%endif --- a/templates/cloud/view_usage.mako +++ /dev/null @@ -1,117 +0,0 @@ -<%inherit file="/base.mako"/> - -<%def name="title()">Cloud home</%def> - -%if message: -<% - try: - messagetype - except: - messagetype = "done" -%> - - - -<p /> -<div class="${messagetype}message"> - ${message} -</div> -%endif - -%if prevInstances: - <h2>Usage report for instance ${prevInstances[0].uci.name}</h2> -%else: - <h2>Selected instance has no record of being used.</h2> -%endif - -<ul class="manage-table-actions"> - <li> - <a class="action-button" href="${h.url_for( action='list' )}"> - <img src="${h.url_for('/static/images/silk/resultset_previous.png')}" /> - <span>Return to cloud management console</span> - </a> - </li> -</ul> - -%if prevInstances: - <table class="mange-table colored" border="0" cellspacing="0" cellpadding="0" width="100%"> - <colgroup width="2%"></colgroup> - <colgroup width="16%"></colgroup> - <colgroup width="16%"></colgroup> - <colgroup width="10%"></colgroup> - <colgroup width="5%"></colgroup> - <tr class="header"> - <th>#</th> - <th>Launch time</th> - <th>Termination time</th> - <th>Time alive</th> - <th>Type</th> - <th></th> - </tr> - <% - total_hours = 0 - %> - %for i, prevInstance in enumerate( prevInstances ): - <tr> - <td>${i+1}</td> - <td> - %if prevInstance.launch_time: - ${str(prevInstance.launch_time)[:16]} UCT - %else: - N/A - %endif - </td> - <td> - %if prevInstance.stop_time: - ${str(prevInstance.stop_time)[:16]} UCT - %else: - N/A - %endif - </td> - <td> - <% - # This is where current time and since duration is calculated - if prevInstance.launch_time is None or prevInstance.stop_time is None: - context.write( 'N/A' ) - else: - context.write( str(h.date.distance_of_time_in_words (prevInstance.launch_time, prevInstance.stop_time ) ) ) - time_delta = prevInstance.stop_time - prevInstance.launch_time - total_hours += time_delta.seconds / 3600 - if time_delta.seconds != 0: - total_hours += 1 - - %> - </td> - <td>${prevInstance.type}</td> - </tr> - %endfor - </table> - <br/>Total number of hours instance was alive: ${total_hours} <br /> - Note that these are just best effort estimates - true usage should be obtained from respective cloud provider. <br /> - <%namespace name="view_cred" file="view_credentials.mako" /> - - <div id="hide_cred_details"> - This instance uses credentials: - <a onclick="document.getElementById('show_cred_details').style.display = 'block'; - document.getElementById('hide_cred_details').style.display = 'none'; return 0" - href="javascript:void(0)"> - ${prevInstances[0].uci.credentials.name} - </a> - </div> - <div id="show_cred_details" style="DISPLAY: none"> - This instance uses credentials: - <a onclick="document.getElementById('hide_cred_details').style.display = 'block'; - document.getElementById('show_cred_details').style.display = 'none'; return 0;" - href="javascript:void(0)"> - ${prevInstances[0].uci.credentials.name} - </a> - ${view_cred.view_cred( prevInstances[0].uci.credentials ) } - </div> - - -%endif - - - - - --- a/templates/cloud/view_credentials.mako +++ /dev/null @@ -1,157 +0,0 @@ -<%inherit file="/base.mako"/> - -<%def name="title()">Cloud credentials</%def> - -<h2>Credentials details</h2> - -<ul class="manage-table-actions"> - <li> - <a class="action-button" href="${h.url_for( action='list' )}"> - <img src="${h.url_for('/static/images/silk/resultset_previous.png')}" /> - <span>Return to cloud management console</span> - </a> - </li> -</ul> - -%if credDetails: - ${view_cred( credDetails )} -%else: - There are no credentials under that name. -%endif - - - - -<%def name="view_cred( credDetails )"> - <table class="mange-table colored" border="0" cellspacing="0" cellpadding="0" width="100%"> - <tr> - <td> Credentials name: </td> - <td> - ${credDetails.name} - <a id="wf-popup" class="popup-arrow" style="display: none;">▼</a> - </td> - <td> - <div popupmenu="wf-popup"> - <a class="action-button" href="${h.url_for( action='edit_credentials', id=trans.security.encode_id(credDetails.id) )}">Edit</a> - <a class="action-button" confirm="Are you sure you want to delete credentials '${credDetails.name}'?" href="${h.url_for( action='delete_credentials', id=trans.security.encode_id(credDetails.id) )}">Delete</a> - </div> - </td> - </tr> - <tr> - <td> Last updated: </td> - <td> ${str(credDetails.update_time)[:16]} - <% - context.write( ' UTC (' ) - context.write( str(h.date.distance_of_time_in_words (credDetails.update_time, h.date.datetime.utcnow() ) ) ) - %> ago) - </td> - </tr> - <tr> - <td> Cloud provider type: </td> - <td> ${str(credDetails.provider.type)}</td> - </tr> - <tr> - <td> Cloud provider name: </td> - <td> ${str(credDetails.provider.name)}</td> - </tr> - <tr> - <td> Access key: </td> - <td> - ${credDetails.access_key} - </td> - </tr> - <tr> - <td> Secret key: </td> - <td> - <div id="shortComment2"> - <a onclick="document.getElementById('fullComment2').style.display = 'block'; - document.getElementById('shortComment2').style.display = 'none'; return 0" - href="javascript:void(0)"> - + Show - </a> - </div> - <div id="fullComment2" style="DISPLAY: none"> - <a onclick="document.getElementById('shortComment2').style.display = 'block'; - document.getElementById('fullComment2').style.display = 'none'; return 0;" - href="javascript:void(0)"> - - Hide - </a><br /> - <nobr>${credDetails.secret_key}</nobr><br/> - </div> - </td> - </tr> - <tr><td id="addl"><b>Additional cloud provider information (if available):</b></td></tr> - %if credDetails.provider.region_connection != None: - <tr> - <td> Region connection: </td> - <td> ${credDetails.provider.region_connection} </td> - </tr> - %endif - %if credDetails.provider.region_name != None: - <tr> - <td> Region name: </td> - <td> ${credDetails.provider.region_name} </td> - </tr> - %endif - %if credDetails.provider.region_endpoint != None: - <tr> - <td> Region endpoint: </td> - <td> ${credDetails.provider.region_endpoint} </td> - </tr> - %endif - %if credDetails.provider.is_secure != None: - <tr> - <td> Is secure: </td> - <td> ${credDetails.provider.is_secure} </td> - </tr> - %endif - %if credDetails.provider.host != None: - <tr> - <td> Host: </td> - <td> ${credDetails.provider.host} </td> - </tr> - %endif - %if credDetails.provider.port != None: - <tr> - <td> Port: </td> - <td> ${credDetails.provider.port} </td> - </tr> - %endif - %if credDetails.provider.proxy != None: - <tr> - <td> Proxy: </td> - <td> ${credDetails.provider.proxy} </td> - </tr> - %endif - %if credDetails.provider.proxy_port != None: - <tr> - <td> Proxy port: </td> - <td> ${credDetails.provider.proxy_port} </td> - </tr> - %endif - %if credDetails.provider.proxy_pass != None: - <tr> - <td> Proxy pass: </td> - <td> ${credDetails.provider.proxy_pass} </td> - </tr> - %endif - %if credDetails.provider.debug != None: - <tr> - <td> Debug: </td> - <td> ${credDetails.provider.debug} </td> - </tr> - %endif - %if credDetails.provider.https_connection_factory != None: - <tr> - <td> HTTPS connection factory: </td> - <td> ${credDetails.provider.https_connection_factory} </td> - </tr> - %endif - %if credDetails.provider.path != None: - <tr> - <td> Path: </td> - <td> ${credDetails.provider.path} </td> - </tr> - %endif - </table> -</%def> --- a/lib/galaxy/cloud/providers/eucalyptus.py +++ /dev/null @@ -1,1039 +0,0 @@ -import subprocess, threading, os, errno, time, datetime -from Queue import Queue, Empty -from datetime import datetime - -from galaxy import model # Database interaction class -from galaxy.model import mapping -from galaxy.datatypes.data import nice_size -from galaxy.util.bunch import Bunch -from galaxy.cloud import UCIwrapper -from Queue import Queue -from sqlalchemy import or_, and_ - -import galaxy.eggs -galaxy.eggs.require("boto") -from boto.ec2.connection import EC2Connection -from boto.ec2.regioninfo import RegionInfo -import boto.exception -import boto - -import logging -log = logging.getLogger( __name__ ) - -uci_states = Bunch( - NEW_UCI = "newUCI", - NEW = "new", - CREATING = "creating", - DELETING_UCI = "deletingUCI", - DELETING = "deleting", - SUBMITTED_UCI = "submittedUCI", - SUBMITTED = "submitted", - SHUTTING_DOWN_UCI = "shutting-downUCI", - SHUTTING_DOWN = "shutting-down", - ADD_STORAGE_UCI = "add-storageUCI", - ADD_STORAGE = "add-storage", - AVAILABLE = "available", - RUNNING = "running", - PENDING = "pending", - ERROR = "error", - DELETED = "deleted", - SNAPSHOT_UCI = "snapshotUCI", - SNAPSHOT = "snapshot" -) - -instance_states = Bunch( - TERMINATED = "terminated", - SUBMITTED = "submitted", - RUNNING = "running", - ADDING = "adding-storage", - PENDING = "pending", - SHUTTING_DOWN = "shutting-down", - ERROR = "error" -) - -store_status = Bunch( - WAITING = "waiting", - IN_USE = "in-use", - ADDING = "adding", - CREATING = "creating", - DELETED = 'deleted', - ERROR = "error" -) - -snapshot_status = Bunch( - SUBMITTED = 'submitted', - PENDING = 'pending', - COMPLETED = 'completed', - DELETE = 'delete', - DELETED= 'deleted', - ERROR = "error" -) - -class EucalyptusCloudProvider( object ): - """ - Eucalyptus-based cloud provider implementation for managing instances. - """ - STOP_SIGNAL = object() - def __init__( self, app ): - self.type = "eucalyptus" # cloud provider type (e.g., ec2, eucalyptus, opennebula) - self.zone = "epc" - self.queue = Queue() - self.sa_session = app.model.context - - self.threads = [] - nworkers = 5 - log.info( "Starting eucalyptus cloud controller workers..." ) - for i in range( nworkers ): - worker = threading.Thread( target=self.run_next ) - worker.start() - self.threads.append( worker ) - log.debug( "%d eucalyptus cloud workers ready", nworkers ) - - def shutdown( self ): - """Attempts to gracefully shut down the monitor thread""" - log.info( "sending stop signal to worker threads in eucalyptus cloud manager" ) - for i in range( len( self.threads ) ): - self.queue.put( self.STOP_SIGNAL ) - log.info( "eucalyptus cloud manager stopped" ) - - def put( self, uci_wrapper ): - """ - Add uci_wrapper object to the end of the request queue to be handled by - this cloud provider. - """ - state = uci_wrapper.get_uci_state() - uci_wrapper.change_state( state.split('U')[0] ) # remove 'UCI' from end of state description (i.e., mark as accepted and ready for processing) - self.queue.put( uci_wrapper ) - - def run_next( self ): - """Process next request, waiting until one is available if necessary.""" - cnt = 0 - while 1: - uci_wrapper = self.queue.get() - uci_state = uci_wrapper.get_uci_state() - if uci_state is self.STOP_SIGNAL: - return - try: - if uci_state==uci_states.NEW: - self.create_uci( uci_wrapper ) - elif uci_state==uci_states.DELETING: - self.delete_uci( uci_wrapper ) - elif uci_state==uci_states.SUBMITTED: - self.start_uci( uci_wrapper ) - #self.dummy_start_uci( uci_wrapper ) - elif uci_state==uci_states.SHUTTING_DOWN: - self.stop_uci( uci_wrapper ) - elif uci_state==uci_states.SNAPSHOT: - self.snapshot_uci( uci_wrapper ) - elif uci_state==uci_states.ADD_STORAGE: - self.add_storage_to_uci( uci_wrapper ) - except: - log.exception( "Uncaught exception executing cloud request." ) - cnt += 1 - - def get_connection( self, uci_wrapper ): - """ - Establishes cloud connection using user's credentials associated with given UCI - """ - log.debug( 'Establishing %s cloud connection.' % self.type ) - provider = uci_wrapper.get_provider() - try: - region = RegionInfo( None, provider.region_name, provider.region_endpoint ) - except Exception, ex: - err = "Selecting region with cloud provider failed: " + str( ex ) - log.error( err ) - uci_wrapper.set_error( err, True ) - return None - try: - conn = EC2Connection( aws_access_key_id=uci_wrapper.get_access_key(), - aws_secret_access_key=uci_wrapper.get_secret_key(), - is_secure=provider.is_secure, - port=provider.port, - region=region, - path=provider.path ) - except boto.exception.EC2ResponseError, e: - err = "Establishing connection with cloud failed: " + str( e ) - log.error( err ) - uci_wrapper.set_error( err, True ) - return None - - return conn - - def check_key_pair( self, uci_wrapper, conn ): - """ - Check if a key pair associated with this UCI exists on cloud provider. - If yes, return key pair name; otherwise, generate a key pair with the cloud - provider and, again, return key pair name. - Key pair name for given UCI is generated from UCI's name and suffix '_kp' - """ - kp = None - kp_name = uci_wrapper.get_name().replace(' ','_') + "_kp" - log.debug( "Checking user's key pair: '%s'" % kp_name ) - try: - kp = conn.get_key_pair( kp_name ) - uci_kp_name = uci_wrapper.get_key_pair_name() - uci_material = uci_wrapper.get_key_pair_material() - if kp != None: - if kp.name != uci_kp_name or uci_material == None: - # key pair exists on the cloud but not in local database, so re-generate it (i.e., delete and then create) - try: - conn.delete_key_pair( kp_name ) - kp = self.create_key_pair( conn, kp_name ) - uci_wrapper.set_key_pair( kp.name, kp.material ) - except boto.exception.EC2ResponseError, e: - err = "EC2 response error while deleting key pair: " + str( e ) - log.error( err ) - uci_wrapper.set_error( err, True ) - else: - try: - kp = self.create_key_pair( conn, kp_name ) - uci_wrapper.set_key_pair( kp.name, kp.material ) - except boto.exception.EC2ResponseError, e: - err = "EC2 response error while creating key pair: " + str( e ) - log.error( err ) - uci_wrapper.set_error( err, True ) - except Exception, ex: - err = "Exception while creating key pair: " + str( ex ) - log.error( err ) - uci_wrapper.set_error( err, True ) - except boto.exception.EC2ResponseError, e: # No keypair under this name exists so create it - if e.code == 'InvalidKeyPair.NotFound': - log.info( "No keypair found, creating keypair '%s'" % kp_name ) - kp = self.create_key_pair( conn, kp_name ) - uci_wrapper.set_key_pair( kp.name, kp.material ) - else: - err = "EC2 response error while retrieving key pair: " + str( e ) - log.error( err ) - uci_wrapper.set_error( err, True ) - - if kp != None: - return kp.name - else: - return None - - def create_key_pair( self, conn, kp_name ): - """ Initiate creation of key pair under kp_name by current cloud provider. """ - try: - return conn.create_key_pair( kp_name ) - except boto.exception.EC2ResponseError, e: - return None - - def get_mi_id( self, uci_wrapper, i_index ): - """ - Get appropriate machine image (mi) ID based on instance type. - """ - i_type = uci_wrapper.get_instance_type( i_index ) - if i_type=='m1.small' or i_type=='c1.medium': - arch = 'i386' - else: - arch = 'x86_64' - - mi = self.sa_session.query( model.CloudImage ).filter_by( deleted=False, provider_type=self.type, architecture=arch ).first() - if mi: - return mi.image_id - else: - err = "Machine image could not be retrieved" - log.error( "%s for UCI '%s'." % (err, uci_wrapper.get_name() ) ) - uci_wrapper.set_error( err+". Contact site administrator to ensure needed machine image is registered.", True ) - return None - - def create_uci( self, uci_wrapper ): - """ - Create User Configured Instance (UCI) - i.e., create storage volume on cloud provider - and register relevant information in local Galaxy database. - """ - conn = self.get_connection( uci_wrapper ) - - # Because only 1 storage volume may be created at UCI config time, index of this storage volume in local Galaxy DB w.r.t - # current UCI is 0; therefore, it can be referenced in following code - log.info( "Creating volume in zone '%s'..." % uci_wrapper.get_uci_availability_zone() ) - if uci_wrapper.get_uci_availability_zone()=='': - log.info( "Availability zone for UCI (i.e., storage volume) was not selected, using default zone: %s" % self.zone ) - uci_wrapper.set_store_availability_zone( self.zone ) - -# log.debug( "Creating volume; using command: conn.create_volume( %s, '%s', snapshot=None )" % ( uci_wrapper.get_store_size( 0 ), uci_wrapper.get_uci_availability_zone() )) -# vol = conn.create_volume( uci_wrapper.get_store_size( 0 ), uci_wrapper.get_uci_availability_zone(), snapshot=None ) -# uci_wrapper.set_store_volume_id( 0, vol.id ) - store = uci_wrapper.get_all_stores_in_status( store_status.ADDING )[0] # Because at UCI creation time only 1 storage volume can be created, reference it directly - - log.info( "Creating storage volume in zone '%s' of size '%s'..." % ( uci_wrapper.get_uci_availability_zone(), store.size ) ) - # Because only 1 storage volume may be created at UCI config time, index of this storage volume in local Galaxy DB w.r.t - # current UCI is 0, so reference it in following methods - vol = conn.create_volume( store.size, uci_wrapper.get_uci_availability_zone(), snapshot=None ) - uci_wrapper.set_store_volume_id( store.id, vol.id ) - - # Retrieve created volume again to get updated status - try: - vl = conn.get_all_volumes( [vol.id] ) - except boto.exception.EC2ResponseError, e: - err = "EC2 response error while retrieving (i.e., updating status) of just created storage volume '" + vol.id + "': " + str( e ) - log.error( err ) - uci_wrapper.set_store_status( vol.id, uci_states.ERROR ) - uci_wrapper.set_error( err, True ) - return - except Exception, ex: - err = "Error while retrieving (i.e., updating status) of just created storage volume '" + vol.id + "': " + str( ex ) - log.error( err ) - uci_wrapper.set_error( err, True ) - return - - if len( vl ) > 0: - # EPC does not allow creation of storage volumes (it deletes one as soon as it is created, so manually set uci_state here) - if vl[0].status == store_status.DELETED: - uci_wrapper.change_state( uci_state=uci_states.AVAILABLE ) - else: - uci_wrapper.change_state( uci_state=vl[0].status ) - uci_wrapper.set_store_status( vol.id, vl[0].status ) - else: - err = "Volume '" + vol.id +"' not found by EC2 after being created." - log.error( err ) - uci_wrapper.set_store_status( vol.id, uci_states.ERROR ) - uci_wrapper.set_error( err, True ) - - def delete_uci( self, uci_wrapper ): - """ - Delete UCI - i.e., delete all storage volumes associated with this UCI. - NOTE that this implies deletion of any and all data associated - with this UCI from the cloud. All data will be deleted. - Information in local Galaxy database is marked as deleted but not actually removed - from the database. - """ - conn = self.get_connection( uci_wrapper ) - vl = [] # volume list - count = 0 # counter for checking if all volumes assoc. w/ UCI were deleted - - # Get all volumes assoc. w/ UCI, delete them from cloud as well as in local DB - vl = uci_wrapper.get_all_stores() - deletedList = [] - failedList = [] - for v in vl: - log.debug( "Deleting volume with id='%s'" % v.volume_id ) - try: - if conn.delete_volume( v.volume_id ): - deletedList.append( v.volume_id ) - v.deleted = True - self.sa_session.add( v ) - self.sa_session.flush() - count += 1 - else: - failedList.append( v.volume_id ) - except boto.exception.EC2ResponseError, e: - err = "EC2 response error while deleting storage volume '" + v.volume_id + "': " + str( e ) - log.error( err ) - uci_wrapper.set_store_error( err, store_id = v.volume_id ) - uci_wrapper.set_error( err, True ) - - # Delete UCI if all of associated - if count == len( vl ): - uci_wrapper.set_deleted() - else: - err = "Deleting following volume(s) failed: "+ str( failedList )+". However, these volumes were successfully deleted: " \ - + str( deletedList ) +". MANUAL intervention and processing needed." - log.error( err ) - uci_wrapper.set_error( err, True ) - - def snapshot_uci( self, uci_wrapper ): - """ - Initiate creation of a snapshot by cloud provider for all storage volumes - associated with this UCI. - """ - if uci_wrapper.get_uci_state() != uci_states.ERROR: - conn = self.get_connection( uci_wrapper ) - - snapshots = uci_wrapper.get_snapshots( status = snapshot_status.SUBMITTED ) - for snapshot in snapshots: - log.debug( "Snapshot DB id: '%s', volume id: '%s'" % ( snapshot.id, snapshot.store.volume_id ) ) - try: - snap = conn.create_snapshot( volume_id=snapshot.store.volume_id ) - snap_id = str( snap ).split(':')[1] - uci_wrapper.set_snapshot_id( snapshot.id, snap_id ) - sh = conn.get_all_snapshots( snap_id ) # get updated status - uci_wrapper.set_snapshot_status( status=sh[0].status, snap_id=snap_id ) - except boto.exception.EC2ResponseError, e: - err = "Cloud provider response error while creating snapshot: " + str( e ) - log.error( err ) - uci_wrapper.set_snapshot_error( error=err, snap_index=snapshot.id, set_status=True ) - uci_wrapper.set_error( err, True ) - return - except Exception, ex: - err = "Error while creating snapshot: " + str( ex ) - log.error( err ) - uci_wrapper.set_snapshot_error( error=err, snap_index=snapshot.id, set_status=True ) - uci_wrapper.set_error( err, True ) - return - - uci_wrapper.change_state( uci_state=uci_states.AVAILABLE ) - -# if uci_wrapper.get_uci_state() != uci_states.ERROR: -# -# snapshots = uci_wrapper.get_snapshots( status = 'submitted' ) -# for snapshot in snapshots: -# uci_wrapper.set_snapshot_id( snapshot.id, None, 'euca_error' ) -# -# log.debug( "Eucalyptus snapshot attempted by user for UCI '%s'" % uci_wrapper.get_name() ) -# uci_wrapper.set_error( "Eucalyptus does not support creation of snapshots at this moment. No snapshot or other changes were performed. \ -# Feel free to resent state of this instance and use it normally.", True ) - - - def add_storage_to_uci( self, uci_wrapper ): - """ Adds more storage to specified UCI """ - uci_wrapper.set_error( "Adding storage to eucalyptus-based clouds is not yet supported.", True ) - - def dummy_start_uci( self, uci_wrapper ): - - uci = uci_wrapper.get_uci() - log.debug( "Would be starting instance '%s'" % uci.name ) -# uci_wrapper.change_state( uci_states.SUBMITTED_UCI ) -# log.debug( "Set UCI state to SUBMITTED_UCI" ) - log.debug( "Sleeping a bit... (%s)" % uci.name ) - time.sleep(10) - log.debug( "Woke up! (%s)" % uci.name ) - - def start_uci( self, uci_wrapper ): - """ - Start instance(s) of given UCI on the cloud. - """ - if uci_wrapper.get_uci_state() != uci_states.ERROR: - conn = self.get_connection( uci_wrapper ) - self.check_key_pair( uci_wrapper, conn ) - if uci_wrapper.get_key_pair_name() == None: - err = "Key pair not found" - log.error( "%s for UCI '%s'." % ( err, uci_wrapper.get_name() ) ) - uci_wrapper.set_error( err + ". Try resetting the state and starting the instance again.", True ) - return - - i_indexes = uci_wrapper.get_instances_indexes( state=instance_states.SUBMITTED ) # Get indexes of i_indexes associated with this UCI that are in 'submitted' state - log.debug( "Starting instances with IDs: '%s' associated with UCI '%s' " % ( i_indexes, uci_wrapper.get_name(), ) ) - if len( i_indexes ) > 0: - for i_index in i_indexes: - # Get machine image for current instance - mi_id = self.get_mi_id( uci_wrapper, i_index ) - log.debug( "mi_id: %s, uci_wrapper.get_key_pair_name(): %s" % ( mi_id, uci_wrapper.get_key_pair_name() ) ) - uci_wrapper.set_mi( i_index, mi_id ) - - if uci_wrapper.get_uci_state() != uci_states.ERROR: - # Start an instance - log.debug( "Starting UCI instance '%s'" % uci_wrapper.get_name() ) - log.debug( "Using following command: conn.run_instances( image_id='%s', key_name='%s', instance_type='%s' )" - % ( mi_id, uci_wrapper.get_key_pair_name(), uci_wrapper.get_instance_type( i_index ) ) ) - reservation = None - try: - reservation = conn.run_instances( image_id=mi_id, - key_name=uci_wrapper.get_key_pair_name(), - instance_type=uci_wrapper.get_instance_type( i_index ) ) - except boto.exception.EC2ResponseError, e: - err = "EC2 response error when starting UCI '"+ uci_wrapper.get_name() +"': " + str( e ) - log.error( err ) - uci_wrapper.set_error( err, True ) - except Exception, ex: - err = "Error when starting UCI '" + uci_wrapper.get_name() + "': " + str( ex ) - log.error( err ) - uci_wrapper.set_error( err, True ) - # Record newly available instance data into local Galaxy database - if reservation: - l_time = datetime.utcnow() -# uci_wrapper.set_instance_launch_time( self.format_time( reservation.instances[0].launch_time ), i_index=i_index ) - uci_wrapper.set_instance_launch_time( l_time, i_index=i_index ) - if not uci_wrapper.uci_launch_time_set(): - uci_wrapper.set_uci_launch_time( l_time ) - try: - uci_wrapper.set_reservation_id( i_index, str( reservation ).split(":")[1] ) - # TODO: if more than a single instance will be started through single reservation, change this reference from element [0] - i_id = str( reservation.instances[0]).split(":")[1] - uci_wrapper.set_instance_id( i_index, i_id ) - s = reservation.instances[0].state - uci_wrapper.change_state( s, i_id, s ) - vol_id = uci_wrapper.get_store_volume_id( store_id=0 ) # TODO: Once more that one vol/UCI is allowed, update this! - uci_wrapper.set_store_status( vol_id, store_status.WAITING ) - log.debug( "Instance of UCI '%s' started, current state: '%s'" % ( uci_wrapper.get_name(), uci_wrapper.get_uci_state() ) ) - except boto.exception.EC2ResponseError, e: - err = "EC2 response error when retrieving instance information for UCI '" + uci_wrapper.get_name() + "': " + str( e ) - log.error( err ) - uci_wrapper.set_error( err, True ) - else: - log.error( "UCI '%s' is in 'error' state, starting instance was aborted." % uci_wrapper.get_name() ) - else: - err = "No instances in state '"+ instance_states.SUBMITTED +"' found for UCI '" + uci_wrapper.get_name() + \ - "'. Nothing to start." - log.error( err ) - uci_wrapper.set_error( err, True ) - else: - log.error( "UCI '%s' is in 'error' state, starting instance was aborted." % uci_wrapper.get_name() ) - - def stop_uci( self, uci_wrapper): - """ - Stop all cloud instances associated with given UCI. - """ - conn = self.get_connection( uci_wrapper ) - - # Get all instances associated with given UCI - il = uci_wrapper.get_instances_ids() # instance list - # Process list of instances and remove any references to empty instance id's - for i in il: - if i is None: - il.remove( i ) - log.debug( 'List of instances being terminated: %s' % il ) - rl = conn.get_all_instances( il ) # Reservation list associated with given instances - - # Initiate shutdown of all instances under given UCI - cnt = 0 - stopped = [] - not_stopped = [] - for r in rl: - for inst in r.instances: - log.debug( "Sending stop signal to instance '%s' associated with reservation '%s' (UCI: %s)." % ( inst, r, uci_wrapper.get_name() ) ) - try: - inst.stop() - uci_wrapper.set_stop_time( datetime.utcnow(), i_id=inst.id ) - uci_wrapper.change_state( instance_id=inst.id, i_state=inst.update() ) - stopped.append( inst ) - except boto.exception.EC2ResponseError, e: - not_stopped.append( inst ) - err = "EC2 response error when stopping instance '" + inst.instance_id + "': " + str( e ) - log.error( err ) - uci_wrapper.set_error( err, True ) - - uci_wrapper.reset_uci_launch_time() - log.debug( "Termination was initiated for all instances of UCI '%s'." % uci_wrapper.get_name() ) - -# dbInstances = get_instances( trans, uci ) #TODO: handle list! -# -# # Get actual cloud instance object -# cloudInstance = get_cloud_instance( conn, dbInstances.instance_id ) -# -# # TODO: Detach persistent storage volume(s) from instance and update volume data in local database -# stores = get_stores( trans, uci ) -# for i, store in enumerate( stores ): -# log.debug( "Detaching volume '%s' to instance '%s'." % ( store.volume_id, dbInstances.instance_id ) ) -# mntDevice = store.device -# volStat = None -## Detaching volume does not work with Eucalyptus Public Cloud, so comment it out -## try: -## volStat = conn.detach_volume( store.volume_id, dbInstances.instance_id, mntDevice ) -## except: -## log.debug ( 'Error detaching volume; still going to try and stop instance %s.' % dbInstances.instance_id ) -# store.attach_time = None -# store.device = None -# store.inst.instance_id = None -# store.status = volStat -# log.debug ( '***** volume status: %s' % volStat ) -# -# # Stop the instance and update status in local database -# cloudInstance.stop() -# dbInstances.stop_time = datetime.utcnow() -# while cloudInstance.state != 'terminated': -# log.debug( "Stopping instance %s state; current state: %s" % ( str( cloudInstance ).split(":")[1], cloudInstance.state ) ) -# time.sleep(3) -# cloudInstance.update() -# dbInstances.state = cloudInstance.state -# -# # Reset relevant UCI fields -# uci.state = 'available' -# uci.launch_time = None -# -# # Persist -# session = trans.sa_session -## session.save_or_update( stores ) -# session.save_or_update( dbInstances ) # TODO: Is this going to work w/ multiple instances stored in dbInstances variable? -# session.save_or_update( uci ) -# session.flush() -# trans.log_event( "User stopped cloud instance '%s'" % uci.name ) -# trans.set_message( "Galaxy instance '%s' stopped." % uci.name ) - - def update( self ): - """ - Run status update on all instances that are in 'running', 'pending', or 'shutting-down' state. - Run status update on all storage volumes whose status is 'in-use', 'creating', or 'None'. - Run status update on all snapshots whose status is 'pending' or 'delete' - Run status update on any zombie UCIs, i.e., UCI's that is in 'submitted' state for an - extended period of time. - - Reason behind this method is to sync state of local DB and real-world resources - """ - log.debug( "Running general status update for %s UCIs..." % self.type ) - # Update instances - instances = self.sa_session.query( model.CloudInstance ) \ - .filter( or_( model.CloudInstance.table.c.state==instance_states.RUNNING, - model.CloudInstance.table.c.state==instance_states.PENDING, - model.CloudInstance.table.c.state==instance_states.SHUTTING_DOWN ) ) \ - .all() - for inst in instances: - if self.type == inst.uci.credentials.provider.type: - log.debug( "[%s] Running general status update on instance '%s'" % ( inst.uci.credentials.provider.type, inst.instance_id ) ) - self.update_instance( inst ) - - # Update storage volume(s) - stores = self.sa_session.query( model.CloudStore ) \ - .filter( or_( model.CloudStore.table.c.status==store_status.IN_USE, - model.CloudStore.table.c.status==store_status.CREATING, - model.CloudStore.table.c.status==store_status.WAITING, - model.CloudStore.table.c.status==None ) ) \ - .all() - for store in stores: - if self.type == store.uci.credentials.provider.type: # and store.volume_id != None: - log.debug( "[%s] Running general status update on store with local database ID: '%s'" % ( store.uci.credentials.provider.type, store.id ) ) - self.update_store( store ) - - # Update pending snapshots or delete ones marked for deletion - snapshots = self.sa_session.query( model.CloudSnapshot ) \ - .filter( or_( model.CloudSnapshot.table.c.status == snapshot_status.PENDING, model.CloudSnapshot.table.c.status == snapshot_status.DELETE ) ) \ - .all() - for snapshot in snapshots: - if self.type == snapshot.uci.credentials.provider.type and snapshot.status == snapshot_status.PENDING: - log.debug( "[%s] Running general status update on snapshot '%s'" % ( snapshot.uci.credentials.provider.type, snapshot.snapshot_id ) ) - self.update_snapshot( snapshot ) - elif self.type == snapshot.uci.credentials.provider.type and snapshot.status == snapshot_status.DELETE: - log.debug( "[%s] Initiating deletion of snapshot '%s'" % ( snapshot.uci.credentials.provider.type, snapshot.snapshot_id ) ) - self.delete_snapshot( snapshot ) - - # Attempt at updating any zombie UCIs (i.e., instances that have been in SUBMITTED state for longer than expected - see below for exact time) - zombies = self.sa_session.query( model.UCI ).filter_by( state=uci_states.SUBMITTED ).all() - for zombie in zombies: - log.debug( "zombie UCI: %s" % zombie.name ) - z_instances = self.sa_session.query( model.CloudInstance ) \ - .filter( or_( model.CloudInstance.table.c.state != instance_states.TERMINATED, - model.CloudInstance.table.c.state == None ) ) \ - .all() - for z_inst in z_instances: - if self.type == z_inst.uci.credentials.provider.type: -# log.debug( "z_inst.id: '%s', state: '%s'" % ( z_inst.id, z_inst.state ) ) - td = datetime.utcnow() - z_inst.update_time -# log.debug( "z_inst.id: %s, time delta is %s sec" % ( z_inst.id, td.seconds ) ) - if td.seconds > 180: # if instance has been in SUBMITTED state for more than 3 minutes - log.debug( "[%s](td=%s) Running zombie repair update on instance with DB id '%s'" % ( z_inst.uci.credentials.provider.type, td.seconds, z_inst.id ) ) - self.process_zombie( z_inst ) - - def update_instance( self, inst ): - """ - Update information in local database for given instance as it is obtained from cloud provider. - Along with updating information about given instance, information about the UCI controlling - this instance is also updated. - """ - # Get credentials associated wit this instance - uci_id = inst.uci_id - uci = self.sa_session.query( model.UCI ).get( uci_id ) - self.sa_session.refresh( uci ) - conn = self.get_connection_from_uci( uci ) - - # Get reservations handle for given instance - try: - rl= conn.get_all_instances( [inst.instance_id] ) - except boto.exception.EC2ResponseError, e: - err = "Retrieving instance(s) from cloud failed for UCI '"+ uci.name +"' during general status update: " + str( e ) - log.error( err ) - uci.error = err - uci.state = uci_states.ERROR - self.sa_session.add( uci ) - self.sa_session.flush() - return None - - # Because references to reservations are deleted shortly after instances have been terminated, getting an empty list as a response to a query - # typically means the instance has successfully shut down but the check was not performed in short enough amount of time. Until an alternative solution - # is found, below code sets state of given UCI to 'error' to indicate to the user something out of ordinary happened. - if len( rl ) == 0: - err = "Instance ID '"+inst.instance_id+"' was not found by the cloud provider. Instance might have crashed or otherwise been terminated."+ \ - "Manual check is recommended." - log.error( err ) - inst.error = err - uci.error = err - inst.state = instance_states.TERMINATED - uci.state = uci_states.ERROR - uci.launch_time = None - self.sa_session.add( inst ) - self.sa_session.add( uci ) - self.sa_session.flush() - # Update instance status in local DB with info from cloud provider - for r in rl: - for i, cInst in enumerate( r.instances ): - try: - s = cInst.update() - log.debug( "Checking state of cloud instance '%s' associated with UCI '%s' and reservation '%s'. State='%s'" % ( cInst, uci.name, r, s ) ) - if s != inst.state: - inst.state = s - self.sa_session.add( inst ) - self.sa_session.flush() - # After instance has shut down, ensure UCI is marked as 'available' - if s == instance_states.TERMINATED and uci.state != uci_states.ERROR: - uci.state = uci_states.AVAILABLE - uci.launch_time = None - self.sa_session.add( uci ) - self.sa_session.flush() - # Making sure state of UCI is updated. Once multiple instances become associated with single UCI, this will need to be changed. - if s != uci.state and s != instance_states.TERMINATED: - uci.state = s - self.sa_session.add( uci ) - self.sa_session.flush() - if cInst.public_dns_name != inst.public_dns: - inst.public_dns = cInst.public_dns_name - self.sa_session.add( inst ) - self.sa_session.flush() - if cInst.private_dns_name != inst.private_dns: - inst.private_dns = cInst.private_dns_name - self.sa_session.add( inst ) - self.sa_session.flush() - except boto.exception.EC2ResponseError, e: - err = "Updating instance status from cloud failed for UCI '"+ uci.name + "' during general status update: " + str( e ) - log.error( err ) - uci.error = err - uci.state = uci_states.ERROR - self.sa_session.add( uci ) - self.sa_session.flush() - return None - - def update_store( self, store ): - """ - Update information in local database for given storage volume as it is obtained from cloud provider. - Along with updating information about given storage volume, information about the UCI controlling - this storage volume is also updated. - """ - # Get credentials associated wit this store - uci_id = store.uci_id - uci = self.sa_session.query( model.UCI ).get( uci_id ) - self.sa_session.refresh( uci ) - conn = self.get_connection_from_uci( uci ) - - if store.volume_id != None: - # Get reservations handle for given store - try: - log.debug( "Updating storage volume command: vl = conn.get_all_volumes( [%s] )" % store.volume_id ) - vl = conn.get_all_volumes( [store.volume_id] ) - except boto.exception.EC2ResponseError, e: - err = "Retrieving volume(s) from cloud failed for UCI '"+ uci.name + "' during general status update: " + str( e ) - log.error( err ) - uci.error = err - uci.state = uci_states.ERROR - self.sa_session.add( uci ) - self.sa_session.flush() - return None - - # Update store status in local DB with info from cloud provider - if len(vl) > 0: - try: - log.debug( "Storage volume '%s' current status: '%s'" % (store.volume_id, vl[0].status ) ) - if store.status != vl[0].status: - # In case something failed during creation of UCI but actual storage volume was created and yet - # UCI state remained as 'new', try to remedy this by updating UCI state here - if ( store.status == None ) and ( store.volume_id != None ): - uci.state = vl[0].status - self.sa_session.add( uci ) - self.sa_session.flush() - # If UCI was marked in state 'CREATING', update its status to reflect new status - elif ( uci.state == uci_states.CREATING ): - # Because Eucalyptus Public Cloud (EPC) deletes volumes immediately after they are created, artificially - # set status of given UCI to 'available' based on storage volume's availability zone (i.e., it's residing - # in EPC as opposed to some other Eucalyptus based cloud that allows creation of storage volumes. - if store.availability_zone == 'epc': - uci.state = uci_states.AVAILABLE - else: - uci.state = vl[0].status - - self.sa_session.add( uci ) - self.sa_session.flush() - - store.status = vl[0].status - self.sa_session.add( store ) - self.sa_session.flush() - if store.inst != None: - if store.inst.instance_id != vl[0].instance_id: - store.inst.instance_id = vl[0].instance_id - self.sa_session.add( store ) - self.sa_session.flush() - if store.attach_time != vl[0].attach_time: - store.attach_time = vl[0].attach_time - self.sa_session.add( store ) - self.sa_session.flush() - if store.device != vl[0].device: - store.device = vl[0].device - self.sa_session.add( store ) - self.sa_session.flush() - except boto.exception.EC2ResponseError, e: - err = "Updating status of volume(s) from cloud failed for UCI '"+ uci.name + "' during general status update: " + str( e ) - log.error( err ) - uci.error = err - uci.state = uci_states.ERROR - self.sa_session.add( uci ) - self.sa_session.flush() - return None - else: - err = "No storage volumes returned by cloud provider on general update" - log.error( "%s for UCI '%s'" % ( err, uci.name ) ) - store.status = store_status.ERROR - store.error = err - uci.error = err - uci.state = uci_states.ERROR - self.sa_session.add( uci ) - self.sa_session.add( store ) - self.sa_session.flush() - else: - err = "Missing storage volume ID in local database on general update. Manual check is needed to check " \ - "if storage volume was actually created by cloud provider." - log.error( "%s (for UCI '%s')" % ( err, uci.name ) ) - store.status = store_status.ERROR - store.error = err - uci.error = err - uci.state = uci_states.ERROR - self.sa_session.add( uci ) - self.sa_session.add( store ) - self.sa_session.flush() - - def update_snapshot( self, snapshot ): - """ - Update information in local database for given snapshot as it is obtained from cloud provider. - Along with updating information about given snapshot, information about the UCI controlling - this snapshot is also updated. - """ - # Get credentials associated wit this store - uci_id = snapshot.uci_id - uci = self.sa_session.query( model.UCI ).get( uci_id ) - self.sa_session.refresh( uci ) - conn = self.get_connection_from_uci( uci ) - - try: - log.debug( "Updating status of snapshot '%s'" % snapshot.snapshot_id ) - snap = conn.get_all_snapshots( [snapshot.snapshot_id] ) - if len( snap ) > 0: - log.debug( "Snapshot '%s' status: %s" % ( snapshot.snapshot_id, snap[0].status ) ) - snapshot.status = snap[0].status - self.sa_session.add( snapshot ) - self.sa_session.flush() - else: - err = "No snapshots returned by EC2 on general update" - log.error( "%s for UCI '%s'" % ( err, uci.name ) ) - snapshot.status = snapshot_status.ERROR - snapshot.error = err - uci.error = err - uci.state = uci_states.ERROR - self.sa_session.add( uci ) - self.sa_session.add( snapshot ) - self.sa_session.flush() - except boto.exception.EC2ResponseError, e: - err = "EC2 response error while updating snapshot status: " + str( e ) - log.error( err ) - snapshot.status = snapshot_status.ERROR - snapshot.error = err - uci.error = err - uci.state = uci_states.ERROR - self.sa_session.add( uci ) - self.sa_session.add( snapshot ) - self.sa_session.flush() - except Exception, ex: - err = "Error while updating snapshot status: " + str( ex ) - log.error( err ) - snapshot.status = snapshot_status.ERROR - snapshot.error = err - uci.error = err - uci.state = uci_states.ERROR - self.sa_session.add( uci ) - self.sa_session.add( snapshot ) - self.sa_session.flush() - - def delete_snapshot( self, snapshot ): - """ - Initiate deletion of given snapshot from cloud provider. - """ - if snapshot.status == snapshot_status.DELETE: - # Get credentials associated wit this store - uci_id = snapshot.uci_id - uci = self.sa_session.query( model.UCI ).get( uci_id ) - self.sa_session.refresh( uci ) - conn = self.get_connection_from_uci( uci ) - - try: - log.debug( "Deleting snapshot '%s'" % snapshot.snapshot_id ) - snap = conn.delete_snapshot( snapshot.snapshot_id ) - if snap == True: - snapshot.deleted = True - snapshot.status = snapshot_status.DELETED - self.sa_session.add( snapshot ) - self.sa_session.flush() - return snap - except boto.exception.EC2ResponseError, e: - err = "EC2 response error while deleting snapshot: " + str( e ) - log.error( err ) - snapshot.status = snapshot_status.ERROR - snapshot.error = err - uci.error = err - uci.state = uci_states.ERROR - self.sa_session.add( uci ) - self.sa_session.add( snapshot ) - self.sa_session.flush() - except Exception, ex: - err = "Error while deleting snapshot: " + str( ex ) - log.error( err ) - snapshot.status = snapshot_status.ERROR - snapshot.error = err - uci.error = err - uci.state = uci_states.ERROR - self.sa_session.add( uci ) - self.sa_session.add( snapshot ) - self.sa_session.flush() - else: - err = "Cannot delete snapshot '"+snapshot.snapshot_id+"' because its status is '"+snapshot.status+"'. Only snapshots with '" + \ - snapshot_status.COMPLETED+"' status can be deleted." - log.error( err ) - snapshot.error = err - self.sa_session.add( snapshot ) - self.sa_session.flush() - - def process_zombie( self, inst ): - """ - Attempt at discovering if starting a cloud instance was successful but local database was not updated - accordingly or if something else failed and instance was never started. Currently, no automatic - repairs are being attempted; instead, appropriate error messages are set. - """ - uci_id = inst.uci_id - uci = self.sa_session.query( model.UCI ).get( uci_id ) - self.sa_session.refresh( uci ) - - # Check if any instance-specific information was written to local DB; if 'yes', set instance and UCI's error message - # suggesting manual check. - if inst.launch_time != None or inst.reservation_id != None or inst.instance_id != None: - # Try to recover state - this is best-case effort, so if something does not work immediately, not - # recovery steps are attempted. Recovery is based on hope that instance_id is available in local DB; if not, - # report as error. - # Fields attempting to be recovered are: reservation_id, instance status, and launch_time - if inst.instance_id != None: - conn = self.get_connection_from_uci( uci ) - rl = conn.get_all_instances( [inst.instance_id] ) # reservation list - # Update local DB with relevant data from instance - if inst.reservation_id == None: - try: - inst.reservation_id = str(rl[0]).split(":")[1] - except: # something failed, so skip - pass - - try: - state = rl[0].instances[0].update() - inst.state = state - uci.state = state - self.sa_session.add( inst ) - self.sa_session.add( uci ) - self.sa_session.flush() - except: # something failed, so skip - pass - - if inst.launch_time == None: - try: - launch_time = self.format_time( rl[0].instances[0].launch_time ) - inst.launch_time = launch_time - self.sa_session.add( inst ) - self.sa_session.flush() - if inst.uci.launch_time == None: - uci.launch_time = launch_time - self.sa_session.add( uci ) - self.sa_session.flush() - except: # something failed, so skip - pass - else: - err = "Starting a machine instance (DB id: '"+str(inst.id)+"') associated with this UCI '" + str(inst.uci.name) + \ - "' seems to have failed. Because it appears that cloud instance might have gotten started, manual check is recommended." - inst.error = err - inst.state = instance_states.ERROR - inst.uci.error = err - inst.uci.state = uci_states.ERROR - log.error( err ) - self.sa_session.add( inst ) - self.sa_session.add( uci ) - self.sa_session.flush() - - else: #Instance most likely never got processed, so set error message suggesting user to try starting instance again. - err = "Starting a machine instance (DB id: '"+str(inst.id)+"') associated with this UCI '" + str(inst.uci.name) + \ - "' seems to have failed. Because it appears that cloud instance never got started, it should be safe to reset state and try " \ - "starting the instance again." - inst.error = err - inst.state = instance_states.ERROR - uci.error = err - uci.state = uci_states.ERROR - log.error( err ) - self.sa_session.add( inst ) - self.sa_session.add( uci ) - self.sa_session.flush() -# uw = UCIwrapper( inst.uci ) -# log.debug( "Try automatically re-submitting UCI '%s'." % uw.get_name() ) - - def get_connection_from_uci( self, uci ): - """ - Establish and return connection to cloud provider. Information needed to do so is obtained - directly from uci database object. - """ - log.debug( 'Establishing %s cloud connection' % self.type ) - a_key = uci.credentials.access_key - s_key = uci.credentials.secret_key - # Get connection - try: - region = RegionInfo( None, uci.credentials.provider.region_name, uci.credentials.provider.region_endpoint ) -# log.debug( "[%s] Using following command to connect to cloud provider: " -# "conn = EC2Connection( aws_access_key_id=%s, " -# "aws_secret_access_key=%s, " -# "port=%s, " -# "is_secure=%s, " -# "region=region, " -# "path=%s )" % ( self.type, a_key, s_key, uci.credentials.provider.is_secure, uci.credentials.provider.port, uci.credentials.provider.path ) ) - conn = EC2Connection( aws_access_key_id=a_key, - aws_secret_access_key=s_key, - is_secure=uci.credentials.provider.is_secure, - port=uci.credentials.provider.port, - region=region, - path=uci.credentials.provider.path ) - except boto.exception.EC2ResponseError, e: - err = "Establishing connection with cloud failed: " + str( e ) - log.error( err ) - uci.error = err - uci.state = uci_states.ERROR - self.sa_session.add( uci ) - self.sa_session.flush() - return None - - return conn - -# def updateUCI( self, uci ): -# """ -# Runs a global status update on all storage volumes and all instances that are -# associated with specified UCI -# """ -# conn = self.get_connection( uci ) -# -# # Update status of storage volumes -# vl = model.CloudStore.filter( model.CloudInstance.table.c.uci_id == uci.id ).all() -# vols = [] -# for v in vl: -# vols.append( v.volume_id ) -# try: -# volumes = conn.get_all_volumes( vols ) -# for i, v in enumerate( volumes ): -# uci.store[i].inst.instance_id = v.instance_id -# uci.store[i].status = v.status -# uci.store[i].device = v.device -# uci.store[i].flush() -# except: -# log.debug( "Error updating status of volume(s) associated with UCI '%s'. Status was not updated." % uci.name ) -# pass -# -# # Update status of instances -# il = model.CloudInstance.filter_by( uci_id=uci.id ).filter( model.CloudInstance.table.c.state != 'terminated' ).all() -# instanceList = [] -# for i in il: -# instanceList.append( i.instance_id ) -# log.debug( 'instanceList: %s' % instanceList ) -# try: -# reservations = conn.get_all_instances( instanceList ) -# for i, r in enumerate( reservations ): -# uci.instance[i].state = r.instances[0].update() -# log.debug('updating instance %s; status: %s' % ( uci.instance[i].instance_id, uci.instance[i].state ) ) -# uci.state = uci.instance[i].state -# uci.instance[i].public_dns = r.instances[0].dns_name -# uci.instance[i].private_dns = r.instances[0].private_dns_name -# uci.instance[i].flush() -# uci.flush() -# except: -# log.debug( "Error updating status of instances associated with UCI '%s'. Instance status was not updated." % uci.name ) -# pass - - # --------- Helper methods ------------ - - def format_time( self, time ): - dict = {'T':' ', 'Z':''} - for i, j in dict.iteritems(): - time = time.replace(i, j) - return time - --- a/templates/cloud/add_credentials.mako +++ /dev/null @@ -1,110 +0,0 @@ -<% _=n_ %> -<%inherit file="/base.mako"/> -<%def name="title()">Add credentials</%def> - -<%def name="javascripts()"> -${parent.javascripts()} -<script type="text/javascript"> -$(function(){ - $("input:text:first").focus(); -}) -</script> -</%def> - -%if header: - ${header} -%endif - -%if providers: - -<div class="form"> - <div class="form-title">Add credentials</div> - <div class="form-body"> - <form name="add_credentials" action="${h.url_for( action='add_credentials' )}" method="post" > - - <% - cls = "form-row" - if error.has_key('cred_error'): - cls += " form-row-error" - %> - <div class="${cls}"> - <label>Credentials name:</label> - <div class="form-row-input"> - <input type="text" name="credName" value="${credName}" size="40"> - </div> - %if error.has_key('cred_error'): - <div class="form-row-error-message">${error['cred_error']}</div> - %endif - <div style="clear: both"></div> - </div> - - - <% - cls = "form-row" - if error.has_key('provider_error'): - cls += " form-row-error" - %> - <div class="${cls}"> - <label>Cloud provider name:</label> - <div class="form-row-input"> - <select name="providerName" style="width:40em"> - <option value="">Select Provider...</option> - %for provider in providers: - <option value="${provider.name}">${provider.name}</option> - %endfor - </select> - <br/>or <a href="${h.url_for( action='add_provider' )}"> - <span>register additional cloud provider</span></a> - </div> - %if error.has_key('provider_error'): - <div class="form-row-error-message">${error['provider_error']}</div> - %endif - <div style="clear: both"></div> - </div> - - <% - cls = "form-row" - if error.has_key('access_key_error'): - cls += " form-row-error" - %> - <div class="${cls}"> - <label>Access key:</label> - <div class="form-row-input"> - <input type="text" name="accessKey" value="${accessKey}" size="40"> - </div> - %if error.has_key('access_key_error'): - <div class="form-row-error-message">${error['access_key_error']}</div> - %endif - <div style="clear: both"></div> - </div> - - - <% - cls = "form-row" - if error.has_key('secret_key_error'): - cls += " form-row-error" - %> - <div class="${cls}"> - <label>Secret key:</label> - <div class="form-row-input"> - <input type="password" name="secretKey" value="${secretKey}" size="40"> - </div> - %if error.has_key('secret_key_error'): - <div class="form-row-error-message">${error['secret_key_error']}</div> - %endif - <div style="clear: both"></div> - </div> - - <div class="form-row"><input type="submit" value="Add"></div> - </form> - </div> -</div> - -%else: - In order to add credentials, desired cloud provider needs to be registered first.<p/> - Register <a href="${h.url_for( action='add_ec2' )}"> - <span>Amazon EC2 (us-east-1 region) automatically</span></a> - or add - <a href="${h.url_for( action='add_provider' )}"> - <span>custom cloud provider</span></a>. -%endif --- a/templates/cloud/list_images.mako +++ /dev/null @@ -1,91 +0,0 @@ -<%inherit file="/base.mako"/> - -<%def name="title()">Cloud home</%def> - -%if message: -<% - try: - messagetype - except: - messagetype = "done" -%> - -<p /> -<div class="${messagetype}message"> - ${message} -</div> -%endif - -<h2>List of registered machine images:</h2> -<ul class="manage-table-actions"> - <li> - <a class="action-button" href="${h.url_for( controller='cloud', action='add_new_image' )}" target="galaxy_main"> - <img src="${h.url_for('/static/images/silk/add.png')}" /> - <span>Add machine image</span> - </a> - </li> -</ul> - -%if images: - <table class="mange-table colored" border="0" cellspacing="0" cellpadding="0" width="100%"> - <colgroup width="2%"></colgroup> - <colgroup width="10%"></colgroup> - <colgroup width="13%"></colgroup> - <colgroup width="55%"></colgroup> - <colgroup width="10%"></colgroup> - <colgroup width="5%"></colgroup> - <colgroup width="5%"></colgroup> - <tr class="header"> - <th>#</th> - <th>Provider type</th> - <th>Machime image ID</th> - <th>Manifest</th> - <th>Architecture</th> - <th>Edit</th> - <th>Delete</th> - <th></th> - </tr> - %for i, image in enumerate( images ): - <tr> - <td>${i+1}</td> - <td> - %if image.provider_type: - ${image.provider_type} - %else: - N/A - %endif - </td> - <td> - %if image.image_id: - ${image.image_id} - %else: - N/A - %endif - </td> - <td> - %if image.manifest: - ${image.manifest} - %else: - N/A - %endif - </td> - <td> - %if image.architecture: - ${image.architecture} - %else: - N/A - %endif - </td> - <td> - <a href="${h.url_for( controller='cloud', action='edit_image', image_id=image.image_id, manifest=image.manifest, id=trans.security.encode_id(image.id) )}">e</a> - </td> - <td> - <a confirm="Are you sure you want to delete machine image '${image.image_id}'? Note that this may result in users' UCI's not to work any more!" - href="${h.url_for( controller='cloud', action='delete_image', id=trans.security.encode_id(image.id) )}">x</a> - </td> - </tr> - %endfor - </table> -%else: - <h3>There are no registered machine images.</h3><br /> -%endif --- a/templates/cloud/configure_uci.mako +++ /dev/null @@ -1,116 +0,0 @@ -<% _=n_ %> -<%inherit file="/base.mako"/> -<%def name="title()">Configure new UCI</%def> - -<%def name="javascripts()"> -${parent.javascripts()} -<script type="text/javascript"> - -var providers_zones = ${h.to_json_string(providersToZones)}; - -$(function(){ - $("input:text:first").focus(); - - $("#credName").change(function() { - var zones = providers_zones[ $(this).val() ]; - var zones_select = $("#zones"); - - zones_select.children().remove(); - - for (var i in zones) { - var zone = zones[i]; - var new_option = $('<option value="' + zone + '">' + zone + '</option>'); - new_option.appendTo(zones_select); - } - - }); -}) -</script> -</%def> - -%if header: - ${header} -%endif - -<div class="form"> - <div class="form-title">Configure new Galaxy instance</div> - <div class="form-body"> - <form name="Configure new UCI" action="${h.url_for( action='configure_new_uci' )}" method="post" > - - <% - cls = "form-row" - if error.has_key('inst_error'): - cls += " form-row-error" - %> - <div class="${cls}"> - <label>Instance name:</label> - <div class="form-row-input"> - <input type="text" name="instanceName" value="${instanceName}" size="40"> - </div> - %if error.has_key('inst_error'): - <div class="form-row-error-message">${error['inst_error']}</div> - %endif - <div style="clear: both"></div> - </div> - - <% - cls = "form-row" - if error.has_key('cred_error'): - cls += " form-row-error" - %> - <div class="${cls}"> - <label>Name of registered credentials to use:</label> - <div class="form-row-input"> - <select id="credName" name="credName" style="width:40em"> - <option value="">Select Credential...</option> - % for cred in credName: - <option value="${cred.name}">${cred.name}</option> - %endfor - </select> - </div> - %if error.has_key('cred_error'): - <div class="form-row-error-message">${error['cred_error']}</div> - %endif - <div style="clear: both"></div> - </div> - - - <% - cls = "form-row" - if error.has_key('vol_error'): - cls += " form-row-error" - %> - <div class="${cls}"> - <label>Permanent storage size (1-1000GB):<br/>(Note: you will be able to add more storage later)</label> - <div class="form-row-input"> - <input type="text" name="volSize" value="${volSize}" size="40"> - </div> - %if error.has_key('vol_error'): - <div class="form-row-error-message">${error['vol_error']}</div> - %endif - <div style="clear: both"></div> - </div> - - <% - cls = "form-row" - if error.has_key('zone_error'): - cls += " form-row-error" - %> - <div class="${cls}"> - <label>Create storage in zone:</label> - <div class="form-row-input"> - <select id="zones" name="zone" style="width:40em"> - </select> - </div> - %if error.has_key('zone_error'): - <div class="form-row-error-message">${error['zone_error']}</div> - %endif - <div style="clear: both"></div> - </div> - - - <div class="form-row"><input type="submit" value="Add"></div> - - </form> - </div> -</div> --- a/templates/cloud/add_provider.mako +++ /dev/null @@ -1,252 +0,0 @@ -<% _=n_ %> -<%inherit file="/base.mako"/> -<%def name="title()">Add provider</%def> - -<%def name="javascripts()"> -${parent.javascripts()} -<script type="text/javascript"> -$(function(){ - - $("input:text:first").focus(); - - $("#type").change(function() { - if ($(this).val() == 'ec2') { - clear(); - $("#autofill").attr( 'disabled', true ); - $("#autofill").attr( 'checked', false ); - $("#name").val( "EC2" ); - $("#region_name").val( "us-east-1" ); - $("#region_endpoint").val( "us-east-1.ec2.amazonaws.com" ); - $("#is_secure").val("1"); - $("#debug").val(""); - $("#path").val("/"); - } - else if ($(this).val() == 'eucalyptus') { - clear(); - $("#autofill").attr( 'disabled', false ); - } - }); -}) - - -function af(){ - - if ( $("#autofill").attr('checked') ) { - $("#name").val("Eucalyptus Public Cloud"); - $("#region_name").val("eucalyptus"); - $("#region_endpoint").val("mayhem9.cs.ucsb.edu"); - $("#is_secure").val("0"); - $("#port").val("8773"); - $("#path").val("/services/Eucalyptus"); - } -} - -function clear() { - $("#name").val(""); - $("#region_name").val(""); - $("#region_endpoint").val(""); - $("#is_secure").val(""); - $("#port").val(""); - $("#proxy").val(""); - $("#proxy_port").val(""); - $("#proxy_user").val(""); - $("#proxy_pass").val(""); - $("#debug").val(""); - $("#https_connection_factory").val(""); - $("#path").val(""); - -} - -</script> -</%def> - -%if header: - ${header} -%endif - -<div class="form"> - <div class="form-title">Add cloud provider</div> - <div class="form-body"> - <form name="add_provider_form" action="${h.url_for( action='add_provider' )}" method="post" > - <% - cls = "form-row" - if error.has_key('type_error'): - cls += " form-row-error" - %> - <div class="${cls}"> - <label>Provider type:</label> - <div class="form-row-input"> - <select id="type" name="type" style="width:40em"> - <option value="">Select Provider...</option> - <option value="eucalyptus">Eucalyptus</option> - <option value="ec2">Amazon EC2</option> - </select> - <br/> - <input type="checkbox" id="autofill" onclick="javascript:af()" disabled="true"> - auto fill using Eucalyptus Public Cloud values - </div> - %if error.has_key('type_error'): - <div class="form-row-error-message">${error['type_error']}</div> - %endif - <div style="clear: both"></div> - </div> - - <% - cls = "form-row" - if error.has_key('name_error'): - cls += " form-row-error" - %> - <div class="${cls}"> - <label>Provider name:</label> - <div class="form-row-input"> - <input type="text" id="name" name="name" value="${name}" size="40"> - </div> - %if error.has_key('name_error'): - <div class="form-row-error-message">${error['name_error']}</div> - %endif - <div style="clear: both"></div> - </div> - - <% - cls = "form-row" - %> - <div class="${cls}"> - <label>Region name:</label> - <div id="region_selection" class="form-row-input"> - <input type="text" name="region_name" id="region_name" value="${region_name}" size="40"> - </div> - <div style="clear: both"></div> - </div> - - <% - cls = "form-row" - %> - <div class="${cls}"> - <label>Region endpoint:</label> - <div class="form-row-input"> - <input type="text" name="region_endpoint" id="region_endpoint" value="${region_endpoint}" size="40"> - </div> - <div style="clear: both"></div> - </div> - - <% - cls = "form-row" - if error.has_key('is_secure_error'): - cls += " form-row-error" - %> - <div class="${cls}"> - <label>Is secure ('O' for False or '1' for True):</label> - <div class="form-row-input"> - <input type="text" name="is_secure" id="is_secure" value="${is_secure}" size="40"> - </div> - %if error.has_key('is_secure_error'): - <div class="form-row-error-message">${error['is_secure_error']}; you entered: '${is_secure}'</div> - %endif - <div style="clear: both"></div> - </div> - - <% - cls = "form-row" - %> - <div class="${cls}"> - <label>Host:</label> - <div class="form-row-input"> - <input type="text" name="host" value="${host}" size="40"> - </div> - <div style="clear: both"></div> - </div> - - <% - cls = "form-row" - %> - <div class="${cls}"> - <label>Port:</label> - <div class="form-row-input"> - <input type="text" name="port" id="port" value="${port}" size="40"> - </div> - <div style="clear: both"></div> - </div> - - <% - cls = "form-row" - %> - <div class="${cls}"> - <label>Proxy:</label> - <div class="form-row-input"> - <input type="text" name="proxy" value="${proxy}" size="40"> - </div> - <div style="clear: both"></div> - </div> - - <% - cls = "form-row" - %> - <div class="${cls}"> - <label>Proxy port:</label> - <div class="form-row-input"> - <input type="text" name="proxy_port" value="${proxy_port}" size="40"> - </div> - <div style="clear: both"></div> - </div> - - <% - cls = "form-row" - %> - <div class="${cls}"> - <label>Proxy user:</label> - <div class="form-row-input"> - <input type="text" name="proxy_user" value="${proxy_user}" size="40"> - </div> - <div style="clear: both"></div> - </div> - - <% - cls = "form-row" - %> - <div class="${cls}"> - <label>Proxy pass:</label> - <div class="form-row-input"> - <input type="text" name="proxy_pass" value="${proxy_pass}" size="40"> - </div> - <div style="clear: both"></div> - </div> - - <% - cls = "form-row" - %> - <div class="${cls}"> - <label>Debug:</label> - <div class="form-row-input"> - <input type="text" name="debug" value="${debug}" size="40"> - </div> - <div style="clear: both"></div> - </div> - - <% - cls = "form-row" - %> - <div class="${cls}"> - <label>HTTPS connection factory:</label> - <div class="form-row-input"> - <input type="text" name="https_connection_factory" value="${https_connection_factory}" size="40"> - </div> - <div style="clear: both"></div> - </div> - - <% - cls = "form-row" - %> - <div class="${cls}"> - <label>Path:</label> - <div class="form-row-input"> - <input type="text" name="path" id="path" value="${path}" size="40"> - </div> - <div style="clear: both"></div> - </div> - - <div class="form-row"><input type="submit" value="Add"></div> - - </form> - - </div> -</div> --- a/templates/cloud/index.mako +++ /dev/null @@ -1,16 +0,0 @@ -<%inherit file="/webapps/galaxy/base_panels.mako"/> - -<%def name="init()"> -<% - self.has_left_panel=False - self.has_right_panel=False - self.active_view="cloud" - self.message_box_visible=False -%> -</%def> - -<%def name="center_panel()"> - - <iframe name="galaxy_main" id="galaxy_main" frameborder="0" style="position: absolute; width: 100%; height: 100%;" src="${h.url_for( controller="cloud", action="list" )}"></iframe> - -</%def> --- a/templates/cloud/add_image.mako +++ /dev/null @@ -1,98 +0,0 @@ -<% _=n_ %> -<%inherit file="/base.mako"/> -<%def name="title()">Add machine image</%def> - -<%def name="javascripts()"> -${parent.javascripts()} -<script type="text/javascript"> -$(function(){ - //$("input:text:first").focus(); -}) -</script> -</%def> - -%if header: - ${header} -%endif - -<div class="form"> - <div class="form-title">Add machine image</div> - <div class="form-body"> - <form name="add_image" action="${h.url_for( action='add_new_image' )}" method="post" > - <% - cls = "form-row" - if error.has_key('provider_error'): - cls += " form-row-error" - %> - <div class="${cls}"> - <label>Cloud provider type:</label> - <div class="form-row-input"> - <select name="provider_type" style="width:40em"> - <option value="">Select Provider Type...</option> - <option value="eucalyptus">Eucalyptus</option> - <option value="ec2">Amazon EC2</option> - </select> - </div> - %if error.has_key('provider_error'): - <div class="form-row-error-message">${error['provider_error']}</div> - %endif - <div style="clear: both"></div> - </div> - - <% - cls = "form-row" - if error.has_key('id_error'): - cls += " form-row-error" - %> - <div class="${cls}"> - <label>Machine Image ID (AMI or EMI):</label> - <div class="form-row-input"> - <input type="text" name="image_id" value="${image_id}" size="40"> - </div> - %if error.has_key('id_error'): - <div class="form-row-error-message">${error['id_error']}</div> - %endif - <div style="clear: both"></div> - </div> - - <% - cls = "form-row" - if error.has_key('manifest_error'): - cls += " form-row-error" - %> - <div class="${cls}"> - <label>Manifest:</label> - <div class="form-row-input"> - <input type="text" name="manifest" value="${manifest}" size="40"> - </div> - %if error.has_key('manifest_error'): - <div class="form-row-error-message">${error['manifest_error']}</div> - %endif - <div style="clear: both"></div> - </div> - - - <% - cls = "form-row" - if error.has_key('arch_error'): - cls += " form-row-error" - %> - <div class="${cls}"> - <label>Image architecture:</label> - <div class="form-row-input"> - <select name="architecture" style="width:40em"> - <option value="">Select Architecture Type...</option> - <option value="i386">i386 (32 bit)</option> - <option value="x86_64">x86_64 (64 bit)</option> - </select> - </div> - %if error.has_key('arch_error'): - <div class="form-row-error-message">${error['arch_error']}</div> - %endif - <div style="clear: both"></div> - </div> - - <div class="form-row"><input type="submit" value="Add"></div> - </form> - </div> -</div> --- a/lib/galaxy/cloud/providers/ec2.py +++ /dev/null @@ -1,1232 +0,0 @@ -import subprocess, threading, os, errno, time, datetime, stat -from Queue import Queue, Empty -from datetime import datetime - -from galaxy import model # Database interaction class -from galaxy.model import mapping -from galaxy.datatypes.data import nice_size -from galaxy.util.bunch import Bunch -from galaxy.cloud import UCIwrapper -from Queue import Queue -from sqlalchemy import or_, and_ - -import galaxy.eggs -galaxy.eggs.require("boto") -from boto.ec2.connection import EC2Connection -from boto.ec2.regioninfo import RegionInfo -import boto.exception -import boto - -import logging -log = logging.getLogger( __name__ ) - -uci_states = Bunch( - NEW_UCI = "newUCI", - NEW = "new", - CREATING = "creating", - DELETING_UCI = "deletingUCI", - DELETING = "deleting", - SUBMITTED_UCI = "submittedUCI", - SUBMITTED = "submitted", - SHUTTING_DOWN_UCI = "shutting-downUCI", - SHUTTING_DOWN = "shutting-down", - ADD_STORAGE_UCI = "add-storageUCI", - ADD_STORAGE = "add-storage", - AVAILABLE = "available", - RUNNING = "running", - PENDING = "pending", - ERROR = "error", - DELETED = "deleted", - SNAPSHOT_UCI = "snapshotUCI", - SNAPSHOT = "snapshot" -) - -instance_states = Bunch( - TERMINATED = "terminated", - SUBMITTED = "submitted", - RUNNING = "running", - ADDING = "adding-storage", - PENDING = "pending", - SHUTTING_DOWN = "shutting-down", - ERROR = "error" -) - -store_status = Bunch( - WAITING = "waiting", - IN_USE = "in-use", - ADDING = "adding", - CREATING = "creating", - DELETED = 'deleted', - ERROR = "error" -) - -snapshot_status = Bunch( - SUBMITTED = 'submitted', - PENDING = 'pending', - COMPLETED = 'completed', - DELETE = 'delete', - DELETED= 'deleted', - ERROR = "error" -) - -class EC2CloudProvider( object ): - """ - Amazon EC2-based cloud provider implementation for managing instances. - """ - STOP_SIGNAL = object() - def __init__( self, app ): - self.type = "ec2" # cloud provider type (e.g., ec2, eucalyptus, opennebula) - self.zone = "us-east-1a" - self.security_group = "galaxyWeb" - self.queue = Queue() - self.sa_session = app.model.context - - self.threads = [] - nworkers = 5 - log.info( "Starting EC2 cloud controller workers..." ) - for i in range( nworkers ): - worker = threading.Thread( target=self.run_next ) - worker.start() - self.threads.append( worker ) - log.debug( "%d EC2 cloud workers ready", nworkers ) - - def shutdown( self ): - """Attempts to gracefully shut down the monitor thread""" - log.info( "sending stop signal to worker threads in EC2 cloud manager" ) - for i in range( len( self.threads ) ): - self.queue.put( self.STOP_SIGNAL ) - log.info( "EC2 cloud manager stopped" ) - - def put( self, uci_wrapper ): - """ - Add uci_wrapper object to the end of the request queue to be handled by - this cloud provider. - """ - state = uci_wrapper.get_uci_state() - uci_wrapper.change_state( state.split('U')[0] ) # remove 'UCI' from end of state description (i.e., mark as accepted and ready for processing) - self.queue.put( uci_wrapper ) - - def run_next( self ): - """Process next request, waiting until one is available if necessary.""" - cnt = 0 - while 1: - - uci_wrapper = self.queue.get() - uci_state = uci_wrapper.get_uci_state() - if uci_state is self.STOP_SIGNAL: - return - try: - if uci_state==uci_states.NEW: - self.create_uci( uci_wrapper ) - elif uci_state==uci_states.DELETING: - self.delete_uci( uci_wrapper ) - elif uci_state==uci_states.SUBMITTED: - self.start_uci( uci_wrapper ) - elif uci_state==uci_states.SHUTTING_DOWN: - self.stop_uci( uci_wrapper ) - elif uci_state==uci_states.SNAPSHOT: - self.snapshot_uci( uci_wrapper ) - elif uci_state==uci_states.ADD_STORAGE: - self.add_storage_to_uci( uci_wrapper ) - #self.dummy_start_uci( uci_wrapper ) - except: - log.exception( "Uncaught exception executing cloud request." ) - cnt += 1 - - def get_connection( self, uci_wrapper ): - """ - Establishes cloud connection using user's credentials associated with given UCI - """ - log.debug( 'Establishing %s cloud connection.' % self.type ) - provider = uci_wrapper.get_provider() - try: - region = RegionInfo( None, provider.region_name, provider.region_endpoint ) - except Exception, ex: - err = "Selecting region with cloud provider failed: " + str( ex ) - log.error( err ) - uci_wrapper.set_error( err, True ) - return None - try: - conn = EC2Connection( aws_access_key_id=uci_wrapper.get_access_key(), - aws_secret_access_key=uci_wrapper.get_secret_key(), - is_secure=provider.is_secure, - region=region, - path=provider.path ) - except boto.exception.EC2ResponseError, e: - err = "Establishing connection with cloud failed: " + str( e ) - log.error( err ) - uci_wrapper.set_error( err, True ) - return None - - return conn - - def check_key_pair( self, uci_wrapper, conn ): - """ - Check if a key pair associated with this UCI exists on cloud provider. - If yes, return key pair name; otherwise, generate a key pair with the cloud - provider and, again, return key pair name. - Key pair name for given UCI is generated from UCI's name and suffix '_kp' - """ - kp = None - kp_name = uci_wrapper.get_name().replace(' ','_') + "_kp" - log.debug( "Checking user's key pair: '%s'" % kp_name ) - try: - kp = conn.get_key_pair( kp_name ) - uci_kp_name = uci_wrapper.get_key_pair_name() - uci_material = uci_wrapper.get_key_pair_material() - if kp != None: - if kp.name != uci_kp_name or uci_material == None: - # key pair exists on the cloud but not in local database, so re-generate it (i.e., delete and then create) - try: - conn.delete_key_pair( kp_name ) - kp = self.create_key_pair( conn, kp_name ) - uci_wrapper.set_key_pair( kp.name, kp.material ) - except boto.exception.EC2ResponseError, e: - err = "EC2 response error while deleting key pair: " + str( e ) - log.error( err ) - uci_wrapper.set_error( err, True ) - else: - try: - kp = self.create_key_pair( conn, kp_name ) - uci_wrapper.set_key_pair( kp.name, kp.material ) - except boto.exception.EC2ResponseError, e: - err = "EC2 response error while creating key pair: " + str( e ) - log.error( err ) - uci_wrapper.set_error( err, True ) - except Exception, ex: - err = "Exception while creating key pair: " + str( ex ) - log.error( err ) - uci_wrapper.set_error( err, True ) - except boto.exception.EC2ResponseError, e: # No keypair under this name exists so create it - if e.code == 'InvalidKeyPair.NotFound': - log.info( "No keypair found, creating keypair '%s'" % kp_name ) - kp = self.create_key_pair( conn, kp_name ) - uci_wrapper.set_key_pair( kp.name, kp.material ) - else: - err = "EC2 response error while retrieving key pair: " + str( e ) - log.error( err ) - uci_wrapper.set_error( err, True ) - - if kp != None: - return kp.name - else: - return None - - def create_key_pair( self, conn, kp_name ): - """ Initiate creation of key pair under kp_name by current cloud provider. """ - try: - return conn.create_key_pair( kp_name ) - except boto.exception.EC2ResponseError, e: - return None - - def get_mi_id( self, uci_wrapper, i_index ): - """ - Get appropriate machine image (mi) based on instance size. - """ - i_type = uci_wrapper.get_instance_type( i_index ) - if i_type=='m1.small' or i_type=='c1.medium': - arch = 'i386' - else: - arch = 'x86_64' - - mi = self.sa_session.query( model.CloudImage ).filter_by( deleted=False, provider_type=self.type, architecture=arch ).first() - if mi: - return mi.image_id - else: - err = "Machine image could not be retrieved" - log.error( "%s for UCI '%s'." % (err, uci_wrapper.get_name() ) ) - uci_wrapper.set_error( err+". Contact site administrator to ensure needed machine image is registered.", True ) - return None - - def create_uci( self, uci_wrapper ): - """ - Create User Configured Instance (UCI) - i.e., create storage volume on cloud provider - and register relevant information in local Galaxy database. - """ - conn = self.get_connection( uci_wrapper ) - if uci_wrapper.get_uci_availability_zone()=='': - log.info( "Availability zone for UCI (i.e., storage volume) was not selected, using default zone: %s" % self.zone ) - uci_wrapper.set_store_availability_zone( self.zone ) - - store = uci_wrapper.get_all_stores_in_status( store_status.ADDING )[0] # Because at UCI creation time only 1 storage volume can be created, reference it directly - - log.info( "Creating storage volume in zone '%s' of size '%s'..." % ( uci_wrapper.get_uci_availability_zone(), store.size ) ) - # Because only 1 storage volume may be created at UCI config time, index of this storage volume in local Galaxy DB w.r.t - # current UCI is 0, so reference it in following methods - vol = conn.create_volume( store.size, uci_wrapper.get_uci_availability_zone(), snapshot=None ) - uci_wrapper.set_store_volume_id( store.id, vol.id ) - - # Retrieve created volume again to get updated status - try: - vl = conn.get_all_volumes( [vol.id] ) - except boto.exception.EC2ResponseError, e: - err = "EC2 response error while retrieving (i.e., updating status) of just created storage volume '" + vol.id + "': " + str( e ) - log.error( err ) - uci_wrapper.set_store_status( vol.id, uci_states.ERROR ) - uci_wrapper.set_error( err, True ) - return - except Exception, ex: - err = "Error while retrieving (i.e., updating status) of just created storage volume '" + vol.id + "': " + str( ex ) - log.error( err ) - uci_wrapper.set_error( err, True ) - return - - if len( vl ) > 0: - uci_wrapper.change_state( uci_state=vl[0].status ) - uci_wrapper.set_store_status( vol.id, vl[0].status ) - else: - err = "Volume '" + vol.id +"' not found by EC2 after being created." - log.error( err ) - uci_wrapper.set_store_status( vol.id, uci_states.ERROR ) - uci_wrapper.set_error( err, True ) - - def delete_uci( self, uci_wrapper ): - """ - Delete UCI - i.e., delete all storage volumes associated with this UCI. - NOTE that this implies deletion of any and all data associated - with this UCI from the cloud. All data will be deleted. - Information in local Galaxy database is marked as deleted but not actually removed - from the database. - """ - conn = self.get_connection( uci_wrapper ) - vl = [] # volume list - count = 0 # counter for checking if all volumes assoc. w/ UCI were deleted - - # Get all volumes assoc. w/ UCI, delete them from cloud as well as in local DB - vl = uci_wrapper.get_all_stores() - deletedList = [] - failedList = [] - for v in vl: - log.debug( "Deleting volume with id='%s'" % v.volume_id ) - try: - if conn.delete_volume( v.volume_id ): - deletedList.append( v.volume_id ) - v.deleted = True - v.status = store_status.DELETED - self.sa_session.add( v ) - self.sa_session.flush() - count += 1 - else: - failedList.append( v.volume_id ) - except boto.exception.EC2ResponseError, e: - err = "EC2 response error while deleting storage volume '" + v.volume_id + "': " + str( e ) - log.error( err ) - uci_wrapper.set_store_error( err, store_id = v.volume_id ) - uci_wrapper.set_error( err, True ) - - # Delete UCI if all of associated - if count == len( vl ): - uci_wrapper.set_deleted() - else: - if uci_wrapper.get_uci_state != uci_states.ERROR: - err = "Deleting following volume(s) failed: " + str( failedList ) + ". However, these volumes were successfully deleted: " \ - + str( deletedList ) + ". MANUAL intervention and processing needed." - log.error( err ) - uci_wrapper.set_error( err, True ) - - def snapshot_uci( self, uci_wrapper ): - """ - Initiate creation of a snapshot by cloud provider for all storage volumes - associated with this UCI. - """ - if uci_wrapper.get_uci_state() != uci_states.ERROR: - conn = self.get_connection( uci_wrapper ) - - snapshots = uci_wrapper.get_snapshots( status = snapshot_status.SUBMITTED ) - for snapshot in snapshots: - log.debug( "Snapshot DB id: '%s', volume id: '%s'" % ( snapshot.id, snapshot.store.volume_id ) ) - try: - snap = conn.create_snapshot( volume_id=snapshot.store.volume_id ) - snap_id = str( snap ).split(':')[1] - uci_wrapper.set_snapshot_id( snapshot.id, snap_id ) - sh = conn.get_all_snapshots( snap_id ) # get updated status - uci_wrapper.set_snapshot_status( status=sh[0].status, snap_id=snap_id ) - except boto.exception.EC2ResponseError, e: - err = "EC2 response error while creating snapshot: " + str( e ) - log.error( err ) - uci_wrapper.set_snapshot_error( error=err, snap_index=snapshot.id, set_status=True ) - uci_wrapper.set_error( err, True ) - return - except Exception, ex: - err = "Error while creating snapshot: " + str( ex ) - log.error( err ) - uci_wrapper.set_snapshot_error( error=err, snap_index=snapshot.id, set_status=True ) - uci_wrapper.set_error( err, True ) - return - - uci_wrapper.change_state( uci_state=uci_states.AVAILABLE ) - - def add_storage_to_uci( self, uci_wrapper ): - """ - Add an additional storage volume to specified UCI by creating the storage volume - on cloud provider, attaching it to currently running instance and adding it to - 'galaxyData' zpool on remote instance. - """ - conn = self.get_connection( uci_wrapper ) - - stores = uci_wrapper.get_all_stores_in_status( store_status.ADDING ) - for store in stores: - vol_size = store.size - availability_zone = uci_wrapper.get_uci_availability_zone() - log.info( "Adding storage volume to UCI '%s' in zone '%s' of size '%s'..." % ( uci_wrapper.get_name(), availability_zone, vol_size ) ) - - try: - vol = conn.create_volume( vol_size, availability_zone, snapshot=None ) - uci_wrapper.set_store_volume_id( store.id, vol.id ) - uci_wrapper.set_store_availability_zone( availability_zone, vol.id ) - log.debug( "New storage volume created: '%s'" % vol.id ) - except boto.exception.EC2ResponseError, e: - err = "EC2 response error while creating storage volume: " + str( e ) - log.error( err ) - uci_wrapper.set_store_error( err, store_id=vol.id ) - uci_wrapper.set_error( err, True ) - return - except Exception, ex: - err = "Error while creating storage volume: " + str( ex ) - log.error( err ) - uci_wrapper.set_error( err, True ) - return - - # Retrieve created volume again to get updated status - try: - vl = conn.get_all_volumes( [vol.id] ) - except boto.exception.EC2ResponseError, e: - err = "EC2 response error while retrieving (i.e., updating status) of just created storage volume '" + vol.id + "': " + str( e ) - log.error( err ) - uci_wrapper.set_store_error( err, store_id=vol.id ) - uci_wrapper.set_error( err, True ) - return - except Exception, ex: - err = "Error while retrieving (i.e., updating status) of just created storage volume '" + vol.id + "': " + str( ex ) - log.error( err ) - uci_wrapper.set_error( err, True ) - return - - # Wait for a while to ensure volume was created - if len( vl ) > 0: - vol_status = vl[0].status # Bc. only single vol is queried, reference it as 0th list element - for i in range( 30 ): - if vol_status != "available": - log.debug( "(%s) Updating volume status; current status: '%s'" % (i, vol_status ) ) - uci_wrapper.change_state( uci_state=vol_status ) - time.sleep(5) - vol_status = vl[0].status - if vol_status == "available": - log.debug( "(%s) New volume status '%s', continuing with file system adjustment." % (i, vol_status ) ) - uci_wrapper.set_store_status( vl[0].id, vol_status ) - break - if i is 29: - err = "Error while creating volume '"+vl[0].id+"'; stuck in state '"+vol_status+"'; deleting volume." - conn.delete_volume( vl[0].id ) - log.error( err ) - uci_wrapper.set_error( err, True ) - uci_wrapper.set_store_error( err, store_id=vol.id ) - conn.delete_volume( vl[0].id ) - uci_wrapper.set_store_deleted( vl[0].id ) - return - else: - err = "Volume '" + vol.id +"' not found by EC2 after being created." - log.error( err ) - uci_wrapper.set_store_error( err, store_id=vol.id ) - uci_wrapper.set_error( err, True ) - return - - # Get private key for given instance - pk = uci_wrapper.get_key_pair_material() - if pk == None: #If pk does not exist, create it - self.check_key_pair( uci_wrapper, conn ) - pk = uci_wrapper.get_key_pair_material() - - # Get working directory for this UCI and store pk into a file - wd = uci_wrapper.get_uci_working_directory() - if not os.path.exists( wd ): - os.mkdir( wd ) - pk_file_path = os.path.join( wd, "pk" ) - - if pk != None: - # Save private key to a file - pk_file = open( pk_file_path, "w" ) - pk_file.write( pk ) - pk_file.close() - else: - err = "ERROR: Private key not available for this UCI." - log.error( err ) - uci_wrapper.set_store_error( err, store_id=vol.id ) - uci_wrapper.set_error( err, True ) - return - - if os.path.exists( pk_file_path ): - # Change permissions of the file - this is required by later used ssh - os.chmod( pk_file_path, stat.S_IRUSR | stat.S_IWUSR ) - - # Get # of storage volumes associated with this UCI to know as which device to connect new volume to the instance - device_num = len( uci_wrapper.get_all_stores_in_status( store_status.IN_USE ) ) + 5 # First device num is 5, so all subsequent ones should follow - - # Get instance that the new storage volume is to be attached to. Although a list is returned, - # only 1 instance can be in 'adding-storage' state (because, for now, only 1 instance is assoc. with - # each UCI) and volume can be attached to only to it - il = uci_wrapper.get_instaces_in_state( instance_states.ADDING ) - if len( il ) > 0: - # Attach new volume to the instance - log.debug( "Attaching new storage volume '%s' to UCI '%s' as device '%s'" % - ( vol.id, uci_wrapper.get_name(), device_num ) ) - try: - vol_status = conn.attach_volume( vol.id, il[0].instance_id, device_num ) - except boto.exception.EC2ResponseError, e: - err = "Attaching just created storage volume '" + vol.id + "'to instance '" + \ - il[0].instance_id + "' as device '" + str( device_num ) + "' failed: " + str( e ) - log.error( err ) - uci_wrapper.set_store_error( err, store_id=vol.id ) - uci_wrapper.set_error( err, True ) - return - # For a while, keep checking attachment status of the new volume - for i in range(30): - log.debug( "Checking attachment status of new volume '%s': '%s'" % ( vol.id, vol_status ) ) - if vol_status == 'attached': - uci_wrapper.set_store_status( vol.id, vol_status ) - uci_wrapper.set_store_device( vol.id, device_num ) - break - if i == 29: - err = "Storage volume '" + vol.id + "' failed to attach to instance '" + il[0].instance_id + \ - "'. Manual check needed." - log.error( err ) - uci_wrapper.set_store_error( err, store_id=vol.id ) - uci_wrapper.set_error( err, False ) - return - - time.sleep(4) - vol_list = conn.get_all_volumes( [vol.id] ) - for v in vol_list: - vol_status = v.attachment_state() - - # Once storage volume is attached, add it to the zpool by issuing system level command - cmd = 'ssh -o StrictHostKeyChecking=no -i '+ pk_file_path +' root@'+il[0].public_dns+' "zpool add galaxyData c7d' + str( device_num )+'"' - log.debug( "Adding new storage volume to zpool cmd: %s" % cmd ) - stdout = os.system( cmd ) - if stdout != 0: - err = "Adding newly created storage volume to zpool on instance '" + il[0].instance_id + \ - "' failed. Error code: " + str( stdout ) - log.error( err ) - uci_wrapper.set_store_error( err, store_id=vol.id ) - uci_wrapper.set_error( err, False ) - return - else: - err = "No instance(s) found in 'adding-storage' state. New disk not added to UCI's zpool." - log.error( err ) - uci_wrapper.set_store_error( err, store_id=vol.id ) - uci_wrapper.set_error( err, True ) - return - - # Update UCI's total storage size - uci_wrapper.set_uci_total_size( uci_wrapper.get_uci_total_size() + vol.size ) - # Reset UCI's and instance's state - uci_wrapper.change_state( uci_state=uci_states.RUNNING, instance_id=il[0].instance_id, i_state=instance_states.RUNNING ) - log.debug( "Successfully added storage volume '%s' to UCI '%s'." % ( vol.id, uci_wrapper.get_name() ) ) - - def dummy_start_uci( self, uci_wrapper ): - - uci = uci_wrapper.get_uci() - log.debug( "Dummy start UCI '%s'" % uci.name ) - - -# uci_wrapper.change_state( uci_state.PENDING ) -# log.debug( "Sleeping a bit... (%s)" % uci.name ) -# time.sleep(20) -# log.debug( "Woke up! (%s)" % uci.name ) - - def start_uci( self, uci_wrapper ): - """ - Start instance(s) of given UCI on the cloud. - """ - if uci_wrapper.get_uci_state() != uci_states.ERROR: - conn = self.get_connection( uci_wrapper ) - self.check_key_pair( uci_wrapper, conn ) - if uci_wrapper.get_key_pair_name() == None: - err = "Key pair not found" - log.error( "%s for UCI '%s'." % ( err, uci_wrapper.get_name() ) ) - uci_wrapper.set_error( err + ". Try resetting the state and starting the instance again.", True ) - return - - i_indexes = uci_wrapper.get_instances_indexes( state=instance_states.SUBMITTED ) # Get indexes of i_indexes associated with this UCI that are in 'submitted' state - log.debug( "Starting instances with IDs: '%s' associated with UCI '%s' " % ( i_indexes, uci_wrapper.get_name(), ) ) - if len( i_indexes ) > 0: - for i_index in i_indexes: - # Get machine image for current instance - mi_id = self.get_mi_id( uci_wrapper, i_index ) - log.debug( "mi_id: %s, uci_wrapper.get_key_pair_name(): %s" % ( mi_id, uci_wrapper.get_key_pair_name() ) ) - uci_wrapper.set_mi( i_index, mi_id ) - - if mi_id != None: - # Check if galaxy security group exists (and create it if it does not) - log.debug( "Setting up '%s' security group." % self.security_group ) - try: - conn.get_all_security_groups( [self.security_group] ) # security groups - except boto.exception.EC2ResponseError, e: - if e.code == 'InvalidGroup.NotFound': - log.info( "No security group found, creating security group '%s'" % self.security_group ) - try: - gSecurityGroup = conn.create_security_group(self.security_group, 'Security group for Galaxy.') - gSecurityGroup.authorize( 'tcp', 80, 80, '0.0.0.0/0' ) # Open HTTP port - gSecurityGroup.authorize( 'tcp', 22, 22, '0.0.0.0/0' ) # Open SSH port - except boto.exception.EC2ResponseError, ee: - err = "EC2 response error while creating security group: " + str( ee ) - log.error( err ) - uci_wrapper.set_error( err, True ) - else: - err = "EC2 response error while retrieving security group: " + str( e ) - log.error( err ) - uci_wrapper.set_error( err, True ) - - - if uci_wrapper.get_uci_state() != uci_states.ERROR: - # Start an instance - log.debug( "Starting instance for UCI '%s'" % uci_wrapper.get_name() ) - #TODO: Once multiple volumes can be attached to a single instance, update 'userdata' composition - # Compose user data; for storage volumes, separate multiple volumes with a colon (:) ensuring that - # the last volume in the list is not followed by a colon. - stores = uci_wrapper.get_all_stores() - volume_ids = "" - if len( stores ) > 0: - for i, store in enumerate( stores ): - volume_ids += store.volume_id - if i < len( stores )-1: - volume_ids += ":" - else: - err = "No storage volumes found that are associated with UCI '%s'" + uci_wrapper.get_name() - log.error( err ) - uci_wrapper.set_error( err, True ) - return - userdata = volume_ids+"|"+uci_wrapper.get_access_key()+"|"+uci_wrapper.get_secret_key() - log.debug( "Using following command: conn.run_instances( image_id='%s', key_name='%s', security_groups=['%s'], user_data=[OMITTED], instance_type='%s', placement='%s' )" - % ( mi_id, uci_wrapper.get_key_pair_name(), self.security_group, uci_wrapper.get_instance_type( i_index ), uci_wrapper.get_uci_availability_zone() ) ) - # Start an instance - reservation = None - try: - reservation = conn.run_instances( image_id=mi_id, - key_name=uci_wrapper.get_key_pair_name(), - security_groups=[self.security_group], - user_data=userdata, - instance_type=uci_wrapper.get_instance_type( i_index ), - placement=uci_wrapper.get_uci_availability_zone() ) - except boto.exception.EC2ResponseError, e: - err = "EC2 response error when starting UCI '"+ uci_wrapper.get_name() +"': " + str( e ) - log.error( err ) - uci_wrapper.set_error( err, True ) - except Exception, ex: - err = "Error when starting UCI '" + uci_wrapper.get_name() + "': " + str( ex ) - log.error( err ) - uci_wrapper.set_error( err, True ) - # Record newly available instance data into local Galaxy database - if reservation: - l_time = datetime.utcnow() - # uci_wrapper.set_instance_launch_time( self.format_time( reservation.instances[0].launch_time ), i_index=i_index ) - uci_wrapper.set_instance_launch_time( l_time, i_index=i_index ) - if not uci_wrapper.uci_launch_time_set(): - uci_wrapper.set_uci_launch_time( l_time ) - try: - uci_wrapper.set_reservation_id( i_index, str( reservation ).split(":")[1] ) - # TODO: if more than a single instance will be started through single reservation, change this reference to element [0] - i_id = str( reservation.instances[0]).split(":")[1] - uci_wrapper.set_instance_id( i_index, i_id ) - s = reservation.instances[0].state - uci_wrapper.change_state( s, i_id, s ) - uci_wrapper.set_security_group_name( self.security_group, i_id=i_id ) - vol_id = uci_wrapper.get_store_volume_id( store_id=0 ) # TODO: Once more that one vol/UCI is allowed, update this! - # Following line is pointless bc. general update updates status of volume to 'available' - # before it actually connects to starting instance... This has been dealt w/ in general update method - #uci_wrapper.set_store_status( vol_id, store_status.WAITING ) - uci_wrapper.set_store_instance( vol_id, i_id ) - log.debug( "Instance of UCI '%s' started, current state: '%s'" % ( uci_wrapper.get_name(), uci_wrapper.get_uci_state() ) ) - except boto.exception.EC2ResponseError, e: - err = "EC2 response error when retrieving instance information for UCI '" + uci_wrapper.get_name() + "': " + str( e ) - log.error( err ) - uci_wrapper.set_error( err, True ) - else: - log.error( "UCI '%s' is in 'error' state, starting instance was aborted." % uci_wrapper.get_name() ) - else: - err = "No instances in state '"+ instance_states.SUBMITTED +"' found for UCI '" + uci_wrapper.get_name() + \ - "'. Nothing to start." - log.error( err ) - uci_wrapper.set_error( err, True ) - else: - log.error( "UCI '%s' is in 'error' state, starting instance was aborted." % uci_wrapper.get_name() ) - - def stop_uci( self, uci_wrapper): - """ - Stop all of cloud instances associated with given UCI. - """ - conn = self.get_connection( uci_wrapper ) - - # Get all instances associated with given UCI - il = uci_wrapper.get_instances_ids() # instance list - # Process list of instances and remove any references to empty instance id's - for i in il: - if i is None: - il.remove( i ) - log.debug( 'List of instances being terminated: %s' % il ) - rl = conn.get_all_instances( il ) # Reservation list associated with given instances - - # Initiate shutdown of all instances under given UCI - cnt = 0 - stopped = [] - not_stopped = [] - for r in rl: - for inst in r.instances: - log.debug( "Sending stop signal to instance '%s' associated with reservation '%s'." % ( inst, r ) ) - try: - inst.stop() - uci_wrapper.set_stop_time( datetime.utcnow(), i_id=inst.id ) - uci_wrapper.change_state( instance_id=inst.id, i_state=inst.update() ) - stopped.append( inst ) - except boto.exception.EC2ResponseError, e: - not_stopped.append( inst ) - err = "EC2 response error when stopping instance '" + inst.instance_id + "': " + str(e) - log.error( err ) - uci_wrapper.set_error( err, True ) - - uci_wrapper.reset_uci_launch_time() - log.debug( "Termination was initiated for all instances of UCI '%s'." % uci_wrapper.get_name() ) - - -# dbInstances = get_instances( trans, uci ) #TODO: handle list! -# -# # Get actual cloud instance object -# cloudInstance = get_cloud_instance( conn, dbInstances.instance_id ) -# -# # TODO: Detach persistent storage volume(s) from instance and update volume data in local database -# stores = get_stores( trans, uci ) -# for i, store in enumerate( stores ): -# log.debug( "Detaching volume '%s' to instance '%s'." % ( store.volume_id, dbInstances.instance_id ) ) -# mntDevice = store.device -# volStat = None -## Detaching volume does not work with Eucalyptus Public Cloud, so comment it out -## try: -## volStat = conn.detach_volume( store.volume_id, dbInstances.instance_id, mntDevice ) -## except: -## log.debug ( 'Error detaching volume; still going to try and stop instance %s.' % dbInstances.instance_id ) -# store.attach_time = None -# store.device = None -# store.i_id = None -# store.status = volStat -# log.debug ( '***** volume status: %s' % volStat ) -# -# -# # Stop the instance and update status in local database -# cloudInstance.stop() -# dbInstances.stop_time = datetime.utcnow() -# while cloudInstance.state != 'terminated': -# log.debug( "Stopping instance %s state; current state: %s" % ( str( cloudInstance ).split(":")[1], cloudInstance.state ) ) -# time.sleep(3) -# cloudInstance.update() -# dbInstances.state = cloudInstance.state -# -# # Reset relevant UCI fields -# uci.state = 'available' -# uci.launch_time = None -# -# # Persist -# session = trans.sa_session -## session.save_or_update( stores ) -# session.save_or_update( dbInstances ) # TODO: Is this going to work w/ multiple instances stored in dbInstances variable? -# session.save_or_update( uci ) -# session.flush() -# trans.log_event( "User stopped cloud instance '%s'" % uci.name ) -# trans.set_message( "Galaxy instance '%s' stopped." % uci.name ) - - def update( self ): - """ - Run status update on all instances that are in 'running', 'pending', or 'shutting-down' state. - Run status update on all storage volumes whose status is 'in-use', 'creating', or 'None'. - Run status update on all snapshots whose status is 'pending' or 'delete' - Run status update on any zombie UCIs, i.e., UCI's that is in 'submitted' state for an - extended period of time. - - Reason behind this method is to sync state of local DB and real-world resources - """ - log.debug( "Running general status update for %s UCIs..." % self.type ) - # Update instances - instances = self.sa_session.query( model.CloudInstance ) \ - .filter( or_( model.CloudInstance.table.c.state==instance_states.RUNNING, - model.CloudInstance.table.c.state==instance_states.PENDING, - model.CloudInstance.table.c.state==instance_states.SHUTTING_DOWN ) ) \ - .all() - for inst in instances: - if self.type == inst.uci.credentials.provider.type: - log.debug( "[%s] Running general status update on instance '%s'" - % ( inst.uci.credentials.provider.type, inst.instance_id ) ) - self.update_instance( inst ) - # Update storage volume(s) associated with current instance - stores = self.sa_session.query( model.CloudStore ) \ - .filter_by( uci_id=inst.uci_id, deleted=False ) \ - .all() - for store in stores: - if self.type == store.uci.credentials.provider.type: # and store.volume_id != None: - log.debug( "[%s] Running general status update on store with local database ID: '%s'" - % ( store.uci.credentials.provider.type, store.id ) ) - self.update_store( store ) - - # Update storage volume(s) - stores = self.sa_session.query( model.CloudStore ) \ - .filter( or_( model.CloudStore.table.c.status==store_status.CREATING, -# model.CloudStore.table.c.status==store_status.IN_USE, -# model.CloudStore.table.c.status==store_status.WAITING, - model.CloudStore.table.c.status==None ) ) \ - .all() - for store in stores: - if self.type == store.uci.credentials.provider.type: # and store.volume_id != None: - log.debug( "[%s] Running general status update on store with local database ID: '%s'" - % ( store.uci.credentials.provider.type, store.id ) ) - self.update_store( store ) -# else: -# log.error( "[%s] There exists an entry for UCI (%s) storage volume without an ID. Storage volume might have been created with " -# "cloud provider though. Manual check is recommended." % ( store.uci.credentials.provider.type, store.uci.name ) ) -# store.uci.error = "There exists an entry in local database for a storage volume without an ID. Storage volume might have been created " \ -# "with cloud provider though. Manual check is recommended. After understanding what happened, local database entry for given " \ -# "storage volume should be updated." -# store.status = store_status.ERROR -# store.uci.state = uci_states.ERROR -# store.uci.flush() -# store.flush() - - # Update pending snapshots or delete ones marked for deletion - snapshots = self.sa_session.query( model.CloudSnapshot ) \ - .filter( or_( model.CloudSnapshot.table.c.status == snapshot_status.PENDING, model.CloudSnapshot.table.c.status == snapshot_status.DELETE ) ) \ - .all() - for snapshot in snapshots: - if self.type == snapshot.uci.credentials.provider.type and snapshot.status == snapshot_status.PENDING: - log.debug( "[%s] Running general status update on snapshot '%s'" % ( snapshot.uci.credentials.provider.type, snapshot.snapshot_id ) ) - self.update_snapshot( snapshot ) - elif self.type == snapshot.uci.credentials.provider.type and snapshot.status == snapshot_status.DELETE: - log.debug( "[%s] Initiating deletion of snapshot '%s'" % ( snapshot.uci.credentials.provider.type, snapshot.snapshot_id ) ) - self.delete_snapshot( snapshot ) - - # Attempt at updating any zombie UCIs (i.e., instances that have been in SUBMITTED state for longer than expected - see below for exact time) - zombies = self.sa_session.query( model.UCI ).filter_by( state=uci_states.SUBMITTED ).all() - for zombie in zombies: - z_instances = self.sa_session.query( model.CloudInstance ) \ - .filter_by( uci_id=zombie.id ) \ - .filter( or_( model.CloudInstance.table.c.state != instance_states.TERMINATED, - model.CloudInstance.table.c.state == None ) ) \ - .all() - for z_inst in z_instances: - if self.type == z_inst.uci.credentials.provider.type: -# log.debug( "z_inst.id: '%s', state: '%s'" % ( z_inst.id, z_inst.state ) ) - td = datetime.utcnow() - z_inst.update_time - if td.seconds > 180: # if instance has been in SUBMITTED state for more than 3 minutes - log.debug( "[%s] Running zombie repair update on instance with DB id '%s'" % ( z_inst.uci.credentials.provider.type, z_inst.id ) ) - self.process_zombie( z_inst ) - - def update_instance( self, inst ): - """ - Update information in local database for given instance as it is obtained from cloud provider. - Along with updating information about given instance, information about the UCI controlling - this instance is also updated. - """ - # Get credentials associated wit this instance - uci_id = inst.uci_id - uci = self.sa_session.query( model.UCI ).get( uci_id ) - self.sa_session.refresh( uci ) - conn = self.get_connection_from_uci( uci ) - - # Get reservations handle for given instance - try: - rl= conn.get_all_instances( [inst.instance_id] ) - except boto.exception.EC2ResponseError, e: - err = "Retrieving instance(s) from cloud failed for UCI '"+ uci.name +"' during general status update: " + str( e ) - log.error( err ) - uci.error = err - uci.state = uci_states.ERROR - self.sa_session.add( uci ) - self.sa_session.flush() - return None - - # Because references to reservations are deleted shortly after instances have been terminated, getting an empty list as a response to a query - # typically means the instance has successfully shut down but the check was not performed in short enough amount of time. Until an alternative solution - # is found, below code sets state of given UCI to 'error' to indicate to the user something out of ordinary happened. - if len( rl ) == 0: - err = "Instance ID '"+inst.instance_id+"' was not found by the cloud provider. Instance might have crashed or otherwise been terminated."+ \ - "Manual check is recommended." - log.error( err ) - inst.error = err - uci.error = err - inst.state = instance_states.TERMINATED - uci.state = uci_states.ERROR - uci.launch_time = None - self.sa_session.add( inst ) - self.sa_session.add( uci ) - self.sa_session.flush() - # Update instance status in local DB with info from cloud provider - for r in rl: - for i, cInst in enumerate( r.instances ): - try: - s = cInst.update() - log.debug( "Checking state of cloud instance '%s' associated with UCI '%s' " \ - "and reservation '%s'. State='%s'" % ( cInst, uci.name, r, s ) ) - if s != inst.state: - inst.state = s - self.sa_session.add( inst ) - self.sa_session.flush() - # After instance has shut down, ensure UCI is marked as 'available' - if s == instance_states.TERMINATED and uci.state != uci_states.ERROR: - uci.state = uci_states.AVAILABLE - uci.launch_time = None - self.sa_session.add( uci ) - self.sa_session.flush() - # Making sure state of UCI is updated. Once multiple instances become associated with single UCI, this will need to be changed. - if s != uci.state and s != instance_states.TERMINATED: - uci.state = s - self.sa_session.add( uci ) - self.sa_session.flush() - if cInst.public_dns_name != inst.public_dns: - inst.public_dns = cInst.public_dns_name - self.sa_session.add( inst ) - self.sa_session.flush() - if cInst.private_dns_name != inst.private_dns: - inst.private_dns = cInst.private_dns_name - self.sa_session.add( inst ) - self.sa_session.flush() - except boto.exception.EC2ResponseError, e: - err = "Updating instance status from cloud failed for UCI '"+ uci.name + \ - "' during general status update: " + str( e ) - log.error( err ) - uci.error = err - uci.state = uci_states.ERROR - self.sa_session.add( uci ) - self.sa_session.flush() - return None - - def update_store( self, store ): - """ - Update information in local database for given storage volume as it is obtained from cloud provider. - Along with updating information about given storage volume, information about the UCI controlling - this storage volume is also updated. - """ - # Get credentials associated wit this store - uci_id = store.uci_id - uci = self.sa_session.query( model.UCI ).get( uci_id ) - self.sa_session.refresh( uci ) - conn = self.get_connection_from_uci( uci ) - - # Get reservations handle for given store - try: - log.debug( "Retrieving reference to storage volume '%s' during update..." % store.volume_id ) - vl = conn.get_all_volumes( [store.volume_id] ) - except boto.exception.EC2ResponseError, e: - err = "Retrieving volume(s) from cloud failed for UCI '"+ uci.name + "' during general status update: " + str( e ) - log.error( err ) - uci.error = err - uci.state = uci_states.ERROR - self.sa_session.add( uci ) - self.sa_session.flush() - return None - - # Update store status in local DB with info from cloud provider - if len(vl) > 0: - try: - log.debug( "General status update for storage volume '%s'; current status: '%s'" % (store.volume_id, vl[0].status ) ) - if store.status != vl[0].status: - # In case something failed during creation of UCI but actual storage volume was created and yet - # UCI state remained as 'new', try to remedy this by updating UCI state here - if ( store.status == None ) and ( store.volume_id != None ): - uci.state = vl[0].status - self.sa_session.add( uci ) - self.sa_session.flush() - # If UCI was marked in state 'CREATING', update its status to reflect new status - elif ( uci.state == uci_states.CREATING ): - uci.state = vl[0].status - self.sa_session.add( uci ) - self.sa_session.flush() - - store.status = vl[0].status - self.sa_session.add( store ) - self.sa_session.flush() - # Boto does not seem to be reporting these values although fields exist so comment them out... -# log.debug( "vl[0].instance_id: '%s'" % vl[0].instance_id ) -# if store.inst != None: -# if store.inst.instance_id != vl[0].instance_id: -# store.inst.instance_id = vl[0].instance_id -# self.sa_session.add( store ) -# self.sa_session.flush() -# log.debug( "vl[0].attach_time: '%s'" % vl[0].attach_time ) -# if store.attach_time != vl[0].attach_time: -# store.attach_time = vl[0].attach_time -# self.sa_session.add( store ) -# self.sa_session.flush() -## log.debug( "vl[0].device: '%s'" % vl[0].device ) -# if store.device != vl[0].device: -# store.device = vl[0].device -# self.sa_session.add( store ) -# self.sa_session.flush() - except boto.exception.EC2ResponseError, e: - err = "Updating status of volume(s) from cloud failed for UCI '"+ uci.name + \ - "' during general status update: " + str( e ) - log.error( err ) - uci.error = err - uci.state = uci_states.ERROR - self.sa_session.add( uci ) - self.sa_session.flush() - return None - else: - err = "No storage volumes returned by cloud provider on general update for volume with id: " + store.volume_id - log.error( "%s for UCI '%s'" % ( err, uci.name ) ) - store.status = store_status.ERROR - store.error = err - uci.error = err - uci.state = uci_states.ERROR - self.sa_session.add( uci ) - self.sa_session.add( store ) - self.sa_session.flush() - - def update_snapshot( self, snapshot ): - """ - Update information in local database for given snapshot as it is obtained from cloud provider. - Along with updating information about given snapshot, information about the UCI controlling - this snapshot is also updated. - """ - # Get credentials associated wit this store - uci_id = snapshot.uci_id - uci = self.sa_session.query( model.UCI ).get( uci_id ) - self.sa_session.refresh( uci ) - conn = self.get_connection_from_uci( uci ) - - try: - log.debug( "Updating status of snapshot '%s'" % snapshot.snapshot_id ) - snap = conn.get_all_snapshots( [snapshot.snapshot_id] ) - if len( snap ) > 0: - log.debug( "Snapshot '%s' status: %s" % ( snapshot.snapshot_id, snap[0].status ) ) - snapshot.status = snap[0].status - self.sa_session.add( snapshot ) - self.sa_session.flush() - else: - err = "No snapshots returned by EC2 on general update" - log.error( "%s for UCI '%s'" % ( err, uci.name ) ) - snapshot.status = snapshot_status.ERROR - snapshot.error = err - uci.error = err - uci.state = uci_states.ERROR - self.sa_session.add( uci ) - self.sa_session.add( snapshot ) - self.sa_session.flush() - except boto.exception.EC2ResponseError, e: - err = "EC2 response error while updating snapshot status: " + str( e ) - log.error( err ) - snapshot.status = snapshot_status.ERROR - snapshot.error = err - uci.error = err - uci.state = uci_states.ERROR - self.sa_session.add( uci ) - self.sa_session.add( snapshot ) - self.sa_session.flush() - except Exception, ex: - err = "Error while updating snapshot status: " + str( ex ) - log.error( err ) - snapshot.status = snapshot_status.ERROR - snapshot.error = err - uci.error = err - uci.state = uci_states.ERROR - self.sa_session.add( uci ) - self.sa_session.add( snapshot ) - self.sa_session.flush() - - def delete_snapshot( self, snapshot ): - """ - Initiate deletion of given snapshot from cloud provider. - """ - if snapshot.status == snapshot_status.DELETE: - # Get credentials associated wit this store - uci_id = snapshot.uci_id - uci = self.sa_session.query( model.UCI ).get( uci_id ) - self.sa_session.refresh( uci ) - conn = self.get_connection_from_uci( uci ) - - try: - log.debug( "Deleting snapshot '%s'" % snapshot.snapshot_id ) - snap = conn.delete_snapshot( snapshot.snapshot_id ) - if snap == True: - snapshot.deleted = True - snapshot.status = snapshot_status.DELETED - self.sa_session.add( snapshot ) - self.sa_session.flush() - return snap - except boto.exception.EC2ResponseError, e: - err = "EC2 response error while deleting snapshot: " + str( e ) - log.error( err ) - snapshot.status = snapshot_status.ERROR - snapshot.error = err - uci.error = err - uci.state = uci_states.ERROR - self.sa_session.add( uci ) - self.sa_session.add( snapshot ) - self.sa_session.flush() - except Exception, ex: - err = "Error while deleting snapshot: " + str( ex ) - log.error( err ) - snapshot.status = snapshot_status.ERROR - snapshot.error = err - uci.error = err - uci.state = uci_states.ERROR - self.sa_session.add( uci ) - self.sa_session.add( snapshot ) - self.sa_session.flush() - else: - err = "Cannot delete snapshot '"+snapshot.snapshot_id+"' because its status is '"+snapshot.status+"'. Only snapshots with '" + \ - snapshot_status.COMPLETED+"' status can be deleted." - log.error( err ) - snapshot.error = err - self.sa_session.add( snapshot ) - self.sa_session.flush() - - def process_zombie( self, inst ): - """ - Attempt at discovering if starting a cloud instance was successful but local database was not updated - accordingly or if something else failed and instance was never started. Currently, no automatic - repairs are being attempted; instead, appropriate error messages are set. - """ - uci_id = inst.uci_id - uci = self.sa_session.query( model.UCI ).get( uci_id ) - self.sa_session.refresh( uci ) - - # Check if any instance-specific information was written to local DB; if 'yes', set instance and UCI's error message - # suggesting manual check. - if inst.launch_time != None or inst.reservation_id != None or inst.instance_id != None: - # Try to recover state - this is best-case effort, so if something does not work immediately, not - # recovery steps are attempted. Recovery is based on hope that instance_id is available in local DB; if not, - # report as error. - # Fields attempting to be recovered are: reservation_id, instance status, and launch_time - if inst.instance_id != None: - conn = self.get_connection_from_uci( uci ) - rl = conn.get_all_instances( [inst.instance_id] ) # reservation list - # Update local DB with relevant data from instance - if inst.reservation_id == None: - try: - inst.reservation_id = str(rl[0]).split(":")[1] - except: # something failed, so skip - pass - - try: - state = rl[0].instances[0].update() - inst.state = state - uci.state = state - self.sa_session.add( inst ) - self.sa_session.add( uci ) - self.sa_session.flush() - except: # something failed, so skip - pass - - if inst.launch_time == None: - try: - launch_time = self.format_time( rl[0].instances[0].launch_time ) - inst.launch_time = launch_time - self.sa_session.add( inst ) - self.sa_session.flush() - if inst.uci.launch_time == None: - uci.launch_time = launch_time - self.sa_session.add( uci ) - self.sa_session.flush() - except: # something failed, so skip - pass - else: - err = "Starting a machine instance (DB id: '"+str(inst.id)+"') associated with this UCI '" + str(inst.uci.name) + \ - "' seems to have failed. Because it appears that cloud instance might have gotten started, manual check is recommended." - inst.error = err - inst.state = instance_states.ERROR - inst.uci.error = err - inst.uci.state = uci_states.ERROR - log.error( err ) - self.sa_session.add( inst ) - self.sa_session.add( uci ) - self.sa_session.flush() - - else: #Instance most likely never got processed, so set error message suggesting user to try starting instance again. - err = "Starting a machine instance (DB id: '"+str(inst.id)+"') associated with this UCI '" + str(inst.uci.name) + \ - "' seems to have failed. Because it appears that cloud instance never got started, it should be safe to reset state and try " \ - "starting the instance again." - inst.error = err - inst.state = instance_states.ERROR - uci.error = err - uci.state = uci_states.ERROR - log.error( err ) - self.sa_session.add( inst ) - self.sa_session.add( uci ) - self.sa_session.flush() -# uw = UCIwrapper( inst.uci ) -# log.debug( "Try automatically re-submitting UCI '%s'." % uw.get_name() ) - - def get_connection_from_uci( self, uci ): - """ - Establish and return connection to cloud provider. Information needed to do so is obtained - directly from uci database object. - """ - log.debug( 'Establishing %s cloud connection' % self.type ) - a_key = uci.credentials.access_key - s_key = uci.credentials.secret_key - # Get connection - try: - region = RegionInfo( None, uci.credentials.provider.region_name, uci.credentials.provider.region_endpoint ) - conn = EC2Connection( aws_access_key_id=a_key, - aws_secret_access_key=s_key, - is_secure=uci.credentials.provider.is_secure, - region=region, - path=uci.credentials.provider.path ) - except boto.exception.EC2ResponseError, e: - err = "Establishing connection with cloud failed: " + str( e ) - log.error( err ) - uci.error = err - uci.state = uci_states.ERROR - self.sa_session.add( uci ) - self.sa_session.flush() - return None - - return conn - -# def updateUCI( self, uci ): -# """ -# Runs a global status update on all storage volumes and all instances that are -# associated with specified UCI -# """ -# conn = self.get_connection( uci ) -# -# # Update status of storage volumes -# vl = model.CloudStore.filter( model.CloudInstance.table.c.uci_id == uci.id ).all() -# vols = [] -# for v in vl: -# vols.append( v.volume_id ) -# try: -# volumes = conn.get_all_volumes( vols ) -# for i, v in enumerate( volumes ): -# uci.store[i].i_id = v.instance_id -# uci.store[i].status = v.status -# uci.store[i].device = v.device -# uci.store[i].flush() -# except: -# log.debug( "Error updating status of volume(s) associated with UCI '%s'. Status was not updated." % uci.name ) -# pass -# -# # Update status of instances -# il = model.CloudInstance.filter_by( uci_id=uci.id ).filter( model.CloudInstance.table.c.state != 'terminated' ).all() -# instanceList = [] -# for i in il: -# instanceList.append( i.instance_id ) -# log.debug( 'instanceList: %s' % instanceList ) -# try: -# reservations = conn.get_all_instances( instanceList ) -# for i, r in enumerate( reservations ): -# uci.instance[i].state = r.instances[0].update() -# log.debug('updating instance %s; status: %s' % ( uci.instance[i].instance_id, uci.instance[i].state ) ) -# uci.state = uci.instance[i].state -# uci.instance[i].public_dns = r.instances[0].dns_name -# uci.instance[i].private_dns = r.instances[0].private_dns_name -# uci.instance[i].flush() -# uci.flush() -# except: -# log.debug( "Error updating status of instances associated with UCI '%s'. Instance status was not updated." % uci.name ) -# pass - - # --------- Helper methods ------------ - - def format_time( self, time ): - dict = {'T':' ', 'Z':''} - for i, j in dict.iteritems(): - time = time.replace(i, j) - return time - --- a/templates/cloud/view_instance.mako +++ /dev/null @@ -1,142 +0,0 @@ -<%inherit file="/base.mako"/> -<%def name="title()">Live instance details</%def> - -<% - # Because of the one-to-many relationship between liveInstance (i.e., UCI) and actual instances, need to know - # which one is currently active. Because only one instance of UCI can be alive at any point in time, simply - # select the most recent one. - # TODO: Once individual UCI's will be able to start more than one instance, this will need to be fixed - #i_id = len(liveInstance.instance) - 1 -%> - -<h2>Live instance details</h2> - -%if liveInstance: - <ul class="manage-table-actions"> - <li> - <a class="action-button" href="${h.url_for( action='list' )}"> - <img src="${h.url_for('/static/images/silk/resultset_previous.png')}" /> - <span>Return to cloud management console</span> - </a> - </li> - </ul> - - <table class="mange-table colored" border="0" cellspacing="0" cellpadding="0" width="100%"> - <tr> - <td> Instance name: </td> - <td> - ${liveInstance.uci.name} - <a id="li-popup" class="popup-arrow" style="display: none;">▼</a> - </td> - <td> - <div popupmenu="li-popup"> - <a class="action-button" href="${h.url_for( action='rename_uci', id=trans.security.encode_id(liveInstance.uci.id) )}">Rename</a> - <a class="action-button" confirm="Are you sure you want to stop instance '${liveInstance.uci.name}'?" href="${h.url_for( action='stop', id=trans.security.encode_id(liveInstance.uci.id) )}">Stop</a> - </div> - </td> - </tr> - <tr> - <td> Date created: </td> - <td> ${str(liveInstance.uci.create_time)[:16]} - <% - context.write( ' UTC (' ) - context.write( str(h.date.distance_of_time_in_words (liveInstance.uci.create_time, h.date.datetime.utcnow() ) ) ) - %> ago) - </td> - </tr> - <tr> - <td> Alive since: </td> - <td> ${str(liveInstance.launch_time)[:16]} - <% - context.write( ' UTC (' ) - context.write( str(h.date.distance_of_time_in_words (liveInstance.launch_time, h.date.datetime.utcnow() ) ) ) - %> ago) - </td> - </tr> - %if liveInstance.instance_id != None: - <tr> - <td> Instance ID: </td> - <td> ${liveInstance.instance_id} </td> - </tr> - %endif - %if liveInstance.reservation_id != None: - <tr> - <td> Reservation ID: </td> - <td> ${liveInstance.reservation_id} </td> - </tr> - %endif - <tr> - <td> AMI: </td> - <td> ${liveInstance.image.image_id} </td> - </tr> - <tr> - <td> State:</td> - <td> ${liveInstance.state} </td> - </tr> - <tr> - <td> Type:</td> - <td> ${liveInstance.type} </td> - </tr> - <tr> - <td> Storage size:</td> - <td> ${liveInstance.uci.total_size} </td> - </tr> - %if liveInstance.public_dns != None and liveInstance.public_dns != '': - <tr> - <td> Public DNS:</td> - <% - lnk="http://"+str(liveInstance.public_dns) - %> - <td><a href="${lnk}" target="_blank">${liveInstance.public_dns}</a></td> - </tr> - %endif - %if liveInstance.private_dns != None and liveInstance.private_dns != '': - <tr> - <td> Private DNS:</td> - <td> ${liveInstance.private_dns}</td> - </tr> - %endif - %if liveInstance.security_group != None: - <tr> - <td> Security group zone:</td> - <td> ${liveInstance.security_group} </td> - </tr> - %endif - %if liveInstance.availability_zone != None: - <tr> - <td> Availabilty zone:</td> - <td> ${liveInstance.availability_zone} </td> - </tr> - %endif - %if liveInstance.uci.key_pair_name != None: - <tr> - <td> Keypair file name:</td> - <td> ${liveInstance.uci.key_pair_name} </td> - </tr> - %endif - %if liveInstance.uci.key_pair_material != None: - <tr> - <td> Keypair material:</td> - <td> - <div id="short"> - <a onclick="document.getElementById('full').style.display = 'block'; - document.getElementById('short').style.display = 'none'; return 0" - href="javascript:void(0)"> - + Show - </a> - </div> - <div id="full" style="DISPLAY: none"> - <a onclick="document.getElementById('short').style.display = 'block'; - document.getElementById('full').style.display = 'none'; return 0;" - href="javascript:void(0)"> - - Hide</a> - ${liveInstance.uci.key_pair_material}<br/> - </div> - </td> - </tr> - %endif - - </table> -%else: - There is no live instance under that name. -%endif --- a/templates/webapps/galaxy/admin/index.mako +++ b/templates/webapps/galaxy/admin/index.mako @@ -118,14 +118,6 @@ <div class="toolTitle"><a href="${h.url_for( controller='requests_admin', action='list')}" target="galaxy_main">Manage requests</a></div></div></div> - <div class="toolSectionTitle"> - <span>Cloud</span> - </div> - <div class="toolSectionBody"> - <div class="toolSectionBg"> - <div class="toolTitle"><a href="${h.url_for( controller='cloud', action='list_machine_images' )}" target="galaxy_main">Manage machine images</a></div> - </div> - </div></div></div></div> --- a/lib/galaxy/web/controllers/cloud.py +++ /dev/null @@ -1,1276 +0,0 @@ -from galaxy.web.base.controller import * - -import pkg_resources -pkg_resources.require( "simplejson" ) -import simplejson -import urllib2 - -from galaxy.tools.parameters import * -from galaxy.tools import DefaultToolState -from galaxy.tools.parameters.grouping import Repeat, Conditional -from galaxy.datatypes.data import Data -from galaxy.util.odict import odict -from galaxy.util.bunch import Bunch -from galaxy.util.topsort import topsort, topsort_levels, CycleError -from galaxy.model.mapping import desc -from galaxy.model.orm import * -from datetime import datetime, timedelta - -pkg_resources.require( "WebHelpers" ) -from webhelpers import * - -# Required for Cloud tab -import galaxy.eggs -galaxy.eggs.require("boto") -from boto.ec2.connection import EC2Connection -from boto.ec2.regioninfo import RegionInfo -from galaxy.cloud import CloudManager -import boto.exception -import boto - -import logging -log = logging.getLogger( __name__ ) - -uci_states = Bunch( - NEW_UCI = "newUCI", - NEW = "new", - CREATING = "creating", - DELETING_UCI = "deletingUCI", - DELETING = "deleting", - SUBMITTED_UCI = "submittedUCI", - SUBMITTED = "submitted", - SHUTTING_DOWN_UCI = "shutting-downUCI", - SHUTTING_DOWN = "shutting-down", - ADD_STORAGE_UCI = "add-storageUCI", - ADD_STORAGE = "add-storage", - AVAILABLE = "available", - RUNNING = "running", - PENDING = "pending", - ERROR = "error", - DELETED = "deleted", - SNAPSHOT_UCI = "snapshotUCI", - SNAPSHOT = "snapshot" -) - -instance_states = Bunch( - TERMINATED = "terminated", - SUBMITTED = "submitted", - RUNNING = "running", - ADDING = "adding-storage", - PENDING = "pending", - SHUTTING_DOWN = "shutting-down", - ERROR = "error" -) - -store_status = Bunch( - WAITING = "waiting", - IN_USE = "in-use", - ADDING = "adding", - CREATING = "creating", - DELETED = 'deleted', - ERROR = "error" -) - -snapshot_status = Bunch( - SUBMITTED = 'submitted', - PENDING = 'pending', - COMPLETED = 'completed', - DELETE = 'delete', - DELETED= 'deleted', - ERROR = "error" -) - -class CloudController( BaseController ): - - @web.expose - def index( self, trans ): - return trans.fill_template( "cloud/index.mako" ) - - @web.expose - @web.require_login( "use Galaxy cloud" ) - def list( self, trans ): - """ - Render cloud main page (management of cloud resources) - """ - user = trans.get_user() - - cloudCredentials = trans.sa_session.query( model.CloudUserCredentials ) \ - .filter_by( user=user ) \ - .filter( model.CloudUserCredentials.table.c.deleted != True ) \ - .order_by( model.CloudUserCredentials.table.c.name ) \ - .all() - - cloudProviders = trans.sa_session.query( model.CloudProvider ) \ - .filter_by( user=user ) \ - .filter( model.CloudProvider.table.c.deleted != True ) \ - .order_by( model.CloudProvider.table.c.name ) \ - .all() - - liveInstances = trans.sa_session.query( model.UCI ) \ - .filter_by( user=user ) \ - .filter( or_( model.UCI.table.c.state==uci_states.RUNNING, - model.UCI.table.c.state==uci_states.PENDING, - model.UCI.table.c.state==uci_states.SUBMITTED, - model.UCI.table.c.state==uci_states.SUBMITTED_UCI, - model.UCI.table.c.state==uci_states.SHUTTING_DOWN, - model.UCI.table.c.state==uci_states.SHUTTING_DOWN_UCI, - model.UCI.table.c.state==uci_states.ADD_STORAGE, - model.UCI.table.c.state==uci_states.ADD_STORAGE_UCI ) ) \ - .order_by( desc( model.UCI.table.c.update_time ) ) \ - .all() - - prevInstances = trans.sa_session.query( model.UCI ) \ - .filter_by( user=user, deleted=False ) \ - .filter( or_( model.UCI.table.c.state==uci_states.AVAILABLE, - model.UCI.table.c.state==uci_states.NEW, - model.UCI.table.c.state==uci_states.NEW_UCI, - model.UCI.table.c.state==uci_states.CREATING, - model.UCI.table.c.state==uci_states.ERROR, - model.UCI.table.c.state==uci_states.DELETED, - model.UCI.table.c.state==uci_states.DELETING, - model.UCI.table.c.state==uci_states.DELETING_UCI, - model.UCI.table.c.state==uci_states.SNAPSHOT, - model.UCI.table.c.state==uci_states.SNAPSHOT_UCI ) ) \ - .order_by( desc( model.UCI.table.c.update_time ) ) \ - .all() - - # Check after update there are instances in pending state; if so, display message - pendingInstances = trans.sa_session.query( model.UCI ) \ - .filter_by( user=user ) \ - .filter( or_( model.UCI.table.c.state==uci_states.PENDING, - model.UCI.table.c.state==uci_states.SUBMITTED, - model.UCI.table.c.state==uci_states.SUBMITTED_UCI ) ) \ - .all() - if pendingInstances: - trans.set_message( "Galaxy instance started. NOTE: Please wait about 5 minutes for the instance to " - "start up. A button to connect to the instance will appear alongside " - "instance description once cloud instance of Galaxy is ready." ) - -# log.debug( "provider.is_secure: '%s'" % trans.sa_session.query( model.CloudProvider).filter_by(id=1).first().is_secure ) -# trans.sa_session.query( model.CloudProvider).filter_by(id=1).first().is_secure=False -# trans.sa_session.flush() -# log.debug( "provider.is_secure: '%s'" % trans.sa_session.query( model.CloudProvider).filter_by(id=1).first().is_secure ) - -# log.debug( "image: '%s'" % model.CloudImage.is_secure ) - - return trans.fill_template( "cloud/configure_cloud.mako", - cloudCredentials = cloudCredentials, - liveInstances = liveInstances, - prevInstances = prevInstances, - cloudProviders = cloudProviders ) - - # ----- UCI methods ----- - - @web.expose - @web.require_login( "use Galaxy cloud" ) - def configure_new_uci( self, trans, instanceName='', credName='', volSize='', zone='' ): - """ - Configure and add new cloud instance to user's instance pool - """ - inst_error = vol_error = cred_error = None - error = {} - user = trans.get_user() - - if instanceName: - # Check if volume size is entered as an integer - try: - volSize = int( volSize ) - except ValueError: - error['vol_error'] = "Volume size must be integer value between 1 and 1000." - - # Create new user configured instance - try: - if trans.sa_session.query( model.UCI ) \ - .filter_by (user=user, deleted=False, name=instanceName ) \ - .first(): - error['inst_error'] = "An instance with that name already exist." - elif instanceName=='' or len( instanceName ) > 255: - error['inst_error'] = "Instance name must be between 1 and 255 characters long." - elif credName=='': - error['cred_error'] = "You must select credentials." - elif volSize == '': - error['vol_error'] = "You must specify volume size as an integer value between 1 and 1000." - elif ( int( volSize ) < 1 ) or ( int( volSize ) > 1000 ): - error['vol_error'] = "Volume size must be integer value between 1 and 1000." - elif zone=='': - error['zone_error'] = "You must select a zone where this UCI will be registered." - else: - # Capture user configured instance information - uci = model.UCI() - uci.name = instanceName - creds = trans.sa_session.query( model.CloudUserCredentials ) \ - .filter( model.CloudUserCredentials.table.c.name==credName ).first() - uci.credentials = creds - uci.user= user - uci.total_size = volSize # This is OK now because new instance is being created and only one storage volume can be created at UCI creation time - uci.state = uci_states.NEW_UCI - - storage = model.CloudStore() - storage.user = user - storage.uci = uci - storage.size = volSize - # If '(any)' zone was selected, just choose the first one that's available - if zone == "(any)": - zones = None - conn = get_connection( trans, creds ) - if conn != None: - try: - zones = conn.get_all_zones() - if len( zones ) > 0: - zone = str( zones[0] ).split(':')[1] - except boto.exception.EC2ResponseError, e: - log.error( "Retrieving zones for credentials '%s' failed: %s" % ( storedCred.name, e ) ) - providersToZones[storedCred.name] = [ "Retrieving zones failed: " + str( e ) ] - storage.availability_zone = zone - storage.status = store_status.ADDING - # Persist - session = trans.sa_session - session.add( uci ) - session.add( storage ) - session.flush() - # Log and display the management page - trans.log_event( "User configured new cloud instance: '%s'" % instanceName ) - trans.set_message( "New Galaxy instance '%s' configured. Once instance status shows 'available' you will be able to start the instance." % instanceName ) - return self.list( trans ) - except AttributeError, ae: - inst_error = "No registered cloud images. You must contact administrator to add some before proceeding." - log.debug("AttributeError when registering new UCI '%s': %s " % ( instanceName, str( ae ) ) ) - else: - storedCreds = trans.sa_session.query( model.CloudUserCredentials ).filter_by( user=user, deleted=False ).all() - if len( storedCreds ) == 0: - return trans.show_error_message( "You must register credentials before configuring a Galaxy cloud instance." ) - # Create dict mapping of cloud-providers-to-zones available by those providers - providersToZones = {} - for storedCred in storedCreds: - zones = None - conn = get_connection( trans, storedCred ) - if conn != None: - avail_zones = [] - try: - zones = conn.get_all_zones() - if len( zones ) > 0: - avail_zones.append( "(any)" ) - for z in zones: - z = str( z ).split(':')[1] - avail_zones.append( z ) - providersToZones[storedCred.name] = avail_zones - except boto.exception.EC2ResponseError, e: - log.error( "Retrieving zones for credentials '%s' failed: %s" % ( storedCred.name, e ) ) - providersToZones[storedCred.name] = [ "Retrieving zones failed: " + str( e ) ] - else: - providersToZones[storedCred.name] = ['Connection with cloud provider could not be established.'] - - # Hard-coded solution - # if storedCred.provider.storedCred.provider.region_name == 'us-east-1': - # ec2_zones = ['us-east-1a', 'us-east-1b', 'us-east-1c', 'us-east-1d'] - # providersToZones[storedCred.name] = ec2_zones - # elif storedCred.provider.region_name == 'eu-west-1': - # ec2_zones = ['eu-west-1a', 'eu-west-1b'] - # providersToZones[storedCred.name] = ec2_zones - # elif storedCred.provider.type == 'eucalyptus': - # providersToZones[storedCred.name] = ['epc'] - # else: - # providersToZones[storedCred.name] = ['Unknown provider zone'] - - return trans.fill_template( "cloud/configure_uci.mako", - instanceName = instanceName, - credName = storedCreds, - volSize = volSize, - zone = zone, - error = error, - providersToZones = providersToZones ) - - @web.expose - @web.require_login( "start Galaxy cloud instance" ) - def start( self, trans, id, type='m1.small' ): - """ - Start a new cloud resource instance - """ - user = trans.get_user() - uci = get_uci( trans, id ) - stores = get_stores( trans, uci ) - # Ensure instance is available and then store relevant data - # into DB to initiate instance startup by cloud manager - if ( len(stores) is not 0 ) and ( uci.state == uci_states.AVAILABLE ): - instance = model.CloudInstance() - instance.user = user - instance.uci = uci - instance.state = instance_states.SUBMITTED - instance.availability_zone = stores[0].availability_zone # Bc. all EBS volumes need to be in the same avail. zone, just check 1st - instance.type = type - uci.state = uci_states.SUBMITTED_UCI - # Persist - session = trans.sa_session - session.add( instance ) - session.add( uci ) - session.flush() - # Log - trans.log_event ("User initiated starting of UCI '%s'." % uci.name ) - trans.set_message( "Galaxy instance started. NOTE: Please wait about 5 minutes for the instance to " - "start up. A button to connect to the instance will appear alongside " - "instance description once cloud instance of Galaxy is ready." ) - return self.list( trans ) - - if len(stores) == 0: - error( "This instance does not have any storage volumes associated it and thus cannot be started." ) - else: - error( "Cannot start instance that is in state '%s'." % uci.state ) - return self.list( trans ) - - @web.expose - @web.require_login( "stop Galaxy cloud instance" ) - def stop( self, trans, id ): - """ - Stop a cloud UCI instance. - """ - uci = get_uci( trans, id ) - if ( uci.state != uci_states.DELETING ) and \ - ( uci.state != uci_states.DELETING_UCI ) and \ - ( uci.state != uci_states.ERROR ) and \ - ( uci.state != uci_states.SHUTTING_DOWN_UCI ) and \ - ( uci.state != uci_states.SHUTTING_DOWN ) and \ - ( uci.state != uci_states.ADD_STORAGE_UCI ) and \ - ( uci.state != uci_states.ADD_STORAGE ) and \ - ( uci.state != uci_states.AVAILABLE ): - uci.state = uci_states.SHUTTING_DOWN_UCI - session = trans.sa_session - session.add( uci ) - session.flush() - trans.log_event( "User stopped cloud instance '%s' (id: %s)" % ( uci.name, uci.id ) ) - trans.set_message( "Stopping of Galaxy instance '%s' initiated." % uci.name ) - - return self.list( trans ) - - trans.show_error_message( "Cannot stop instance that is in state '%s'." % uci.state ) - return self.list( trans ) - - @web.expose - @web.require_login( "use Galaxy cloud" ) - def set_uci_state( self, trans, id, state='available', clear_error=True ): - """ - Sets state of UCI to given state, optionally resets error field, and resets UCI's launch time field to 'None'. - """ - uci = get_uci( trans, id ) - uci.state = state - if clear_error: - uci.error = None - uci.launch_time = None - trans.sa_session.flush() - trans.set_message( "Instance '%s' state reset." % uci.name ) - return self.list( trans ) - - @web.expose - @web.require_login( "view instance details" ) - def view_uci_details( self, trans, id=None ): - """ - View details about running instance - """ - uci = get_uci( trans, id ) - instances = get_instances( trans, uci ) # TODO: Handle list (will probably need to be done in mako template) - - return trans.fill_template( "cloud/view_instance.mako", - liveInstance = instances ) - - @web.expose - @web.require_login( "use Galaxy cloud" ) - def rename_uci( self, trans, id, new_name=None ): - instance = get_uci( trans, id ) - if new_name is not None: - if len(new_name) > 255: - error( "Instance name must be less than 255 characters long." ) - user = trans.get_user() - name_exists = trans.sa_session.query( model.UCI ) \ - .filter_by( user=user, name=new_name ) \ - .first() - if name_exists: - error( "Specified name ('%s') is already used by an existing instance. Please choose an alternative name." % new_name ) - - # Update name in local DB - instance.name = new_name - trans.sa_session.flush() - trans.set_message( "Instance renamed to '%s'." % new_name ) - return self.list( trans ) - else: - return trans.show_form( - web.FormBuilder( url_for( id=trans.security.encode_id(instance.id) ), "Rename instance", submit_text="Rename" ) - .add_text( "new_name", "Instance name", value=instance.name ) ) - - @web.expose - @web.require_login( "use Galaxy cloud" ) - def uci_usage_report( self, trans, id ): - user = trans.get_user() - id = trans.security.decode_id( id ) - - prevInstances = trans.sa_session.query( model.CloudInstance ) \ - .filter_by( user=user, state=instance_states.TERMINATED, uci_id=id ) \ - .order_by( desc( model.CloudInstance.table.c.update_time ) ) \ - .all() - - return trans.fill_template( "cloud/view_usage.mako", prevInstances = prevInstances ) - - @web.expose - @web.require_login( "delete user configured Galaxy cloud instance" ) - def delete_uci( self, trans, id ): - """ - Deletes User Configured Instance (UCI) from the cloud and local database. NOTE that this implies deletion of - any and all storage associated with this UCI! - """ - uci = get_uci( trans, id ) - - if ( uci.state != uci_states.DELETING_UCI ) and ( uci.state != uci_states.DELETING ) and ( uci.state != uci_states.ERROR ): - name = uci.name - uci.state = uci_states.DELETING_UCI - session = trans.sa_session - session.add( uci ) - session.flush() - trans.log_event( "User marked cloud instance '%s' for deletion." % name ) - trans.set_message( "Galaxy instance '%s' marked for deletion." % name ) - return self.list( trans ) - - if uci.state != uci_states.ERROR: - trans.set_message( "Cannot delete instance in state ERROR." ) - else: - trans.set_message( "Instance '%s' is already marked for deletion." % uci.name ) - return self.list( trans ) - - # ----- Snapshot methods ----- - - @web.expose - @web.require_login( "use Galaxy cloud" ) - def create_snapshot( self, trans, id ): - user = trans.get_user() - id = trans.security.decode_id( id ) - uci = get_uci( trans, id ) - - stores = trans.sa_session.query( model.CloudStore ) \ - .filter_by( user=user, deleted=False, uci_id=id ) \ - .all() - - if ( len( stores ) > 0 ) and ( uci.state == uci_states.AVAILABLE ): - for store in stores: - snapshot = model.CloudSnapshot() - snapshot.user = user - snapshot.uci = uci - snapshot.store = store - snapshot.status = snapshot_status.SUBMITTED - uci.state = uci_states.SNAPSHOT_UCI - # Persist - session = trans.sa_session - session.add( snapshot ) - session.add( uci ) - session.flush() - elif len( stores ) == 0: - error( "No storage volumes found that are associated with this instance." ) - else: - error( "Snapshot can be created only for an instance that is in 'available' state." ) - - # Log and display the management page - trans.log_event( "User initiated creation of new snapshot." ) - trans.set_message( "Creation of new snapshot initiated. " ) - return self.list( trans ) - - @web.expose - @web.require_login( "use Galaxy cloud" ) - def view_snapshots( self, trans, id=None ): - """ - View details about any snapshots associated with given UCI - """ - user = trans.get_user() - id = trans.security.decode_id( id ) - - snaps = trans.sa_session.query( model.CloudSnapshot ) \ - .filter_by( user=user, uci_id=id, deleted=False ) \ - .order_by( desc( model.CloudSnapshot.table.c.update_time ) ) \ - .all() - - return trans.fill_template( "cloud/view_snapshots.mako", - snaps = snaps ) - - @web.expose - @web.require_login( "use Galaxy cloud" ) - def delete_snapshot( self, trans, uci_id=None, snap_id=None ): - """ - Initiates deletion of a snapshot - """ - user = trans.get_user() - snap_id = trans.security.decode_id( snap_id ) - # Set snapshot as 'ready for deletion' to be picked up by general updater - snap = trans.sa_session.query( model.CloudSnapshot ).get( snap_id ) - - if snap.status == snapshot_status.COMPLETED: - snap.status = snapshot_status.DELETE - trans.sa_session.add( snap ) - trans.sa_session.flush() - trans.set_message( "Snapshot '%s' is marked for deletion. Once the deletion is complete, it will no longer be visible in this list. " - "Please note that this process may take up to a minute." % snap.snapshot_id ) - else: - error( "Only snapshots in state 'completed' can be deleted. See the cloud provider directly " - "if you believe the snapshot is available and can be deleted." ) - - # Display new list of snapshots - uci_id = trans.security.decode_id( uci_id ) - snaps = trans.sa_session.query( model.CloudSnapshot ) \ - .filter_by( user=user, uci_id=uci_id, deleted=False ) \ - .order_by( desc( model.CloudSnapshot.table.c.update_time ) ) \ - .all() - - return trans.fill_template( "cloud/view_snapshots.mako", - snaps = snaps ) - - # ----- Storage methods ----- - - @web.expose - @web.require_login( "add instance storage" ) - def add_storage( self, trans, id, vol_size=None ): - error = None - uci = get_uci( trans, id ) - stores = get_stores_in_status( trans, uci, store_status.IN_USE ) - - # Start adding of storage making sure given UCI is running and that at least one - # storage volume is attached to it (this is needed to by cloud controller to know - # as which device to attach the new storage volume) - if uci.state == uci_states.RUNNING and len( stores ) > 0: - if vol_size is not None: - try: - vol_size = int( vol_size ) - except ValueError: - error = "Volume size must be integer value between 1 and 1000." - - if not error: - user = trans.get_user() - - storage = model.CloudStore() - storage.user = user - storage.uci = uci - storage.size = vol_size - storage.status = store_status.ADDING - - # Set state of instance - NOTE that this code will only work (with code in cloud controller) - # for scenario where a UCI is associated with *1* compute instance!!! - instances = get_instances( trans, uci ) - instances.state = instance_states.ADDING - - uci.state = uci_states.ADD_STORAGE_UCI - # Persist - session = trans.sa_session - session.add( instances ) - session.add( storage ) - session.add( uci ) - session.flush() - # Log and display the management page - trans.log_event( "User added storage volume to UCI: '%s'" % uci.name ) - trans.set_message( "Adding of storage to instance '%s' initiated." % uci.name ) - return self.list( trans ) - else: - error( "Storage can only be added to instances that are in state 'RUNNING' with existing " \ - "storage volume(s) already attached." ) - - return trans.show_form( - web.FormBuilder( url_for( id=trans.security.encode_id(uci.id) ), "Add storage to an instance", submit_text="Add" ) - .add_text( "vol_size", "Storage size (1-1000 GB)", value='', error=error ) ) - - # ----- Image methods ----- - @web.expose - @web.require_admin - def add_new_image( self, trans, provider_type='', image_id='', manifest='', architecture='', state=None ): - #id_error = arch_error = provider_error = manifest_error = None - error = {} - if provider_type or image_id or manifest or architecture: - if provider_type=='': - error['provider_error'] = "You must select cloud provider type for this machine image." - elif image_id=='' or len( image_id ) > 255: - error['id_error'] = "Image ID must be between 1 and 255 characters long." - elif trans.sa_session.query( model.CloudImage ) \ - .filter_by( deleted=False ) \ - .filter( model.CloudImage.table.c.image_id == image_id ) \ - .first(): - error['id_error'] = "Image with ID '" + image_id + "' is already registered. \ - Please choose another ID." - elif architecture=='': - error['arch_error'] = "You must select architecture type for this machine image." - else: - # Create new image - image = model.CloudImage() - image.provider_type = provider_type - image.image_id = image_id - image.manifest = manifest - image.architecture = architecture - # Persist - session = trans.sa_session - session.add( image ) - session.flush() - # Log and display the management page - trans.log_event( "New cloud image added: '%s'" % image.image_id ) - trans.set_message( "Cloud image '%s' added." % image.image_id ) - if state: - image.state = state - images = trans.sa_session.query( model.CloudImage ).all() - return trans.fill_template( '/cloud/list_images.mako', images=images ) - - return trans.fill_template( "cloud/add_image.mako", - provider_type = provider_type, - image_id = image_id, - manifest = manifest, - architecture = architecture, - error = error ) -# return trans.show_form( -# web.FormBuilder( web.url_for(), "Add new cloud image", submit_text="Add" ) -# .add_text( "provider_type", "Provider type", value='ec2 or eucalyptus', error=provider_error ) -# .add_text( "image_id", "Machine Image ID (AMI or EMI)", value='', error=id_error ) -# .add_text( "manifest", "Manifest", value='', error=manifest_error ) -# .add_text( "architecture", "Architecture", value='i386 or x86_64', error=arch_error ) ) - - @web.expose - @web.require_login( "use Galaxy cloud" ) - def list_machine_images( self, trans ): - images = trans.sa_session.query( model.CloudImage ).filter_by( deleted=False ).all() - return trans.fill_template( '/cloud/list_images.mako', images=images ) - - @web.expose - @web.require_admin - def delete_image( self, trans, id=None ): - if not isinstance( id, int ): - id = trans.security.decode_id( id ) - - image = trans.sa_session.query( model.CloudImage ).get( id ) - image.deleted = True - trans.sa_session.add( image ) - trans.sa_session.flush() - return self.list_machine_images( trans ) - - @web.expose - @web.require_admin - def edit_image( self, trans, provider_type='', image_id='', manifest='', architecture='', id='', edited=False ): - error = {} - if not isinstance( id, int ): - id = trans.security.decode_id( id ) - - if not edited: - image = trans.sa_session.query( model.CloudImage ).get( id ) - return trans.fill_template( "cloud/edit_image.mako", - image = image, - error = error - ) - else: - image = trans.sa_session.query( model.CloudImage ).get( id ) - if image_id=='' or len( image_id ) > 255: - error['id_error'] = "Image ID must be between 1 and 255 characters in length." - elif trans.sa_session.query( model.CloudImage ) \ - .filter_by( deleted=False ) \ - .filter( and_( model.CloudImage.table.c.id != image.id, model.CloudImage.table.c.image_id==image_id ) ) \ - .first(): - error['id_error'] = "Image with ID '" + image_id + "' already exist. Please choose an alternative name." - elif architecture=='' or len( architecture ) > 255: - error['arch_error'] = "Architecture type must be between 1 and 255 characters long." - if error: - return trans.fill_template( "cloud/edit_image.mako", - image = image, - error = error - ) - else: - image.image_id = image_id - image.manifest = manifest - image.architecture = architecture - # Persist - session = trans.sa_session - session.add( image ) - session.flush() - # Log and display the management page - trans.set_message( "Machine image '%s' edited." % image.image_id ) - return self.list_machine_images( trans ) - - # ----- Credentials methods ----- - - @web.expose - @web.require_login( "add credentials" ) - def add_credentials( self, trans, credName='', accessKey='', secretKey='', providerName='' ): - """ - Add user's cloud credentials stored under name `credName`. - """ - user = trans.get_user() - error = {} - - if credName or providerName or accessKey or secretKey: - if credName=='' or len( credName ) > 255: - error['cred_error'] = "Credentials name must be between 1 and 255 characters in length." - elif trans.sa_session.query( model.CloudUserCredentials ) \ - .filter_by( user=user, deleted=False ) \ - .filter( model.CloudUserCredentials.table.c.name == credName ) \ - .first(): - error['cred_error'] = "Credentials with that name already exist." - elif providerName=='': - error['provider_error'] = "You must select cloud provider associated with these credentials." - elif accessKey=='' or len( accessKey ) > 255: - error['access_key_error'] = "Access key must be between 1 and 255 characters long." - elif secretKey=='' or len( secretKey ) > 255: - error['secret_key_error'] = "Secret key must be between 1 and 255 characters long." - else: - # Create new user stored credentials - credentials = model.CloudUserCredentials() - credentials.name = credName - credentials.user = user - credentials.access_key = accessKey - credentials.secret_key = secretKey - provider = get_provider( trans, providerName ) - credentials.provider = provider - # Persist - session = trans.sa_session - session.add( credentials ) - session.flush() - # Log and display the management page - trans.log_event( "User added new credentials" ) - trans.set_message( "Credential '%s' created" % credentials.name ) - return self.list( trans ) - - providers = trans.sa_session.query( model.CloudProvider ).filter_by( user=user ).all() - return trans.fill_template( "cloud/add_credentials.mako", - credName = credName, - providerName = providerName, - accessKey = accessKey, - secretKey = secretKey, - error = error, - providers = providers - ) - - @web.expose - @web.require_login( "use Galaxy cloud" ) - def edit_credentials( self, trans, id, credName=None, accessKey=None, secretKey=None, edited=False ): - error = {} - if not edited: - credentials = get_stored_credentials( trans, id ) - return trans.fill_template( "cloud/edit_credentials.mako", - credential = credentials, - error = error - ) - else: - user = trans.get_user() - credentials = get_stored_credentials( trans, id ) - if credName=='' or len( credName ) > 255: - error['cred_error'] = "Credentials name must be between 1 and 255 characters in length." - elif trans.sa_session.query( model.CloudUserCredentials ) \ - .filter_by( user=user ) \ - .filter( and_( model.CloudUserCredentials.table.c.id != credentials.id, model.CloudUserCredentials.table.c.name==credName ) ) \ - .first(): - error['cred_error'] = "Credentials with name '" + credName + "' already exist. Please choose an alternative name." - elif accessKey=='' or len( accessKey ) > 255: - error['access_key_error'] = "Access key must be between 1 and 255 characters long." - elif secretKey=='' or len( secretKey ) > 255: - error['secret_key_error'] = "Secret key must be between 1 and 255 characters long." - - if error: - return trans.fill_template( "cloud/edit_credentials.mako", - credential = credentials, - error = error - ) - else: - # Edit user stored credentials - credentials.name = credName - credentials.access_key = accessKey - credentials.secret_key = secretKey - # Persist - session = trans.sa_session - session.add( credentials ) - session.flush() - # Log and display the management page - trans.set_message( "Credential '%s' edited." % credentials.name ) - return self.list( trans ) - - @web.expose - @web.require_login( "view credentials" ) - def view_credentials( self, trans, id=None ): - """ - View details for user credentials - """ - # Load credentials from database - stored = get_stored_credentials( trans, id ) - - return trans.fill_template( "cloud/view_credentials.mako", - credDetails = stored ) - - @web.expose - @web.require_login( "test cloud credentials" ) - def test_cred( self, trans, id=None ): - """ - Tests credentials provided by user with selected cloud provider - """ - - @web.expose - @web.require_login( "delete credentials" ) - def delete_credentials( self, trans, id=None ): - """ - Delete user's cloud credentials checking that no registered instances are tied to given credentials. - """ - # Load credentials from database - user = trans.get_user() - stored = get_stored_credentials( trans, id ) - # Check if there are any UCIs that depend on these credentials - UCI = None - UCI = trans.sa_session.query( model.UCI ) \ - .filter_by( user=user, credentials_id=stored.id, deleted=False ) \ - .first() - - if UCI == None: - # Delete and save - stored.deleted = True - trans.sa_session.add( stored ) - trans.sa_session.flush() - # Display the management page - trans.set_message( "Credentials '%s' deleted." % stored.name ) - return self.list( trans ) - else: - error( "Existing instance(s) depend on credentials '%s'. You must delete those instances before being able \ - to delete these credentials." % stored.name ) - return self.list( trans ) - - # ----- Provider methods ----- - - @web.expose - @web.require_login( "add provider" ) - def add_provider( self, trans, name='', type='', region_name='', region_endpoint='', is_secure='', host='', port='', proxy='', proxy_port='', - proxy_user='', proxy_pass='', debug='', https_connection_factory='', path='' ): - user = trans.get_user() - error = {} - - if region_name or region_endpoint or name or is_secure or port or proxy or debug or path: - try: - is_secure = int(is_secure) - except ValueError: - error['is_secure_error'] = "Field 'is secure' can only take on an integer value '0' or '1'" - - if trans.sa_session.query( model.CloudProvider ) \ - .filter_by (user=user, name=name) \ - .filter( model.CloudProvider.table.c.deleted != True ) \ - .first(): - error['name_error'] = "A provider with that name already exist." - elif name=='' or len( name ) > 255: - error['name_error'] = "Provider name must be between 1 and 255 characters long." - elif type=='': - error['type_error'] = "Provider type must be selected." - elif not (is_secure == 0 or is_secure == 1): - error['is_secure_error'] = "Field 'is secure' can only take on an integer value '0' or '1'" - else: - provider = model.CloudProvider() - provider.user = user - provider.type = type - provider.name = name - if region_name: - provider.region_name = region_name - else: - provider.region_name = None - - if region_endpoint: - provider.region_endpoint = region_endpoint - else: - provider.region_endpoint = None - - if is_secure==0: - provider.is_secure = False - else: - provider.is_secure = True - - if host: - provider.host = host - else: - provider.host = None - - if port: - provider.port = port - else: - provider.port = None - - if proxy: - provider.proxy = proxy - else: - provider.proxy = None - - if proxy_port: - provider.proxy_port = proxy_port - else: - provider.proxy_port = None - - if proxy_user: - provider.proxy_user = proxy_user - else: - provider.proxy_user = None - - if proxy_pass: - provider.proxy_pass = proxy_pass - else: - provider.proxy_pass = None - - if debug: - provider.debug = debug - else: - provider.debug = None - - if https_connection_factory: - provider.https_connection_factory = https_connection_factory - else: - provider.https_connection_factory = None - - provider.path = path - # Persist - session = trans.sa_session - session.add( provider ) - session.flush() - # Log and display the management page - trans.log_event( "User configured new cloud provider: '%s'" % name ) - trans.set_message( "New cloud provider '%s' added." % name ) - return self.list( trans ) - - return trans.fill_template( "cloud/add_provider.mako", - name = name, - type = type, - region_name = region_name, - region_endpoint = region_endpoint, - is_secure = is_secure, - host = host, - port = port, - proxy = proxy, - proxy_port = proxy_port, - proxy_user = proxy_user, - proxy_pass = proxy_pass, - debug = debug, - https_connection_factory = https_connection_factory, - path = path, - error = error - ) - - @web.expose - @web.require_login( "add Amazon EC2 provider" ) - def add_ec2( self, trans ): - """ Default provider setup for Amazon's EC2. """ - self.add_provider( trans, name='Amazon EC2', type='ec2', region_name='us-east-1', region_endpoint='us-east-1.ec2.amazonaws.com', is_secure=1, path='/' ) - return self.add( trans ) - - @web.expose - @web.require_login( "use Galaxy cloud" ) - def view_provider( self, trans, id=None ): - """ - View details about given cloud provider - """ - # Load credentials from database - provider = get_provider_by_id( trans, id ) - - return trans.fill_template( "cloud/view_provider.mako", - provider = provider ) - - @web.expose - @web.require_login( "use Galaxy cloud" ) - def edit_provider( self, trans, id, name='', type='', region_name='', region_endpoint='', is_secure='', host='', port='', proxy='', proxy_port='', - proxy_user='', proxy_pass='', debug='', https_connection_factory='', path='', edited=False ): - error = {} - if edited == False: - provider = get_provider_by_id( trans, id ) - return trans.fill_template( "cloud/edit_provider.mako", - provider = provider, - error = error - ) - else: - user = trans.get_user() - provider = get_provider_by_id( trans, id ) - - try: - is_secure = int(is_secure) - except ValueError: - error['is_secure_error'] = "Field 'is secure' can only take on an integer value '0' or '1'" - - if name=='' or len( name ) > 255: - error['name_error'] = "Cloud provider name must be between 1 and 255 characters in length." - elif trans.sa_session.query( model.CloudProvider ) \ - .filter_by( user=user ) \ - .filter( and_( model.CloudProvider.table.c.id != provider.id, model.CloudProvider.table.c.name == name ) ) \ - .first(): - error['name_error'] = "Cloud provider with name '" + name + "' already exist. Please choose an alternative name." - elif not ( is_secure == 0 or is_secure == 1): - error['is_secure_error'] = "Field 'is secure' can only take on an integer value '0' or '1'" - - if error: - return trans.fill_template( "cloud/edit_provider.mako", - provider = provider, - error = error - ) - else: - provider.name = name - if region_name and region_name != 'None': - provider.region_name = region_name - else: - provider.region_name = None - - if region_endpoint and region_endpoint != 'None': - provider.region_endpoint = region_endpoint - else: - provider.region_endpoint = None - - if is_secure==0: - provider.is_secure = False - else: - provider.is_secure = True - - if host and host != 'None': - provider.host = host - else: - provider.host = None - - if port and port != 'None': - provider.port = port - else: - provider.port = None - - if proxy and proxy != 'None': - provider.proxy = proxy - else: - provider.proxy = None - - if proxy_port and proxy_port != 'None': - provider.proxy_port = proxy_port - else: - provider.proxy_port = None - - if proxy_user and proxy_user != 'None': - provider.proxy_user = proxy_user - else: - provider.proxy_user = None - - if proxy_pass and proxy_pass != 'None': - provider.proxy_pass = proxy_pass - else: - provider.proxy_pass = None - - if debug and debug != 'None': - provider.debug = debug - else: - provider.debug = None - - if https_connection_factory and https_connection_factory != 'None': - provider.https_connection_factory = https_connection_factory - else: - provider.https_connection_factory = None - - if path and path != 'None': - provider.path = path - else: - provider.path = None - # Persist - session = trans.sa_session - session.add( provider ) - session.flush() - # Log and display the management page - trans.log_event( "User edited cloud provider: '%s'" % name ) - trans.set_message( "Cloud provider '%s' edited." % name ) - return self.list( trans ) - - @web.expose - @web.require_login( "delete credentials" ) - def delete_provider( self, trans, id=None ): - """ - Delete use-registered cloud provider checking that no registered credentials are tied to given provider. - """ - # Load provider from database - user = trans.get_user() - provider = get_provider_by_id( trans, id ) - creds = trans.sa_session.query( model.CloudUserCredentials ) \ - .filter_by( user=user, provider_id=provider.id ) \ - .filter( model.CloudUserCredentials.table.c.deleted != True ) \ - .all() - - if len( creds ) == 0: - # Delete and save - #sess = trans.sa_session - provider.deleted = True - trans.sa_session.add( provider ) - trans.sa_session.flush() - # Display the management page - trans.set_message( "Cloud provider '%s' deleted." % provider.name ) - return self.list( trans ) - - error( "Existing credentails depend on cloud provider '%s'. You must delete those credentials before being able \ - to delete this cloud provider." % provider.name ) - return self.list( trans ) - - # ----- AJAX methods ----- - - @web.json - def json_update( self, trans ): - user = trans.get_user() - UCIs = trans.sa_session.query( model.UCI ).filter_by( user=user, deleted=False ).all() - insd = {} # instance name-state dict - for uci in UCIs: - dict = {} - dict['id'] = uci.id - dict['state'] = uci.state - dict['total_size'] = uci.total_size - if uci.error != None: - dict['error'] = str( uci.error ) - else: - dict['error'] = None - if uci.launch_time != None: - dict['launch_time'] = str( uci.launch_time ) - dict['time_ago'] = str( date.distance_of_time_in_words( uci.launch_time, date.datetime.utcnow() ) ) - else: - dict['launch_time'] = None - dict['time_ago'] = None - insd[uci.name] = dict - return insd - - @web.json - def link_update( self, trans, uci_id=0 ): - ild = {} # instance-link-dict - dict = {} - dict['uci_id'] = uci_id - try: - user = trans.get_user() - # TODO: This query can assumes only one instance under given UCI can be running (i.e., started). - inst = trans.sa_session.query( model.CloudInstance ).filter_by( user=user, uci_id=uci_id, state=uci_states.RUNNING ).first() - urllib2.urlopen( "http://" + inst.public_dns ) - dict['public_dns'] = inst.public_dns - dict['inst_id'] = inst.id - ild['data'] = dict - return ild - except urllib2.URLError: - dict['public_dns'] = False - ild['data'] = dict - return ild - -## ---- Utility methods ------------------------------------------------------- - -def get_provider( trans, name ): - user = trans.get_user() - return trans.sa_session.query( model.CloudProvider ) \ - .filter_by (user=user, name=name) \ - .first() - -def get_provider_by_id( trans, id, check_ownership=True ): - # Check if 'id' is in int (i.e., it was called from this program) or - # it was passed from the web (in which case decode it) - if not isinstance( id, int ): - id = trans.security.decode_id( id ) - - stored = trans.sa_session.query( model.CloudProvider ).get( id ) - if not stored: - error( "Cloud provider not found" ) - # Verify ownership - user = trans.get_user() - if not user: - error( "Must be logged in to use the cloud." ) - if check_ownership and not( stored.user == user ): - error( "Cloud provider '%s' is not registered by current user." % stored.name ) - # Looks good - return stored - -def get_stored_credentials( trans, id, check_ownership=True ): - """ - Get StoredUserCredentials from the database by id, verifying ownership. - """ - # Check if 'id' is in int (i.e., it was called from this program) or - # it was passed from the web (in which case decode it) - if not isinstance( id, int ): - id = trans.security.decode_id( id ) - - stored = trans.sa_session.query( model.CloudUserCredentials ).get( id ) - if not stored: - error( "Credentials not found" ) - # Verify ownership - user = trans.get_user() - if not user: - error( "Must be logged in to use the cloud." ) - if check_ownership and not( stored.user == user ): - error( "Credentials are not owned by current user." ) - # Looks good - return stored - -def get_uci( trans, id, check_ownership=True ): - """ - Get a UCI object from the database by id, verifying ownership. - """ - # Check if 'id' is in int (i.e., it was called from this program) or - # it was passed from the web (in which case decode it) - if not isinstance( id, int ): - id = trans.security.decode_id( id ) - - live = trans.sa_session.query( model.UCI ).get( id ) - if not live: - error( "Galaxy instance not found." ) - # Verify ownership - user = trans.get_user() - if not user: - error( "Must be logged in to use the cloud." ) - if check_ownership and not( live.user == user ): - error( "Instance is not owned by current user." ) - # Looks good - return live - -def get_stores( trans, uci ): - """ - Get stores objects that are associated with given uci and are not in 'error' status - """ - user = trans.get_user() - stores = trans.sa_session.query( model.CloudStore ) \ - .filter_by( user=user, uci_id=uci.id, deleted=False ) \ - .filter( model.CloudStore.table.c.status != store_status.ERROR ) \ - .all() - - return stores - -def get_stores_in_status( trans, uci, status ): - """ - Get stores objects that are associated with given uci and are not have given status - """ - user = trans.get_user() - stores = trans.sa_session.query( model.CloudStore ) \ - .filter_by( user=user, uci_id=uci.id, status=status ) \ - .all() - - return stores - -def get_instances( trans, uci ): - """ - Get objects of instances that are pending or running and are connected to the given uci object - """ - user = trans.get_user() - instances = trans.sa_session.query( model.CloudInstance ) \ - .filter_by( user=user, uci_id=uci.id ) \ - .filter( or_(model.CloudInstance.table.c.state==instance_states.RUNNING, model.CloudInstance.table.c.state==instance_states.PENDING ) ) \ - .first() - #.all() #TODO: return all but need to edit calling method(s) to handle list - - return instances - -def get_instances_in_state( trans, uci, state ): - """ - Get objects of instances that are in specified state and are connected to the given uci object - """ - user = trans.get_user() - instances = trans.sa_session.query( model.CloudInstance ) \ - .filter_by( user=user, uci_id=uci.id, state=state ) \ - .all() - - return instances - -def get_connection( trans, creds ): - """ - Establishes cloud connection using user's credentials - """ - log.debug( 'Establishing cloud connection.' ) -# user = trans.get_user() -# creds = trans.sa_session.query( model.CloudUserCredentials ) \ -# .filter_by( user=user, name=credName ) \ -# .first() - #.filter( model.CloudUserCredentials.table.c.deleted != True ) \ MOVE TO LINE ABOVE ONCE DELETE COLUMS ARE IMPLEMENTED - - if creds: - a_key = creds.access_key - s_key = creds.secret_key - try: - euca_region = RegionInfo( None, creds.provider.region_name, creds.provider.region_endpoint ) - conn = EC2Connection( aws_access_key_id=a_key, - aws_secret_access_key=s_key, - is_secure=creds.provider.is_secure, - port=creds.provider.port, - region=euca_region, - path=creds.provider.path ) - except boto.exception.EC2ResponseError, e: - log.error( "Establishing connection with cloud failed: %s" % str(e) ) - return None - - return conn --- /dev/null +++ b/lib/galaxy/model/migrate/versions/0050_drop_cloud_tables.py @@ -0,0 +1,147 @@ +from sqlalchemy import * +from migrate import * + +import datetime +now = datetime.datetime.utcnow + +# Need our custom types, but don't import anything else from model +from galaxy.model.custom_types import * + +import logging +log = logging.getLogger( __name__ ) + +metadata = MetaData( migrate_engine ) + +def display_migration_details(): + print + print "========================================" + print "This script drops tables that were associated with the old Galaxy Cloud functionality." + print "========================================" + +CloudImage_table = Table( "cloud_image", metadata, + Column( "id", Integer, primary_key=True ), + Column( "create_time", DateTime, default=now ), + Column( "update_time", DateTime, default=now, onupdate=now ), + Column( "provider_type", TEXT ), + Column( "image_id", TEXT, nullable=False ), + Column( "manifest", TEXT ), + Column( "state", TEXT ), + Column( "architecture", TEXT ), + Column( "deleted", Boolean, default=False ) ) + +""" UserConfiguredInstance (UCI) table """ +UCI_table = Table( "cloud_uci", metadata, + Column( "id", Integer, primary_key=True ), + Column( "create_time", DateTime, default=now ), + Column( "update_time", DateTime, default=now, onupdate=now ), + Column( "user_id", Integer, ForeignKey( "galaxy_user.id" ), index=True, nullable=False ), + Column( "credentials_id", Integer, ForeignKey( "cloud_user_credentials.id" ), index=True ), + Column( "key_pair_name", TEXT ), + Column( "key_pair_material", TEXT ), + Column( "name", TEXT ), + Column( "state", TEXT ), + Column( "error", TEXT ), + Column( "total_size", Integer ), + Column( "launch_time", DateTime ), + Column( "deleted", Boolean, default=False ) ) + +CloudInstance_table = Table( "cloud_instance", metadata, + Column( "id", Integer, primary_key=True ), + Column( "create_time", DateTime, default=now ), + Column( "update_time", DateTime, default=now, onupdate=now ), + Column( "launch_time", DateTime ), + Column( "stop_time", DateTime ), + Column( "user_id", Integer, ForeignKey( "galaxy_user.id" ), index=True, nullable=False ), + Column( "uci_id", Integer, ForeignKey( "cloud_uci.id" ), index=True ), + Column( "type", TEXT ), + Column( "reservation_id", TEXT ), + Column( "instance_id", TEXT ), + Column( "mi_id", Integer, ForeignKey( "cloud_image.id" ), index=True ), + Column( "state", TEXT ), + Column( "error", TEXT ), + Column( "public_dns", TEXT ), + Column( "private_dns", TEXT ), + Column( "security_group", TEXT ), + Column( "availability_zone", TEXT ) ) + +CloudStore_table = Table( "cloud_store", metadata, + Column( "id", Integer, primary_key=True ), + Column( "create_time", DateTime, default=now ), + Column( "update_time", DateTime, default=now, onupdate=now ), + Column( "attach_time", DateTime ), + Column( "user_id", Integer, ForeignKey( "galaxy_user.id" ), index=True, nullable=False ), + Column( "uci_id", Integer, ForeignKey( "cloud_uci.id" ), index=True, nullable=False ), + Column( "volume_id", TEXT ), + Column( "size", Integer, nullable=False ), + Column( "availability_zone", TEXT ), + Column( "inst_id", Integer, ForeignKey( "cloud_instance.id" ) ), + Column( "status", TEXT ), + Column( "device", TEXT ), + Column( "space_consumed", Integer ), + Column( "error", TEXT ), + Column( "deleted", Boolean, default=False ) ) + +CloudSnapshot_table = Table( "cloud_snapshot", metadata, + Column( "id", Integer, primary_key=True ), + Column( "create_time", DateTime, default=now ), + Column( "update_time", DateTime, default=now, onupdate=now ), + Column( "user_id", Integer, ForeignKey( "galaxy_user.id" ), index=True, nullable=False ), + Column( "uci_id", Integer, ForeignKey( "cloud_uci.id" ), index=True ), + Column( "store_id", Integer, ForeignKey( "cloud_store.id" ), index=True, nullable=False ), + Column( "snapshot_id", TEXT ), + Column( "status", TEXT ), + Column( "description", TEXT ), + Column( "error", TEXT ), + Column( "deleted", Boolean, default=False ) ) + +CloudUserCredentials_table = Table( "cloud_user_credentials", metadata, + Column( "id", Integer, primary_key=True ), + Column( "create_time", DateTime, default=now ), + Column( "update_time", DateTime, default=now, onupdate=now ), + Column( "user_id", Integer, ForeignKey( "galaxy_user.id" ), index=True, nullable=False ), + Column( "provider_id", Integer, ForeignKey( "cloud_provider.id" ), index=True, nullable=False ), + Column( "name", TEXT ), + Column( "access_key", TEXT ), + Column( "secret_key", TEXT ), + Column( "deleted", Boolean, default=False ) ) + +CloudProvider_table = Table( "cloud_provider", metadata, + Column( "id", Integer, primary_key=True ), + Column( "create_time", DateTime, default=now ), + Column( "update_time", DateTime, default=now, onupdate=now ), + Column( "user_id", Integer, ForeignKey( "galaxy_user.id" ), index=True, nullable=False ), + Column( "type", TEXT, nullable=False ), + Column( "name", TEXT ), + Column( "region_connection", TEXT ), + Column( "region_name", TEXT ), + Column( "region_endpoint", TEXT ), + Column( "is_secure", Boolean ), + Column( "host", TEXT ), + Column( "port", Integer ), + Column( "proxy", TEXT ), + Column( "proxy_port", TEXT ), + Column( "proxy_user", TEXT ), + Column( "proxy_pass", TEXT ), + Column( "debug", Integer ), + Column( "https_connection_factory", TEXT ), + Column( "path", TEXT ), + Column( "deleted", Boolean, default=False ) ) + +def upgrade(): + display_migration_details() + # Load existing tables + metadata.reflect() + try: + CloudProvider_table.drop() + CloudUserCredentials_table.drop() + CloudImage_table.drop() + UCI_table.drop() + CloudInstance_table.drop() + CloudStore_table.drop() + CloudSnapshot_table.drop() + except Exception, e: + log.debug( "Dropping cloud tables failed: %s" % str( e ) ) + + +def downgrade(): + pass --- a/lib/galaxy/model/mapping.py +++ b/lib/galaxy/model/mapping.py @@ -409,117 +409,6 @@ GalaxySessionToHistoryAssociation.table Column( "session_id", Integer, ForeignKey( "galaxy_session.id" ), index=True ), Column( "history_id", Integer, ForeignKey( "history.id" ), index=True ) ) -# *************************** Start cloud tables*********************************** -CloudImage.table = Table( "cloud_image", metadata, - Column( "id", Integer, primary_key=True ), - Column( "create_time", DateTime, default=now ), - Column( "update_time", DateTime, default=now, onupdate=now ), - Column( "provider_type", TEXT ), - Column( "image_id", TEXT, nullable=False ), - Column( "manifest", TEXT ), - Column( "state", TEXT ), - Column( "architecture", TEXT ), - Column( "deleted", Boolean, default=False ) ) - -""" UserConfiguredInstance (UCI) table """ -UCI.table = Table( "cloud_uci", metadata, - Column( "id", Integer, primary_key=True ), - Column( "create_time", DateTime, default=now ), - Column( "update_time", DateTime, default=now, onupdate=now ), - Column( "user_id", Integer, ForeignKey( "galaxy_user.id" ), index=True, nullable=False ), - Column( "credentials_id", Integer, ForeignKey( "cloud_user_credentials.id" ), index=True ), - Column( "key_pair_name", TEXT ), - Column( "key_pair_material", TEXT ), - Column( "name", TEXT ), - Column( "state", TEXT ), - Column( "error", TEXT ), - Column( "total_size", Integer ), - Column( "launch_time", DateTime ), - Column( "deleted", Boolean, default=False ) ) - -CloudInstance.table = Table( "cloud_instance", metadata, - Column( "id", Integer, primary_key=True ), - Column( "create_time", DateTime, default=now ), - Column( "update_time", DateTime, default=now, onupdate=now ), - Column( "launch_time", DateTime ), - Column( "stop_time", DateTime ), - Column( "user_id", Integer, ForeignKey( "galaxy_user.id" ), index=True, nullable=False ), - Column( "uci_id", Integer, ForeignKey( "cloud_uci.id" ), index=True ), - Column( "type", TEXT ), - Column( "reservation_id", TEXT ), - Column( "instance_id", TEXT ), - Column( "mi_id", Integer, ForeignKey( "cloud_image.id" ), index=True ), - Column( "state", TEXT ), - Column( "error", TEXT ), - Column( "public_dns", TEXT ), - Column( "private_dns", TEXT ), - Column( "security_group", TEXT ), - Column( "availability_zone", TEXT ) ) - -CloudStore.table = Table( "cloud_store", metadata, - Column( "id", Integer, primary_key=True ), - Column( "create_time", DateTime, default=now ), - Column( "update_time", DateTime, default=now, onupdate=now ), - Column( "attach_time", DateTime ), - Column( "user_id", Integer, ForeignKey( "galaxy_user.id" ), index=True, nullable=False ), - Column( "uci_id", Integer, ForeignKey( "cloud_uci.id" ), index=True, nullable=False ), - Column( "volume_id", TEXT ), - Column( "size", Integer, nullable=False ), - Column( "availability_zone", TEXT ), - Column( "inst_id", Integer, ForeignKey( "cloud_instance.id" ) ), - Column( "status", TEXT ), - Column( "device", TEXT ), - Column( "space_consumed", Integer ), - Column( "error", TEXT ), - Column( "deleted", Boolean, default=False ) ) - -CloudSnapshot.table = Table( "cloud_snapshot", metadata, - Column( "id", Integer, primary_key=True ), - Column( "create_time", DateTime, default=now ), - Column( "update_time", DateTime, default=now, onupdate=now ), - Column( "user_id", Integer, ForeignKey( "galaxy_user.id" ), index=True, nullable=False ), - Column( "uci_id", Integer, ForeignKey( "cloud_uci.id" ), index=True ), - Column( "store_id", Integer, ForeignKey( "cloud_store.id" ), index=True, nullable=False ), - Column( "snapshot_id", TEXT ), - Column( "status", TEXT ), - Column( "description", TEXT ), - Column( "error", TEXT ), - Column( "deleted", Boolean, default=False ) ) - -CloudUserCredentials.table = Table( "cloud_user_credentials", metadata, - Column( "id", Integer, primary_key=True ), - Column( "create_time", DateTime, default=now ), - Column( "update_time", DateTime, default=now, onupdate=now ), - Column( "user_id", Integer, ForeignKey( "galaxy_user.id" ), index=True, nullable=False ), - Column( "provider_id", Integer, ForeignKey( "cloud_provider.id" ), index=True, nullable=False ), - Column( "name", TEXT ), - Column( "access_key", TEXT ), - Column( "secret_key", TEXT ), - Column( "deleted", Boolean, default=False ) ) - -CloudProvider.table = Table( "cloud_provider", metadata, - Column( "id", Integer, primary_key=True ), - Column( "create_time", DateTime, default=now ), - Column( "update_time", DateTime, default=now, onupdate=now ), - Column( "user_id", Integer, ForeignKey( "galaxy_user.id" ), index=True, nullable=False ), - Column( "type", TEXT, nullable=False ), - Column( "name", TEXT ), - Column( "region_connection", TEXT ), - Column( "region_name", TEXT ), - Column( "region_endpoint", TEXT ), - Column( "is_secure", Boolean ), - Column( "host", TEXT ), - Column( "port", Integer ), - Column( "proxy", TEXT ), - Column( "proxy_port", TEXT ), - Column( "proxy_user", TEXT ), - Column( "proxy_pass", TEXT ), - Column( "debug", Integer ), - Column( "https_connection_factory", TEXT ), - Column( "path", TEXT ), - Column( "deleted", Boolean, default=False ) ) -# *************************** End cloud tables*********************************** - StoredWorkflow.table = Table( "stored_workflow", metadata, Column( "id", Integer, primary_key=True ), Column( "create_time", DateTime, default=now ), @@ -1284,42 +1173,6 @@ assign_mapper( context, GalaxySessionToH HistoryDatasetAssociation.mapper.add_property( "creating_job_associations", relation( JobToOutputDatasetAssociation ) ) -# vvvvvvvvvvvvvvvv Start cloud table mappings vvvvvvvvvvvvvvvv -assign_mapper( context, CloudImage, CloudImage.table ) - -assign_mapper( context, UCI, UCI.table, - properties=dict( user=relation( User ), - credentials=relation( CloudUserCredentials ), - instance=relation( CloudInstance, backref='uci' ), - store=relation( CloudStore, backref='uci', cascade='all, delete-orphan' ), - snapshot=relation( CloudSnapshot, backref='uci' ) - ) ) - -assign_mapper( context, CloudInstance, CloudInstance.table, - properties=dict( user=relation( User ), - image=relation( CloudImage ) - ) ) - -assign_mapper( context, CloudStore, CloudStore.table, - properties=dict( user=relation( User ), - inst=relation( CloudInstance ), - snapshot=relation( CloudSnapshot, backref="store" ) - ) ) - -assign_mapper( context, CloudSnapshot, CloudSnapshot.table, - properties=dict( user=relation( User ) - ) ) - -assign_mapper( context, CloudProvider, CloudProvider.table, - properties=dict( user=relation( User ) - ) ) - -assign_mapper( context, CloudUserCredentials, CloudUserCredentials.table, - properties=dict( user=relation( User), - provider=relation( CloudProvider ) - ) ) -# ^^^^^^^^^^^^^^^ End cloud table mappings ^^^^^^^^^^^^^^^^^^ - assign_mapper( context, Workflow, Workflow.table, properties=dict( steps=relation( WorkflowStep, backref='workflow', order_by=asc(WorkflowStep.table.c.order_index), --- a/lib/galaxy/app.py +++ b/lib/galaxy/app.py @@ -73,8 +73,6 @@ class UniverseApplication( object ): # FIXME: These are exposed directly for backward compatibility self.job_queue = self.job_manager.job_queue self.job_stop_queue = self.job_manager.job_stop_queue - # Start the cloud manager - self.cloud_manager = cloud.CloudManager( self ) # Track Store ## self.track_store = store.TrackStoreManager( self.config.track_store_path ) --- a/templates/cloud/view_snapshots.mako +++ /dev/null @@ -1,90 +0,0 @@ -<%inherit file="/base.mako"/> - -<%def name="title()">Snapshots</%def> - -%if message: -<% - try: - messagetype - except: - messagetype = "done" -%> - - - -<p /> -<div class="${messagetype}message"> - ${message} -</div> -%endif - -%if snaps: - <h2>Snapshots for instance ${snaps[0].uci.name}</h2> -%else: - <h2>Selected instance has no recorded or associated snapshots.</h2> -%endif - -<ul class="manage-table-actions"> - <li> - <a class="action-button" href="${h.url_for( action='list' )}"> - <img src="${h.url_for('/static/images/silk/resultset_previous.png')}" /> - <span>Return to cloud management console</span> - </a> - </li> -</ul> - -%if snaps: - <table class="mange-table colored" border="0" cellspacing="0" cellpadding="0" width="100%"> - <colgroup width="2%"></colgroup> - <colgroup width="16%"></colgroup> - <colgroup width="16%"></colgroup> - <colgroup width="10%"></colgroup> - <colgroup width="5%"></colgroup> - <tr class="header"> - <th>#</th> - <th>Create time</th> - <th>Snapshot ID</th> - <th>Status</th> - <th>Delete?</th> - <th></th> - </tr> - <% - total_hours = 0 - %> - %for i, snap in enumerate( snaps ): - <tr> - <td>${i+1}</td> - <td> - %if snap.create_time: - ${str(snap.create_time)[:16]} UCT - %else: - N/A - %endif - </td> - <td> - %if snap.snapshot_id: - ${snap.snapshot_id} - %else: - N/A - %endif - </td> - <td> - %if snap.status: - ${snap.status} - %else: - N/A - %endif - </td> - <td> - <a confirm="Are you sure you want to delete snapshot '${snap.snapshot_id}'?" - href="${h.url_for( controller='cloud', action='delete_snapshot', uci_id=trans.security.encode_id(snap.uci.id), snap_id=trans.security.encode_id(snap.id) )}">x</a> - </td> - </tr> - %endfor - </table> -%endif - - - - - --- a/eggs.ini +++ b/eggs.ini @@ -54,7 +54,6 @@ wsgiref = 0.1.2 Babel = 0.9.4 wchartype = 0.1 Whoosh = 0.3.18 -boto = 1.8d ; extra version information [tags] --- a/templates/cloud/configure_cloud.mako +++ /dev/null @@ -1,360 +0,0 @@ -<%inherit file="/base.mako"/> - -<%def name="title()">Cloud home</%def> - -%if message: -<% - try: - messagetype - except: - messagetype = "done" -%> - -<div class="${messagetype}message"> - ${message} -</div> -%endif - -<%def name="javascripts()"> -${parent.javascripts()} -${h.js( "jquery" )} - -<script type="text/javascript"> -function trim19(str){ - var str = str.replace(/^\s\s*/, ''), - ws = /\s/, - i = str.length; - while (ws.test(str.charAt(--i))); - return str.slice(0, i + 1); -} - - function update_state() { - $.getJSON( "${h.url_for( action='json_update' )}", {}, function ( data ) { - for (var i in data) { - var elem = '#' + data[i].id; - // Because of different list managing 'live' vs. 'available' instances, reload url on various state changes - old_state = $(elem + "-state").text(); - prev_old_state = trim19( $(elem + "-state-p").text() ); - new_state = data[i].state; - error_msg = data[i].error; - //console.log( "old_state[%d] = %s", i, old_state ); - //console.log( "prev_old_state[%d] = %s", i, prev_old_state ); - //console.log( "new_state[%d] = %s", i, new_state ); - //console.log( "error_msg[%d] = %s", i, error_msg ); - if ( ( old_state=='pending' && new_state=='running' ) || ( old_state=='shutting-down' && new_state=='available' ) || \ - ( old_state=='running' && new_state=='available' ) || ( old_state=='running' && new_state=='error' ) || \ - ( old_state=='pending' && new_state=='available' ) || ( old_state=='submitted' && new_state=='available' ) || \ - ( prev_old_state.match('creating') && new_state=='available' ) ) { - var url = "${h.url_for( controller='cloud', action='list')}"; - location.replace( url ); - } - else if ( ( ( old_state != 'error' && old_state != '' ) && new_state == 'error' ) || ( !prev_old_state.match('error') && new_state == 'error' ) ) { - var url = "${h.url_for( controller='cloud', action='list')}"; - location.replace( url ); - } - - if ( new_state=='shutting-down' || new_state=='shutting-downUCI' ) { - $(elem + "-link").text( "" ); - } - - // Check if Galaxy website is accessible on given instance; if so, provide link. Otherwise, wait more. - else if ( ( $(elem+"-link").text().match('starting') || $(elem+"-link").text()=='' ) && new_state=='running' ) { - //console.log ( 'elem.text: ' + $(elem+"-link").text() ); - $.getJSON( "${h.url_for( action='link_update' )}", { uci_id: data[i].id }, function ( data ) { - for (var i in data) { - var dns = data[i].public_dns; - var uci = '#' + data[i].uci_id; - if( !dns ) { - $(uci+"-link").text( 'Galaxy starting...' ); - // http://stackoverflow.com/questions/275931/how-do-you-make-an-element-flash-i... - //$(uci+"-link").stop().animate({ fontSize: "14px" }, 1000).animate({ fontSize: "12px" }, 1000); - } - else { - $(uci+"-link").html( '<div align="right"><a class="action-button" href="http://'+dns+'" target="_blank">' + - '<span>Access Galaxy</span>'+ - '<img src="' + "${h.url_for( '/static/images/silk/resultset_next.png' )}" + '" /></div>' ); - //$(uci+"-link").stop().animate({ fontSize: "14px" }, 1000).animate({ fontSize: "12px" }, 1000); - } - } - }); - } - - // Update 'size', 'state' and 'time alive' fields - $(elem + "-size").text( data[i].total_size ); - $(elem + "-state").text( data[i].state ); - if ( new_state != 'error' ) { // Because 'error' state is handled as a JS link, don't include it in update - $(elem + "-state-p").text( data[i].state ); - } - - if (data[i].launch_time) { - $(elem + "-launch_time").text( data[i].launch_time.substring(0, 16 ) + " UTC (" + data[i].time_ago + ")" ); - } - else { - $(elem + "-launch_time").text( "N/A" ); - } - } - }); - setTimeout("update_state()", 15000); - } - - $(function() { - update_state(); - }); - -</script> -</%def> - -<h2>Galaxy in the clouds</h2> - -%if cloudProviders: - ## Manage user-registered cloud providers - <h3>Your registered cloud providers</h3> - <ul class="manage-table-actions"> - <li> - <a class="action-button" href="${h.url_for( action='add_provider' )}"> - <img src="${h.url_for('/static/images/silk/add.png')}" /> - <span>Add provider</span> - </a> - </li> - </ul> - - <table class="mange-table colored" border="0" cellspacing="0" cellpadding="0" width="100%"> - <tr class="header"> - <th>Provider name</th> - <th>Provider type</th> - <th></th> - </tr> - %for i, cloudProvder in enumerate( cloudProviders ): - <tr> - <td> - ${cloudProvder.name} - <a id="cp-${i}-popup" class="popup-arrow" style="display: none;">▼</a> - </td> - <td> - ${cloudProvder.type} - </td> - <td> - <div popupmenu="cp-${i}-popup"> - <a class="action-button" href="${h.url_for( action='view_provider', id=trans.security.encode_id(cloudProvder.id) )}">View</a> - <a class="action-button" href="${h.url_for( action='edit_provider', id=trans.security.encode_id(cloudProvder.id) )}">Edit</a> - <a class="action-button" confirm="Are you sure you want to delete cloud provider '${cloudProvder.name}'?" href="${h.url_for( action='delete_provider', id=trans.security.encode_id(cloudProvder.id) )}">Delete</a> - </div> - </td> - </tr> - %endfor - </table> - - - ## ***************************************************** - ## Manage user credentials - <h3>Your registered cloud credentials</h3> - - %if cloudCredentials: - <ul class="manage-table-actions"> - <li> - <a class="action-button" href="${h.url_for( action='add_credentials' )}"> - <img src="${h.url_for('/static/images/silk/add.png')}" /> - <span>Add credentials</span> - </a> - </li> - </ul> - <table class="mange-table colored" border="0" cellspacing="0" cellpadding="0" width="100%"> - <tr class="header"> - <th>Credentials name</th> - <th>Provider name (type)</th> - <th></th> - </tr> - - %for i, cloudCredential in enumerate( cloudCredentials ): - <tr> - <td> - ${cloudCredential.name} - <a id="cr-${i}-popup" class="popup-arrow" style="display: none;">▼</a> - </td> - <td> - ${cloudCredential.provider.name} - (${cloudCredential.provider.type}) - </td> - <td> - <div popupmenu="cr-${i}-popup"> - <a class="action-button" href="${h.url_for( action='view_credentials', id=trans.security.encode_id(cloudCredential.id) )}">View</a> - <a class="action-button" href="${h.url_for( action='edit_credentials', id=trans.security.encode_id(cloudCredential.id) )}">Edit</a> - <a class="action-button" confirm="Are you sure you want to delete credentials '${cloudCredential.name}'?" href="${h.url_for( action='delete_credentials', id=trans.security.encode_id(cloudCredential.id) )}">Delete</a> - </div> - </td> - </tr> - %endfor - </table> - - ## ***************************************************** - ## Manage live instances - <p /> - <h3>Manage your cloud instances</h3> - <ul class="manage-table-actions"> - <li> - <a class="action-button" href="${h.url_for( action='configure_new_uci' )}"> - <img src="${h.url_for('/static/images/silk/add.png')}" /> - <span>Configure new instance</span> - </a> - </li> - </ul> - - <table class="mange-table colored" border="0" cellspacing="0" cellpadding="0" width="100%"> - <colgroup width="40%"></colgroup> - <colgroup width="15%"></colgroup> - <colgroup width="10%"></colgroup> - <colgroup width="25%"></colgroup> - <colgroup width="10%"></colgroup> - <tr class="header"> - <th>Live instance name (credentials)</th> - <th>Storage size (GB)</th> - <th>State</th> - <th>Alive since</th> - <th></th> - <th></th> - </tr> - %if liveInstances: - %for i, liveInstance in enumerate( liveInstances ): - <tr> - <td> - ${liveInstance.name} (${liveInstance.credentials.name}) - <a id="li-${i}-popup" class="popup-arrow" style="display: none;">▼</a> - </td> - <td id="${ liveInstance.id }-size">${str(liveInstance.total_size)}</td> - <td id="${ liveInstance.id }-state">${str(liveInstance.state)}</td> - <td id="${ liveInstance.id }-launch_time"> - ##${str(liveInstance.launch_time)[:16]} - <% - #from datetime import datetime - #from datetime import timedelta - - # DB stores all times in GMT, so adjust for difference (4 hours) - #adjustedStarttime = liveInstance.update_time - timedelta(hours=4) - - #delta = datetime.now() - adjustedStarttime - #context.write( str(datetime.utcnow() ) ) - #context.write( str(delta) ) - - # This is where current time and since duration is calculated - if liveInstance.launch_time is None: - context.write( 'N/A' ) - else: - context.write( str( liveInstance.launch_time )[:16] ) - context.write( ' UTC (' ) - context.write( str(h.date.distance_of_time_in_words (liveInstance.launch_time, h.date.datetime.utcnow() ) ) ) - context.write( ')' ) - %> - </td> - ## Handled by JavaScript function - <td id="${ liveInstance.id }-link"></td> - <td> - <div popupmenu="li-${i}-popup"> - <a class="action-button" confirm="Are you sure you want to stop instance '${liveInstance.name}'?" href="${h.url_for( action='stop', id=trans.security.encode_id(liveInstance.id) )}">Stop</a> - <a class="action-button" href="${h.url_for( action='rename_uci', id=trans.security.encode_id(liveInstance.id) )}">Rename</a> - <a class="action-button" href="${h.url_for( action='view_uci_details', id=trans.security.encode_id(liveInstance.id) )}">View details</a> - <a class="action-button" href="${h.url_for( action='add_storage', id=trans.security.encode_id(liveInstance.id) )}">Add storage</a> - <a class="action-button" href="${h.url_for( action='uci_usage_report', id=trans.security.encode_id(liveInstance.id) )}">Usage report</a> - </div> - </td> - </tr> - %endfor - %else: - <tr> - <td>Currently, you have no live instances.</td> - </tr> - %endif - </table> - <p /> - ## ***************************************************** - ## Manage previously configured instances - ## <table class="mange-table noHR" border="0" cellspacing="0" cellpadding="0" width="100%"> - <table class="mange-table colored" border="0" cellspacing="0" cellpadding="0" width="100%"> - <colgroup width="40%"></colgroup> - <colgroup width="15%"></colgroup> - <colgroup width="10%"></colgroup> - <colgroup width="25%"></colgroup> - <colgroup width="10%"></colgroup> - <tr class="header"> - <th>Configured instance name (credentials)</th> - <th>Storage size (GB)</th> - <th>State</th> - <th></th> - <th></th> - </tr> - - %if prevInstances: - %for i, prevInstance in enumerate( prevInstances ): - <tr> - <td> - ${prevInstance.name} (${prevInstance.credentials.name}) - <a id="pi-${i}-popup" class="popup-arrow" style="display: none;">▼</a> - </td> - <td>${str(prevInstance.total_size)}</td> - <td id="${ prevInstance.id }-state-p"> - <%state = str(prevInstance.state)%> - %if state =='error': - <div id="${prevInstance.name}-short"> - <a onclick="document.getElementById('${prevInstance.name}-full').style.display = 'block'; - document.getElementById('${prevInstance.name}-short').style.display = 'none'; return 0" - href="javascript:void(0)"> - error - </a> - </div> - <div id="${prevInstance.name}-full" style="DISPLAY: none"> - <a onclick="document.getElementById('${prevInstance.name}-short').style.display = 'block'; - document.getElementById('${prevInstance.name}-full').style.display = 'none'; return 0;" - href="javascript:void(0)"> - error:</a><br /> - ${str(prevInstance.error)} - <p /> - <div style="font-size:10px;"> - <a href="${h.url_for( action='set_uci_state', id=trans.security.encode_id(prevInstance.id), state='available' )}">reset state</a> - </div> - </div> - %else: - ${str(prevInstance.state)} - %endif - </td> - <td> - <div popupmenu="pi-${i}-popup"> - <a class="action-button" href="${h.url_for( action='uci_usage_report', id=trans.security.encode_id(prevInstance.id) )}">Usage report</a> - <a class="action-button" href="${h.url_for( action='rename_uci', id=trans.security.encode_id(prevInstance.id) )}">Rename</a> - <a class="action-button" href="${h.url_for( action='start', id=trans.security.encode_id(prevInstance.id), type='m1.small' )}"> Start m1.small</a> - <a class="action-button" href="${h.url_for( action='start', id=trans.security.encode_id(prevInstance.id), type='c1.medium' )}"> Start c1.medium</a> - <a class="action-button" href="${h.url_for( action='start', id=trans.security.encode_id(prevInstance.id), type='m1.large' )}"> Start m1.large</a> - <a class="action-button" href="${h.url_for( action='start', id=trans.security.encode_id(prevInstance.id), type='m1.xlarge' )}"> Start m1.xlarge</a> - <a class="action-button" href="${h.url_for( action='start', id=trans.security.encode_id(prevInstance.id), type='c1.xlarge' )}"> Start c1.xlarge</a> - <a class="action-button" href="${h.url_for( action='start', id=trans.security.encode_id(prevInstance.id), type='m2.2xlarge' )}"> Start m2.2xlarge</a> - <a class="action-button" href="${h.url_for( action='start', id=trans.security.encode_id(prevInstance.id), type='m2.4xlarge' )}"> Start m2.4xlarge</a> - <a class="action-button" href="${h.url_for( action='create_snapshot', id=trans.security.encode_id(prevInstance.id) )}">Create snapshot</a> - <a class="action-button" href="${h.url_for( action='view_snapshots', id=trans.security.encode_id(prevInstance.id) )}">View snapshots</a> - <a class="action-button" confirm="Are you sure you want to delete instance '${prevInstance.name}'? This will delete all of your data assocaiated with this instance!" href="${h.url_for( action='delete_uci', id=trans.security.encode_id(prevInstance.id) )}">Delete</a> - </div> - </td> - </tr> - %endfor - %else: - <tr> - <td>You have no previously configured instances (or they are all currently alive).</td> - </tr> - %endif - </table> - - %else: - You have no credentials associated with your Galaxy account: - <a class="action-button" href="${h.url_for( action='add_credentials' )}"> - <img src="${h.url_for('/static/images/silk/add.png')}" /> - <span>add credentials</span> - </a> - or - <a href="http://aws.amazon.com/" target="_blank"> - open AWS account with Amazon</a>. - %endif - -%else: - You have no cloud providers registered with your Galaxy account: - <a class="action-button" href="${h.url_for( action='add_provider' )}"> - <img src="${h.url_for('/static/images/silk/add.png')}" /> - <span>add provider now</span> - </a> -%endif --- a/universe_wsgi.ini.sample +++ b/universe_wsgi.ini.sample @@ -201,11 +201,6 @@ use_interactive = True # Should default dataset access permissions be private for new users; default is False (datasets are public) new_user_dataset_access_role_default_private = False -# ---- Cloud Management -------------------------------------------------------- - -# Uncomment following line to enable cloud management mode (only leaves the Cloud, Help and User tabs on) -# cloud_controller_instance = True - # ---- Job Execution -------------------------------------------------------- # If running multiple Galaxy processes, one can be designated as the job --- a/lib/galaxy/cloud/__init__.py +++ /dev/null @@ -1,734 +0,0 @@ -import logging, threading, sys, os, time, subprocess, string, tempfile, re, traceback, shutil - -from galaxy import util, model, config -from galaxy.model import mapping -from galaxy.model.orm import lazyload -from galaxy.datatypes.tabular import * -from galaxy.datatypes.interval import * -from galaxy.datatypes import metadata -from galaxy.util.bunch import Bunch -from sqlalchemy import or_ - -import pkg_resources -pkg_resources.require( "PasteDeploy" ) - -from paste.deploy.converters import asbool - -from Queue import Queue, Empty - -log = logging.getLogger( __name__ ) - -uci_states = Bunch( - NEW_UCI = "newUCI", - NEW = "new", - CREATING = "creating", - DELETING_UCI = "deletingUCI", - DELETING = "deleting", - DELETED = "deleted", - SUBMITTED_UCI = "submittedUCI", - SUBMITTED = "submitted", - SHUTTING_DOWN_UCI = "shutting-downUCI", - SHUTTING_DOWN = "shutting-down", - ADD_STORAGE_UCI = "add-storageUCI", - ADD_STORAGE = "add-storage", - AVAILABLE = "available", - RUNNING = "running", - PENDING = "pending", - ERROR = "error", - SNAPSHOT_UCI = "snapshotUCI", - SNAPSHOT = "snapshot" -) -instance_states = Bunch( - TERMINATED = "terminated", - SUBMITTED = "submitted", - RUNNING = "running", - ADDING = "adding-storage", - PENDING = "pending", - SHUTTING_DOWN = "shutting-down", - ERROR = "error" -) - -store_status = Bunch( - WAITING = "waiting", - IN_USE = "in-use", - ADDING = "adding", - CREATING = "creating", - DELETED = 'deleted', - ERROR = "error" -) - -snapshot_status = Bunch( - SUBMITTED = 'submitted', - PENDING = 'pending', - COMPLETED = 'completed', - DELETE = 'delete', - DELETED= 'deleted', - ERROR = "error" -) - -class CloudManager( object ): - """ - Highest level interface to cloud management. - """ - def __init__( self, app ): - self.app = app - self.sa_session = app.model.context - if self.app.config.enable_cloud_execution == True: - # The dispatcher manager for underlying cloud instances - implements and contacts individual cloud providers - self.provider = CloudProvider( app ) - # Monitor for updating status of cloud instances - self.cloud_monitor = CloudMonitor( self.app, self.provider ) - else: - self.job_queue = self.job_stop_queue = NoopCloudMonitor() - - def shutdown( self ): - self.cloud_monitor.shutdown() - -class Sleeper( object ): - """ - Provides a 'sleep' method that sleeps for a number of seconds *unless* - the notify method is called (from a different thread). - """ - def __init__( self ): - self.condition = threading.Condition() - def sleep( self, seconds ): - self.condition.acquire() - self.condition.wait( seconds ) - self.condition.release() - def wake( self ): - self.condition.acquire() - self.condition.notify() - self.condition.release() - -class CloudMonitor( object ): - """ - Cloud manager, waits for user to instantiate a cloud instance and then invokes a - CloudProvider. - """ - STOP_SIGNAL = object() - def __init__( self, app, provider ): - """Start the cloud manager""" - self.app = app - # Keep track of the pid that started the cloud manager, only it - # has valid threads - self.parent_pid = os.getpid() - self.sa_session = app.model.context - - # Contains requests that are waiting (only use from monitor thread) - self.waiting = [] - - # Helper for interruptable sleep - self.sleeper = Sleeper() - self.running = True - self.provider = provider - self.monitor_thread = threading.Thread( target=self.__monitor ) - self.monitor_thread.start() - log.info( "Cloud manager started" ) - - def __monitor( self ): - """ - Daemon that continuously monitors cloud instance requests as well as state - of running instances. - """ - # HACK: Delay until after forking, we need a way to do post fork notification!!! - time.sleep( 10 ) - - cnt = 0 # Run global update only periodically so keep counter variable - while self.running: - try: - self.__monitor_step() - if cnt%30 == 0: # Run global update every 30 iterations (1 minute) - self.provider.update() - cnt = 0 - except: - log.exception( "Exception in cloud manager monitor_step" ) - # Sleep - cnt += 1 - self.sleeper.sleep( 2 ) - - def __monitor_step( self ): - """ - Called repeatedly by `monitor` to process cloud instance requests. - TODO: Update following description to match the code - Gets any new cloud instance requests from the database, then iterates - over all new and waiting jobs to check the state of the jobs each - depends on. If the job has dependencies that have not finished, it - it goes to the waiting queue. If the job has dependencies with errors, - it is marked as having errors and removed from the queue. Otherwise, - the job is dispatched. - """ - model = self.app.model - new_requests = [] - - for r in self.sa_session.query( model.UCI ) \ - .filter( or_( model.UCI.table.c.state==uci_states.NEW_UCI, - model.UCI.table.c.state==uci_states.SUBMITTED_UCI, - model.UCI.table.c.state==uci_states.SHUTTING_DOWN_UCI, - model.UCI.table.c.state==uci_states.DELETING_UCI, - model.UCI.table.c.state==uci_states.SNAPSHOT_UCI, - model.UCI.table.c.state==uci_states.ADD_STORAGE_UCI ) ) \ - .all(): - uci_wrapper = UCIwrapper( r, self.app ) - new_requests.append( uci_wrapper ) - - for uci_wrapper in new_requests: - self.sa_session.expunge_all() - self.put( uci_wrapper ) - - def put( self, uci_wrapper ): - """Add a request to the queue.""" - self.provider.put( uci_wrapper ) - self.sleeper.wake() - - def shutdown( self ): - """Attempts to gracefully shut down the worker thread""" - if self.parent_pid != os.getpid(): - # We're not the real queue, do nothing - return - else: - log.info( "Sending stop signal to worker thread" ) - self.running = False - self.sleeper.wake() - log.info( "cloud manager stopped" ) - self.dispatcher.shutdown() - -class UCIwrapper( object ): - """ - Wraps 'model.UCI' with convenience methods for state management - """ - def __init__( self, uci, app ): - self.uci_id = uci.id - self.app = app - self.sa_session = self.app.model.context - base_directory = os.path.join( self.app.config.job_working_directory, "cloud" ) - self.working_directory = os.path.join( base_directory, str( self.uci_id ) ) -# log.debug( "Cloud controller working directory for UCI DB ID '%s': '%s'" % ( self.uci_id, self.working_directory ) ) - if not os.path.exists( base_directory ): - os.mkdir( base_directory ) - - - # --------- Setter methods ----------------- - - def change_state( self, uci_state=None, instance_id=None, i_state=None ): - """ - Sets state for UCI and/or UCI's instance with instance_id as provided by cloud provider and stored in local - Galaxy database. - Need to provide either: (1) state for the UCI, or (2) instance_id and it's state, or (3) all arguments. - """ -# log.debug( "Changing state - new uci_state: %s, instance_id: %s, i_state: %s" % ( uci_state, instance_id, i_state ) ) - if uci_state is not None: - uci = self.sa_session.query( model.UCI ).get( self.uci_id ) - self.sa_session.refresh( uci ) - uci.state = uci_state - self.sa_session.flush() - if ( instance_id is not None ) and ( i_state is not None ): - instance = self.sa_session.query( model.CloudInstance ).filter_by( uci_id=self.uci_id, instance_id=instance_id).first() - instance.state = i_state - self.sa_session.add( instance ) - self.sa_session.flush() - - def set_mi( self, i_index, mi_id ): - """ - Sets Machine Image (MI), e.g., 'ami-66fa190f', for UCI's instance with given index as it - is stored in local Galaxy database. - """ - mi = self.sa_session.query( model.CloudImage ).filter( model.CloudImage.table.c.image_id==mi_id ).first() - instance = self.sa_session.query( model.CloudInstance ).get( i_index ) - instance.image = mi - self.sa_session.add( instance ) - self.sa_session.flush() - - def set_key_pair( self, key_name, key_material=None ): - """ - Sets key pair value for current UCI. - """ - uci = self.sa_session.query( model.UCI ).get( self.uci_id ) - self.sa_session.refresh( uci ) - uci.key_pair_name = key_name - if key_material is not None: - uci.key_pair_material = key_material - self.sa_session.flush() - - def set_instance_launch_time( self, launch_time, i_index=None, i_id=None ): - """ - Stores launch time in local database for instance with specified index - i_index (as it is stored in local - Galaxy database) or with specified instance ID - i_id (as obtained from the cloud provider AND stored - in local Galaxy Database). Either 'i_index' or 'i_id' needs to be provided. - """ - if i_index != None: - instance = self.sa_session.query( model.CloudInstance ).get( i_index ) - elif i_id != None: - instance = self.sa_session.query( model.CloudInstance ).filter_by( uci_id=self.uci_id, instance_id=i_id).first() - else: - return None - - instance.launch_time = launch_time - self.sa_session.add( instance ) - self.sa_session.flush() - - def set_uci_launch_time( self, launch_time ): - uci = self.sa_session.query( model.UCI ).get( self.uci_id ) - self.sa_session.refresh( uci ) - uci.launch_time = launch_time - self.sa_session.add( uci ) - self.sa_session.flush() - - def set_stop_time( self, stop_time, i_index=None, i_id=None ): - """ - Stores stop time in local database for instance with specified index - i_index (as it is stored in local - Galaxy database) or with specified instance ID - i_id (as obtained from the cloud provider AND stored - in local Galaxy Database). Either 'i_index' or 'i_id' needs to be provided. - """ - if i_index != None: - instance = self.sa_session.query( model.CloudInstance ).get( i_index ) - elif i_id != None: - instance = self.sa_session.query( model.CloudInstance ).filter_by( uci_id=self.uci_id, instance_id=i_id).first() - else: - return None - - instance.stop_time = stop_time - self.sa_session.add( instance ) - self.sa_session.flush() - - def set_security_group_name( self, security_group_name, i_index=None, i_id=None ): - """ - Stores security group name in local database for instance with specified index - i_index (as it is stored in local - Galaxy database) or with specified instance ID - i_id (as obtained from the cloud provider AND stored - in local Galaxy Database). Either 'i_index' or 'i_id' needs to be provided. - """ - if i_index != None: - instance = self.sa_session.query( model.CloudInstance ).get( i_index ) - elif i_id != None: - instance = self.sa_session.query( model.CloudInstance ).filter_by( uci_id=self.uci_id, instance_id=i_id).first() - else: - return None - - instance.security_group = security_group_name - self.sa_session.add( instance ) - self.sa_session.flush() - - def set_reservation_id( self, i_index, reservation_id ): - instance = self.sa_session.query( model.CloudInstance ).get( i_index ) - instance.reservation_id = reservation_id - self.sa_session.add( instance ) - self.sa_session.flush() - - def set_instance_id( self, i_index, instance_id ): - """ - i_index refers to UCI's instance ID as stored in local database - instance_id refers to real-world, cloud resource ID (e.g., 'i-78hd823a') - """ - instance = self.sa_session.query( model.CloudInstance ).get( i_index ) - instance.instance_id = instance_id - self.sa_session.add( instance ) - self.sa_session.flush() - -# def set_public_dns( self, instance_id, public_dns ): -# uci = self.sa_session.query( model.UCI ).get( self.uci_id ) -# self.sa_session.refresh( uci ) -# uci.instance[instance_id].public_dns = public_dns -# uci.instance[instance_id].flush() -# -# def set_private_dns( self, instance_id, private_dns ): -# uci = self.sa_session.query( model.UCI ).get( self.uci_id ) -# self.sa_session.refresh( uci ) -# uci.instance[instance_id].private_dns = private_dns -# uci.instance[instance_id].flush() - - def reset_uci_launch_time( self ): - uci = self.sa_session.query( model.UCI ).get( self.uci_id ) - self.sa_session.refresh( uci ) - uci.launch_time = None - self.sa_session.add( uci ) - self.sa_session.flush() - - def set_error( self, error, set_state=False ): - """ - Sets error field of given UCI in local Galaxy database as well as any instances associated with - this UCI whose state is 'None' or 'SUBMITTED'. If set_state is set to 'true', - method also sets state of give UCI and corresponding instances to 'error' - """ - uci = self.sa_session.query( model.UCI ).get( self.uci_id ) - self.sa_session.refresh( uci ) - uci.error = error - if set_state: - uci.state = uci_states.ERROR - # Process all instances associated with this UCI - instances = self.sa_session.query( model.CloudInstance ) \ - .filter_by( uci=uci ) \ - .filter( or_( model.CloudInstance.table.c.state==None, model.CloudInstance.table.c.state==instance_states.SUBMITTED ) ) \ - .all() - for i in instances: - i.error = error - i.state = instance_states.ERROR - self.sa_session.add( i ) - self.sa_session.flush() - - self.sa_session.add( uci ) - self.sa_session.flush() - - def set_deleted( self ): - uci = self.sa_session.query( model.UCI ).get( self.uci_id ) - self.sa_session.refresh( uci ) - uci.state = uci_states.DELETED # for bookkeeping reasons, mark as deleted but don't actually delete. - uci.deleted = True - self.sa_session.add( uci ) - self.sa_session.flush() - -# def set_store_device( self, store_id, device ): -# uci = self.sa_session.query( model.UCI ).get( self.uci_id ) -# self.sa_session.refresh( uci ) -# uci.store[store_id].device = device -# uci.store[store_id].flush() - - def set_uci_total_size( self, total_size ): - uci = self.sa_session.query( model.UCI ).get( self.uci_id ) - self.sa_session.refresh( uci ) - uci.total_size = total_size - self.sa_session.add( uci ) - self.sa_session.flush() - - def set_store_error( self, error, store_index=None, store_id=None ): - if store_index != None: - store = self.sa_session.query( model.CloudStore ).get( store_index ) - elif store_id != None: - store = self.sa_session.query( model.CloudStore ).filter_by( volume_id = store_id ).first() - else: - return None - - store.error = error - store.status = store_status.ERROR - self.sa_session.add( store ) - self.sa_session.flush() - - def set_store_status( self, vol_id, status ): - vol = self.sa_session.query( model.CloudStore ).filter( model.CloudStore.table.c.volume_id == vol_id ).first() - vol.status = status - self.sa_session.add( vol ) - self.sa_session.flush() - - def set_store_availability_zone( self, availability_zone, vol_id=None ): - """ - Sets availability zone of storage volumes for either ALL volumes associated with current - UCI or for the volume whose volume ID (e.g., 'vol-39F80512') is provided as argument. - """ - if vol_id is not None: - vol = self.sa_session.query( model.CloudStore ).filter( model.CloudStore.table.c.volume_id == vol_id ).all() - else: - vol = self.sa_session.query( model.CloudStore ).filter( model.CloudStore.table.c.uci_id == self.uci_id ).all() - - for v in vol: - v.availability_zone = availability_zone - self.sa_session.add( v ) - self.sa_session.flush() - - def set_store_volume_id( self, store_index, volume_id ): - """ - Given store index as it is stored in local database, set volume ID as it is registered - on the cloud provider (e.g., vol-39890501) - """ - - if store_index != None: - store = self.sa_session.query( model.CloudStore ).get( store_index ) - store.volume_id = volume_id - self.sa_session.add( store ) - self.sa_session.flush() - else: - return None - -# uci = self.sa_session.query( model.UCI ).get( self.uci_id ) -# self.sa_session.refresh( uci ) -# uci.store[store_index].volume_id = volume_id -# #uci.store[store_index].flush() -# self.sa_session.add( uci ) -# self.sa_session.flush() - - def set_store_instance( self, vol_id, instance_id ): - """ - Stores instance ID that given store volume is attached to. Store volume ID should - be given in following format: 'vol-78943248' - """ - vol = self.sa_session.query( model.CloudStore ).filter( model.CloudStore.table.c.volume_id == vol_id ).first() - inst = self.sa_session.query( model.CloudInstance ).filter_by( instance_id=instance_id ).first() - vol.inst = inst - self.sa_session.add( vol ) - self.sa_session.flush() - - def set_store_device( self, vol_id, device ): - """ - Stores instance ID that given store volume is attached to. Store volume ID should - be given in following format: 'vol-78943248' - """ - vol = self.sa_session.query( model.CloudStore ).filter( model.CloudStore.table.c.volume_id == vol_id ).first() - vol.device = str( device ) - self.sa_session.add( vol ) - self.sa_session.flush() - - def set_store_deleted( self, vol_id, status=None ): - """ - Set storage volume as deleted in local database. Optionally, set the volume status too. - """ - vol = self.sa_session.query( model.CloudStore ).filter( model.CloudStore.table.c.volume_id == vol_id ).first() - vol.deleted = True - if status != None: - vol.status = status - self.sa_session.add( vol ) - self.sa_session.flush() - - def set_snapshot_id( self, snap_index, id ): - snap = model.CloudSnapshot.get( snap_index ) - - snap.snapshot_id = id - self.sa_session.add( snap ) - self.sa_session.flush() - - def set_snapshot_status( self, status, snap_index=None, snap_id=None ): - if snap_index != None: - snap = self.sa_session.query( model.CloudSnapshot ).get( snap_index ) - elif snap_id != None: - snap = self.sa_session.query( model.CloudSnapshot ).filter_by( snapshot_id = snap_id).first() - else: - return - snap.status = status - self.sa_session.add( snap ) - self.sa_session.flush() - - def set_snapshot_error( self, error, snap_index=None, snap_id=None, set_status=False ): - if snap_index != None: - snap = self.sa_session.query( model.CloudSnapshot ).get( snap_index ) - elif snap_id != None: - snap = self.sa_session.query( model.CloudSnapshot ).filter_by( snapshot_id = snap_id).first() - else: - return - snap.error = error - if set_status: - snap.status = snapshot_status.ERROR - - self.sa_session.add( snap ) - self.sa_session.flush() - - # --------- Getter methods ----------------- - - def get_provider_type( self ): - """ Returns type of cloud provider associated with given UCI. """ - uci = self.sa_session.query( model.UCI ).get( self.uci_id ) - self.sa_session.refresh( uci ) - return uci.credentials.provider.type - - def get_provider( self ): - """ Returns database object of cloud provider associated with credentials of given UCI. """ - uci = self.sa_session.query( model.UCI ).get( self.uci_id ) - self.sa_session.refresh( uci ) - return uci.credentials.provider - - def get_instance_type( self, i_index ): - instance = self.sa_session.query( model.CloudInstance ).get( i_index ) - self.sa_session.refresh( instance ) - return instance.type - - def get_uci_state( self ): - uci = self.sa_session.query( model.UCI ).get( self.uci_id ) - self.sa_session.refresh( uci ) - return uci.state - - def get_instances_indexes( self, state=None ): - """ - Returns indexes of instances associated with given UCI as they are stored in local Galaxy database and - whose state corresponds to passed argument. Returned values enable indexing instances from local Galaxy database. - """ - uci = self.sa_session.query( model.UCI ).get( self.uci_id ) - self.sa_session.refresh( uci ) - instances = self.sa_session.query( model.CloudInstance ) \ - .filter_by( uci=uci ) \ - .filter( model.CloudInstance.table.c.state==state ) \ - .all() - il = [] - for i in instances: - il.append( i.id ) - - return il - - def get_instance_state( self, instance_id ): - uci = self.sa_session.query( model.UCI ).get( self.uci_id ) - self.sa_session.refresh( uci ) - return uci.instance[instance_id].state - - def get_instaces_in_state( self, state ): - """ Get database objects of all instances associated with this UCI in given state. """ - return self.sa_session.query( model.CloudInstance ) \ - .filter_by( uci_id=self.uci_id, state = state ) \ - .all() - - def get_instances_ids( self ): - """ - Returns list IDs of all instances' associated with this UCI that are not in 'terminated' or - 'error' but the state is defined (i.e., state is not None) - (e.g., return value: ['i-402906D2', 'i-q0290dsD2'] ). - """ - il = self.sa_session.query( model.CloudInstance ) \ - .filter_by( uci_id=self.uci_id ) \ - .filter( or_( model.CloudInstance.table.c.state != 'terminated', - model.CloudInstance.table.c.state != 'error', - model.CloudInstance.table.c.state != None ) ) \ - .all() - instanceList = [] - for i in il: - instanceList.append( i.instance_id ) - return instanceList - - def get_name( self ): - uci = self.sa_session.query( model.UCI ).get( self.uci_id ) - self.sa_session.refresh( uci ) - return uci.name - - def get_key_pair_name( self ): - """ - Returns keypair name associated with given UCI. - """ - uci = self.sa_session.query( model.UCI ).get( self.uci_id ) - self.sa_session.refresh( uci ) - return uci.key_pair_name - - def get_key_pair_material( self ): - """ - Returns keypair material (i.e., private key) associated with given UCI. - """ - uci = self.sa_session.query( model.UCI ).get( self.uci_id ) - self.sa_session.refresh( uci ) - return uci.key_pair_material - - def get_security_group_name( self, i_index=None, i_id=None ): - """ - Given EITHER instance index as it is stored in local Galaxy database OR instance ID as it is - obtained from cloud provider and stored in local Galaxy database, return security group name associated - with given instance. - """ - if i_index != None: - instance = self.sa_session.query( model.CloudInstance ).get( i_index ) - return instance.security_group - elif i_id != None: - instance = self.sa_session.query( model.CloudInstance ).filter_by( uci_id=self.uci_id, instance_id=i_id).first() - return instance.security_group - - def get_access_key( self ): - uci = self.sa_session.query( model.UCI ).get( self.uci_id ) - self.sa_session.refresh( uci ) - return uci.credentials.access_key - - def get_secret_key( self ): - uci = self.sa_session.query( model.UCI ).get( self.uci_id ) - self.sa_session.refresh( uci ) - return uci.credentials.secret_key - - def get_mi_id( self, instance_id=0 ): - uci = self.sa_session.query( model.UCI ).get( self.uci_id ) - self.sa_session.refresh( uci ) - return uci.instance[instance_id].image.image_id - - def get_public_dns( self, instance_id=0 ): - uci = self.sa_session.query( model.UCI ).get( self.uci_id ) - self.sa_session.refresh( uci ) - return uci.instance[instance_id].public_dns - - def get_private_dns( self, instance_id=0 ): - uci = self.sa_session.query( model.UCI ).get( self.uci_id ) - self.sa_session.refresh( uci ) - return uci.instance[instance_id].private_dns - - def get_uci_availability_zone( self ): - """ - Returns UCI's availability zone. - Because all of storage volumes associated with a given UCI must be in the same - availability zone, availability of a UCI is determined by availability zone of - any one storage volume. - """ - uci = self.sa_session.query( model.UCI ).get( self.uci_id ) - self.sa_session.refresh( uci ) - return uci.store[0].availability_zone - - def get_uci_total_size( self ): - uci = self.sa_session.query( model.UCI ).get( self.uci_id ) - self.sa_session.refresh( uci ) - return uci.total_size - - def get_store_size( self, store_id=0 ): - uci = self.sa_session.query( model.UCI ).get( self.uci_id ) - self.sa_session.refresh( uci ) - return uci.store[store_id].size - - def get_store_volume_id( self, store_id=0 ): - """ - Given store ID associated with this UCI, get volume ID as it is registered - on the cloud provider (e.g., 'vol-39890501') - """ - uci = self.sa_session.query( model.UCI ).get( self.uci_id ) - self.sa_session.refresh( uci ) - return uci.store[store_id].volume_id - - def get_all_stores_in_status( self, status ): - """ - Return database objects of all stores associated with this UCI that have their - status set to value passed as parameter. - """ - return self.sa_session.query( model.CloudStore ).filter_by( deleted=False, uci_id=self.uci_id, status=status ).all() - - def get_all_stores( self ): - """ Returns all storage volumes' database objects associated with this UCI that have not been marked as 'deleted'. """ - return self.sa_session.query( model.CloudStore ) \ - .filter_by( deleted=False, uci_id=self.uci_id ) \ - .all() - - def get_snapshots( self, status=None ): - """ Returns database objects for all snapshots associated with this UCI and in given status.""" - return self.sa_session.query( model.CloudSnapshot ).filter_by( uci_id=self.uci_id, status=status ).all() - - def get_uci( self ): - """ Returns database object for given UCI. """ - uci = self.sa_session.query( model.UCI ).get( self.uci_id ) - self.sa_session.refresh( uci ) - return uci - - def get_uci_working_directory( self ): - return self.working_directory - - def uci_launch_time_set( self ): - uci = self.sa_session.query( model.UCI ).get( self.uci_id ) - self.sa_session.refresh( uci ) - return uci.launch_time - -class CloudProvider( object ): - def __init__( self, app ): - import providers.eucalyptus - import providers.ec2 - - self.app = app - self.cloud_provider = {} - self.cloud_provider["eucalyptus"] = providers.eucalyptus.EucalyptusCloudProvider( app ) - self.cloud_provider["ec2"] = providers.ec2.EC2CloudProvider( app ) - - def put( self, uci_wrapper ): - """ Put given request for UCI manipulation into provider's request queue.""" - self.cloud_provider[uci_wrapper.get_provider_type()].put( uci_wrapper ) - - def update( self ): - """ - Runs a global status update across all providers for all UCIs in state other than 'terminated' and 'available'. - Reason behind this method is to sync state of local DB and real world resources. - """ - for provider in self.cloud_provider.keys(): -# log.debug( "Running global update for provider: '%s'" % provider ) - self.cloud_provider[provider].update() - - def shutdown( self ): - for runner in self.cloud_provider.itervalues(): - runner.shutdown() - -class NoopCloudMonitor( object ): - """ - Implements the CloudMonitor interface but does nothing - """ - def put( self, *args ): - return - def shutdown( self ): - return -