- galaxy-dev - lists.galaxyproject.org

[hg] galaxy 3103: Change cloud table creation order
by Greg Von Kuster 23 Nov '09

23 Nov '09

details: http://www.bx.psu.edu/hg/galaxy/rev/cffcf1cae94c changeset: 3103:cffcf1cae94c user: James Taylor <james(a)jamestaylor.org> date: Wed Nov 18 10:43:08 2009 -0500 description: Change cloud table creation order diffstat: lib/galaxy/model/migrate/versions/0026_cloud_tables.py | 18 +++++++++++------- 1 files changed, 11 insertions(+), 7 deletions(-) diffs (37 lines): diff -r b39f3d484220 -r cffcf1cae94c lib/galaxy/model/migrate/versions/0026_cloud_tables.py --- a/lib/galaxy/model/migrate/versions/0026_cloud_tables.py Wed Nov 18 09:52:38 2009 -0500 +++ b/lib/galaxy/model/migrate/versions/0026_cloud_tables.py Wed Nov 18 10:43:08 2009 -0500 @@ -132,21 +132,25 @@ # Load existing tables metadata.reflect() + CloudProvider_table.create() + CloudUserCredentials_table.create() + CloudImage_table.create() UCI_table.create() - CloudUserCredentials_table.create() + CloudStore_table.create() CloudSnapshot_table.create() CloudInstance_table.create() - CloudProvider_table.create() def downgrade(): metadata.reflect() + CloudInstance_table.drop() + CloudSnapshot_table.drop() + CloudStore_table.drop() + + UCI_table.drop() CloudImage_table.drop() - CloudInstance_table.drop() - CloudStore_table.drop() - CloudSnapshot_table.drop() + CloudUserCredentials_table.drop() - UCI_table.drop() - CloudProvider_table.drop() \ No newline at end of file + CloudProvider_table.drop()

1 0

[hg] galaxy 3102: Unicode fix: make it possible to display non-a...
by Greg Von Kuster 23 Nov '09

23 Nov '09

details: http://www.bx.psu.edu/hg/galaxy/rev/b39f3d484220 changeset: 3102:b39f3d484220 user: jeremy goecks <jeremy.goecks(a)emory.edu> date: Wed Nov 18 09:52:38 2009 -0500 description: Unicode fix: make it possible to display non-acii characters in dataset name. diffstat: lib/galaxy/datatypes/data.py | 5 ++++- templates/root/history_common.mako | 2 +- 2 files changed, 5 insertions(+), 2 deletions(-) diffs (27 lines): diff -r 03eb232d1111 -r b39f3d484220 lib/galaxy/datatypes/data.py --- a/lib/galaxy/datatypes/data.py Tue Nov 17 19:07:19 2009 -0500 +++ b/lib/galaxy/datatypes/data.py Wed Nov 18 09:52:38 2009 -0500 @@ -146,7 +146,10 @@ def display_name(self, dataset): """Returns formatted html of dataset name""" try: - return escape(dataset.name) + if type ( dataset.name ) is unicode: + return escape( dataset.name ) + else: + return escape( unicode( dataset.name, 'utf-8 ') ) except: return "name unavailable" def display_info(self, dataset): diff -r 03eb232d1111 -r b39f3d484220 templates/root/history_common.mako --- a/templates/root/history_common.mako Tue Nov 17 19:07:19 2009 -0500 +++ b/templates/root/history_common.mako Wed Nov 18 09:52:38 2009 -0500 @@ -43,7 +43,7 @@ %endif </div> <span class="state-icon"></span> - <span class="historyItemTitle"><b>${hid}: ${data.display_name().decode('utf-8')}</b></span> + <span class="historyItemTitle"><b>${hid}: ${data.display_name()}</b></span> </div> ## Body for history items, extra info and actions, data "peek"

1 0

[hg] galaxy 3101: merge
by Greg Von Kuster 23 Nov '09

23 Nov '09

details: http://www.bx.psu.edu/hg/galaxy/rev/03eb232d1111 changeset: 3101:03eb232d1111 user: jeremy goecks <jeremy.goecks(a)emory.edu> date: Tue Nov 17 19:07:19 2009 -0500 description: merge diffstat: eggs.ini | 6 +- lib/galaxy/app.py | 4 +- lib/galaxy/cloud/__init__.py | 653 ++++++++++++ lib/galaxy/cloud/providers/ec2.py | 995 ++++++++++++++++++ lib/galaxy/cloud/providers/eucalyptus.py | 1019 ++++++++++++++++++ lib/galaxy/config.py | 6 + lib/galaxy/model/__init__.py | 54 + lib/galaxy/model/mapping.py | 147 ++ lib/galaxy/model/migrate/versions/0026_cloud_tables.py | 152 ++ lib/galaxy/web/controllers/cloud.py | 1188 ++++++++++++++++++++++ lib/galaxy/web/controllers/forms.py | 252 ++- lib/galaxy/web/controllers/library_common.py | 2 +- lib/galaxy/web/controllers/requests.py | 302 +++-- lib/galaxy/web/controllers/requests_admin.py | 663 +++++++---- static/images/silk/resultset_previous.png | templates/admin/forms/edit_form.mako | 2 +- templates/admin/forms/grid.mako | 1 + templates/admin/forms/manage_forms.mako | 76 - templates/admin/forms/show_form_read_only.mako | 4 +- templates/admin/index.mako | 9 + templates/admin/requests/create_request_type.mako | 92 +- templates/admin/requests/grid.mako | 218 +---- templates/admin/requests/manage_request_types.mako | 69 +- templates/admin/requests/show_request.mako | 2 +- templates/admin/requests/view_request_type.mako | 70 +- templates/base_panels.mako | 14 +- templates/cloud/add_credentials.mako | 110 ++ templates/cloud/add_image.mako | 98 + templates/cloud/add_provider.mako | 252 ++++ templates/cloud/configure_cloud.mako | 354 ++++++ templates/cloud/configure_uci.mako | 116 ++ templates/cloud/edit_credentials.mako | 91 + templates/cloud/edit_image.mako | 92 + templates/cloud/edit_provider.mako | 261 ++++ templates/cloud/index.mako | 16 + templates/cloud/list_images.mako | 90 + templates/cloud/view_credentials.mako | 157 ++ templates/cloud/view_instance.mako | 140 ++ templates/cloud/view_provider.mako | 126 ++ templates/cloud/view_snapshots.mako | 90 + templates/cloud/view_usage.mako | 117 ++ templates/requests/grid.mako | 218 +---- templates/requests/show_request.mako | 2 +- templates/root/index.mako | 13 +- test/base/twilltestcase.py | 34 +- test/functional/test_forms_and_requests.py | 44 +- test/functional/test_user_info.py | 9 +- universe_wsgi.ini.sample | 5 + 48 files changed, 7244 insertions(+), 1191 deletions(-) diffs (truncated from 9241 to 3000 lines): diff -r 90723c58b1a6 -r 03eb232d1111 eggs.ini --- a/eggs.ini Tue Nov 17 19:06:42 2009 -0500 +++ b/eggs.ini Tue Nov 17 19:07:19 2009 -0500 @@ -52,6 +52,7 @@ wsgiref = 0.1.2 Babel = 0.9.4 wchartype = 0.1 +boto = 1.8d ; extra version information [tags] @@ -60,7 +61,7 @@ MySQL_python = _5.0.67_static python_lzo = _static bx_python = _dev_r4bf1f32e6b76 -GeneTrack = _dev_raa786e9fc131d998e532a1aef39d108850c9e93d +GeneTrack = _dev_e380f21c704218622155b9d230a44b3c9c452524 SQLAlchemy = _dev_r6498 ; nose = .dev_r7156749efc58 @@ -82,7 +83,7 @@ decorator = http://pypi.python.org/packages/source/d/decorator/decorator-3.1.2.tar.gz docutils = http://downloads.sourceforge.net/docutils/docutils-0.4.tar.gz elementtree = http://effbot.org/downloads/elementtree-1.2.6-20050316.tar.gz -GeneTrack = http://github.com/ialbert/genetrack-central/tarball/aa786e9fc131d998e532a1a… +GeneTrack = http://github.com/ialbert/genetrack-central/tarball/e380f21c704218622155b9d… lrucache = http://evan.prodromou.name/lrucache/lrucache-0.2.tar.gz Mako = http://www.makotemplates.org/downloads/Mako-0.2.5.tar.gz nose = http://pypi.python.org/packages/source/n/nose/nose-0.11.1.tar.gz @@ -103,3 +104,4 @@ wsgiref = http://pypi.python.org/packages/source/w/wsgiref/wsgiref-0.1.2.zip Babel = http://ftp.edgewall.com/pub/babel/Babel-0.9.4.zip wchartype = http://ginstrom.com/code/wchartype-0.1.zip +boto = http://boto.googlecode.com/files/boto-1.8d.tar.gz \ No newline at end of file diff -r 90723c58b1a6 -r 03eb232d1111 lib/galaxy/app.py --- a/lib/galaxy/app.py Tue Nov 17 19:06:42 2009 -0500 +++ b/lib/galaxy/app.py Tue Nov 17 19:07:19 2009 -0500 @@ -1,6 +1,6 @@ import sys, os, atexit -from galaxy import config, jobs, util, tools, web +from galaxy import config, jobs, util, tools, web, cloud ## from galaxy.tracks import store from galaxy.web import security import galaxy.model @@ -68,6 +68,8 @@ # FIXME: These are exposed directly for backward compatibility self.job_queue = self.job_manager.job_queue self.job_stop_queue = self.job_manager.job_stop_queue + # Start the cloud manager + self.cloud_manager = cloud.CloudManager( self ) # Track Store ## self.track_store = store.TrackStoreManager( self.config.track_store_path ) diff -r 90723c58b1a6 -r 03eb232d1111 lib/galaxy/cloud/__init__.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/lib/galaxy/cloud/__init__.py Tue Nov 17 19:07:19 2009 -0500 @@ -0,0 +1,653 @@ +import logging, threading, sys, os, time, subprocess, string, tempfile, re, traceback, shutil + +from galaxy import util, model, config +from galaxy.model import mapping +from galaxy.model.orm import lazyload +from galaxy.datatypes.tabular import * +from galaxy.datatypes.interval import * +from galaxy.datatypes import metadata +from galaxy.util.bunch import Bunch +from sqlalchemy import or_ + +import pkg_resources +pkg_resources.require( "PasteDeploy" ) + +from paste.deploy.converters import asbool + +from Queue import Queue, Empty + +log = logging.getLogger( __name__ ) + +uci_states = Bunch( + NEW_UCI = "newUCI", + NEW = "new", + CREATING = "creating", + DELETING_UCI = "deletingUCI", + DELETING = "deleting", + DELETED = "deleted", + SUBMITTED_UCI = "submittedUCI", + SUBMITTED = "submitted", + SHUTTING_DOWN_UCI = "shutting-downUCI", + SHUTTING_DOWN = "shutting-down", + AVAILABLE = "available", + RUNNING = "running", + PENDING = "pending", + ERROR = "error", + SNAPSHOT_UCI = "snapshotUCI", + SNAPSHOT = "snapshot" +) +instance_states = Bunch( + TERMINATED = "terminated", + SUBMITTED = "submitted", + RUNNING = "running", + PENDING = "pending", + SHUTTING_DOWN = "shutting-down", + ERROR = "error" +) + +snapshot_status = Bunch( + SUBMITTED = 'submitted', + PENDING = 'pending', + COMPLETED = 'completed', + DELETE = 'delete', + DELETED= 'deleted', + ERROR = "error" +) + +class CloudManager( object ): + """ + Highest level interface to cloud management. + """ + def __init__( self, app ): + self.app = app + self.sa_session = app.model.context + if self.app.config.enable_cloud_execution == True: + # The dispatcher manager for underlying cloud instances - implements and contacts individual cloud providers + self.provider = CloudProvider( app ) + # Monitor for updating status of cloud instances + self.cloud_monitor = CloudMonitor( self.app, self.provider ) + else: + self.job_queue = self.job_stop_queue = NoopCloudMonitor() + + def shutdown( self ): + self.cloud_monitor.shutdown() + +class Sleeper( object ): + """ + Provides a 'sleep' method that sleeps for a number of seconds *unless* + the notify method is called (from a different thread). + """ + def __init__( self ): + self.condition = threading.Condition() + def sleep( self, seconds ): + self.condition.acquire() + self.condition.wait( seconds ) + self.condition.release() + def wake( self ): + self.condition.acquire() + self.condition.notify() + self.condition.release() + +class CloudMonitor( object ): + """ + Cloud manager, waits for user to instantiate a cloud instance and then invokes a + CloudProvider. + """ + STOP_SIGNAL = object() + def __init__( self, app, provider ): + """Start the cloud manager""" + self.app = app + # Keep track of the pid that started the cloud manager, only it + # has valid threads + self.parent_pid = os.getpid() + self.sa_session = app.model.context + + # Contains requests that are waiting (only use from monitor thread) + self.waiting = [] + + # Helper for interruptable sleep + self.sleeper = Sleeper() + self.running = True + self.provider = provider + self.monitor_thread = threading.Thread( target=self.__monitor ) + self.monitor_thread.start() + log.info( "Cloud manager started" ) + + def __monitor( self ): + """ + Daemon that continuously monitors cloud instance requests as well as state + of running instances. + """ + # HACK: Delay until after forking, we need a way to do post fork notification!!! + time.sleep( 10 ) + + cnt = 0 # Run global update only periodically so keep counter variable + while self.running: + try: + self.__monitor_step() + if cnt%30 == 0: # Run global update every 30 iterations (1 minute) + self.provider.update() + cnt = 0 + except: + log.exception( "Exception in cloud manager monitor_step" ) + # Sleep + cnt += 1 + self.sleeper.sleep( 2 ) + + def __monitor_step( self ): + """ + Called repeatedly by `monitor` to process cloud instance requests. + TODO: Update following description to match the code + Gets any new cloud instance requests from the database, then iterates + over all new and waiting jobs to check the state of the jobs each + depends on. If the job has dependencies that have not finished, it + it goes to the waiting queue. If the job has dependencies with errors, + it is marked as having errors and removed from the queue. Otherwise, + the job is dispatched. + """ + model = self.app.model + new_requests = [] + + for r in self.sa_session.query( model.UCI ) \ + .filter( or_( model.UCI.table.c.state==uci_states.NEW_UCI, + model.UCI.table.c.state==uci_states.SUBMITTED_UCI, + model.UCI.table.c.state==uci_states.SHUTTING_DOWN_UCI, + model.UCI.table.c.state==uci_states.DELETING_UCI, + model.UCI.table.c.state==uci_states.SNAPSHOT_UCI ) ) \ + .all(): + uci_wrapper = UCIwrapper( r, self.app ) + new_requests.append( uci_wrapper ) + + for uci_wrapper in new_requests: + self.sa_session.expunge_all() + self.put( uci_wrapper ) + + def put( self, uci_wrapper ): + """Add a request to the queue.""" + self.provider.put( uci_wrapper ) + self.sleeper.wake() + + def shutdown( self ): + """Attempts to gracefully shut down the worker thread""" + if self.parent_pid != os.getpid(): + # We're not the real queue, do nothing + return + else: + log.info( "Sending stop signal to worker thread" ) + self.running = False + self.sleeper.wake() + log.info( "cloud manager stopped" ) + self.dispatcher.shutdown() + +class UCIwrapper( object ): + """ + Wraps 'model.UCI' with convenience methods for state management + """ + def __init__( self, uci, app ): + self.uci_id = uci.id + self.app = app + self.sa_session = self.app.model.context + + # --------- Setter methods ----------------- + + def change_state( self, uci_state=None, instance_id=None, i_state=None ): + """ + Sets state for UCI and/or UCI's instance with instance_id as provided by cloud provider and stored in local + Galaxy database. + Need to provide either: (1) state for the UCI, or (2) instance_id and it's state, or (3) all arguments. + """ +# log.debug( "Changing state - new uci_state: %s, instance_id: %s, i_state: %s" % ( uci_state, instance_id, i_state ) ) + if uci_state is not None: + uci = self.sa_session.query( model.UCI ).get( self.uci_id ) + self.sa_session.refresh( uci ) + uci.state = uci_state + self.sa_session.flush() + if ( instance_id is not None ) and ( i_state is not None ): + instance = self.sa_session.query( model.CloudInstance ).filter_by( uci_id=self.uci_id, instance_id=instance_id).first() + instance.state = i_state + self.sa_session.add( instance ) + self.sa_session.flush() + + def set_mi( self, i_index, mi_id ): + """ + Sets Machine Image (MI), e.g., 'ami-66fa190f', for UCI's instance with given index as it + is stored in local Galaxy database. + """ + mi = self.sa_session.query( model.CloudImage ).filter( model.CloudImage.table.c.image_id==mi_id ).first() + instance = self.sa_session.query( model.CloudInstance ).get( i_index ) + instance.image = mi + self.sa_session.add( instance ) + self.sa_session.flush() + + def set_key_pair( self, key_name, key_material=None ): + """ + Sets key pair value for current UCI. + """ + uci = self.sa_session.query( model.UCI ).get( self.uci_id ) + self.sa_session.refresh( uci ) + uci.key_pair_name = key_name + if key_material is not None: + uci.key_pair_material = key_material + self.sa_session.flush() + + def set_instance_launch_time( self, launch_time, i_index=None, i_id=None ): + """ + Stores launch time in local database for instance with specified index - i_index (as it is stored in local + Galaxy database) or with specified instance ID - i_id (as obtained from the cloud provider AND stored + in local Galaxy Database). Either 'i_index' or 'i_id' needs to be provided. + """ + if i_index != None: + instance = self.sa_session.query( model.CloudInstance ).get( i_index ) + elif i_id != None: + instance = self.sa_session.query( model.CloudInstance ).filter_by( uci_id=self.uci_id, instance_id=i_id).first() + else: + return None + + instance.launch_time = launch_time + self.sa_session.add( instance ) + self.sa_session.flush() + + def set_uci_launch_time( self, launch_time ): + uci = self.sa_session.query( model.UCI ).get( self.uci_id ) + self.sa_session.refresh( uci ) + uci.launch_time = launch_time + self.sa_session.add( uci ) + self.sa_session.flush() + + def set_stop_time( self, stop_time, i_index=None, i_id=None ): + """ + Stores stop time in local database for instance with specified index - i_index (as it is stored in local + Galaxy database) or with specified instance ID - i_id (as obtained from the cloud provider AND stored + in local Galaxy Database). Either 'i_index' or 'i_id' needs to be provided. + """ + if i_index != None: + instance = self.sa_session.query( model.CloudInstance ).get( i_index ) + elif i_id != None: + instance = self.sa_session.query( model.CloudInstance ).filter_by( uci_id=self.uci_id, instance_id=i_id).first() + else: + return None + + instance.stop_time = stop_time + self.sa_session.add( instance ) + self.sa_session.flush() + + def set_security_group_name( self, security_group_name, i_index=None, i_id=None ): + """ + Stores security group name in local database for instance with specified index - i_index (as it is stored in local + Galaxy database) or with specified instance ID - i_id (as obtained from the cloud provider AND stored + in local Galaxy Database). Either 'i_index' or 'i_id' needs to be provided. + """ + if i_index != None: + instance = self.sa_session.query( model.CloudInstance ).get( i_index ) + elif i_id != None: + instance = self.sa_session.query( model.CloudInstance ).filter_by( uci_id=self.uci_id, instance_id=i_id).first() + else: + return None + + instance.security_group = security_group_name + self.sa_session.add( instance ) + self.sa_session.flush() + + def set_reservation_id( self, i_index, reservation_id ): + instance = self.sa_session.query( model.CloudInstance ).get( i_index ) + instance.reservation_id = reservation_id + self.sa_session.add( instance ) + self.sa_session.flush() + + def set_instance_id( self, i_index, instance_id ): + """ + i_index refers to UCI's instance ID as stored in local database + instance_id refers to real-world, cloud resource ID (e.g., 'i-78hd823a') + """ + instance = self.sa_session.query( model.CloudInstance ).get( i_index ) + instance.instance_id = instance_id + self.sa_session.add( instance ) + self.sa_session.flush() + +# def set_public_dns( self, instance_id, public_dns ): +# uci = self.sa_session.query( model.UCI ).get( self.uci_id ) +# self.sa_session.refresh( uci ) +# uci.instance[instance_id].public_dns = public_dns +# uci.instance[instance_id].flush() +# +# def set_private_dns( self, instance_id, private_dns ): +# uci = self.sa_session.query( model.UCI ).get( self.uci_id ) +# self.sa_session.refresh( uci ) +# uci.instance[instance_id].private_dns = private_dns +# uci.instance[instance_id].flush() + + def reset_uci_launch_time( self ): + uci = self.sa_session.query( model.UCI ).get( self.uci_id ) + self.sa_session.refresh( uci ) + uci.launch_time = None + self.sa_session.add( uci ) + self.sa_session.flush() + + def set_error( self, error, set_state=False ): + """ + Sets error field of given UCI in local Galaxy database as well as any instances associated with + this UCI whose state is 'None' or 'SUBMITTED'. If set_state is set to 'true', + method also sets state of give UCI and corresponding instances to 'error' + """ + uci = self.sa_session.query( model.UCI ).get( self.uci_id ) + self.sa_session.refresh( uci ) + uci.error = error + if set_state: + uci.state = uci_states.ERROR + # Process all instances associated with this UCI + instances = self.sa_session.query( model.CloudInstance ) \ + .filter_by( uci=uci ) \ + .filter( or_( model.CloudInstance.table.c.state==None, model.CloudInstance.table.c.state==instance_states.SUBMITTED ) ) \ + .all() + for i in instances: + i.error = error + i.state = instance_states.ERROR + self.sa_session.add( i ) + self.sa_session.flush() + + self.sa_session.add( uci ) + self.sa_session.flush() + + def set_deleted( self ): + uci = self.sa_session.query( model.UCI ).get( self.uci_id ) + self.sa_session.refresh( uci ) + uci.state = uci_states.DELETED # for bookkeeping reasons, mark as deleted but don't actually delete. + uci.deleted = True + self.sa_session.add( uci ) + self.sa_session.flush() + + def set_store_device( self, store_id, device ): + uci = self.sa_session.query( model.UCI ).get( self.uci_id ) + self.sa_session.refresh( uci ) + uci.store[store_id].device = device + uci.store[store_id].flush() + + def set_store_error( self, error, store_index=None, store_id=None ): + if store_index != None: + store = self.sa_session.query( model.CloudStore ).get( store_index ) + elif store_id != None: + store = self.sa_session.query( model.CloudStore ).filter_by( volume_id = store_id ).first() + else: + return None + + store.error = error + self.sa_session.add( store ) + self.sa_session.flush() + + def set_store_status( self, vol_id, status ): + vol = self.sa_session.query( model.CloudStore ).filter( model.CloudStore.table.c.volume_id == vol_id ).first() + vol.status = status + self.sa_session.add( vol ) + self.sa_session.flush() + + def set_store_availability_zone( self, availability_zone, vol_id=None ): + """ + Sets availability zone of storage volumes for either ALL volumes associated with current + UCI or for the volume whose volume ID (e.g., 'vol-39F80512') is provided as argument. + """ + if vol_id is not None: + vol = self.sa_session.query( model.CloudStore ).filter( model.CloudStore.table.c.volume_id == vol_id ).all() + else: + vol = self.sa_session.query( model.CloudStore ).filter( model.CloudStore.table.c.uci_id == self.uci_id ).all() + + for v in vol: + v.availability_zone = availability_zone + self.sa_session.add( v ) + self.sa_session.flush() + + def set_store_volume_id( self, store_index, volume_id ): + """ + Given store index associated with this UCI in local database, set volume ID as it is registered + on the cloud provider (e.g., vol-39890501) + """ + uci = self.sa_session.query( model.UCI ).get( self.uci_id ) + self.sa_session.refresh( uci ) + uci.store[store_index].volume_id = volume_id + #uci.store[store_index].flush() + self.sa_session.add( uci ) + self.sa_session.flush() + + def set_store_instance( self, vol_id, instance_id ): + """ + Stores instance ID that given store volume is attached to. Store volume ID should + be given in following format: 'vol-78943248' + """ + vol = self.sa_session.query( model.CloudStore ).filter( model.CloudStore.table.c.volume_id == vol_id ).first() + vol.i_id = instance_id + self.sa_session.add( vol ) + self.sa_session.flush() + + def set_snapshot_id( self, snap_index, id ): + snap = model.CloudSnapshot.get( snap_index ) + + snap.snapshot_id = id + self.sa_session.add( snap ) + self.sa_session.flush() + + def set_snapshot_status( self, status, snap_index=None, snap_id=None ): + if snap_index != None: + snap = self.sa_session.query( model.CloudSnapshot ).get( snap_index ) + elif snap_id != None: + snap = self.sa_session.query( model.CloudSnapshot ).filter_by( snapshot_id = snap_id).first() + else: + return + snap.status = status + self.sa_session.add( snap ) + self.sa_session.flush() + + def set_snapshot_error( self, error, snap_index=None, snap_id=None, set_status=False ): + if snap_index != None: + snap = self.sa_session.query( model.CloudSnapshot ).get( snap_index ) + elif snap_id != None: + snap = self.sa_session.query( model.CloudSnapshot ).filter_by( snapshot_id = snap_id).first() + else: + return + snap.error = error + if set_status: + snap.status = snapshot_status.ERROR + + self.sa_session.add( snap ) + self.sa_session.flush() + + # --------- Getter methods ----------------- + + def get_provider_type( self ): + """ Returns type of cloud provider associated with given UCI. """ + uci = self.sa_session.query( model.UCI ).get( self.uci_id ) + self.sa_session.refresh( uci ) + return uci.credentials.provider.type + + def get_provider( self ): + """ Returns database object of cloud provider associated with credentials of given UCI. """ + uci = self.sa_session.query( model.UCI ).get( self.uci_id ) + self.sa_session.refresh( uci ) + return uci.credentials.provider + + def get_instance_type( self, i_index ): + instance = self.sa_session.query( model.CloudInstance ).get( i_index ) + self.sa_session.refresh( instance ) + return instance.type + + def get_uci_state( self ): + uci = self.sa_session.query( model.UCI ).get( self.uci_id ) + self.sa_session.refresh( uci ) + return uci.state + + def get_instances_indexes( self, state=None ): + """ + Returns indexes of instances associated with given UCI as they are stored in local Galaxy database and + whose state corresponds to passed argument. Returned values enable indexing instances from local Galaxy database. + """ + uci = self.sa_session.query( model.UCI ).get( self.uci_id ) + self.sa_session.refresh( uci ) + instances = self.sa_session.query( model.CloudInstance ) \ + .filter_by( uci=uci ) \ + .filter( model.CloudInstance.table.c.state==state ) \ + .all() + il = [] + for i in instances: + il.append( i.id ) + + return il + + def get_instance_state( self, instance_id ): + uci = self.sa_session.query( model.UCI ).get( self.uci_id ) + self.sa_session.refresh( uci ) + return uci.instance[instance_id].state + + def get_instances_ids( self ): + """ + Returns list IDs of all instances' associated with this UCI that are not in 'terminated' or + 'error' but the state is defined (i.e., state is not None) + (e.g., return value: ['i-402906D2', 'i-q0290dsD2'] ). + """ + il = self.sa_session.query( model.CloudInstance ) \ + .filter_by( uci_id=self.uci_id ) \ + .filter( or_( model.CloudInstance.table.c.state != 'terminated', + model.CloudInstance.table.c.state != 'error', + model.CloudInstance.table.c.state != None ) ) \ + .all() + instanceList = [] + for i in il: + instanceList.append( i.instance_id ) + return instanceList + + def get_name( self ): + uci = self.sa_session.query( model.UCI ).get( self.uci_id ) + self.sa_session.refresh( uci ) + return uci.name + + def get_key_pair_name( self ): + """ + Returns keypair name associated with given UCI. + """ + uci = self.sa_session.query( model.UCI ).get( self.uci_id ) + self.sa_session.refresh( uci ) + return uci.key_pair_name + + def get_key_pair_material( self ): + """ + Returns keypair material (i.e., private key) associated with given UCI. + """ + uci = self.sa_session.query( model.UCI ).get( self.uci_id ) + self.sa_session.refresh( uci ) + return uci.key_pair_material + + def get_security_group_name( self, i_index=None, i_id=None ): + """ + Given EITHER instance index as it is stored in local Galaxy database OR instance ID as it is + obtained from cloud provider and stored in local Galaxy database, return security group name associated + with given instance. + """ + if i_index != None: + instance = self.sa_session.query( model.CloudInstance ).get( i_index ) + return instance.security_group + elif i_id != None: + instance = self.sa_session.query( model.CloudInstance ).filter_by( uci_id=self.uci_id, instance_id=i_id).first() + return instance.security_group + + def get_access_key( self ): + uci = self.sa_session.query( model.UCI ).get( self.uci_id ) + self.sa_session.refresh( uci ) + return uci.credentials.access_key + + def get_secret_key( self ): + uci = self.sa_session.query( model.UCI ).get( self.uci_id ) + self.sa_session.refresh( uci ) + return uci.credentials.secret_key + + def get_mi_id( self, instance_id=0 ): + uci = self.sa_session.query( model.UCI ).get( self.uci_id ) + self.sa_session.refresh( uci ) + return uci.instance[instance_id].mi_id + + def get_public_dns( self, instance_id=0 ): + uci = self.sa_session.query( model.UCI ).get( self.uci_id ) + self.sa_session.refresh( uci ) + return uci.instance[instance_id].public_dns + + def get_private_dns( self, instance_id=0 ): + uci = self.sa_session.query( model.UCI ).get( self.uci_id ) + self.sa_session.refresh( uci ) + return uci.instance[instance_id].private_dns + + def get_uci_availability_zone( self ): + """ + Returns UCI's availability zone. + Because all of storage volumes associated with a given UCI must be in the same + availability zone, availability of a UCI is determined by availability zone of + any one storage volume. + """ + uci = self.sa_session.query( model.UCI ).get( self.uci_id ) + self.sa_session.refresh( uci ) + return uci.store[0].availability_zone + + def get_store_size( self, store_id=0 ): + uci = self.sa_session.query( model.UCI ).get( self.uci_id ) + self.sa_session.refresh( uci ) + return uci.store[store_id].size + + def get_store_volume_id( self, store_id=0 ): + """ + Given store ID associated with this UCI, get volume ID as it is registered + on the cloud provider (e.g., 'vol-39890501') + """ + uci = self.sa_session.query( model.UCI ).get( self.uci_id ) + self.sa_session.refresh( uci ) + return uci.store[store_id].volume_id + + def get_all_stores( self ): + """ Returns all storage volumes' database objects associated with this UCI. """ + return self.sa_session.query( model.CloudStore ).filter( model.CloudStore.table.c.uci_id == self.uci_id ).all() + + def get_snapshots( self, status=None ): + """ Returns database objects for all snapshots associated with this UCI and in given status.""" + return self.sa_session.query( model.CloudSnapshot ).filter_by( uci_id=self.uci_id, status=status ).all() + + def get_uci( self ): + """ Returns database object for given UCI. """ + uci = self.sa_session.query( model.UCI ).get( self.uci_id ) + self.sa_session.refresh( uci ) + return uci + + def uci_launch_time_set( self ): + uci = self.sa_session.query( model.UCI ).get( self.uci_id ) + self.sa_session.refresh( uci ) + return uci.launch_time + +class CloudProvider( object ): + def __init__( self, app ): + import providers.eucalyptus + import providers.ec2 + + self.app = app + self.cloud_provider = {} + self.cloud_provider["eucalyptus"] = providers.eucalyptus.EucalyptusCloudProvider( app ) + self.cloud_provider["ec2"] = providers.ec2.EC2CloudProvider( app ) + + def put( self, uci_wrapper ): + """ Put given request for UCI manipulation into provider's request queue.""" + self.cloud_provider[uci_wrapper.get_provider_type()].put( uci_wrapper ) + + def update( self ): + """ + Runs a global status update across all providers for all UCIs in state other than 'terminated' and 'available'. + Reason behind this method is to sync state of local DB and real world resources. + """ + for provider in self.cloud_provider.keys(): +# log.debug( "Running global update for provider: '%s'" % provider ) + self.cloud_provider[provider].update() + + def shutdown( self ): + for runner in self.cloud_provider.itervalues(): + runner.shutdown() + +class NoopCloudMonitor( object ): + """ + Implements the CloudMonitor interface but does nothing + """ + def put( self, *args ): + return + def shutdown( self ): + return + diff -r 90723c58b1a6 -r 03eb232d1111 lib/galaxy/cloud/providers/ec2.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/lib/galaxy/cloud/providers/ec2.py Tue Nov 17 19:07:19 2009 -0500 @@ -0,0 +1,995 @@ +import subprocess, threading, os, errno, time, datetime +from Queue import Queue, Empty +from datetime import datetime + +from galaxy import model # Database interaction class +from galaxy.model import mapping +from galaxy.datatypes.data import nice_size +from galaxy.util.bunch import Bunch +from galaxy.cloud import UCIwrapper +from Queue import Queue +from sqlalchemy import or_, and_ + +import galaxy.eggs +galaxy.eggs.require("boto") +from boto.ec2.connection import EC2Connection +from boto.ec2.regioninfo import RegionInfo +import boto.exception +import boto + +import logging +log = logging.getLogger( __name__ ) + +uci_states = Bunch( + NEW_UCI = "newUCI", + NEW = "new", + CREATING = "creating", + DELETING_UCI = "deletingUCI", + DELETING = "deleting", + SUBMITTED_UCI = "submittedUCI", + SUBMITTED = "submitted", + SHUTTING_DOWN_UCI = "shutting-downUCI", + SHUTTING_DOWN = "shutting-down", + AVAILABLE = "available", + RUNNING = "running", + PENDING = "pending", + ERROR = "error", + DELETED = "deleted", + SNAPSHOT_UCI = "snapshotUCI", + SNAPSHOT = "snapshot" +) + +instance_states = Bunch( + TERMINATED = "terminated", + SUBMITTED = "submitted", + RUNNING = "running", + PENDING = "pending", + SHUTTING_DOWN = "shutting-down", + ERROR = "error" +) + +store_status = Bunch( + IN_USE = "in-use", + CREATING = "creating", + DELETED = 'deleted', + ERROR = "error" +) + +snapshot_status = Bunch( + SUBMITTED = 'submitted', + PENDING = 'pending', + COMPLETED = 'completed', + DELETE = 'delete', + DELETED= 'deleted', + ERROR = "error" +) + +class EC2CloudProvider( object ): + """ + Amazon EC2-based cloud provider implementation for managing instances. + """ + STOP_SIGNAL = object() + def __init__( self, app ): + self.type = "ec2" # cloud provider type (e.g., ec2, eucalyptus, opennebula) + self.zone = "us-east-1a" + self.security_group = "galaxyWeb" + self.queue = Queue() + self.sa_session = app.model.context + + self.threads = [] + nworkers = 5 + log.info( "Starting EC2 cloud controller workers..." ) + for i in range( nworkers ): + worker = threading.Thread( target=self.run_next ) + worker.start() + self.threads.append( worker ) + log.debug( "%d EC2 cloud workers ready", nworkers ) + + def shutdown( self ): + """Attempts to gracefully shut down the monitor thread""" + log.info( "sending stop signal to worker threads in EC2 cloud manager" ) + for i in range( len( self.threads ) ): + self.queue.put( self.STOP_SIGNAL ) + log.info( "EC2 cloud manager stopped" ) + + def put( self, uci_wrapper ): + # Get rid of UCI from state description + state = uci_wrapper.get_uci_state() + uci_wrapper.change_state( state.split('U')[0] ) # remove 'UCI' from end of state description (i.e., mark as accepted and ready for processing) + self.queue.put( uci_wrapper ) + + def run_next( self ): + """Run the next job, waiting until one is available if necessary""" + cnt = 0 + while 1: + + uci_wrapper = self.queue.get() + uci_state = uci_wrapper.get_uci_state() + if uci_state is self.STOP_SIGNAL: + return + try: + if uci_state==uci_states.NEW: + self.create_uci( uci_wrapper ) + elif uci_state==uci_states.DELETING: + self.delete_uci( uci_wrapper ) + elif uci_state==uci_states.SUBMITTED: + self.start_uci( uci_wrapper ) + elif uci_state==uci_states.SHUTTING_DOWN: + self.stop_uci( uci_wrapper ) + elif uci_state==uci_states.SNAPSHOT: + self.snapshot_uci( uci_wrapper ) + except: + log.exception( "Uncaught exception executing cloud request." ) + cnt += 1 + + def get_connection( self, uci_wrapper ): + """ + Establishes EC2 cloud connection using user's credentials associated with given UCI + """ + log.debug( 'Establishing %s cloud connection.' % self.type ) + provider = uci_wrapper.get_provider() + try: + region = RegionInfo( None, provider.region_name, provider.region_endpoint ) + except Exception, ex: + err = "Selecting region with cloud provider failed: " + str( ex ) + log.error( err ) + uci_wrapper.set_error( err, True ) + return None + try: + conn = EC2Connection( aws_access_key_id=uci_wrapper.get_access_key(), + aws_secret_access_key=uci_wrapper.get_secret_key(), + is_secure=provider.is_secure, + region=region, + path=provider.path ) + except boto.exception.EC2ResponseError, e: + err = "Establishing connection with cloud failed: " + str( e ) + log.error( err ) + uci_wrapper.set_error( err, True ) + return None + + return conn + + def check_key_pair( self, uci_wrapper, conn ): + """ + Generate key pair using user's credentials + """ + kp = None + kp_name = uci_wrapper.get_name().replace(' ','_') + "_kp" + log.debug( "Checking user's key pair: '%s'" % kp_name ) + try: + kp = conn.get_key_pair( kp_name ) + uci_kp_name = uci_wrapper.get_key_pair_name() + uci_material = uci_wrapper.get_key_pair_material() + if kp != None: + if kp.name != uci_kp_name or uci_material == None: + # key pair exists on the cloud but not in local database, so re-generate it (i.e., delete and then create) + try: + conn.delete_key_pair( kp_name ) + kp = self.create_key_pair( conn, kp_name ) + uci_wrapper.set_key_pair( kp.name, kp.material ) + except boto.exception.EC2ResponseError, e: + err = "EC2 response error while deleting key pair: " + str( e ) + log.error( err ) + uci_wrapper.set_error( err, True ) + else: + try: + kp = self.create_key_pair( conn, kp_name ) + uci_wrapper.set_key_pair( kp.name, kp.material ) + except boto.exception.EC2ResponseError, e: + err = "EC2 response error while creating key pair: " + str( e ) + log.error( err ) + uci_wrapper.set_error( err, True ) + except Exception, ex: + err = "Exception while creating key pair: " + str( ex ) + log.error( err ) + uci_wrapper.set_error( err, True ) + except boto.exception.EC2ResponseError, e: # No keypair under this name exists so create it + if e.code == 'InvalidKeyPair.NotFound': + log.info( "No keypair found, creating keypair '%s'" % kp_name ) + kp = self.create_key_pair( conn, kp_name ) + uci_wrapper.set_key_pair( kp.name, kp.material ) + else: + err = "EC2 response error while retrieving key pair: " + str( e ) + log.error( err ) + uci_wrapper.set_error( err, True ) + + if kp != None: + return kp.name + else: + return None + + def create_key_pair( self, conn, kp_name ): + try: + return conn.create_key_pair( kp_name ) + except boto.exception.EC2ResponseError, e: + return None + + def get_mi_id( self, uci_wrapper, i_index ): + """ + Get appropriate machine image (mi) based on instance size. + """ + i_type = uci_wrapper.get_instance_type( i_index ) + if i_type=='m1.small' or i_type=='c1.medium': + arch = 'i386' + else: + arch = 'x86_64' + + mi = self.sa_session.query( model.CloudImage ).filter_by( deleted=False, provider_type=self.type, architecture=arch ).first() + if mi: + return mi.image_id + else: + err = "Machine image could not be retrieved" + log.error( "%s for UCI '%s'." % (err, uci_wrapper.get_name() ) ) + uci_wrapper.set_error( err+". Contact site administrator to ensure needed machine image is registered.", True ) + return None + + def create_uci( self, uci_wrapper ): + """ + Creates User Configured Instance (UCI). Essentially, creates storage volume on cloud provider + and registers relevant information in Galaxy database. + """ + conn = self.get_connection( uci_wrapper ) + if uci_wrapper.get_uci_availability_zone()=='': + log.info( "Availability zone for UCI (i.e., storage volume) was not selected, using default zone: %s" % self.zone ) + uci_wrapper.set_store_availability_zone( self.zone ) + + log.info( "Creating volume in zone '%s'..." % uci_wrapper.get_uci_availability_zone() ) + # Because only 1 storage volume may be created at UCI config time, index of this storage volume in local Galaxy DB w.r.t + # current UCI is 0, so reference it in following methods + vol = conn.create_volume( uci_wrapper.get_store_size( 0 ), uci_wrapper.get_uci_availability_zone(), snapshot=None ) + uci_wrapper.set_store_volume_id( 0, vol.id ) + + # Wait for a while to ensure volume was created +# vol_status = vol.status +# for i in range( 30 ): +# if vol_status is not "available": +# log.debug( 'Updating volume status; current status: %s' % vol_status ) +# vol_status = vol.status +# time.sleep(3) +# if i is 29: +# log.debug( "Error while creating volume '%s'; stuck in state '%s'; deleting volume." % ( vol.id, vol_status ) ) +# conn.delete_volume( vol.id ) +# uci_wrapper.change_state( uci_state='error' ) +# return + + # Retrieve created volume again to get updated status + try: + vl = conn.get_all_volumes( [vol.id] ) + except boto.exception.EC2ResponseError, e: + err = "EC2 response error while retrieving (i.e., updating status) of just created storage volume '" + vol.id + "': " + str( e ) + log.error( err ) + uci_wrapper.set_store_status( vol.id, uci_states.ERROR ) + uci_wrapper.set_error( err, True ) + return + except Exception, ex: + err = "Error while retrieving (i.e., updating status) of just created storage volume '" + vol.id + "': " + str( ex ) + log.error( err ) + uci_wrapper.set_error( err, True ) + return + + if len( vl ) > 0: + uci_wrapper.change_state( uci_state=vl[0].status ) + uci_wrapper.set_store_status( vol.id, vl[0].status ) + else: + err = "Volume '" + vol.id +"' not found by EC2 after being created." + log.error( err ) + uci_wrapper.set_store_status( vol.id, uci_states.ERROR ) + uci_wrapper.set_error( err, True ) + + def delete_uci( self, uci_wrapper ): + """ + Deletes UCI. NOTE that this implies deletion of any and all data associated + with this UCI from the cloud. All data will be deleted. + """ + conn = self.get_connection( uci_wrapper ) + vl = [] # volume list + count = 0 # counter for checking if all volumes assoc. w/ UCI were deleted + + # Get all volumes assoc. w/ UCI, delete them from cloud as well as in local DB + vl = uci_wrapper.get_all_stores() + deletedList = [] + failedList = [] + for v in vl: + log.debug( "Deleting volume with id='%s'" % v.volume_id ) + try: + if conn.delete_volume( v.volume_id ): + deletedList.append( v.volume_id ) + v.deleted = True + self.sa_session.add( v ) + self.sa_session.flush() + count += 1 + else: + failedList.append( v.volume_id ) + except boto.exception.EC2ResponseError, e: + err = "EC2 response error while deleting storage volume '" + v.volume_id + "': " + str( e ) + log.error( err ) + uci_wrapper.set_store_error( err, store_id = v.volume_id ) + uci_wrapper.set_error( err, True ) + + # Delete UCI if all of associated + if count == len( vl ): + uci_wrapper.set_deleted() + else: + err = "Deleting following volume(s) failed: " + str( failedList ) + ". However, these volumes were successfully deleted: " \ + + str( deletedList ) + ". MANUAL intervention and processing needed." + log.error( err ) + uci_wrapper.set_error( err, True ) + + def snapshot_uci( self, uci_wrapper ): + """ + Creates snapshot of all storage volumes associated with this UCI. + """ + if uci_wrapper.get_uci_state() != uci_states.ERROR: + conn = self.get_connection( uci_wrapper ) + + snapshots = uci_wrapper.get_snapshots( status = snapshot_status.SUBMITTED ) + for snapshot in snapshots: + log.debug( "Snapshot DB id: '%s', volume id: '%s'" % ( snapshot.id, snapshot.store.volume_id ) ) + try: + snap = conn.create_snapshot( volume_id=snapshot.store.volume_id ) + snap_id = str( snap ).split(':')[1] + uci_wrapper.set_snapshot_id( snapshot.id, snap_id ) + sh = conn.get_all_snapshots( snap_id ) # get updated status + uci_wrapper.set_snapshot_status( status=sh[0].status, snap_id=snap_id ) + except boto.exception.EC2ResponseError, e: + err = "EC2 response error while creating snapshot: " + str( e ) + log.error( err ) + uci_wrapper.set_snapshot_error( error=err, snap_index=snapshot.id, set_status=True ) + uci_wrapper.set_error( err, True ) + return + except Exception, ex: + err = "Error while creating snapshot: " + str( ex ) + log.error( err ) + uci_wrapper.set_snapshot_error( error=err, snap_index=snapshot.id, set_status=True ) + uci_wrapper.set_error( err, True ) + return + + uci_wrapper.change_state( uci_state=uci_states.AVAILABLE ) + + def add_storage_to_uci( self, name ): + """ Adds more storage to specified UCI + TODO""" + + def dummy_start_uci( self, uci_wrapper ): + + uci = uci_wrapper.get_uci() + log.debug( "Would be starting instance '%s'" % uci.name ) + uci_wrapper.change_state( uci_state.PENDING ) +# log.debug( "Sleeping a bit... (%s)" % uci.name ) +# time.sleep(20) +# log.debug( "Woke up! (%s)" % uci.name ) + + def start_uci( self, uci_wrapper ): + """ + Starts instance(s) of given UCI on the cloud. + """ + if uci_wrapper.get_uci_state() != uci_states.ERROR: + conn = self.get_connection( uci_wrapper ) + self.check_key_pair( uci_wrapper, conn ) + if uci_wrapper.get_key_pair_name() == None: + err = "Key pair not found" + log.error( "%s for UCI '%s'." % ( err, uci_wrapper.get_name() ) ) + uci_wrapper.set_error( err + ". Try resetting the state and starting the instance again.", True ) + return + + i_indexes = uci_wrapper.get_instances_indexes( state=instance_states.SUBMITTED ) # Get indexes of i_indexes associated with this UCI that are in 'submitted' state + log.debug( "Starting instances with IDs: '%s' associated with UCI '%s' " % ( i_indexes, uci_wrapper.get_name(), ) ) + if len( i_indexes ) > 0: + for i_index in i_indexes: + # Get machine image for current instance + mi_id = self.get_mi_id( uci_wrapper, i_index ) + log.debug( "mi_id: %s, uci_wrapper.get_key_pair_name(): %s" % ( mi_id, uci_wrapper.get_key_pair_name() ) ) + uci_wrapper.set_mi( i_index, mi_id ) + + if mi_id != None: + # Check if galaxy security group exists (and create it if it does not) + log.debug( "Setting up '%s' security group." % self.security_group ) + try: + conn.get_all_security_groups( [self.security_group] ) # security groups + except boto.exception.EC2ResponseError, e: + if e.code == 'InvalidGroup.NotFound': + log.info( "No security group found, creating security group '%s'" % self.security_group ) + try: + gSecurityGroup = conn.create_security_group(self.security_group, 'Security group for Galaxy.') + gSecurityGroup.authorize( 'tcp', 80, 80, '0.0.0.0/0' ) # Open HTTP port + gSecurityGroup.authorize( 'tcp', 22, 22, '0.0.0.0/0' ) # Open SSH port + except boto.exception.EC2ResponseError, ee: + err = "EC2 response error while creating security group: " + str( ee ) + log.error( err ) + uci_wrapper.set_error( err, True ) + else: + err = "EC2 response error while retrieving security group: " + str( e ) + log.error( err ) + uci_wrapper.set_error( err, True ) + + + if uci_wrapper.get_uci_state() != uci_states.ERROR: + # Start an instance + log.debug( "Starting instance for UCI '%s'" % uci_wrapper.get_name() ) + #TODO: Once multiple volumes can be attached to a single instance, update 'userdata' composition + userdata = uci_wrapper.get_store_volume_id()+"|"+uci_wrapper.get_access_key()+"|"+uci_wrapper.get_secret_key() + log.debug( "Using following command: conn.run_instances( image_id='%s', key_name='%s', security_groups=['%s'], user_data=[OMITTED], instance_type='%s', placement='%s' )" + % ( mi_id, uci_wrapper.get_key_pair_name(), self.security_group, uci_wrapper.get_instance_type( i_index ), uci_wrapper.get_uci_availability_zone() ) ) + reservation = None + try: + reservation = conn.run_instances( image_id=mi_id, + key_name=uci_wrapper.get_key_pair_name(), + security_groups=[self.security_group], + user_data=userdata, + instance_type=uci_wrapper.get_instance_type( i_index ), + placement=uci_wrapper.get_uci_availability_zone() ) + except boto.exception.EC2ResponseError, e: + err = "EC2 response error when starting UCI '"+ uci_wrapper.get_name() +"': " + str( e ) + log.error( err ) + uci_wrapper.set_error( err, True ) + except Exception, ex: + err = "Error when starting UCI '" + uci_wrapper.get_name() + "': " + str( ex ) + log.error( err ) + uci_wrapper.set_error( err, True ) + # Record newly available instance data into local Galaxy database + if reservation: + l_time = datetime.utcnow() + # uci_wrapper.set_instance_launch_time( self.format_time( reservation.instances[0].launch_time ), i_index=i_index ) + uci_wrapper.set_instance_launch_time( l_time, i_index=i_index ) + if not uci_wrapper.uci_launch_time_set(): + uci_wrapper.set_uci_launch_time( l_time ) + try: + uci_wrapper.set_reservation_id( i_index, str( reservation ).split(":")[1] ) + # TODO: if more than a single instance will be started through single reservation, change this reference to element [0] + i_id = str( reservation.instances[0]).split(":")[1] + uci_wrapper.set_instance_id( i_index, i_id ) + s = reservation.instances[0].state + uci_wrapper.change_state( s, i_id, s ) + uci_wrapper.set_security_group_name( self.security_group, i_id=i_id ) + log.debug( "Instance of UCI '%s' started, current state: '%s'" % ( uci_wrapper.get_name(), uci_wrapper.get_uci_state() ) ) + except boto.exception.EC2ResponseError, e: + err = "EC2 response error when retrieving instance information for UCI '" + uci_wrapper.get_name() + "': " + str( e ) + log.error( err ) + uci_wrapper.set_error( err, True ) + else: + log.error( "UCI '%s' is in 'error' state, starting instance was aborted." % uci_wrapper.get_name() ) + else: + err = "No instances in state '"+ instance_states.SUBMITTED +"' found for UCI '" + uci_wrapper.get_name() + \ + "'. Nothing to start." + log.error( err ) + uci_wrapper.set_error( err, True ) + else: + log.error( "UCI '%s' is in 'error' state, starting instance was aborted." % uci_wrapper.get_name() ) + + def stop_uci( self, uci_wrapper): + """ + Stops all of cloud instances associated with given UCI. + """ + conn = self.get_connection( uci_wrapper ) + + # Get all instances associated with given UCI + il = uci_wrapper.get_instances_ids() # instance list + # Process list of instances and remove any references to empty instance id's + for i in il: + if i is None: + il.remove( i ) + log.debug( 'List of instances being terminated: %s' % il ) + rl = conn.get_all_instances( il ) # Reservation list associated with given instances + + # Initiate shutdown of all instances under given UCI + cnt = 0 + stopped = [] + not_stopped = [] + for r in rl: + for inst in r.instances: + log.debug( "Sending stop signal to instance '%s' associated with reservation '%s'." % ( inst, r ) ) + try: + inst.stop() + uci_wrapper.set_stop_time( datetime.utcnow(), i_id=inst.id ) + uci_wrapper.change_state( instance_id=inst.id, i_state=inst.update() ) + stopped.append( inst ) + except boto.exception.EC2ResponseError, e: + not_stopped.append( inst ) + err = "EC2 response error when stopping instance '" + inst.instance_id + "': " + str(e) + log.error( err ) + uci_wrapper.set_error( err, True ) + + uci_wrapper.reset_uci_launch_time() + log.debug( "Termination was initiated for all instances of UCI '%s'." % uci_wrapper.get_name() ) + + +# dbInstances = get_instances( trans, uci ) #TODO: handle list! +# +# # Get actual cloud instance object +# cloudInstance = get_cloud_instance( conn, dbInstances.instance_id ) +# +# # TODO: Detach persistent storage volume(s) from instance and update volume data in local database +# stores = get_stores( trans, uci ) +# for i, store in enumerate( stores ): +# log.debug( "Detaching volume '%s' to instance '%s'." % ( store.volume_id, dbInstances.instance_id ) ) +# mntDevice = store.device +# volStat = None +## Detaching volume does not work with Eucalyptus Public Cloud, so comment it out +## try: +## volStat = conn.detach_volume( store.volume_id, dbInstances.instance_id, mntDevice ) +## except: +## log.debug ( 'Error detaching volume; still going to try and stop instance %s.' % dbInstances.instance_id ) +# store.attach_time = None +# store.device = None +# store.i_id = None +# store.status = volStat +# log.debug ( '***** volume status: %s' % volStat ) +# +# +# # Stop the instance and update status in local database +# cloudInstance.stop() +# dbInstances.stop_time = datetime.utcnow() +# while cloudInstance.state != 'terminated': +# log.debug( "Stopping instance %s state; current state: %s" % ( str( cloudInstance ).split(":")[1], cloudInstance.state ) ) +# time.sleep(3) +# cloudInstance.update() +# dbInstances.state = cloudInstance.state +# +# # Reset relevant UCI fields +# uci.state = 'available' +# uci.launch_time = None +# +# # Persist +# session = trans.sa_session +## session.save_or_update( stores ) +# session.save_or_update( dbInstances ) # TODO: Is this going to work w/ multiple instances stored in dbInstances variable? +# session.save_or_update( uci ) +# session.flush() +# trans.log_event( "User stopped cloud instance '%s'" % uci.name ) +# trans.set_message( "Galaxy instance '%s' stopped." % uci.name ) + + def update( self ): + """ + Runs a global status update on all instances that are in 'running', 'pending', or 'shutting-down' state. + Also, runs update on all storage volumes that are in 'in-use', 'creating', or 'None' state. + Reason behind this method is to sync state of local DB and real-world resources + """ + log.debug( "Running general status update for EC2 UCIs..." ) + # Update instances + instances = self.sa_session.query( model.CloudInstance ) \ + .filter( or_( model.CloudInstance.table.c.state==instance_states.RUNNING, + model.CloudInstance.table.c.state==instance_states.PENDING, + model.CloudInstance.table.c.state==instance_states.SHUTTING_DOWN ) ) \ + .all() + for inst in instances: + if self.type == inst.uci.credentials.provider.type: + log.debug( "[%s] Running general status update on instance '%s'" % ( inst.uci.credentials.provider.type, inst.instance_id ) ) + self.update_instance( inst ) + + # Update storage volume(s) + stores = self.sa_session.query( model.CloudStore ) \ + .filter( or_( model.CloudStore.table.c.status==store_status.IN_USE, + model.CloudStore.table.c.status==store_status.CREATING, + model.CloudStore.table.c.status==None ) ) \ + .all() + for store in stores: + if self.type == store.uci.credentials.provider.type: # and store.volume_id != None: + log.debug( "[%s] Running general status update on store with local database ID: '%s'" % ( store.uci.credentials.provider.type, store.id ) ) + self.update_store( store ) +# else: +# log.error( "[%s] There exists an entry for UCI (%s) storage volume without an ID. Storage volume might have been created with " +# "cloud provider though. Manual check is recommended." % ( store.uci.credentials.provider.type, store.uci.name ) ) +# store.uci.error = "There exists an entry in local database for a storage volume without an ID. Storage volume might have been created " \ +# "with cloud provider though. Manual check is recommended. After understanding what happened, local database entry for given " \ +# "storage volume should be updated." +# store.status = store_status.ERROR +# store.uci.state = uci_states.ERROR +# store.uci.flush() +# store.flush() + + # Update pending snapshots or delete ones marked for deletion + snapshots = self.sa_session.query( model.CloudSnapshot ) \ + .filter( or_( model.CloudSnapshot.table.c.status == snapshot_status.PENDING, model.CloudSnapshot.table.c.status == snapshot_status.DELETE ) ) \ + .all() + for snapshot in snapshots: + if self.type == snapshot.uci.credentials.provider.type and snapshot.status == snapshot_status.PENDING: + log.debug( "[%s] Running general status update on snapshot '%s'" % ( snapshot.uci.credentials.provider.type, snapshot.snapshot_id ) ) + self.update_snapshot( snapshot ) + elif self.type == snapshot.uci.credentials.provider.type and snapshot.status == snapshot_status.DELETE: + log.debug( "[%s] Initiating deletion of snapshot '%s'" % ( snapshot.uci.credentials.provider.type, snapshot.snapshot_id ) ) + self.delete_snapshot( snapshot ) + + # Attempt at updating any zombie UCIs (i.e., instances that have been in SUBMITTED state for longer than expected - see below for exact time) + zombies = self.sa_session.query( model.UCI ).filter_by( state=uci_states.SUBMITTED ).all() + for zombie in zombies: + z_instances = self.sa_session.query( model.CloudInstance ) \ + .filter_by( uci_id=zombie.id ) \ + .filter( or_( model.CloudInstance.table.c.state != instance_states.TERMINATED, + model.CloudInstance.table.c.state == None ) ) \ + .all() + for z_inst in z_instances: + if self.type == z_inst.uci.credentials.provider.type: +# log.debug( "z_inst.id: '%s', state: '%s'" % ( z_inst.id, z_inst.state ) ) + td = datetime.utcnow() - z_inst.update_time + if td.seconds > 180: # if instance has been in SUBMITTED state for more than 3 minutes + log.debug( "[%s] Running zombie repair update on instance with DB id '%s'" % ( z_inst.uci.credentials.provider.type, z_inst.id ) ) + self.process_zombie( z_inst ) + + def update_instance( self, inst ): + + # Get credentials associated wit this instance + uci_id = inst.uci_id + uci = self.sa_session.query( model.UCI ).get( uci_id ) + self.sa_session.refresh( uci ) + conn = self.get_connection_from_uci( uci ) + + # Get reservations handle for given instance + try: + rl= conn.get_all_instances( [inst.instance_id] ) + except boto.exception.EC2ResponseError, e: + err = "Retrieving instance(s) from cloud failed for UCI '"+ uci.name +"' during general status update: " + str( e ) + log.error( err ) + uci.error = err + uci.state = uci_states.ERROR + self.sa_session.add( uci ) + self.sa_session.flush() + return None + + # Because references to reservations are deleted shortly after instances have been terminated, getting an empty list as a response to a query + # typically means the instance has successfully shut down but the check was not performed in short enough amount of time. Until an alternative solution + # is found, below code sets state of given UCI to 'error' to indicate to the user something out of ordinary happened. + if len( rl ) == 0: + err = "Instance ID '"+inst.instance_id+"' was not found by the cloud provider. Instance might have crashed or otherwise been terminated."+ \ + "Manual check is recommended." + log.error( err ) + inst.error = err + uci.error = err + inst.state = instance_states.TERMINATED + uci.state = uci_states.ERROR + uci.launch_time = None + self.sa_session.add( inst ) + self.sa_session.add( uci ) + self.sa_session.flush() + # Update instance status in local DB with info from cloud provider + for r in rl: + for i, cInst in enumerate( r.instances ): + try: + s = cInst.update() + log.debug( "Checking state of cloud instance '%s' associated with UCI '%s' and reservation '%s'. State='%s'" % ( cInst, uci.name, r, s ) ) + if s != inst.state: + inst.state = s + self.sa_session.add( inst ) + self.sa_session.flush() + # After instance has shut down, ensure UCI is marked as 'available' + if s == instance_states.TERMINATED and uci.state != uci_states.ERROR: + uci.state = uci_states.AVAILABLE + uci.launch_time = None + self.sa_session.add( uci ) + self.sa_session.flush() + # Making sure state of UCI is updated. Once multiple instances become associated with single UCI, this will need to be changed. + if s != uci.state and s != instance_states.TERMINATED: + uci.state = s + self.sa_session.add( uci ) + self.sa_session.flush() + if cInst.public_dns_name != inst.public_dns: + inst.public_dns = cInst.public_dns_name + self.sa_session.add( inst ) + self.sa_session.flush() + if cInst.private_dns_name != inst.private_dns: + inst.private_dns = cInst.private_dns_name + self.sa_session.add( inst ) + self.sa_session.flush() + except boto.exception.EC2ResponseError, e: + err = "Updating instance status from cloud failed for UCI '"+ uci.name + "' during general status update: " + str( e ) + log.error( err ) + uci.error = err + uci.state = uci_states.ERROR + self.sa_session.add( uci ) + self.sa_session.flush() + return None + + def update_store( self, store ): + # Get credentials associated wit this store + uci_id = store.uci_id + uci = self.sa_session.query( model.UCI ).get( uci_id ) + self.sa_session.refresh( uci ) + conn = self.get_connection_from_uci( uci ) + + # Get reservations handle for given store + try: + log.debug( "Updating storage volume command: vl = conn.get_all_volumes( [%s] )" % store.volume_id ) + vl = conn.get_all_volumes( [store.volume_id] ) + except boto.exception.EC2ResponseError, e: + err = "Retrieving volume(s) from cloud failed for UCI '"+ uci.name + "' during general status update: " + str( e ) + log.error( err ) + uci.error = err + uci.state = uci_states.ERROR + self.sa_session.add( uci ) + self.sa_session.flush() + return None + + # Update store status in local DB with info from cloud provider + if len(vl) > 0: + try: + if store.status != vl[0].status: + # In case something failed during creation of UCI but actual storage volume was created and yet + # UCI state remained as 'new', try to remedy this by updating UCI state here + if ( store.status == None ) and ( store.volume_id != None ): + uci.state = vl[0].status + self.sa_session.add( uci ) + self.sa_session.flush() + # If UCI was marked in state 'CREATING', update its status to reflect new status + elif ( uci.state == uci_states.CREATING ): + uci.state = vl[0].status + self.sa_session.add( uci ) + self.sa_session.flush() + + store.status = vl[0].status + self.sa_session.add( store ) + self.sa_session.flush() + if store.i_id != vl[0].instance_id: + store.i_id = vl[0].instance_id + self.sa_session.add( store ) + self.sa_session.flush() + if store.attach_time != vl[0].attach_time: + store.attach_time = vl[0].attach_time + self.sa_session.add( store ) + self.sa_session.flush() + if store.device != vl[0].device: + store.device = vl[0].device + self.sa_session.add( store ) + self.sa_session.flush() + except boto.exception.EC2ResponseError, e: + err = "Updating status of volume(s) from cloud failed for UCI '"+ uci.name + "' during general status update: " + str( e ) + log.error( err ) + uci.error = err + uci.state = uci_states.ERROR + self.sa_session.add( uci ) + self.sa_session.flush() + return None + else: + err = "No storage volumes returned by cloud provider on general update" + log.error( "%s for UCI '%s'" % ( err, uci.name ) ) + store.status = store_status.ERROR + store.error = err + uci.error = err + uci.state = uci_states.ERROR + self.sa_session.add( uci ) + self.sa_session.add( store ) + self.sa_session.flush() + + def update_snapshot( self, snapshot ): + # Get credentials associated wit this store + uci_id = snapshot.uci_id + uci = self.sa_session.query( model.UCI ).get( uci_id ) + self.sa_session.refresh( uci ) + conn = self.get_connection_from_uci( uci ) + + try: + log.debug( "Updating status of snapshot '%s'" % snapshot.snapshot_id ) + snap = conn.get_all_snapshots( [snapshot.snapshot_id] ) + if len( snap ) > 0: + log.debug( "Snapshot '%s' status: %s" % ( snapshot.snapshot_id, snap[0].status ) ) + snapshot.status = snap[0].status + self.sa_session.add( snapshot ) + self.sa_session.flush() + else: + err = "No snapshots returned by EC2 on general update" + log.error( "%s for UCI '%s'" % ( err, uci.name ) ) + snapshot.status = snapshot_status.ERROR + snapshot.error = err + uci.error = err + uci.state = uci_states.ERROR + self.sa_session.add( uci ) + self.sa_session.add( snapshot ) + self.sa_session.flush() + except boto.exception.EC2ResponseError, e: + err = "EC2 response error while updating snapshot status: " + str( e ) + log.error( err ) + snapshot.status = snapshot_status.ERROR + snapshot.error = err + uci.error = err + uci.state = uci_states.ERROR + self.sa_session.add( uci ) + self.sa_session.add( snapshot ) + self.sa_session.flush() + except Exception, ex: + err = "Error while updating snapshot status: " + str( ex ) + log.error( err ) + snapshot.status = snapshot_status.ERROR + snapshot.error = err + uci.error = err + uci.state = uci_states.ERROR + self.sa_session.add( uci ) + self.sa_session.add( snapshot ) + self.sa_session.flush() + + def delete_snapshot( self, snapshot ): + if snapshot.status == snapshot_status.DELETE: + # Get credentials associated wit this store + uci_id = snapshot.uci_id + uci = self.sa_session.query( model.UCI ).get( uci_id ) + self.sa_session.refresh( uci ) + conn = self.get_connection_from_uci( uci ) + + try: + log.debug( "Deleting snapshot '%s'" % snapshot.snapshot_id ) + snap = conn.delete_snapshot( snapshot.snapshot_id ) + if snap == True: + snapshot.deleted = True + snapshot.status = snapshot_status.DELETED + self.sa_session.add( snapshot ) + self.sa_session.flush() + return snap + except boto.exception.EC2ResponseError, e: + err = "EC2 response error while deleting snapshot: " + str( e ) + log.error( err ) + snapshot.status = snapshot_status.ERROR + snapshot.error = err + uci.error = err + uci.state = uci_states.ERROR + self.sa_session.add( uci ) + self.sa_session.add( snapshot ) + self.sa_session.flush() + except Exception, ex: + err = "Error while deleting snapshot: " + str( ex ) + log.error( err ) + snapshot.status = snapshot_status.ERROR + snapshot.error = err + uci.error = err + uci.state = uci_states.ERROR + self.sa_session.add( uci ) + self.sa_session.add( snapshot ) + self.sa_session.flush() + else: + err = "Cannot delete snapshot '"+snapshot.snapshot_id+"' because its status is '"+snapshot.status+"'. Only snapshots with '" + \ + snapshot_status.COMPLETED+"' status can be deleted." + log.error( err ) + snapshot.error = err + self.sa_session.add( snapshot ) + self.sa_session.flush() + + def process_zombie( self, inst ): + """ + Attempt at discovering if starting an instance was successful but local database was not updated + accordingly or if something else failed and instance was never started. Currently, no automatic + repairs are being attempted; instead, appropriate error messages are set. + """ + uci_id = inst.uci_id + uci = self.sa_session.query( model.UCI ).get( uci_id ) + self.sa_session.refresh( uci ) + + # Check if any instance-specific information was written to local DB; if 'yes', set instance and UCI's error message + # suggesting manual check. + if inst.launch_time != None or inst.reservation_id != None or inst.instance_id != None: + # Try to recover state - this is best-case effort, so if something does not work immediately, not + # recovery steps are attempted. Recovery is based on hope that instance_id is available in local DB; if not, + # report as error. + # Fields attempting to be recovered are: reservation_id, instance status, and launch_time + if inst.instance_id != None: + conn = self.get_connection_from_uci( uci ) + rl = conn.get_all_instances( [inst.instance_id] ) # reservation list + # Update local DB with relevant data from instance + if inst.reservation_id == None: + try: + inst.reservation_id = str(rl[0]).split(":")[1] + except: # something failed, so skip + pass + + try: + state = rl[0].instances[0].update() + inst.state = state + uci.state = state + self.sa_session.add( inst ) + self.sa_session.add( uci ) + self.sa_session.flush() + except: # something failed, so skip + pass + + if inst.launch_time == None: + try: + launch_time = self.format_time( rl[0].instances[0].launch_time ) + inst.launch_time = launch_time + self.sa_session.add( inst ) + self.sa_session.flush() + if inst.uci.launch_time == None: + uci.launch_time = launch_time + self.sa_session.add( uci ) + self.sa_session.flush() + except: # something failed, so skip + pass + else: + err = "Starting a machine instance (DB id: '"+str(inst.id)+"') associated with this UCI '" + str(inst.uci.name) + \ + "' seems to have failed. Because it appears that cloud instance might have gotten started, manual check is recommended." + inst.error = err + inst.state = instance_states.ERROR + inst.uci.error = err + inst.uci.state = uci_states.ERROR + log.error( err ) + self.sa_session.add( inst ) + self.sa_session.add( uci ) + self.sa_session.flush() + + else: #Instance most likely never got processed, so set error message suggesting user to try starting instance again. + err = "Starting a machine instance (DB id: '"+str(inst.id)+"') associated with this UCI '" + str(inst.uci.name) + \ + "' seems to have failed. Because it appears that cloud instance never got started, it should be safe to reset state and try " \ + "starting the instance again." + inst.error = err + inst.state = instance_states.ERROR + uci.error = err + uci.state = uci_states.ERROR + log.error( err ) + self.sa_session.add( inst ) + self.sa_session.add( uci ) + self.sa_session.flush() +# uw = UCIwrapper( inst.uci ) +# log.debug( "Try automatically re-submitting UCI '%s'." % uw.get_name() ) + + def get_connection_from_uci( self, uci ): + """ + Establishes and returns connection to cloud provider. Information needed to do so is obtained + directly from uci database object. + """ + log.debug( 'Establishing %s cloud connection' % self.type ) + a_key = uci.credentials.access_key + s_key = uci.credentials.secret_key + # Get connection + try: + region = RegionInfo( None, uci.credentials.provider.region_name, uci.credentials.provider.region_endpoint ) + conn = EC2Connection( aws_access_key_id=a_key, + aws_secret_access_key=s_key, + is_secure=uci.credentials.provider.is_secure, + region=region, + path=uci.credentials.provider.path ) + except boto.exception.EC2ResponseError, e: + err = "Establishing connection with cloud failed: " + str( e ) + log.error( err ) + uci.error = err + uci.state = uci_states.ERROR + self.sa_session.add( uci ) + self.sa_session.flush() + return None + + return conn + +# def updateUCI( self, uci ): +# """ +# Runs a global status update on all storage volumes and all instances that are +# associated with specified UCI +# """ +# conn = self.get_connection( uci ) +# +# # Update status of storage volumes +# vl = model.CloudStore.filter( model.CloudInstance.table.c.uci_id == uci.id ).all() +# vols = [] +# for v in vl: +# vols.append( v.volume_id ) +# try: +# volumes = conn.get_all_volumes( vols ) +# for i, v in enumerate( volumes ): +# uci.store[i].i_id = v.instance_id +# uci.store[i].status = v.status +# uci.store[i].device = v.device +# uci.store[i].flush() +# except: +# log.debug( "Error updating status of volume(s) associated with UCI '%s'. Status was not updated." % uci.name ) +# pass +# +# # Update status of instances +# il = model.CloudInstance.filter_by( uci_id=uci.id ).filter( model.CloudInstance.table.c.state != 'terminated' ).all() +# instanceList = [] +# for i in il: +# instanceList.append( i.instance_id ) +# log.debug( 'instanceList: %s' % instanceList ) +# try: +# reservations = conn.get_all_instances( instanceList ) +# for i, r in enumerate( reservations ): +# uci.instance[i].state = r.instances[0].update() +# log.debug('updating instance %s; status: %s' % ( uci.instance[i].instance_id, uci.instance[i].state ) ) +# uci.state = uci.instance[i].state +# uci.instance[i].public_dns = r.instances[0].dns_name +# uci.instance[i].private_dns = r.instances[0].private_dns_name +# uci.instance[i].flush() +# uci.flush() +# except: +# log.debug( "Error updating status of instances associated with UCI '%s'. Instance status was not updated." % uci.name ) +# pass + + # --------- Helper methods ------------ + + def format_time( self, time ): + dict = {'T':' ', 'Z':''} + for i, j in dict.iteritems(): + time = time.replace(i, j) + return time + \ No newline at end of file diff -r 90723c58b1a6 -r 03eb232d1111 lib/galaxy/cloud/providers/eucalyptus.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/lib/galaxy/cloud/providers/eucalyptus.py Tue Nov 17 19:07:19 2009 -0500 @@ -0,0 +1,1019 @@ +import subprocess, threading, os, errno, time, datetime +from Queue import Queue, Empty +from datetime import datetime + +from galaxy import model # Database interaction class +from galaxy.model import mapping +from galaxy.datatypes.data import nice_size +from galaxy.util.bunch import Bunch +from galaxy.cloud import UCIwrapper +from Queue import Queue +from sqlalchemy import or_, and_ + +import galaxy.eggs +galaxy.eggs.require("boto") +from boto.ec2.connection import EC2Connection +from boto.ec2.regioninfo import RegionInfo +import boto.exception +import boto + +import logging +log = logging.getLogger( __name__ ) + +uci_states = Bunch( + NEW_UCI = "newUCI", + NEW = "new", + CREATING = "creating", + DELETING_UCI = "deletingUCI", + DELETING = "deleting", + SUBMITTED_UCI = "submittedUCI", + SUBMITTED = "submitted", + SHUTTING_DOWN_UCI = "shutting-downUCI", + SHUTTING_DOWN = "shutting-down", + AVAILABLE = "available", + RUNNING = "running", + PENDING = "pending", + ERROR = "error", + DELETED = "deleted", + SNAPSHOT_UCI = "snapshotUCI", + SNAPSHOT = "snapshot" +) + +instance_states = Bunch( + TERMINATED = "terminated", + SUBMITTED = "submitted", + RUNNING = "running", + PENDING = "pending", + SHUTTING_DOWN = "shutting-down", + ERROR = "error" +) + +store_status = Bunch( + IN_USE = "in-use", + CREATING = "creating", + DELETED = 'deleted', + ERROR = "error" +) + +snapshot_status = Bunch( + SUBMITTED = 'submitted', + PENDING = 'pending', + COMPLETED = 'completed', + DELETE = 'delete', + DELETED= 'deleted', + ERROR = "error" +) + +class EucalyptusCloudProvider( object ): + """ + Eucalyptus-based cloud provider implementation for managing instances. + """ + STOP_SIGNAL = object() + def __init__( self, app ): + self.type = "eucalyptus" # cloud provider type (e.g., ec2, eucalyptus, opennebula) + self.zone = "epc" + self.queue = Queue() + self.sa_session = app.model.context + + self.threads = [] + nworkers = 5 + log.info( "Starting eucalyptus cloud controller workers..." ) + for i in range( nworkers ): + worker = threading.Thread( target=self.run_next ) + worker.start() + self.threads.append( worker ) + log.debug( "%d eucalyptus cloud workers ready", nworkers ) + + def shutdown( self ): + """Attempts to gracefully shut down the monitor thread""" + log.info( "sending stop signal to worker threads in eucalyptus cloud manager" ) + for i in range( len( self.threads ) ): + self.queue.put( self.STOP_SIGNAL ) + log.info( "eucalyptus cloud manager stopped" ) + + def put( self, uci_wrapper ): + """ + Adds uci_wrapper object to the end of the request queue to be handled by + this cloud provider. + """ + state = uci_wrapper.get_uci_state() + uci_wrapper.change_state( state.split('U')[0] ) # remove 'UCI' from end of state description (i.e., mark as accepted and ready for processing) + self.queue.put( uci_wrapper ) + + def run_next( self ): + """Process next request, waiting until one is available if necessary.""" + cnt = 0 + while 1: + uci_wrapper = self.queue.get() + uci_state = uci_wrapper.get_uci_state() + if uci_state is self.STOP_SIGNAL: + return + try: + if uci_state==uci_states.NEW: + self.create_uci( uci_wrapper ) + elif uci_state==uci_states.DELETING: + self.delete_uci( uci_wrapper ) + elif uci_state==uci_states.SUBMITTED: + self.start_uci( uci_wrapper ) + #self.dummy_start_uci( uci_wrapper ) + elif uci_state==uci_states.SHUTTING_DOWN: + self.stop_uci( uci_wrapper ) + elif uci_state==uci_states.SNAPSHOT: + self.snapshot_uci( uci_wrapper ) + except: + log.exception( "Uncaught exception executing cloud request." ) + cnt += 1 + + def get_connection( self, uci_wrapper ): + """ + Establishes cloud connection using user's credentials associated with given UCI + """ + log.debug( 'Establishing %s cloud connection.' % self.type ) + provider = uci_wrapper.get_provider() + try: + region = RegionInfo( None, provider.region_name, provider.region_endpoint ) + except Exception, ex: + err = "Selecting region with cloud provider failed: " + str( ex ) + log.error( err ) + uci_wrapper.set_error( err, True ) + return None + try: + conn = EC2Connection( aws_access_key_id=uci_wrapper.get_access_key(), + aws_secret_access_key=uci_wrapper.get_secret_key(), + is_secure=provider.is_secure, + port=provider.port, + region=region, + path=provider.path ) + except boto.exception.EC2ResponseError, e: + err = "Establishing connection with cloud failed: " + str( e ) + log.error( err ) + uci_wrapper.set_error( err, True ) + return None + + return conn + + def check_key_pair( self, uci_wrapper, conn ): + """ + Check if a key pair associated with this UCI exists on cloud provider. + If yes, return key pair name; otherwise, generate a key pair with the cloud + provider and, again, return key pair name. + Key pair name for given UCI is generated from UCI's name and suffix '_kp' + """ + kp = None + kp_name = uci_wrapper.get_name().replace(' ','_') + "_kp" + log.debug( "Checking user's key pair: '%s'" % kp_name ) + try: + kp = conn.get_key_pair( kp_name ) + uci_kp_name = uci_wrapper.get_key_pair_name() + uci_material = uci_wrapper.get_key_pair_material() + if kp != None: + if kp.name != uci_kp_name or uci_material == None: + # key pair exists on the cloud but not in local database, so re-generate it (i.e., delete and then create) + try: + conn.delete_key_pair( kp_name ) + kp = self.create_key_pair( conn, kp_name ) + uci_wrapper.set_key_pair( kp.name, kp.material ) + except boto.exception.EC2ResponseError, e: + err = "EC2 response error while deleting key pair: " + str( e ) + log.error( err ) + uci_wrapper.set_error( err, True ) + else: + try: + kp = self.create_key_pair( conn, kp_name ) + uci_wrapper.set_key_pair( kp.name, kp.material ) + except boto.exception.EC2ResponseError, e: + err = "EC2 response error while creating key pair: " + str( e ) + log.error( err ) + uci_wrapper.set_error( err, True ) + except Exception, ex: + err = "Exception while creating key pair: " + str( ex ) + log.error( err ) + uci_wrapper.set_error( err, True ) + except boto.exception.EC2ResponseError, e: # No keypair under this name exists so create it + if e.code == 'InvalidKeyPair.NotFound': + log.info( "No keypair found, creating keypair '%s'" % kp_name ) + kp = self.create_key_pair( conn, kp_name ) + uci_wrapper.set_key_pair( kp.name, kp.material ) + else: + err = "EC2 response error while retrieving key pair: " + str( e ) + log.error( err ) + uci_wrapper.set_error( err, True ) + + if kp != None: + return kp.name + else: + return None + + def create_key_pair( self, conn, kp_name ): + """ Initiate creation of key pair under kp_name by current cloud provider. """ + try: + return conn.create_key_pair( kp_name ) + except boto.exception.EC2ResponseError, e: + return None + + def get_mi_id( self, uci_wrapper, i_index ): + """ + Get appropriate machine image (mi) ID based on instance type. + """ + i_type = uci_wrapper.get_instance_type( i_index ) + if i_type=='m1.small' or i_type=='c1.medium': + arch = 'i386' + else: + arch = 'x86_64' + + mi = self.sa_session.query( model.CloudImage ).filter_by( deleted=False, provider_type=self.type, architecture=arch ).first() + if mi: + return mi.image_id + else: + err = "Machine image could not be retrieved" + log.error( "%s for UCI '%s'." % (err, uci_wrapper.get_name() ) ) + uci_wrapper.set_error( err+". Contact site administrator to ensure needed machine image is registered.", True ) + return None + + def create_uci( self, uci_wrapper ): + """ + Create User Configured Instance (UCI) - i.e., create storage volume on cloud provider + and register relevant information in local Galaxy database. + """ + conn = self.get_connection( uci_wrapper ) + + # Because only 1 storage volume may be created at UCI config time, index of this storage volume in local Galaxy DB w.r.t + # current UCI is 0; therefore, it can be referenced in following code + log.info( "Creating volume in zone '%s'..." % uci_wrapper.get_uci_availability_zone() ) + if uci_wrapper.get_uci_availability_zone()=='': + log.info( "Availability zone for UCI (i.e., storage volume) was not selected, using default zone: %s" % self.zone ) + uci_wrapper.set_store_availability_zone( self.zone ) + + log.debug( "Creating volume; using command: conn.create_volume( %s, '%s', snapshot=None )" % ( uci_wrapper.get_store_size( 0 ), uci_wrapper.get_uci_availability_zone() )) + vol = conn.create_volume( uci_wrapper.get_store_size( 0 ), uci_wrapper.get_uci_availability_zone(), snapshot=None ) + uci_wrapper.set_store_volume_id( 0, vol.id ) + + # Retrieve created volume again to get updated status + try: + vl = conn.get_all_volumes( [vol.id] ) + except boto.exception.EC2ResponseError, e: + err = "EC2 response error while retrieving (i.e., updating status) of just created storage volume '" + vol.id + "': " + str( e ) + log.error( err ) + uci_wrapper.set_store_status( vol.id, uci_states.ERROR ) + uci_wrapper.set_error( err, True ) + return + except Exception, ex: + err = "Error while retrieving (i.e., updating status) of just created storage volume '" + vol.id + "': " + str( ex ) + log.error( err ) + uci_wrapper.set_error( err, True ) + return + + if len( vl ) > 0: + # EPC does not allow creation of storage volumes (it deletes one as soon as it is created, so manually set uci_state here) + if vl[0].status == store_status.DELETED: + uci_wrapper.change_state( uci_state=uci_states.AVAILABLE ) + else: + uci_wrapper.change_state( uci_state=vl[0].status ) + uci_wrapper.set_store_status( vol.id, vl[0].status ) + else: + err = "Volume '" + vol.id +"' not found by EC2 after being created." + log.error( err ) + uci_wrapper.set_store_status( vol.id, uci_states.ERROR ) + uci_wrapper.set_error( err, True ) + + def delete_uci( self, uci_wrapper ): + """ + Delete UCI - i.e., delete all storage volumes associated with this UCI. + NOTE that this implies deletion of any and all data associated + with this UCI from the cloud. All data will be deleted. + Information in local Galaxy database is marked as deleted but not actually removed + from the database. + """ + conn = self.get_connection( uci_wrapper ) + vl = [] # volume list + count = 0 # counter for checking if all volumes assoc. w/ UCI were deleted + + # Get all volumes assoc. w/ UCI, delete them from cloud as well as in local DB + vl = uci_wrapper.get_all_stores() + deletedList = [] + failedList = [] + for v in vl: + log.debug( "Deleting volume with id='%s'" % v.volume_id ) + try: + if conn.delete_volume( v.volume_id ): + deletedList.append( v.volume_id ) + v.deleted = True + self.sa_session.add( v ) + self.sa_session.flush() + count += 1 + else: + failedList.append( v.volume_id ) + except boto.exception.EC2ResponseError, e: + err = "EC2 response error while deleting storage volume '" + v.volume_id + "': " + str( e ) + log.error( err ) + uci_wrapper.set_store_error( err, store_id = v.volume_id ) + uci_wrapper.set_error( err, True ) + + # Delete UCI if all of associated + if count == len( vl ): + uci_wrapper.set_deleted() + else: + err = "Deleting following volume(s) failed: "+ str( failedList )+". However, these volumes were successfully deleted: " \ + + str( deletedList ) +". MANUAL intervention and processing needed." + log.error( err ) + uci_wrapper.set_error( err, True ) + + def snapshot_uci( self, uci_wrapper ): + """ + Initiate creation of a snapshot by cloud provider for all storage volumes + associated with this UCI. + """ + if uci_wrapper.get_uci_state() != uci_states.ERROR: + conn = self.get_connection( uci_wrapper ) + + snapshots = uci_wrapper.get_snapshots( status = snapshot_status.SUBMITTED ) + for snapshot in snapshots: + log.debug( "Snapshot DB id: '%s', volume id: '%s'" % ( snapshot.id, snapshot.store.volume_id ) ) + try: + snap = conn.create_snapshot( volume_id=snapshot.store.volume_id ) + snap_id = str( snap ).split(':')[1] + uci_wrapper.set_snapshot_id( snapshot.id, snap_id ) + sh = conn.get_all_snapshots( snap_id ) # get updated status + uci_wrapper.set_snapshot_status( status=sh[0].status, snap_id=snap_id ) + except boto.exception.EC2ResponseError, e: + err = "Cloud provider response error while creating snapshot: " + str( e ) + log.error( err ) + uci_wrapper.set_snapshot_error( error=err, snap_index=snapshot.id, set_status=True ) + uci_wrapper.set_error( err, True ) + return + except Exception, ex: + err = "Error while creating snapshot: " + str( ex ) + log.error( err ) + uci_wrapper.set_snapshot_error( error=err, snap_index=snapshot.id, set_status=True ) + uci_wrapper.set_error( err, True ) + return + + uci_wrapper.change_state( uci_state=uci_states.AVAILABLE ) + +# if uci_wrapper.get_uci_state() != uci_states.ERROR: +# +# snapshots = uci_wrapper.get_snapshots( status = 'submitted' ) +# for snapshot in snapshots: +# uci_wrapper.set_snapshot_id( snapshot.id, None, 'euca_error' ) +# +# log.debug( "Eucalyptus snapshot attempted by user for UCI '%s'" % uci_wrapper.get_name() ) +# uci_wrapper.set_error( "Eucalyptus does not support creation of snapshots at this moment. No snapshot or other changes were performed. \ +# Feel free to resent state of this instance and use it normally.", True ) + + + def add_storage_to_uci( self, uci_wrapper ): + """ Adds more storage to specified UCI """ + + def dummy_start_uci( self, uci_wrapper ): + + uci = uci_wrapper.get_uci() + log.debug( "Would be starting instance '%s'" % uci.name ) +# uci_wrapper.change_state( uci_states.SUBMITTED_UCI ) +# log.debug( "Set UCI state to SUBMITTED_UCI" ) + log.debug( "Sleeping a bit... (%s)" % uci.name ) + time.sleep(10) + log.debug( "Woke up! (%s)" % uci.name ) + + def start_uci( self, uci_wrapper ): + """ + Start instance(s) of given UCI on the cloud. + """ + if uci_wrapper.get_uci_state() != uci_states.ERROR: + conn = self.get_connection( uci_wrapper ) + self.check_key_pair( uci_wrapper, conn ) + if uci_wrapper.get_key_pair_name() == None: + err = "Key pair not found" + log.error( "%s for UCI '%s'." % ( err, uci_wrapper.get_name() ) ) + uci_wrapper.set_error( err + ". Try resetting the state and starting the instance again.", True ) + return + + i_indexes = uci_wrapper.get_instances_indexes( state=instance_states.SUBMITTED ) # Get indexes of i_indexes associated with this UCI that are in 'submitted' state + log.debug( "Starting instances with IDs: '%s' associated with UCI '%s' " % ( i_indexes, uci_wrapper.get_name(), ) ) + if len( i_indexes ) > 0: + for i_index in i_indexes: + # Get machine image for current instance + mi_id = self.get_mi_id( uci_wrapper, i_index ) + log.debug( "mi_id: %s, uci_wrapper.get_key_pair_name(): %s" % ( mi_id, uci_wrapper.get_key_pair_name() ) ) + uci_wrapper.set_mi( i_index, mi_id ) + + if uci_wrapper.get_uci_state() != uci_states.ERROR: + # Start an instance + log.debug( "Starting UCI instance '%s'" % uci_wrapper.get_name() ) + log.debug( "Using following command: conn.run_instances( image_id='%s', key_name='%s', instance_type='%s' )" + % ( mi_id, uci_wrapper.get_key_pair_name(), uci_wrapper.get_instance_type( i_index ) ) ) + reservation = None + try: + reservation = conn.run_instances( image_id=mi_id, + key_name=uci_wrapper.get_key_pair_name(), + instance_type=uci_wrapper.get_instance_type( i_index ) ) + except boto.exception.EC2ResponseError, e: + err = "EC2 response error when starting UCI '"+ uci_wrapper.get_name() +"': " + str( e ) + log.error( err ) + uci_wrapper.set_error( err, True ) + except Exception, ex: + err = "Error when starting UCI '" + uci_wrapper.get_name() + "': " + str( ex ) + log.error( err ) + uci_wrapper.set_error( err, True ) + # Record newly available instance data into local Galaxy database + if reservation: + l_time = datetime.utcnow() +# uci_wrapper.set_instance_launch_time( self.format_time( reservation.instances[0].launch_time ), i_index=i_index ) + uci_wrapper.set_instance_launch_time( l_time, i_index=i_index ) + if not uci_wrapper.uci_launch_time_set(): + uci_wrapper.set_uci_launch_time( l_time ) + try: + uci_wrapper.set_reservation_id( i_index, str( reservation ).split(":")[1] ) + # TODO: if more than a single instance will be started through single reservation, change this reference from element [0] + i_id = str( reservation.instances[0]).split(":")[1] + uci_wrapper.set_instance_id( i_index, i_id ) + s = reservation.instances[0].state + uci_wrapper.change_state( s, i_id, s ) + log.debug( "Instance of UCI '%s' started, current state: '%s'" % ( uci_wrapper.get_name(), uci_wrapper.get_uci_state() ) ) + except boto.exception.EC2ResponseError, e: + err = "EC2 response error when retrieving instance information for UCI '" + uci_wrapper.get_name() + "': " + str( e ) + log.error( err ) + uci_wrapper.set_error( err, True ) + else: + log.error( "UCI '%s' is in 'error' state, starting instance was aborted." % uci_wrapper.get_name() ) + else: + err = "No instances in state '"+ instance_states.SUBMITTED +"' found for UCI '" + uci_wrapper.get_name() + \ + "'. Nothing to start." + log.error( err ) + uci_wrapper.set_error( err, True ) + else: + log.error( "UCI '%s' is in 'error' state, starting instance was aborted." % uci_wrapper.get_name() ) + + def stop_uci( self, uci_wrapper): + """ + Stop all cloud instances associated with given UCI. + """ + conn = self.get_connection( uci_wrapper ) + + # Get all instances associated with given UCI + il = uci_wrapper.get_instances_ids() # instance list + # Process list of instances and remove any references to empty instance id's + for i in il: + if i is None: + il.remove( i ) + log.debug( 'List of instances being terminated: %s' % il ) + rl = conn.get_all_instances( il ) # Reservation list associated with given instances + + # Initiate shutdown of all instances under given UCI + cnt = 0 + stopped = [] + not_stopped = [] + for r in rl: + for inst in r.instances: + log.debug( "Sending stop signal to instance '%s' associated with reservation '%s' (UCI: %s)." % ( inst, r, uci_wrapper.get_name() ) ) + try: + inst.stop() + uci_wrapper.set_stop_time( datetime.utcnow(), i_id=inst.id ) + uci_wrapper.change_state( instance_id=inst.id, i_state=inst.update() ) + stopped.append( inst ) + except boto.exception.EC2ResponseError, e: + not_stopped.append( inst ) + err = "EC2 response error when stopping instance '" + inst.instance_id + "': " + str( e ) + log.error( err ) + uci_wrapper.set_error( err, True ) + + uci_wrapper.reset_uci_launch_time() + log.debug( "Termination was initiated for all instances of UCI '%s'." % uci_wrapper.get_name() ) + +# dbInstances = get_instances( trans, uci ) #TODO: handle list! +# +# # Get actual cloud instance object +# cloudInstance = get_cloud_instance( conn, dbInstances.instance_id ) +# +# # TODO: Detach persistent storage volume(s) from instance and update volume data in local database +# stores = get_stores( trans, uci ) +# for i, store in enumerate( stores ): +# log.debug( "Detaching volume '%s' to instance '%s'." % ( store.volume_id, dbInstances.instance_id ) ) +# mntDevice = store.device +# volStat = None +## Detaching volume does not work with Eucalyptus Public Cloud, so comment it out +## try: +## volStat = conn.detach_volume( store.volume_id, dbInstances.instance_id, mntDevice ) +## except: +## log.debug ( 'Error detaching volume; still going to try and stop instance %s.' % dbInstances.instance_id ) +# store.attach_time = None +# store.device = None +# store.i_id = None +# store.status = volStat +# log.debug ( '***** volume status: %s' % volStat ) +# +# # Stop the instance and update status in local database +# cloudInstance.stop() +# dbInstances.stop_time = datetime.utcnow() +# while cloudInstance.state != 'terminated': +# log.debug( "Stopping instance %s state; current state: %s" % ( str( cloudInstance ).split(":")[1], cloudInstance.state ) ) +# time.sleep(3) +# cloudInstance.update() +# dbInstances.state = cloudInstance.state +# +# # Reset relevant UCI fields +# uci.state = 'available' +# uci.launch_time = None +# +# # Persist +# session = trans.sa_session +## session.save_or_update( stores ) +# session.save_or_update( dbInstances ) # TODO: Is this going to work w/ multiple instances stored in dbInstances variable? +# session.save_or_update( uci ) +# session.flush() +# trans.log_event( "User stopped cloud instance '%s'" % uci.name ) +# trans.set_message( "Galaxy instance '%s' stopped." % uci.name ) + + def update( self ): + """ + Run status update on all instances that are in 'running', 'pending', or 'shutting-down' state. + Run status update on all storage volumes whose status is 'in-use', 'creating', or 'None'. + Run status update on all snapshots whose status is 'pending' or 'delete' + Run status update on any zombie UCIs, i.e., UCI's that is in 'submitted' state for an + extended period of time. + + Reason behind this method is to sync state of local DB and real-world resources + """ + log.debug( "Running general status update for EPC UCIs..." ) + # Update instances + instances = self.sa_session.query( model.CloudInstance ) \ + .filter( or_( model.CloudInstance.table.c.state==instance_states.RUNNING, + model.CloudInstance.table.c.state==instance_states.PENDING, + model.CloudInstance.table.c.state==instance_states.SHUTTING_DOWN ) ) \ + .all() + for inst in instances: + if self.type == inst.uci.credentials.provider.type: + log.debug( "[%s] Running general status update on instance '%s'" % ( inst.uci.credentials.provider.type, inst.instance_id ) ) + self.update_instance( inst ) + + # Update storage volume(s) + stores = self.sa_session.query( model.CloudStore ) \ + .filter( or_( model.CloudStore.table.c.status==store_status.IN_USE, + model.CloudStore.table.c.status==store_status.CREATING, + model.CloudStore.table.c.status==None ) ) \ + .all() + for store in stores: + if self.type == store.uci.credentials.provider.type: # and store.volume_id != None: + log.debug( "[%s] Running general status update on store with local database ID: '%s'" % ( store.uci.credentials.provider.type, store.id ) ) + self.update_store( store ) + + # Update pending snapshots or delete ones marked for deletion + snapshots = self.sa_session.query( model.CloudSnapshot ) \ + .filter( or_( model.CloudSnapshot.table.c.status == snapshot_status.PENDING, model.CloudSnapshot.table.c.status == snapshot_status.DELETE ) ) \ + .all() + for snapshot in snapshots: + if self.type == snapshot.uci.credentials.provider.type and snapshot.status == snapshot_status.PENDING: + log.debug( "[%s] Running general status update on snapshot '%s'" % ( snapshot.uci.credentials.provider.type, snapshot.snapshot_id ) ) + self.update_snapshot( snapshot ) + elif self.type == snapshot.uci.credentials.provider.type and snapshot.status == snapshot_status.DELETE: + log.debug( "[%s] Initiating deletion of snapshot '%s'" % ( snapshot.uci.credentials.provider.type, snapshot.snapshot_id ) ) + self.delete_snapshot( snapshot ) + + # Attempt at updating any zombie UCIs (i.e., instances that have been in SUBMITTED state for longer than expected - see below for exact time) + zombies = self.sa_session.query( model.UCI ).filter_by( state=uci_states.SUBMITTED ).all() + for zombie in zombies: + log.debug( "zombie UCI: %s" % zombie.name ) + z_instances = self.sa_session.query( model.CloudInstance ) \ + .filter( or_( model.CloudInstance.table.c.state != instance_states.TERMINATED, + model.CloudInstance.table.c.state == None ) ) \ + .all() + for z_inst in z_instances: + if self.type == z_inst.uci.credentials.provider.type: +# log.debug( "z_inst.id: '%s', state: '%s'" % ( z_inst.id, z_inst.state ) ) + td = datetime.utcnow() - z_inst.update_time +# log.debug( "z_inst.id: %s, time delta is %s sec" % ( z_inst.id, td.seconds ) ) + if td.seconds > 180: # if instance has been in SUBMITTED state for more than 3 minutes + log.debug( "[%s](td=%s) Running zombie repair update on instance with DB id '%s'" % ( z_inst.uci.credentials.provider.type, td.seconds, z_inst.id ) ) + self.process_zombie( z_inst ) + + def update_instance( self, inst ): + """ + Update information in local database for given instance as it is obtained from cloud provider. + Along with updating information about given instance, information about the UCI controlling + this instance is also updated. + """ + # Get credentials associated wit this instance + uci_id = inst.uci_id + uci = self.sa_session.query( model.UCI ).get( uci_id ) + self.sa_session.refresh( uci ) + conn = self.get_connection_from_uci( uci ) + + # Get reservations handle for given instance + try: + rl= conn.get_all_instances( [inst.instance_id] ) + except boto.exception.EC2ResponseError, e: + err = "Retrieving instance(s) from cloud failed for UCI '"+ uci.name +"' during general status update: " + str( e ) + log.error( err ) + uci.error = err + uci.state = uci_states.ERROR + self.sa_session.add( uci ) + self.sa_session.flush() + return None + + # Because references to reservations are deleted shortly after instances have been terminated, getting an empty list as a response to a query + # typically means the instance has successfully shut down but the check was not performed in short enough amount of time. Until an alternative solution + # is found, below code sets state of given UCI to 'error' to indicate to the user something out of ordinary happened. + if len( rl ) == 0: + err = "Instance ID '"+inst.instance_id+"' was not found by the cloud provider. Instance might have crashed or otherwise been terminated."+ \ + "Manual check is recommended." + log.error( err ) + inst.error = err + uci.error = err + inst.state = instance_states.TERMINATED + uci.state = uci_states.ERROR + uci.launch_time = None + self.sa_session.add( inst ) + self.sa_session.add( uci ) + self.sa_session.flush() + # Update instance status in local DB with info from cloud provider + for r in rl: + for i, cInst in enumerate( r.instances ): + try: + s = cInst.update() + log.debug( "Checking state of cloud instance '%s' associated with UCI '%s' and reservation '%s'. State='%s'" % ( cInst, uci.name, r, s ) ) + if s != inst.state: + inst.state = s + self.sa_session.add( inst ) + self.sa_session.flush() + # After instance has shut down, ensure UCI is marked as 'available' + if s == instance_states.TERMINATED and uci.state != uci_states.ERROR: + uci.state = uci_states.AVAILABLE + uci.launch_time = None + self.sa_session.add( uci ) + self.sa_session.flush() + # Making sure state of UCI is updated. Once multiple instances become associated with single UCI, this will need to be changed. + if s != uci.state and s != instance_states.TERMINATED: + uci.state = s + self.sa_session.add( uci ) + self.sa_session.flush() + if cInst.public_dns_name != inst.public_dns: + inst.public_dns = cInst.public_dns_name + self.sa_session.add( inst ) + self.sa_session.flush() + if cInst.private_dns_name != inst.private_dns: + inst.private_dns = cInst.private_dns_name + self.sa_session.add( inst ) + self.sa_session.flush() + except boto.exception.EC2ResponseError, e: + err = "Updating instance status from cloud failed for UCI '"+ uci.name + "' during general status update: " + str( e ) + log.error( err ) + uci.error = err + uci.state = uci_states.ERROR + self.sa_session.add( uci ) + self.sa_session.flush() + return None + + def update_store( self, store ): + """ + Update information in local database for given storage volume as it is obtained from cloud provider. + Along with updating information about given storage volume, information about the UCI controlling + this storage volume is also updated. + """ + # Get credentials associated wit this store + uci_id = store.uci_id + uci = self.sa_session.query( model.UCI ).get( uci_id ) + self.sa_session.refresh( uci ) + conn = self.get_connection_from_uci( uci ) + + if store.volume_id != None: + # Get reservations handle for given store + try: + log.debug( "Updating storage volume command: vl = conn.get_all_volumes( [%s] )" % store.volume_id ) + vl = conn.get_all_volumes( [store.volume_id] ) + except boto.exception.EC2ResponseError, e: + err = "Retrieving volume(s) from cloud failed for UCI '"+ uci.name + "' during general status update: " + str( e ) + log.error( err ) + uci.error = err + uci.state = uci_states.ERROR + self.sa_session.add( uci ) + self.sa_session.flush() + return None + + # Update store status in local DB with info from cloud provider + if len(vl) > 0: + try: + if store.status != vl[0].status: + # In case something failed during creation of UCI but actual storage volume was created and yet + # UCI state remained as 'new', try to remedy this by updating UCI state here + if ( store.status == None ) and ( store.volume_id != None ): + uci.state = vl[0].status + self.sa_session.add( uci ) + self.sa_session.flush() + # If UCI was marked in state 'CREATING', update its status to reflect new status + elif ( uci.state == uci_states.CREATING ): + # Because Eucalyptus Public Cloud (EPC) deletes volumes immediately after they are created, artificially + # set status of given UCI to 'available' based on storage volume's availability zone (i.e., it's residing + # in EPC as opposed to some other Eucalyptus based cloud that allows creation of storage volumes. + if store.availability_zone == 'epc': + uci.state = uci_states.AVAILABLE + else: + uci.state = vl[0].status + + self.sa_session.add( uci ) + self.sa_session.flush() + + store.status = vl[0].status + self.sa_session.add( store ) + self.sa_session.flush() + if store.i_id != vl[0].instance_id: + store.i_id = vl[0].instance_id + self.sa_session.add( store ) + self.sa_session.flush() + if store.attach_time != vl[0].attach_time: + store.attach_time = vl[0].attach_time + self.sa_session.add( store ) + self.sa_session.flush() + if store.device != vl[0].device: + store.device = vl[0].device + self.sa_session.add( store ) + self.sa_session.flush() + except boto.exception.EC2ResponseError, e: + err = "Updating status of volume(s) from cloud failed for UCI '"+ uci.name + "' during general status update: " + str( e ) + log.error( err ) + uci.error = err + uci.state = uci_states.ERROR + self.sa_session.add( uci ) + self.sa_session.flush() + return None + else: + err = "No storage volumes returned by cloud provider on general update" + log.error( "%s for UCI '%s'" % ( err, uci.name ) ) + store.status = store_status.ERROR + store.error = err + uci.error = err + uci.state = uci_states.ERROR + self.sa_session.add( uci ) + self.sa_session.add( store ) + self.sa_session.flush() + else: + err = "Missing storage volume ID in local database on general update. Manual check is needed to check " \ + "if storage volume was actually created by cloud provider." + log.error( "%s (for UCI '%s')" % ( err, uci.name ) ) + store.status = store_status.ERROR + store.error = err + uci.error = err + uci.state = uci_states.ERROR + self.sa_session.add( uci ) + self.sa_session.add( store ) + self.sa_session.flush() + + def update_snapshot( self, snapshot ): + """ + Update information in local database for given snapshot as it is obtained from cloud provider. + Along with updating information about given snapshot, information about the UCI controlling + this snapshot is also updated. + """ + # Get credentials associated wit this store + uci_id = snapshot.uci_id + uci = self.sa_session.query( model.UCI ).get( uci_id ) + self.sa_session.refresh( uci ) + conn = self.get_connection_from_uci( uci ) + + try: + log.debug( "Updating status of snapshot '%s'" % snapshot.snapshot_id ) + snap = conn.get_all_snapshots( [snapshot.snapshot_id] ) + if len( snap ) > 0: + log.debug( "Snapshot '%s' status: %s" % ( snapshot.snapshot_id, snap[0].status ) ) + snapshot.status = snap[0].status + self.sa_session.add( snapshot ) + self.sa_session.flush() + else: + err = "No snapshots returned by EC2 on general update" + log.error( "%s for UCI '%s'" % ( err, uci.name ) ) + snapshot.status = snapshot_status.ERROR + snapshot.error = err + uci.error = err + uci.state = uci_states.ERROR + self.sa_session.add( uci ) + self.sa_session.add( snapshot ) + self.sa_session.flush() + except boto.exception.EC2ResponseError, e: + err = "EC2 response error while updating snapshot status: " + str( e ) + log.error( err ) + snapshot.status = snapshot_status.ERROR + snapshot.error = err + uci.error = err + uci.state = uci_states.ERROR + self.sa_session.add( uci ) + self.sa_session.add( snapshot ) + self.sa_session.flush() + except Exception, ex: + err = "Error while updating snapshot status: " + str( ex ) + log.error( err ) + snapshot.status = snapshot_status.ERROR + snapshot.error = err + uci.error = err + uci.state = uci_states.ERROR + self.sa_session.add( uci ) + self.sa_session.add( snapshot ) + self.sa_session.flush() + + def delete_snapshot( self, snapshot ): + """ + Initiate deletion of given snapshot from cloud provider. + """ + if snapshot.status == snapshot_status.DELETE: + # Get credentials associated wit this store + uci_id = snapshot.uci_id + uci = self.sa_session.query( model.UCI ).get( uci_id ) + self.sa_session.refresh( uci ) + conn = self.get_connection_from_uci( uci ) + + try: + log.debug( "Deleting snapshot '%s'" % snapshot.snapshot_id ) + snap = conn.delete_snapshot( snapshot.snapshot_id ) + if snap == True: + snapshot.deleted = True + snapshot.status = snapshot_status.DELETED + self.sa_session.add( snapshot ) + self.sa_session.flush() + return snap + except boto.exception.EC2ResponseError, e: + err = "EC2 response error while deleting snapshot: " + str( e ) + log.error( err ) + snapshot.status = snapshot_status.ERROR + snapshot.error = err + uci.error = err + uci.state = uci_states.ERROR + self.sa_session.add( uci ) + self.sa_session.add( snapshot ) + self.sa_session.flush() + except Exception, ex: + err = "Error while deleting snapshot: " + str( ex ) + log.error( err ) + snapshot.status = snapshot_status.ERROR + snapshot.error = err + uci.error = err + uci.state = uci_states.ERROR + self.sa_session.add( uci ) + self.sa_session.add( snapshot ) + self.sa_session.flush() + else: + err = "Cannot delete snapshot '"+snapshot.snapshot_id+"' because its status is '"+snapshot.status+"'. Only snapshots with '" + \ + snapshot_status.COMPLETED+"' status can be deleted." + log.error( err ) + snapshot.error = err + self.sa_session.add( snapshot ) + self.sa_session.flush() + + def process_zombie( self, inst ): + """ + Attempt at discovering if starting a cloud instance was successful but local database was not updated + accordingly or if something else failed and instance was never started. Currently, no automatic + repairs are being attempted; instead, appropriate error messages are set. + """ + uci_id = inst.uci_id + uci = self.sa_session.query( model.UCI ).get( uci_id ) + self.sa_session.refresh( uci ) + + # Check if any instance-specific information was written to local DB; if 'yes', set instance and UCI's error message + # suggesting manual check. + if inst.launch_time != None or inst.reservation_id != None or inst.instance_id != None: + # Try to recover state - this is best-case effort, so if something does not work immediately, not + # recovery steps are attempted. Recovery is based on hope that instance_id is available in local DB; if not, + # report as error. + # Fields attempting to be recovered are: reservation_id, instance status, and launch_time + if inst.instance_id != None: + conn = self.get_connection_from_uci( uci ) + rl = conn.get_all_instances( [inst.instance_id] ) # reservation list + # Update local DB with relevant data from instance + if inst.reservation_id == None: + try: + inst.reservation_id = str(rl[0]).split(":")[1] + except: # something failed, so skip + pass + + try: + state = rl[0].instances[0].update() + inst.state = state + uci.state = state + self.sa_session.add( inst ) + self.sa_session.add( uci ) + self.sa_session.flush() + except: # something failed, so skip + pass + + if inst.launch_time == None: + try: + launch_time = self.format_time( rl[0].instances[0].launch_time ) + inst.launch_time = launch_time + self.sa_session.add( inst ) + self.sa_session.flush() + if inst.uci.launch_time == None: + uci.launch_time = launch_time + self.sa_session.add( uci ) + self.sa_session.flush() + except: # something failed, so skip + pass + else: + err = "Starting a machine instance (DB id: '"+str(inst.id)+"') associated with this UCI '" + str(inst.uci.name) + \ + "' seems to have failed. Because it appears that cloud instance might have gotten started, manual check is recommended." + inst.error = err + inst.state = instance_states.ERROR + inst.uci.error = err + inst.uci.state = uci_states.ERROR + log.error( err ) + self.sa_session.add( inst ) + self.sa_session.add( uci ) + self.sa_session.flush() + + else: #Instance most likely never got processed, so set error message suggesting user to try starting instance again. + err = "Starting a machine instance (DB id: '"+str(inst.id)+"') associated with this UCI '" + str(inst.uci.name) + \ + "' seems to have failed. Because it appears that cloud instance never got started, it should be safe to reset state and try " \ + "starting the instance again." + inst.error = err + inst.state = instance_states.ERROR + uci.error = err + uci.state = uci_states.ERROR + log.error( err ) + self.sa_session.add( inst ) + self.sa_session.add( uci ) + self.sa_session.flush() +# uw = UCIwrapper( inst.uci ) +# log.debug( "Try automatically re-submitting UCI '%s'." % uw.get_name() ) + + def get_connection_from_uci( self, uci ): + """ + Establishes and returns connection to cloud provider. Information needed to do so is obtained + directly from uci database object. + """ + log.debug( 'Establishing %s cloud connection' % self.type ) + a_key = uci.credentials.access_key + s_key = uci.credentials.secret_key + # Get connection + try: + region = RegionInfo( None, uci.credentials.provider.region_name, uci.credentials.provider.region_endpoint ) + log.debug( "[%s] Using following command to connect to cloud provider: " + "conn = EC2Connection( aws_access_key_id=%s, " + "aws_secret_access_key=%s, " + "port=%s, " + "is_secure=%s, " + "region=region, " + "path=%s )" % ( self.type, a_key, s_key, uci.credentials.provider.is_secure, uci.credentials.provider.port, uci.credentials.provider.path ) ) + conn = EC2Connection( aws_access_key_id=a_key, + aws_secret_access_key=s_key, + is_secure=uci.credentials.provider.is_secure, + port=uci.credentials.provider.port, + region=region, + path=uci.credentials.provider.path ) + except boto.exception.EC2ResponseError, e: + err = "Establishing connection with cloud failed: " + str( e ) + log.error( err ) + uci.error = err + uci.state = uci_states.ERROR + self.sa_session.add( uci ) + self.sa_session.flush() + return None + + return conn + +# def updateUCI( self, uci ): +# """ +# Runs a global status update on all storage volumes and all instances that are +# associated with specified UCI +# """ +# conn = self.get_connection( uci ) +# +# # Update status of storage volumes +# vl = model.CloudStore.filter( model.CloudInstance.table.c.uci_id == uci.id ).all() +# vols = [] +# for v in vl: +# vols.append( v.volume_id ) +# try: +# volumes = conn.get_all_volumes( vols ) +# for i, v in enumerate( volumes ): +# uci.store[i].i_id = v.instance_id +# uci.store[i].status = v.status +# uci.store[i].device = v.device +# uci.store[i].flush() +# except: +# log.debug( "Error updating status of volume(s) associated with UCI '%s'. Status was not updated." % uci.name ) +# pass +# +# # Update status of instances +# il = model.CloudInstance.filter_by( uci_id=uci.id ).filter( model.CloudInstance.table.c.state != 'terminated' ).all() +# instanceList = [] +# for i in il: +# instanceList.append( i.instance_id ) +# log.debug( 'instanceList: %s' % instanceList ) +# try: +# reservations = conn.get_all_instances( instanceList ) +# for i, r in enumerate( reservations ): +# uci.instance[i].state = r.instances[0].update() +# log.debug('updating instance %s; status: %s' % ( uci.instance[i].instance_id, uci.instance[i].state ) ) +# uci.state = uci.instance[i].state +# uci.instance[i].public_dns = r.instances[0].dns_name +# uci.instance[i].private_dns = r.instances[0].private_dns_name +# uci.instance[i].flush() +# uci.flush() +# except: +# log.debug( "Error updating status of instances associated with UCI '%s'. Instance status was not updated." % uci.name ) +# pass + + # --------- Helper methods ------------ + + def format_time( self, time ): + dict = {'T':' ', 'Z':''} + for i, j in dict.iteritems(): + time = time.replace(i, j) + return time + \ No newline at end of file diff -r 90723c58b1a6 -r 03eb232d1111 lib/galaxy/config.py --- a/lib/galaxy/config.py Tue Nov 17 19:06:42 2009 -0500 +++ b/lib/galaxy/config.py Tue Nov 17 19:07:19 2009 -0500 @@ -113,6 +113,12 @@ except ConfigParser.NoSectionError: self.tool_runners = [] self.datatypes_config = kwargs.get( 'datatypes_config_file', 'datatypes_conf.xml' ) + # Cloud configuration options + self.cloud_controller_instance = string_as_bool( kwargs.get( 'cloud_controller_instance', 'False' ) ) + if self.cloud_controller_instance == True: + self.enable_cloud_execution = string_as_bool( kwargs.get( 'enable_cloud_execution', 'True' ) ) + else: + self.enable_cloud_execution = string_as_bool( kwargs.get( 'enable_cloud_execution', 'False' ) ) def get( self, key, default ): return self.config_dict.get( key, default ) def get_bool( self, key, default ): diff -r 90723c58b1a6 -r 03eb232d1111 lib/galaxy/model/__init__.py --- a/lib/galaxy/model/__init__.py Tue Nov 17 19:06:42 2009 -0500 +++ b/lib/galaxy/model/__init__.py Tue Nov 17 19:07:19 2009 -0500 @@ -38,6 +38,7 @@ self.username = None # Relationships self.histories = [] + self.credentials = [] def set_password_cleartext( self, cleartext ): """Set 'self.password' to the digest of 'cleartext'.""" @@ -1049,7 +1050,60 @@ def __init__( self, galaxy_session, history ): self.galaxy_session = galaxy_session self.history = history + +class CloudImage( object ): + def __init__( self ): + self.id = None + self.instance_id = None + self.state = None +class UCI( object ): + def __init__( self ): + self.id = None + self.user = None + +class CloudInstance( object ): + def __init__( self ): + self.id = None + self.user = None + self.name = None + self.instance_id = None + self.mi = None + self.state = None + self.public_dns = None + self.availability_zone = None + +class CloudStore( object ): + def __init__( self ): + self.id = None + self.volume_id = None + self.i_id = None + self.user = None + self.size = None + self.availability_zone = None + +class CloudSnapshot( object ): + def __init__( self ): + self.id = None + self.user = None + self.store_id = None + self.snapshot_id = None + +class CloudProvider( object ): + def __init__( self ): + self.id = None + self.user = None + self.type = None + +class CloudUserCredentials( object ): + def __init__( self ): + self.id = None + self.user = None + self.name = None + self.accessKey = None + self.secretKey = None + self.credentials = [] + class StoredWorkflow( object ): def __init__( self ): self.id = None diff -r 90723c58b1a6 -r 03eb232d1111 lib/galaxy/model/mapping.py --- a/lib/galaxy/model/mapping.py Tue Nov 17 19:06:42 2009 -0500 +++ b/lib/galaxy/model/mapping.py Tue Nov 17 19:07:19 2009 -0500 @@ -390,6 +390,117 @@ Column( "session_id", Integer, ForeignKey( "galaxy_session.id" ), index=True ), Column( "history_id", Integer, ForeignKey( "history.id" ), index=True ) ) +# *************************** Start cloud tables*********************************** +CloudImage.table = Table( "cloud_image", metadata, + Column( "id", Integer, primary_key=True ), + Column( "create_time", DateTime, default=now ), + Column( "update_time", DateTime, default=now, onupdate=now ), + Column( "provider_type", TEXT ), + Column( "image_id", TEXT, nullable=False ), + Column( "manifest", TEXT ), + Column( "state", TEXT ), + Column( "architecture", TEXT ), + Column( "deleted", Boolean, default=False ) ) + +""" UserConfiguredInstance (UCI) table """ +UCI.table = Table( "cloud_uci", metadata, + Column( "id", Integer, primary_key=True ), + Column( "create_time", DateTime, default=now ), + Column( "update_time", DateTime, default=now, onupdate=now ), + Column( "user_id", Integer, ForeignKey( "galaxy_user.id" ), index=True, nullable=False ), + Column( "credentials_id", Integer, ForeignKey( "cloud_user_credentials.id" ), index=True ), + Column( "key_pair_name", TEXT ), + Column( "key_pair_material", TEXT ), + Column( "name", TEXT ), + Column( "state", TEXT ), + Column( "error", TEXT ), + Column( "total_size", Integer ), + Column( "launch_time", DateTime ), + Column( "deleted", Boolean, default=False ) ) + +CloudInstance.table = Table( "cloud_instance", metadata, + Column( "id", Integer, primary_key=True ), + Column( "create_time", DateTime, default=now ), + Column( "update_time", DateTime, default=now, onupdate=now ), + Column( "launch_time", DateTime ), + Column( "stop_time", DateTime ), + Column( "user_id", Integer, ForeignKey( "galaxy_user.id" ), index=True, nullable=False ), + Column( "uci_id", Integer, ForeignKey( "cloud_uci.id" ), index=True ), + Column( "type", TEXT ), + Column( "reservation_id", TEXT ), + Column( "instance_id", TEXT ), + Column( "mi_id", TEXT, ForeignKey( "cloud_image.image_id" ), index=True ), + Column( "state", TEXT ), + Column( "error", TEXT ), + Column( "public_dns", TEXT ), + Column( "private_dns", TEXT ), + Column( "security_group", TEXT ), + Column( "availability_zone", TEXT ) ) + +CloudStore.table = Table( "cloud_store", metadata, + Column( "id", Integer, primary_key=True ), + Column( "create_time", DateTime, default=now ), + Column( "update_time", DateTime, default=now, onupdate=now ), + Column( "attach_time", DateTime ), + Column( "user_id", Integer, ForeignKey( "galaxy_user.id" ), index=True, nullable=False ), + Column( "uci_id", Integer, ForeignKey( "cloud_uci.id" ), index=True, nullable=False ), + Column( "volume_id", TEXT ), + Column( "size", Integer, nullable=False ), + Column( "availability_zone", TEXT ), + Column( "i_id", TEXT, ForeignKey( "cloud_instance.instance_id" ) ), + Column( "status", TEXT ), + Column( "device", TEXT ), + Column( "space_consumed", Integer ), + Column( "error", TEXT ), + Column( "deleted", Boolean, default=False ) ) + +CloudSnapshot.table = Table( "cloud_snapshot", metadata, + Column( "id", Integer, primary_key=True ), + Column( "create_time", DateTime, default=now ), + Column( "update_time", DateTime, default=now, onupdate=now ), + Column( "user_id", Integer, ForeignKey( "galaxy_user.id" ), index=True, nullable=False ), + Column( "uci_id", Integer, ForeignKey( "cloud_uci.id" ), index=True ), + Column( "store_id", Integer, ForeignKey( "cloud_store.id" ), index=True, nullable=False ), + Column( "snapshot_id", TEXT ), + Column( "status", TEXT ), + Column( "description", TEXT ), + Column( "error", TEXT ), + Column( "deleted", Boolean, default=False ) ) + +CloudUserCredentials.table = Table( "cloud_user_credentials", metadata, + Column( "id", Integer, primary_key=True ), + Column( "create_time", DateTime, default=now ), + Column( "update_time", DateTime, default=now, onupdate=now ), + Column( "user_id", Integer, ForeignKey( "galaxy_user.id" ), index=True, nullable=False ), + Column( "provider_id", Integer, ForeignKey( "cloud_provider.id" ), index=True, nullable=False ), + Column( "name", TEXT ), + Column( "access_key", TEXT ), + Column( "secret_key", TEXT ), + Column( "deleted", Boolean, default=False ) ) + +CloudProvider.table = Table( "cloud_provider", metadata, + Column( "id", Integer, primary_key=True ), + Column( "create_time", DateTime, default=now ), + Column( "update_time", DateTime, default=now, onupdate=now ), + Column( "user_id", Integer, ForeignKey( "galaxy_user.id" ), index=True, nullable=False ), + Column( "type", TEXT, nullable=False ), + Column( "name", TEXT ), + Column( "region_connection", TEXT ), + Column( "region_name", TEXT ), + Column( "region_endpoint", TEXT ), + Column( "is_secure", Boolean ), + Column( "host", TEXT ), + Column( "port", Integer ), + Column( "proxy", TEXT ), + Column( "proxy_port", TEXT ), + Column( "proxy_user", TEXT ), + Column( "proxy_pass", TEXT ), + Column( "debug", Integer ), + Column( "https_connection_factory", TEXT ), + Column( "path", TEXT ), + Column( "deleted", Boolean, default=False ) ) +# *************************** End cloud tables*********************************** + StoredWorkflow.table = Table( "stored_workflow", metadata, Column( "id", Integer, primary_key=True ), Column( "create_time", DateTime, default=now ), @@ -1004,6 +1115,42 @@ output_step=relation( WorkflowStep, backref="output_connections", cascade="all", primaryjoin=( WorkflowStepConnection.table.c.output_step_id == WorkflowStep.table.c.id ) ) ) ) +# vvvvvvvvvvvvvvvv Start cloud table mappings vvvvvvvvvvvvvvvv +assign_mapper( context, CloudImage, CloudImage.table ) + +assign_mapper( context, UCI, UCI.table, + properties=dict( user=relation( User ), + credentials=relation( CloudUserCredentials ), + instance=relation( CloudInstance, backref='uci' ), + store=relation( CloudStore, backref='uci', cascade='all, delete-orphan' ), + snapshot=relation( CloudSnapshot, backref='uci' ) + ) ) + +assign_mapper( context, CloudInstance, CloudInstance.table, + properties=dict( user=relation( User ), + image=relation( CloudImage ) + ) ) + +assign_mapper( context, CloudStore, CloudStore.table, + properties=dict( user=relation( User ), + i=relation( CloudInstance ), + snapshot=relation( CloudSnapshot, backref="store" ) + ) ) + +assign_mapper( context, CloudSnapshot, CloudSnapshot.table, + properties=dict( user=relation( User ) + ) ) + +assign_mapper( context, CloudProvider, CloudProvider.table, + properties=dict( user=relation( User ) + ) ) + +assign_mapper( context, CloudUserCredentials, CloudUserCredentials.table, + properties=dict( user=relation( User), + provider=relation( CloudProvider ) + ) ) +# ^^^^^^^^^^^^^^^ End cloud table mappings ^^^^^^^^^^^^^^^^^^ + assign_mapper( context, StoredWorkflow, StoredWorkflow.table, properties=dict( user=relation( User ), workflows=relation( Workflow, backref='stored_workflow', diff -r 90723c58b1a6 -r 03eb232d1111 lib/galaxy/model/migrate/versions/0026_cloud_tables.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/lib/galaxy/model/migrate/versions/0026_cloud_tables.py Tue Nov 17 19:07:19 2009 -0500 @@ -0,0 +1,152 @@ +from sqlalchemy import * +from migrate import * + +import datetime +now = datetime.datetime.utcnow + +# Need our custom types, but don't import anything else from model +from galaxy.model.custom_types import *

1 0

[hg] galaxy 3100: 'Insert history link' functionality added to p...
by Greg Von Kuster 23 Nov '09

23 Nov '09

details: http://www.bx.psu.edu/hg/galaxy/rev/90723c58b1a6 changeset: 3100:90723c58b1a6 user: jeremy goecks <jeremy.goecks(a)emory.edu> date: Tue Nov 17 19:06:42 2009 -0500 description: 'Insert history link' functionality added to pages. A user can select 1 or more histories and create links that point to a view of each history. diffstat: lib/galaxy/web/controllers/history.py | 23 +++++ lib/galaxy/web/controllers/page.py | 4 +- static/scripts/jquery.wymeditor.js | 91 ---------------------- static/wymeditor/lang/en.js | 4 +- templates/grid_base_async.mako | 10 ++- templates/page/editor.mako | 117 +++++++++++++++++++--------- templates/page/select_histories_grid.mako | 6 + templates/root/history.mako | 13 +++ 8 files changed, 133 insertions(+), 135 deletions(-) diffs (421 lines): diff -r 4c3cddd02b09 -r 90723c58b1a6 lib/galaxy/web/controllers/history.py --- a/lib/galaxy/web/controllers/history.py Mon Nov 16 18:38:32 2009 -0500 +++ b/lib/galaxy/web/controllers/history.py Tue Nov 17 19:06:42 2009 -0500 @@ -369,7 +369,30 @@ history.name = new_name trans.sa_session.add( history ) trans.sa_session.flush() + + @web.expose + @web.require_login( "get history name" ) + def get_name_async( self, trans, id=None ): + """ Returns the name for a given history. """ + history = get_history( trans, id, False ) + # To get name: user must own history, history must be importable. + if history.user == trans.get_user() or history.importable or trans.get_user() in history.users_shared_with: + return history.name + return + + @web.expose + @web.require_login( "set history's importable flag" ) + def set_importable_async( self, trans, id=None, importable=False ): + """ Set history's importable attribute. """ + history = get_history( trans, id, True ) + + if history: + history.importable = importable + trans.sa_session.flush() + + return + @web.expose def name_autocomplete_data( self, trans, q=None, limit=None, timestamp=None ): """Return autocomplete data for history names""" diff -r 4c3cddd02b09 -r 90723c58b1a6 lib/galaxy/web/controllers/page.py --- a/lib/galaxy/web/controllers/page.py Mon Nov 16 18:38:32 2009 -0500 +++ b/lib/galaxy/web/controllers/page.py Tue Nov 17 19:06:42 2009 -0500 @@ -116,13 +116,13 @@ # Grid definition. title = "Saved Histories" - template = "grid_base_async.mako" + template = "/page/select_histories_grid.mako" async_template = "grid_body_async.mako" model_class = model.History default_filter = { "deleted" : "False" , "shared" : "All" } default_sort_key = "-update_time" use_paging = True - num_rows_per_page = 5 + num_rows_per_page = 10 columns = [ NameColumn( "Name", key="name", model_class=model.History, filterable="advanced" ), grids.TagsColumn( "Tags", "tags", model.History, model.HistoryTagAssociation, filterable="advanced"), diff -r 4c3cddd02b09 -r 90723c58b1a6 static/scripts/jquery.wymeditor.js --- a/static/scripts/jquery.wymeditor.js Mon Nov 16 18:38:32 2009 -0500 +++ b/static/scripts/jquery.wymeditor.js Tue Nov 17 19:06:42 2009 -0500 @@ -295,67 +295,6 @@ }); -/* - Galaxy code that integrates into the WYM Editor. - */ -var Galaxy = -{ - /* - Galaxy constants for WYM Editor: - TOOLS - A string replaced by the galaxy toolbar's HTML. - TOOLS_ITEMS - A string replaced by the galaxy toolbar items. - INSERT_HISTORY - Command: open the insert history dialog. - INSERT_DATASET - Command: open the insert dataset dialog. - DIALOG_HISTORY - A dialog to insert a history. - DIALOG_DATASET - A dialog to insert a dataset. - */ - TOOLS : "{Galaxy_Tools}", - TOOLS_ITEMS : "{Galaxy_Tools_Items}", - INSERT_HISTORY : "InsertHistory", - INSERT_DATASET : "InsertDataset", - DIALOG_HISTORY : "DialogHistory", - DIALOG_DATASET : "DialogDataset", - - // Tool items overview. - toolsItems: [ - {'name': "InsertHistory", 'title': 'History', 'css': 'galaxy_tools_insert_history_link'}, - {'name': "InsertDataset", 'title': 'Dataset', 'css': 'galaxy_dataset'} - ], - - // Tools HTML. - toolsHtml: "<div class='wym_tools wym_section'>" - + "<h2>" + this.TOOLS + "</h2>" - + "<ul>" - + this.TOOLS_ITEMS - + "</ul>" - + "</div>", - - // Insert history dialog. - dialogHistoryHtml: "<body class='wym_dialog wym_dialog_history'" - + " onload='WYMeditor.INIT_DIALOG(" + WYMeditor.INDEX + ")'" - + ">" - + "<form>" - + "<fieldset>" - + "<input type='hidden' class='wym_dialog_type' value='" - + this.DIALOG_HISTORY - + "' />" - + "<legend>{Link}</legend>" - + "<div class='row'>" - + "<label>{Title}</label>" - + "<input type='text' class='wym_title' value='' size='40' />" - + "</div>" - + "<div class='row row-indent'>" - + "<input class='wym_submit' type='button'" - + " value='{Add}' />" - + "<input class='wym_cancel' type='button'" - + "value='{Cancel}' />" - + "</div>" - + "</fieldset>" - + "</form>" - + "</body>", -}; - - /********** JQUERY **********/ /** @@ -414,7 +353,6 @@ + "<div class='wym_area_right'>" + WYMeditor.CONTAINERS + WYMeditor.CLASSES - + Galaxy.TOOLS + "</div>" + "<div class='wym_area_main'>" + WYMeditor.HTML @@ -445,8 +383,6 @@ + "<h2>{Tools}</h2>" + "<ul>" + WYMeditor.TOOLS_ITEMS - // Add Galaxy Tools. - //+ Galaxy.TOOLS_ITEMS + "</ul>" + "</div>", @@ -821,7 +757,6 @@ boxHtml = h.replaceAll(boxHtml, WYMeditor.LOGO, this._options.logoHtml); boxHtml = h.replaceAll(boxHtml, WYMeditor.TOOLS, this._options.toolsHtml); - boxHtml = h.replaceAll(boxHtml, Galaxy.TOOLS, Galaxy.toolsHtml); boxHtml = h.replaceAll(boxHtml, WYMeditor.CONTAINERS,this._options.containersHtml); boxHtml = h.replaceAll(boxHtml, WYMeditor.CLASSES, this._options.classesHtml); boxHtml = h.replaceAll(boxHtml, WYMeditor.HTML, this._options.htmlHtml); @@ -846,24 +781,6 @@ boxHtml = h.replaceAll(boxHtml, WYMeditor.TOOLS_ITEMS, sTools); - // Construct Galaxy tools list. - var galaxyTools = eval(Galaxy.toolsItems); - sTools = ""; - for(var i = 0; i < galaxyTools.length; i++) { - var galaxyTool = galaxyTools[i]; - if(galaxyTool.name && galaxyTool.title) { - var sTool = this._options.toolsItemHtml; - var sTool = h.replaceAll(sTool, WYMeditor.TOOL_NAME, galaxyTool.name); - sTool = h.replaceAll(sTool, WYMeditor.TOOL_TITLE, this._options.stringDelimiterLeft - + galaxyTool.title - + this._options.stringDelimiterRight); - sTool = h.replaceAll(sTool, WYMeditor.TOOL_CLASS, galaxyTool.css); - sTools += sTool; - } - } - - //boxHtml = h.replaceAll(boxHtml, Galaxy.TOOLS_ITEMS, sTools); - //construct classes list var aClasses = eval(this._options.classesItems); var sClasses = ""; @@ -1029,10 +946,6 @@ this.dialog(WYMeditor.PREVIEW, this._options.dialogFeaturesPreview); break; - case Galaxy.INSERT_HISTORY: - this.dialog(Galaxy.DIALOG_HISTORY); - break; - default: this._exec(cmd); break; @@ -1263,10 +1176,6 @@ case(WYMeditor.PREVIEW): sBodyHtml = this._options.dialogPreviewHtml; break; - case(Galaxy.DIALOG_HISTORY): - sBodyHtml = Galaxy.dialogHistoryHtml; - break; - default: sBodyHtml = bodyHtml; } diff -r 4c3cddd02b09 -r 90723c58b1a6 static/wymeditor/lang/en.js --- a/static/wymeditor/lang/en.js Mon Nov 16 18:38:32 2009 -0500 +++ b/static/wymeditor/lang/en.js Tue Nov 17 19:06:42 2009 -0500 @@ -43,7 +43,7 @@ Source_Code: 'Source code', // Galaxy replacements. - History: 'History', - Dataset: 'Dataset', + Galaxy_History_Link: 'Insert Link to History', + Galaxy_Dataset_Link: 'Insert Link to Dataset', }; diff -r 4c3cddd02b09 -r 90723c58b1a6 templates/grid_base_async.mako --- a/templates/grid_base_async.mako Mon Nov 16 18:38:32 2009 -0500 +++ b/templates/grid_base_async.mako Tue Nov 17 19:06:42 2009 -0500 @@ -407,7 +407,11 @@ // Update grid. function update_grid(maintain_page_links) { + // If there's an operation in the args, do POST; otherwise, do GET. + var operation = url_args['operation']; + var method = (operation != null && operation != undefined ? "POST" : "GET" ); $.ajax({ + type: method, url: "${h.url_for()}", data: url_args, error: function() { alert( "Grid refresh failed" ) }, @@ -504,9 +508,11 @@ <%namespace file="./grid_common_async.mako" import="*" /> ## Print grid header. -<%def name="render_grid_header()"> +<%def name="render_grid_header(include_title)"> <div class="grid-header"> - <h2>${grid.title}</h2> + %if include_title: + <h2>${grid.title}</h2> + %endif %if grid.global_actions: <ul class="manage-table-actions"> diff -r 4c3cddd02b09 -r 90723c58b1a6 templates/page/editor.mako --- a/templates/page/editor.mako Mon Nov 16 18:38:32 2009 -0500 +++ b/templates/page/editor.mako Tue Nov 17 19:06:42 2009 -0500 @@ -39,6 +39,13 @@ <script type='text/javascript' src="${h.url_for('/static/scripts/jquery.autocomplete.js')}"> </script> <script type="text/javascript"> + + // Useful Galaxy stuff. + var Galaxy = + { + DIALOG_HISTORY_LINK : "history_link", + }; + ## Completely replace WYM's dialog handling WYMeditor.editor.prototype.dialog = function( dialogType, dialogFeatures, bodyHtml ) { @@ -187,45 +194,73 @@ } // HISTORY DIALOG - if ( dialogType == Galaxy.DIALOG_HISTORY ) { - show_modal( - "Insert History", - "<div class='row'>" - + "<label>History Name</label><br>" - + "<input id='history_name_input' type='text' class='wym_galaxy_history_name' value='' size='40' />" - + "</div>" - + "<div class='row'>" - + "<label>Select History</label><br>" - + "<input type='text' class='wym_galaxy_history_selected' value='' size='40' />" - + "</div>" - , - { - "Insert": function() { - var sUrl = jQuery(wym._options.hrefSelector).val(); - if(sUrl.length > 0) { - - wym._exec(WYMeditor.CREATE_LINK, sStamp); - - jQuery("a[href=" + sStamp + "]", wym._doc.body) - .attr(WYMeditor.HREF, sUrl) - .attr(WYMeditor.TITLE, jQuery(wym._options.titleSelector).val()); - hide_modal(); - - // TODO: remove autocomplete. - }, - "Cancel": function() { - hide_modal(); - - // TODO: remove autocomplete. - } + if ( dialogType == Galaxy.DIALOG_HISTORY_LINK ) { + $.ajax( + { + url: "${h.url_for( action='list_histories_for_selection' )}", + data: {}, + error: function() { alert( "Grid refresh failed" ) }, + success: function(table_html) + { + show_modal( + "Insert Link to History", + table_html + + "<div><input id='make-importable' type='checkbox' checked/>" + + "Publish the selected histories so that they can viewed by everyone.</div>" + , + { + "Insert": function() + { + // Make histories public/importable? + var make_importable = false; + if ( $('#make-importable:checked').val() !== null ) + make_importable = true; + + // Insert links to history for each checked item. + var item_ids = new Array(); + $('input[name=id]:checked').each(function() { + var item_id = $(this).val(); + + // Make history importable? + if (make_importable) + $.ajax({ + type: "POST", + url: '${h.url_for( controller='history', action='set_importable_async' )}', + data: { id: item_id, importable: 'True' }, + error: function() { alert('Make history importable failed; id=' + item_id) } + }); + + // Insert link. + wym._exec(WYMeditor.CREATE_LINK, sStamp); + if ( $("a[href=" + sStamp + "]", wym._doc.body).length != 0) + { + // Link created from selected text; add href and title. + $("a[href=" + sStamp + "]", wym._doc.body) + .attr(WYMeditor.HREF, '${h.url_for( controller='history', action='view' )}' + '?id=' + item_id) + .attr(WYMeditor.TITLE, "History" + item_id); + } + else + { + // User selected no text; create link from scratch and use default text. + + // Get history name. + $.get( '${h.url_for( controller='history', action='get_name_async' )}?id=' + item_id, function( history_name ) { + var href = '${h.url_for( controller='history', action='view' )}?id=' + item_id; + wym.insert("<a href='" + href + "'>History '" + history_name + "'</a>"); + }); + } + }); + + hide_modal(); + }, + "Cancel": function() + { + hide_modal(); + } + } + ); } - ); - - // Set up autocomplete for name input. - var t = $("#history_name_input"); - var autocomplete_options = - { selectFirst: false, autoFill: false, highlight: false, mustMatch: false }; - t.autocomplete("${h.url_for( controller='history', action='name_autocomplete_data' )}", autocomplete_options); + }); } }; </script> @@ -272,6 +307,7 @@ {'name': 'Unlink', 'title': 'Unlink', 'css': 'wym_tools_unlink'}, {'name': 'InsertImage', 'title': 'Image', 'css': 'wym_tools_image'}, {'name': 'InsertTable', 'title': 'Table', 'css': 'wym_tools_table'}, + {'name': 'Insert Galaxy History Link', 'title' : 'Galaxy_History_Link', 'css' : 'galaxy_tools_insert_history_link'} ] }); ## Get the editor object @@ -320,6 +356,11 @@ window.document.location = "${next_url}"; } }); + + // Initialize 'Insert history link' button. + $('.galaxy_tools_insert_history_link').children().click( function() { + editor.dialog(Galaxy.DIALOG_HISTORY_LINK); + }); }); </script> </%def> diff -r 4c3cddd02b09 -r 90723c58b1a6 templates/page/select_histories_grid.mako --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/templates/page/select_histories_grid.mako Tue Nov 17 19:06:42 2009 -0500 @@ -0,0 +1,6 @@ +## Template generates a grid that enables user to select histories. +<%namespace file="../grid_base_async.mako" import="*" /> + +${javascripts()} +${render_grid_header(False)} +${render_grid_table()} \ No newline at end of file diff -r 4c3cddd02b09 -r 90723c58b1a6 templates/root/history.mako --- a/templates/root/history.mako Mon Nov 16 18:38:32 2009 -0500 +++ b/templates/root/history.mako Tue Nov 17 19:06:42 2009 -0500 @@ -246,6 +246,19 @@ } }); }; + + //TODO: this function is a duplicate of array_length defined in galaxy.base.js ; not sure why it needs to be redefined here (due to streaming?). + // Returns the number of keys (elements) in an array/dictionary. + var array_length = function(an_array) + { + if (an_array.length) + return an_array.length; + + var count = 0; + for (element in an_array) + count++; + return count; + }; // // Function provides text for tagging toggle link.

1 0

[hg] galaxy 3096: Completed port to SQLalchemy 0.5. Also, fixed ...
by Greg Von Kuster 23 Nov '09

23 Nov '09

details: http://www.bx.psu.edu/hg/galaxy/rev/5e2fd6248f71 changeset: 3096:5e2fd6248f71 user: Enis Afgan <afgane(a)gmail.com> date: Tue Nov 17 15:51:39 2009 -0500 description: Completed port to SQLalchemy 0.5. Also, fixed couple of bugs, cleaned up cloud provider code, tested w/ EC2. diffstat: lib/galaxy/cloud/providers/ec2.py | 42 ++++++++++---------- lib/galaxy/cloud/providers/eucalyptus.py | 44 +++++++++++----------- lib/galaxy/web/controllers/cloud.py | 14 ++++-- templates/cloud/configure_cloud.mako | 24 ++--------- 4 files changed, 57 insertions(+), 67 deletions(-) diffs (414 lines): diff -r 39502dd3fd23 -r 5e2fd6248f71 lib/galaxy/cloud/providers/ec2.py --- a/lib/galaxy/cloud/providers/ec2.py Mon Nov 16 20:37:47 2009 -0500 +++ b/lib/galaxy/cloud/providers/ec2.py Tue Nov 17 15:51:39 2009 -0500 @@ -109,15 +109,15 @@ return try: if uci_state==uci_states.NEW: - self.createUCI( uci_wrapper ) + self.create_uci( uci_wrapper ) elif uci_state==uci_states.DELETING: - self.deleteUCI( uci_wrapper ) + self.delete_uci( uci_wrapper ) elif uci_state==uci_states.SUBMITTED: - self.startUCI( uci_wrapper ) + self.start_uci( uci_wrapper ) elif uci_state==uci_states.SHUTTING_DOWN: - self.stopUCI( uci_wrapper ) + self.stop_uci( uci_wrapper ) elif uci_state==uci_states.SNAPSHOT: - self.snapshotUCI( uci_wrapper ) + self.snapshot_uci( uci_wrapper ) except: log.exception( "Uncaught exception executing cloud request." ) cnt += 1 @@ -223,7 +223,7 @@ uci_wrapper.set_error( err+". Contact site administrator to ensure needed machine image is registered.", True ) return None - def createUCI( self, uci_wrapper ): + def create_uci( self, uci_wrapper ): """ Creates User Configured Instance (UCI). Essentially, creates storage volume on cloud provider and registers relevant information in Galaxy database. @@ -276,7 +276,7 @@ uci_wrapper.set_store_status( vol.id, uci_states.ERROR ) uci_wrapper.set_error( err, True ) - def deleteUCI( self, uci_wrapper ): + def delete_uci( self, uci_wrapper ): """ Deletes UCI. NOTE that this implies deletion of any and all data associated with this UCI from the cloud. All data will be deleted. @@ -315,7 +315,7 @@ log.error( err ) uci_wrapper.set_error( err, True ) - def snapshotUCI( self, uci_wrapper ): + def snapshot_uci( self, uci_wrapper ): """ Creates snapshot of all storage volumes associated with this UCI. """ @@ -346,11 +346,11 @@ uci_wrapper.change_state( uci_state=uci_states.AVAILABLE ) - def addStorageToUCI( self, name ): + def add_storage_to_uci( self, name ): """ Adds more storage to specified UCI TODO""" - def dummyStartUCI( self, uci_wrapper ): + def dummy_start_uci( self, uci_wrapper ): uci = uci_wrapper.get_uci() log.debug( "Would be starting instance '%s'" % uci.name ) @@ -359,7 +359,7 @@ # time.sleep(20) # log.debug( "Woke up! (%s)" % uci.name ) - def startUCI( self, uci_wrapper ): + def start_uci( self, uci_wrapper ): """ Starts instance(s) of given UCI on the cloud. """ @@ -456,7 +456,7 @@ else: log.error( "UCI '%s' is in 'error' state, starting instance was aborted." % uci_wrapper.get_name() ) - def stopUCI( self, uci_wrapper): + def stop_uci( self, uci_wrapper): """ Stops all of cloud instances associated with given UCI. """ @@ -467,7 +467,7 @@ # Process list of instances and remove any references to empty instance id's for i in il: if i is None: - l.remove( i ) + il.remove( i ) log.debug( 'List of instances being terminated: %s' % il ) rl = conn.get_all_instances( il ) # Reservation list associated with given instances @@ -554,7 +554,7 @@ for inst in instances: if self.type == inst.uci.credentials.provider.type: log.debug( "[%s] Running general status update on instance '%s'" % ( inst.uci.credentials.provider.type, inst.instance_id ) ) - self.updateInstance( inst ) + self.update_instance( inst ) # Update storage volume(s) stores = self.sa_session.query( model.CloudStore ) \ @@ -565,7 +565,7 @@ for store in stores: if self.type == store.uci.credentials.provider.type: # and store.volume_id != None: log.debug( "[%s] Running general status update on store with local database ID: '%s'" % ( store.uci.credentials.provider.type, store.id ) ) - self.updateStore( store ) + self.update_store( store ) # else: # log.error( "[%s] There exists an entry for UCI (%s) storage volume without an ID. Storage volume might have been created with " # "cloud provider though. Manual check is recommended." % ( store.uci.credentials.provider.type, store.uci.name ) ) @@ -579,7 +579,7 @@ # Update pending snapshots or delete ones marked for deletion snapshots = self.sa_session.query( model.CloudSnapshot ) \ - .filter_by( status=snapshot_status.PENDING, status=snapshot_status.DELETE ) \ + .filter( or_( model.CloudSnapshot.table.c.status == snapshot_status.PENDING, model.CloudSnapshot.table.c.status == snapshot_status.DELETE ) ) \ .all() for snapshot in snapshots: if self.type == snapshot.uci.credentials.provider.type and snapshot.status == snapshot_status.PENDING: @@ -603,9 +603,9 @@ td = datetime.utcnow() - z_inst.update_time if td.seconds > 180: # if instance has been in SUBMITTED state for more than 3 minutes log.debug( "[%s] Running zombie repair update on instance with DB id '%s'" % ( z_inst.uci.credentials.provider.type, z_inst.id ) ) - self.processZombie( z_inst ) + self.process_zombie( z_inst ) - def updateInstance( self, inst ): + def update_instance( self, inst ): # Get credentials associated wit this instance uci_id = inst.uci_id @@ -678,7 +678,7 @@ self.sa_session.flush() return None - def updateStore( self, store ): + def update_store( self, store ): # Get credentials associated wit this store uci_id = store.uci_id uci = self.sa_session.query( model.UCI ).get( uci_id ) @@ -748,7 +748,7 @@ self.sa_session.add( store ) self.sa_session.flush() - def updateSnapshot( self, snapshot ): + def update_snapshot( self, snapshot ): # Get credentials associated wit this store uci_id = snapshot.uci_id uci = self.sa_session.query( model.UCI ).get( uci_id ) @@ -839,7 +839,7 @@ self.sa_session.add( snapshot ) self.sa_session.flush() - def processZombie( self, inst ): + def process_zombie( self, inst ): """ Attempt at discovering if starting an instance was successful but local database was not updated accordingly or if something else failed and instance was never started. Currently, no automatic diff -r 39502dd3fd23 -r 5e2fd6248f71 lib/galaxy/cloud/providers/eucalyptus.py --- a/lib/galaxy/cloud/providers/eucalyptus.py Mon Nov 16 20:37:47 2009 -0500 +++ b/lib/galaxy/cloud/providers/eucalyptus.py Tue Nov 17 15:51:39 2009 -0500 @@ -110,16 +110,16 @@ return try: if uci_state==uci_states.NEW: - self.createUCI( uci_wrapper ) + self.create_uci( uci_wrapper ) elif uci_state==uci_states.DELETING: - self.deleteUCI( uci_wrapper ) + self.delete_uci( uci_wrapper ) elif uci_state==uci_states.SUBMITTED: - self.startUCI( uci_wrapper ) - #self.dummyStartUCI( uci_wrapper ) + self.start_uci( uci_wrapper ) + #self.dummy_start_uci( uci_wrapper ) elif uci_state==uci_states.SHUTTING_DOWN: - self.stopUCI( uci_wrapper ) + self.stop_uci( uci_wrapper ) elif uci_state==uci_states.SNAPSHOT: - self.snapshotUCI( uci_wrapper ) + self.snapshot_uci( uci_wrapper ) except: log.exception( "Uncaught exception executing cloud request." ) cnt += 1 @@ -230,7 +230,7 @@ uci_wrapper.set_error( err+". Contact site administrator to ensure needed machine image is registered.", True ) return None - def createUCI( self, uci_wrapper ): + def create_uci( self, uci_wrapper ): """ Create User Configured Instance (UCI) - i.e., create storage volume on cloud provider and register relevant information in local Galaxy database. @@ -276,7 +276,7 @@ uci_wrapper.set_store_status( vol.id, uci_states.ERROR ) uci_wrapper.set_error( err, True ) - def deleteUCI( self, uci_wrapper ): + def delete_uci( self, uci_wrapper ): """ Delete UCI - i.e., delete all storage volumes associated with this UCI. NOTE that this implies deletion of any and all data associated @@ -318,7 +318,7 @@ log.error( err ) uci_wrapper.set_error( err, True ) - def snapshotUCI( self, uci_wrapper ): + def snapshot_uci( self, uci_wrapper ): """ Initiate creation of a snapshot by cloud provider for all storage volumes associated with this UCI. @@ -361,10 +361,10 @@ # Feel free to resent state of this instance and use it normally.", True ) - def addStorageToUCI( self, uci_wrapper ): + def add_storage_to_uci( self, uci_wrapper ): """ Adds more storage to specified UCI """ - def dummyStartUCI( self, uci_wrapper ): + def dummy_start_uci( self, uci_wrapper ): uci = uci_wrapper.get_uci() log.debug( "Would be starting instance '%s'" % uci.name ) @@ -374,7 +374,7 @@ time.sleep(10) log.debug( "Woke up! (%s)" % uci.name ) - def startUCI( self, uci_wrapper ): + def start_uci( self, uci_wrapper ): """ Start instance(s) of given UCI on the cloud. """ @@ -443,7 +443,7 @@ else: log.error( "UCI '%s' is in 'error' state, starting instance was aborted." % uci_wrapper.get_name() ) - def stopUCI( self, uci_wrapper): + def stop_uci( self, uci_wrapper): """ Stop all cloud instances associated with given UCI. """ @@ -454,7 +454,7 @@ # Process list of instances and remove any references to empty instance id's for i in il: if i is None: - l.remove( i ) + il.remove( i ) log.debug( 'List of instances being terminated: %s' % il ) rl = conn.get_all_instances( il ) # Reservation list associated with given instances @@ -543,7 +543,7 @@ for inst in instances: if self.type == inst.uci.credentials.provider.type: log.debug( "[%s] Running general status update on instance '%s'" % ( inst.uci.credentials.provider.type, inst.instance_id ) ) - self.updateInstance( inst ) + self.update_instance( inst ) # Update storage volume(s) stores = self.sa_session.query( model.CloudStore ) \ @@ -554,11 +554,11 @@ for store in stores: if self.type == store.uci.credentials.provider.type: # and store.volume_id != None: log.debug( "[%s] Running general status update on store with local database ID: '%s'" % ( store.uci.credentials.provider.type, store.id ) ) - self.updateStore( store ) + self.update_store( store ) # Update pending snapshots or delete ones marked for deletion snapshots = self.sa_session.query( model.CloudSnapshot ) \ - .filter_by( status=snapshot_status.PENDING, status=snapshot_status.DELETE ) \ + .filter( or_( model.CloudSnapshot.table.c.status == snapshot_status.PENDING, model.CloudSnapshot.table.c.status == snapshot_status.DELETE ) ) \ .all() for snapshot in snapshots: if self.type == snapshot.uci.credentials.provider.type and snapshot.status == snapshot_status.PENDING: @@ -583,9 +583,9 @@ # log.debug( "z_inst.id: %s, time delta is %s sec" % ( z_inst.id, td.seconds ) ) if td.seconds > 180: # if instance has been in SUBMITTED state for more than 3 minutes log.debug( "[%s](td=%s) Running zombie repair update on instance with DB id '%s'" % ( z_inst.uci.credentials.provider.type, td.seconds, z_inst.id ) ) - self.processZombie( z_inst ) + self.process_zombie( z_inst ) - def updateInstance( self, inst ): + def update_instance( self, inst ): """ Update information in local database for given instance as it is obtained from cloud provider. Along with updating information about given instance, information about the UCI controlling @@ -662,7 +662,7 @@ self.sa_session.flush() return None - def updateStore( self, store ): + def update_store( self, store ): """ Update information in local database for given storage volume as it is obtained from cloud provider. Along with updating information about given storage volume, information about the UCI controlling @@ -756,7 +756,7 @@ self.sa_session.add( store ) self.sa_session.flush() - def updateSnapshot( self, snapshot ): + def update_snapshot( self, snapshot ): """ Update information in local database for given snapshot as it is obtained from cloud provider. Along with updating information about given snapshot, information about the UCI controlling @@ -855,7 +855,7 @@ self.sa_session.add( snapshot ) self.sa_session.flush() - def processZombie( self, inst ): + def process_zombie( self, inst ): """ Attempt at discovering if starting a cloud instance was successful but local database was not updated accordingly or if something else failed and instance was never started. Currently, no automatic diff -r 39502dd3fd23 -r 5e2fd6248f71 lib/galaxy/web/controllers/cloud.py --- a/lib/galaxy/web/controllers/cloud.py Mon Nov 16 20:37:47 2009 -0500 +++ b/lib/galaxy/web/controllers/cloud.py Tue Nov 17 15:51:39 2009 -0500 @@ -515,7 +515,7 @@ error['provider_error'] = "You must select cloud provider type for this machine image." elif image_id=='' or len( image_id ) > 255: error['id_error'] = "Image ID must be between 1 and 255 characters long." - elif trans.sa_session.query( model.CloudUserCredentials ) \ + elif trans.sa_session.query( model.CloudImage ) \ .filter_by( deleted=False ) \ .filter( model.CloudImage.table.c.image_id == image_id ) \ .first(): @@ -558,7 +558,7 @@ @web.expose @web.require_login( "use Galaxy cloud" ) def list_machine_images( self, trans ): - images = trans.sa_session.query( model.CloudImage ).filter( model.CloudImage.table.c.deleted != True ).all() + images = trans.sa_session.query( model.CloudImage ).filter_by( deleted=False ).all() return trans.fill_template( '/cloud/list_images.mako', images=images ) @web.expose @@ -1028,15 +1028,19 @@ @web.json def json_update( self, trans ): user = trans.get_user() - UCIs = trans.sa_session.query( model.UCI ).filter_by( user=user ).filter( model.UCI.table.c.deleted != True ).all() + UCIs = trans.sa_session.query( model.UCI ).filter_by( user=user, deleted=False ).all() insd = {} # instance name-state dict for uci in UCIs: dict = {} dict['id'] = uci.id dict['state'] = uci.state + if uci.error != None: + dict['error'] = str( uci.error ) + else: + dict['error'] = None if uci.launch_time != None: - dict['launch_time'] = str(uci.launch_time) - dict['time_ago'] = str(date.distance_of_time_in_words(uci.launch_time, date.datetime.utcnow() ) ) + dict['launch_time'] = str( uci.launch_time ) + dict['time_ago'] = str( date.distance_of_time_in_words( uci.launch_time, date.datetime.utcnow() ) ) else: dict['launch_time'] = None dict['time_ago'] = None diff -r 39502dd3fd23 -r 5e2fd6248f71 templates/cloud/configure_cloud.mako --- a/templates/cloud/configure_cloud.mako Mon Nov 16 20:37:47 2009 -0500 +++ b/templates/cloud/configure_cloud.mako Tue Nov 17 15:51:39 2009 -0500 @@ -36,10 +36,11 @@ old_state = $(elem + "-state").text(); prev_old_state = trim19( $(elem + "-state-p").text() ); new_state = data[i].state; + error_msg = data[i].error; //console.log( "old_state[%d] = %s", i, old_state ); //console.log( "prev_old_state[%d] = %s", i, prev_old_state ); //console.log( "new_state[%d] = %s", i, new_state ); - //console.log( trim19(prev_old_state) ); + //console.log( "error_msg[%d] = %s", i, error_msg ); if ( ( old_state=='pending' && new_state=='running' ) || ( old_state=='shutting-down' && new_state=='available' ) || \ ( old_state=='running' && new_state=='available' ) || ( old_state=='running' && new_state=='error' ) || \ ( old_state=='pending' && new_state=='available' ) || ( old_state=='submitted' && new_state=='available' ) || \ @@ -47,21 +48,11 @@ var url = "${h.url_for( controller='cloud', action='list')}"; location.replace( url ); } - else if ( ( old_state=='running' && new_state=='error' ) || ( old_state=='pending' && new_state=='error' ) || \ - ( old_state=='submitted' && new_state=='error' ) || ( old_state=='submittedUCI' && new_state=='error' ) || \ - ( old_state=='shutting-down' && new_state=='error' ) || ( prev_old_state.match('newUCI') && new_state=='error' ) || \ - ( prev_old_state.match('new') && new_state=='error' ) || ( prev_old_state.match('deletingUCI') && new_state=='error' ) ) { - // TODO: Following clause causes constant page refresh for an exception thrown as a result of instance not starting correctly - need alternative method! - //( prev_old_state.match('available') && new_state=='error' ) || ( prev_old_state.match('deleting') && new_state=='error' ) \ - + else if ( ( ( old_state != 'error' && old_state != '' ) && new_state == 'error' ) || ( !prev_old_state.match('error') && new_state == 'error' ) ) { var url = "${h.url_for( controller='cloud', action='list')}"; location.replace( url ); } - if ( prev_old_state.match('deletingUCI') || prev_old_state.match('deleting') ) { - setTimeout("update_state()", 3000); - } - if ( new_state=='shutting-down' || new_state=='shutting-downUCI' ) { $(elem + "-link").text( "" ); } @@ -90,15 +81,10 @@ // Update 'state' and 'time alive' fields $(elem + "-state").text( data[i].state ); - if ( ( prev_old_state.match('newUCI') && new_state=='new' ) || ( prev_old_state.match('newUCI') && new_state=='available' ) || \ - ( prev_old_state.match('newUCI') && new_state=='creating' ) || ( prev_old_state.match('new') && new_state=='creating' ) || \ - ( prev_old_state.match('new') && new_state=='available' ) || \ - ( prev_old_state.match('deletingUCI') && new_state=='deleted' ) || ( prev_old_state.match('deleting') && new_state=='deleted' ) || \ - ( prev_old_state.match('available') && new_state=='error' ) || ( prev_old_state.match('deleting') && new_state=='error' ) ) { - // TODO: on state change from available->error and deleting->error page should be refreshed but that causes problems with - // constant refreshings depending on what error message is so at least do it here... + if ( new_state != 'error' ) { // Because 'error' state is handled as a JS link, don't include it in update $(elem + "-state-p").text( data[i].state ); } + if (data[i].launch_time) { $(elem + "-launch_time").text( data[i].launch_time.substring(0, 16 ) + " UTC (" + data[i].time_ago + ")" ); }

1 0

[hg] galaxy 3099: Fix cloud execution on startup
by Greg Von Kuster 23 Nov '09

23 Nov '09

details: http://www.bx.psu.edu/hg/galaxy/rev/655868f3ef98 changeset: 3099:655868f3ef98 user: Enis Afgan <afgane(a)gmail.com> date: Tue Nov 17 16:39:05 2009 -0500 description: Fix cloud execution on startup diffstat: lib/galaxy/cloud/__init__.py | 2 +- lib/galaxy/config.py | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diffs (30 lines): diff -r 37406d8ad116 -r 655868f3ef98 lib/galaxy/cloud/__init__.py --- a/lib/galaxy/cloud/__init__.py Tue Nov 17 16:16:26 2009 -0500 +++ b/lib/galaxy/cloud/__init__.py Tue Nov 17 16:39:05 2009 -0500 @@ -61,7 +61,7 @@ def __init__( self, app ): self.app = app self.sa_session = app.model.context - if self.app.config.get_bool( "enable_cloud_execution", True ): + if self.app.config.enable_cloud_execution == True: # The dispatcher manager for underlying cloud instances - implements and contacts individual cloud providers self.provider = CloudProvider( app ) # Monitor for updating status of cloud instances diff -r 37406d8ad116 -r 655868f3ef98 lib/galaxy/config.py --- a/lib/galaxy/config.py Tue Nov 17 16:16:26 2009 -0500 +++ b/lib/galaxy/config.py Tue Nov 17 16:39:05 2009 -0500 @@ -114,11 +114,11 @@ self.tool_runners = [] self.datatypes_config = kwargs.get( 'datatypes_config_file', 'datatypes_conf.xml' ) # Cloud configuration options - self.cloud_controller_instance = string_as_bool( kwargs.get( 'cloud_controller_instance', False ) ) + self.cloud_controller_instance = string_as_bool( kwargs.get( 'cloud_controller_instance', 'False' ) ) if self.cloud_controller_instance == True: - self.enable_cloud_execution = string_as_bool( kwargs.get( 'enable_cloud_execution', True ) ) + self.enable_cloud_execution = string_as_bool( kwargs.get( 'enable_cloud_execution', 'True' ) ) else: - self.enable_cloud_execution = string_as_bool( kwargs.get( 'enable_cloud_execution', False ) ) + self.enable_cloud_execution = string_as_bool( kwargs.get( 'enable_cloud_execution', 'False' ) ) def get( self, key, default ): return self.config_dict.get( key, default ) def get_bool( self, key, default ):

1 0

[hg] galaxy 3098: merge
by Greg Von Kuster 23 Nov '09

23 Nov '09

details: http://www.bx.psu.edu/hg/galaxy/rev/37406d8ad116 changeset: 3098:37406d8ad116 user: Enis Afgan <afgane(a)gmail.com> date: Tue Nov 17 16:16:26 2009 -0500 description: merge diffstat: datatypes_conf.xml.sample | 429 +++++++------ dist-eggs.ini | 9 +- eggs.ini | 13 +- lib/galaxy/datatypes/binary.py | 156 +++++ lib/galaxy/datatypes/data.py | 93 +- lib/galaxy/datatypes/genetics.py | 180 ++--- lib/galaxy/datatypes/images.py | 120 --- lib/galaxy/datatypes/metadata.py | 4 +- lib/galaxy/datatypes/registry.py | 16 +- lib/galaxy/datatypes/tracks.py | 9 +- lib/galaxy/jobs/__init__.py | 22 +- lib/galaxy/jobs/runners/local.py | 4 +- lib/galaxy/jobs/runners/pbs.py | 17 +- lib/galaxy/tools/actions/metadata.py | 1 + lib/galaxy/web/controllers/dataset.py | 61 +- lib/galaxy/web/controllers/forms.py | 252 +++++--- lib/galaxy/web/controllers/history.py | 34 +- lib/galaxy/web/controllers/library_common.py | 2 +- lib/galaxy/web/controllers/page.py | 60 +- lib/galaxy/web/controllers/requests.py | 302 +++++---- lib/galaxy/web/controllers/requests_admin.py | 663 +++++++++++++------- lib/galaxy/web/controllers/user.py | 2 - lib/galaxy/web/framework/__init__.py | 10 +- lib/galaxy/web/framework/helpers/grids.py | 8 +- run.sh | 19 + scripts/check_eggs.py | 14 +- scripts/check_python.py | 19 +- scripts/cleanup_datasets/cleanup_datasets.py | 4 +- scripts/scramble/scripts/pysqlite.py | 44 +- scripts/set_metadata.py | 15 +- static/scripts/galaxy.base.js | 20 +- templates/admin/forms/edit_form.mako | 2 +- templates/admin/forms/grid.mako | 1 + templates/admin/forms/manage_forms.mako | 76 -- templates/admin/forms/show_form_read_only.mako | 4 +- templates/admin/requests/create_request_type.mako | 92 +- templates/admin/requests/grid.mako | 218 +------- templates/admin/requests/manage_request_types.mako | 69 +-- templates/admin/requests/show_request.mako | 2 +- templates/admin/requests/view_request_type.mako | 70 +- templates/dataset/errors.mako | 17 +- templates/grid_base.mako | 4 +- templates/grid_base_async.mako | 698 ++++++++++++++++++++++ templates/grid_body_async.mako | 5 + templates/grid_common_async.mako | 155 +++++ templates/mobile/history/detail.mako | 2 +- templates/requests/grid.mako | 218 +------- templates/requests/show_request.mako | 2 +- templates/root/history_common.mako | 4 +- templates/tagging_common.mako | 13 - templates/tool_form.mako | 16 +- test/base/twilltestcase.py | 43 +- test/functional/test_forms_and_requests.py | 44 +- test/functional/test_get_data.py | 620 ++++++++++++++++--- test/functional/test_history_functions.py | 6 + test/functional/test_sniffing_and_metadata_settings.py | 262 -------- test/functional/test_user_info.py | 9 +- tools/data_source/upload.py | 45 +- tools/data_source/upload.xml | 6 + tools/extract/extract_genomic_dna.xml | 4 +- tools/extract/liftOver_wrapper.xml | 2 +- 61 files changed, 3055 insertions(+), 2256 deletions(-) diffs (truncated from 7153 to 3000 lines): diff -r 555afd0bf457 -r 37406d8ad116 datatypes_conf.xml.sample --- a/datatypes_conf.xml.sample Tue Nov 17 16:14:54 2009 -0500 +++ b/datatypes_conf.xml.sample Tue Nov 17 16:16:26 2009 -0500 @@ -1,211 +1,224 @@ <?xml version="1.0"?> <datatypes> - <registration converters_path="lib/galaxy/datatypes/converters"> - <datatype extension="ab1" type="galaxy.datatypes.images:Ab1" mimetype="application/octet-stream" display_in_upload="true"/> - <datatype extension="axt" type="galaxy.datatypes.sequence:Axt" display_in_upload="true"/> - <datatype extension="bam" type="galaxy.datatypes.images:Bam" mimetype="application/octet-stream"/> - <datatype extension="bed" type="galaxy.datatypes.interval:Bed" display_in_upload="true"> - <converter file="bed_to_gff_converter.xml" target_datatype="gff"/> - <converter file="interval_to_coverage.xml" target_datatype="coverage"/> - <converter file="bed_to_interval_index_converter.xml" target_datatype="interval_index"/> - </datatype> - <datatype extension="binseq.zip" type="galaxy.datatypes.images:Binseq" mimetype="application/zip" display_in_upload="true"/> - <datatype extension="len" type="galaxy.datatypes.chrominfo:ChromInfo" display_in_upload="true"> -  - </datatype> - <datatype extension="coverage" type="galaxy.datatypes.coverage:LastzCoverage" display_in_upload="true"> - <indexer file="coverage.xml" /> - </datatype> - <datatype extension="customtrack" type="galaxy.datatypes.interval:CustomTrack"/> - <datatype extension="csfasta" type="galaxy.datatypes.sequence:csFasta" display_in_upload="true"/> - <datatype extension="data" type="galaxy.datatypes.data:Data" mimetype="application/octet-stream"/> - <datatype extension="fasta" type="galaxy.datatypes.sequence:Fasta" display_in_upload="true"> - <converter file="fasta_to_tabular_converter.xml" target_datatype="tabular"/> - </datatype> - <datatype extension="fastq" type="galaxy.datatypes.sequence:Fastq" display_in_upload="true"/> - <datatype extension="fastqsanger" type="galaxy.datatypes.sequence:FastqSanger" display_in_upload="true"/> - <datatype extension="genetrack" type="galaxy.datatypes.tracks:GeneTrack"/> - <datatype extension="gff" type="galaxy.datatypes.interval:Gff" display_in_upload="true"> - <converter file="gff_to_bed_converter.xml" target_datatype="bed"/> - </datatype> - <datatype extension="gff3" type="galaxy.datatypes.interval:Gff3" display_in_upload="true"/> - <datatype extension="gif" type="galaxy.datatypes.images:Image" mimetype="image/gif"/> - <datatype extension="gmaj.zip" type="galaxy.datatypes.images:Gmaj" mimetype="application/zip"/> - <datatype extension="html" type="galaxy.datatypes.images:Html" mimetype="text/html"/> - <datatype extension="interval" type="galaxy.datatypes.interval:Interval" display_in_upload="true"> - <converter file="interval_to_bed_converter.xml" target_datatype="bed"/> - <indexer file="interval_awk.xml" /> - </datatype> - <datatype extension="jpg" type="galaxy.datatypes.images:Image" mimetype="image/jpeg"/> - <datatype extension="laj" type="galaxy.datatypes.images:Laj"/> - <datatype extension="lav" type="galaxy.datatypes.sequence:Lav" display_in_upload="true"/> - <datatype extension="maf" type="galaxy.datatypes.sequence:Maf" display_in_upload="true"> - <converter file="maf_to_fasta_converter.xml" target_datatype="fasta"/> - <converter file="maf_to_interval_converter.xml" target_datatype="interval"/> - </datatype> - <datatype extension="pdf" type="galaxy.datatypes.images:Image" mimetype="application/pdf"/> - <datatype extension="png" type="galaxy.datatypes.images:Image" mimetype="image/png"/> - <datatype extension="qualsolexa" type="galaxy.datatypes.qualityscore:QualityScoreSolexa" display_in_upload="true"/> - <datatype extension="qualsolid" type="galaxy.datatypes.qualityscore:QualityScoreSOLiD" display_in_upload="true"/> - <datatype extension="qual454" type="galaxy.datatypes.qualityscore:QualityScore454" display_in_upload="true"/> - <datatype extension="sam" type="galaxy.datatypes.tabular:Sam" display_in_upload="true"/> - <datatype extension="scf" type="galaxy.datatypes.images:Scf" mimetype="application/octet-stream" display_in_upload="true"/> - <datatype extension="taxonomy" type="galaxy.datatypes.tabular:Taxonomy" display_in_upload="true"/> - <datatype extension="tabular" type="galaxy.datatypes.tabular:Tabular" display_in_upload="true"/> - <datatype extension="txt" type="galaxy.datatypes.data:Text" display_in_upload="true"/> - <datatype extension="blastxml" type="galaxy.datatypes.xml:BlastXml" display_in_upload="true"/> - <datatype extension="txtseq.zip" type="galaxy.datatypes.images:Txtseq" mimetype="application/zip" display_in_upload="true"/> - <datatype extension="wig" type="galaxy.datatypes.interval:Wiggle" display_in_upload="true"> - <converter file="wiggle_to_array_tree_converter.xml" target_datatype="array_tree"/> - </datatype> - <datatype extension="array_tree" type="galaxy.datatypes.data:Data" /> - <datatype extension="interval_index" type="galaxy.datatypes.data:Data" /> -  - <datatype extension="acedb" type="galaxy.datatypes.data:Text"/> - <datatype extension="asn1" type="galaxy.datatypes.data:Text"/> - <datatype extension="btwisted" type="galaxy.datatypes.data:Text"/> - <datatype extension="cai" type="galaxy.datatypes.data:Text"/> - <datatype extension="charge" type="galaxy.datatypes.data:Text"/> - <datatype extension="checktrans" type="galaxy.datatypes.data:Text"/> - <datatype extension="chips" type="galaxy.datatypes.data:Text"/> - <datatype extension="clustal" type="galaxy.datatypes.data:Text"/> - <datatype extension="codata" type="galaxy.datatypes.data:Text"/> - <datatype extension="codcmp" type="galaxy.datatypes.data:Text"/> - <datatype extension="coderet" type="galaxy.datatypes.data:Text"/> - <datatype extension="compseq" type="galaxy.datatypes.data:Text"/> - <datatype extension="cpgplot" type="galaxy.datatypes.data:Text"/> - <datatype extension="cpgreport" type="galaxy.datatypes.data:Text"/> - <datatype extension="cusp" type="galaxy.datatypes.data:Text"/> - <datatype extension="cut" type="galaxy.datatypes.data:Text"/> - <datatype extension="dan" type="galaxy.datatypes.data:Text"/> - <datatype extension="dbmotif" type="galaxy.datatypes.data:Text"/> - <datatype extension="diffseq" type="galaxy.datatypes.data:Text"/> - <datatype extension="digest" type="galaxy.datatypes.data:Text"/> - <datatype extension="dreg" type="galaxy.datatypes.data:Text"/> - <datatype extension="einverted" type="galaxy.datatypes.data:Text"/> - <datatype extension="embl" type="galaxy.datatypes.data:Text"/> - <datatype extension="epestfind" type="galaxy.datatypes.data:Text"/> - <datatype extension="equicktandem" type="galaxy.datatypes.data:Text"/> - <datatype extension="est2genome" type="galaxy.datatypes.data:Text"/> - <datatype extension="etandem" type="galaxy.datatypes.data:Text"/> - <datatype extension="excel" type="galaxy.datatypes.data:Text"/> - <datatype extension="feattable" type="galaxy.datatypes.data:Text"/> - <datatype extension="fitch" type="galaxy.datatypes.data:Text"/> - <datatype extension="freak" type="galaxy.datatypes.data:Text"/> - <datatype extension="fuzznuc" type="galaxy.datatypes.data:Text"/> - <datatype extension="fuzzpro" type="galaxy.datatypes.data:Text"/> - <datatype extension="fuzztran" type="galaxy.datatypes.data:Text"/> - <datatype extension="garnier" type="galaxy.datatypes.data:Text"/> - <datatype extension="gcg" type="galaxy.datatypes.data:Text"/> - <datatype extension="geecee" type="galaxy.datatypes.data:Text"/> - <datatype extension="genbank" type="galaxy.datatypes.data:Text"/> - <datatype extension="helixturnhelix" type="galaxy.datatypes.data:Text"/> - <datatype extension="hennig86" type="galaxy.datatypes.data:Text"/> - <datatype extension="hmoment" type="galaxy.datatypes.data:Text"/> - <datatype extension="ig" type="galaxy.datatypes.data:Text"/> - <datatype extension="isochore" type="galaxy.datatypes.data:Text"/> - <datatype extension="jackknifer" type="galaxy.datatypes.data:Text"/> - <datatype extension="jackknifernon" type="galaxy.datatypes.data:Text"/> - <datatype extension="markx10" type="galaxy.datatypes.data:Text"/> - <datatype extension="markx1" type="galaxy.datatypes.data:Text"/> - <datatype extension="markx0" type="galaxy.datatypes.data:Text"/> - <datatype extension="markx3" type="galaxy.datatypes.data:Text"/> - <datatype extension="markx2" type="galaxy.datatypes.data:Text"/> - <datatype extension="match" type="galaxy.datatypes.data:Text"/> - <datatype extension="mega" type="galaxy.datatypes.data:Text"/> - <datatype extension="meganon" type="galaxy.datatypes.data:Text"/> - <datatype extension="motif" type="galaxy.datatypes.data:Text"/> - <datatype extension="msf" type="galaxy.datatypes.data:Text"/> - <datatype extension="nametable" type="galaxy.datatypes.data:Text"/> - <datatype extension="ncbi" type="galaxy.datatypes.data:Text"/> - <datatype extension="needle" type="galaxy.datatypes.data:Text"/> - <datatype extension="newcpgreport" type="galaxy.datatypes.data:Text"/> - <datatype extension="newcpgseek" type="galaxy.datatypes.data:Text"/> - <datatype extension="nexus" type="galaxy.datatypes.data:Text"/> - <datatype extension="nexusnon" type="galaxy.datatypes.data:Text"/> - <datatype extension="noreturn" type="galaxy.datatypes.data:Text"/> - <datatype extension="pair" type="galaxy.datatypes.data:Text"/> - <datatype extension="palindrome" type="galaxy.datatypes.data:Text"/> - <datatype extension="pepcoil" type="galaxy.datatypes.data:Text"/> - <datatype extension="pepinfo" type="galaxy.datatypes.data:Text"/> - <datatype extension="pepstats" type="galaxy.datatypes.data:Text"/> - <datatype extension="phylip" type="galaxy.datatypes.data:Text"/> - <datatype extension="phylipnon" type="galaxy.datatypes.data:Text"/> - <datatype extension="pir" type="galaxy.datatypes.data:Text"/> - <datatype extension="polydot" type="galaxy.datatypes.data:Text"/> - <datatype extension="preg" type="galaxy.datatypes.data:Text"/> - <datatype extension="prettyseq" type="galaxy.datatypes.data:Text"/> - <datatype extension="primersearch" type="galaxy.datatypes.data:Text"/> - <datatype extension="regions" type="galaxy.datatypes.data:Text"/> - <datatype extension="score" type="galaxy.datatypes.data:Text"/> - <datatype extension="selex" type="galaxy.datatypes.data:Text"/> - <datatype extension="seqtable" type="galaxy.datatypes.data:Text"/> - <datatype extension="showfeat" type="galaxy.datatypes.data:Text"/> - <datatype extension="showorf" type="galaxy.datatypes.data:Text"/> - <datatype extension="simple" type="galaxy.datatypes.data:Text"/> - <datatype extension="sixpack" type="galaxy.datatypes.data:Text"/> - <datatype extension="srs" type="galaxy.datatypes.data:Text"/> - <datatype extension="srspair" type="galaxy.datatypes.data:Text"/> - <datatype extension="staden" type="galaxy.datatypes.data:Text"/> - <datatype extension="strider" type="galaxy.datatypes.data:Text"/> - <datatype extension="supermatcher" type="galaxy.datatypes.data:Text"/> - <datatype extension="swiss" type="galaxy.datatypes.data:Text"/> - <datatype extension="syco" type="galaxy.datatypes.data:Text"/> - <datatype extension="table" type="galaxy.datatypes.data:Text"/> - <datatype extension="textsearch" type="galaxy.datatypes.data:Text"/> - <datatype extension="vectorstrip" type="galaxy.datatypes.data:Text"/> - <datatype extension="wobble" type="galaxy.datatypes.data:Text"/> - <datatype extension="wordcount" type="galaxy.datatypes.data:Text"/> - <datatype extension="tagseq" type="galaxy.datatypes.data:Text"/> -  -  - <datatype extension="gg" type="galaxy.datatypes.genetics:GenomeGraphs"/> - <datatype extension="rgenetics" type="galaxy.datatypes.genetics:Rgenetics"/> -  - <datatype extension="lped" type="galaxy.datatypes.genetics:Lped" display_in_upload="true"/> -  - <datatype extension="pbed" type="galaxy.datatypes.genetics:Pbed" display_in_upload="true"/> -  - <datatype extension="eigenstratgeno" type="galaxy.datatypes.genetics:Eigenstratgeno"/> -  - <datatype extension="eigenstratpca" type="galaxy.datatypes.genetics:Eigenstratpca"/> -  - <datatype extension="fped" type="galaxy.datatypes.genetics:Fped"/> -  - <datatype extension="lmap" type="galaxy.datatypes.genetics:Lmap"/> -  - <datatype extension="fphe" type="galaxy.datatypes.genetics:Fphe"/> -  - <datatype extension="pphe" type="galaxy.datatypes.genetics:Pphe"/> - <datatype extension="snptest" type="galaxy.datatypes.genetics:Snptest"/> - <datatype extension="snpmatrix" type="galaxy.datatypes.genetics:SNPMatrix"/> - <datatype extension="xls" type="galaxy.datatypes.tabular:Tabular"/> -  - </registration> - <sniffers> -  - <sniffer type="galaxy.datatypes.xml:BlastXml"/> - <sniffer type="galaxy.datatypes.sequence:Maf"/> - <sniffer type="galaxy.datatypes.sequence:Lav"/> - <sniffer type="galaxy.datatypes.sequence:csFasta"/> - <sniffer type="galaxy.datatypes.qualityscore:QualityScoreSOLiD"/> - <sniffer type="galaxy.datatypes.qualityscore:QualityScore454"/> - <sniffer type="galaxy.datatypes.sequence:Fasta"/> - <sniffer type="galaxy.datatypes.sequence:Fastq"/> - <sniffer type="galaxy.datatypes.interval:Wiggle"/> - <sniffer type="galaxy.datatypes.images:Html"/> - <sniffer type="galaxy.datatypes.sequence:Axt"/> - <sniffer type="galaxy.datatypes.interval:Bed"/> - <sniffer type="galaxy.datatypes.interval:CustomTrack"/> - <sniffer type="galaxy.datatypes.interval:Gff"/> - <sniffer type="galaxy.datatypes.interval:Gff3"/> - <sniffer type="galaxy.datatypes.interval:Interval"/> - <sniffer type="galaxy.datatypes.tabular:Sam"/> - </sniffers> + <registration converters_path="lib/galaxy/datatypes/converters"> + <datatype extension="ab1" type="galaxy.datatypes.binary:Ab1" mimetype="application/octet-stream" display_in_upload="true"/> + <datatype extension="axt" type="galaxy.datatypes.sequence:Axt" display_in_upload="true"/> + <datatype extension="bam" type="galaxy.datatypes.binary:Bam" mimetype="application/octet-stream"/> + <datatype extension="bed" type="galaxy.datatypes.interval:Bed" display_in_upload="true"> + <converter file="bed_to_gff_converter.xml" target_datatype="gff"/> + <converter file="interval_to_coverage.xml" target_datatype="coverage"/> + <converter file="bed_to_interval_index_converter.xml" target_datatype="interval_index"/> + </datatype> + <datatype extension="binseq.zip" type="galaxy.datatypes.binary:Binseq" mimetype="application/zip" display_in_upload="true"/> + <datatype extension="len" type="galaxy.datatypes.chrominfo:ChromInfo" display_in_upload="true"> +  + </datatype> + <datatype extension="coverage" type="galaxy.datatypes.coverage:LastzCoverage" display_in_upload="true"> + <indexer file="coverage.xml" /> + </datatype> + <datatype extension="customtrack" type="galaxy.datatypes.interval:CustomTrack"/> + <datatype extension="csfasta" type="galaxy.datatypes.sequence:csFasta" display_in_upload="true"/> + <datatype extension="data" type="galaxy.datatypes.data:Data" mimetype="application/octet-stream"/> + <datatype extension="fasta" type="galaxy.datatypes.sequence:Fasta" display_in_upload="true"> + <converter file="fasta_to_tabular_converter.xml" target_datatype="tabular"/> + </datatype> + <datatype extension="fastq" type="galaxy.datatypes.sequence:Fastq" display_in_upload="true"/> + <datatype extension="fastqsanger" type="galaxy.datatypes.sequence:FastqSanger" display_in_upload="true"/> + <datatype extension="genetrack" type="galaxy.datatypes.tracks:GeneTrack"/> + <datatype extension="gff" type="galaxy.datatypes.interval:Gff" display_in_upload="true"> + <converter file="gff_to_bed_converter.xml" target_datatype="bed"/> + </datatype> + <datatype extension="gff3" type="galaxy.datatypes.interval:Gff3" display_in_upload="true"/> + <datatype extension="gif" type="galaxy.datatypes.images:Image" mimetype="image/gif"/> + <datatype extension="gmaj.zip" type="galaxy.datatypes.images:Gmaj" mimetype="application/zip"/> + <datatype extension="html" type="galaxy.datatypes.images:Html" mimetype="text/html"/> + <datatype extension="interval" type="galaxy.datatypes.interval:Interval" display_in_upload="true"> + <converter file="interval_to_bed_converter.xml" target_datatype="bed"/> + <indexer file="interval_awk.xml" /> + </datatype> + <datatype extension="jpg" type="galaxy.datatypes.images:Image" mimetype="image/jpeg"/> + <datatype extension="laj" type="galaxy.datatypes.images:Laj"/> + <datatype extension="lav" type="galaxy.datatypes.sequence:Lav" display_in_upload="true"/> + <datatype extension="maf" type="galaxy.datatypes.sequence:Maf" display_in_upload="true"> + <converter file="maf_to_fasta_converter.xml" target_datatype="fasta"/> + <converter file="maf_to_interval_converter.xml" target_datatype="interval"/> + </datatype> + <datatype extension="pdf" type="galaxy.datatypes.images:Image" mimetype="application/pdf"/> + <datatype extension="png" type="galaxy.datatypes.images:Image" mimetype="image/png"/> + <datatype extension="qualsolexa" type="galaxy.datatypes.qualityscore:QualityScoreSolexa" display_in_upload="true"/> + <datatype extension="qualsolid" type="galaxy.datatypes.qualityscore:QualityScoreSOLiD" display_in_upload="true"/> + <datatype extension="qual454" type="galaxy.datatypes.qualityscore:QualityScore454" display_in_upload="true"/> + <datatype extension="sam" type="galaxy.datatypes.tabular:Sam" display_in_upload="true"/> + <datatype extension="scf" type="galaxy.datatypes.binary:Scf" mimetype="application/octet-stream" display_in_upload="true"/> + <datatype extension="sff" type="galaxy.datatypes.binary:Sff" mimetype="application/octet-stream" display_in_upload="true"/> + <datatype extension="taxonomy" type="galaxy.datatypes.tabular:Taxonomy" display_in_upload="true"/> + <datatype extension="tabular" type="galaxy.datatypes.tabular:Tabular" display_in_upload="true"/> + <datatype extension="txt" type="galaxy.datatypes.data:Text" display_in_upload="true"/> + <datatype extension="blastxml" type="galaxy.datatypes.xml:BlastXml" display_in_upload="true"/> + <datatype extension="txtseq.zip" type="galaxy.datatypes.data:Txtseq" mimetype="application/zip" display_in_upload="true"/> + <datatype extension="wig" type="galaxy.datatypes.interval:Wiggle" display_in_upload="true"> + <converter file="wiggle_to_array_tree_converter.xml" target_datatype="array_tree"/> + </datatype> + <datatype extension="array_tree" type="galaxy.datatypes.data:Data" /> + <datatype extension="interval_index" type="galaxy.datatypes.data:Data" /> +  + <datatype extension="acedb" type="galaxy.datatypes.data:Text"/> + <datatype extension="asn1" type="galaxy.datatypes.data:Text"/> + <datatype extension="btwisted" type="galaxy.datatypes.data:Text"/> + <datatype extension="cai" type="galaxy.datatypes.data:Text"/> + <datatype extension="charge" type="galaxy.datatypes.data:Text"/> + <datatype extension="checktrans" type="galaxy.datatypes.data:Text"/> + <datatype extension="chips" type="galaxy.datatypes.data:Text"/> + <datatype extension="clustal" type="galaxy.datatypes.data:Text"/> + <datatype extension="codata" type="galaxy.datatypes.data:Text"/> + <datatype extension="codcmp" type="galaxy.datatypes.data:Text"/> + <datatype extension="coderet" type="galaxy.datatypes.data:Text"/> + <datatype extension="compseq" type="galaxy.datatypes.data:Text"/> + <datatype extension="cpgplot" type="galaxy.datatypes.data:Text"/> + <datatype extension="cpgreport" type="galaxy.datatypes.data:Text"/> + <datatype extension="cusp" type="galaxy.datatypes.data:Text"/> + <datatype extension="cut" type="galaxy.datatypes.data:Text"/> + <datatype extension="dan" type="galaxy.datatypes.data:Text"/> + <datatype extension="dbmotif" type="galaxy.datatypes.data:Text"/> + <datatype extension="diffseq" type="galaxy.datatypes.data:Text"/> + <datatype extension="digest" type="galaxy.datatypes.data:Text"/> + <datatype extension="dreg" type="galaxy.datatypes.data:Text"/> + <datatype extension="einverted" type="galaxy.datatypes.data:Text"/> + <datatype extension="embl" type="galaxy.datatypes.data:Text"/> + <datatype extension="epestfind" type="galaxy.datatypes.data:Text"/> + <datatype extension="equicktandem" type="galaxy.datatypes.data:Text"/> + <datatype extension="est2genome" type="galaxy.datatypes.data:Text"/> + <datatype extension="etandem" type="galaxy.datatypes.data:Text"/> + <datatype extension="excel" type="galaxy.datatypes.data:Text"/> + <datatype extension="feattable" type="galaxy.datatypes.data:Text"/> + <datatype extension="fitch" type="galaxy.datatypes.data:Text"/> + <datatype extension="freak" type="galaxy.datatypes.data:Text"/> + <datatype extension="fuzznuc" type="galaxy.datatypes.data:Text"/> + <datatype extension="fuzzpro" type="galaxy.datatypes.data:Text"/> + <datatype extension="fuzztran" type="galaxy.datatypes.data:Text"/> + <datatype extension="garnier" type="galaxy.datatypes.data:Text"/> + <datatype extension="gcg" type="galaxy.datatypes.data:Text"/> + <datatype extension="geecee" type="galaxy.datatypes.data:Text"/> + <datatype extension="genbank" type="galaxy.datatypes.data:Text"/> + <datatype extension="helixturnhelix" type="galaxy.datatypes.data:Text"/> + <datatype extension="hennig86" type="galaxy.datatypes.data:Text"/> + <datatype extension="hmoment" type="galaxy.datatypes.data:Text"/> + <datatype extension="ig" type="galaxy.datatypes.data:Text"/> + <datatype extension="isochore" type="galaxy.datatypes.data:Text"/> + <datatype extension="jackknifer" type="galaxy.datatypes.data:Text"/> + <datatype extension="jackknifernon" type="galaxy.datatypes.data:Text"/> + <datatype extension="markx10" type="galaxy.datatypes.data:Text"/> + <datatype extension="markx1" type="galaxy.datatypes.data:Text"/> + <datatype extension="markx0" type="galaxy.datatypes.data:Text"/> + <datatype extension="markx3" type="galaxy.datatypes.data:Text"/> + <datatype extension="markx2" type="galaxy.datatypes.data:Text"/> + <datatype extension="match" type="galaxy.datatypes.data:Text"/> + <datatype extension="mega" type="galaxy.datatypes.data:Text"/> + <datatype extension="meganon" type="galaxy.datatypes.data:Text"/> + <datatype extension="motif" type="galaxy.datatypes.data:Text"/> + <datatype extension="msf" type="galaxy.datatypes.data:Text"/> + <datatype extension="nametable" type="galaxy.datatypes.data:Text"/> + <datatype extension="ncbi" type="galaxy.datatypes.data:Text"/> + <datatype extension="needle" type="galaxy.datatypes.data:Text"/> + <datatype extension="newcpgreport" type="galaxy.datatypes.data:Text"/> + <datatype extension="newcpgseek" type="galaxy.datatypes.data:Text"/> + <datatype extension="nexus" type="galaxy.datatypes.data:Text"/> + <datatype extension="nexusnon" type="galaxy.datatypes.data:Text"/> + <datatype extension="noreturn" type="galaxy.datatypes.data:Text"/> + <datatype extension="pair" type="galaxy.datatypes.data:Text"/> + <datatype extension="palindrome" type="galaxy.datatypes.data:Text"/> + <datatype extension="pepcoil" type="galaxy.datatypes.data:Text"/> + <datatype extension="pepinfo" type="galaxy.datatypes.data:Text"/> + <datatype extension="pepstats" type="galaxy.datatypes.data:Text"/> + <datatype extension="phylip" type="galaxy.datatypes.data:Text"/> + <datatype extension="phylipnon" type="galaxy.datatypes.data:Text"/> + <datatype extension="pir" type="galaxy.datatypes.data:Text"/> + <datatype extension="polydot" type="galaxy.datatypes.data:Text"/> + <datatype extension="preg" type="galaxy.datatypes.data:Text"/> + <datatype extension="prettyseq" type="galaxy.datatypes.data:Text"/> + <datatype extension="primersearch" type="galaxy.datatypes.data:Text"/> + <datatype extension="regions" type="galaxy.datatypes.data:Text"/> + <datatype extension="score" type="galaxy.datatypes.data:Text"/> + <datatype extension="selex" type="galaxy.datatypes.data:Text"/> + <datatype extension="seqtable" type="galaxy.datatypes.data:Text"/> + <datatype extension="showfeat" type="galaxy.datatypes.data:Text"/> + <datatype extension="showorf" type="galaxy.datatypes.data:Text"/> + <datatype extension="simple" type="galaxy.datatypes.data:Text"/> + <datatype extension="sixpack" type="galaxy.datatypes.data:Text"/> + <datatype extension="srs" type="galaxy.datatypes.data:Text"/> + <datatype extension="srspair" type="galaxy.datatypes.data:Text"/> + <datatype extension="staden" type="galaxy.datatypes.data:Text"/> + <datatype extension="strider" type="galaxy.datatypes.data:Text"/> + <datatype extension="supermatcher" type="galaxy.datatypes.data:Text"/> + <datatype extension="swiss" type="galaxy.datatypes.data:Text"/> + <datatype extension="syco" type="galaxy.datatypes.data:Text"/> + <datatype extension="table" type="galaxy.datatypes.data:Text"/> + <datatype extension="textsearch" type="galaxy.datatypes.data:Text"/> + <datatype extension="vectorstrip" type="galaxy.datatypes.data:Text"/> + <datatype extension="wobble" type="galaxy.datatypes.data:Text"/> + <datatype extension="wordcount" type="galaxy.datatypes.data:Text"/> + <datatype extension="tagseq" type="galaxy.datatypes.data:Text"/> +  +  + <datatype extension="affybatch" type="galaxy.datatypes.genetics:Affybatch" display_in_upload="true"/> +  + <datatype extension="eigenstratgeno" type="galaxy.datatypes.genetics:Eigenstratgeno"/> +  + <datatype extension="eigenstratpca" type="galaxy.datatypes.genetics:Eigenstratpca"/> + <datatype extension="eset" type="galaxy.datatypes.genetics:Eset" display_in_upload="true" /> +  + <datatype extension="fped" type="galaxy.datatypes.genetics:Fped" display_in_upload="true"/> +  + <datatype extension="fphe" type="galaxy.datatypes.genetics:Fphe" display_in_upload="true" mimetype="text/html"/> +  + <datatype extension="gg" type="galaxy.datatypes.genetics:GenomeGraphs"/> +  + <datatype extension="lmap" type="galaxy.datatypes.genetics:Lmap" display_in_upload="true"/> + <datatype extension="malist" type="galaxy.datatypes.genetics:MAlist" display_in_upload="true"/> +  + <datatype extension="lped" type="galaxy.datatypes.genetics:Lped" display_in_upload="true"> + <converter file="lped_to_fped_converter.xml" target_datatype="fped"/> + <converter file="lped_to_pbed_converter.xml" target_datatype="pbed"/> + </datatype> +  + <datatype extension="pbed" type="galaxy.datatypes.genetics:Pbed" display_in_upload="true"> + <converter file="pbed_to_lped_converter.xml" target_datatype="lped"/> + </datatype> + <datatype extension="pheno" type="galaxy.datatypes.genetics:Pheno"/> +  + <datatype extension="pphe" type="galaxy.datatypes.genetics:Pphe" display_in_upload="true" mimetype="text/html"/> + <datatype extension="rexpbase" type="galaxy.datatypes.genetics:RexpBase"/> + <datatype extension="rgenetics" type="galaxy.datatypes.genetics:Rgenetics"/> + <datatype extension="snptest" type="galaxy.datatypes.genetics:Snptest" display_in_upload="true"/> + <datatype extension="snpmatrix" type="galaxy.datatypes.genetics:SNPMatrix" display_in_upload="true"/> + <datatype extension="xls" type="galaxy.datatypes.tabular:Tabular"/> +  + </registration> + <sniffers> +  + <sniffer type="galaxy.datatypes.binary:Sff"/> + <sniffer type="galaxy.datatypes.xml:BlastXml"/> + <sniffer type="galaxy.datatypes.sequence:Maf"/> + <sniffer type="galaxy.datatypes.sequence:Lav"/> + <sniffer type="galaxy.datatypes.sequence:csFasta"/> + <sniffer type="galaxy.datatypes.qualityscore:QualityScoreSOLiD"/> + <sniffer type="galaxy.datatypes.qualityscore:QualityScore454"/> + <sniffer type="galaxy.datatypes.sequence:Fasta"/> + <sniffer type="galaxy.datatypes.sequence:Fastq"/> + <sniffer type="galaxy.datatypes.interval:Wiggle"/> + <sniffer type="galaxy.datatypes.images:Html"/> + <sniffer type="galaxy.datatypes.sequence:Axt"/> + <sniffer type="galaxy.datatypes.interval:Bed"/> + <sniffer type="galaxy.datatypes.interval:CustomTrack"/> + <sniffer type="galaxy.datatypes.interval:Gff"/> + <sniffer type="galaxy.datatypes.interval:Gff3"/> + <sniffer type="galaxy.datatypes.interval:Interval"/> + <sniffer type="galaxy.datatypes.tabular:Sam"/> + </sniffers> </datatypes> diff -r 555afd0bf457 -r 37406d8ad116 dist-eggs.ini --- a/dist-eggs.ini Tue Nov 17 16:14:54 2009 -0500 +++ b/dist-eggs.ini Tue Nov 17 16:16:26 2009 -0500 @@ -23,6 +23,9 @@ py2.5-macosx-10.3-fat-ucs2 = medeski.bx.psu.edu /usr/local/bin/python2.5 py2.6-macosx-10.3-fat-ucs2 = medeski.bx.psu.edu /usr/local/bin/python2.6 py2.5-macosx-10.5-i386-ucs2 = lion.bx.psu.edu /usr/bin/python2.5 +py2.4-solaris-2.10-i86pc-ucs2 = thumper.bx.psu.edu /depot/projects/pythons/solaris-2.10-i86pc-ucs2/bin/python2.4 +py2.5-solaris-2.10-i86pc-ucs2 = thumper.bx.psu.edu /depot/projects/pythons/solaris-2.10-i86pc-ucs2/bin/python2.5 +py2.6-solaris-2.10-i86pc-ucs2 = thumper.bx.psu.edu /depot/projects/pythons/solaris-2.10-i86pc-ucs2/bin/python2.6 py2.4-solaris-2.11-i86pc-ucs2 = victory.bx.psu.edu /depot/projects/pythons/solaris-2.11-i86pc-ucs2/bin/python2.4 py2.5-solaris-2.11-i86pc-ucs2 = victory.bx.psu.edu /depot/projects/pythons/solaris-2.11-i86pc-ucs2/bin/python2.5 py2.6-solaris-2.11-i86pc-ucs2 = victory.bx.psu.edu /depot/projects/pythons/solaris-2.11-i86pc-ucs2/bin/python2.6 @@ -47,9 +50,9 @@ py2.5-macosx = py2.5-macosx-10.3-fat-ucs2 py2.5-macosx-10.5-i386-ucs2 py2.6-macosx = py2.6-macosx-10.3-fat-ucs2 macosx = py2.4-macosx py2.5-macosx py2.6-macosx -py2.4-solaris = py2.4-solaris-2.11-i86pc-ucs2 py2.4-solaris-2.10-sun4u-ucs2 -py2.5-solaris = py2.5-solaris-2.11-i86pc-ucs2 py2.5-solaris-2.10-sun4u-ucs2 -py2.6-solaris = py2.6-solaris-2.11-i86pc-ucs2 py2.6-solaris-2.10-sun4u-ucs2 +py2.4-solaris = py2.4-solaris-2.10-i86pc-ucs2 py2.4-solaris-2.11-i86pc-ucs2 py2.4-solaris-2.10-sun4u-ucs2 +py2.5-solaris = py2.5-solaris-2.10-i86pc-ucs2 py2.5-solaris-2.11-i86pc-ucs2 py2.5-solaris-2.10-sun4u-ucs2 +py2.6-solaris = py2.6-solaris-2.10-i86pc-ucs2 py2.6-solaris-2.11-i86pc-ucs2 py2.6-solaris-2.10-sun4u-ucs2 solaris = py2.4-solaris py2.5-solaris py2.6-solaris py2.4-all = py2.4-linux py2.4-macosx py2.4-solaris py2.5-all = py2.5-linux py2.5-macosx py2.5-solaris diff -r 555afd0bf457 -r 37406d8ad116 eggs.ini --- a/eggs.ini Tue Nov 17 16:14:54 2009 -0500 +++ b/eggs.ini Tue Nov 17 16:16:26 2009 -0500 @@ -19,7 +19,7 @@ pbs_python = 2.9.4 psycopg2 = 2.0.6 pycrypto = 2.0.1 -pysqlite = 2.3.5 +pysqlite = 2.5.6 python_lzo = 1.08 threadframe = 0.2 guppy = 0.1.8 @@ -57,11 +57,12 @@ ; extra version information [tags] psycopg2 = _8.2.6_static -pysqlite = _3.5.4_static +pysqlite = _static MySQL_python = _5.0.67_static python_lzo = _static bx_python = _dev_r4bf1f32e6b76 -GeneTrack = _dev_raa786e9fc131d998e532a1aef39d108850c9e93d +GeneTrack = _dev_e380f21c704218622155b9d230a44b3c9c452524 +SQLAlchemy = _dev_r6498 ; nose = .dev_r7156749efc58 ; source location, necessary for scrambling @@ -73,7 +74,7 @@ pbs_python = http://ftp.sara.nl/pub/outgoing/pbs_python-2.9.4.tar.gz psycopg2 = http://initd.org/pub/software/psycopg/PSYCOPG-2-0/psycopg2-2.0.6.tar.gz ftp://ftp-archives.postgresql.org/pub/source/v8.2.6/postgresql-8.2.6.tar.bz2 pycrypto = http://www.amk.ca/files/python/crypto/pycrypto-2.0.1.tar.gz -pysqlite = http://initd.org/pub/software/pysqlite/releases/2.3/2.3.5/pysqlite-2.3.5.ta… http://www.sqlite.org/sqlite-source-3_5_4.zip +pysqlite = http://pypi.python.org/packages/source/p/pysqlite/pysqlite-2.5.6.tar.gz python_lzo = http://www.oberhumer.com/opensource/lzo/download/LZO-v1/python-lzo-1.08.tar… http://www.oberhumer.com/opensource/lzo/download/LZO-v1/lzo-1.08.tar.gz threadframe = http://www.majid.info/python/threadframe/threadframe-0.2.tar.gz guppy = http://pypi.python.org/packages/source/g/guppy/guppy-0.1.8.tar.gz @@ -82,7 +83,7 @@ decorator = http://pypi.python.org/packages/source/d/decorator/decorator-3.1.2.tar.gz docutils = http://downloads.sourceforge.net/docutils/docutils-0.4.tar.gz elementtree = http://effbot.org/downloads/elementtree-1.2.6-20050316.tar.gz -GeneTrack = http://github.com/ialbert/genetrack-central/tarball/aa786e9fc131d998e532a1a… +GeneTrack = http://github.com/ialbert/genetrack-central/tarball/e380f21c704218622155b9d… lrucache = http://evan.prodromou.name/lrucache/lrucache-0.2.tar.gz Mako = http://www.makotemplates.org/downloads/Mako-0.2.5.tar.gz nose = http://pypi.python.org/packages/source/n/nose/nose-0.11.1.tar.gz @@ -93,7 +94,7 @@ PSI = http://pypi.python.org/packages/source/P/PSI/PSI-0.3b1.1.tar.gz Routes = http://pypi.python.org/packages/source/R/Routes/Routes-1.11.tar.gz simplejson = http://cheeseshop.python.org/packages/source/s/simplejson/simplejson-1.5.ta… -SQLAlchemy = http://pypi.python.org/packages/source/S/SQLAlchemy/SQLAlchemy-0.5.6.tar.gz +SQLAlchemy = http://dist.g2.bx.psu.edu/SQLAlchemy-0.5.6_r6498.tar.bz2 sqlalchemy_migrate = http://pypi.python.org/packages/source/s/sqlalchemy-migrate/sqlalchemy-migr… Tempita = http://pypi.python.org/packages/source/T/Tempita/Tempita-0.1.tar.gz twill = http://darcs.idyll.org/~t/projects/twill-0.9.tar.gz diff -r 555afd0bf457 -r 37406d8ad116 lib/galaxy/datatypes/binary.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/lib/galaxy/datatypes/binary.py Tue Nov 17 16:16:26 2009 -0500 @@ -0,0 +1,156 @@ +""" +Binary classes +""" + +import data, logging, binascii +from galaxy.datatypes.metadata import MetadataElement +from galaxy.datatypes import metadata +from galaxy.datatypes.sniff import * +from urllib import urlencode, quote_plus +import zipfile +import os, subprocess, tempfile + +log = logging.getLogger(__name__) + +sniffable_binary_formats = [ 'sff' ] +# Currently these supported binary data types must be manually set on upload +unsniffable_binary_formats = [ 'ab1', 'scf' ] + +class Binary( data.Data ): + """Binary data""" + def set_peek( self, dataset ): + """Set the peek and blurb text""" + if not dataset.dataset.purged: + dataset.peek = 'binary data' + dataset.blurb = 'data' + else: + dataset.peek = 'file does not exist' + dataset.blurb = 'file purged from disk' + +class Ab1( Binary ): + """Class describing an ab1 binary sequence file""" + file_ext = "ab1" + def set_peek( self, dataset ): + if not dataset.dataset.purged: + export_url = "/history_add_to?" + urlencode( {'history_id':dataset.history_id,'ext':'ab1','name':'ab1 sequence','info':'Sequence file','dbkey':dataset.dbkey} ) + dataset.peek = "Binary ab1 sequence file" + dataset.blurb = data.nice_size( dataset.get_size() ) + else: + dataset.peek = 'file does not exist' + dataset.blurb = 'file purged from disk' + def display_peek( self, dataset ): + try: + return dataset.peek + except: + return "Binary ab1 sequence file (%s)" % ( data.nice_size( dataset.get_size() ) ) + +class Bam( Binary ): + """Class describing a BAM binary file""" + file_ext = "bam" + MetadataElement( name="bam_index", desc="BAM Index File", param=metadata.FileParameter, readonly=True, no_value=None, visible=False, optional=True ) + def init_meta( self, dataset, copy_from=None ): + Binary.init_meta( self, dataset, copy_from=copy_from ) + def set_meta( self, dataset, overwrite = True, **kwd ): + """ + Sets index for BAM file. + """ + index_file = dataset.metadata.bam_index + if not index_file: + index_file = dataset.metadata.spec['bam_index'].param.new_file( dataset = dataset ) + tmp_dir = tempfile.gettempdir() + tmpf1 = tempfile.NamedTemporaryFile( dir=tmp_dir ) + tmpf1bai = '%s.bai' % tmpf1.name + try: + os.system( 'cd %s' % tmp_dir ) + os.system( 'cp %s %s' % ( dataset.file_name, tmpf1.name ) ) + os.system( 'samtools index %s' % tmpf1.name ) + os.system( 'cp %s %s' % ( tmpf1bai, index_file.file_name ) ) + except Exception, ex: + sys.stderr.write( 'There was a problem creating the index for the BAM file\n%s\n' + str( ex ) ) + tmpf1.close() + if os.path.exists( tmpf1bai ): + os.remove( tmpf1bai ) + dataset.metadata.bam_index = index_file + def set_peek( self, dataset ): + if not dataset.dataset.purged: + export_url = "/history_add_to?" + urlencode( {'history_id':dataset.history_id,'ext':'bam','name':'bam alignments','info':'Alignments file','dbkey':dataset.dbkey} ) + dataset.peek = "Binary bam alignments file" + dataset.blurb = data.nice_size( dataset.get_size() ) + else: + dataset.peek = 'file does not exist' + dataset.blurb = 'file purged from disk' + def display_peek( self, dataset ): + try: + return dataset.peek + except: + return "Binary bam alignments file (%s)" % ( data.nice_size( dataset.get_size() ) ) + def get_mime( self ): + """Returns the mime type of the datatype""" + return 'application/octet-stream' + +class Binseq( Binary ): + """Class describing a zip archive of binary sequence files""" + file_ext = "binseq.zip" + def set_peek( self, dataset ): + if not dataset.dataset.purged: + zip_file = zipfile.ZipFile( dataset.file_name, "r" ) + num_files = len( zip_file.namelist() ) + dataset.peek = "Archive of %s binary sequence files" % ( str( num_files ) ) + dataset.blurb = data.nice_size( dataset.get_size() ) + else: + dataset.peek = 'file does not exist' + dataset.blurb = 'file purged from disk' + def display_peek( self, dataset ): + try: + return dataset.peek + except: + return "Binary sequence file archive (%s)" % ( data.nice_size( dataset.get_size() ) ) + def get_mime( self ): + """Returns the mime type of the datatype""" + return 'application/zip' + +class Scf( Binary ): + """Class describing an scf binary sequence file""" + file_ext = "scf" + def set_peek( self, dataset ): + if not dataset.dataset.purged: + export_url = "/history_add_to?" + urlencode({'history_id':dataset.history_id,'ext':'scf','name':'scf sequence','info':'Sequence file','dbkey':dataset.dbkey}) + dataset.peek = "Binary scf sequence file" + dataset.blurb = data.nice_size( dataset.get_size() ) + else: + dataset.peek = 'file does not exist' + dataset.blurb = 'file purged from disk' + def display_peek( self, dataset ): + try: + return dataset.peek + except: + return "Binary scf sequence file (%s)" % ( data.nice_size( dataset.get_size() ) ) + +class Sff( Binary ): + """ Standard Flowgram Format (SFF) """ + file_ext = "sff" + def __init__( self, **kwd ): + Binary.__init__( self, **kwd ) + def sniff( self, filename ): + # The first 4 bytes of any sff file is '.sff', and the file is binary. For details + # about the format, see http://www.ncbi.nlm.nih.gov/Traces/trace.cgi?cmd=show&f=formats&m=doc&s=for… + try: + header = open( filename ).read(4) + if binascii.b2a_hex( header ) == binascii.hexlify( '.sff' ): + return True + return False + except Exception, e: + return False + def set_peek( self, dataset ): + if not dataset.dataset.purged: + export_url = "/history_add_to?" + urlencode( {'history_id':dataset.history_id,'ext':'sff','name':'sff file','info':'sff file','dbkey':dataset.dbkey} ) + dataset.peek = "Binary sff file" + dataset.blurb = data.nice_size( dataset.get_size() ) + else: + dataset.peek = 'file does not exist' + dataset.blurb = 'file purged from disk' + def display_peek( self, dataset ): + try: + return dataset.peek + except: + return "Binary sff file (%s)" % ( data.nice_size( dataset.get_size() ) ) diff -r 555afd0bf457 -r 37406d8ad116 lib/galaxy/datatypes/data.py --- a/lib/galaxy/datatypes/data.py Tue Nov 17 16:14:54 2009 -0500 +++ b/lib/galaxy/datatypes/data.py Tue Nov 17 16:16:26 2009 -0500 @@ -1,4 +1,4 @@ -import logging, os, sys, time, tempfile, binascii +import logging, os, sys, time, tempfile from galaxy import util from galaxy.util.odict import odict from galaxy.util.bunch import Bunch @@ -40,20 +40,18 @@ """ __metaclass__ = DataMeta - - """Add metadata elements""" + # Add metadata elements MetadataElement( name="dbkey", desc="Database/Build", default="?", param=metadata.DBKeyParameter, multiple=False, no_value="?" ) - - """Stores the set of display applications, and viewing methods, supported by this datatype """ + # Stores the set of display applications, and viewing methods, supported by this datatype supported_display_apps = {} - - """If False, the peek is regenerated whenever a dataset of this type is copied""" + # If False, the peek is regenerated whenever a dataset of this type is copied copy_safe_peek = True - - is_binary = True #The dataset contains binary data --> do not space_to_tab or convert newlines, etc. Allow binary file uploads of this type when True. - - allow_datatype_change = True #Allow user to change between this datatype and others. If False, this datatype cannot be changed from or into. - + # The dataset contains binary data --> do not space_to_tab or convert newlines, etc. + # Allow binary file uploads of this type when True. + is_binary = True + # Allow user to change between this datatype and others. If False, this datatype + # cannot be changed from or into. + allow_datatype_change = True #Composite datatypes composite_type = None composite_files = odict() @@ -162,6 +160,11 @@ info = info.replace( '\r', '<br/>' ) if info.find( '\n' ) >= 0: info = info.replace( '\n', '<br/>' ) + + # Convert to unicode to display non-ascii characters. + if type( info ) is not unicode: + info = unicode( info, 'utf-8') + return info except: return "info unavailable" @@ -270,8 +273,6 @@ def add_composite_file( self, name, **kwds ): #self.composite_files = self.composite_files.copy() self.composite_files[ name ] = self.__new_composite_file( name, **kwds ) - - def __substitute_composite_key( self, key, composite_file, dataset = None ): if composite_file.substitute_name_with_metadata: if dataset: @@ -303,7 +304,6 @@ return files def generate_auto_primary_file( self, dataset = None ): raise Exception( "generate_auto_primary_file is not implemented for this datatype." ) - @property def has_resolution(self): return False @@ -364,23 +364,37 @@ dataset.peek = 'file does not exist' dataset.blurb = 'file purged from disk' -class Binary( Data ): - """Binary data""" +class Txtseq( Data ): + """Class describing a zip archive of text sequence files""" + file_ext = "txtseq.zip" def set_peek( self, dataset ): - """Set the peek and blurb text""" if not dataset.dataset.purged: - dataset.peek = 'binary data' - dataset.blurb = 'data' + zip_file = zipfile.ZipFile( dataset.file_name, "r" ) + num_files = len( zip_file.namelist() ) + dataset.peek = "Archive of %s text sequence files" % ( str( num_files ) ) + dataset.blurb = data.nice_size( dataset.get_size() ) else: dataset.peek = 'file does not exist' dataset.blurb = 'file purged from disk' + def display_peek(self, dataset): + try: + return dataset.peek + except: + return "Text sequence file archive (%s)" % ( data.nice_size( dataset.get_size() ) ) + def get_mime(self): + """Returns the mime type of the datatype""" + return 'application/zip' + +class Newick( Text ): + pass + +# ------------- Utility methods -------------- def get_test_fname( fname ): """Returns test data filename""" path, name = os.path.split(__file__) full_path = os.path.join( path, 'test', fname ) return full_path - def nice_size(size): """ Returns a readably formatted string with the size @@ -406,7 +420,6 @@ out = "%.1f %s" % (size, word) return out return '??? bytes' - def get_file_peek( file_name, is_multi_byte=False, WIDTH=256, LINE_COUNT=5 ): """ Returns the first LINE_COUNT lines wrapped to WIDTH @@ -443,7 +456,6 @@ else: text = unicode( '\n'.join( lines ), 'utf-8' ) return text - def get_line_count(file_name): """Returns the number of lines in a file that are neither null nor comments""" count = 0 @@ -452,38 +464,3 @@ if line and line[0] != '#': count += 1 return count - -class Newick( Text ): - pass - -class Sff( Binary ): - """ Standard Flowgram Format (SFF) """ - file_ext = "sff" - def __init__( self, **kwd ): - Binary.__init__(self, **kwd) - def init_meta( self, dataset, copy_from=None ): - Binary.init_meta( self, dataset, copy_from=copy_from ) - def sniff( self, filename ): - ''' - The first 4 bytes of any sff file is '.sff' - - >>> fname = get_test_fname( '1.sff' ) - >>> Sff().sniff( fname ) - True - ''' - header = open( filename ).read(4) - if binascii.b2a_hex( header ) == binascii.hexlify( '.sff' ): - return True - return False - def set_peek( self, dataset ): - if not dataset.dataset.purged: - dataset.peek = "Binary sff file" - dataset.blurb = nice_size( dataset.get_size() ) - else: - dataset.peek = 'file does not exist' - dataset.blurb = 'file purged from disk' - def display_peek(self, dataset): - try: - return dataset.peek - except: - return "sff file (%s)" % ( nice_size( dataset.get_size() ) ) diff -r 555afd0bf457 -r 37406d8ad116 lib/galaxy/datatypes/genetics.py --- a/lib/galaxy/datatypes/genetics.py Tue Nov 17 16:14:54 2009 -0500 +++ b/lib/galaxy/datatypes/genetics.py Tue Nov 17 16:16:26 2009 -0500 @@ -1,6 +1,5 @@ """ rgenetics datatypes -Use at your peril Ross Lazarus for the rgenetics and galaxy projects @@ -11,7 +10,6 @@ ross lazarus for rgenetics august 20 2007 """ - import logging, os, sys, time, tempfile, shutil, string, glob import data from galaxy import util @@ -26,8 +24,7 @@ from galaxy.datatypes.interval import Interval from galaxy.util.hash_util import * -gal_Log = logging.getLogger(__name__) -verbose = False +log = logging.getLogger(__name__) class GenomeGraphs(Interval): @@ -48,10 +45,8 @@ """Initialize datatype, by adding GBrowse display app""" Interval.__init__(self, **kwd) self.add_display_app ( 'ucsc', 'display at UCSC', 'as_ucsc_display_file', 'ucsc_links' ) - def as_ucsc_display_file( self, dataset, **kwd ): return open( dataset.file_name ) - def set_meta( self, dataset, overwrite = True, **kwd ): i = 0 for i, line in enumerate( file ( dataset.file_name ) ): @@ -66,7 +61,6 @@ except: pass Interval.set_meta( self, dataset, overwrite = overwrite, skip = i ) - def make_html_table( self, dataset, skipchars=[] ): """Create HTML table, used for displaying peek""" out = ['<table cellspacing="0" cellpadding="3">'] @@ -82,7 +76,6 @@ except Exception, exc: out = "Can't create peek %s" % exc return out - def get_estimated_display_viewport( self, dataset ): """ Return a chrom, start, stop tuple for viewing a file. There are slight differences between gff 2 and gff 3 @@ -118,7 +111,6 @@ return ( seqid, str( start ), str( stop ) ) else: return ( '', '', '' ) - def gbrowse_links( self, dataset, type, app, base_url ): ret_val = [] if dataset.has_data: @@ -132,7 +124,6 @@ link = "%s?start=%s&stop=%s&ref=%s&dbkey=%s" % ( site_url, start, stop, seqid, dataset.dbkey ) ret_val.append( ( site_name, link ) ) return ret_val - def ucsc_links( self, dataset, type, app, base_url ): ret_val = [] if dataset.has_data: @@ -160,10 +151,8 @@ link = '%s?redirect_url=%s&display_url=%s' % ( internal_url, redirect_url, display_url ) ret_val.append( (site_name, link) ) else: - gal_Log.debug('@@@ gg ucsc_links - no viewport_tuple') + log.debug('@@@ gg ucsc_links - no viewport_tuple') return ret_val - - def sniff( self, filename ): """ Determines whether the file is in gff format @@ -202,20 +191,17 @@ except: return False - - class rgTabList(Tabular): - """ for sampleid and for featureid lists of exclusions or inclusions in the clean tool + """ + for sampleid and for featureid lists of exclusions or inclusions in the clean tool featureid subsets on statistical criteria -> specialized display such as gg """ file_ext = "rgTList" - def __init__(self, **kwd): """Initialize featurelistt datatype""" Tabular.__init__( self, **kwd ) self.column_names = [] - def make_html_table( self, dataset, skipchars=[] ): """Create HTML table, used for displaying peek""" out = ['<table cellspacing="0" cellpadding="3">'] @@ -236,23 +222,24 @@ out = "Can't create peek %s" % exc return out - class rgSampleList(rgTabList): - """ for sampleid exclusions or inclusions in the clean tool - output from QC eg excess het, gender error, ibd pair member,eigen outlier,excess mendel errors,... - since they can be uploaded, should be flexible - but they are persistent at least - same infrastructure for expression? + """ + for sampleid exclusions or inclusions in the clean tool + output from QC eg excess het, gender error, ibd pair member,eigen outlier,excess mendel errors,... + since they can be uploaded, should be flexible + but they are persistent at least + same infrastructure for expression? """ file_ext = "rgSList" def __init__(self, **kwd): - """Initialize samplelist datatype""" + """ + Initialize samplelist datatype + """ rgTabList.__init__( self, **kwd ) self.column_names[0] = 'FID' self.column_names[1] = 'IID' # this is what Plink wants as at 2009 - def sniff(self,filename): """ """ @@ -264,10 +251,11 @@ return False class rgFeatureList( rgTabList ): - """ for featureid lists of exclusions or inclusions in the clean tool - output from QC eg low maf, high missingness, bad hwe in controls, excess mendel errors,... - featureid subsets on statistical criteria -> specialized display such as gg - same infrastructure for expression? + """ + for featureid lists of exclusions or inclusions in the clean tool + output from QC eg low maf, high missingness, bad hwe in controls, excess mendel errors,... + featureid subsets on statistical criteria -> specialized display such as gg + same infrastructure for expression? """ file_ext = "rgFList" @@ -276,26 +264,23 @@ rgTabList.__init__( self, **kwd ) for i,s in enumerate(['#FeatureId', 'Chr', 'Genpos', 'Mappos']): self.column_names[i] = s - class Rgenetics(Html): - """class to use for rgenetics""" - - MetadataElement( name="base_name", desc="base name for all transformed versions of this genetic dataset", default="rgenetics", - readonly=True, set_in_upload=True) + """ + class to use for rgenetics + """ + MetadataElement( name="base_name", desc="base name for all transformed versions of this genetic dataset", default="rgenetics", readonly=True, set_in_upload=True) composite_type = 'auto_primary_file' allow_datatype_change = False file_ext = 'rgenetics' - def missing_meta( self, dataset=None, **kwargs): """Checks for empty meta values""" for key, value in dataset.metadata.items(): if not value: return True return False - def generate_primary_file( self, dataset = None ): rval = ['<html><head><title>Rgenetics Galaxy Composite Dataset </title></head><p/>'] rval.append('<div>This composite dataset is composed of the following files:<p/><ul>') @@ -306,9 +291,9 @@ rval.append( '<li><a href="%s" type="application/binary">%s</a>%s' % ( composite_name, composite_name, opt_text ) ) rval.append( '</ul></div></html>' ) return "\n".join( rval ) - def regenerate_primary_file(self,dataset): - """cannot do this until we are setting metadata + """ + cannot do this until we are setting metadata """ def fix(oldpath,newbase): old,e = os.path.splitext(oldpath) @@ -332,30 +317,25 @@ f.write("\n".join( rval )) f.write('\n') f.close() - def set_meta( self, dataset, **kwd ): - - """for lped/pbed eg - + """ + for lped/pbed eg """ if kwd.get('overwrite') == False: - if verbose: - gal_Log.debug('@@@ rgenetics set_meta called with overwrite = False') + #log.debug('@@@ rgenetics set_meta called with overwrite = False') return True try: efp = dataset.extra_files_path except: - if verbose: - gal_Log.debug('@@@rgenetics set_meta failed %s - dataset %s has no efp ?' % (sys.exc_info()[0], dataset.name)) + #log.debug('@@@rgenetics set_meta failed %s - dataset %s has no efp ?' % (sys.exc_info()[0], dataset.name)) return False try: - flist = os.listdir(efp) - except: - if verbose: gal_Log.debug('@@@rgenetics set_meta failed %s - dataset %s has no efp ?' % (sys.exc_info()[0],dataset.name)) + flist = os.listdir(efp) + except: + #log.debug('@@@rgenetics set_meta failed %s - dataset %s has no efp ?' % (sys.exc_info()[0],dataset.name)) return False if len(flist) == 0: - if verbose: - gal_Log.debug('@@@rgenetics set_meta failed - %s efp %s is empty?' % (dataset.name,efp)) + #log.debug('@@@rgenetics set_meta failed - %s efp %s is empty?' % (dataset.name,efp)) return False bn = None for f in flist: @@ -372,9 +352,9 @@ dataset.blurb = 'Composite file - Rgenetics Galaxy toolkit' return True - class SNPMatrix(Rgenetics): - """fake class to distinguish different species of Rgenetics data collections + """ + fake class to distinguish different species of Rgenetics data collections """ file_ext="snpmatrix" @@ -385,9 +365,9 @@ else: dataset.peek = 'file does not exist' dataset.blurb = 'file purged from disk' - def sniff(self,filename): - """ need to check the file header hex code + """ + need to check the file header hex code """ infile = open(dataset.file_name, "b") head = infile.read(16) @@ -397,9 +377,9 @@ else: return True - class Lped(Rgenetics): - """fake class to distinguish different species of Rgenetics data collections + """ + fake class to distinguish different species of Rgenetics data collections """ file_ext="lped" @@ -408,9 +388,9 @@ self.add_composite_file( '%s.ped', description = 'Pedigree File', substitute_name_with_metadata = 'base_name', is_binary = True ) self.add_composite_file( '%s.map', description = 'Map File', substitute_name_with_metadata = 'base_name', is_binary = True ) - class Pphe(Rgenetics): - """fake class to distinguish different species of Rgenetics data collections + """ + fake class to distinguish different species of Rgenetics data collections """ file_ext="pphe" @@ -418,14 +398,15 @@ Rgenetics.__init__(self, **kwd) self.add_composite_file( '%s.pphe', description = 'Plink Phenotype File', substitute_name_with_metadata = 'base_name' ) - class Lmap(Rgenetics): - """fake class to distinguish different species of Rgenetics data collections + """ + fake class to distinguish different species of Rgenetics data collections """ file_ext="lmap" class Fphe(Rgenetics): - """fake class to distinguish different species of Rgenetics data collections + """ + fake class to distinguish different species of Rgenetics data collections """ file_ext="fphe" @@ -434,7 +415,8 @@ self.add_composite_file( '%s.fphe', description = 'FBAT Phenotype File', substitute_name_with_metadata = 'base_name' ) class Phe(Rgenetics): - """fake class to distinguish different species of Rgenetics data collections + """ + fake class to distinguish different species of Rgenetics data collections """ file_ext="phe" @@ -442,10 +424,9 @@ Rgenetics.__init__(self, **kwd) self.add_composite_file( '%s.phe', description = 'Phenotype File', substitute_name_with_metadata = 'base_name' ) - - class Fped(Rgenetics): - """fake class to distinguish different species of Rgenetics data collections + """ + fake class to distinguish different species of Rgenetics data collections """ file_ext="fped" @@ -453,9 +434,9 @@ Rgenetics.__init__(self, **kwd) self.add_composite_file( '%s.fped', description = 'FBAT format pedfile', substitute_name_with_metadata = 'base_name' ) - class Pbed(Rgenetics): - """fake class to distinguish different species of Rgenetics data collections + """ + fake class to distinguish different species of Rgenetics data collections """ file_ext="pbed" @@ -466,7 +447,8 @@ self.add_composite_file( '%s.fam', substitute_name_with_metadata = 'base_name', is_binary = True ) class Eigenstratgeno(Rgenetics): - """fake class to distinguish different species of Rgenetics data collections + """ + fake class to distinguish different species of Rgenetics data collections """ file_ext="eigenstratgeno" @@ -475,11 +457,10 @@ self.add_composite_file( '%s.eigenstratgeno', substitute_name_with_metadata = 'base_name', is_binary = True ) self.add_composite_file( '%s.ind', substitute_name_with_metadata = 'base_name', is_binary = True ) self.add_composite_file( '%s.map', substitute_name_with_metadata = 'base_name', is_binary = True ) - - class Eigenstratpca(Rgenetics): - """fake class to distinguish different species of Rgenetics data collections + """ + fake class to distinguish different species of Rgenetics data collections """ file_ext="eigenstratpca" @@ -487,22 +468,21 @@ Rgenetics.__init__(self, **kwd) self.add_composite_file( '%s.eigenstratpca', description = 'Eigenstrat PCA file', substitute_name_with_metadata = 'base_name' ) - class Snptest(Rgenetics): - """fake class to distinguish different species of Rgenetics data collections + """ + fake class to distinguish different species of Rgenetics data collections """ file_ext="snptest" - class Pheno(Tabular): """ base class for pheno files """ file_ext = 'pheno' - class RexpBase( Html ): - """base class for BioC data structures in Galaxy + """ + base class for BioC data structures in Galaxy must be constructed with the pheno data in place since that goes into the metadata for each instance """ @@ -518,18 +498,16 @@ composite_type = 'auto_primary_file' allow_datatype_change = False - def __init__( self, **kwd ): Html.__init__(self,**kwd) self.add_composite_file( '%s.pheno', description = 'Phenodata tab text file', substitute_name_with_metadata = 'base_name', is_binary=True) - def generate_primary_file( self, dataset = None ): - """ This is called only at upload to write the html file + """ + This is called only at upload to write the html file cannot rename the datasets here - they come with the default unfortunately """ return '<html><head></head><body>AutoGenerated Primary File for Composite Dataset</body></html>' - def get_phecols(self, phenolist=[], maxConc=20): """ sept 2009: cannot use whitespace to split - make a more complex structure here @@ -555,7 +533,7 @@ else: for col,code in enumerate(row): # keep column order correct if col >= totcols: - gal_Log.warning('### get_phecols error in pheno file - row %d col %d (%s) longer than header %s' % (nrows, col, row, head)) + log.warning('### get_phecols error in pheno file - row %d col %d (%s) longer than header %s' % (nrows, col, row, head)) else: concordance[col].setdefault(code,0) # first one is zero concordance[col][code] += 1 @@ -601,10 +579,9 @@ res = [('no usable phenotype columns found',[('?',0),]),] return res - - def get_pheno(self,dataset): - """expects a .pheno file in the extra_files_dir - ugh + """ + expects a .pheno file in the extra_files_dir - ugh note that R is wierd and adds the row.name in the header so the columns are all wrong - unless you tell it not to. A file can be written as @@ -620,11 +597,12 @@ else: p = [] return '\n'.join(p) - def set_peek( self, dataset ): - """expects a .pheno file in the extra_files_dir - ugh + """ + expects a .pheno file in the extra_files_dir - ugh note that R is wierd and does not include the row.name in - the header. why?""" + the header. why? + """ if not dataset.dataset.purged: pp = os.path.join(dataset.extra_files_path,'%s.pheno' % dataset.metadata.base_name) try: @@ -636,17 +614,14 @@ else: dataset.peek = 'file does not exist\n' dataset.blurb = 'file purged from disk' - def get_peek( self, dataset ): - """expects a .pheno file in the extra_files_dir - ugh - """ + """expects a .pheno file in the extra_files_dir - ugh""" pp = os.path.join(dataset.extra_files_path,'%s.pheno' % dataset.metadata.base_name) try: p = file(pp,'r').readlines() except: p = ['##failed to find %s' % pp] return ''.join(p[:5]) - def get_file_peek(self,filename): """ can't really peek at a filename - need the extra_files_path and such? @@ -657,7 +632,6 @@ except: pass return ''.join(h[:5]) - def regenerate_primary_file(self,dataset): """cannot do this until we are setting metadata """ @@ -672,24 +646,19 @@ f.write("\n".join( rval )) f.write('\n') f.close() - - """Add metadata elements""" def init_meta( self, dataset, copy_from=None ): + """Add metadata elements""" if copy_from: dataset.metadata = copy_from.metadata - def set_meta( self, dataset, **kwd ): - """ NOTE we apply the tabular machinary to the phenodata extracted from a BioC eSet or affybatch. - """ try: flist = os.listdir(dataset.extra_files_path) except: - if verbose: - gal_Log.debug('@@@rexpression set_meta failed - no dataset?') + #log.debug('@@@rexpression set_meta failed - no dataset?') return False bn = None for f in flist: @@ -727,7 +696,6 @@ if not dataset.blurb: dataset.blurb = 'R loadable BioC expression object for the Rexpression Galaxy toolkit' return True - def make_html_table( self, pp='nothing supplied from peek\n'): """Create HTML table, used for displaying peek""" out = ['<table cellspacing="0" cellpadding="3">',] @@ -750,20 +718,16 @@ except Exception, exc: out = "Can't create html table %s" % str( exc ) return out - def display_peek( self, dataset ): """Returns formatted html of peek""" out=self.make_html_table(dataset.peek) return out - def get_mime(self): """Returns the mime type of the datatype""" return 'text/html' - class Affybatch( RexpBase ): """derived class for BioC data structures in Galaxy """ - file_ext = "affybatch" def __init__( self, **kwd ): @@ -780,7 +744,6 @@ self.add_composite_file( '%s.eset', description = 'ESet R object saved to file', substitute_name_with_metadata = 'base_name', is_binary = True ) - class MAlist( RexpBase ): """derived class for BioC data structures in Galaxy """ file_ext = "malist" @@ -790,9 +753,6 @@ self.add_composite_file( '%s.malist', description = 'MAlist R object saved to file', substitute_name_with_metadata = 'base_name', is_binary = True ) - if __name__ == '__main__': import doctest, sys doctest.testmod(sys.modules[__name__]) - - diff -r 555afd0bf457 -r 37406d8ad116 lib/galaxy/datatypes/images.py --- a/lib/galaxy/datatypes/images.py Tue Nov 17 16:14:54 2009 -0500 +++ b/lib/galaxy/datatypes/images.py Tue Nov 17 16:16:26 2009 -0500 @@ -13,82 +13,6 @@ log = logging.getLogger(__name__) -class Ab1( data.Data ): - """Class describing an ab1 binary sequence file""" - file_ext = "ab1" - def set_peek( self, dataset ): - if not dataset.dataset.purged: - export_url = "/history_add_to?" + urlencode({'history_id':dataset.history_id,'ext':'ab1','name':'ab1 sequence','info':'Sequence file','dbkey':dataset.dbkey}) - dataset.peek = "Binary ab1 sequence file" - dataset.blurb = data.nice_size( dataset.get_size() ) - else: - dataset.peek = 'file does not exist' - dataset.blurb = 'file purged from disk' - def display_peek(self, dataset): - try: - return dataset.peek - except: - return "Binary ab1 sequence file (%s)" % ( data.nice_size( dataset.get_size() ) ) - -class Scf( data.Data ): - """Class describing an scf binary sequence file""" - file_ext = "scf" - def set_peek( self, dataset ): - if not dataset.dataset.purged: - export_url = "/history_add_to?" + urlencode({'history_id':dataset.history_id,'ext':'scf','name':'scf sequence','info':'Sequence file','dbkey':dataset.dbkey}) - dataset.peek = "Binary scf sequence file" - dataset.blurb = data.nice_size( dataset.get_size() ) - else: - dataset.peek = 'file does not exist' - dataset.blurb = 'file purged from disk' - def display_peek(self, dataset): - try: - return dataset.peek - except: - return "Binary scf sequence file (%s)" % ( data.nice_size( dataset.get_size() ) ) - -class Binseq( data.Data ): - """Class describing a zip archive of binary sequence files""" - file_ext = "binseq.zip" - def set_peek( self, dataset ): - if not dataset.dataset.purged: - zip_file = zipfile.ZipFile( dataset.file_name, "r" ) - num_files = len( zip_file.namelist() ) - dataset.peek = "Archive of %s binary sequence files" % ( str( num_files ) ) - dataset.blurb = data.nice_size( dataset.get_size() ) - else: - dataset.peek = 'file does not exist' - dataset.blurb = 'file purged from disk' - def display_peek(self, dataset): - try: - return dataset.peek - except: - return "Binary sequence file archive (%s)" % ( data.nice_size( dataset.get_size() ) ) - def get_mime(self): - """Returns the mime type of the datatype""" - return 'application/zip' - -class Txtseq( data.Data ): - """Class describing a zip archive of text sequence files""" - file_ext = "txtseq.zip" - def set_peek( self, dataset ): - if not dataset.dataset.purged: - zip_file = zipfile.ZipFile( dataset.file_name, "r" ) - num_files = len( zip_file.namelist() ) - dataset.peek = "Archive of %s text sequence files" % ( str( num_files ) ) - dataset.blurb = data.nice_size( dataset.get_size() ) - else: - dataset.peek = 'file does not exist' - dataset.blurb = 'file purged from disk' - def display_peek(self, dataset): - try: - return dataset.peek - except: - return "Text sequence file archive (%s)" % ( data.nice_size( dataset.get_size() ) ) - def get_mime(self): - """Returns the mime type of the datatype""" - return 'application/zip' - class Image( data.Data ): """Class describing an image""" def set_peek( self, dataset ): @@ -236,47 +160,3 @@ return dataset.peek except: return "peek unavailable" - -class Bam( data.Binary ): - """Class describing a BAM binary file""" - file_ext = "bam" - MetadataElement( name="bam_index", desc="BAM Index File", param=metadata.FileParameter, readonly=True, no_value=None, visible=False, optional=True ) - def init_meta( self, dataset, copy_from=None ): - data.Binary.init_meta( self, dataset, copy_from=copy_from ) - def set_meta( self, dataset, overwrite = True, **kwd ): - """ - Sets index for BAM file. - """ - index_file = dataset.metadata.bam_index - if not index_file: - index_file = dataset.metadata.spec['bam_index'].param.new_file( dataset = dataset ) - tmp_dir = tempfile.gettempdir() - tmpf1 = tempfile.NamedTemporaryFile(dir=tmp_dir) - tmpf1bai = '%s.bai' % tmpf1.name - try: - os.system('cd %s' % tmp_dir) - os.system('cp %s %s' % (dataset.file_name, tmpf1.name)) - os.system('samtools index %s' % tmpf1.name) - os.system('cp %s %s' % (tmpf1bai, index_file.file_name)) - except Exception, ex: - sys.stderr.write('There was a problem creating the index for the BAM file\n%s\n' + str(ex)) - tmpf1.close() - if os.path.exists(tmpf1bai): - os.remove(tmpf1bai) - dataset.metadata.bam_index = index_file - def set_peek( self, dataset ): - if not dataset.dataset.purged: - export_url = "/history_add_to?" + urlencode({'history_id':dataset.history_id,'ext':'bam','name':'bam alignments','info':'Alignments file','dbkey':dataset.dbkey}) - dataset.peek = "Binary bam alignments file" - dataset.blurb = data.nice_size( dataset.get_size() ) - else: - dataset.peek = 'file does not exist' - dataset.blurb = 'file purged from disk' - def display_peek(self, dataset): - try: - return dataset.peek - except: - return "Binary bam alignments file (%s)" % ( data.nice_size( dataset.get_size() ) ) - def get_mime(self): - """Returns the mime type of the datatype""" - return 'application/octet-stream' diff -r 555afd0bf457 -r 37406d8ad116 lib/galaxy/datatypes/metadata.py --- a/lib/galaxy/datatypes/metadata.py Tue Nov 17 16:14:54 2009 -0500 +++ b/lib/galaxy/datatypes/metadata.py Tue Nov 17 16:16:26 2009 -0500 @@ -509,7 +509,7 @@ # need to make different keys for them, since ids can overlap return "%s_%d" % ( dataset.__class__.__name__, dataset.id ) def setup_external_metadata( self, datasets, sa_session, exec_dir=None, tmp_dir=None, dataset_files_path=None, - output_fnames=None, config_root=None, datatypes_config=None, kwds={} ): + output_fnames=None, config_root=None, datatypes_config=None, job_metadata=None, kwds={} ): #fill in metadata_files_dict and return the command with args required to set metadata def __metadata_files_list_to_cmd_line( metadata_files ): def __get_filename_override(): @@ -564,7 +564,7 @@ sa_session.flush() metadata_files_list.append( metadata_files ) #return command required to build - return "%s %s %s %s %s %s" % ( os.path.join( exec_dir, 'set_metadata.sh' ), dataset_files_path, tmp_dir, config_root, datatypes_config, " ".join( map( __metadata_files_list_to_cmd_line, metadata_files_list ) ) ) + return "%s %s %s %s %s %s %s" % ( os.path.join( exec_dir, 'set_metadata.sh' ), dataset_files_path, tmp_dir, config_root, datatypes_config, job_metadata, " ".join( map( __metadata_files_list_to_cmd_line, metadata_files_list ) ) ) def external_metadata_set_successfully( self, dataset, sa_session ): metadata_files = self.get_output_filenames_by_dataset( dataset, sa_session ) diff -r 555afd0bf457 -r 37406d8ad116 lib/galaxy/datatypes/registry.py --- a/lib/galaxy/datatypes/registry.py Tue Nov 17 16:14:54 2009 -0500 +++ b/lib/galaxy/datatypes/registry.py Tue Nov 17 16:16:26 2009 -0500 @@ -3,7 +3,7 @@ """ import os, tempfile import logging -import data, tabular, interval, images, sequence, qualityscore, genetics, xml, coverage, tracks, chrominfo +import data, tabular, interval, images, sequence, qualityscore, genetics, xml, coverage, tracks, chrominfo, binary import galaxy.util from galaxy.util.odict import odict @@ -109,11 +109,11 @@ #default values if len(self.datatypes_by_extension) < 1: self.datatypes_by_extension = { - 'ab1' : images.Ab1(), + 'ab1' : binary.Ab1(), 'axt' : sequence.Axt(), - 'bam' : images.Bam(), + 'bam' : binary.Bam(), 'bed' : interval.Bed(), - 'binseq.zip' : images.Binseq(), + 'binseq.zip' : binary.Binseq(), 'blastxml' : xml.BlastXml(), 'coverage' : coverage.LastzCoverage(), 'customtrack' : interval.CustomTrack(), @@ -132,12 +132,12 @@ 'qualsolexa' : qualityscore.QualityScoreSolexa(), 'qual454' : qualityscore.QualityScore454(), 'sam' : tabular.Sam(), - 'scf' : images.Scf(), - 'sff' : data.Sff(), + 'scf' : binary.Scf(), + 'sff' : binary.Sff(), 'tabular' : tabular.Tabular(), 'taxonomy' : tabular.Taxonomy(), 'txt' : data.Text(), - 'txtseq.zip' : images.Txtseq(), + 'txtseq.zip' : data.Txtseq(), 'wig' : interval.Wiggle() } self.mimetypes_by_extension = { @@ -174,7 +174,7 @@ # because some formats are much more flexibly defined than others. if len(self.sniff_order) < 1: self.sniff_order = [ - data.Sff(), + binary.Sff(), xml.BlastXml(), sequence.Maf(), sequence.Lav(), diff -r 555afd0bf457 -r 37406d8ad116 lib/galaxy/datatypes/tracks.py --- a/lib/galaxy/datatypes/tracks.py Tue Nov 17 16:14:54 2009 -0500 +++ b/lib/galaxy/datatypes/tracks.py Tue Nov 17 16:16:26 2009 -0500 @@ -3,10 +3,7 @@ """ import data -import logging -import re -import binascii -from cgi import escape +import tabular, binascii, logging from galaxy.datatypes.metadata import MetadataElement from galaxy.datatypes import metadata import galaxy.model @@ -17,7 +14,7 @@ log = logging.getLogger(__name__) -class GeneTrack( data.Binary ): +class GeneTrack( tabular.Tabular ): file_ext = "genetrack" MetadataElement( name="genetrack", default="data.genetrack", desc="HDF index", readonly=True, visible=True, no_value=0 ) @@ -27,7 +24,7 @@ super( GeneTrack, self ).__init__( **kwargs ) self.add_display_app( 'genetrack', 'View in', '', 'genetrack_link' ) def get_display_links( self, dataset, type, app, base_url, target_frame='galaxy_main', **kwd ): - return data.Binary.get_display_links( self, dataset, type, app, base_url, target_frame=target_frame, **kwd ) + return data.Data.get_display_links( self, dataset, type, app, base_url, target_frame=target_frame, **kwd ) def genetrack_link( self, hda, type, app, base_url ): ret_val = [] if hda.has_data: diff -r 555afd0bf457 -r 37406d8ad116 lib/galaxy/jobs/__init__.py --- a/lib/galaxy/jobs/__init__.py Tue Nov 17 16:14:54 2009 -0500 +++ b/lib/galaxy/jobs/__init__.py Tue Nov 17 16:16:26 2009 -0500 @@ -139,10 +139,15 @@ JobWrapper( job, None, self ).fail( 'This tool was disabled before the job completed. Please contact your Galaxy administrator, or' ) elif job.job_runner_name is None: log.debug( "no runner: %s is still in queued state, adding to the jobs queue" %job.id ) - self.queue.put( ( job.id, job.tool_id ) ) + if self.track_jobs_in_database: + job.state = model.Job.states.NEW + else: + self.queue.put( ( job.id, job.tool_id ) ) else: job_wrapper = JobWrapper( job, self.app.toolbox.tools_by_id[ job.tool_id ], self ) self.dispatcher.recover( job, job_wrapper ) + if self.sa_session.dirty: + self.sa_session.flush() def __monitor( self ): """ @@ -526,6 +531,7 @@ # If the tool was expected to set the extension, attempt to retrieve it if dataset.ext == 'auto': dataset.extension = context.get( 'ext', 'data' ) + dataset.init_meta( copy_from=dataset ) #if a dataset was copied, it won't appear in our dictionary: #either use the metadata from originating output dataset, or call set_meta on the copies #it would be quicker to just copy the metadata from the originating output dataset, @@ -715,14 +721,15 @@ for outfile in [ str( o ) for o in output_paths ]: sizes.append( ( outfile, os.stat( outfile ).st_size ) ) return sizes - def setup_external_metadata( self, exec_dir = None, tmp_dir = None, dataset_files_path = None, config_root = None, datatypes_config = None, **kwds ): + def setup_external_metadata( self, exec_dir = None, tmp_dir = None, dataset_files_path = None, config_root = None, datatypes_config = None, set_extension = True, **kwds ): # extension could still be 'auto' if this is the upload tool. job = self.sa_session.query( model.Job ).get( self.job_id ) - for output_dataset_assoc in job.output_datasets: - if output_dataset_assoc.dataset.ext == 'auto': - context = self.get_dataset_finish_context( dict(), output_dataset_assoc.dataset.dataset ) - output_dataset_assoc.dataset.extension = context.get( 'ext', 'data' ) - self.sa_session.flush() + if set_extension: + for output_dataset_assoc in job.output_datasets: + if output_dataset_assoc.dataset.ext == 'auto': + context = self.get_dataset_finish_context( dict(), output_dataset_assoc.dataset.dataset ) + output_dataset_assoc.dataset.extension = context.get( 'ext', 'data' ) + self.sa_session.flush() if tmp_dir is None: #this dir should should relative to the exec_dir tmp_dir = self.app.config.new_file_path @@ -739,6 +746,7 @@ dataset_files_path = dataset_files_path, config_root = config_root, datatypes_config = datatypes_config, + job_metadata = os.path.join( self.working_directory, TOOL_PROVIDED_JOB_METADATA_FILE ), **kwds ) class DefaultJobDispatcher( object ): diff -r 555afd0bf457 -r 37406d8ad116 lib/galaxy/jobs/runners/local.py --- a/lib/galaxy/jobs/runners/local.py Tue Nov 17 16:14:54 2009 -0500 +++ b/lib/galaxy/jobs/runners/local.py Tue Nov 17 16:16:26 2009 -0500 @@ -106,7 +106,9 @@ #this is terminatable when output dataset/job is deleted #so that long running set_meta()s can be cancelled without having to reboot the server if job_wrapper.get_state() not in [ model.Job.states.ERROR, model.Job.states.DELETED ] and self.app.config.set_metadata_externally and job_wrapper.output_paths: - external_metadata_script = job_wrapper.setup_external_metadata( output_fnames = job_wrapper.get_output_fnames(), kwds = { 'overwrite' : False } ) #we don't want to overwrite metadata that was copied over in init_meta(), as per established behavior + external_metadata_script = job_wrapper.setup_external_metadata( output_fnames = job_wrapper.get_output_fnames(), + set_extension = True, + kwds = { 'overwrite' : False } ) #we don't want to overwrite metadata that was copied over in init_meta(), as per established behavior log.debug( 'executing external set_meta script for job %d: %s' % ( job_wrapper.job_id, external_metadata_script ) ) external_metadata_proc = subprocess.Popen( args = external_metadata_script, shell = True, diff -r 555afd0bf457 -r 37406d8ad116 lib/galaxy/jobs/runners/pbs.py --- a/lib/galaxy/jobs/runners/pbs.py Tue Nov 17 16:14:54 2009 -0500 +++ b/lib/galaxy/jobs/runners/pbs.py Tue Nov 17 16:16:26 2009 -0500 @@ -29,7 +29,6 @@ fi cd %s %s -%s """ pbs_symlink_template = """#!/bin/sh @@ -178,7 +177,9 @@ pbs_queue_name = self.determine_pbs_queue( runner_url ) c = pbs.pbs_connect( pbs_server_name ) if c <= 0: - raise Exception( "Connection to PBS server for submit failed" ) + job_wrapper.fail( "Unable to queue job for execution. Resubmitting the job may succeed." ) + log.error( "Connection to PBS server for submit failed" ) + return # define job attributes ofile = "%s/%s.o" % (self.app.config.cluster_files_directory, job_wrapper.job_id) @@ -221,11 +222,15 @@ if self.app.config.pbs_stage_path != '': script = pbs_symlink_template % (job_wrapper.galaxy_lib_dir, " ".join(job_wrapper.get_input_fnames() + output_files), self.app.config.pbs_stage_path, exec_dir, command_line) else: + script = pbs_template % ( job_wrapper.galaxy_lib_dir, exec_dir, command_line ) if self.app.config.set_metadata_externally: - external_metadata_script = job_wrapper.setup_external_metadata( exec_dir = os.path.abspath( os.getcwd() ), tmp_dir = self.app.config.new_file_path, dataset_files_path = self.app.model.Dataset.file_path, output_fnames = output_fnames, kwds = { 'overwrite' : False } ) #we don't want to overwrite metadata that was copied over in init_meta(), as per established behavior - else: - external_metadata_script = "" - script = pbs_template % ( job_wrapper.galaxy_lib_dir, exec_dir, command_line, external_metadata_script ) + script += "cd %s\n" % os.path.abspath( os.getcwd() ) + script += "%s\n" % job_wrapper.setup_external_metadata( exec_dir = os.path.abspath( os.getcwd() ), + tmp_dir = self.app.config.new_file_path, + dataset_files_path = self.app.model.Dataset.file_path, + output_fnames = output_fnames, + set_extension = False, + kwds = { 'overwrite' : False } ) #we don't want to overwrite metadata that was copied over in init_meta(), as per established behavior job_file = "%s/%s.sh" % (self.app.config.cluster_files_directory, job_wrapper.job_id) fh = file(job_file, "w") fh.write(script) diff -r 555afd0bf457 -r 37406d8ad116 lib/galaxy/tools/actions/metadata.py --- a/lib/galaxy/tools/actions/metadata.py Tue Nov 17 16:14:54 2009 -0500 +++ b/lib/galaxy/tools/actions/metadata.py Tue Nov 17 16:16:26 2009 -0500 @@ -41,6 +41,7 @@ output_fnames = None, config_root = None, datatypes_config = None, + job_metadata = None, kwds = { 'overwrite' : True } ) incoming[ '__SET_EXTERNAL_METADATA_COMMAND_LINE__' ] = cmd_line for name, value in tool.params_to_strings( incoming, trans.app ).iteritems(): diff -r 555afd0bf457 -r 37406d8ad116 lib/galaxy/web/controllers/dataset.py --- a/lib/galaxy/web/controllers/dataset.py Tue Nov 17 16:14:54 2009 -0500 +++ b/lib/galaxy/web/controllers/dataset.py Tue Nov 17 16:16:26 2009 -0500 @@ -20,25 +20,38 @@ This error report was sent from the Galaxy instance hosted on the server "${host}" ----------------------------------------------------------------------------- -This is in reference to output dataset ${dataset_id}. +This is in reference to dataset id ${dataset_id} from history id ${history_id} +----------------------------------------------------------------------------- +You should be able to view the history containing the related history item + +${hid}: ${history_item_name} + +by logging in as a Galaxy admin user to the Galaxy instance referenced above +and pointing your browser to the following link. + +${history_view_link} ----------------------------------------------------------------------------- The user '${email}' provided the following information: + ${message} ----------------------------------------------------------------------------- job id: ${job_id} -tool id: ${tool_id} +tool id: ${job_tool_id} +----------------------------------------------------------------------------- +job command line: +${job_command_line} ----------------------------------------------------------------------------- job stderr: -${stderr} +${job_stderr} ----------------------------------------------------------------------------- job stdout: -${stdout} +${job_stdout} ----------------------------------------------------------------------------- job info: -${info} +${job_info} ----------------------------------------------------------------------------- job traceback: -${traceback} +${job_traceback} ----------------------------------------------------------------------------- (This is an automated message). """ @@ -103,41 +116,45 @@ @web.expose def errors( self, trans, id ): - dataset = trans.sa_session.query( model.HistoryDatasetAssociation ).get( id ) - return trans.fill_template( "dataset/errors.mako", dataset=dataset ) - + hda = trans.sa_session.query( model.HistoryDatasetAssociation ).get( id ) + return trans.fill_template( "dataset/errors.mako", hda=hda ) @web.expose def stderr( self, trans, id ): dataset = trans.sa_session.query( model.HistoryDatasetAssociation ).get( id ) job = dataset.creating_job_associations[0].job trans.response.set_content_type( 'text/plain' ) return job.stderr - @web.expose def report_error( self, trans, id, email='', message="" ): smtp_server = trans.app.config.smtp_server if smtp_server is None: - return trans.show_error_message( "Sorry, mail is not configured for this galaxy instance" ) + return trans.show_error_message( "Mail is not configured for this galaxy instance" ) to_address = trans.app.config.error_email_to if to_address is None: - return trans.show_error_message( "Sorry, error reporting has been disabled for this galaxy instance" ) + return trans.show_error_message( "Error reporting has been disabled for this galaxy instance" ) # Get the dataset and associated job - dataset = trans.sa_session.query( model.HistoryDatasetAssociation ).get( id ) - job = dataset.creating_job_associations[0].job + hda = trans.sa_session.query( model.HistoryDatasetAssociation ).get( id ) + job = hda.creating_job_associations[0].job # Get the name of the server hosting the Galaxy instance from which this report originated host = trans.request.host + history_view_link = "%s/history/view?id=%s" % ( str( host ), trans.security.encode_id( hda.history_id ) ) # Build the email message msg = MIMEText( string.Template( error_report_template ) .safe_substitute( host=host, - dataset_id=dataset.id, + dataset_id=hda.dataset_id, + history_id=hda.history_id, + hid=hda.hid, + history_item_name=hda.get_display_name(), + history_view_link=history_view_link, + job_id=job.id, + job_tool_id=job.tool_id, + job_command_line=job.command_line, + job_stderr=job.stderr, + job_stdout=job.stdout, + job_info=job.info, + job_traceback=job.traceback, email=email, - message=message, - job_id=job.id, - tool_id=job.tool_id, - stderr=job.stderr, - stdout=job.stdout, - traceback=job.traceback, - info=job.info ) ) + message=message ) ) frm = to_address # Check email a bit email = email.strip() diff -r 555afd0bf457 -r 37406d8ad116 lib/galaxy/web/controllers/forms.py --- a/lib/galaxy/web/controllers/forms.py Tue Nov 17 16:14:54 2009 -0500 +++ b/lib/galaxy/web/controllers/forms.py Tue Nov 17 16:16:26 2009 -0500 @@ -8,9 +8,71 @@ from elementtree.ElementTree import XML, Element from galaxy.util.odict import odict import copy +from galaxy.web.framework.helpers import time_ago, iff, grids log = logging.getLogger( __name__ ) +class FormsGrid( grids.Grid ): + # Custom column types + class NameColumn( grids.TextColumn ): + def get_value(self, trans, grid, form): + return form.latest_form.name + class DescriptionColumn( grids.TextColumn ): + def get_value(self, trans, grid, form): + return form.latest_form.desc + class TypeColumn( grids.TextColumn ): + def get_value(self, trans, grid, form): + return form.latest_form.type + class DeletedColumn( grids.GridColumn ): + def get_accepted_filters( self ): + """ Returns a list of accepted filters for this column. """ + accepted_filter_labels_and_vals = { "active" : "False", "deleted" : "True", "all": "All" } + accepted_filters = [] + for label, val in accepted_filter_labels_and_vals.items(): + args = { self.key: val } + accepted_filters.append( grids.GridColumnFilter( label, args) ) + return accepted_filters + # Grid definition + title = "Forms" + template = "admin/forms/grid.mako" + model_class = model.FormDefinitionCurrent + default_sort_key = "-create_time" + num_rows_per_page = 50 + preserve_state = True + use_paging = True + default_filter = dict( deleted="False" ) + columns = [ + NameColumn( "Name", + key="name", + model_class=model.FormDefinition, + link=( lambda item: iff( item.deleted, None, dict( operation="view", id=item.id ) ) ), + attach_popup=True, + filterable="advanced" ), + DescriptionColumn( "Description", + key='desc', + model_class=model.FormDefinition, + filterable="advanced" ), + TypeColumn( "Type" ), + DeletedColumn( "Deleted", + key="deleted", + visible=False, + filterable="advanced" ) + ] + columns.append( grids.MulticolFilterColumn( "Search", + cols_to_filter=[ columns[0], columns[1] ], + key="free-text-search", + visible=False, + filterable="standard" ) ) + operations = [ + grids.GridOperation( "Edit", allow_multiple=False, condition=( lambda item: not item.deleted ) ), + grids.GridOperation( "Delete", allow_multiple=True, condition=( lambda item: not item.deleted ) ), + grids.GridOperation( "Undelete", condition=( lambda item: item.deleted ) ), + ] + global_actions = [ + grids.GridAction( "Create new form", dict( controller='forms', + action='new' ) ) + ] + class Forms( BaseController ): # Empty form field empty_field = { 'label': '', @@ -20,38 +82,38 @@ 'type': BaseField.form_field_types()[0], 'selectlist': [], 'layout': 'none' } + forms_grid = FormsGrid() + @web.expose @web.require_admin - def index( self, trans, **kwd ): - params = util.Params( kwd ) - msg = util.restore_text( params.get( 'msg', '' ) ) - messagetype = params.get( 'messagetype', 'done' ) - return trans.fill_template( "/sample/index.mako", - default_action=params.get( 'default_action', None ), - msg=msg, - messagetype=messagetype ) - @web.expose - @web.require_admin - def manage( self, trans, **kwd ): - params = util.Params( kwd ) - msg = util.restore_text( params.get( 'msg', '' ) ) - messagetype = params.get( 'messagetype', 'done' ) - show_filter = params.get( 'show_filter', 'Active' ) - return self._show_forms_list(trans, msg, messagetype, show_filter) - def _show_forms_list(self, trans, msg, messagetype, show_filter='Active'): - all_forms = trans.sa_session.query( trans.app.model.FormDefinitionCurrent ) - if show_filter == 'All': - forms_list = all_forms - elif show_filter == 'Deleted': - forms_list = [form for form in all_forms if form.deleted] - else: - forms_list = [form for form in all_forms if not form.deleted] - return trans.fill_template( '/admin/forms/manage_forms.mako', - fdc_list=forms_list, - all_forms=all_forms, - show_filter=show_filter, - msg=msg, - messagetype=messagetype ) + def manage( self, trans, **kwd ): + if 'operation' in kwd: + operation = kwd['operation'].lower() + if not kwd.get( 'id', None ): + return trans.response.send_redirect( web.url_for( controller='forms', + action='manage', + status='error', + message="Invalid form ID") ) + if operation == "view": + return self.__view( trans, **kwd ) + elif operation == "delete": + return self.__delete( trans, **kwd ) + elif operation == "undelete": + return self.__undelete( trans, **kwd ) + elif operation == "edit": + return self.__edit( trans, **kwd ) + return self.forms_grid( trans, **kwd ) + def __view(self, trans, **kwd): + try: + fdc = trans.sa_session.query( trans.app.model.FormDefinitionCurrent )\ + .get( trans.security.decode_id(kwd['id']) ) + except: + return trans.response.send_redirect( web.url_for( controller='forms', + action='manage', + msg='Invalid form', + messagetype='error' ) ) + return trans.fill_template( '/admin/forms/show_form_read_only.mako', + form=fdc.latest_form ) def __form_types_widget(self, trans, selected='none'): form_type_selectbox = SelectField( 'form_type_selectbox', refresh_on_change=True, @@ -86,13 +148,14 @@ self.__get_saved_form( fd ) if self.__imported_from_file: return trans.response.send_redirect( web.url_for( controller='forms', - action='edit', - show_form=True, - form_id=fd.id) ) + action='manage', + operation='edit', + id=trans.security.encode_id(fd.current.id)) ) else: return trans.response.send_redirect( web.url_for( controller='forms', - action='edit', - form_id=fd.id, + action='manage', + operation='edit', + id=trans.security.encode_id(fd.current.id), add_field_button='Add field', name=fd.name, description=fd.desc, @@ -105,35 +168,43 @@ inputs=inputs, msg=msg, messagetype=messagetype ) - @web.expose - @web.require_admin - def delete( self, trans, **kwd ): - params = util.Params( kwd ) - msg = util.restore_text( params.get( 'msg', '' ) ) - messagetype = params.get( 'messagetype', 'done' ) - fd = trans.sa_session.query( trans.app.model.FormDefinition ).get( int( util.restore_text( params.form_id ) ) ) - fd.form_definition_current.deleted = True - trans.sa_session.add( fd.form_definition_current ) - trans.sa_session.flush() - return self._show_forms_list(trans, - msg='The form definition named %s is deleted.' % fd.name, - messagetype='done') - @web.expose - @web.require_admin - def undelete( self, trans, **kwd ): - params = util.Params( kwd ) - msg = util.restore_text( params.get( 'msg', '' ) ) - messagetype = params.get( 'messagetype', 'done' ) - fd = trans.sa_session.query( trans.app.model.FormDefinition ).get( int( util.restore_text( params.form_id ) ) ) - fd.form_definition_current.deleted = False - trans.sa_session.add( fd.form_definition_current ) - trans.sa_session.flush() - return self._show_forms_list(trans, - msg='The form definition named %s is undeleted.' % fd.name, - messagetype='done') - @web.expose - @web.require_admin - def edit( self, trans, **kwd ): + def __delete( self, trans, **kwd ): + id_list = util.listify( kwd['id'] ) + delete_failed = [] + for id in id_list: + try: + fdc = trans.sa_session.query( trans.app.model.FormDefinitionCurrent ).get( trans.security.decode_id(id) ) + except: + return trans.response.send_redirect( web.url_for( controller='forms', + action='manage', + message='Invalid form', + status='error' ) ) + fdc.deleted = True + trans.sa_session.add( fdc ) + trans.sa_session.flush() + return trans.response.send_redirect( web.url_for( controller='forms', + action='manage', + message='%i form(s) is deleted.' % len(id_list), + status='done') ) + def __undelete( self, trans, **kwd ): + id_list = util.listify( kwd['id'] ) + delete_failed = [] + for id in id_list: + try: + fdc = trans.sa_session.query( trans.app.model.FormDefinitionCurrent ).get( trans.security.decode_id(id) ) + except: + return trans.response.send_redirect( web.url_for( controller='forms', + action='manage', + message='Invalid form', + status='error' ) ) + fdc.deleted = False + trans.sa_session.add( fdc ) + trans.sa_session.flush() + return trans.response.send_redirect( web.url_for( controller='forms', + action='manage', + message='%i form(s) is undeleted.' % len(id_list), + status='done') ) + def __edit( self, trans, **kwd ): ''' This callback method is for handling all the editing functions like renaming fields, adding/deleting fields, changing fields attributes. @@ -142,17 +213,28 @@ msg = util.restore_text( params.get( 'msg', '' ) ) messagetype = params.get( 'messagetype', 'done' ) try: - fd = trans.sa_session.query( trans.app.model.FormDefinition ).get( int( params.get( 'form_id', None ) ) ) + fdc = trans.sa_session.query( trans.app.model.FormDefinitionCurrent ).get( trans.security.decode_id(kwd['id']) ) except: return trans.response.send_redirect( web.url_for( controller='forms', action='manage', - msg='Invalid form', - messagetype='error' ) ) + message='Invalid form', + status='error' ) ) + fd = fdc.latest_form # - # Show the form for editing + # Save changes # - if params.get( 'show_form', False ): + if params.get( 'save_changes_button', False ): + fd_new, msg = self.__save_form( trans, fdc_id=fd.form_definition_current.id, **kwd ) + # if validation error encountered while saving the form, show the + # unsaved form, with the error message + if not fd_new: + current_form = self.__get_form( trans, **kwd ) + return self.__show( trans=trans, form=fd, current_form=current_form, + msg=msg, messagetype='error', **kwd ) + # everything went fine. form saved successfully. Show the saved form + fd = fd_new current_form = self.__get_saved_form( fd ) + msg = "The form '%s' has been updated with the changes." % fd.name return self.__show( trans=trans, form=fd, current_form=current_form, msg=msg, messagetype=messagetype, **kwd ) # @@ -193,31 +275,6 @@ return self.__show( trans=trans, form=fd, current_form=current_form, msg=msg, messagetype=messagetype, **kwd ) # - # Save changes - # - elif params.get( 'save_changes_button', False ): - fd_new, msg = self.__save_form( trans, fdc_id=fd.form_definition_current.id, **kwd ) - # if validation error encountered while saving the form, show the - # unsaved form, with the error message - if not fd_new: - current_form = self.__get_form( trans, **kwd ) - return self.__show( trans=trans, form=fd, current_form=current_form, - msg=msg, messagetype='error', **kwd ) - # everything went fine. form saved successfully. Show the saved form - fd = fd_new - current_form = self.__get_saved_form( fd ) - msg = "The form '%s' has been updated with the changes." % fd.name - return self.__show( trans=trans, form=fd, current_form=current_form, - msg=msg, messagetype=messagetype, **kwd ) - # - # Show form read-only - # - elif params.get( 'read_only', False ): - return trans.fill_template( '/admin/forms/show_form_read_only.mako', - form=fd, - msg=msg, - messagetype=messagetype ) - # # Add SelectField option # elif 'Add' in kwd.values(): @@ -234,6 +291,13 @@ current_form = self.__get_form( trans, **kwd ) return self.__show( trans=trans, form=fd, current_form=current_form, msg=msg, messagetype=messagetype, **kwd ) + # + # Show the form for editing + # + else: + current_form = self.__get_saved_form( fd ) + return self.__show( trans=trans, form=fd, current_form=current_form, + msg=msg, messagetype=messagetype, **kwd ) def __add_selectbox_option( self, trans, fd, msg, messagetype, **kwd ): ''' diff -r 555afd0bf457 -r 37406d8ad116 lib/galaxy/web/controllers/history.py --- a/lib/galaxy/web/controllers/history.py Tue Nov 17 16:14:54 2009 -0500 +++ b/lib/galaxy/web/controllers/history.py Tue Nov 17 16:16:26 2009 -0500 @@ -87,7 +87,7 @@ # Grid definition title = "Saved Histories" model_class = model.History - template='/history/grid.mako' + template='/grid_base.mako' default_sort_key = "-create_time" columns = [ NameColumn( "Name", key="name", model_class=model.History, @@ -110,14 +110,14 @@ ) operations = [ - grids.GridOperation( "Switch", allow_multiple=False, condition=( lambda item: not item.deleted ) ), - grids.GridOperation( "Share", condition=( lambda item: not item.deleted ) ), - grids.GridOperation( "Unshare", condition=( lambda item: not item.deleted ) ), - grids.GridOperation( "Rename", condition=( lambda item: not item.deleted ) ), - grids.GridOperation( "Delete", condition=( lambda item: not item.deleted ) ), - grids.GridOperation( "Undelete", condition=( lambda item: item.deleted ) ), - grids.GridOperation( "Enable import via link", condition=( lambda item: item.deleted ) ), - grids.GridOperation( "Disable import via link", condition=( lambda item: item.deleted ) ) + grids.GridOperation( "Switch", allow_multiple=False, condition=( lambda item: not item.deleted ), async_compatible=True ), + grids.GridOperation( "Share", condition=( lambda item: not item.deleted ), async_compatible=False ), + grids.GridOperation( "Unshare", condition=( lambda item: not item.deleted ), async_compatible=False ), + grids.GridOperation( "Rename", condition=( lambda item: not item.deleted ), async_compatible=False ), + grids.GridOperation( "Delete", condition=( lambda item: not item.deleted ), async_compatible=True ), + grids.GridOperation( "Undelete", condition=( lambda item: item.deleted ), async_compatible=True ), + grids.GridOperation( "Enable import via link", condition=( lambda item: item.deleted ), async_compatible=True ), + grids.GridOperation( "Disable import via link", condition=( lambda item: item.deleted ), async_compatible=True ) ] standard_filters = [ grids.GridColumnFilter( "Active", args=dict( deleted=False ) ), @@ -262,7 +262,7 @@ n_deleted += 1 status = SUCCESS if n_deleted: - message_parts.append( "Deleted %d histories. " % n_deleted ) + message_parts.append( "Deleted %d %s. " % ( n_deleted, iff( n_deleted != 1, "histories", "history" ) ) ) if deleted_current: message_parts.append( "Your active history was deleted, a new empty history is now active. " ) status = INFO @@ -290,7 +290,7 @@ status = SUCCESS message_parts = [] if n_undeleted: - message_parts.append( "Undeleted %d histories." % n_undeleted ) + message_parts.append( "Undeleted %d %s. " % ( n_undeleted, iff( n_undeleted != 1, "histories", "history" ) ) ) if n_already_purged: message_parts.append( "%d histories have already been purged and cannot be undeleted." % n_already_purged ) status = WARNING @@ -438,23 +438,20 @@ Warning! If you import this history, you will lose your current history. Click <a href="%s">here</a> to confirm. """ % web.url_for( id=id, confirm=True ) ) - @web.expose def view( self, trans, id=None ): """View a history. If a history is importable, then it is viewable by any user.""" - # Get history to view. if not id: return trans.show_error_message( "You must specify a history you want to view." ) history_to_view = get_history( trans, id, False) - # Integrity checks. if not history_to_view: - return trans.show_error_message( "The specified history does not exist.") + return trans.show_error_message( "The specified history does not exist." ) + # Admin users can view any history # TODO: Use a new flag to determine if history is viewable? - if not history_to_view.importable: - error( "The owner of this history has not published this history." ) - + if not trans.user_is_admin and not history_to_view.importable: + error( "Either you are not allowed to view this history or the owner of this history has not published it." ) # View history. query = trans.sa_session.query( model.HistoryDatasetAssociation ) \ .filter( model.HistoryDatasetAssociation.history == history_to_view ) \ @@ -469,7 +466,6 @@ datasets = query.all(), user_owns_history = user_owns_history, show_deleted = False ) - @web.expose @web.require_login( "share histories with other users" ) def share( self, trans, id=None, email="", **kwd ): diff -r 555afd0bf457 -r 37406d8ad116 lib/galaxy/web/controllers/library_common.py --- a/lib/galaxy/web/controllers/library_common.py Tue Nov 17 16:14:54 2009 -0500 +++ b/lib/galaxy/web/controllers/library_common.py Tue Nov 17 16:16:26 2009 -0500 @@ -117,7 +117,7 @@ uploaded_dataset.link_data_only = True uploaded_dataset.data.file_name = os.path.abspath( path ) trans.sa_session.add( uploaded_dataset.data ) - trans.sa_session.data.flush() + trans.sa_session.flush() return uploaded_dataset def get_server_dir_uploaded_datasets( self, trans, params, full_dir, import_dir_desc, library_bunch, err_redirect, msg ): files = [] diff -r 555afd0bf457 -r 37406d8ad116 lib/galaxy/web/controllers/page.py --- a/lib/galaxy/web/controllers/page.py Tue Nov 17 16:14:54 2009 -0500 +++ b/lib/galaxy/web/controllers/page.py Tue Nov 17 16:16:26 2009 -0500 @@ -1,6 +1,7 @@ from galaxy.web.base.controller import * from galaxy.web.framework.helpers import time_ago, grids from galaxy.util.sanitize_html import sanitize_html +from galaxy.util.odict import odict import re @@ -69,21 +70,66 @@ ] def apply_default_filter( self, trans, query, **kwargs ): return query.filter_by( deleted=False, published=True ) - - -class NameColumn( grids.TextColumn ): - def get_value(self, trans, grid, history): - return history.get_display_name() class HistorySelectionGrid( grids.Grid ): + # Custom columns. + class NameColumn( grids.TextColumn ): + def get_value(self, trans, grid, history): + return history.get_display_name() + + class DeletedColumn( grids.GridColumn ): + def get_accepted_filters( self ): + """ Returns a list of accepted filters for this column. """ + accepted_filter_labels_and_vals = { "active" : "False", "deleted" : "True", "all": "All" } + accepted_filters = [] + for label, val in accepted_filter_labels_and_vals.items(): + args = { self.key: val } + accepted_filters.append( grids.GridColumnFilter( label, args) ) + return accepted_filters + + class SharingColumn( grids.GridColumn ): + def filter( self, db_session, query, column_filter ): + """ Modify query to filter histories by sharing status. """ + if column_filter == "All": + pass + elif column_filter: + if column_filter == "private": + query = query.filter( model.History.users_shared_with == None ) + query = query.filter( model.History.importable == False ) + elif column_filter == "shared": + query = query.filter( model.History.users_shared_with != None ) + elif column_filter == "importable": + query = query.filter( model.History.importable == True ) + return query + def get_accepted_filters( self ): + """ Returns a list of accepted filters for this column. """ + accepted_filter_labels_and_vals = odict() + accepted_filter_labels_and_vals["private"] = "private" + accepted_filter_labels_and_vals["shared"] = "shared" + accepted_filter_labels_and_vals["importable"] = "importable" + accepted_filter_labels_and_vals["all"] = "All" + accepted_filters = [] + for label, val in accepted_filter_labels_and_vals.items(): + args = { self.key: val } + accepted_filters.append( grids.GridColumnFilter( label, args) ) + return accepted_filters + # Grid definition. title = "Saved Histories" + template = "grid_base_async.mako" + async_template = "grid_body_async.mako" model_class = model.History + default_filter = { "deleted" : "False" , "shared" : "All" } default_sort_key = "-update_time" + use_paging = True + num_rows_per_page = 5 columns = [ NameColumn( "Name", key="name", model_class=model.History, filterable="advanced" ), grids.TagsColumn( "Tags", "tags", model.History, model.HistoryTagAssociation, filterable="advanced"), grids.GridColumn( "Last Updated", key="update_time", format=time_ago ), + # Columns that are valid for filtering but are not visible. + DeletedColumn( "Deleted", key="deleted", visible=False, filterable="advanced" ), + SharingColumn( "Shared", key="shared", visible=False, filterable="advanced" ), ] columns.append( grids.MulticolFilterColumn( @@ -91,6 +137,8 @@ cols_to_filter=[ columns[0], columns[1] ], key="free-text-search", visible=False, filterable="standard" ) ) + def apply_default_filter( self, trans, query, **kwargs ): + return query.filter_by( user=trans.user, purged=False ) class PageController( BaseController ): @@ -268,4 +316,4 @@ @web.require_login("select a history from saved histories") def list_histories_for_selection( self, trans, **kwargs ): # Render the list view - return self._history_selection_grid( trans, status=status, message=message, **kwargs ) \ No newline at end of file + return self._history_selection_grid( trans, **kwargs ) \ No newline at end of file diff -r 555afd0bf457 -r 37406d8ad116 lib/galaxy/web/controllers/requests.py --- a/lib/galaxy/web/controllers/requests.py Tue Nov 17 16:14:54 2009 -0500 +++ b/lib/galaxy/web/controllers/requests.py Tue Nov 17 16:16:26 2009 -0500 @@ -12,57 +12,109 @@ log = logging.getLogger( __name__ ) -class RequestsListGrid( grids.Grid ): +class RequestsGrid( grids.Grid ): + # Custom column types + class NameColumn( grids.TextColumn ): + def get_value(self, trans, grid, request): + return request.name + class DescriptionColumn( grids.TextColumn ): + def get_value(self, trans, grid, request): + return request.desc + class SamplesColumn( grids.GridColumn ): + def get_value(self, trans, grid, request): + return str(len(request.samples)) + class TypeColumn( grids.TextColumn ): + def get_value(self, trans, grid, request): + return request.type.name + class LastUpdateColumn( grids.TextColumn ): + def get_value(self, trans, grid, request): + return request.update_time + class StateColumn( grids.GridColumn ): + def filter( self, db_session, query, column_filter ): + """ Modify query to filter request by state. """ + if column_filter == "All": + return query + if column_filter: + query = query.filter( model.Request.state == column_filter ) + return query + def get_accepted_filters( self ): + """ Returns a list of accepted filters for this column. """ + accepted_filter_labels_and_vals = [ model.Request.states.UNSUBMITTED, + model.Request.states.SUBMITTED, + model.Request.states.COMPLETE, + "All"] + accepted_filters = [] + for val in accepted_filter_labels_and_vals: + label = val.lower() + args = { self.key: val } + accepted_filters.append( grids.GridColumnFilter( label, args) ) + return accepted_filters + class DeletedColumn( grids.GridColumn ): + def get_accepted_filters( self ): + """ Returns a list of accepted filters for this column. """ + accepted_filter_labels_and_vals = { "active" : "False", "deleted" : "True", "all": "All" } + accepted_filters = [] + for label, val in accepted_filter_labels_and_vals.items(): + args = { self.key: val } + accepted_filters.append( grids.GridColumnFilter( label, args) ) + return accepted_filters + # Grid definition title = "Sequencing Requests" - template = '/requests/grid.mako' + template = 'requests/grid.mako' model_class = model.Request default_sort_key = "-create_time" - show_filter = model.Request.states.UNSUBMITTED + num_rows_per_page = 50 + preserve_state = True + use_paging = True + default_filter = dict( deleted="False", state=model.Request.states.UNSUBMITTED) columns = [ - grids.GridColumn( "Name", key="name", - link=( lambda item: iff( item.deleted, None, dict( operation="show_request", id=item.id ) ) ), - attach_popup=True ), - grids.GridColumn( "Description", key='desc'), - grids.GridColumn( "Sample(s)", method='number_of_samples', - link=( lambda item: iff( item.deleted, None, dict( operation="show_request", id=item.id ) ) ), ), - grids.GridColumn( "Type", key="request_type_id", method='get_request_type'), - grids.GridColumn( "Last update", key="update_time", format=time_ago ), - grids.GridColumn( "State", key='state'), + NameColumn( "Name", + key="name", + model_class=model.Request, + link=( lambda item: iff( item.deleted, None, dict( operation="show_request", id=item.id ) ) ), + attach_popup=True, + filterable="advanced" ), + DescriptionColumn( "Description", + key='desc', + model_class=model.Request, + filterable="advanced" ), + SamplesColumn( "Sample(s)", + link=( lambda item: iff( item.deleted, None, dict( operation="show_request", id=item.id ) ) ), ), + TypeColumn( "Type" ), + LastUpdateColumn( "Last update", + format=time_ago ), + StateColumn( "State", + key='state', + filterable="advanced"), + DeletedColumn( "Deleted", + key="deleted", + visible=True, + filterable="advanced" ) ] + columns.append( grids.MulticolFilterColumn( "Search", + cols_to_filter=[ columns[0], columns[1] ], + key="free-text-search", + visible=False, + filterable="standard" ) ) operations = [ grids.GridOperation( "Submit", allow_multiple=False, condition=( lambda item: not item.deleted and item.unsubmitted() and item.samples ) ), grids.GridOperation( "Edit", allow_multiple=False, condition=( lambda item: not item.deleted and item.unsubmitted() ) ), - grids.GridOperation( "Delete", allow_multiple=False, condition=( lambda item: not item.deleted and item.unsubmitted() ) ), - grids.GridOperation( "Undelete", allow_multiple=False, condition=( lambda item: item.deleted ) ) + grids.GridOperation( "Delete", allow_multiple=True, condition=( lambda item: not item.deleted and item.unsubmitted() ) ), + grids.GridOperation( "Undelete", allow_multiple=True, condition=( lambda item: item.deleted ) ) ] - standard_filters = [ - grids.GridColumnFilter( model.Request.states.UNSUBMITTED, - args=dict( state=model.Request.states.UNSUBMITTED, deleted=False ) ), - grids.GridColumnFilter( model.Request.states.SUBMITTED, - args=dict( state=model.Request.states.SUBMITTED, deleted=False ) ), - grids.GridColumnFilter( model.Request.states.COMPLETE, args=dict( state=model.Request.states.COMPLETE, deleted=False ) ), - grids.GridColumnFilter( "Deleted", args=dict( deleted=True ) ), - grids.GridColumnFilter( "All", args={} ) + global_actions = [ + grids.GridAction( "Create new request", dict( controller='requests', + action='new', + select_request_type='True' ) ) ] - #default_filter = dict( deleted=False ) - def get_current_item( self, trans ): - return None - def get_request_type(self, trans, request): - return request.type.name - def apply_default_filter( self, trans, query, **kwargs ): - query = query.filter_by( user=trans.user ) - if self.default_filter: - return query.filter_by( **self.default_filter ) - else: - return query - def number_of_samples(self, trans, request): - return str(len(request.samples)) - def get_state(self, trans, request): - return request.state + def apply_default_filter( self, trans, query, **kwd ): + return query.filter_by( user=trans.user ) + def build_initial_query( self, session ): + return session.query( self.model_class ) class Requests( BaseController ): - request_grid = RequestsListGrid() + request_grid = RequestsGrid() @web.expose @web.require_login( "create/submit sequencing requests" ) @@ -71,50 +123,43 @@ @web.expose @web.require_login( "create/submit sequencing requests" ) - def list( self, trans, **kwargs ): + def list( self, trans, **kwd ): ''' List all request made by the current user ''' - status = message = None - self.request_grid.default_filter = dict(state=trans.app.model.Request.states.UNSUBMITTED, - deleted=False) - if 'operation' in kwargs: - operation = kwargs['operation'].lower() + + if 'operation' in kwd: + operation = kwd['operation'].lower() + if not kwd.get( 'id', None ): + return trans.response.send_redirect( web.url_for( controller='requests', + action='list', + status='error', + message="Invalid request ID") ) if operation == "show_request": - id = trans.security.decode_id(kwargs['id']) - return self.__show_request(trans, id, kwargs.get('add_sample', False)) + return self.__show_request( trans, **kwd ) elif operation == "submit": - id = trans.security.decode_id(kwargs['id']) - return self.__submit_request(trans, id) + return self.__submit_request( trans, **kwd ) elif operation == "delete": - id = trans.security.decode_id(kwargs['id']) - return self.__delete_request(trans, id) + return self.__delete_request( trans, **kwd ) elif operation == "undelete": - id = trans.security.decode_id(kwargs['id']) - return self.__undelete_request(trans, id) + return self.__undelete_request( trans, **kwd ) elif operation == "edit": - id = trans.security.decode_id(kwargs['id']) - return self.__edit_request(trans, id) - if 'show_filter' in kwargs.keys(): - if kwargs['show_filter'] == 'All': - self.request_grid.default_filter = {} - elif kwargs['show_filter'] == 'Deleted': - self.request_grid.default_filter = dict(deleted=True) - else: - self.request_grid.default_filter = dict(state=kwargs['show_filter'], deleted=False) - self.request_grid.show_filter = kwargs.get('show_filter', trans.app.model.Request.states.UNSUBMITTED) + return self.__edit_request( trans, **kwd ) # Render the list view - return self.request_grid( trans, **kwargs ) + return self.request_grid( trans, **kwd ) - def __show_request(self, trans, id, add_sample=False): + def __show_request(self, trans, **kwd): + params = util.Params( kwd ) + msg = util.restore_text( params.get( 'msg', '' ) ) + messagetype = params.get( 'messagetype', 'done' ) + add_sample = params.get('add_sample', False) try: - request = trans.sa_session.query( trans.app.model.Request ).get( id ) + request = trans.sa_session.query( trans.app.model.Request ).get( trans.security.decode_id(kwd['id']) ) except: return trans.response.send_redirect( web.url_for( controller='requests', action='list', status='error', - message="Invalid request ID", - **kwd) ) + message="Invalid request ID" ) ) current_samples = [] for s in request.samples: current_samples.append([s.name, s.values.content]) @@ -122,10 +167,11 @@ current_samples.append(['Sample_%i' % (len(current_samples)+1),['' for field in request.type.sample_form.fields]]) return trans.fill_template( '/requests/show_request.mako', request=request, - request_details=self.request_details(trans, id), + request_details=self.request_details(trans, request.id), current_samples = current_samples, sample_copy=self.__copy_sample(current_samples), - details='hide', edit_mode='False') + details='hide', edit_mode='False', + msg=msg, messagetype=messagetype ) def request_details(self, trans, id): ''' Shows the request details @@ -685,7 +731,7 @@ message="Invalid request ID", **kwd) ) if params.get('show', False) == 'True': - return self.__edit_request(trans, request.id, **kwd) + return self.__edit_request(trans, **kwd) elif params.get('save_changes_request_button', False) == 'Save changes' \ or params.get('edit_samples_button', False) == 'Edit samples': request_type = trans.sa_session.query( trans.app.model.RequestType ).get( int( params.select_request_type ) ) @@ -714,11 +760,11 @@ messagetype='done', **new_kwd) ) elif params.get('refresh', False) == 'true': - return self.__edit_request(trans, request.id, **kwd) + return self.__edit_request(trans, **kwd) - def __edit_request(self, trans, id, **kwd): + def __edit_request(self, trans, **kwd): try: - request = trans.sa_session.query( trans.app.model.Request ).get( id ) + request = trans.sa_session.query( trans.app.model.Request ).get( trans.security.decode_id(kwd['id']) ) except: msg = "Invalid request ID" log.warn( msg ) @@ -758,59 +804,61 @@ msg=msg, messagetype=messagetype) return self.__show_request_form(trans) - def __delete_request(self, trans, id): - try: - request = trans.sa_session.query( trans.app.model.Request ).get( id ) - except: - msg = "Invalid request ID" - log.warn( msg ) - return trans.response.send_redirect( web.url_for( controller='requests', - action='list', - status='error', - message=msg, - **kwd) ) - # change request's submitted field - if not request.unsubmitted(): - return trans.response.send_redirect( web.url_for( controller='requests', - action='list', - status='error', - message='This request cannot be deleted as it is already been submitted', - **kwd) ) - request.deleted = True - trans.sa_session.add( request ) - trans.sa_session.flush() - kwd = {} - kwd['id'] = trans.security.encode_id(request.id) + def __delete_request(self, trans, **kwd): + id_list = util.listify( kwd['id'] ) + delete_failed = [] + for id in id_list: + try: + request = trans.sa_session.query( trans.app.model.Request ).get( trans.security.decode_id(id) ) + except: + msg = "Invalid request ID" + log.warn( msg ) + return trans.response.send_redirect( web.url_for( controller='requests', + action='list', + status='error', + message=msg, + **kwd) ) + # a request cannot be deleted once its submitted + if not request.unsubmitted(): + delete_failed.append(request.name) + else: + request.deleted = True + trans.sa_session.add( request ) + trans.sa_session.flush() + if not len(delete_failed): + msg = '%i request(s) has been deleted.' % len(id_list) + status = 'done' + else: + msg = '%i request(s) has been deleted. %i request %s could not be deleted as they have been submitted.' % (len(id_list)-len(delete_failed), + len(delete_failed), str(delete_failed)) + status = 'warning' + return trans.response.send_redirect( web.url_for( controller='requests', + action='list', + status=status, + message=msg) ) + def __undelete_request(self, trans, **kwd): + id_list = util.listify( kwd['id'] ) + for id in id_list: + try: + request = trans.sa_session.query( trans.app.model.Request ).get( trans.security.decode_id(id) ) + except: + msg = "Invalid request ID" + log.warn( msg ) + return trans.response.send_redirect( web.url_for( controller='requests', + action='list', + status='error', + message=msg, + **kwd) ) + request.deleted = False + trans.sa_session.add( request ) + trans.sa_session.flush() return trans.response.send_redirect( web.url_for( controller='requests', action='list', status='done', - message='The request <b>%s</b> has been deleted.' % request.name, - **kwd) ) - def __undelete_request(self, trans, id): + message='%i request(s) has been undeleted.' % len(id_list) ) ) + def __submit_request(self, trans, **kwd): try: - request = trans.sa_session.query( trans.app.model.Request ).get( id ) - except: - msg = "Invalid request ID" - log.warn( msg ) - return trans.response.send_redirect( web.url_for( controller='requests', - action='list', - status='error', - message=msg, - **kwd) ) - # change request's submitted field - request.deleted = False - trans.sa_session.add( request ) - trans.sa_session.flush() - kwd = {} - kwd['id'] = trans.security.encode_id(request.id) - return trans.response.send_redirect( web.url_for( controller='requests', - action='list', - status='done', - message='The request <b>%s</b> has been undeleted.' % request.name, - **kwd) ) - def __submit_request(self, trans, id): - try: - request = trans.sa_session.query( trans.app.model.Request ).get( id ) + request = trans.sa_session.query( trans.app.model.Request ).get( trans.security.decode_id(kwd['id']) ) except: msg = "Invalid request ID" log.warn( msg ) @@ -837,14 +885,12 @@ request.state = request.states.SUBMITTED trans.sa_session.add( request ) trans.sa_session.flush() - kwd = {} - kwd['id'] = trans.security.encode_id(request.id) - kwd['status'] = 'done' - kwd['message'] = 'The request <b>%s</b> has been submitted.' % request.name return trans.response.send_redirect( web.url_for( controller='requests', action='list', - show_filter=trans.app.model.Request.states.SUBMITTED, - **kwd) ) + id=trans.security.encode_id(request.id), + status='done', + message='The request <b>%s</b> has been submitted.' % request.name + ) ) @web.expose @web.require_login( "create/submit sequencing requests" ) def show_events(self, trans, **kwd): diff -r 555afd0bf457 -r 37406d8ad116 lib/galaxy/web/controllers/requests_admin.py --- a/lib/galaxy/web/controllers/requests_admin.py Tue Nov 17 16:14:54 2009 -0500 +++ b/lib/galaxy/web/controllers/requests_admin.py Tue Nov 17 16:16:26 2009 -0500 @@ -15,63 +15,187 @@ # ---- Request Grid ------------------------------------------------------------ # -class RequestsListGrid( grids.Grid ): +class RequestsGrid( grids.Grid ): + # Custom column types + class NameColumn( grids.TextColumn ): + def get_value(self, trans, grid, request): + return request.name + class DescriptionColumn( grids.TextColumn ): + def get_value(self, trans, grid, request): + return request.desc + class SamplesColumn( grids.GridColumn ): + def get_value(self, trans, grid, request): + return str(len(request.samples)) + class TypeColumn( grids.TextColumn ): + def get_value(self, trans, grid, request): + return request.type.name + class LastUpdateColumn( grids.TextColumn ): + def get_value(self, trans, grid, request): + return request.update_time + class StateColumn( grids.GridColumn ): + def filter( self, db_session, query, column_filter ): + """ Modify query to filter request by state. """ + if column_filter == "All": + return query + if column_filter: + query = query.filter( model.Request.state == column_filter ) + return query + def get_accepted_filters( self ): + """ Returns a list of accepted filters for this column. """ + accepted_filter_labels_and_vals = [ model.Request.states.UNSUBMITTED, + model.Request.states.SUBMITTED, + model.Request.states.COMPLETE, + "All"] + accepted_filters = [] + for val in accepted_filter_labels_and_vals: + label = val.lower() + args = { self.key: val } + accepted_filters.append( grids.GridColumnFilter( label, args) ) + return accepted_filters + class UserColumn( grids.TextColumn ): + def get_value(self, trans, grid, request): + return request.user.email + class DeletedColumn( grids.GridColumn ): + def get_accepted_filters( self ): + """ Returns a list of accepted filters for this column. """ + accepted_filter_labels_and_vals = { "active" : "False", "deleted" : "True", "all": "All" } + accepted_filters = [] + for label, val in accepted_filter_labels_and_vals.items(): + args = { self.key: val } + accepted_filters.append( grids.GridColumnFilter( label, args) ) + return accepted_filters + # Grid definition title = "Sequencing Requests" template = "admin/requests/grid.mako" model_class = model.Request default_sort_key = "-create_time" - show_filter = model.Request.states.SUBMITTED + num_rows_per_page = 50 + preserve_state = True + use_paging = True + default_filter = dict( deleted="False", state=model.Request.states.SUBMITTED) columns = [ - grids.GridColumn( "Name", key="name", - link=( lambda item: iff( item.deleted, None, dict( operation="show_request", id=item.id ) ) ), - attach_popup=True ), - grids.GridColumn( "Description", key="desc"), - grids.GridColumn( "Sample(s)", method='number_of_samples', - link=( lambda item: iff( item.deleted, None, dict( operation="show_request", id=item.id ) ) ), ), - grids.GridColumn( "Type", key="request_type_id", method='get_request_type'), - grids.GridColumn( "Last update", key="update_time", format=time_ago ), - grids.GridColumn( "State", key='state'), - grids.GridColumn( "User", key="user_id", method='get_user') - + NameColumn( "Name", + key="name", + model_class=model.Request, + link=( lambda item: iff( item.deleted, None, dict( operation="show_request", id=item.id ) ) ), + attach_popup=True, + filterable="advanced" ), + DescriptionColumn( "Description", + key='desc', + model_class=model.Request, + filterable="advanced" ), + SamplesColumn( "Sample(s)", + link=( lambda item: iff( item.deleted, None, dict( operation="show_request", id=item.id ) ) ), ), + TypeColumn( "Type" ), + LastUpdateColumn( "Last update", + format=time_ago ), + StateColumn( "State", + key='state', + filterable="advanced"), + UserColumn( "User", + key='user.email', + model_class=model.Request, + filterable="advanced" ), + DeletedColumn( "Deleted", + key="deleted", + visible=True, + filterable="advanced" ) ] + columns.append( grids.MulticolFilterColumn( "Search", + cols_to_filter=[ columns[0], columns[1], columns[6] ], + key="free-text-search", + visible=False, + filterable="standard" ) ) operations = [ grids.GridOperation( "Submit", allow_multiple=False, condition=( lambda item: not item.deleted and item.unsubmitted() and item.samples ) ), grids.GridOperation( "Edit", allow_multiple=False, condition=( lambda item: not item.deleted ) ), grids.GridOperation( "Reject", allow_multiple=False, condition=( lambda item: not item.deleted and item.submitted() ) ), - grids.GridOperation( "Delete", allow_multiple=False, condition=( lambda item: not item.deleted and item.unsubmitted() ) ), + grids.GridOperation( "Delete", allow_multiple=True, condition=( lambda item: not item.deleted and item.unsubmitted() ) ), grids.GridOperation( "Undelete", condition=( lambda item: item.deleted ) ), ] - standard_filters = [ - grids.GridColumnFilter( model.Request.states.UNSUBMITTED, - args=dict( state=model.Request.states.UNSUBMITTED, deleted=False ) ), - grids.GridColumnFilter( model.Request.states.SUBMITTED, - args=dict( state=model.Request.states.SUBMITTED, deleted=False ) ), - grids.GridColumnFilter( model.Request.states.COMPLETE, args=dict( state=model.Request.states.COMPLETE, deleted=False ) ), - grids.GridColumnFilter( "Deleted", args=dict( deleted=True ) ), - grids.GridColumnFilter( "All", args=dict( deleted=False ) ) + global_actions = [ + grids.GridAction( "Create new request", dict( controller='requests_admin', + action='new', + select_request_type='True' ) ) ] - def get_user(self, trans, request): - return trans.sa_session.query( trans.app.model.User ).get( request.user_id ).email - def get_current_item( self, trans ): - return None - def get_request_type(self, trans, request): - request_type = trans.sa_session.query( trans.app.model.RequestType ).get( request.request_type_id ) - return request_type.name - def number_of_samples(self, trans, request): - return str(len(request.samples)) - def apply_default_filter( self, trans, query, **kwargs ): - if self.default_filter: - return query.filter_by( **self.default_filter ) - else: - return query - + +# +# ---- Request Type Gridr ------------------------------------------------------ +# +class RequestTypeGrid( grids.Grid ): + # Custom column types + class NameColumn( grids.TextColumn ): + def get_value(self, trans, grid, request_type): + return request_type.name + class DescriptionColumn( grids.TextColumn ): + def get_value(self, trans, grid, request_type): + return request_type.desc + class RequestFormColumn( grids.TextColumn ): + def get_value(self, trans, grid, request_type): + return request_type.request_form.name + class SampleFormColumn( grids.TextColumn ): + def get_value(self, trans, grid, request_type): + return request_type.sample_form.name + class DeletedColumn( grids.GridColumn ): + def get_accepted_filters( self ): + """ Returns a list of accepted filters for this column. """ + accepted_filter_labels_and_vals = { "active" : "False", "deleted" : "True", "all": "All" } + accepted_filters = [] + for label, val in accepted_filter_labels_and_vals.items(): + args = { self.key: val } + accepted_filters.append( grids.GridColumnFilter( label, args) ) + return accepted_filters + # Grid definition + title = "Requests Types" + template = "admin/requests/manage_request_types.mako" + model_class = model.RequestType + default_sort_key = "-create_time" + num_rows_per_page = 50 + preserve_state = True + use_paging = True + default_filter = dict( deleted="False" ) + columns = [ + NameColumn( "Name", + key="name", + model_class=model.RequestType, + link=( lambda item: iff( item.deleted, None, dict( operation="view", id=item.id ) ) ), + attach_popup=True, + filterable="advanced" ), + DescriptionColumn( "Description", + key='desc', + model_class=model.Request, + filterable="advanced" ), + RequestFormColumn( "Request Form", + link=( lambda item: iff( item.deleted, None, dict( operation="view_form", id=item.request_form.id ) ) ), ), + SampleFormColumn( "Sample Form", + link=( lambda item: iff( item.deleted, None, dict( operation="view_form", id=item.sample_form.id ) ) ), ), + DeletedColumn( "Deleted", + key="deleted", + visible=False, + filterable="advanced" ) + ] + columns.append( grids.MulticolFilterColumn( "Search", + cols_to_filter=[ columns[0], columns[1] ], + key="free-text-search", + visible=False, + filterable="standard" ) ) + operations = [ + #grids.GridOperation( "Update", allow_multiple=False, condition=( lambda item: not item.deleted ) ), + grids.GridOperation( "Delete", allow_multiple=True, condition=( lambda item: not item.deleted ) ), + grids.GridOperation( "Undelete", condition=( lambda item: item.deleted ) ), + ] + global_actions = [ + grids.GridAction( "Create new request type", dict( controller='requests_admin', + action='create_request_type' ) ) + ] # # ---- Request Controller ------------------------------------------------------ #

1 0

[hg] galaxy 3097: Moved cloud control flag in universe.ini file ...
by Greg Von Kuster 23 Nov '09

23 Nov '09

details: http://www.bx.psu.edu/hg/galaxy/rev/555afd0bf457 changeset: 3097:555afd0bf457 user: Enis Afgan <afgane(a)gmail.com> date: Tue Nov 17 16:14:54 2009 -0500 description: Moved cloud control flag in universe.ini file to appropriate location. diffstat: universe_wsgi.ini.sample | 10 +++++----- 1 files changed, 5 insertions(+), 5 deletions(-) diffs (27 lines): diff -r 5e2fd6248f71 -r 555afd0bf457 universe_wsgi.ini.sample --- a/universe_wsgi.ini.sample Tue Nov 17 15:51:39 2009 -0500 +++ b/universe_wsgi.ini.sample Tue Nov 17 16:14:54 2009 -0500 @@ -18,11 +18,6 @@ # [filter:gzip] # use = egg:Paste#gzip -# ---- Cloud Management -------------------------------------------------------- - -# Uncomment following line to enable cloud management mode (only leaves the Cloud, Help and User tabs on) -# cloud_controller_instance = True - # ---- Galaxy Web Interface ------------------------------------------------- [app:main] @@ -197,6 +192,11 @@ # Should default dataset access permissions be private for new users; default is False (datasets are public) new_user_dataset_access_role_default_private = False +# ---- Cloud Management -------------------------------------------------------- + +# Uncomment following line to enable cloud management mode (only leaves the Cloud, Help and User tabs on) +# cloud_controller_instance = True + # ---- Job Execution -------------------------------------------------------- # Number of concurrent jobs to run (local job runner)

1 0

[hg] galaxy 3095: Ported DB interaction to be compatible with Sq...
by Greg Von Kuster 23 Nov '09

23 Nov '09

details: http://www.bx.psu.edu/hg/galaxy/rev/39502dd3fd23 changeset: 3095:39502dd3fd23 user: Enis Afgan <afgane(a)gmail.com> date: Mon Nov 16 20:37:47 2009 -0500 description: Ported DB interaction to be compatible with Sqlalchemy 0.5.6 diffstat: lib/galaxy/cloud/__init__.py | 425 +++++++++++++++++++---------------- lib/galaxy/cloud/providers/ec2.py | 341 ++++++++++++++++----------- lib/galaxy/cloud/providers/eucalyptus.py | 432 ++++++++++++++++++++++-------------- lib/galaxy/config.py | 9 +- lib/galaxy/web/controllers/cloud.py | 151 +++++------ templates/cloud/configure_cloud.mako | 3 +- 6 files changed, 768 insertions(+), 593 deletions(-) diffs (2608 lines): diff -r 7d013eb98022 -r 39502dd3fd23 lib/galaxy/cloud/__init__.py --- a/lib/galaxy/cloud/__init__.py Thu Nov 12 16:36:07 2009 -0500 +++ b/lib/galaxy/cloud/__init__.py Mon Nov 16 20:37:47 2009 -0500 @@ -60,6 +60,7 @@ """ def __init__( self, app ): self.app = app + self.sa_session = app.model.context if self.app.config.get_bool( "enable_cloud_execution", True ): # The dispatcher manager for underlying cloud instances - implements and contacts individual cloud providers self.provider = CloudProvider( app ) @@ -99,6 +100,7 @@ # Keep track of the pid that started the cloud manager, only it # has valid threads self.parent_pid = os.getpid() + self.sa_session = app.model.context # Contains requests that are waiting (only use from monitor thread) self.waiting = [] @@ -122,7 +124,6 @@ cnt = 0 # Run global update only periodically so keep counter variable while self.running: try: -# log.debug( "Calling monitor_step" ) self.__monitor_step() if cnt%30 == 0: # Run global update every 30 iterations (1 minute) self.provider.update() @@ -144,27 +145,23 @@ it is marked as having errors and removed from the queue. Otherwise, the job is dispatched. """ - # Get an orm (object relational mapping) session - session = mapping.Session() + model = self.app.model new_requests = [] - for r in session.query( model.UCI ) \ - .filter( or_( model.UCI.c.state==uci_states.NEW_UCI, - model.UCI.c.state==uci_states.SUBMITTED_UCI, - model.UCI.c.state==uci_states.SHUTTING_DOWN_UCI, - model.UCI.c.state==uci_states.DELETING_UCI, - model.UCI.c.state==uci_states.SNAPSHOT_UCI ) ) \ + for r in self.sa_session.query( model.UCI ) \ + .filter( or_( model.UCI.table.c.state==uci_states.NEW_UCI, + model.UCI.table.c.state==uci_states.SUBMITTED_UCI, + model.UCI.table.c.state==uci_states.SHUTTING_DOWN_UCI, + model.UCI.table.c.state==uci_states.DELETING_UCI, + model.UCI.table.c.state==uci_states.SNAPSHOT_UCI ) ) \ .all(): - uci_wrapper = UCIwrapper( r ) + uci_wrapper = UCIwrapper( r, self.app ) new_requests.append( uci_wrapper ) for uci_wrapper in new_requests: - session.clear() + self.sa_session.expunge_all() self.put( uci_wrapper ) - - # Done with the session - mapping.Session.remove() - + def put( self, uci_wrapper ): """Add a request to the queue.""" self.provider.put( uci_wrapper ) @@ -186,8 +183,10 @@ """ Wraps 'model.UCI' with convenience methods for state management """ - def __init__( self, uci ): + def __init__( self, uci, app ): self.uci_id = uci.id + self.app = app + self.sa_session = self.app.model.context # --------- Setter methods ----------------- @@ -199,56 +198,61 @@ """ # log.debug( "Changing state - new uci_state: %s, instance_id: %s, i_state: %s" % ( uci_state, instance_id, i_state ) ) if uci_state is not None: - uci = model.UCI.get( self.uci_id ) - uci.refresh() + uci = self.sa_session.query( model.UCI ).get( self.uci_id ) + self.sa_session.refresh( uci ) uci.state = uci_state - uci.flush() + self.sa_session.flush() if ( instance_id is not None ) and ( i_state is not None ): - instance = model.CloudInstance.filter_by( uci_id=self.uci_id, instance_id=instance_id).first() + instance = self.sa_session.query( model.CloudInstance ).filter_by( uci_id=self.uci_id, instance_id=instance_id).first() instance.state = i_state - instance.flush() + self.sa_session.add( instance ) + self.sa_session.flush() def set_mi( self, i_index, mi_id ): """ Sets Machine Image (MI), e.g., 'ami-66fa190f', for UCI's instance with given index as it is stored in local Galaxy database. """ - mi = model.CloudImage.filter( model.CloudImage.c.image_id==mi_id ).first() - instance = model.CloudInstance.get( i_index ) + mi = self.sa_session.query( model.CloudImage ).filter( model.CloudImage.table.c.image_id==mi_id ).first() + instance = self.sa_session.query( model.CloudInstance ).get( i_index ) instance.image = mi - instance.flush() + self.sa_session.add( instance ) + self.sa_session.flush() def set_key_pair( self, key_name, key_material=None ): """ Sets key pair value for current UCI. """ - uci = model.UCI.get( self.uci_id ) - uci.refresh() + uci = self.sa_session.query( model.UCI ).get( self.uci_id ) + self.sa_session.refresh( uci ) uci.key_pair_name = key_name if key_material is not None: uci.key_pair_material = key_material - uci.flush() + self.sa_session.flush() - def set_launch_time( self, launch_time, i_index=None, i_id=None ): + def set_instance_launch_time( self, launch_time, i_index=None, i_id=None ): """ Stores launch time in local database for instance with specified index - i_index (as it is stored in local Galaxy database) or with specified instance ID - i_id (as obtained from the cloud provider AND stored in local Galaxy Database). Either 'i_index' or 'i_id' needs to be provided. """ if i_index != None: - instance = model.CloudInstance.get( i_index ) - instance.launch_time = launch_time - instance.flush() + instance = self.sa_session.query( model.CloudInstance ).get( i_index ) elif i_id != None: - instance = model.CloudInstance.filter_by( uci_id=self.uci_id, instance_id=i_id).first() - instance.launch_time = launch_time - instance.flush() + instance = self.sa_session.query( model.CloudInstance ).filter_by( uci_id=self.uci_id, instance_id=i_id).first() + else: + return None + + instance.launch_time = launch_time + self.sa_session.add( instance ) + self.sa_session.flush() def set_uci_launch_time( self, launch_time ): - uci = model.UCI.get( self.uci_id ) - uci.refresh() + uci = self.sa_session.query( model.UCI ).get( self.uci_id ) + self.sa_session.refresh( uci ) uci.launch_time = launch_time - uci.flush() + self.sa_session.add( uci ) + self.sa_session.flush() def set_stop_time( self, stop_time, i_index=None, i_id=None ): """ @@ -257,20 +261,16 @@ in local Galaxy Database). Either 'i_index' or 'i_id' needs to be provided. """ if i_index != None: - instance = model.CloudInstance.get( i_index ) - instance.stop_time = stop_time - instance.flush() + instance = self.sa_session.query( model.CloudInstance ).get( i_index ) elif i_id != None: - instance = model.CloudInstance.filter_by( uci_id=self.uci_id, instance_id=i_id).first() - instance.stop_time = stop_time - instance.flush() + instance = self.sa_session.query( model.CloudInstance ).filter_by( uci_id=self.uci_id, instance_id=i_id).first() + else: + return None + + instance.stop_time = stop_time + self.sa_session.add( instance ) + self.sa_session.flush() - def reset_uci_launch_time( self ): - uci = model.UCI.get( self.uci_id ) - uci.refresh() - uci.launch_time = None - uci.flush() - def set_security_group_name( self, security_group_name, i_index=None, i_id=None ): """ Stores security group name in local database for instance with specified index - i_index (as it is stored in local @@ -278,90 +278,107 @@ in local Galaxy Database). Either 'i_index' or 'i_id' needs to be provided. """ if i_index != None: - instance = model.CloudInstance.get( i_index ) - instance.security_group = security_group_name - instance.flush() + instance = self.sa_session.query( model.CloudInstance ).get( i_index ) elif i_id != None: - instance = model.CloudInstance.filter_by( uci_id=self.uci_id, instance_id=i_id).first() - instance.security_group = security_group_name - instance.flush() + instance = self.sa_session.query( model.CloudInstance ).filter_by( uci_id=self.uci_id, instance_id=i_id).first() + else: + return None + + instance.security_group = security_group_name + self.sa_session.add( instance ) + self.sa_session.flush() def set_reservation_id( self, i_index, reservation_id ): - instance = model.CloudInstance.get( i_index ) + instance = self.sa_session.query( model.CloudInstance ).get( i_index ) instance.reservation_id = reservation_id - instance.flush() + self.sa_session.add( instance ) + self.sa_session.flush() def set_instance_id( self, i_index, instance_id ): """ i_index refers to UCI's instance ID as stored in local database instance_id refers to real-world, cloud resource ID (e.g., 'i-78hd823a') """ - instance = model.CloudInstance.get( i_index ) + instance = self.sa_session.query( model.CloudInstance ).get( i_index ) instance.instance_id = instance_id - instance.flush() + self.sa_session.add( instance ) + self.sa_session.flush() - def set_public_dns( self, instance_id, public_dns ): - uci = model.UCI.get( self.uci_id ) - uci.refresh() - uci.instance[instance_id].public_dns = public_dns - uci.instance[instance_id].flush() +# def set_public_dns( self, instance_id, public_dns ): +# uci = self.sa_session.query( model.UCI ).get( self.uci_id ) +# self.sa_session.refresh( uci ) +# uci.instance[instance_id].public_dns = public_dns +# uci.instance[instance_id].flush() +# +# def set_private_dns( self, instance_id, private_dns ): +# uci = self.sa_session.query( model.UCI ).get( self.uci_id ) +# self.sa_session.refresh( uci ) +# uci.instance[instance_id].private_dns = private_dns +# uci.instance[instance_id].flush() - def set_private_dns( self, instance_id, private_dns ): - uci = model.UCI.get( self.uci_id ) - uci.refresh() - uci.instance[instance_id].private_dns = private_dns - uci.instance[instance_id].flush() - + def reset_uci_launch_time( self ): + uci = self.sa_session.query( model.UCI ).get( self.uci_id ) + self.sa_session.refresh( uci ) + uci.launch_time = None + self.sa_session.add( uci ) + self.sa_session.flush() + + def set_error( self, error, set_state=False ): + """ + Sets error field of given UCI in local Galaxy database as well as any instances associated with + this UCI whose state is 'None' or 'SUBMITTED'. If set_state is set to 'true', + method also sets state of give UCI and corresponding instances to 'error' + """ + uci = self.sa_session.query( model.UCI ).get( self.uci_id ) + self.sa_session.refresh( uci ) + uci.error = error + if set_state: + uci.state = uci_states.ERROR + # Process all instances associated with this UCI + instances = self.sa_session.query( model.CloudInstance ) \ + .filter_by( uci=uci ) \ + .filter( or_( model.CloudInstance.table.c.state==None, model.CloudInstance.table.c.state==instance_states.SUBMITTED ) ) \ + .all() + for i in instances: + i.error = error + i.state = instance_states.ERROR + self.sa_session.add( i ) + self.sa_session.flush() + + self.sa_session.add( uci ) + self.sa_session.flush() + + def set_deleted( self ): + uci = self.sa_session.query( model.UCI ).get( self.uci_id ) + self.sa_session.refresh( uci ) + uci.state = uci_states.DELETED # for bookkeeping reasons, mark as deleted but don't actually delete. + uci.deleted = True + self.sa_session.add( uci ) + self.sa_session.flush() + def set_store_device( self, store_id, device ): - uci = model.UCI.get( self.uci_id ) - uci.refresh() + uci = self.sa_session.query( model.UCI ).get( self.uci_id ) + self.sa_session.refresh( uci ) uci.store[store_id].device = device uci.store[store_id].flush() def set_store_error( self, error, store_index=None, store_id=None ): if store_index != None: - store = model.CloudStore.get( store_index ) + store = self.sa_session.query( model.CloudStore ).get( store_index ) elif store_id != None: - store = model.CloudStore.filter_by( volume_id = store_id ).first() + store = self.sa_session.query( model.CloudStore ).filter_by( volume_id = store_id ).first() else: return None store.error = error - store.flush() + self.sa_session.add( store ) + self.sa_session.flush() def set_store_status( self, vol_id, status ): - vol = model.CloudStore.filter( model.CloudStore.c.volume_id == vol_id ).first() + vol = self.sa_session.query( model.CloudStore ).filter( model.CloudStore.table.c.volume_id == vol_id ).first() vol.status = status - vol.flush() - - def set_snapshot_id( self, snap_index, id ): - snap = model.CloudSnapshot.get( snap_index ) - snap.snapshot_id = id - snap.flush() - - def set_snapshot_status( self, status, snap_index=None, snap_id=None ): - if snap_index != None: - snap = model.CloudSnapshot.get( snap_index ) - elif snap_id != None: - snap = model.CloudSnapshot.filter_by( snapshot_id = snap_id).first() - else: - return - snap.status = status - snap.flush() - - def set_snapshot_error( self, error, snap_index=None, snap_id=None, set_status=False ): - if snap_index != None: - snap = model.CloudSnapshot.get( snap_index ) - elif snap_id != None: - snap = model.CloudSnapshot.filter_by( snapshot_id = snap_id).first() - else: - return - snap.error = error - - if set_status: - snap.status = snapshot_status.ERROR - - snap.flush() + self.sa_session.add( vol ) + self.sa_session.flush() def set_store_availability_zone( self, availability_zone, vol_id=None ): """ @@ -369,78 +386,91 @@ UCI or for the volume whose volume ID (e.g., 'vol-39F80512') is provided as argument. """ if vol_id is not None: - vol = model.CloudStore.filter( model.CloudStore.c.volume_id == vol_id ).all() + vol = self.sa_session.query( model.CloudStore ).filter( model.CloudStore.table.c.volume_id == vol_id ).all() else: - vol = model.CloudStore.filter( model.CloudStore.c.uci_id == self.uci_id ).all() + vol = self.sa_session.query( model.CloudStore ).filter( model.CloudStore.table.c.uci_id == self.uci_id ).all() for v in vol: v.availability_zone = availability_zone - v.flush() + self.sa_session.add( v ) + self.sa_session.flush() - def set_store_volume_id( self, store_id, volume_id ): + def set_store_volume_id( self, store_index, volume_id ): """ - Given store ID associated with this UCI, set volume ID as it is registered + Given store index associated with this UCI in local database, set volume ID as it is registered on the cloud provider (e.g., vol-39890501) """ - uci = model.UCI.get( self.uci_id ) - uci.refresh() - uci.store[store_id].volume_id = volume_id - uci.store[store_id].flush() + uci = self.sa_session.query( model.UCI ).get( self.uci_id ) + self.sa_session.refresh( uci ) + uci.store[store_index].volume_id = volume_id + #uci.store[store_index].flush() + self.sa_session.add( uci ) + self.sa_session.flush() def set_store_instance( self, vol_id, instance_id ): """ Stores instance ID that given store volume is attached to. Store volume ID should be given in following format: 'vol-78943248' """ - vol = model.CloudStore.filter( model.CloudStore.c.volume_id == vol_id ).first() + vol = self.sa_session.query( model.CloudStore ).filter( model.CloudStore.table.c.volume_id == vol_id ).first() vol.i_id = instance_id - vol.flush() + self.sa_session.add( vol ) + self.sa_session.flush() + + def set_snapshot_id( self, snap_index, id ): + snap = model.CloudSnapshot.get( snap_index ) - def set_error( self, error, set_state=False ): - """ - Sets error field of given UCI in local Galaxy database as well as any instances associated with - this UCI whose state is 'None' or 'SUBMITTED'. If set_state is set to 'true', - method also sets state of give UCI and corresponding instances to 'error' - """ - uci = model.UCI.get( self.uci_id ) - uci.refresh() - uci.error = error - if set_state: - uci.state = uci_states.ERROR - instances = model.CloudInstance \ - .filter_by( uci=uci ) \ - .filter( or_( model.CloudInstance.c.state==None, model.CloudInstance.c.state==instance_states.SUBMITTED ) ) \ - .all() - for i in instances: - i.error = error - i.state = instance_states.ERROR - i.flush() - uci.flush() + snap.snapshot_id = id + self.sa_session.add( snap ) + self.sa_session.flush() + + def set_snapshot_status( self, status, snap_index=None, snap_id=None ): + if snap_index != None: + snap = self.sa_session.query( model.CloudSnapshot ).get( snap_index ) + elif snap_id != None: + snap = self.sa_session.query( model.CloudSnapshot ).filter_by( snapshot_id = snap_id).first() + else: + return + snap.status = status + self.sa_session.add( snap ) + self.sa_session.flush() - def set_deleted( self ): - uci = model.UCI.get( self.uci_id ) - uci.refresh() - uci.state = uci_states.DELETED # for bookkeeping reasons, mark as deleted but don't actually delete. - uci.deleted = True - uci.flush() + def set_snapshot_error( self, error, snap_index=None, snap_id=None, set_status=False ): + if snap_index != None: + snap = self.sa_session.query( model.CloudSnapshot ).get( snap_index ) + elif snap_id != None: + snap = self.sa_session.query( model.CloudSnapshot ).filter_by( snapshot_id = snap_id).first() + else: + return + snap.error = error + if set_status: + snap.status = snapshot_status.ERROR + + self.sa_session.add( snap ) + self.sa_session.flush() # --------- Getter methods ----------------- def get_provider_type( self ): """ Returns type of cloud provider associated with given UCI. """ - uci = model.UCI.get( self.uci_id ) - uci.refresh() -# cred_id = uci.credentials_id -# cred = model.CloudUserCredentials.get( cred_id ) + uci = self.sa_session.query( model.UCI ).get( self.uci_id ) + self.sa_session.refresh( uci ) return uci.credentials.provider.type - def get_type( self, i_index ): - instance = model.CloudInstance.get( i_index ) + def get_provider( self ): + """ Returns database object of cloud provider associated with credentials of given UCI. """ + uci = self.sa_session.query( model.UCI ).get( self.uci_id ) + self.sa_session.refresh( uci ) + return uci.credentials.provider + + def get_instance_type( self, i_index ): + instance = self.sa_session.query( model.CloudInstance ).get( i_index ) + self.sa_session.refresh( instance ) return instance.type - def get_state( self ): - uci = model.UCI.get( self.uci_id ) - uci.refresh() + def get_uci_state( self ): + uci = self.sa_session.query( model.UCI ).get( self.uci_id ) + self.sa_session.refresh( uci ) return uci.state def get_instances_indexes( self, state=None ): @@ -448,9 +478,12 @@ Returns indexes of instances associated with given UCI as they are stored in local Galaxy database and whose state corresponds to passed argument. Returned values enable indexing instances from local Galaxy database. """ - uci = model.UCI.get( self.uci_id ) - uci.refresh() - instances = model.CloudInstance.filter_by( uci=uci ).filter( model.CloudInstance.c.state==state ).all() + uci = self.sa_session.query( model.UCI ).get( self.uci_id ) + self.sa_session.refresh( uci ) + instances = self.sa_session.query( model.CloudInstance ) \ + .filter_by( uci=uci ) \ + .filter( model.CloudInstance.table.c.state==state ) \ + .all() il = [] for i in instances: il.append( i.id ) @@ -458,40 +491,46 @@ return il def get_instance_state( self, instance_id ): - uci = model.UCI.get( self.uci_id ) - uci.refresh() + uci = self.sa_session.query( model.UCI ).get( self.uci_id ) + self.sa_session.refresh( uci ) return uci.instance[instance_id].state def get_instances_ids( self ): """ - Returns list IDs of all instances' associated with this UCI that are not in 'terminated' state - (e.g., ['i-402906D2', 'i-q0290dsD2'] ). + Returns list IDs of all instances' associated with this UCI that are not in 'terminated' or + 'error' but the state is defined (i.e., state is not None) + (e.g., return value: ['i-402906D2', 'i-q0290dsD2'] ). """ - il = model.CloudInstance.filter_by( uci_id=self.uci_id ).filter( model.CloudInstance.c.state != 'terminated' ).all() + il = self.sa_session.query( model.CloudInstance ) \ + .filter_by( uci_id=self.uci_id ) \ + .filter( or_( model.CloudInstance.table.c.state != 'terminated', + model.CloudInstance.table.c.state != 'error', + model.CloudInstance.table.c.state != None ) ) \ + .all() instanceList = [] for i in il: instanceList.append( i.instance_id ) return instanceList def get_name( self ): - uci = model.UCI.get( self.uci_id ) - uci.refresh() + uci = self.sa_session.query( model.UCI ).get( self.uci_id ) + self.sa_session.refresh( uci ) return uci.name def get_key_pair_name( self ): """ Returns keypair name associated with given UCI. """ - uci = model.UCI.get( self.uci_id ) - uci.refresh() + uci = self.sa_session.query( model.UCI ).get( self.uci_id ) + self.sa_session.refresh( uci ) return uci.key_pair_name def get_key_pair_material( self ): """ Returns keypair material (i.e., private key) associated with given UCI. """ - uci = model.UCI.get( self.uci_id ) - uci.refresh() + uci = self.sa_session.query( model.UCI ).get( self.uci_id ) + self.sa_session.refresh( uci ) return uci.key_pair_material def get_security_group_name( self, i_index=None, i_id=None ): @@ -501,35 +540,35 @@ with given instance. """ if i_index != None: - instance = model.CloudInstance.get( i_index ) + instance = self.sa_session.query( model.CloudInstance ).get( i_index ) return instance.security_group elif i_id != None: - instance = model.CloudInstance.filter_by( uci_id=self.uci_id, instance_id=i_id).first() + instance = self.sa_session.query( model.CloudInstance ).filter_by( uci_id=self.uci_id, instance_id=i_id).first() return instance.security_group def get_access_key( self ): - uci = model.UCI.get( self.uci_id ) - uci.refresh() + uci = self.sa_session.query( model.UCI ).get( self.uci_id ) + self.sa_session.refresh( uci ) return uci.credentials.access_key def get_secret_key( self ): - uci = model.UCI.get( self.uci_id ) - uci.refresh() + uci = self.sa_session.query( model.UCI ).get( self.uci_id ) + self.sa_session.refresh( uci ) return uci.credentials.secret_key def get_mi_id( self, instance_id=0 ): - uci = model.UCI.get( self.uci_id ) - uci.refresh() + uci = self.sa_session.query( model.UCI ).get( self.uci_id ) + self.sa_session.refresh( uci ) return uci.instance[instance_id].mi_id def get_public_dns( self, instance_id=0 ): - uci = model.UCI.get( self.uci_id ) - uci.refresh() + uci = self.sa_session.query( model.UCI ).get( self.uci_id ) + self.sa_session.refresh( uci ) return uci.instance[instance_id].public_dns def get_private_dns( self, instance_id=0 ): - uci = model.UCI.get( self.uci_id ) - uci.refresh() + uci = self.sa_session.query( model.UCI ).get( self.uci_id ) + self.sa_session.refresh( uci ) return uci.instance[instance_id].private_dns def get_uci_availability_zone( self ): @@ -539,13 +578,13 @@ availability zone, availability of a UCI is determined by availability zone of any one storage volume. """ - uci = model.UCI.get( self.uci_id ) - uci.refresh() + uci = self.sa_session.query( model.UCI ).get( self.uci_id ) + self.sa_session.refresh( uci ) return uci.store[0].availability_zone def get_store_size( self, store_id=0 ): - uci = model.UCI.get( self.uci_id ) - uci.refresh() + uci = self.sa_session.query( model.UCI ).get( self.uci_id ) + self.sa_session.refresh( uci ) return uci.store[store_id].size def get_store_volume_id( self, store_id=0 ): @@ -553,33 +592,27 @@ Given store ID associated with this UCI, get volume ID as it is registered on the cloud provider (e.g., 'vol-39890501') """ - uci = model.UCI.get( self.uci_id ) - uci.refresh() + uci = self.sa_session.query( model.UCI ).get( self.uci_id ) + self.sa_session.refresh( uci ) return uci.store[store_id].volume_id def get_all_stores( self ): """ Returns all storage volumes' database objects associated with this UCI. """ - return model.CloudStore.filter( model.CloudStore.c.uci_id == self.uci_id ).all() + return self.sa_session.query( model.CloudStore ).filter( model.CloudStore.table.c.uci_id == self.uci_id ).all() def get_snapshots( self, status=None ): """ Returns database objects for all snapshots associated with this UCI and in given status.""" - return model.CloudSnapshot.filter_by( uci_id=self.uci_id, status=status ).all() + return self.sa_session.query( model.CloudSnapshot ).filter_by( uci_id=self.uci_id, status=status ).all() def get_uci( self ): """ Returns database object for given UCI. """ - uci = model.UCI.get( self.uci_id ) - uci.refresh() + uci = self.sa_session.query( model.UCI ).get( self.uci_id ) + self.sa_session.refresh( uci ) return uci - def get_provider( self ): - """ Returns database object of cloud provider associated with credentials of given UCI. """ - uci = model.UCI.get( self.uci_id ) - uci.refresh() - return uci.credentials.provider - def uci_launch_time_set( self ): - uci = model.UCI.get( self.uci_id ) - uci.refresh() + uci = self.sa_session.query( model.UCI ).get( self.uci_id ) + self.sa_session.refresh( uci ) return uci.launch_time class CloudProvider( object ): diff -r 7d013eb98022 -r 39502dd3fd23 lib/galaxy/cloud/providers/ec2.py --- a/lib/galaxy/cloud/providers/ec2.py Thu Nov 12 16:36:07 2009 -0500 +++ b/lib/galaxy/cloud/providers/ec2.py Mon Nov 16 20:37:47 2009 -0500 @@ -74,6 +74,7 @@ self.zone = "us-east-1a" self.security_group = "galaxyWeb" self.queue = Queue() + self.sa_session = app.model.context self.threads = [] nworkers = 5 @@ -84,13 +85,26 @@ self.threads.append( worker ) log.debug( "%d EC2 cloud workers ready", nworkers ) + def shutdown( self ): + """Attempts to gracefully shut down the monitor thread""" + log.info( "sending stop signal to worker threads in EC2 cloud manager" ) + for i in range( len( self.threads ) ): + self.queue.put( self.STOP_SIGNAL ) + log.info( "EC2 cloud manager stopped" ) + + def put( self, uci_wrapper ): + # Get rid of UCI from state description + state = uci_wrapper.get_uci_state() + uci_wrapper.change_state( state.split('U')[0] ) # remove 'UCI' from end of state description (i.e., mark as accepted and ready for processing) + self.queue.put( uci_wrapper ) + def run_next( self ): """Run the next job, waiting until one is available if necessary""" cnt = 0 while 1: uci_wrapper = self.queue.get() - uci_state = uci_wrapper.get_state() + uci_state = uci_wrapper.get_uci_state() if uci_state is self.STOP_SIGNAL: return try: @@ -194,13 +208,13 @@ """ Get appropriate machine image (mi) based on instance size. """ - i_type = uci_wrapper.get_type( i_index ) + i_type = uci_wrapper.get_instance_type( i_index ) if i_type=='m1.small' or i_type=='c1.medium': arch = 'i386' else: arch = 'x86_64' - mi = model.CloudImage.filter_by( deleted=False, provider_type=self.type, architecture=arch ).first() + mi = self.sa_session.query( model.CloudImage ).filter_by( deleted=False, provider_type=self.type, architecture=arch ).first() if mi: return mi.image_id else: @@ -209,19 +223,6 @@ uci_wrapper.set_error( err+". Contact site administrator to ensure needed machine image is registered.", True ) return None - def shutdown( self ): - """Attempts to gracefully shut down the monitor thread""" - log.info( "sending stop signal to worker threads in EC2 cloud manager" ) - for i in range( len( self.threads ) ): - self.queue.put( self.STOP_SIGNAL ) - log.info( "EC2 cloud manager stopped" ) - - def put( self, uci_wrapper ): - # Get rid of UCI from state description - state = uci_wrapper.get_state() - uci_wrapper.change_state( state.split('U')[0] ) # remove 'UCI' from end of state description (i.e., mark as accepted and ready for processing) - self.queue.put( uci_wrapper ) - def createUCI( self, uci_wrapper ): """ Creates User Configured Instance (UCI). Essentially, creates storage volume on cloud provider @@ -294,7 +295,8 @@ if conn.delete_volume( v.volume_id ): deletedList.append( v.volume_id ) v.deleted = True - v.flush() + self.sa_session.add( v ) + self.sa_session.flush() count += 1 else: failedList.append( v.volume_id ) @@ -308,8 +310,8 @@ if count == len( vl ): uci_wrapper.set_deleted() else: - err = "Deleting following volume(s) failed: "+failedList+". However, these volumes were successfully deleted: "+deletedList+". \ - MANUAL intervention and processing needed." + err = "Deleting following volume(s) failed: " + str( failedList ) + ". However, these volumes were successfully deleted: " \ + + str( deletedList ) + ". MANUAL intervention and processing needed." log.error( err ) uci_wrapper.set_error( err, True ) @@ -317,7 +319,7 @@ """ Creates snapshot of all storage volumes associated with this UCI. """ - if uci_wrapper.get_state() != uci_states.ERROR: + if uci_wrapper.get_uci_state() != uci_states.ERROR: conn = self.get_connection( uci_wrapper ) snapshots = uci_wrapper.get_snapshots( status = snapshot_status.SUBMITTED ) @@ -361,7 +363,7 @@ """ Starts instance(s) of given UCI on the cloud. """ - if uci_wrapper.get_state() != uci_states.ERROR: + if uci_wrapper.get_uci_state() != uci_states.ERROR: conn = self.get_connection( uci_wrapper ) self.check_key_pair( uci_wrapper, conn ) if uci_wrapper.get_key_pair_name() == None: @@ -379,70 +381,73 @@ log.debug( "mi_id: %s, uci_wrapper.get_key_pair_name(): %s" % ( mi_id, uci_wrapper.get_key_pair_name() ) ) uci_wrapper.set_mi( i_index, mi_id ) - # Check if galaxy security group exists (and create it if it does not) - log.debug( "Setting up '%s' security group." % self.security_group ) - try: - conn.get_all_security_groups( [self.security_group] ) # security groups - except boto.exception.EC2ResponseError, e: - if e.code == 'InvalidGroup.NotFound': - log.info( "No security group found, creating security group '%s'" % self.security_group ) - try: - gSecurityGroup = conn.create_security_group(self.security_group, 'Security group for Galaxy.') - gSecurityGroup.authorize( 'tcp', 80, 80, '0.0.0.0/0' ) # Open HTTP port - gSecurityGroup.authorize( 'tcp', 22, 22, '0.0.0.0/0' ) # Open SSH port - except boto.exception.EC2ResponseError, ee: - err = "EC2 response error while creating security group: " + str( ee ) + if mi_id != None: + # Check if galaxy security group exists (and create it if it does not) + log.debug( "Setting up '%s' security group." % self.security_group ) + try: + conn.get_all_security_groups( [self.security_group] ) # security groups + except boto.exception.EC2ResponseError, e: + if e.code == 'InvalidGroup.NotFound': + log.info( "No security group found, creating security group '%s'" % self.security_group ) + try: + gSecurityGroup = conn.create_security_group(self.security_group, 'Security group for Galaxy.') + gSecurityGroup.authorize( 'tcp', 80, 80, '0.0.0.0/0' ) # Open HTTP port + gSecurityGroup.authorize( 'tcp', 22, 22, '0.0.0.0/0' ) # Open SSH port + except boto.exception.EC2ResponseError, ee: + err = "EC2 response error while creating security group: " + str( ee ) + log.error( err ) + uci_wrapper.set_error( err, True ) + else: + err = "EC2 response error while retrieving security group: " + str( e ) log.error( err ) uci_wrapper.set_error( err, True ) - else: - err = "EC2 response error while retrieving security group: " + str( e ) - log.error( err ) - uci_wrapper.set_error( err, True ) - - if uci_wrapper.get_state() != uci_states.ERROR: - # Start an instance - log.debug( "Starting instance for UCI '%s'" % uci_wrapper.get_name() ) - #TODO: Once multiple volumes can be attached to a single instance, update 'userdata' composition - userdata = uci_wrapper.get_store_volume_id()+"|"+uci_wrapper.get_access_key()+"|"+uci_wrapper.get_secret_key() - log.debug( "Using following command: conn.run_instances( image_id='%s', key_name='%s', security_groups=['%s'], user_data=[OMITTED], instance_type='%s', placement='%s' )" - % ( mi_id, uci_wrapper.get_key_pair_name(), self.security_group, uci_wrapper.get_type( i_index ), uci_wrapper.get_uci_availability_zone() ) ) - reservation = None - try: - reservation = conn.run_instances( image_id=mi_id, - key_name=uci_wrapper.get_key_pair_name(), - security_groups=[self.security_group], - user_data=userdata, - instance_type=uci_wrapper.get_type( i_index ), - placement=uci_wrapper.get_uci_availability_zone() ) - except boto.exception.EC2ResponseError, e: - err = "EC2 response error when starting UCI '"+ uci_wrapper.get_name() +"': " + str( e ) - log.error( err ) - uci_wrapper.set_error( err, True ) - except Exception, ex: - err = "Error when starting UCI '" + uci_wrapper.get_name() + "': " + str( ex ) - log.error( err ) - uci_wrapper.set_error( err, True ) - # Record newly available instance data into local Galaxy database - if reservation: - uci_wrapper.set_launch_time( self.format_time( reservation.instances[0].launch_time ), i_index=i_index ) - if not uci_wrapper.uci_launch_time_set(): - uci_wrapper.set_uci_launch_time( self.format_time( reservation.instances[0].launch_time ) ) + + if uci_wrapper.get_uci_state() != uci_states.ERROR: + # Start an instance + log.debug( "Starting instance for UCI '%s'" % uci_wrapper.get_name() ) + #TODO: Once multiple volumes can be attached to a single instance, update 'userdata' composition + userdata = uci_wrapper.get_store_volume_id()+"|"+uci_wrapper.get_access_key()+"|"+uci_wrapper.get_secret_key() + log.debug( "Using following command: conn.run_instances( image_id='%s', key_name='%s', security_groups=['%s'], user_data=[OMITTED], instance_type='%s', placement='%s' )" + % ( mi_id, uci_wrapper.get_key_pair_name(), self.security_group, uci_wrapper.get_instance_type( i_index ), uci_wrapper.get_uci_availability_zone() ) ) + reservation = None try: - uci_wrapper.set_reservation_id( i_index, str( reservation ).split(":")[1] ) - # TODO: if more than a single instance will be started through single reservation, change this reference to element [0] - i_id = str( reservation.instances[0]).split(":")[1] - uci_wrapper.set_instance_id( i_index, i_id ) - s = reservation.instances[0].state - uci_wrapper.change_state( s, i_id, s ) - uci_wrapper.set_security_group_name( self.security_group, i_id=i_id ) - log.debug( "Instance of UCI '%s' started, current state: '%s'" % ( uci_wrapper.get_name(), uci_wrapper.get_state() ) ) + reservation = conn.run_instances( image_id=mi_id, + key_name=uci_wrapper.get_key_pair_name(), + security_groups=[self.security_group], + user_data=userdata, + instance_type=uci_wrapper.get_instance_type( i_index ), + placement=uci_wrapper.get_uci_availability_zone() ) except boto.exception.EC2ResponseError, e: - err = "EC2 response error when retrieving instance information for UCI '" + uci_wrapper.get_name() + "': " + str( e ) + err = "EC2 response error when starting UCI '"+ uci_wrapper.get_name() +"': " + str( e ) log.error( err ) uci_wrapper.set_error( err, True ) - else: - log.error( "UCI '%s' is in 'error' state, starting instance was aborted." % uci_wrapper.get_name() ) + except Exception, ex: + err = "Error when starting UCI '" + uci_wrapper.get_name() + "': " + str( ex ) + log.error( err ) + uci_wrapper.set_error( err, True ) + # Record newly available instance data into local Galaxy database + if reservation: + l_time = datetime.utcnow() + # uci_wrapper.set_instance_launch_time( self.format_time( reservation.instances[0].launch_time ), i_index=i_index ) + uci_wrapper.set_instance_launch_time( l_time, i_index=i_index ) + if not uci_wrapper.uci_launch_time_set(): + uci_wrapper.set_uci_launch_time( l_time ) + try: + uci_wrapper.set_reservation_id( i_index, str( reservation ).split(":")[1] ) + # TODO: if more than a single instance will be started through single reservation, change this reference to element [0] + i_id = str( reservation.instances[0]).split(":")[1] + uci_wrapper.set_instance_id( i_index, i_id ) + s = reservation.instances[0].state + uci_wrapper.change_state( s, i_id, s ) + uci_wrapper.set_security_group_name( self.security_group, i_id=i_id ) + log.debug( "Instance of UCI '%s' started, current state: '%s'" % ( uci_wrapper.get_name(), uci_wrapper.get_uci_state() ) ) + except boto.exception.EC2ResponseError, e: + err = "EC2 response error when retrieving instance information for UCI '" + uci_wrapper.get_name() + "': " + str( e ) + log.error( err ) + uci_wrapper.set_error( err, True ) + else: + log.error( "UCI '%s' is in 'error' state, starting instance was aborted." % uci_wrapper.get_name() ) else: err = "No instances in state '"+ instance_states.SUBMITTED +"' found for UCI '" + uci_wrapper.get_name() + \ "'. Nothing to start." @@ -459,8 +464,13 @@ # Get all instances associated with given UCI il = uci_wrapper.get_instances_ids() # instance list + # Process list of instances and remove any references to empty instance id's + for i in il: + if i is None: + l.remove( i ) + log.debug( 'List of instances being terminated: %s' % il ) rl = conn.get_all_instances( il ) # Reservation list associated with given instances - + # Initiate shutdown of all instances under given UCI cnt = 0 stopped = [] @@ -536,18 +546,22 @@ """ log.debug( "Running general status update for EC2 UCIs..." ) # Update instances - instances = model.CloudInstance.filter( or_( model.CloudInstance.c.state==instance_states.RUNNING, - model.CloudInstance.c.state==instance_states.PENDING, - model.CloudInstance.c.state==instance_states.SHUTTING_DOWN ) ).all() + instances = self.sa_session.query( model.CloudInstance ) \ + .filter( or_( model.CloudInstance.table.c.state==instance_states.RUNNING, + model.CloudInstance.table.c.state==instance_states.PENDING, + model.CloudInstance.table.c.state==instance_states.SHUTTING_DOWN ) ) \ + .all() for inst in instances: if self.type == inst.uci.credentials.provider.type: log.debug( "[%s] Running general status update on instance '%s'" % ( inst.uci.credentials.provider.type, inst.instance_id ) ) self.updateInstance( inst ) # Update storage volume(s) - stores = model.CloudStore.filter( or_( model.CloudStore.c.status==store_status.IN_USE, - model.CloudStore.c.status==store_status.CREATING, - model.CloudStore.c.status==None ) ).all() + stores = self.sa_session.query( model.CloudStore ) \ + .filter( or_( model.CloudStore.table.c.status==store_status.IN_USE, + model.CloudStore.table.c.status==store_status.CREATING, + model.CloudStore.table.c.status==None ) ) \ + .all() for store in stores: if self.type == store.uci.credentials.provider.type: # and store.volume_id != None: log.debug( "[%s] Running general status update on store with local database ID: '%s'" % ( store.uci.credentials.provider.type, store.id ) ) @@ -564,7 +578,9 @@ # store.flush() # Update pending snapshots or delete ones marked for deletion - snapshots = model.CloudSnapshot.filter_by( status=snapshot_status.PENDING, status=snapshot_status.DELETE ).all() + snapshots = self.sa_session.query( model.CloudSnapshot ) \ + .filter_by( status=snapshot_status.PENDING, status=snapshot_status.DELETE ) \ + .all() for snapshot in snapshots: if self.type == snapshot.uci.credentials.provider.type and snapshot.status == snapshot_status.PENDING: log.debug( "[%s] Running general status update on snapshot '%s'" % ( snapshot.uci.credentials.provider.type, snapshot.snapshot_id ) ) @@ -574,11 +590,12 @@ self.delete_snapshot( snapshot ) # Attempt at updating any zombie UCIs (i.e., instances that have been in SUBMITTED state for longer than expected - see below for exact time) - zombies = model.UCI.filter_by( state=uci_states.SUBMITTED ).all() + zombies = self.sa_session.query( model.UCI ).filter_by( state=uci_states.SUBMITTED ).all() for zombie in zombies: - z_instances = model.CloudInstance.filter_by( uci_id=zombie.id) \ - .filter( or_( model.CloudInstance.c.state != instance_states.TERMINATED, - model.CloudInstance.c.state == None ) ) \ + z_instances = self.sa_session.query( model.CloudInstance ) \ + .filter_by( uci_id=zombie.id ) \ + .filter( or_( model.CloudInstance.table.c.state != instance_states.TERMINATED, + model.CloudInstance.table.c.state == None ) ) \ .all() for z_inst in z_instances: if self.type == z_inst.uci.credentials.provider.type: @@ -592,8 +609,8 @@ # Get credentials associated wit this instance uci_id = inst.uci_id - uci = model.UCI.get( uci_id ) - uci.refresh() + uci = self.sa_session.query( model.UCI ).get( uci_id ) + self.sa_session.refresh( uci ) conn = self.get_connection_from_uci( uci ) # Get reservations handle for given instance @@ -604,6 +621,8 @@ log.error( err ) uci.error = err uci.state = uci_states.ERROR + self.sa_session.add( uci ) + self.sa_session.flush() return None # Because references to reservations are deleted shortly after instances have been terminated, getting an empty list as a response to a query @@ -618,8 +637,9 @@ inst.state = instance_states.TERMINATED uci.state = uci_states.ERROR uci.launch_time = None - inst.flush() - uci.flush() + self.sa_session.add( inst ) + self.sa_session.add( uci ) + self.sa_session.flush() # Update instance status in local DB with info from cloud provider for r in rl: for i, cInst in enumerate( r.instances ): @@ -628,45 +648,54 @@ log.debug( "Checking state of cloud instance '%s' associated with UCI '%s' and reservation '%s'. State='%s'" % ( cInst, uci.name, r, s ) ) if s != inst.state: inst.state = s - inst.flush() + self.sa_session.add( inst ) + self.sa_session.flush() # After instance has shut down, ensure UCI is marked as 'available' if s == instance_states.TERMINATED and uci.state != uci_states.ERROR: uci.state = uci_states.AVAILABLE uci.launch_time = None - uci.flush() + self.sa_session.add( uci ) + self.sa_session.flush() # Making sure state of UCI is updated. Once multiple instances become associated with single UCI, this will need to be changed. if s != uci.state and s != instance_states.TERMINATED: uci.state = s - uci.flush() + self.sa_session.add( uci ) + self.sa_session.flush() if cInst.public_dns_name != inst.public_dns: inst.public_dns = cInst.public_dns_name - inst.flush() + self.sa_session.add( inst ) + self.sa_session.flush() if cInst.private_dns_name != inst.private_dns: inst.private_dns = cInst.private_dns_name - inst.flush() + self.sa_session.add( inst ) + self.sa_session.flush() except boto.exception.EC2ResponseError, e: err = "Updating instance status from cloud failed for UCI '"+ uci.name + "' during general status update: " + str( e ) log.error( err ) uci.error = err uci.state = uci_states.ERROR + self.sa_session.add( uci ) + self.sa_session.flush() return None def updateStore( self, store ): # Get credentials associated wit this store uci_id = store.uci_id - uci = model.UCI.get( uci_id ) - uci.refresh() + uci = self.sa_session.query( model.UCI ).get( uci_id ) + self.sa_session.refresh( uci ) conn = self.get_connection_from_uci( uci ) # Get reservations handle for given store try: + log.debug( "Updating storage volume command: vl = conn.get_all_volumes( [%s] )" % store.volume_id ) vl = conn.get_all_volumes( [store.volume_id] ) except boto.exception.EC2ResponseError, e: err = "Retrieving volume(s) from cloud failed for UCI '"+ uci.name + "' during general status update: " + str( e ) log.error( err ) uci.error = err uci.state = uci_states.ERROR - uci.flush() + self.sa_session.add( uci ) + self.sa_session.flush() return None # Update store status in local DB with info from cloud provider @@ -677,29 +706,36 @@ # UCI state remained as 'new', try to remedy this by updating UCI state here if ( store.status == None ) and ( store.volume_id != None ): uci.state = vl[0].status - uci.flush() + self.sa_session.add( uci ) + self.sa_session.flush() # If UCI was marked in state 'CREATING', update its status to reflect new status elif ( uci.state == uci_states.CREATING ): uci.state = vl[0].status - uci.flush() + self.sa_session.add( uci ) + self.sa_session.flush() store.status = vl[0].status - store.flush() + self.sa_session.add( store ) + self.sa_session.flush() if store.i_id != vl[0].instance_id: store.i_id = vl[0].instance_id - store.flush() + self.sa_session.add( store ) + self.sa_session.flush() if store.attach_time != vl[0].attach_time: store.attach_time = vl[0].attach_time - store.flush() + self.sa_session.add( store ) + self.sa_session.flush() if store.device != vl[0].device: store.device = vl[0].device - store.flush() + self.sa_session.add( store ) + self.sa_session.flush() except boto.exception.EC2ResponseError, e: err = "Updating status of volume(s) from cloud failed for UCI '"+ uci.name + "' during general status update: " + str( e ) log.error( err ) uci.error = err uci.state = uci_states.ERROR - uci.flush() + self.sa_session.add( uci ) + self.sa_session.flush() return None else: err = "No storage volumes returned by cloud provider on general update" @@ -708,14 +744,15 @@ store.error = err uci.error = err uci.state = uci_states.ERROR - uci.flush() - store.flush() + self.sa_session.add( uci ) + self.sa_session.add( store ) + self.sa_session.flush() def updateSnapshot( self, snapshot ): # Get credentials associated wit this store uci_id = snapshot.uci_id - uci = model.UCI.get( uci_id ) - uci.refresh() + uci = self.sa_session.query( model.UCI ).get( uci_id ) + self.sa_session.refresh( uci ) conn = self.get_connection_from_uci( uci ) try: @@ -724,7 +761,8 @@ if len( snap ) > 0: log.debug( "Snapshot '%s' status: %s" % ( snapshot.snapshot_id, snap[0].status ) ) snapshot.status = snap[0].status - snapshot.flush() + self.sa_session.add( snapshot ) + self.sa_session.flush() else: err = "No snapshots returned by EC2 on general update" log.error( "%s for UCI '%s'" % ( err, uci.name ) ) @@ -732,8 +770,9 @@ snapshot.error = err uci.error = err uci.state = uci_states.ERROR - uci.flush() - snapshot.flush() + self.sa_session.add( uci ) + self.sa_session.add( snapshot ) + self.sa_session.flush() except boto.exception.EC2ResponseError, e: err = "EC2 response error while updating snapshot status: " + str( e ) log.error( err ) @@ -741,8 +780,9 @@ snapshot.error = err uci.error = err uci.state = uci_states.ERROR - uci.flush() - snapshot.flush() + self.sa_session.add( uci ) + self.sa_session.add( snapshot ) + self.sa_session.flush() except Exception, ex: err = "Error while updating snapshot status: " + str( ex ) log.error( err ) @@ -750,15 +790,16 @@ snapshot.error = err uci.error = err uci.state = uci_states.ERROR - uci.flush() - snapshot.flush() + self.sa_session.add( uci ) + self.sa_session.add( snapshot ) + self.sa_session.flush() def delete_snapshot( self, snapshot ): if snapshot.status == snapshot_status.DELETE: # Get credentials associated wit this store uci_id = snapshot.uci_id - uci = model.UCI.get( uci_id ) - uci.refresh() + uci = self.sa_session.query( model.UCI ).get( uci_id ) + self.sa_session.refresh( uci ) conn = self.get_connection_from_uci( uci ) try: @@ -767,7 +808,8 @@ if snap == True: snapshot.deleted = True snapshot.status = snapshot_status.DELETED - snapshot.flush() + self.sa_session.add( snapshot ) + self.sa_session.flush() return snap except boto.exception.EC2ResponseError, e: err = "EC2 response error while deleting snapshot: " + str( e ) @@ -776,8 +818,9 @@ snapshot.error = err uci.error = err uci.state = uci_states.ERROR - uci.flush() - snapshot.flush() + self.sa_session.add( uci ) + self.sa_session.add( snapshot ) + self.sa_session.flush() except Exception, ex: err = "Error while deleting snapshot: " + str( ex ) log.error( err ) @@ -785,14 +828,16 @@ snapshot.error = err uci.error = err uci.state = uci_states.ERROR - uci.flush() - snapshot.flush() + self.sa_session.add( uci ) + self.sa_session.add( snapshot ) + self.sa_session.flush() else: err = "Cannot delete snapshot '"+snapshot.snapshot_id+"' because its status is '"+snapshot.status+"'. Only snapshots with '" + \ snapshot_status.COMPLETED+"' status can be deleted." log.error( err ) snapshot.error = err - snapshot.flush() + self.sa_session.add( snapshot ) + self.sa_session.flush() def processZombie( self, inst ): """ @@ -800,6 +845,10 @@ accordingly or if something else failed and instance was never started. Currently, no automatic repairs are being attempted; instead, appropriate error messages are set. """ + uci_id = inst.uci_id + uci = self.sa_session.query( model.UCI ).get( uci_id ) + self.sa_session.refresh( uci ) + # Check if any instance-specific information was written to local DB; if 'yes', set instance and UCI's error message # suggesting manual check. if inst.launch_time != None or inst.reservation_id != None or inst.instance_id != None: @@ -820,9 +869,10 @@ try: state = rl[0].instances[0].update() inst.state = state - inst.uci.state = state - inst.flush() - inst.uci.flush() + uci.state = state + self.sa_session.add( inst ) + self.sa_session.add( uci ) + self.sa_session.flush() except: # something failed, so skip pass @@ -830,10 +880,12 @@ try: launch_time = self.format_time( rl[0].instances[0].launch_time ) inst.launch_time = launch_time - inst.flush() + self.sa_session.add( inst ) + self.sa_session.flush() if inst.uci.launch_time == None: - inst.uci.launch_time = launch_time - inst.uci.flush() + uci.launch_time = launch_time + self.sa_session.add( uci ) + self.sa_session.flush() except: # something failed, so skip pass else: @@ -844,8 +896,9 @@ inst.uci.error = err inst.uci.state = uci_states.ERROR log.error( err ) - inst.flush() - inst.uci.flush() + self.sa_session.add( inst ) + self.sa_session.add( uci ) + self.sa_session.flush() else: #Instance most likely never got processed, so set error message suggesting user to try starting instance again. err = "Starting a machine instance (DB id: '"+str(inst.id)+"') associated with this UCI '" + str(inst.uci.name) + \ @@ -853,11 +906,12 @@ "starting the instance again." inst.error = err inst.state = instance_states.ERROR - inst.uci.error = err - inst.uci.state = uci_states.ERROR + uci.error = err + uci.state = uci_states.ERROR log.error( err ) - inst.flush() - inst.uci.flush() + self.sa_session.add( inst ) + self.sa_session.add( uci ) + self.sa_session.flush() # uw = UCIwrapper( inst.uci ) # log.debug( "Try automatically re-submitting UCI '%s'." % uw.get_name() ) @@ -882,7 +936,8 @@ log.error( err ) uci.error = err uci.state = uci_states.ERROR - uci.flush() + self.sa_session.add( uci ) + self.sa_session.flush() return None return conn @@ -895,7 +950,7 @@ # conn = self.get_connection( uci ) # # # Update status of storage volumes -# vl = model.CloudStore.filter( model.CloudInstance.c.uci_id == uci.id ).all() +# vl = model.CloudStore.filter( model.CloudInstance.table.c.uci_id == uci.id ).all() # vols = [] # for v in vl: # vols.append( v.volume_id ) @@ -911,7 +966,7 @@ # pass # # # Update status of instances -# il = model.CloudInstance.filter_by( uci_id=uci.id ).filter( model.CloudInstance.c.state != 'terminated' ).all() +# il = model.CloudInstance.filter_by( uci_id=uci.id ).filter( model.CloudInstance.table.c.state != 'terminated' ).all() # instanceList = [] # for i in il: # instanceList.append( i.instance_id ) diff -r 7d013eb98022 -r 39502dd3fd23 lib/galaxy/cloud/providers/eucalyptus.py --- a/lib/galaxy/cloud/providers/eucalyptus.py Thu Nov 12 16:36:07 2009 -0500 +++ b/lib/galaxy/cloud/providers/eucalyptus.py Mon Nov 16 20:37:47 2009 -0500 @@ -73,6 +73,7 @@ self.type = "eucalyptus" # cloud provider type (e.g., ec2, eucalyptus, opennebula) self.zone = "epc" self.queue = Queue() + self.sa_session = app.model.context self.threads = [] nworkers = 5 @@ -83,12 +84,28 @@ self.threads.append( worker ) log.debug( "%d eucalyptus cloud workers ready", nworkers ) + def shutdown( self ): + """Attempts to gracefully shut down the monitor thread""" + log.info( "sending stop signal to worker threads in eucalyptus cloud manager" ) + for i in range( len( self.threads ) ): + self.queue.put( self.STOP_SIGNAL ) + log.info( "eucalyptus cloud manager stopped" ) + + def put( self, uci_wrapper ): + """ + Adds uci_wrapper object to the end of the request queue to be handled by + this cloud provider. + """ + state = uci_wrapper.get_uci_state() + uci_wrapper.change_state( state.split('U')[0] ) # remove 'UCI' from end of state description (i.e., mark as accepted and ready for processing) + self.queue.put( uci_wrapper ) + def run_next( self ): - """Run the next job, waiting until one is available if necessary""" + """Process next request, waiting until one is available if necessary.""" cnt = 0 while 1: uci_wrapper = self.queue.get() - uci_state = uci_wrapper.get_state() + uci_state = uci_wrapper.get_uci_state() if uci_state is self.STOP_SIGNAL: return try: @@ -109,7 +126,7 @@ def get_connection( self, uci_wrapper ): """ - Establishes eucalyptus cloud connection using user's credentials associated with given UCI + Establishes cloud connection using user's credentials associated with given UCI """ log.debug( 'Establishing %s cloud connection.' % self.type ) provider = uci_wrapper.get_provider() @@ -137,7 +154,10 @@ def check_key_pair( self, uci_wrapper, conn ): """ - Generate key pair using user's credentials + Check if a key pair associated with this UCI exists on cloud provider. + If yes, return key pair name; otherwise, generate a key pair with the cloud + provider and, again, return key pair name. + Key pair name for given UCI is generated from UCI's name and suffix '_kp' """ kp = None kp_name = uci_wrapper.get_name().replace(' ','_') + "_kp" @@ -185,6 +205,7 @@ return None def create_key_pair( self, conn, kp_name ): + """ Initiate creation of key pair under kp_name by current cloud provider. """ try: return conn.create_key_pair( kp_name ) except boto.exception.EC2ResponseError, e: @@ -192,15 +213,15 @@ def get_mi_id( self, uci_wrapper, i_index ): """ - Get appropriate machine image (mi) based on instance size. + Get appropriate machine image (mi) ID based on instance type. """ - i_type = uci_wrapper.get_type( i_index ) + i_type = uci_wrapper.get_instance_type( i_index ) if i_type=='m1.small' or i_type=='c1.medium': arch = 'i386' else: arch = 'x86_64' - mi = model.CloudImage.filter_by( deleted=False, provider_type=self.type, architecture=arch ).first() + mi = self.sa_session.query( model.CloudImage ).filter_by( deleted=False, provider_type=self.type, architecture=arch ).first() if mi: return mi.image_id else: @@ -209,23 +230,10 @@ uci_wrapper.set_error( err+". Contact site administrator to ensure needed machine image is registered.", True ) return None - def shutdown( self ): - """Attempts to gracefully shut down the monitor thread""" - log.info( "sending stop signal to worker threads in eucalyptus cloud manager" ) - for i in range( len( self.threads ) ): - self.queue.put( self.STOP_SIGNAL ) - log.info( "eucalyptus cloud manager stopped" ) - - def put( self, uci_wrapper ): - # Get rid of UCI from state description - state = uci_wrapper.get_state() - uci_wrapper.change_state( state.split('U')[0] ) # remove 'UCI' from end of state description (i.e., mark as accepted and ready for processing) - self.queue.put( uci_wrapper ) - def createUCI( self, uci_wrapper ): """ - Creates User Configured Instance (UCI). Essentially, creates storage volume on cloud provider - and registers relevant information in Galaxy database. + Create User Configured Instance (UCI) - i.e., create storage volume on cloud provider + and register relevant information in local Galaxy database. """ conn = self.get_connection( uci_wrapper ) @@ -270,8 +278,11 @@ def deleteUCI( self, uci_wrapper ): """ - Deletes UCI. NOTE that this implies deletion of any and all data associated + Delete UCI - i.e., delete all storage volumes associated with this UCI. + NOTE that this implies deletion of any and all data associated with this UCI from the cloud. All data will be deleted. + Information in local Galaxy database is marked as deleted but not actually removed + from the database. """ conn = self.get_connection( uci_wrapper ) vl = [] # volume list @@ -287,7 +298,8 @@ if conn.delete_volume( v.volume_id ): deletedList.append( v.volume_id ) v.deleted = True - v.flush() + self.sa_session.add( v ) + self.sa_session.flush() count += 1 else: failedList.append( v.volume_id ) @@ -301,16 +313,17 @@ if count == len( vl ): uci_wrapper.set_deleted() else: - err = "Deleting following volume(s) failed: "+failedList+". However, these volumes were successfully deleted: "+deletedList+". \ - MANUAL intervention and processing needed." + err = "Deleting following volume(s) failed: "+ str( failedList )+". However, these volumes were successfully deleted: " \ + + str( deletedList ) +". MANUAL intervention and processing needed." log.error( err ) uci_wrapper.set_error( err, True ) def snapshotUCI( self, uci_wrapper ): """ - Creates snapshot of all storage volumes associated with this UCI. + Initiate creation of a snapshot by cloud provider for all storage volumes + associated with this UCI. """ - if uci_wrapper.get_state() != uci_states.ERROR: + if uci_wrapper.get_uci_state() != uci_states.ERROR: conn = self.get_connection( uci_wrapper ) snapshots = uci_wrapper.get_snapshots( status = snapshot_status.SUBMITTED ) @@ -337,7 +350,7 @@ uci_wrapper.change_state( uci_state=uci_states.AVAILABLE ) -# if uci_wrapper.get_state() != uci_states.ERROR: +# if uci_wrapper.get_uci_state() != uci_states.ERROR: # # snapshots = uci_wrapper.get_snapshots( status = 'submitted' ) # for snapshot in snapshots: @@ -363,9 +376,9 @@ def startUCI( self, uci_wrapper ): """ - Starts instance(s) of given UCI on the cloud. + Start instance(s) of given UCI on the cloud. """ - if uci_wrapper.get_state() != uci_states.ERROR: + if uci_wrapper.get_uci_state() != uci_states.ERROR: conn = self.get_connection( uci_wrapper ) self.check_key_pair( uci_wrapper, conn ) if uci_wrapper.get_key_pair_name() == None: @@ -383,16 +396,16 @@ log.debug( "mi_id: %s, uci_wrapper.get_key_pair_name(): %s" % ( mi_id, uci_wrapper.get_key_pair_name() ) ) uci_wrapper.set_mi( i_index, mi_id ) - if uci_wrapper.get_state() != uci_states.ERROR: + if uci_wrapper.get_uci_state() != uci_states.ERROR: # Start an instance log.debug( "Starting UCI instance '%s'" % uci_wrapper.get_name() ) log.debug( "Using following command: conn.run_instances( image_id='%s', key_name='%s', instance_type='%s' )" - % ( mi_id, uci_wrapper.get_key_pair_name(), uci_wrapper.get_type( i_index ) ) ) + % ( mi_id, uci_wrapper.get_key_pair_name(), uci_wrapper.get_instance_type( i_index ) ) ) reservation = None try: reservation = conn.run_instances( image_id=mi_id, key_name=uci_wrapper.get_key_pair_name(), - instance_type=uci_wrapper.get_type( i_index ) ) + instance_type=uci_wrapper.get_instance_type( i_index ) ) except boto.exception.EC2ResponseError, e: err = "EC2 response error when starting UCI '"+ uci_wrapper.get_name() +"': " + str( e ) log.error( err ) @@ -403,9 +416,11 @@ uci_wrapper.set_error( err, True ) # Record newly available instance data into local Galaxy database if reservation: - uci_wrapper.set_launch_time( self.format_time( reservation.instances[0].launch_time ), i_index=i_index ) + l_time = datetime.utcnow() +# uci_wrapper.set_instance_launch_time( self.format_time( reservation.instances[0].launch_time ), i_index=i_index ) + uci_wrapper.set_instance_launch_time( l_time, i_index=i_index ) if not uci_wrapper.uci_launch_time_set(): - uci_wrapper.set_uci_launch_time( self.format_time( reservation.instances[0].launch_time ) ) + uci_wrapper.set_uci_launch_time( l_time ) try: uci_wrapper.set_reservation_id( i_index, str( reservation ).split(":")[1] ) # TODO: if more than a single instance will be started through single reservation, change this reference from element [0] @@ -413,7 +428,7 @@ uci_wrapper.set_instance_id( i_index, i_id ) s = reservation.instances[0].state uci_wrapper.change_state( s, i_id, s ) - log.debug( "Instance of UCI '%s' started, current state: '%s'" % ( uci_wrapper.get_name(), uci_wrapper.get_state() ) ) + log.debug( "Instance of UCI '%s' started, current state: '%s'" % ( uci_wrapper.get_name(), uci_wrapper.get_uci_state() ) ) except boto.exception.EC2ResponseError, e: err = "EC2 response error when retrieving instance information for UCI '" + uci_wrapper.get_name() + "': " + str( e ) log.error( err ) @@ -430,12 +445,16 @@ def stopUCI( self, uci_wrapper): """ - Stops all of cloud instances associated with given UCI. + Stop all cloud instances associated with given UCI. """ conn = self.get_connection( uci_wrapper ) # Get all instances associated with given UCI il = uci_wrapper.get_instances_ids() # instance list + # Process list of instances and remove any references to empty instance id's + for i in il: + if i is None: + l.remove( i ) log.debug( 'List of instances being terminated: %s' % il ) rl = conn.get_all_instances( il ) # Reservation list associated with given instances @@ -506,41 +525,41 @@ def update( self ): """ - Runs a global status update on all instances that are in 'running', 'pending', or 'shutting-down' state. - Also, runs update on all storage volumes that are in 'in-use', 'creating', or 'None' state. + Run status update on all instances that are in 'running', 'pending', or 'shutting-down' state. + Run status update on all storage volumes whose status is 'in-use', 'creating', or 'None'. + Run status update on all snapshots whose status is 'pending' or 'delete' + Run status update on any zombie UCIs, i.e., UCI's that is in 'submitted' state for an + extended period of time. + Reason behind this method is to sync state of local DB and real-world resources """ log.debug( "Running general status update for EPC UCIs..." ) # Update instances - instances = model.CloudInstance.filter( or_( model.CloudInstance.c.state==instance_states.RUNNING, - model.CloudInstance.c.state==instance_states.PENDING, - model.CloudInstance.c.state==instance_states.SHUTTING_DOWN ) ).all() + instances = self.sa_session.query( model.CloudInstance ) \ + .filter( or_( model.CloudInstance.table.c.state==instance_states.RUNNING, + model.CloudInstance.table.c.state==instance_states.PENDING, + model.CloudInstance.table.c.state==instance_states.SHUTTING_DOWN ) ) \ + .all() for inst in instances: if self.type == inst.uci.credentials.provider.type: log.debug( "[%s] Running general status update on instance '%s'" % ( inst.uci.credentials.provider.type, inst.instance_id ) ) self.updateInstance( inst ) # Update storage volume(s) - stores = model.CloudStore.filter( or_( model.CloudStore.c.status==store_status.IN_USE, - model.CloudStore.c.status==store_status.CREATING, - model.CloudStore.c.status==None ) ).all() + stores = self.sa_session.query( model.CloudStore ) \ + .filter( or_( model.CloudStore.table.c.status==store_status.IN_USE, + model.CloudStore.table.c.status==store_status.CREATING, + model.CloudStore.table.c.status==None ) ) \ + .all() for store in stores: if self.type == store.uci.credentials.provider.type: # and store.volume_id != None: log.debug( "[%s] Running general status update on store with local database ID: '%s'" % ( store.uci.credentials.provider.type, store.id ) ) self.updateStore( store ) -# else: -# log.error( "[%s] There exists an entry for UCI (%s) storage volume without an ID. Storage volume might have been created with " -# "cloud provider though. Manual check is recommended." % ( store.uci.credentials.provider.type, store.uci.name ) ) -# store.uci.error = "There exists an entry in local database for a storage volume without an ID. Storage volume might have been created " \ -# "with cloud provider though. Manual check is recommended. After understanding what happened, local database entry for given " \ -# "storage volume should be updated." -# store.status = store_status.ERROR -# store.uci.state = uci_states.ERROR -# store.uci.flush() -# store.flush() # Update pending snapshots or delete ones marked for deletion - snapshots = model.CloudSnapshot.filter_by( status=snapshot_status.PENDING, status=snapshot_status.DELETE ).all() + snapshots = self.sa_session.query( model.CloudSnapshot ) \ + .filter_by( status=snapshot_status.PENDING, status=snapshot_status.DELETE ) \ + .all() for snapshot in snapshots: if self.type == snapshot.uci.credentials.provider.type and snapshot.status == snapshot_status.PENDING: log.debug( "[%s] Running general status update on snapshot '%s'" % ( snapshot.uci.credentials.provider.type, snapshot.snapshot_id ) ) @@ -550,29 +569,34 @@ self.delete_snapshot( snapshot ) # Attempt at updating any zombie UCIs (i.e., instances that have been in SUBMITTED state for longer than expected - see below for exact time) - zombies = model.UCI.filter_by( state=uci_states.SUBMITTED ).all() + zombies = self.sa_session.query( model.UCI ).filter_by( state=uci_states.SUBMITTED ).all() for zombie in zombies: log.debug( "zombie UCI: %s" % zombie.name ) - z_instances = model.CloudInstance \ - .filter_by( uci_id=zombie.id, state=None ) \ + z_instances = self.sa_session.query( model.CloudInstance ) \ + .filter( or_( model.CloudInstance.table.c.state != instance_states.TERMINATED, + model.CloudInstance.table.c.state == None ) ) \ .all() for z_inst in z_instances: if self.type == z_inst.uci.credentials.provider.type: # log.debug( "z_inst.id: '%s', state: '%s'" % ( z_inst.id, z_inst.state ) ) td = datetime.utcnow() - z_inst.update_time - log.debug( "z_inst.id: %s, time delta is %s sec" % ( z_inst.id, td.seconds ) ) +# log.debug( "z_inst.id: %s, time delta is %s sec" % ( z_inst.id, td.seconds ) ) if td.seconds > 180: # if instance has been in SUBMITTED state for more than 3 minutes log.debug( "[%s](td=%s) Running zombie repair update on instance with DB id '%s'" % ( z_inst.uci.credentials.provider.type, td.seconds, z_inst.id ) ) self.processZombie( z_inst ) def updateInstance( self, inst ): - + """ + Update information in local database for given instance as it is obtained from cloud provider. + Along with updating information about given instance, information about the UCI controlling + this instance is also updated. + """ # Get credentials associated wit this instance uci_id = inst.uci_id - uci = model.UCI.get( uci_id ) - uci.refresh() + uci = self.sa_session.query( model.UCI ).get( uci_id ) + self.sa_session.refresh( uci ) conn = self.get_connection_from_uci( uci ) - + # Get reservations handle for given instance try: rl= conn.get_all_instances( [inst.instance_id] ) @@ -581,10 +605,12 @@ log.error( err ) uci.error = err uci.state = uci_states.ERROR + self.sa_session.add( uci ) + self.sa_session.flush() return None - # Because EPC deletes references to reservations after a short while after instances have terminated, getting an empty list as a response to a query - # typically means the instance has successfully shut down but the check was not performed in short enough amount of time. Until alternative solution + # Because references to reservations are deleted shortly after instances have been terminated, getting an empty list as a response to a query + # typically means the instance has successfully shut down but the check was not performed in short enough amount of time. Until an alternative solution # is found, below code sets state of given UCI to 'error' to indicate to the user something out of ordinary happened. if len( rl ) == 0: err = "Instance ID '"+inst.instance_id+"' was not found by the cloud provider. Instance might have crashed or otherwise been terminated."+ \ @@ -595,109 +621,151 @@ inst.state = instance_states.TERMINATED uci.state = uci_states.ERROR uci.launch_time = None - inst.flush() - uci.flush() + self.sa_session.add( inst ) + self.sa_session.add( uci ) + self.sa_session.flush() # Update instance status in local DB with info from cloud provider for r in rl: for i, cInst in enumerate( r.instances ): try: s = cInst.update() - log.debug( "Checking state of cloud instance '%s' associated with reservation '%s'. State='%s'" % ( cInst, r, s ) ) + log.debug( "Checking state of cloud instance '%s' associated with UCI '%s' and reservation '%s'. State='%s'" % ( cInst, uci.name, r, s ) ) if s != inst.state: inst.state = s - inst.flush() - # After instance has shut down, ensure UCI is marked as 'available' - if s == instance_states.TERMINATED and uci.state != uci_states.ERROR: + self.sa_session.add( inst ) + self.sa_session.flush() + # After instance has shut down, ensure UCI is marked as 'available' + if s == instance_states.TERMINATED and uci.state != uci_states.ERROR: uci.state = uci_states.AVAILABLE uci.launch_time = None - uci.flush() + self.sa_session.add( uci ) + self.sa_session.flush() # Making sure state of UCI is updated. Once multiple instances become associated with single UCI, this will need to be changed. if s != uci.state and s != instance_states.TERMINATED: uci.state = s - uci.flush() + self.sa_session.add( uci ) + self.sa_session.flush() if cInst.public_dns_name != inst.public_dns: inst.public_dns = cInst.public_dns_name - inst.flush() + self.sa_session.add( inst ) + self.sa_session.flush() if cInst.private_dns_name != inst.private_dns: inst.private_dns = cInst.private_dns_name - inst.flush() + self.sa_session.add( inst ) + self.sa_session.flush() except boto.exception.EC2ResponseError, e: err = "Updating instance status from cloud failed for UCI '"+ uci.name + "' during general status update: " + str( e ) log.error( err ) uci.error = err uci.state = uci_states.ERROR + self.sa_session.add( uci ) + self.sa_session.flush() return None - + def updateStore( self, store ): + """ + Update information in local database for given storage volume as it is obtained from cloud provider. + Along with updating information about given storage volume, information about the UCI controlling + this storage volume is also updated. + """ # Get credentials associated wit this store uci_id = store.uci_id - uci = model.UCI.get( uci_id ) - uci.refresh() + uci = self.sa_session.query( model.UCI ).get( uci_id ) + self.sa_session.refresh( uci ) conn = self.get_connection_from_uci( uci ) - - try: - vl = conn.get_all_volumes( [store.volume_id] ) - except boto.exception.EC2ResponseError, e: - err = "Retrieving volume(s) from cloud failed for UCI '"+ uci.name + "' during general status update: " + str( e ) - log.error( err ) - uci.error = err - uci.state = uci_states.ERROR - uci.flush() - return None - # Update store status in local DB with info from cloud provider - if len(vl) > 0: + if store.volume_id != None: + # Get reservations handle for given store try: - if store.status != vl[0].status: - # In case something failed during creation of UCI but actual storage volume was created and yet - # UCI state remained as 'new', try to remedy this by updating UCI state here - if ( store.status == None ) and ( store.volume_id != None ): - uci.state = vl[0].status - uci.flush() - # If UCI was marked in state 'CREATING', update its status to reflect new status - elif ( uci.state == uci_states.CREATING ): - # Because Eucalyptus Public Cloud (EPC) deletes volumes immediately after they are created, artificially - # set status of given UCI to 'available' based on storage volume's availability zone (i.e., it's residing - # in EPC as opposed to some other Eucalyptus based cloud that allows creation of storage volumes. - if store.availability_zone == 'epc': - uci.state = uci_states.AVAILABLE - else: - uci.state = vl[0].status - uci.flush() - - store.status = vl[0].status - store.flush() - if store.i_id != vl[0].instance_id: - store.i_id = vl[0].instance_id - store.flush() - if store.attach_time != vl[0].attach_time: - store.attach_time = vl[0].attach_time - store.flush() - if store.device != vl[0].device: - store.device = vl[0].device - store.flush() + log.debug( "Updating storage volume command: vl = conn.get_all_volumes( [%s] )" % store.volume_id ) + vl = conn.get_all_volumes( [store.volume_id] ) except boto.exception.EC2ResponseError, e: - err = "Updating status of volume(s) from cloud failed for UCI '"+ uci.name + "' during general status update: " + str( e ) + err = "Retrieving volume(s) from cloud failed for UCI '"+ uci.name + "' during general status update: " + str( e ) log.error( err ) uci.error = err uci.state = uci_states.ERROR - uci.flush() + self.sa_session.add( uci ) + self.sa_session.flush() return None + + # Update store status in local DB with info from cloud provider + if len(vl) > 0: + try: + if store.status != vl[0].status: + # In case something failed during creation of UCI but actual storage volume was created and yet + # UCI state remained as 'new', try to remedy this by updating UCI state here + if ( store.status == None ) and ( store.volume_id != None ): + uci.state = vl[0].status + self.sa_session.add( uci ) + self.sa_session.flush() + # If UCI was marked in state 'CREATING', update its status to reflect new status + elif ( uci.state == uci_states.CREATING ): + # Because Eucalyptus Public Cloud (EPC) deletes volumes immediately after they are created, artificially + # set status of given UCI to 'available' based on storage volume's availability zone (i.e., it's residing + # in EPC as opposed to some other Eucalyptus based cloud that allows creation of storage volumes. + if store.availability_zone == 'epc': + uci.state = uci_states.AVAILABLE + else: + uci.state = vl[0].status + + self.sa_session.add( uci ) + self.sa_session.flush() + + store.status = vl[0].status + self.sa_session.add( store ) + self.sa_session.flush() + if store.i_id != vl[0].instance_id: + store.i_id = vl[0].instance_id + self.sa_session.add( store ) + self.sa_session.flush() + if store.attach_time != vl[0].attach_time: + store.attach_time = vl[0].attach_time + self.sa_session.add( store ) + self.sa_session.flush() + if store.device != vl[0].device: + store.device = vl[0].device + self.sa_session.add( store ) + self.sa_session.flush() + except boto.exception.EC2ResponseError, e: + err = "Updating status of volume(s) from cloud failed for UCI '"+ uci.name + "' during general status update: " + str( e ) + log.error( err ) + uci.error = err + uci.state = uci_states.ERROR + self.sa_session.add( uci ) + self.sa_session.flush() + return None + else: + err = "No storage volumes returned by cloud provider on general update" + log.error( "%s for UCI '%s'" % ( err, uci.name ) ) + store.status = store_status.ERROR + store.error = err + uci.error = err + uci.state = uci_states.ERROR + self.sa_session.add( uci ) + self.sa_session.add( store ) + self.sa_session.flush() else: - err = "No storage volumes returned by cloud provider on general update" - log.error( "%s for UCI '%s'" % ( err, uci.name ) ) + err = "Missing storage volume ID in local database on general update. Manual check is needed to check " \ + "if storage volume was actually created by cloud provider." + log.error( "%s (for UCI '%s')" % ( err, uci.name ) ) store.status = store_status.ERROR store.error = err uci.error = err uci.state = uci_states.ERROR - uci.flush() - store.flush() - + self.sa_session.add( uci ) + self.sa_session.add( store ) + self.sa_session.flush() + def updateSnapshot( self, snapshot ): + """ + Update information in local database for given snapshot as it is obtained from cloud provider. + Along with updating information about given snapshot, information about the UCI controlling + this snapshot is also updated. + """ # Get credentials associated wit this store uci_id = snapshot.uci_id - uci = model.UCI.get( uci_id ) - uci.refresh() + uci = self.sa_session.query( model.UCI ).get( uci_id ) + self.sa_session.refresh( uci ) conn = self.get_connection_from_uci( uci ) try: @@ -706,25 +774,28 @@ if len( snap ) > 0: log.debug( "Snapshot '%s' status: %s" % ( snapshot.snapshot_id, snap[0].status ) ) snapshot.status = snap[0].status - snapshot.flush() + self.sa_session.add( snapshot ) + self.sa_session.flush() else: - err = "No snapshots returned by cloud provider on general update" + err = "No snapshots returned by EC2 on general update" log.error( "%s for UCI '%s'" % ( err, uci.name ) ) snapshot.status = snapshot_status.ERROR snapshot.error = err uci.error = err uci.state = uci_states.ERROR - uci.flush() - snapshot.flush() + self.sa_session.add( uci ) + self.sa_session.add( snapshot ) + self.sa_session.flush() except boto.exception.EC2ResponseError, e: - err = "Cloud provider response error while updating snapshot status: " + str( e ) + err = "EC2 response error while updating snapshot status: " + str( e ) log.error( err ) snapshot.status = snapshot_status.ERROR snapshot.error = err uci.error = err uci.state = uci_states.ERROR - uci.flush() - snapshot.flush() + self.sa_session.add( uci ) + self.sa_session.add( snapshot ) + self.sa_session.flush() except Exception, ex: err = "Error while updating snapshot status: " + str( ex ) log.error( err ) @@ -732,15 +803,19 @@ snapshot.error = err uci.error = err uci.state = uci_states.ERROR - uci.flush() - snapshot.flush() - + self.sa_session.add( uci ) + self.sa_session.add( snapshot ) + self.sa_session.flush() + def delete_snapshot( self, snapshot ): + """ + Initiate deletion of given snapshot from cloud provider. + """ if snapshot.status == snapshot_status.DELETE: # Get credentials associated wit this store uci_id = snapshot.uci_id - uci = model.UCI.get( uci_id ) - uci.refresh() + uci = self.sa_session.query( model.UCI ).get( uci_id ) + self.sa_session.refresh( uci ) conn = self.get_connection_from_uci( uci ) try: @@ -749,7 +824,8 @@ if snap == True: snapshot.deleted = True snapshot.status = snapshot_status.DELETED - snapshot.flush() + self.sa_session.add( snapshot ) + self.sa_session.flush() return snap except boto.exception.EC2ResponseError, e: err = "EC2 response error while deleting snapshot: " + str( e ) @@ -758,8 +834,9 @@ snapshot.error = err uci.error = err uci.state = uci_states.ERROR - uci.flush() - snapshot.flush() + self.sa_session.add( uci ) + self.sa_session.add( snapshot ) + self.sa_session.flush() except Exception, ex: err = "Error while deleting snapshot: " + str( ex ) log.error( err ) @@ -767,21 +844,27 @@ snapshot.error = err uci.error = err uci.state = uci_states.ERROR - uci.flush() - snapshot.flush() + self.sa_session.add( uci ) + self.sa_session.add( snapshot ) + self.sa_session.flush() else: err = "Cannot delete snapshot '"+snapshot.snapshot_id+"' because its status is '"+snapshot.status+"'. Only snapshots with '" + \ snapshot_status.COMPLETED+"' status can be deleted." log.error( err ) snapshot.error = err - snapshot.flush() - + self.sa_session.add( snapshot ) + self.sa_session.flush() + def processZombie( self, inst ): """ - Attempt at discovering if starting an instance was successful but local database was not updated + Attempt at discovering if starting a cloud instance was successful but local database was not updated accordingly or if something else failed and instance was never started. Currently, no automatic repairs are being attempted; instead, appropriate error messages are set. """ + uci_id = inst.uci_id + uci = self.sa_session.query( model.UCI ).get( uci_id ) + self.sa_session.refresh( uci ) + # Check if any instance-specific information was written to local DB; if 'yes', set instance and UCI's error message # suggesting manual check. if inst.launch_time != None or inst.reservation_id != None or inst.instance_id != None: @@ -790,7 +873,7 @@ # report as error. # Fields attempting to be recovered are: reservation_id, instance status, and launch_time if inst.instance_id != None: - conn = self.get_connection_from_uci( inst.uci ) + conn = self.get_connection_from_uci( uci ) rl = conn.get_all_instances( [inst.instance_id] ) # reservation list # Update local DB with relevant data from instance if inst.reservation_id == None: @@ -802,9 +885,10 @@ try: state = rl[0].instances[0].update() inst.state = state - inst.uci.state = state - inst.flush() - inst.uci.flush() + uci.state = state + self.sa_session.add( inst ) + self.sa_session.add( uci ) + self.sa_session.flush() except: # something failed, so skip pass @@ -812,10 +896,12 @@ try: launch_time = self.format_time( rl[0].instances[0].launch_time ) inst.launch_time = launch_time - inst.flush() + self.sa_session.add( inst ) + self.sa_session.flush() if inst.uci.launch_time == None: - inst.uci.launch_time = launch_time - inst.uci.flush() + uci.launch_time = launch_time + self.sa_session.add( uci ) + self.sa_session.flush() except: # something failed, so skip pass else: @@ -826,8 +912,9 @@ inst.uci.error = err inst.uci.state = uci_states.ERROR log.error( err ) - inst.flush() - inst.uci.flush() + self.sa_session.add( inst ) + self.sa_session.add( uci ) + self.sa_session.flush() else: #Instance most likely never got processed, so set error message suggesting user to try starting instance again. err = "Starting a machine instance (DB id: '"+str(inst.id)+"') associated with this UCI '" + str(inst.uci.name) + \ @@ -835,11 +922,12 @@ "starting the instance again." inst.error = err inst.state = instance_states.ERROR - inst.uci.error = err - inst.uci.state = uci_states.ERROR + uci.error = err + uci.state = uci_states.ERROR log.error( err ) - inst.flush() - inst.uci.flush() + self.sa_session.add( inst ) + self.sa_session.add( uci ) + self.sa_session.flush() # uw = UCIwrapper( inst.uci ) # log.debug( "Try automatically re-submitting UCI '%s'." % uw.get_name() ) @@ -848,16 +936,23 @@ Establishes and returns connection to cloud provider. Information needed to do so is obtained directly from uci database object. """ - log.debug( 'Establishing %s cloud connection.' % self.type ) + log.debug( 'Establishing %s cloud connection' % self.type ) a_key = uci.credentials.access_key s_key = uci.credentials.secret_key # Get connection try: region = RegionInfo( None, uci.credentials.provider.region_name, uci.credentials.provider.region_endpoint ) + log.debug( "[%s] Using following command to connect to cloud provider: " + "conn = EC2Connection( aws_access_key_id=%s, " + "aws_secret_access_key=%s, " + "port=%s, " + "is_secure=%s, " + "region=region, " + "path=%s )" % ( self.type, a_key, s_key, uci.credentials.provider.is_secure, uci.credentials.provider.port, uci.credentials.provider.path ) ) conn = EC2Connection( aws_access_key_id=a_key, aws_secret_access_key=s_key, - is_secure=uci.credentials.provider.is_secure, - port=uci.credentials.provider.port, + is_secure=uci.credentials.provider.is_secure, + port=uci.credentials.provider.port, region=region, path=uci.credentials.provider.path ) except boto.exception.EC2ResponseError, e: @@ -865,7 +960,8 @@ log.error( err ) uci.error = err uci.state = uci_states.ERROR - uci.flush() + self.sa_session.add( uci ) + self.sa_session.flush() return None return conn @@ -878,7 +974,7 @@ # conn = self.get_connection( uci ) # # # Update status of storage volumes -# vl = model.CloudStore.filter( model.CloudInstance.c.uci_id == uci.id ).all() +# vl = model.CloudStore.filter( model.CloudInstance.table.c.uci_id == uci.id ).all() # vols = [] # for v in vl: # vols.append( v.volume_id ) @@ -894,7 +990,7 @@ # pass # # # Update status of instances -# il = model.CloudInstance.filter_by( uci_id=uci.id ).filter( model.CloudInstance.c.state != 'terminated' ).all() +# il = model.CloudInstance.filter_by( uci_id=uci.id ).filter( model.CloudInstance.table.c.state != 'terminated' ).all() # instanceList = [] # for i in il: # instanceList.append( i.instance_id ) diff -r 7d013eb98022 -r 39502dd3fd23 lib/galaxy/config.py --- a/lib/galaxy/config.py Thu Nov 12 16:36:07 2009 -0500 +++ b/lib/galaxy/config.py Mon Nov 16 20:37:47 2009 -0500 @@ -114,12 +114,11 @@ self.tool_runners = [] self.datatypes_config = kwargs.get( 'datatypes_config_file', 'datatypes_conf.xml' ) # Cloud configuration options - self.cloud_controller_instance = string_as_bool( kwargs.get( 'cloud_controller_instance', 'False' ) ) - self.cloud_provider = kwargs.get( 'cloud_provider', None ) - if self.cloud_controller_instance: - self.enable_cloud_execution = string_as_bool( kwargs.get( 'enable_cloud_execution', 'True' ) ) + self.cloud_controller_instance = string_as_bool( kwargs.get( 'cloud_controller_instance', False ) ) + if self.cloud_controller_instance == True: + self.enable_cloud_execution = string_as_bool( kwargs.get( 'enable_cloud_execution', True ) ) else: - self.enable_cloud_execution = string_as_bool( kwargs.get( 'enable_cloud_execution', 'False' ) ) + self.enable_cloud_execution = string_as_bool( kwargs.get( 'enable_cloud_execution', False ) ) def get( self, key, default ): return self.config_dict.get( key, default ) def get_bool( self, key, default ): diff -r 7d013eb98022 -r 39502dd3fd23 lib/galaxy/web/controllers/cloud.py --- a/lib/galaxy/web/controllers/cloud.py Thu Nov 12 16:36:07 2009 -0500 +++ b/lib/galaxy/web/controllers/cloud.py Mon Nov 16 20:37:47 2009 -0500 @@ -91,48 +91,48 @@ cloudCredentials = trans.sa_session.query( model.CloudUserCredentials ) \ .filter_by( user=user ) \ - .filter( model.CloudUserCredentials.c.deleted != True ) \ - .order_by( model.CloudUserCredentials.c.name ) \ + .filter( model.CloudUserCredentials.table.c.deleted != True ) \ + .order_by( model.CloudUserCredentials.table.c.name ) \ .all() cloudProviders = trans.sa_session.query( model.CloudProvider ) \ .filter_by( user=user ) \ - .filter( model.CloudProvider.c.deleted != True ) \ - .order_by( model.CloudProvider.c.name ) \ + .filter( model.CloudProvider.table.c.deleted != True ) \ + .order_by( model.CloudProvider.table.c.name ) \ .all() liveInstances = trans.sa_session.query( model.UCI ) \ .filter_by( user=user ) \ - .filter( or_( model.UCI.c.state==uci_states.RUNNING, - model.UCI.c.state==uci_states.PENDING, - model.UCI.c.state==uci_states.SUBMITTED, - model.UCI.c.state==uci_states.SUBMITTED_UCI, - model.UCI.c.state==uci_states.SHUTTING_DOWN, - model.UCI.c.state==uci_states.SHUTTING_DOWN_UCI ) ) \ - .order_by( desc( model.UCI.c.update_time ) ) \ + .filter( or_( model.UCI.table.c.state==uci_states.RUNNING, + model.UCI.table.c.state==uci_states.PENDING, + model.UCI.table.c.state==uci_states.SUBMITTED, + model.UCI.table.c.state==uci_states.SUBMITTED_UCI, + model.UCI.table.c.state==uci_states.SHUTTING_DOWN, + model.UCI.table.c.state==uci_states.SHUTTING_DOWN_UCI ) ) \ + .order_by( desc( model.UCI.table.c.update_time ) ) \ .all() prevInstances = trans.sa_session.query( model.UCI ) \ .filter_by( user=user, deleted=False ) \ - .filter( or_( model.UCI.c.state==uci_states.AVAILABLE, - model.UCI.c.state==uci_states.NEW, - model.UCI.c.state==uci_states.NEW_UCI, - model.UCI.c.state==uci_states.CREATING, - model.UCI.c.state==uci_states.ERROR, - model.UCI.c.state==uci_states.DELETED, - model.UCI.c.state==uci_states.DELETING, - model.UCI.c.state==uci_states.DELETING_UCI, - model.UCI.c.state==uci_states.SNAPSHOT, - model.UCI.c.state==uci_states.SNAPSHOT_UCI ) ) \ - .order_by( desc( model.UCI.c.update_time ) ) \ + .filter( or_( model.UCI.table.c.state==uci_states.AVAILABLE, + model.UCI.table.c.state==uci_states.NEW, + model.UCI.table.c.state==uci_states.NEW_UCI, + model.UCI.table.c.state==uci_states.CREATING, + model.UCI.table.c.state==uci_states.ERROR, + model.UCI.table.c.state==uci_states.DELETED, + model.UCI.table.c.state==uci_states.DELETING, + model.UCI.table.c.state==uci_states.DELETING_UCI, + model.UCI.table.c.state==uci_states.SNAPSHOT, + model.UCI.table.c.state==uci_states.SNAPSHOT_UCI ) ) \ + .order_by( desc( model.UCI.table.c.update_time ) ) \ .all() # Check after update there are instances in pending state; if so, display message pendingInstances = trans.sa_session.query( model.UCI ) \ .filter_by( user=user ) \ - .filter( or_( model.UCI.c.state==uci_states.PENDING, - model.UCI.c.state==uci_states.SUBMITTED, - model.UCI.c.state==uci_states.SUBMITTED_UCI ) ) \ + .filter( or_( model.UCI.table.c.state==uci_states.PENDING, + model.UCI.table.c.state==uci_states.SUBMITTED, + model.UCI.table.c.state==uci_states.SUBMITTED_UCI ) ) \ .all() if pendingInstances: trans.set_message( "Galaxy instance started. NOTE: Please wait about 5 minutes for the instance to " @@ -173,7 +173,7 @@ # Create new user configured instance try: - if trans.app.model.UCI \ + if trans.sa_session.query( model.UCI ) \ .filter_by (user=user, deleted=False, name=instanceName ) \ .first(): error['inst_error'] = "An instance with that name already exist." @@ -191,8 +191,8 @@ # Capture user configured instance information uci = model.UCI() uci.name = instanceName - uci.credentials = trans.app.model.CloudUserCredentials.filter( - trans.app.model.CloudUserCredentials.table.c.name==credName ).first() + uci.credentials = trans.sa_session.query( model.CloudUserCredentials ) \ + .filter( model.CloudUserCredentials.table.c.name==credName ).first() uci.user= user uci.total_size = volSize # This is OK now because new instance is being created and only one storage volume can be created at UCI creation time uci.state = uci_states.NEW_UCI @@ -204,8 +204,8 @@ storage.availability_zone = zone # Persist session = trans.sa_session - session.save_or_update( uci ) - session.save_or_update( storage ) + session.add( uci ) + session.add( storage ) session.flush() # Log and display the management page trans.log_event( "User configured new cloud instance: '%s'" % instanceName ) @@ -278,8 +278,8 @@ uci.state = uci_states.SUBMITTED_UCI # Persist session = trans.sa_session - session.save_or_update( instance ) - session.save_or_update( uci ) + session.add( instance ) + session.add( uci ) session.flush() # Log trans.log_event ("User initiated starting of UCI '%s'." % uci.name ) @@ -309,7 +309,7 @@ ( uci.state != uci_states.AVAILABLE ): uci.state = uci_states.SHUTTING_DOWN_UCI session = trans.sa_session - session.save_or_update( uci ) + session.add( uci ) session.flush() trans.log_event( "User stopped cloud instance '%s' (id: %s)" % ( uci.name, uci.id ) ) trans.set_message( "Stopping of Galaxy instance '%s' initiated." % uci.name ) @@ -378,7 +378,7 @@ prevInstances = trans.sa_session.query( model.CloudInstance ) \ .filter_by( user=user, state=instance_states.TERMINATED, uci_id=id ) \ - .order_by( desc( model.CloudInstance.c.update_time ) ) \ + .order_by( desc( model.CloudInstance.table.c.update_time ) ) \ .all() return trans.fill_template( "cloud/view_usage.mako", prevInstances = prevInstances ) @@ -396,7 +396,7 @@ name = uci.name uci.state = uci_states.DELETING_UCI session = trans.sa_session - session.save_or_update( uci ) + session.add( uci ) session.flush() trans.log_event( "User marked cloud instance '%s' for deletion." % name ) trans.set_message( "Galaxy instance '%s' marked for deletion." % name ) @@ -431,8 +431,8 @@ uci.state = uci_states.SNAPSHOT_UCI # Persist session = trans.sa_session - session.save_or_update( snapshot ) - session.save_or_update( uci ) + session.add( snapshot ) + session.add( uci ) session.flush() elif len( stores ) == 0: error( "No storage volumes found that are associated with this instance." ) @@ -455,7 +455,7 @@ snaps = trans.sa_session.query( model.CloudSnapshot ) \ .filter_by( user=user, uci_id=id, deleted=False ) \ - .order_by( desc( model.CloudSnapshot.c.update_time ) ) \ + .order_by( desc( model.CloudSnapshot.table.c.update_time ) ) \ .all() return trans.fill_template( "cloud/view_snapshots.mako", @@ -474,7 +474,8 @@ if snap.status == snapshot_status.COMPLETED: snap.status = snapshot_status.DELETE - snap.flush() + trans.sa_session.add( snap ) + trans.sa_session.flush() trans.set_message( "Snapshot '%s' is marked for deletion. Once the deletion is complete, it will no longer be visible in this list. " "Please note that this process may take up to a minute." % snap.snapshot_id ) else: @@ -485,7 +486,7 @@ uci_id = trans.security.decode_id( uci_id ) snaps = trans.sa_session.query( model.CloudSnapshot ) \ .filter_by( user=user, uci_id=uci_id, deleted=False ) \ - .order_by( desc( model.CloudSnapshot.c.update_time ) ) \ + .order_by( desc( model.CloudSnapshot.table.c.update_time ) ) \ .all() return trans.fill_template( "cloud/view_snapshots.mako", @@ -514,9 +515,9 @@ error['provider_error'] = "You must select cloud provider type for this machine image." elif image_id=='' or len( image_id ) > 255: error['id_error'] = "Image ID must be between 1 and 255 characters long." - elif trans.app.model.CloudUserCredentials \ + elif trans.sa_session.query( model.CloudUserCredentials ) \ .filter_by( deleted=False ) \ - .filter( trans.app.model.CloudImage.table.c.image_id == image_id ) \ + .filter( model.CloudImage.table.c.image_id == image_id ) \ .first(): error['id_error'] = "Image with ID '" + image_id + "' is already registered. \ Please choose another ID." @@ -531,7 +532,7 @@ image.architecture = architecture # Persist session = trans.sa_session - session.save_or_update( image ) + session.add( image ) session.flush() # Log and display the management page trans.log_event( "New cloud image added: '%s'" % image.image_id ) @@ -557,7 +558,7 @@ @web.expose @web.require_login( "use Galaxy cloud" ) def list_machine_images( self, trans ): - images = trans.sa_session.query( model.CloudImage ).filter( trans.app.model.CloudImage.table.c.deleted != True ).all() + images = trans.sa_session.query( model.CloudImage ).filter( model.CloudImage.table.c.deleted != True ).all() return trans.fill_template( '/cloud/list_images.mako', images=images ) @web.expose @@ -568,7 +569,8 @@ image = trans.sa_session.query( model.CloudImage ).get( id ) image.deleted = True - image.flush() + trans.sa_session.add( image ) + trans.sa_session.flush() return self.list_machine_images( trans ) @web.expose @@ -588,9 +590,9 @@ image = trans.sa_session.query( model.CloudImage ).get( id ) if image_id=='' or len( image_id ) > 255: error['id_error'] = "Image ID must be between 1 and 255 characters in length." - elif trans.app.model.CloudImage \ + elif trans.sa_session.query( model.CloudImage ) \ .filter_by( deleted=False ) \ - .filter( and_( trans.app.model.CloudImage.table.c.id != image.id, trans.app.model.CloudImage.table.c.image_id==image_id ) ) \ + .filter( and_( model.CloudImage.table.c.id != image.id, model.CloudImage.table.c.image_id==image_id ) ) \ .first(): error['id_error'] = "Image with ID '" + image_id + "' already exist. Please choose an alternative name." elif architecture=='' or len( architecture ) > 255: @@ -606,7 +608,7 @@ image.architecture = architecture # Persist session = trans.sa_session - session.save_or_update( image ) + session.add( image ) session.flush() # Log and display the management page trans.set_message( "Machine image '%s' edited." % image.image_id ) @@ -626,9 +628,9 @@ if credName or providerName or accessKey or secretKey: if credName=='' or len( credName ) > 255: error['cred_error'] = "Credentials name must be between 1 and 255 characters in length." - elif trans.app.model.CloudUserCredentials \ + elif trans.sa_session.query( model.CloudUserCredentials ) \ .filter_by( user=user, deleted=False ) \ - .filter( trans.app.model.CloudUserCredentials.table.c.name == credName ) \ + .filter( model.CloudUserCredentials.table.c.name == credName ) \ .first(): error['cred_error'] = "Credentials with that name already exist." elif providerName=='': @@ -648,7 +650,7 @@ credentials.provider = provider # Persist session = trans.sa_session - session.save_or_update( credentials ) + session.add( credentials ) session.flush() # Log and display the management page trans.log_event( "User added new credentials" ) @@ -680,9 +682,9 @@ credentials = get_stored_credentials( trans, id ) if credName=='' or len( credName ) > 255: error['cred_error'] = "Credentials name must be between 1 and 255 characters in length." - elif trans.app.model.CloudUserCredentials \ + elif trans.sa_session.query( model.CloudUserCredentials ) \ .filter_by( user=user ) \ - .filter( and_( trans.app.model.CloudUserCredentials.table.c.id != credentials.id, trans.app.model.CloudUserCredentials.table.c.name==credName ) ) \ + .filter( and_( model.CloudUserCredentials.table.c.id != credentials.id, model.CloudUserCredentials.table.c.name==credName ) ) \ .first(): error['cred_error'] = "Credentials with name '" + credName + "' already exist. Please choose an alternative name." elif accessKey=='' or len( accessKey ) > 255: @@ -702,7 +704,7 @@ credentials.secret_key = secretKey # Persist session = trans.sa_session - session.save_or_update( credentials ) + session.add( credentials ) session.flush() # Log and display the management page trans.set_message( "Credential '%s' edited." % credentials.name ) @@ -745,7 +747,8 @@ if UCI == None: # Delete and save stored.deleted = True - stored.flush() + trans.sa_session.add( stored ) + trans.sa_session.flush() # Display the management page trans.set_message( "Credentials '%s' deleted." % stored.name ) return self.list( trans ) @@ -769,9 +772,9 @@ except ValueError: error['is_secure_error'] = "Field 'is secure' can only take on an integer value '0' or '1'" - if trans.app.model.CloudProvider \ + if trans.sa_session.query( model.CloudProvider ) \ .filter_by (user=user, name=name) \ - .filter( model.CloudProvider.c.deleted != True ) \ + .filter( model.CloudProvider.table.c.deleted != True ) \ .first(): error['name_error'] = "A provider with that name already exist." elif name=='' or len( name ) > 255: @@ -843,7 +846,7 @@ provider.path = path # Persist session = trans.sa_session - session.save_or_update( provider ) + session.add( provider ) session.flush() # Log and display the management page trans.log_event( "User configured new cloud provider: '%s'" % name ) @@ -909,9 +912,9 @@ if name=='' or len( name ) > 255: error['name_error'] = "Cloud provider name must be between 1 and 255 characters in length." - elif trans.app.model.CloudProvider \ + elif trans.sa_session.query( model.CloudProvider ) \ .filter_by( user=user ) \ - .filter( and_( trans.app.model.CloudProvider.table.c.id != provider.id, trans.app.model.CloudProvider.table.c.name == name ) ) \ + .filter( and_( model.CloudProvider.table.c.id != provider.id, model.CloudProvider.table.c.name == name ) ) \ .first(): error['name_error'] = "Cloud provider with name '" + name + "' already exist. Please choose an alternative name." elif not ( is_secure == 0 or is_secure == 1): @@ -985,7 +988,7 @@ provider.path = None # Persist session = trans.sa_session - session.save_or_update( provider ) + session.add( provider ) session.flush() # Log and display the management page trans.log_event( "User edited cloud provider: '%s'" % name ) @@ -1003,14 +1006,15 @@ provider = get_provider_by_id( trans, id ) creds = trans.sa_session.query( model.CloudUserCredentials ) \ .filter_by( user=user, provider_id=provider.id ) \ - .filter( model.CloudUserCredentials.c.deleted != True ) \ + .filter( model.CloudUserCredentials.table.c.deleted != True ) \ .all() if len( creds ) == 0: # Delete and save #sess = trans.sa_session provider.deleted = True - provider.flush() + trans.sa_session.add( provider ) + trans.sa_session.flush() # Display the management page trans.set_message( "Cloud provider '%s' deleted." % provider.name ) return self.list( trans ) @@ -1024,7 +1028,7 @@ @web.json def json_update( self, trans ): user = trans.get_user() - UCIs = trans.sa_session.query( model.UCI ).filter_by( user=user ).filter( model.UCI.c.deleted != True ).all() + UCIs = trans.sa_session.query( model.UCI ).filter_by( user=user ).filter( model.UCI.table.c.deleted != True ).all() insd = {} # instance name-state dict for uci in UCIs: dict = {} @@ -1062,7 +1066,7 @@ def get_provider( trans, name ): user = trans.get_user() - return trans.app.model.CloudProvider \ + return trans.sa_session.query( model.CloudProvider ) \ .filter_by (user=user, name=name) \ .first() @@ -1126,19 +1130,6 @@ # Looks good return live -def get_mi( trans, uci, size='m1.small' ): - """ - Get appropriate machine image (mi) based on instance size. - TODO: Dummy method - need to implement logic - For valid sizes, see http://aws.amazon.com/ec2/instance-types/ - """ - if uci.credentials.provider.type == 'ec2': - return trans.app.model.CloudImage.filter( - trans.app.model.CloudImage.table.c.id==2).first() - else: - return trans.app.model.CloudImage.filter( - trans.app.model.CloudImage.table.c.id==1).first() - def get_stores( trans, uci ): """ Get stores objects that are connected to uci object @@ -1146,7 +1137,7 @@ user = trans.get_user() stores = trans.sa_session.query( model.CloudStore ) \ .filter_by( user=user, uci_id=uci.id ) \ - .filter( model.CloudStore.c.status != store_status.ERROR ) \ + .filter( model.CloudStore.table.c.status != store_status.ERROR ) \ .all() return stores @@ -1173,7 +1164,7 @@ # creds = trans.sa_session.query( model.CloudUserCredentials ) \ # .filter_by( user=user, name=credName ) \ # .first() - #.filter( model.CloudUserCredentials.c.deleted != True ) \ MOVE TO LINE ABOVE ONCE DELETE COLUMS ARE IMPLEMENTED + #.filter( model.CloudUserCredentials.table.c.deleted != True ) \ MOVE TO LINE ABOVE ONCE DELETE COLUMS ARE IMPLEMENTED if creds: a_key = creds.access_key diff -r 7d013eb98022 -r 39502dd3fd23 templates/cloud/configure_cloud.mako --- a/templates/cloud/configure_cloud.mako Thu Nov 12 16:36:07 2009 -0500 +++ b/templates/cloud/configure_cloud.mako Mon Nov 16 20:37:47 2009 -0500 @@ -90,8 +90,9 @@ // Update 'state' and 'time alive' fields $(elem + "-state").text( data[i].state ); - if ( ( prev_old_state.match('newUCI') && new_state=='new' ) || \ + if ( ( prev_old_state.match('newUCI') && new_state=='new' ) || ( prev_old_state.match('newUCI') && new_state=='available' ) || \ ( prev_old_state.match('newUCI') && new_state=='creating' ) || ( prev_old_state.match('new') && new_state=='creating' ) || \ + ( prev_old_state.match('new') && new_state=='available' ) || \ ( prev_old_state.match('deletingUCI') && new_state=='deleted' ) || ( prev_old_state.match('deleting') && new_state=='deleted' ) || \ ( prev_old_state.match('available') && new_state=='error' ) || ( prev_old_state.match('deleting') && new_state=='error' ) ) { // TODO: on state change from available->error and deleting->error page should be refreshed but that causes problems with

1 0

[hg] galaxy 3092: Cleaned up code in main cloud controller.
by Greg Von Kuster 23 Nov '09

23 Nov '09

details: http://www.bx.psu.edu/hg/galaxy/rev/118dc385752b changeset: 3092:118dc385752b user: Enis Afgan <afgane(a)gmail.com> date: Thu Nov 12 14:24:43 2009 -0500 description: Cleaned up code in main cloud controller. diffstat: lib/galaxy/cloud/providers/ec2.py | 16 +- lib/galaxy/cloud/providers/eucalyptus.py | 12 +- lib/galaxy/web/controllers/cloud.py | 632 +++++++++++++++++----------------- templates/admin/index.mako | 4 +- templates/cloud/add_credentials.mako | 2 +- templates/cloud/add_image.mako | 2 +- templates/cloud/configure_cloud.mako | 30 +- templates/cloud/configure_uci.mako | 2 +- templates/cloud/edit_credentials.mako | 2 +- templates/cloud/edit_image.mako | 2 +- templates/cloud/list_images.mako | 6 +- templates/cloud/view.mako | 157 -------- templates/cloud/viewInstance.mako | 140 ------- templates/cloud/view_credentials.mako | 157 ++++++++ templates/cloud/view_instance.mako | 140 +++++++ templates/cloud/view_usage.mako | 2 +- 16 files changed, 660 insertions(+), 646 deletions(-) diffs (1644 lines): diff -r e21f605d2766 -r 118dc385752b lib/galaxy/cloud/providers/ec2.py --- a/lib/galaxy/cloud/providers/ec2.py Thu Nov 12 12:42:17 2009 -0500 +++ b/lib/galaxy/cloud/providers/ec2.py Thu Nov 12 14:24:43 2009 -0500 @@ -48,7 +48,7 @@ ERROR = "error" ) -store_states = Bunch( +store_status = Bunch( IN_USE = "in-use", CREATING = "creating", DELETED = 'deleted', @@ -105,14 +105,14 @@ elif uci_state==uci_states.SNAPSHOT: self.snapshotUCI( uci_wrapper ) except: - log.exception( "Uncaught exception executing request." ) + log.exception( "Uncaught exception executing cloud request." ) cnt += 1 def get_connection( self, uci_wrapper ): """ Establishes EC2 cloud connection using user's credentials associated with given UCI """ - log.debug( 'Establishing %s cloud connection' % self.type ) + log.debug( 'Establishing %s cloud connection.' % self.type ) provider = uci_wrapper.get_provider() try: region = RegionInfo( None, provider.region_name, provider.region_endpoint ) @@ -127,8 +127,8 @@ is_secure=provider.is_secure, region=region, path=provider.path ) - except Exception, ex: - err = "Establishing connection with cloud failed: " + str( ex ) + except boto.exception.EC2ResponseError, e: + err = "Establishing connection with cloud failed: " + str( e ) log.error( err ) uci_wrapper.set_error( err, True ) return None @@ -545,8 +545,8 @@ self.updateInstance( inst ) # Update storage volume(s) - stores = model.CloudStore.filter( or_( model.CloudStore.c.status==store_states.IN_USE, - model.CloudStore.c.status==store_states.CREATING, + stores = model.CloudStore.filter( or_( model.CloudStore.c.status==store_status.IN_USE, + model.CloudStore.c.status==store_status.CREATING, model.CloudStore.c.status==None ) ).all() for store in stores: if self.type == store.uci.credentials.provider.type: # and store.volume_id != None: @@ -558,7 +558,7 @@ # store.uci.error = "There exists an entry in local database for a storage volume without an ID. Storage volume might have been created " \ # "with cloud provider though. Manual check is recommended. After understanding what happened, local database entry for given " \ # "storage volume should be updated." -# store.status = store_states.ERROR +# store.status = store_status.ERROR # store.uci.state = uci_states.ERROR # store.uci.flush() # store.flush() diff -r e21f605d2766 -r 118dc385752b lib/galaxy/cloud/providers/eucalyptus.py --- a/lib/galaxy/cloud/providers/eucalyptus.py Thu Nov 12 12:42:17 2009 -0500 +++ b/lib/galaxy/cloud/providers/eucalyptus.py Thu Nov 12 14:24:43 2009 -0500 @@ -114,9 +114,9 @@ log.debug( 'Establishing %s cloud connection.' % self.type ) provider = uci_wrapper.get_provider() try: - euca_region = RegionInfo( None, provider.region_name, provider.region_endpoint ) - except Exception, e: - err = "Selecting region with cloud provider failed: " + str( e ) + region = RegionInfo( None, provider.region_name, provider.region_endpoint ) + except Exception, ex: + err = "Selecting region with cloud provider failed: " + str( ex ) log.error( err ) uci_wrapper.set_error( err, True ) return None @@ -125,7 +125,7 @@ aws_secret_access_key=uci_wrapper.get_secret_key(), is_secure=provider.is_secure, port=provider.port, - region=euca_region, + region=region, path=provider.path ) except boto.exception.EC2ResponseError, e: err = "Establishing connection with cloud failed: " + str( e ) @@ -853,12 +853,12 @@ s_key = uci.credentials.secret_key # Get connection try: - euca_region = RegionInfo( None, uci.credentials.provider.region_name, uci.credentials.provider.region_endpoint ) + region = RegionInfo( None, uci.credentials.provider.region_name, uci.credentials.provider.region_endpoint ) conn = EC2Connection( aws_access_key_id=a_key, aws_secret_access_key=s_key, is_secure=uci.credentials.provider.is_secure, port=uci.credentials.provider.port, - region=euca_region, + region=region, path=uci.credentials.provider.path ) except boto.exception.EC2ResponseError, e: err = "Establishing connection with cloud failed: " + str( e ) diff -r e21f605d2766 -r 118dc385752b lib/galaxy/web/controllers/cloud.py --- a/lib/galaxy/web/controllers/cloud.py Thu Nov 12 12:42:17 2009 -0500 +++ b/lib/galaxy/web/controllers/cloud.py Thu Nov 12 14:24:43 2009 -0500 @@ -59,7 +59,7 @@ ERROR = "error" ) -store_states = Bunch( +store_status = Bunch( IN_USE = "in-use", CREATING = "creating", DELETED = 'deleted', @@ -135,9 +135,9 @@ model.UCI.c.state==uci_states.SUBMITTED_UCI ) ) \ .all() if pendingInstances: - trans.set_message( "Galaxy instance started. Note that it will take several minutes for the instance to start " - "(typically, 3-5 minutes). Once the instance is running and Galaxy is available, " - "a button to connect to the instance will then appear alongside instance description." ) + trans.set_message( "Galaxy instance started. NOTE: Please wait about 5 minutes for the instance to " + "start up. A button to connect to the instance will appear alongside " + "instance description once cloud instance of Galaxy is ready." ) # log.debug( "provider.is_secure: '%s'" % trans.sa_session.query( model.CloudProvider).filter_by(id=1).first().is_secure ) # trans.sa_session.query( model.CloudProvider).filter_by(id=1).first().is_secure=False @@ -152,200 +152,11 @@ prevInstances = prevInstances, cloudProviders = cloudProviders ) - @web.expose - @web.require_login( "start Galaxy cloud instance" ) - def start( self, trans, id, type='m1.small' ): - """ - Start a new cloud resource instance - """ - user = trans.get_user() - uci = get_uci( trans, id ) - stores = get_stores( trans, uci ) - # Ensure instance is available and then store relevant data - # into DB to initiate instance startup by cloud manager - if ( len(stores) is not 0 ) and ( uci.state == uci_states.AVAILABLE ): - instance = model.CloudInstance() - instance.user = user - instance.uci = uci - instance.state = instance_states.SUBMITTED - instance.availability_zone = stores[0].availability_zone # Bc. all EBS volumes need to be in the same avail. zone, just check 1st - instance.type = type - uci.state = uci_states.SUBMITTED_UCI - # Persist - session = trans.sa_session - session.save_or_update( instance ) - session.save_or_update( uci ) - session.flush() - # Log - trans.log_event ("User initiated starting of UCI '%s'." % uci.name ) - trans.set_message( "Galaxy instance started. NOTE: Please wait about 5 minutes for the instance to " - "start up. A button to connect to the instance will appear alongside " - "instance description once cloud instance of Galaxy is ready." ) - return self.list( trans ) - - if len(stores) == 0: - error( "This instance does not have any storage volumes associated it and thus cannot be started." ) - else: - error( "Cannot start instance that is in state '%s'." % uci.state ) - return self.list( trans ) - - @web.expose - @web.require_login( "stop Galaxy cloud instance" ) - def stop( self, trans, id ): - """ - Stop a cloud UCI instance. - """ - uci = get_uci( trans, id ) - if ( uci.state != uci_states.DELETING ) and \ - ( uci.state != uci_states.DELETING_UCI ) and \ - ( uci.state != uci_states.ERROR ) and \ - ( uci.state != uci_states.SHUTTING_DOWN_UCI ) and \ - ( uci.state != uci_states.SHUTTING_DOWN ) and \ - ( uci.state != uci_states.AVAILABLE ): - uci.state = uci_states.SHUTTING_DOWN_UCI - session = trans.sa_session - session.save_or_update( uci ) - session.flush() - trans.log_event( "User stopped cloud instance '%s' (id: %s)" % ( uci.name, uci.id ) ) - trans.set_message( "Stopping of Galaxy instance '%s' initiated." % uci.name ) - - return self.list( trans ) - - trans.show_error_message( "Cannot stop instance that is in state '%s'." % uci.state ) - return self.list( trans ) - - @web.expose - @web.require_login( "delete user configured Galaxy cloud instance" ) - def deleteInstance( self, trans, id ): - """ - Deletes User Configured Instance (UCI) from the cloud and local database. NOTE that this implies deletion of - any and all storage associated with this UCI! - """ - uci = get_uci( trans, id ) - - if ( uci.state != uci_states.DELETING_UCI ) and ( uci.state != uci_states.DELETING ) and ( uci.state != uci_states.ERROR ): - name = uci.name - uci.state = uci_states.DELETING_UCI - session = trans.sa_session - session.save_or_update( uci ) - session.flush() - trans.log_event( "User marked cloud instance '%s' for deletion." % name ) - trans.set_message( "Galaxy instance '%s' marked for deletion." % name ) - return self.list( trans ) - - if uci.state != uci_states.ERROR: - trans.set_message( "Cannot delete instance in state ERROR." ) - else: - trans.set_message( "Instance '%s' is already marked for deletion." % uci.name ) - return self.list( trans ) + # ----- UCI methods ----- @web.expose @web.require_login( "use Galaxy cloud" ) - def create_snapshot( self, trans, id ): - user = trans.get_user() - id = trans.security.decode_id( id ) - uci = get_uci( trans, id ) - - stores = trans.sa_session.query( model.CloudStore ) \ - .filter_by( user=user, deleted=False, uci_id=id ) \ - .all() - - if ( len( stores ) > 0 ) and ( uci.state == uci_states.AVAILABLE ): - for store in stores: - snapshot = model.CloudSnapshot() - snapshot.user = user - snapshot.uci = uci - snapshot.store = store - snapshot.status = snapshot_status.SUBMITTED - uci.state = uci_states.SNAPSHOT_UCI - # Persist - session = trans.sa_session - session.save_or_update( snapshot ) - session.save_or_update( uci ) - session.flush() - elif len( stores ) == 0: - error( "No storage volumes found that are associated with this instance." ) - else: - error( "Snapshot can be created only for an instance that is in 'available' state." ) - - # Log and display the management page - trans.log_event( "User initiated creation of new snapshot." ) - trans.set_message( "Creation of new snapshot initiated. " ) - return self.list( trans ) - - @web.expose - @web.require_login( "use Galaxy cloud" ) - def view_snapshots( self, trans, id=None ): - """ - View details about any snapshots associated with given UCI - """ - user = trans.get_user() - id = trans.security.decode_id( id ) - - snaps = trans.sa_session.query( model.CloudSnapshot ) \ - .filter_by( user=user, uci_id=id, deleted=False ) \ - .order_by( desc( model.CloudSnapshot.c.update_time ) ) \ - .all() - - return trans.fill_template( "cloud/view_snapshots.mako", - snaps = snaps ) - - @web.expose - @web.require_login( "use Galaxy cloud" ) - def delete_snapshot( self, trans, uci_id=None, snap_id=None ): - """ - Initiates deletion of a snapshot - """ - user = trans.get_user() - snap_id = trans.security.decode_id( snap_id ) - # Set snapshot as 'ready for deletion' to be picked up by general updater - snap = trans.sa_session.query( model.CloudSnapshot ).get( snap_id ) - - if snap.status == snapshot_status.COMPLETED: - snap.status = snapshot_status.DELETE - snap.flush() - trans.set_message( "Snapshot '%s' is marked for deletion. Once the deletion is complete, it will no longer be visible in this list. " - "Please note that this process may take up to a minute." % snap.snapshot_id ) - else: - error( "Only snapshots in state 'completed' can be deleted. See the cloud provider directly " - "if you believe the snapshot is available and can be deleted." ) - - # Display new list of snapshots - uci_id = trans.security.decode_id( uci_id ) - snaps = trans.sa_session.query( model.CloudSnapshot ) \ - .filter_by( user=user, uci_id=uci_id, deleted=False ) \ - .order_by( desc( model.CloudSnapshot.c.update_time ) ) \ - .all() - - return trans.fill_template( "cloud/view_snapshots.mako", - snaps = snaps ) - - @web.expose - @web.require_login( "add instance storage" ) - def addStorage( self, trans, id ): - instance = get_uci( trans, id ) - - - error( "Adding storage to instance '%s' is not supported yet." % instance.name ) - - return self.list( trans ) - - @web.expose - @web.require_login( "use Galaxy cloud" ) - def usageReport( self, trans, id ): - user = trans.get_user() - id = trans.security.decode_id( id ) - - prevInstances = trans.sa_session.query( model.CloudInstance ) \ - .filter_by( user=user, state=instance_states.TERMINATED, uci_id=id ) \ - .order_by( desc( model.CloudInstance.c.update_time ) ) \ - .all() - - return trans.fill_template( "cloud/view_usage.mako", prevInstances = prevInstances ) - - @web.expose - @web.require_login( "use Galaxy cloud" ) - def configureNew( self, trans, instanceName='', credName='', volSize='', zone='' ): + def configure_new_uci( self, trans, instanceName='', credName='', volSize='', zone='' ): """ Configure and add new cloud instance to user's instance pool """ @@ -447,8 +258,255 @@ providersToZones = providersToZones ) @web.expose + @web.require_login( "start Galaxy cloud instance" ) + def start( self, trans, id, type='m1.small' ): + """ + Start a new cloud resource instance + """ + user = trans.get_user() + uci = get_uci( trans, id ) + stores = get_stores( trans, uci ) + # Ensure instance is available and then store relevant data + # into DB to initiate instance startup by cloud manager + if ( len(stores) is not 0 ) and ( uci.state == uci_states.AVAILABLE ): + instance = model.CloudInstance() + instance.user = user + instance.uci = uci + instance.state = instance_states.SUBMITTED + instance.availability_zone = stores[0].availability_zone # Bc. all EBS volumes need to be in the same avail. zone, just check 1st + instance.type = type + uci.state = uci_states.SUBMITTED_UCI + # Persist + session = trans.sa_session + session.save_or_update( instance ) + session.save_or_update( uci ) + session.flush() + # Log + trans.log_event ("User initiated starting of UCI '%s'." % uci.name ) + trans.set_message( "Galaxy instance started. NOTE: Please wait about 5 minutes for the instance to " + "start up. A button to connect to the instance will appear alongside " + "instance description once cloud instance of Galaxy is ready." ) + return self.list( trans ) + + if len(stores) == 0: + error( "This instance does not have any storage volumes associated it and thus cannot be started." ) + else: + error( "Cannot start instance that is in state '%s'." % uci.state ) + return self.list( trans ) + + @web.expose + @web.require_login( "stop Galaxy cloud instance" ) + def stop( self, trans, id ): + """ + Stop a cloud UCI instance. + """ + uci = get_uci( trans, id ) + if ( uci.state != uci_states.DELETING ) and \ + ( uci.state != uci_states.DELETING_UCI ) and \ + ( uci.state != uci_states.ERROR ) and \ + ( uci.state != uci_states.SHUTTING_DOWN_UCI ) and \ + ( uci.state != uci_states.SHUTTING_DOWN ) and \ + ( uci.state != uci_states.AVAILABLE ): + uci.state = uci_states.SHUTTING_DOWN_UCI + session = trans.sa_session + session.save_or_update( uci ) + session.flush() + trans.log_event( "User stopped cloud instance '%s' (id: %s)" % ( uci.name, uci.id ) ) + trans.set_message( "Stopping of Galaxy instance '%s' initiated." % uci.name ) + + return self.list( trans ) + + trans.show_error_message( "Cannot stop instance that is in state '%s'." % uci.state ) + return self.list( trans ) + + @web.expose + @web.require_login( "use Galaxy cloud" ) + def set_uci_state( self, trans, id, state='available', clear_error=True ): + """ + Sets state of UCI to given state, optionally resets error field, and resets UCI's launch time field to 'None'. + """ + uci = get_uci( trans, id ) + uci.state = state + if clear_error: + uci.error = None + uci.launch_time = None + trans.sa_session.flush() + trans.set_message( "Instance '%s' state reset." % uci.name ) + return self.list( trans ) + + @web.expose + @web.require_login( "view instance details" ) + def view_uci_details( self, trans, id=None ): + """ + View details about running instance + """ + uci = get_uci( trans, id ) + instances = get_instances( trans, uci ) # TODO: Handle list (will probably need to be done in mako template) + + return trans.fill_template( "cloud/view_instance.mako", + liveInstance = instances ) + + @web.expose + @web.require_login( "use Galaxy cloud" ) + def rename_uci( self, trans, id, new_name=None ): + instance = get_uci( trans, id ) + if new_name is not None: + if len(new_name) > 255: + error( "Instance name must be less than 255 characters long." ) + user = trans.get_user() + name_exists = trans.sa_session.query( model.UCI ) \ + .filter_by( user=user, name=new_name ) \ + .first() + if name_exists: + error( "Specified name ('%s') is already used by an existing instance. Please choose an alternative name." % new_name ) + + # Update name in local DB + instance.name = new_name + trans.sa_session.flush() + trans.set_message( "Instance renamed to '%s'." % new_name ) + return self.list( trans ) + else: + return trans.show_form( + web.FormBuilder( url_for( id=trans.security.encode_id(instance.id) ), "Rename instance", submit_text="Rename" ) + .add_text( "new_name", "Instance name", value=instance.name ) ) + + @web.expose + @web.require_login( "use Galaxy cloud" ) + def uci_usage_report( self, trans, id ): + user = trans.get_user() + id = trans.security.decode_id( id ) + + prevInstances = trans.sa_session.query( model.CloudInstance ) \ + .filter_by( user=user, state=instance_states.TERMINATED, uci_id=id ) \ + .order_by( desc( model.CloudInstance.c.update_time ) ) \ + .all() + + return trans.fill_template( "cloud/view_usage.mako", prevInstances = prevInstances ) + + @web.expose + @web.require_login( "delete user configured Galaxy cloud instance" ) + def delete_uci( self, trans, id ): + """ + Deletes User Configured Instance (UCI) from the cloud and local database. NOTE that this implies deletion of + any and all storage associated with this UCI! + """ + uci = get_uci( trans, id ) + + if ( uci.state != uci_states.DELETING_UCI ) and ( uci.state != uci_states.DELETING ) and ( uci.state != uci_states.ERROR ): + name = uci.name + uci.state = uci_states.DELETING_UCI + session = trans.sa_session + session.save_or_update( uci ) + session.flush() + trans.log_event( "User marked cloud instance '%s' for deletion." % name ) + trans.set_message( "Galaxy instance '%s' marked for deletion." % name ) + return self.list( trans ) + + if uci.state != uci_states.ERROR: + trans.set_message( "Cannot delete instance in state ERROR." ) + else: + trans.set_message( "Instance '%s' is already marked for deletion." % uci.name ) + return self.list( trans ) + + # ----- Snapshot methods ----- + + @web.expose + @web.require_login( "use Galaxy cloud" ) + def create_snapshot( self, trans, id ): + user = trans.get_user() + id = trans.security.decode_id( id ) + uci = get_uci( trans, id ) + + stores = trans.sa_session.query( model.CloudStore ) \ + .filter_by( user=user, deleted=False, uci_id=id ) \ + .all() + + if ( len( stores ) > 0 ) and ( uci.state == uci_states.AVAILABLE ): + for store in stores: + snapshot = model.CloudSnapshot() + snapshot.user = user + snapshot.uci = uci + snapshot.store = store + snapshot.status = snapshot_status.SUBMITTED + uci.state = uci_states.SNAPSHOT_UCI + # Persist + session = trans.sa_session + session.save_or_update( snapshot ) + session.save_or_update( uci ) + session.flush() + elif len( stores ) == 0: + error( "No storage volumes found that are associated with this instance." ) + else: + error( "Snapshot can be created only for an instance that is in 'available' state." ) + + # Log and display the management page + trans.log_event( "User initiated creation of new snapshot." ) + trans.set_message( "Creation of new snapshot initiated. " ) + return self.list( trans ) + + @web.expose + @web.require_login( "use Galaxy cloud" ) + def view_snapshots( self, trans, id=None ): + """ + View details about any snapshots associated with given UCI + """ + user = trans.get_user() + id = trans.security.decode_id( id ) + + snaps = trans.sa_session.query( model.CloudSnapshot ) \ + .filter_by( user=user, uci_id=id, deleted=False ) \ + .order_by( desc( model.CloudSnapshot.c.update_time ) ) \ + .all() + + return trans.fill_template( "cloud/view_snapshots.mako", + snaps = snaps ) + + @web.expose + @web.require_login( "use Galaxy cloud" ) + def delete_snapshot( self, trans, uci_id=None, snap_id=None ): + """ + Initiates deletion of a snapshot + """ + user = trans.get_user() + snap_id = trans.security.decode_id( snap_id ) + # Set snapshot as 'ready for deletion' to be picked up by general updater + snap = trans.sa_session.query( model.CloudSnapshot ).get( snap_id ) + + if snap.status == snapshot_status.COMPLETED: + snap.status = snapshot_status.DELETE + snap.flush() + trans.set_message( "Snapshot '%s' is marked for deletion. Once the deletion is complete, it will no longer be visible in this list. " + "Please note that this process may take up to a minute." % snap.snapshot_id ) + else: + error( "Only snapshots in state 'completed' can be deleted. See the cloud provider directly " + "if you believe the snapshot is available and can be deleted." ) + + # Display new list of snapshots + uci_id = trans.security.decode_id( uci_id ) + snaps = trans.sa_session.query( model.CloudSnapshot ) \ + .filter_by( user=user, uci_id=uci_id, deleted=False ) \ + .order_by( desc( model.CloudSnapshot.c.update_time ) ) \ + .all() + + return trans.fill_template( "cloud/view_snapshots.mako", + snaps = snaps ) + + # ----- Storage methods ----- + + @web.expose + @web.require_login( "add instance storage" ) + def add_storage( self, trans, id ): + instance = get_uci( trans, id ) + + + error( "Adding storage to instance '%s' is not supported yet." % instance.name ) + + return self.list( trans ) + + # ----- Image methods ----- + @web.expose @web.require_admin - def addNewImage( self, trans, provider_type='', image_id='', manifest='', architecture='', state=None ): + def add_new_image( self, trans, provider_type='', image_id='', manifest='', architecture='', state=None ): #id_error = arch_error = provider_error = manifest_error = None error = {} if provider_type or image_id or manifest or architecture: @@ -498,24 +556,24 @@ @web.expose @web.require_login( "use Galaxy cloud" ) - def listMachineImages( self, trans ): + def list_machine_images( self, trans ): images = trans.sa_session.query( model.CloudImage ).filter( trans.app.model.CloudImage.table.c.deleted != True ).all() return trans.fill_template( '/cloud/list_images.mako', images=images ) @web.expose @web.require_admin - def deleteImage( self, trans, id=None ): + def delete_image( self, trans, id=None ): if not isinstance( id, int ): id = trans.security.decode_id( id ) image = trans.sa_session.query( model.CloudImage ).get( id ) image.deleted = True image.flush() - return self.listMachineImages( trans ) + return self.list_machine_images( trans ) @web.expose @web.require_admin - def editImage( self, trans, provider_type='', image_id='', manifest='', architecture='', id='', edited=False ): + def edit_image( self, trans, provider_type='', image_id='', manifest='', architecture='', id='', edited=False ): error = {} if not isinstance( id, int ): id = trans.security.decode_id( id ) @@ -552,93 +610,13 @@ session.flush() # Log and display the management page trans.set_message( "Machine image '%s' edited." % image.image_id ) - return self.listMachineImages( trans ) - - @web.expose - @web.require_login( "use Galaxy cloud" ) - def edit( self, trans, id, credName=None, accessKey=None, secretKey=None, edited=False ): - error = {} - if not edited: - credentials = get_stored_credentials( trans, id ) - return trans.fill_template( "cloud/edit_credentials.mako", - credential = credentials, - error = error - ) - else: - user = trans.get_user() - credentials = get_stored_credentials( trans, id ) - if credName=='' or len( credName ) > 255: - error['cred_error'] = "Credentials name must be between 1 and 255 characters in length." - elif trans.app.model.CloudUserCredentials \ - .filter_by( user=user ) \ - .filter( and_( trans.app.model.CloudUserCredentials.table.c.id != credentials.id, trans.app.model.CloudUserCredentials.table.c.name==credName ) ) \ - .first(): - error['cred_error'] = "Credentials with name '" + credName + "' already exist. Please choose an alternative name." - elif accessKey=='' or len( accessKey ) > 255: - error['access_key_error'] = "Access key must be between 1 and 255 characters long." - elif secretKey=='' or len( secretKey ) > 255: - error['secret_key_error'] = "Secret key must be between 1 and 255 characters long." + return self.list_machine_images( trans ) - if error: - return trans.fill_template( "cloud/edit_credentials.mako", - credential = credentials, - error = error - ) - else: - # Edit user stored credentials - credentials.name = credName - credentials.access_key = accessKey - credentials.secret_key = secretKey - # Persist - session = trans.sa_session - session.save_or_update( credentials ) - session.flush() - # Log and display the management page - trans.set_message( "Credential '%s' edited." % credentials.name ) - return self.list( trans ) - - @web.expose - @web.require_login( "use Galaxy cloud" ) - def renameInstance( self, trans, id, new_name=None ): - instance = get_uci( trans, id ) - if new_name is not None: - if len(new_name) > 255: - error( "Instance name must be less than 255 characters long." ) - user = trans.get_user() - name_exists = trans.sa_session.query( model.UCI ) \ - .filter_by( user=user, name=new_name ) \ - .first() - if name_exists: - error( "Specified name ('%s') is already used by an existing instance. Please choose an alternative name." % new_name ) - - # Update name in local DB - instance.name = new_name - trans.sa_session.flush() - trans.set_message( "Instance renamed to '%s'." % new_name ) - return self.list( trans ) - else: - return trans.show_form( - web.FormBuilder( url_for( id=trans.security.encode_id(instance.id) ), "Rename instance", submit_text="Rename" ) - .add_text( "new_name", "Instance name", value=instance.name ) ) - - @web.expose - @web.require_login( "use Galaxy cloud" ) - def set_uci_state( self, trans, id, state='available', clear_error=True ): - """ - Sets state of UCI to given state, optionally resets error field, and resets UCI's launch time field to 'None'. - """ - uci = get_uci( trans, id ) - uci.state = state - if clear_error: - uci.error = None - uci.launch_time = None - trans.sa_session.flush() - trans.set_message( "Instance '%s' state reset." % uci.name ) - return self.list( trans ) - + # ----- Credentials methods ----- + @web.expose @web.require_login( "add credentials" ) - def add( self, trans, credName='', accessKey='', secretKey='', providerName='' ): + def add_credentials( self, trans, credName='', accessKey='', secretKey='', providerName='' ): """ Add user's cloud credentials stored under name `credName`. """ @@ -688,15 +666,58 @@ ) @web.expose + @web.require_login( "use Galaxy cloud" ) + def edit_credentials( self, trans, id, credName=None, accessKey=None, secretKey=None, edited=False ): + error = {} + if not edited: + credentials = get_stored_credentials( trans, id ) + return trans.fill_template( "cloud/edit_credentials.mako", + credential = credentials, + error = error + ) + else: + user = trans.get_user() + credentials = get_stored_credentials( trans, id ) + if credName=='' or len( credName ) > 255: + error['cred_error'] = "Credentials name must be between 1 and 255 characters in length." + elif trans.app.model.CloudUserCredentials \ + .filter_by( user=user ) \ + .filter( and_( trans.app.model.CloudUserCredentials.table.c.id != credentials.id, trans.app.model.CloudUserCredentials.table.c.name==credName ) ) \ + .first(): + error['cred_error'] = "Credentials with name '" + credName + "' already exist. Please choose an alternative name." + elif accessKey=='' or len( accessKey ) > 255: + error['access_key_error'] = "Access key must be between 1 and 255 characters long." + elif secretKey=='' or len( secretKey ) > 255: + error['secret_key_error'] = "Secret key must be between 1 and 255 characters long." + + if error: + return trans.fill_template( "cloud/edit_credentials.mako", + credential = credentials, + error = error + ) + else: + # Edit user stored credentials + credentials.name = credName + credentials.access_key = accessKey + credentials.secret_key = secretKey + # Persist + session = trans.sa_session + session.save_or_update( credentials ) + session.flush() + # Log and display the management page + trans.set_message( "Credential '%s' edited." % credentials.name ) + return self.list( trans ) + + @web.expose @web.require_login( "view credentials" ) - def view( self, trans, id=None ): + def view_credentials( self, trans, id=None ): """ View details for user credentials """ # Load credentials from database stored = get_stored_credentials( trans, id ) - return trans.fill_template( "cloud/view.mako", + return trans.fill_template( "cloud/view_credentials.mako", credDetails = stored ) @web.expose @@ -707,42 +728,33 @@ """ @web.expose - @web.require_login( "view instance details" ) - def viewInstance( self, trans, id=None ): - """ - View details about running instance - """ - uci = get_uci( trans, id ) - instances = get_instances( trans, uci ) # TODO: Handle list (will probably need to be done in mako template) - - return trans.fill_template( "cloud/viewInstance.mako", - liveInstance = instances ) - - @web.expose @web.require_login( "delete credentials" ) - def delete( self, trans, id=None ): + def delete_credentials( self, trans, id=None ): """ Delete user's cloud credentials checking that no registered instances are tied to given credentials. """ # Load credentials from database user = trans.get_user() stored = get_stored_credentials( trans, id ) - UCIs = trans.sa_session.query( model.UCI ) \ - .filter_by( user=user, credentials_id=stored.id ) \ - .filter( model.UCI.c.deleted != True ) \ - .all() + # Check if there are any UCIs that depend on these credentials + UCI = None + UCI = trans.sa_session.query( model.UCI ) \ + .filter_by( user=user, credentials_id=stored.id, deleted=False ) \ + .first() - if len(UCIs) == 0: + if UCI == None: # Delete and save stored.deleted = True stored.flush() # Display the management page trans.set_message( "Credentials '%s' deleted." % stored.name ) return self.list( trans ) - - error( "Existing instance(s) depend on credentials '%s'. You must delete those instances before being able \ - to delete these credentials." % stored.name ) - return self.list( trans ) + else: + error( "Existing instance(s) depend on credentials '%s'. You must delete those instances before being able \ + to delete these credentials." % stored.name ) + return self.list( trans ) + + # ----- Provider methods ----- @web.expose @web.require_login( "add provider" ) @@ -1007,6 +1019,8 @@ to delete this cloud provider." % provider.name ) return self.list( trans ) + # ----- AJAX methods ----- + @web.json def json_update( self, trans ): user = trans.get_user() @@ -1132,7 +1146,7 @@ user = trans.get_user() stores = trans.sa_session.query( model.CloudStore ) \ .filter_by( user=user, uci_id=uci.id ) \ - .filter( model.CloudStore.c.status != store_states.ERROR ) \ + .filter( model.CloudStore.c.status != store_status.ERROR ) \ .all() return stores diff -r e21f605d2766 -r 118dc385752b templates/admin/index.mako --- a/templates/admin/index.mako Thu Nov 12 12:42:17 2009 -0500 +++ b/templates/admin/index.mako Thu Nov 12 14:24:43 2009 -0500 @@ -127,8 +127,8 @@ </div> <div class="toolSectionBody"> <div class="toolSectionBg"> - <div class="toolTitle"><a href="${h.url_for( controller='cloud', action='listMachineImages' )}" target="galaxy_main">List machine images</a></div> - <div class="toolTitle"><a href="${h.url_for( controller='cloud', action='addNewImage' )}" target="galaxy_main">Add machine image</a></div> + <div class="toolTitle"><a href="${h.url_for( controller='cloud', action='list_machine_images' )}" target="galaxy_main">List machine images</a></div> + <div class="toolTitle"><a href="${h.url_for( controller='cloud', action='add_new_image' )}" target="galaxy_main">Add machine image</a></div> </div> </div> </div> diff -r e21f605d2766 -r 118dc385752b templates/cloud/add_credentials.mako --- a/templates/cloud/add_credentials.mako Thu Nov 12 12:42:17 2009 -0500 +++ b/templates/cloud/add_credentials.mako Thu Nov 12 14:24:43 2009 -0500 @@ -20,7 +20,7 @@ <div class="form"> <div class="form-title">Add credentials</div> <div class="form-body"> - <form name="Add credentials" action="${h.url_for( action='add' )}" method="post" > + <form name="add_credentials" action="${h.url_for( action='add_credentials' )}" method="post" > <% cls = "form-row" diff -r e21f605d2766 -r 118dc385752b templates/cloud/add_image.mako --- a/templates/cloud/add_image.mako Thu Nov 12 12:42:17 2009 -0500 +++ b/templates/cloud/add_image.mako Thu Nov 12 14:24:43 2009 -0500 @@ -18,7 +18,7 @@ <div class="form"> <div class="form-title">Add machine image</div> <div class="form-body"> - <form name="add_image" action="${h.url_for( action='addNewImage' )}" method="post" > + <form name="add_image" action="${h.url_for( action='add_new_image' )}" method="post" > <% cls = "form-row" if error.has_key('provider_error'): diff -r e21f605d2766 -r 118dc385752b templates/cloud/configure_cloud.mako --- a/templates/cloud/configure_cloud.mako Thu Nov 12 12:42:17 2009 -0500 +++ b/templates/cloud/configure_cloud.mako Thu Nov 12 14:24:43 2009 -0500 @@ -156,7 +156,7 @@ %if cloudCredentials: <ul class="manage-table-actions"> <li> - <a class="action-button" href="${h.url_for( action='add' )}"> + <a class="action-button" href="${h.url_for( action='add_credentials' )}"> <img src="${h.url_for('/static/images/silk/add.png')}" /> <span>Add credentials</span> </a> @@ -181,9 +181,9 @@ </td> <td> <div popupmenu="cr-${i}-popup"> - <a class="action-button" href="${h.url_for( action='view', id=trans.security.encode_id(cloudCredential.id) )}">View</a> - <a class="action-button" href="${h.url_for( action='edit', id=trans.security.encode_id(cloudCredential.id) )}">Edit</a> - <a class="action-button" confirm="Are you sure you want to delete credentials '${cloudCredential.name}'?" href="${h.url_for( action='delete', id=trans.security.encode_id(cloudCredential.id) )}">Delete</a> + <a class="action-button" href="${h.url_for( action='view_credentials', id=trans.security.encode_id(cloudCredential.id) )}">View</a> + <a class="action-button" href="${h.url_for( action='edit_credentials', id=trans.security.encode_id(cloudCredential.id) )}">Edit</a> + <a class="action-button" confirm="Are you sure you want to delete credentials '${cloudCredential.name}'?" href="${h.url_for( action='delete_credentials', id=trans.security.encode_id(cloudCredential.id) )}">Delete</a> </div> </td> </tr> @@ -196,7 +196,7 @@ <h3>Manage your cloud instances</h3> <ul class="manage-table-actions"> <li> - <a class="action-button" href="${h.url_for( action='configureNew' )}"> + <a class="action-button" href="${h.url_for( action='configure_new_uci' )}"> <img src="${h.url_for('/static/images/silk/add.png')}" /> <span>Configure new instance</span> </a> @@ -253,10 +253,10 @@ <td id="${ liveInstance.id }-link"></td> <td> <div popupmenu="li-${i}-popup"> - <a class="action-button" confirm="Are you sure you want to stop instance '${liveInstance.name}'? Please note that this may take up to 1 minute during which time the page will not refresh." href="${h.url_for( action='stop', id=trans.security.encode_id(liveInstance.id) )}">Stop</a> - <a class="action-button" href="${h.url_for( action='renameInstance', id=trans.security.encode_id(liveInstance.id) )}">Rename</a> - <a class="action-button" href="${h.url_for( action='viewInstance', id=trans.security.encode_id(liveInstance.id) )}">View details</a> - <a class="action-button" href="${h.url_for( action='usageReport', id=trans.security.encode_id(liveInstance.id) )}">Usage report</a> + <a class="action-button" confirm="Are you sure you want to stop instance '${liveInstance.name}'?" href="${h.url_for( action='stop', id=trans.security.encode_id(liveInstance.id) )}">Stop</a> + <a class="action-button" href="${h.url_for( action='rename_uci', id=trans.security.encode_id(liveInstance.id) )}">Rename</a> + <a class="action-button" href="${h.url_for( action='view_uci_details', id=trans.security.encode_id(liveInstance.id) )}">View details</a> + <a class="action-button" href="${h.url_for( action='uci_usage_report', id=trans.security.encode_id(liveInstance.id) )}">Usage report</a> </div> </td> </tr> @@ -322,12 +322,12 @@ <div popupmenu="pi-${i}-popup"> <a class="action-button" href="${h.url_for( action='start', id=trans.security.encode_id(prevInstance.id), type='m1.small' )}"> Start m1.small</a> <a class="action-button" href="${h.url_for( action='start', id=trans.security.encode_id(prevInstance.id), type='c1.medium' )}"> Start c1.medium</a> - <a class="action-button" href="${h.url_for( action='renameInstance', id=trans.security.encode_id(prevInstance.id) )}">Rename</a> - <a class="action-button" href="${h.url_for( action='create_snapshot', id=trans.security.encode_id(prevInstance.id) )}">Create snapshot</a> + <a class="action-button" href="${h.url_for( action='rename_uci', id=trans.security.encode_id(prevInstance.id) )}">Rename</a> + <a class="action-button" href="${h.url_for( action='uci_usage_report', id=trans.security.encode_id(prevInstance.id) )}">Usage report</a> + <a class="action-button" href="${h.url_for( action='create_snapshot', id=trans.security.encode_id(prevInstance.id) )}">Create snapshot</a> <a class="action-button" href="${h.url_for( action='view_snapshots', id=trans.security.encode_id(prevInstance.id) )}">View snapshots</a> - <a class="action-button" href="${h.url_for( action='addStorage', id=trans.security.encode_id(prevInstance.id) )}" target="_parent">Add storage</a> - <a class="action-button" href="${h.url_for( action='usageReport', id=trans.security.encode_id(prevInstance.id) )}">Usage report</a> - <a class="action-button" confirm="Are you sure you want to delete instance '${prevInstance.name}'? This will delete all of your data assocaiated with this instance!" href="${h.url_for( action='deleteInstance', id=trans.security.encode_id(prevInstance.id) )}">Delete</a> + <a class="action-button" href="${h.url_for( action='add_storage', id=trans.security.encode_id(prevInstance.id) )}" target="_parent">Add storage</a> + <a class="action-button" confirm="Are you sure you want to delete instance '${prevInstance.name}'? This will delete all of your data assocaiated with this instance!" href="${h.url_for( action='delete_uci', id=trans.security.encode_id(prevInstance.id) )}">Delete</a> </div> </td> </tr> @@ -341,7 +341,7 @@ %else: You have no credentials associated with your Galaxy account: - <a class="action-button" href="${h.url_for( action='add' )}"> + <a class="action-button" href="${h.url_for( action='add_credentials' )}"> <img src="${h.url_for('/static/images/silk/add.png')}" /> <span>add credentials</span> </a> diff -r e21f605d2766 -r 118dc385752b templates/cloud/configure_uci.mako --- a/templates/cloud/configure_uci.mako Thu Nov 12 12:42:17 2009 -0500 +++ b/templates/cloud/configure_uci.mako Thu Nov 12 14:24:43 2009 -0500 @@ -35,7 +35,7 @@ <div class="form"> <div class="form-title">Configure new Galaxy instance</div> <div class="form-body"> - <form name="Configure new UCI" action="${h.url_for( action='configureNew' )}" method="post" > + <form name="Configure new UCI" action="${h.url_for( action='configure_new_uci' )}" method="post" > <% cls = "form-row" diff -r e21f605d2766 -r 118dc385752b templates/cloud/edit_credentials.mako --- a/templates/cloud/edit_credentials.mako Thu Nov 12 12:42:17 2009 -0500 +++ b/templates/cloud/edit_credentials.mako Thu Nov 12 14:24:43 2009 -0500 @@ -20,7 +20,7 @@ <div class="form"> <div class="form-title">Edit credentials</div> <div class="form-body"> - <form name="edit_credentials" action="${h.url_for( action='edit', id=trans.security.encode_id(credential.id), edited="true" )}" method="post" > + <form name="edit_credentials" action="${h.url_for( action='edit_credentials', id=trans.security.encode_id(credential.id), edited="true" )}" method="post" > <% cls = "form-row" diff -r e21f605d2766 -r 118dc385752b templates/cloud/edit_image.mako --- a/templates/cloud/edit_image.mako Thu Nov 12 12:42:17 2009 -0500 +++ b/templates/cloud/edit_image.mako Thu Nov 12 14:24:43 2009 -0500 @@ -20,7 +20,7 @@ <div class="form"> <div class="form-title">Edit image</div> <div class="form-body"> - <form name="edit_image" action="${h.url_for( action='editImage', id=trans.security.encode_id(image.id), edited="true" )}" method="post" > + <form name="edit_image" action="${h.url_for( action='edit_image', id=trans.security.encode_id(image.id), edited="true" )}" method="post" > <% cls = "form-row" if error.has_key('provider_error'): diff -r e21f605d2766 -r 118dc385752b templates/cloud/list_images.mako --- a/templates/cloud/list_images.mako Thu Nov 12 12:42:17 2009 -0500 +++ b/templates/cloud/list_images.mako Thu Nov 12 14:24:43 2009 -0500 @@ -69,18 +69,18 @@ %endif </td> <td> - <a href="${h.url_for( controller='cloud', action='editImage', image_id=image.image_id, manifest=image.manifest, id=trans.security.encode_id(image.id) )}">e</a> + <a href="${h.url_for( controller='cloud', action='edit_image', image_id=image.image_id, manifest=image.manifest, id=trans.security.encode_id(image.id) )}">e</a> </td> <td> <a confirm="Are you sure you want to delete machine image '${image.image_id}'? Note that this may result in users' UCI's not to work any more!" - href="${h.url_for( controller='cloud', action='deleteImage', id=trans.security.encode_id(image.id) )}">x</a> + href="${h.url_for( controller='cloud', action='delete_image', id=trans.security.encode_id(image.id) )}">x</a> </td> </tr> %endfor </table> %else: <h3>There are no registered machine images.</h3><br /> - <a href="${h.url_for( controller='cloud', action='addNewImage' )}" target="galaxy_main">Add machine image now?</a> + <a href="${h.url_for( controller='cloud', action='add_new_image' )}" target="galaxy_main">Add machine image now?</a> %endif diff -r e21f605d2766 -r 118dc385752b templates/cloud/view.mako --- a/templates/cloud/view.mako Thu Nov 12 12:42:17 2009 -0500 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,157 +0,0 @@ -<%inherit file="/base.mako"/> - -<%def name="title()">Cloud credentials</%def> - -<h2>Credentials details</h2> - -<ul class="manage-table-actions"> - <li> - <a class="action-button" href="${h.url_for( action='list' )}"> - <img src="${h.url_for('/static/images/silk/resultset_previous.png')}" /> - <span>Return to cloud management console</span> - </a> - </li> -</ul> - -%if credDetails: - ${view_cred( credDetails )} -%else: - There are no credentials under that name. -%endif - - - - -<%def name="view_cred( credDetails )"> - <table class="mange-table colored" border="0" cellspacing="0" cellpadding="0" width="100%"> - <tr> - <td> Credentials name: </td> - <td> - ${credDetails.name} - <a id="wf-popup" class="popup-arrow" style="display: none;">▼</a> - </td> - <td> - <div popupmenu="wf-popup"> - <a class="action-button" href="${h.url_for( action='rename', id=trans.security.encode_id(credDetails.id) )}">Rename</a> - <a class="action-button" confirm="Are you sure you want to delete credentials '${credDetails.name}'?" href="${h.url_for( action='delete', id=trans.security.encode_id(credDetails.id) )}">Delete</a> - </div> - </td> - </tr> - <tr> - <td> Last updated: </td> - <td> ${str(credDetails.update_time)[:16]} - <% - context.write( ' UTC (' ) - context.write( str(h.date.distance_of_time_in_words (credDetails.update_time, h.date.datetime.utcnow() ) ) ) - %> ago) - </td> - </tr> - <tr> - <td> Cloud provider type: </td> - <td> ${str(credDetails.provider.type)}</td> - </tr> - <tr> - <td> Cloud provider name: </td> - <td> ${str(credDetails.provider.name)}</td> - </tr> - <tr> - <td> Access key: </td> - <td> - ${credDetails.access_key} - </td> - </tr> - <tr> - <td> Secret key: </td> - <td> - <div id="shortComment2"> - <a onclick="document.getElementById('fullComment2').style.display = 'block'; - document.getElementById('shortComment2').style.display = 'none'; return 0" - href="javascript:void(0)"> - + Show - </a> - </div> - <div id="fullComment2" style="DISPLAY: none"> - <a onclick="document.getElementById('shortComment2').style.display = 'block'; - document.getElementById('fullComment2').style.display = 'none'; return 0;" - href="javascript:void(0)"> - - Hide - </a><br /> - <nobr>${credDetails.secret_key}</nobr><br/> - </div> - </td> - </tr> - <tr><td id="addl"><b>Additional cloud provider information (if available):</b></td></tr> - %if credDetails.provider.region_connection != None: - <tr> - <td> Region connection: </td> - <td> ${credDetails.provider.region_connection} </td> - </tr> - %endif - %if credDetails.provider.region_name != None: - <tr> - <td> Region name: </td> - <td> ${credDetails.provider.region_name} </td> - </tr> - %endif - %if credDetails.provider.region_endpoint != None: - <tr> - <td> Region endpoint: </td> - <td> ${credDetails.provider.region_endpoint} </td> - </tr> - %endif - %if credDetails.provider.is_secure != None: - <tr> - <td> Is secure: </td> - <td> ${credDetails.provider.is_secure} </td> - </tr> - %endif - %if credDetails.provider.host != None: - <tr> - <td> Host: </td> - <td> ${credDetails.provider.host} </td> - </tr> - %endif - %if credDetails.provider.port != None: - <tr> - <td> Port: </td> - <td> ${credDetails.provider.port} </td> - </tr> - %endif - %if credDetails.provider.proxy != None: - <tr> - <td> Proxy: </td> - <td> ${credDetails.provider.proxy} </td> - </tr> - %endif - %if credDetails.provider.proxy_port != None: - <tr> - <td> Proxy port: </td> - <td> ${credDetails.provider.proxy_port} </td> - </tr> - %endif - %if credDetails.provider.proxy_pass != None: - <tr> - <td> Proxy pass: </td> - <td> ${credDetails.provider.proxy_pass} </td> - </tr> - %endif - %if credDetails.provider.debug != None: - <tr> - <td> Debug: </td> - <td> ${credDetails.provider.debug} </td> - </tr> - %endif - %if credDetails.provider.https_connection_factory != None: - <tr> - <td> HTTPS connection factory: </td> - <td> ${credDetails.provider.https_connection_factory} </td> - </tr> - %endif - %if credDetails.provider.path != None: - <tr> - <td> Path: </td> - <td> ${credDetails.provider.path} </td> - </tr> - %endif - </table> -</%def> diff -r e21f605d2766 -r 118dc385752b templates/cloud/viewInstance.mako --- a/templates/cloud/viewInstance.mako Thu Nov 12 12:42:17 2009 -0500 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,140 +0,0 @@ -<%inherit file="/base.mako"/> -<%def name="title()">Live instance details</%def> - -<% - # Because of the one-to-many relationship between liveInstance (i.e., UCI) and actual instances, need to know - # which one is currently active. Because only one instance of UCI can be alive at any point in time, simply - # select the most recent one. - # TODO: Once individual UCI's will be able to start more than one instance, this will need to be fixed - #i_id = len(liveInstance.instance) - 1 -%> - -<h2>Live instance details</h2> - -%if liveInstance: - <ul class="manage-table-actions"> - <li> - <a class="action-button" href="${h.url_for( action='list' )}"> - <img src="${h.url_for('/static/images/silk/resultset_previous.png')}" /> - <span>Return to cloud management console</span> - </a> - </li> - </ul> - - <table class="mange-table colored" border="0" cellspacing="0" cellpadding="0" width="100%"> - <tr> - <td> Instance name: </td> - <td> - ${liveInstance.uci.name} - <a id="li-popup" class="popup-arrow" style="display: none;">▼</a> - </td> - <td> - <div popupmenu="li-popup"> - <a class="action-button" href="${h.url_for( action='renameInstance', id=trans.security.encode_id(liveInstance.uci.id) )}">Rename</a> - <a class="action-button" confirm="Are you sure you want to stop instance '${liveInstance.uci.name}'?" href="${h.url_for( action='stop', id=trans.security.encode_id(liveInstance.uci.id) )}">Stop</a> - </div> - </td> - </tr> - <tr> - <td> Date created: </td> - <td> ${str(liveInstance.uci.create_time)[:16]} - <% - context.write( ' UTC (' ) - context.write( str(h.date.distance_of_time_in_words (liveInstance.uci.create_time, h.date.datetime.utcnow() ) ) ) - %> ago) - </td> - </tr> - <tr> - <td> Alive since: </td> - <td> ${str(liveInstance.launch_time)[:16]} - <% - context.write( ' UTC (' ) - context.write( str(h.date.distance_of_time_in_words (liveInstance.launch_time, h.date.datetime.utcnow() ) ) ) - %> ago) - </td> - </tr> - %if liveInstance.instance_id != None: - <tr> - <td> Instance ID: </td> - <td> ${liveInstance.instance_id} </td> - </tr> - %endif - %if liveInstance.reservation_id != None: - <tr> - <td> Reservation ID: </td> - <td> ${liveInstance.reservation_id} </td> - </tr> - %endif - <tr> - <td> AMI: </td> - <td> ${liveInstance.mi_id} </td> - </tr> - <tr> - <td> State:</td> - <td> ${liveInstance.state} </td> - </tr> - <tr> - <td> Type:</td> - <td> ${liveInstance.type} </td> - </tr> - <tr> - <td> Storage size:</td> - <td> ${liveInstance.uci.total_size} </td> - </tr> - <tr> - <td> Public DNS:</td> - <% - lnk="http://"+str(liveInstance.public_dns) - %> - <td> <a href="${lnk}" target="_blank">${liveInstance.public_dns}</a></td> - </tr> - %if liveInstance.private_dns != None: - <tr> - <td> Private DNS:</td> - <td> ${liveInstance.private_dns}</td> - </tr> - %endif - %if liveInstance.security_group != None: - <tr> - <td> Security group zone:</td> - <td> ${liveInstance.security_group} </td> - </tr> - %endif - %if liveInstance.availability_zone != None: - <tr> - <td> Availabilty zone:</td> - <td> ${liveInstance.availability_zone} </td> - </tr> - %endif - %if liveInstance.uci.key_pair_name != None: - <tr> - <td> Keypair file name:</td> - <td> ${liveInstance.uci.key_pair_name} </td> - </tr> - %endif - %if liveInstance.uci.key_pair_material != None: - <tr> - <td> Keypair material:</td> - <td> - <div id="short"> - <a onclick="document.getElementById('full').style.display = 'block'; - document.getElementById('short').style.display = 'none'; return 0" - href="javascript:void(0)"> - + Show - </a> - </div> - <div id="full" style="DISPLAY: none"> - <a onclick="document.getElementById('short').style.display = 'block'; - document.getElementById('full').style.display = 'none'; return 0;" - href="javascript:void(0)"> - - Hide</a> - ${liveInstance.uci.key_pair_material}<br/> - </div> - </td> - </tr> - %endif - - </table> -%else: - There is no live instance under that name. -%endif diff -r e21f605d2766 -r 118dc385752b templates/cloud/view_credentials.mako --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/templates/cloud/view_credentials.mako Thu Nov 12 14:24:43 2009 -0500 @@ -0,0 +1,157 @@ +<%inherit file="/base.mako"/> + +<%def name="title()">Cloud credentials</%def> + +<h2>Credentials details</h2> + +<ul class="manage-table-actions"> + <li> + <a class="action-button" href="${h.url_for( action='list' )}"> + <img src="${h.url_for('/static/images/silk/resultset_previous.png')}" /> + <span>Return to cloud management console</span> + </a> + </li> +</ul> + +%if credDetails: + ${view_cred( credDetails )} +%else: + There are no credentials under that name. +%endif + + + + +<%def name="view_cred( credDetails )"> + <table class="mange-table colored" border="0" cellspacing="0" cellpadding="0" width="100%"> + <tr> + <td> Credentials name: </td> + <td> + ${credDetails.name} + <a id="wf-popup" class="popup-arrow" style="display: none;">▼</a> + </td> + <td> + <div popupmenu="wf-popup"> + <a class="action-button" href="${h.url_for( action='edit_credentials', id=trans.security.encode_id(credDetails.id) )}">Edit</a> + <a class="action-button" confirm="Are you sure you want to delete credentials '${credDetails.name}'?" href="${h.url_for( action='delete_credentials', id=trans.security.encode_id(credDetails.id) )}">Delete</a> + </div> + </td> + </tr> + <tr> + <td> Last updated: </td> + <td> ${str(credDetails.update_time)[:16]} + <% + context.write( ' UTC (' ) + context.write( str(h.date.distance_of_time_in_words (credDetails.update_time, h.date.datetime.utcnow() ) ) ) + %> ago) + </td> + </tr> + <tr> + <td> Cloud provider type: </td> + <td> ${str(credDetails.provider.type)}</td> + </tr> + <tr> + <td> Cloud provider name: </td> + <td> ${str(credDetails.provider.name)}</td> + </tr> + <tr> + <td> Access key: </td> + <td> + ${credDetails.access_key} + </td> + </tr> + <tr> + <td> Secret key: </td> + <td> + <div id="shortComment2"> + <a onclick="document.getElementById('fullComment2').style.display = 'block'; + document.getElementById('shortComment2').style.display = 'none'; return 0" + href="javascript:void(0)"> + + Show + </a> + </div> + <div id="fullComment2" style="DISPLAY: none"> + <a onclick="document.getElementById('shortComment2').style.display = 'block'; + document.getElementById('fullComment2').style.display = 'none'; return 0;" + href="javascript:void(0)"> + - Hide + </a><br /> + <nobr>${credDetails.secret_key}</nobr><br/> + </div> + </td> + </tr> + <tr><td id="addl"><b>Additional cloud provider information (if available):</b></td></tr> + %if credDetails.provider.region_connection != None: + <tr> + <td> Region connection: </td> + <td> ${credDetails.provider.region_connection} </td> + </tr> + %endif + %if credDetails.provider.region_name != None: + <tr> + <td> Region name: </td> + <td> ${credDetails.provider.region_name} </td> + </tr> + %endif + %if credDetails.provider.region_endpoint != None: + <tr> + <td> Region endpoint: </td> + <td> ${credDetails.provider.region_endpoint} </td> + </tr> + %endif + %if credDetails.provider.is_secure != None: + <tr> + <td> Is secure: </td> + <td> ${credDetails.provider.is_secure} </td> + </tr> + %endif + %if credDetails.provider.host != None: + <tr> + <td> Host: </td> + <td> ${credDetails.provider.host} </td> + </tr> + %endif + %if credDetails.provider.port != None: + <tr> + <td> Port: </td> + <td> ${credDetails.provider.port} </td> + </tr> + %endif + %if credDetails.provider.proxy != None: + <tr> + <td> Proxy: </td> + <td> ${credDetails.provider.proxy} </td> + </tr> + %endif + %if credDetails.provider.proxy_port != None: + <tr> + <td> Proxy port: </td> + <td> ${credDetails.provider.proxy_port} </td> + </tr> + %endif + %if credDetails.provider.proxy_pass != None: + <tr> + <td> Proxy pass: </td> + <td> ${credDetails.provider.proxy_pass} </td> + </tr> + %endif + %if credDetails.provider.debug != None: + <tr> + <td> Debug: </td> + <td> ${credDetails.provider.debug} </td> + </tr> + %endif + %if credDetails.provider.https_connection_factory != None: + <tr> + <td> HTTPS connection factory: </td> + <td> ${credDetails.provider.https_connection_factory} </td> + </tr> + %endif + %if credDetails.provider.path != None: + <tr> + <td> Path: </td> + <td> ${credDetails.provider.path} </td> + </tr> + %endif + </table> +</%def> diff -r e21f605d2766 -r 118dc385752b templates/cloud/view_instance.mako --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/templates/cloud/view_instance.mako Thu Nov 12 14:24:43 2009 -0500 @@ -0,0 +1,140 @@ +<%inherit file="/base.mako"/> +<%def name="title()">Live instance details</%def> + +<% + # Because of the one-to-many relationship between liveInstance (i.e., UCI) and actual instances, need to know + # which one is currently active. Because only one instance of UCI can be alive at any point in time, simply + # select the most recent one. + # TODO: Once individual UCI's will be able to start more than one instance, this will need to be fixed + #i_id = len(liveInstance.instance) - 1 +%> + +<h2>Live instance details</h2> + +%if liveInstance: + <ul class="manage-table-actions"> + <li> + <a class="action-button" href="${h.url_for( action='list' )}"> + <img src="${h.url_for('/static/images/silk/resultset_previous.png')}" /> + <span>Return to cloud management console</span> + </a> + </li> + </ul> + + <table class="mange-table colored" border="0" cellspacing="0" cellpadding="0" width="100%"> + <tr> + <td> Instance name: </td> + <td> + ${liveInstance.uci.name} + <a id="li-popup" class="popup-arrow" style="display: none;">▼</a> + </td> + <td> + <div popupmenu="li-popup"> + <a class="action-button" href="${h.url_for( action='rename_uci', id=trans.security.encode_id(liveInstance.uci.id) )}">Rename</a> + <a class="action-button" confirm="Are you sure you want to stop instance '${liveInstance.uci.name}'?" href="${h.url_for( action='stop', id=trans.security.encode_id(liveInstance.uci.id) )}">Stop</a> + </div> + </td> + </tr> + <tr> + <td> Date created: </td> + <td> ${str(liveInstance.uci.create_time)[:16]} + <% + context.write( ' UTC (' ) + context.write( str(h.date.distance_of_time_in_words (liveInstance.uci.create_time, h.date.datetime.utcnow() ) ) ) + %> ago) + </td> + </tr> + <tr> + <td> Alive since: </td> + <td> ${str(liveInstance.launch_time)[:16]} + <% + context.write( ' UTC (' ) + context.write( str(h.date.distance_of_time_in_words (liveInstance.launch_time, h.date.datetime.utcnow() ) ) ) + %> ago) + </td> + </tr> + %if liveInstance.instance_id != None: + <tr> + <td> Instance ID: </td> + <td> ${liveInstance.instance_id} </td> + </tr> + %endif + %if liveInstance.reservation_id != None: + <tr> + <td> Reservation ID: </td> + <td> ${liveInstance.reservation_id} </td> + </tr> + %endif + <tr> + <td> AMI: </td> + <td> ${liveInstance.mi_id} </td> + </tr> + <tr> + <td> State:</td> + <td> ${liveInstance.state} </td> + </tr> + <tr> + <td> Type:</td> + <td> ${liveInstance.type} </td> + </tr> + <tr> + <td> Storage size:</td> + <td> ${liveInstance.uci.total_size} </td> + </tr> + <tr> + <td> Public DNS:</td> + <% + lnk="http://"+str(liveInstance.public_dns) + %> + <td> <a href="${lnk}" target="_blank">${liveInstance.public_dns}</a></td> + </tr> + %if liveInstance.private_dns != None: + <tr> + <td> Private DNS:</td> + <td> ${liveInstance.private_dns}</td> + </tr> + %endif + %if liveInstance.security_group != None: + <tr> + <td> Security group zone:</td> + <td> ${liveInstance.security_group} </td> + </tr> + %endif + %if liveInstance.availability_zone != None: + <tr> + <td> Availabilty zone:</td> + <td> ${liveInstance.availability_zone} </td> + </tr> + %endif + %if liveInstance.uci.key_pair_name != None: + <tr> + <td> Keypair file name:</td> + <td> ${liveInstance.uci.key_pair_name} </td> + </tr> + %endif + %if liveInstance.uci.key_pair_material != None: + <tr> + <td> Keypair material:</td> + <td> + <div id="short"> + <a onclick="document.getElementById('full').style.display = 'block'; + document.getElementById('short').style.display = 'none'; return 0" + href="javascript:void(0)"> + + Show + </a> + </div> + <div id="full" style="DISPLAY: none"> + <a onclick="document.getElementById('short').style.display = 'block'; + document.getElementById('full').style.display = 'none'; return 0;" + href="javascript:void(0)"> + - Hide</a> + ${liveInstance.uci.key_pair_material}<br/> + </div> + </td> + </tr> + %endif + + </table> +%else: + There is no live instance under that name. +%endif diff -r e21f605d2766 -r 118dc385752b templates/cloud/view_usage.mako --- a/templates/cloud/view_usage.mako Thu Nov 12 12:42:17 2009 -0500 +++ b/templates/cloud/view_usage.mako Thu Nov 12 14:24:43 2009 -0500 @@ -88,7 +88,7 @@ </table> <br/>Total number of hours instance was alive: ${total_hours} <br /> Note that these are just best effort estimates - true usage should be obtained from respective cloud provider. <br /> - <%namespace name="view_cred" file="view.mako" /> + <%namespace name="view_cred" file="view_credentials.mako" /> <div id="hide_cred_details"> This instance uses credentials:

1 0