- galaxy-dev - lists.galaxyproject.org

[hg] galaxy 3133: Merge
by Greg Von Kuster 02 Dec '09

02 Dec '09

details: http://www.bx.psu.edu/hg/galaxy/rev/e75b55cc9e08 changeset: 3133:e75b55cc9e08 user: Enis Afgan <afgane(a)gmail.com> date: Tue Dec 01 09:53:36 2009 -0500 description: Merge diffstat: lib/galaxy/config.py | 1 + lib/galaxy/model/__init__.py | 10 +++++ lib/galaxy/model/mapping.py | 13 ++++++ lib/galaxy/model/migrate/versions/0029_user_actions.py | 47 +++++++++++++++++++++++ lib/galaxy/web/controllers/dataset.py | 2 +- lib/galaxy/web/controllers/history.py | 2 +- lib/galaxy/web/controllers/tag.py | 24 ++++++++--- lib/galaxy/web/framework/__init__.py | 17 ++++++++ lib/galaxy/web/framework/helpers/grids.py | 13 +++++- templates/dataset/edit_attributes.mako | 2 +- templates/history/view.mako | 2 +- templates/root/history.mako | 2 +- templates/tagging_common.mako | 8 ++-- universe_wsgi.ini.sample | 3 + 14 files changed, 128 insertions(+), 18 deletions(-) diffs (335 lines): diff -r efed28fa6a5e -r e75b55cc9e08 lib/galaxy/config.py --- a/lib/galaxy/config.py Mon Nov 30 16:47:18 2009 -0500 +++ b/lib/galaxy/config.py Tue Dec 01 09:53:36 2009 -0500 @@ -79,6 +79,7 @@ self.use_heartbeat = string_as_bool( kwargs.get( 'use_heartbeat', 'False' ) ) self.use_memdump = string_as_bool( kwargs.get( 'use_memdump', 'False' ) ) self.log_memory_usage = string_as_bool( kwargs.get( 'log_memory_usage', 'False' ) ) + self.log_actions = string_as_bool( kwargs.get( 'log_actions', 'False' ) ) self.log_events = string_as_bool( kwargs.get( 'log_events', 'False' ) ) self.ucsc_display_sites = kwargs.get( 'ucsc_display_sites', "main,test,archaea,ucla" ).lower().split(",") self.gbrowse_display_sites = kwargs.get( 'gbrowse_display_sites', "main,test,tair" ).lower().split(",") diff -r efed28fa6a5e -r e75b55cc9e08 lib/galaxy/model/__init__.py --- a/lib/galaxy/model/__init__.py Mon Nov 30 16:47:18 2009 -0500 +++ b/lib/galaxy/model/__init__.py Tue Dec 01 09:53:36 2009 -0500 @@ -1441,6 +1441,16 @@ def __init__( self, name=None, value=None ): self.name = name self.value = value + +class UserAction( object ): + def __init__( self, id=None, create_time=None, user_id=None, session_id=None, action=None, params=None, context=None): + self.id = id + self.create_time = create_time + self.user_id = user_id + self.session_id = session_id + self.action = action + self.params = params + self.context = context ## ---- Utility methods ------------------------------------------------------- diff -r efed28fa6a5e -r e75b55cc9e08 lib/galaxy/model/mapping.py --- a/lib/galaxy/model/mapping.py Mon Nov 30 16:47:18 2009 -0500 +++ b/lib/galaxy/model/mapping.py Tue Dec 01 09:53:36 2009 -0500 @@ -743,6 +743,15 @@ Column( "name", Unicode( 255 ), index=True), Column( "value", Unicode( 1024 ) ) ) +UserAction.table = Table( "user_action", metadata, + Column( "id", Integer, primary_key=True ), + Column( "create_time", DateTime, default=now ), + Column( "user_id", Integer, ForeignKey( "galaxy_user.id" ), index=True ), + Column( "session_id", Integer, ForeignKey( "galaxy_session.id" ), index=True ), + Column( "action", Unicode( 255 ) ), + Column( "context", Unicode( 512 ) ), + Column( "params", Unicode( 1024 ) ) ) + # With the tables defined we can define the mappers and setup the # relationships between the model objects. @@ -1237,6 +1246,10 @@ assign_mapper( context, UserPreference, UserPreference.table, properties = {} ) + +assign_mapper( context, UserAction, UserAction.table, + properties = dict( user=relation( User.mapper ) ) + ) def db_next_hid( self ): """ diff -r efed28fa6a5e -r e75b55cc9e08 lib/galaxy/model/migrate/versions/0029_user_actions.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/lib/galaxy/model/migrate/versions/0029_user_actions.py Tue Dec 01 09:53:36 2009 -0500 @@ -0,0 +1,47 @@ +""" +This migration script adds a user actions table to Galaxy. +""" + +from sqlalchemy import * +from migrate import * + +import datetime +now = datetime.datetime.utcnow + +import logging +log = logging.getLogger( __name__ ) + +metadata = MetaData( migrate_engine ) + +def display_migration_details(): + print "" + print "This migration script adds a user actions table to Galaxy." + print "" + + +# New table to store user actions. +UserAction_table = Table( "user_action", metadata, + Column( "id", Integer, primary_key=True ), + Column( "create_time", DateTime, default=now ), + Column( "user_id", Integer, ForeignKey( "galaxy_user.id" ), index=True ), + Column( "session_id", Integer, ForeignKey( "galaxy_session.id" ), index=True ), + Column( "action", Unicode( 255 ) ), + Column( "context", Unicode( 512 ) ), + Column( "params", Unicode( 1024 ) ) ) + +def upgrade(): + display_migration_details() + metadata.reflect() + try: + UserAction_table.create() + except Exception, e: + print str(e) + log.debug( "Creating user_action table failed: %s" % str( e ) ) + +def downgrade(): + metadata.reflect() + try: + UserAction_table.drop() + except Exception, e: + print str(e) + log.debug( "Dropping user_action table failed: %s" % str( e ) ) \ No newline at end of file diff -r efed28fa6a5e -r e75b55cc9e08 lib/galaxy/web/controllers/dataset.py --- a/lib/galaxy/web/controllers/dataset.py Mon Nov 30 16:47:18 2009 -0500 +++ b/lib/galaxy/web/controllers/dataset.py Tue Dec 01 09:53:36 2009 -0500 @@ -87,7 +87,7 @@ link=( lambda item: iff( item.history.deleted, None, dict( operation="switch", id=item.id ) ) ), filterable="advanced" ), HistoryColumn( "History", key="history", link=( lambda item: iff( item.history.deleted, None, dict( operation="switch_history", id=item.id ) ) ) ), - grids.TagsColumn( "Tags", "tags", model.HistoryDatasetAssociation, model.HistoryDatasetAssociationTagAssociation, filterable="advanced" ), + grids.TagsColumn( "Tags", "tags", model.HistoryDatasetAssociation, model.HistoryDatasetAssociationTagAssociation, filterable="advanced", grid_name="HistoryDatasetAssocationListGrid" ), StatusColumn( "Status", key="deleted", attach_popup=False ), grids.GridColumn( "Created", key="create_time", format=time_ago ), grids.GridColumn( "Last Updated", key="update_time", format=time_ago ), diff -r efed28fa6a5e -r e75b55cc9e08 lib/galaxy/web/controllers/history.py --- a/lib/galaxy/web/controllers/history.py Mon Nov 30 16:47:18 2009 -0500 +++ b/lib/galaxy/web/controllers/history.py Tue Dec 01 09:53:36 2009 -0500 @@ -94,7 +94,7 @@ link=( lambda history: iff( history.deleted, None, dict( operation="Switch", id=history.id ) ) ), attach_popup=True, filterable="advanced" ), DatasetsByStateColumn( "Datasets (by state)", ncells=4 ), - grids.TagsColumn( "Tags", "tags", model.History, model.HistoryTagAssociation, filterable="advanced"), + grids.TagsColumn( "Tags", "tags", model.History, model.HistoryTagAssociation, filterable="advanced", grid_name="HistoryListGrid" ), StatusColumn( "Status", attach_popup=False ), grids.GridColumn( "Created", key="create_time", format=time_ago ), grids.GridColumn( "Last Updated", key="update_time", format=time_ago ), diff -r efed28fa6a5e -r e75b55cc9e08 lib/galaxy/web/controllers/tag.py --- a/lib/galaxy/web/controllers/tag.py Mon Nov 30 16:47:18 2009 -0500 +++ b/lib/galaxy/web/controllers/tag.py Tue Dec 01 09:53:36 2009 -0500 @@ -28,28 +28,38 @@ @web.expose @web.require_login( "Add tag to an item." ) - def add_tag_async( self, trans, id=None, item_class=None, new_tag=None ): + def add_tag_async( self, trans, id=None, item_class=None, new_tag=None, context=None ): """ Add tag to an item. """ - item = self._get_item(trans, item_class, trans.security.decode_id(id)) - self._do_security_check(trans, item) + # Check that user owns item. + item = self._get_item(trans, item_class, trans.security.decode_id( id ) ) + self._do_security_check( trans, item ) + # Apply tag. self.tag_handler.apply_item_tags( trans.sa_session, item, new_tag.encode('utf-8') ) trans.sa_session.flush() + # Log. + params = dict( item_id=item.id, item_class=item_class, tag=new_tag) + trans.log_action( unicode( "tag"), context, params ) + @web.expose @web.require_login( "Remove tag from an item." ) - def remove_tag_async( self, trans, id=None, item_class=None, tag_name=None ): + def remove_tag_async( self, trans, id=None, item_class=None, tag_name=None, context=None ): """ Remove tag from an item. """ + + # Check that user owns item. item = self._get_item(trans, item_class, trans.security.decode_id(id)) - self._do_security_check(trans, item) + # Remove tag. self.tag_handler.remove_item_tag( trans, item, tag_name.encode('utf-8') ) - #print tag_name - #print unicode(tag_name) trans.sa_session.flush() + # Log. + params = dict( item_id=item.id, item_class=item_class, tag=tag_name) + trans.log_action( unicode( "untag"), context, params ) + # Retag an item. All previous tags are deleted and new tags are applied. @web.expose @web.require_login( "Apply a new set of tags to an item; previous tags are deleted." ) diff -r efed28fa6a5e -r e75b55cc9e08 lib/galaxy/web/framework/__init__.py --- a/lib/galaxy/web/framework/__init__.py Mon Nov 30 16:47:18 2009 -0500 +++ b/lib/galaxy/web/framework/__init__.py Tue Dec 01 09:53:36 2009 -0500 @@ -10,6 +10,7 @@ import base import pickle from galaxy import util +from galaxy.util.json import to_json_string pkg_resources.require( "simplejson" ) import simplejson @@ -170,6 +171,22 @@ to allow migration toward a more SQLAlchemy 0.4 style of use. """ return self.app.model.context.current + def log_action( self, action, context, params): + """ + Application-level logging of user actions. + """ + if self.app.config.log_actions: + action = self.app.model.UserAction(action=action, context=context, params=unicode( to_json_string( params ) ) ) + try: + action.user = self.user + except: + action.user = None + try: + action.session_id = self.galaxy_session.id + except: + action.session_id = None + self.sa_session.add( action ) + self.sa_session.flush() def log_event( self, message, tool_id=None, **kwargs ): """ Application level logging. Still needs fleshing out (log levels and such) diff -r efed28fa6a5e -r e75b55cc9e08 lib/galaxy/web/framework/helpers/grids.py --- a/lib/galaxy/web/framework/helpers/grids.py Mon Nov 30 16:47:18 2009 -0500 +++ b/lib/galaxy/web/framework/helpers/grids.py Tue Dec 01 09:53:36 2009 -0500 @@ -200,6 +200,13 @@ trans.get_user().preferences[pref_name] = unicode( to_json_string( sort_key ) ) trans.sa_session.flush() + # Log grid view. + context = unicode( self.__class__.__name__ ) + params = cur_filter_dict.copy() + params['sort'] = sort_key + params['async'] = ( 'async' in kwargs ) + trans.log_action( unicode( "grid.view"), context, params ) + # Render grid. def url( *args, **kwargs ): # Only include sort/filter arguments if not linking to another @@ -340,17 +347,19 @@ # Generic column that supports tagging. class TagsColumn( TextColumn ): - def __init__( self, col_name, key, model_class, model_tag_association_class, filterable ): + def __init__( self, col_name, key, model_class, model_tag_association_class, filterable, grid_name=None ): GridColumn.__init__(self, col_name, key=key, model_class=model_class, filterable=filterable) self.model_tag_association_class = model_tag_association_class # Tags cannot be sorted. self.sortable = False + # Column-specific attributes. self.tag_elt_id_gen = 0 + self.grid_name = grid_name def get_value( self, trans, grid, item ): self.tag_elt_id_gen += 1 elt_id="tagging-elt" + str( self.tag_elt_id_gen ) div_elt = "<div id=%s></div>" % elt_id - return div_elt + trans.fill_template( "/tagging_common.mako", trans=trans, tagged_item=item, + return div_elt + trans.fill_template( "/tagging_common.mako", trans=trans, tagged_item=item, elt_context=self.grid_name, elt_id = elt_id, in_form="true", input_size="20", tag_click_fn="add_tag_to_grid_filter" ) def filter( self, db_session, query, column_filter ): """ Modify query to filter model_class by tag. Multiple filters are ANDed. """ diff -r efed28fa6a5e -r e75b55cc9e08 templates/dataset/edit_attributes.mako --- a/templates/dataset/edit_attributes.mako Mon Nov 30 16:47:18 2009 -0500 +++ b/templates/dataset/edit_attributes.mako Tue Dec 01 09:53:36 2009 -0500 @@ -59,7 +59,7 @@ </div> <div style="clear: both"></div> </div> - ${render_tagging_element(data, "dataset-tag-area", use_toggle_link="false", in_form="true", input_size="30")} + ${render_tagging_element(data, "dataset-tag-area", "edit_attributes.mako", use_toggle_link="false", in_form="true", input_size="30")} %endif %for name, spec in data.metadata.spec.items(): %if spec.visible: diff -r efed28fa6a5e -r e75b55cc9e08 templates/history/view.mako --- a/templates/history/view.mako Mon Nov 30 16:47:18 2009 -0500 +++ b/templates/history/view.mako Tue Dec 01 09:53:36 2009 -0500 @@ -327,7 +327,7 @@ %if trans.get_user() is not None: <div id='history-tag-area' class="tag-element"></div> - ${render_tagging_element(history, "history-tag-area", use_toggle_link='false', get_toggle_link_text_fn='get_toggle_link_text', editable=user_owns_history)} + ${render_tagging_element(history, "history-tag-area", "history/view.mako", use_toggle_link='false', get_toggle_link_text_fn='get_toggle_link_text', editable=user_owns_history)} %endif %if not datasets: diff -r efed28fa6a5e -r e75b55cc9e08 templates/root/history.mako --- a/templates/root/history.mako Mon Nov 30 16:47:18 2009 -0500 +++ b/templates/root/history.mako Tue Dec 01 09:53:36 2009 -0500 @@ -325,7 +325,7 @@ %if trans.get_user() is not None: <div id='history-tag-area' class="tag-element"></div> - ${render_tagging_element(history, "history-tag-area", get_toggle_link_text_fn='get_toggle_link_text')} + ${render_tagging_element( tagged_item=history, elt_id="history-tag-area", elt_context="history.mako", get_toggle_link_text_fn='get_toggle_link_text' )} %endif %if not datasets: diff -r efed28fa6a5e -r e75b55cc9e08 templates/tagging_common.mako --- a/templates/tagging_common.mako Mon Nov 30 16:47:18 2009 -0500 +++ b/templates/tagging_common.mako Tue Dec 01 09:53:36 2009 -0500 @@ -1,10 +1,10 @@ ## Render a tagging element if there is a tagged_item. %if tagged_item is not None and elt_id is not None: - ${render_tagging_element(tagged_item, elt_id=elt_id, in_form=in_form, input_size=input_size, tag_click_fn=tag_click_fn)} + ${render_tagging_element(tagged_item=tagged_item, elt_id=elt_id, elt_context=elt_context, in_form=in_form, input_size=input_size, tag_click_fn=tag_click_fn)} %endif ## Render the tags 'tags' as an autocomplete element. -<%def name="render_tagging_element(tagged_item, elt_id, use_toggle_link='true', in_form='false', input_size='15', tag_click_fn='default_tag_click_fn', get_toggle_link_text_fn='default_get_toggle_link_text_fn', editable='true')"> +<%def name="render_tagging_element(tagged_item, elt_id, elt_context, use_toggle_link='true', in_form='false', input_size='15', tag_click_fn='default_tag_click_fn', get_toggle_link_text_fn='default_get_toggle_link_text_fn', editable='true')"> <script type="text/javascript"> // // Set up autocomplete tagger. @@ -75,8 +75,8 @@ tag_click_fn: ${tag_click_fn}, <% tagged_item_id = trans.security.encode_id(tagged_item.id) %> ajax_autocomplete_tag_url: "${h.url_for( controller='tag', action='tag_autocomplete_data', id=tagged_item_id, item_class=tagged_item.__class__.__name__ )}", - ajax_add_tag_url: "${h.url_for( controller='tag', action='add_tag_async', id=tagged_item_id, item_class=tagged_item.__class__.__name__ )}", - ajax_delete_tag_url: "${h.url_for( controller='tag', action='remove_tag_async', id=tagged_item_id, item_class=tagged_item.__class__.__name__ )}", + ajax_add_tag_url: "${h.url_for( controller='tag', action='add_tag_async', id=tagged_item_id, item_class=tagged_item.__class__.__name__, context=elt_context )}", + ajax_delete_tag_url: "${h.url_for( controller='tag', action='remove_tag_async', id=tagged_item_id, item_class=tagged_item.__class__.__name__, context=elt_context )}", delete_tag_img: "${h.url_for('/static/images/delete_tag_icon_gray.png')}", delete_tag_img_rollover: "${h.url_for('/static/images/delete_tag_icon_white.png')}", add_tag_img: "${h.url_for('/static/images/add_icon.png')}", diff -r efed28fa6a5e -r e75b55cc9e08 universe_wsgi.ini.sample --- a/universe_wsgi.ini.sample Mon Nov 30 16:47:18 2009 -0500 +++ b/universe_wsgi.ini.sample Tue Dec 01 09:53:36 2009 -0500 @@ -144,6 +144,9 @@ # Log events log_events = True +# Log user actions +log_actions = True + # Configuration for debugging middleware debug = True use_lint = False

1 0

[hg] galaxy 3130: Framework for recording user actions for repor...
by Greg Von Kuster 02 Dec '09

02 Dec '09

details: http://www.bx.psu.edu/hg/galaxy/rev/f98643c26eb7 changeset: 3130:f98643c26eb7 user: jeremy goecks <jeremy.goecks(a)emory.edu> date: Mon Nov 30 17:53:25 2009 -0500 description: Framework for recording user actions for reports; framework is modelled after event recording. Currently two types of actions are recorded: (a) tagging/untagging and (b) filtering/searching in grids. diffstat: lib/galaxy/config.py | 1 + lib/galaxy/model/__init__.py | 10 +++++ lib/galaxy/model/mapping.py | 13 ++++++ lib/galaxy/model/migrate/versions/0029_user_actions.py | 47 +++++++++++++++++++++++ lib/galaxy/web/controllers/dataset.py | 2 +- lib/galaxy/web/controllers/history.py | 2 +- lib/galaxy/web/controllers/tag.py | 24 ++++++++--- lib/galaxy/web/framework/__init__.py | 17 ++++++++ lib/galaxy/web/framework/helpers/grids.py | 13 +++++- templates/dataset/edit_attributes.mako | 2 +- templates/history/view.mako | 2 +- templates/root/history.mako | 2 +- templates/tagging_common.mako | 8 ++-- universe_wsgi.ini.sample | 3 + 14 files changed, 128 insertions(+), 18 deletions(-) diffs (335 lines): diff -r 9d4945bbdcf5 -r f98643c26eb7 lib/galaxy/config.py --- a/lib/galaxy/config.py Mon Nov 30 15:33:45 2009 -0500 +++ b/lib/galaxy/config.py Mon Nov 30 17:53:25 2009 -0500 @@ -79,6 +79,7 @@ self.use_heartbeat = string_as_bool( kwargs.get( 'use_heartbeat', 'False' ) ) self.use_memdump = string_as_bool( kwargs.get( 'use_memdump', 'False' ) ) self.log_memory_usage = string_as_bool( kwargs.get( 'log_memory_usage', 'False' ) ) + self.log_actions = string_as_bool( kwargs.get( 'log_actions', 'False' ) ) self.log_events = string_as_bool( kwargs.get( 'log_events', 'False' ) ) self.ucsc_display_sites = kwargs.get( 'ucsc_display_sites', "main,test,archaea,ucla" ).lower().split(",") self.gbrowse_display_sites = kwargs.get( 'gbrowse_display_sites', "main,test,tair" ).lower().split(",") diff -r 9d4945bbdcf5 -r f98643c26eb7 lib/galaxy/model/__init__.py --- a/lib/galaxy/model/__init__.py Mon Nov 30 15:33:45 2009 -0500 +++ b/lib/galaxy/model/__init__.py Mon Nov 30 17:53:25 2009 -0500 @@ -1441,6 +1441,16 @@ def __init__( self, name=None, value=None ): self.name = name self.value = value + +class UserAction( object ): + def __init__( self, id=None, create_time=None, user_id=None, session_id=None, action=None, params=None, context=None): + self.id = id + self.create_time = create_time + self.user_id = user_id + self.session_id = session_id + self.action = action + self.params = params + self.context = context ## ---- Utility methods ------------------------------------------------------- diff -r 9d4945bbdcf5 -r f98643c26eb7 lib/galaxy/model/mapping.py --- a/lib/galaxy/model/mapping.py Mon Nov 30 15:33:45 2009 -0500 +++ b/lib/galaxy/model/mapping.py Mon Nov 30 17:53:25 2009 -0500 @@ -743,6 +743,15 @@ Column( "name", Unicode( 255 ), index=True), Column( "value", Unicode( 1024 ) ) ) +UserAction.table = Table( "user_action", metadata, + Column( "id", Integer, primary_key=True ), + Column( "create_time", DateTime, default=now ), + Column( "user_id", Integer, ForeignKey( "galaxy_user.id" ), index=True ), + Column( "session_id", Integer, ForeignKey( "galaxy_session.id" ), index=True ), + Column( "action", Unicode( 255 ) ), + Column( "context", Unicode( 512 ) ), + Column( "params", Unicode( 1024 ) ) ) + # With the tables defined we can define the mappers and setup the # relationships between the model objects. @@ -1237,6 +1246,10 @@ assign_mapper( context, UserPreference, UserPreference.table, properties = {} ) + +assign_mapper( context, UserAction, UserAction.table, + properties = dict( user=relation( User.mapper ) ) + ) def db_next_hid( self ): """ diff -r 9d4945bbdcf5 -r f98643c26eb7 lib/galaxy/model/migrate/versions/0029_user_actions.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/lib/galaxy/model/migrate/versions/0029_user_actions.py Mon Nov 30 17:53:25 2009 -0500 @@ -0,0 +1,47 @@ +""" +This migration script adds a user actions table to Galaxy. +""" + +from sqlalchemy import * +from migrate import * + +import datetime +now = datetime.datetime.utcnow + +import logging +log = logging.getLogger( __name__ ) + +metadata = MetaData( migrate_engine ) + +def display_migration_details(): + print "" + print "This migration script adds a user actions table to Galaxy." + print "" + + +# New table to store user actions. +UserAction_table = Table( "user_action", metadata, + Column( "id", Integer, primary_key=True ), + Column( "create_time", DateTime, default=now ), + Column( "user_id", Integer, ForeignKey( "galaxy_user.id" ), index=True ), + Column( "session_id", Integer, ForeignKey( "galaxy_session.id" ), index=True ), + Column( "action", Unicode( 255 ) ), + Column( "context", Unicode( 512 ) ), + Column( "params", Unicode( 1024 ) ) ) + +def upgrade(): + display_migration_details() + metadata.reflect() + try: + UserAction_table.create() + except Exception, e: + print str(e) + log.debug( "Creating user_action table failed: %s" % str( e ) ) + +def downgrade(): + metadata.reflect() + try: + UserAction_table.drop() + except Exception, e: + print str(e) + log.debug( "Dropping user_action table failed: %s" % str( e ) ) \ No newline at end of file diff -r 9d4945bbdcf5 -r f98643c26eb7 lib/galaxy/web/controllers/dataset.py --- a/lib/galaxy/web/controllers/dataset.py Mon Nov 30 15:33:45 2009 -0500 +++ b/lib/galaxy/web/controllers/dataset.py Mon Nov 30 17:53:25 2009 -0500 @@ -87,7 +87,7 @@ link=( lambda item: iff( item.history.deleted, None, dict( operation="switch", id=item.id ) ) ), filterable="advanced" ), HistoryColumn( "History", key="history", link=( lambda item: iff( item.history.deleted, None, dict( operation="switch_history", id=item.id ) ) ) ), - grids.TagsColumn( "Tags", "tags", model.HistoryDatasetAssociation, model.HistoryDatasetAssociationTagAssociation, filterable="advanced" ), + grids.TagsColumn( "Tags", "tags", model.HistoryDatasetAssociation, model.HistoryDatasetAssociationTagAssociation, filterable="advanced", grid_name="HistoryDatasetAssocationListGrid" ), StatusColumn( "Status", key="deleted", attach_popup=False ), grids.GridColumn( "Created", key="create_time", format=time_ago ), grids.GridColumn( "Last Updated", key="update_time", format=time_ago ), diff -r 9d4945bbdcf5 -r f98643c26eb7 lib/galaxy/web/controllers/history.py --- a/lib/galaxy/web/controllers/history.py Mon Nov 30 15:33:45 2009 -0500 +++ b/lib/galaxy/web/controllers/history.py Mon Nov 30 17:53:25 2009 -0500 @@ -94,7 +94,7 @@ link=( lambda history: iff( history.deleted, None, dict( operation="Switch", id=history.id ) ) ), attach_popup=True, filterable="advanced" ), DatasetsByStateColumn( "Datasets (by state)", ncells=4 ), - grids.TagsColumn( "Tags", "tags", model.History, model.HistoryTagAssociation, filterable="advanced"), + grids.TagsColumn( "Tags", "tags", model.History, model.HistoryTagAssociation, filterable="advanced", grid_name="HistoryListGrid" ), StatusColumn( "Status", attach_popup=False ), grids.GridColumn( "Created", key="create_time", format=time_ago ), grids.GridColumn( "Last Updated", key="update_time", format=time_ago ), diff -r 9d4945bbdcf5 -r f98643c26eb7 lib/galaxy/web/controllers/tag.py --- a/lib/galaxy/web/controllers/tag.py Mon Nov 30 15:33:45 2009 -0500 +++ b/lib/galaxy/web/controllers/tag.py Mon Nov 30 17:53:25 2009 -0500 @@ -28,28 +28,38 @@ @web.expose @web.require_login( "Add tag to an item." ) - def add_tag_async( self, trans, id=None, item_class=None, new_tag=None ): + def add_tag_async( self, trans, id=None, item_class=None, new_tag=None, context=None ): """ Add tag to an item. """ - item = self._get_item(trans, item_class, trans.security.decode_id(id)) - self._do_security_check(trans, item) + # Check that user owns item. + item = self._get_item(trans, item_class, trans.security.decode_id( id ) ) + self._do_security_check( trans, item ) + # Apply tag. self.tag_handler.apply_item_tags( trans.sa_session, item, new_tag.encode('utf-8') ) trans.sa_session.flush() + # Log. + params = dict( item_id=item.id, item_class=item_class, tag=new_tag) + trans.log_action( unicode( "tag"), context, params ) + @web.expose @web.require_login( "Remove tag from an item." ) - def remove_tag_async( self, trans, id=None, item_class=None, tag_name=None ): + def remove_tag_async( self, trans, id=None, item_class=None, tag_name=None, context=None ): """ Remove tag from an item. """ + + # Check that user owns item. item = self._get_item(trans, item_class, trans.security.decode_id(id)) - self._do_security_check(trans, item) + # Remove tag. self.tag_handler.remove_item_tag( trans, item, tag_name.encode('utf-8') ) - #print tag_name - #print unicode(tag_name) trans.sa_session.flush() + # Log. + params = dict( item_id=item.id, item_class=item_class, tag=tag_name) + trans.log_action( unicode( "untag"), context, params ) + # Retag an item. All previous tags are deleted and new tags are applied. @web.expose @web.require_login( "Apply a new set of tags to an item; previous tags are deleted." ) diff -r 9d4945bbdcf5 -r f98643c26eb7 lib/galaxy/web/framework/__init__.py --- a/lib/galaxy/web/framework/__init__.py Mon Nov 30 15:33:45 2009 -0500 +++ b/lib/galaxy/web/framework/__init__.py Mon Nov 30 17:53:25 2009 -0500 @@ -10,6 +10,7 @@ import base import pickle from galaxy import util +from galaxy.util.json import to_json_string pkg_resources.require( "simplejson" ) import simplejson @@ -170,6 +171,22 @@ to allow migration toward a more SQLAlchemy 0.4 style of use. """ return self.app.model.context.current + def log_action( self, action, context, params): + """ + Application-level logging of user actions. + """ + if self.app.config.log_actions: + action = self.app.model.UserAction(action=action, context=context, params=unicode( to_json_string( params ) ) ) + try: + action.user = self.user + except: + action.user = None + try: + action.session_id = self.galaxy_session.id + except: + action.session_id = None + self.sa_session.add( action ) + self.sa_session.flush() def log_event( self, message, tool_id=None, **kwargs ): """ Application level logging. Still needs fleshing out (log levels and such) diff -r 9d4945bbdcf5 -r f98643c26eb7 lib/galaxy/web/framework/helpers/grids.py --- a/lib/galaxy/web/framework/helpers/grids.py Mon Nov 30 15:33:45 2009 -0500 +++ b/lib/galaxy/web/framework/helpers/grids.py Mon Nov 30 17:53:25 2009 -0500 @@ -200,6 +200,13 @@ trans.get_user().preferences[pref_name] = unicode( to_json_string( sort_key ) ) trans.sa_session.flush() + # Log grid view. + context = unicode( self.__class__.__name__ ) + params = cur_filter_dict.copy() + params['sort'] = sort_key + params['async'] = ( 'async' in kwargs ) + trans.log_action( unicode( "grid.view"), context, params ) + # Render grid. def url( *args, **kwargs ): # Only include sort/filter arguments if not linking to another @@ -340,17 +347,19 @@ # Generic column that supports tagging. class TagsColumn( TextColumn ): - def __init__( self, col_name, key, model_class, model_tag_association_class, filterable ): + def __init__( self, col_name, key, model_class, model_tag_association_class, filterable, grid_name=None ): GridColumn.__init__(self, col_name, key=key, model_class=model_class, filterable=filterable) self.model_tag_association_class = model_tag_association_class # Tags cannot be sorted. self.sortable = False + # Column-specific attributes. self.tag_elt_id_gen = 0 + self.grid_name = grid_name def get_value( self, trans, grid, item ): self.tag_elt_id_gen += 1 elt_id="tagging-elt" + str( self.tag_elt_id_gen ) div_elt = "<div id=%s></div>" % elt_id - return div_elt + trans.fill_template( "/tagging_common.mako", trans=trans, tagged_item=item, + return div_elt + trans.fill_template( "/tagging_common.mako", trans=trans, tagged_item=item, elt_context=self.grid_name, elt_id = elt_id, in_form="true", input_size="20", tag_click_fn="add_tag_to_grid_filter" ) def filter( self, db_session, query, column_filter ): """ Modify query to filter model_class by tag. Multiple filters are ANDed. """ diff -r 9d4945bbdcf5 -r f98643c26eb7 templates/dataset/edit_attributes.mako --- a/templates/dataset/edit_attributes.mako Mon Nov 30 15:33:45 2009 -0500 +++ b/templates/dataset/edit_attributes.mako Mon Nov 30 17:53:25 2009 -0500 @@ -59,7 +59,7 @@ </div> <div style="clear: both"></div> </div> - ${render_tagging_element(data, "dataset-tag-area", use_toggle_link="false", in_form="true", input_size="30")} + ${render_tagging_element(data, "dataset-tag-area", "edit_attributes.mako", use_toggle_link="false", in_form="true", input_size="30")} %endif %for name, spec in data.metadata.spec.items(): %if spec.visible: diff -r 9d4945bbdcf5 -r f98643c26eb7 templates/history/view.mako --- a/templates/history/view.mako Mon Nov 30 15:33:45 2009 -0500 +++ b/templates/history/view.mako Mon Nov 30 17:53:25 2009 -0500 @@ -327,7 +327,7 @@ %if trans.get_user() is not None: <div id='history-tag-area' class="tag-element"></div> - ${render_tagging_element(history, "history-tag-area", use_toggle_link='false', get_toggle_link_text_fn='get_toggle_link_text', editable=user_owns_history)} + ${render_tagging_element(history, "history-tag-area", "history/view.mako", use_toggle_link='false', get_toggle_link_text_fn='get_toggle_link_text', editable=user_owns_history)} %endif %if not datasets: diff -r 9d4945bbdcf5 -r f98643c26eb7 templates/root/history.mako --- a/templates/root/history.mako Mon Nov 30 15:33:45 2009 -0500 +++ b/templates/root/history.mako Mon Nov 30 17:53:25 2009 -0500 @@ -325,7 +325,7 @@ %if trans.get_user() is not None: <div id='history-tag-area' class="tag-element"></div> - ${render_tagging_element(history, "history-tag-area", get_toggle_link_text_fn='get_toggle_link_text')} + ${render_tagging_element( tagged_item=history, elt_id="history-tag-area", elt_context="history.mako", get_toggle_link_text_fn='get_toggle_link_text' )} %endif %if not datasets: diff -r 9d4945bbdcf5 -r f98643c26eb7 templates/tagging_common.mako --- a/templates/tagging_common.mako Mon Nov 30 15:33:45 2009 -0500 +++ b/templates/tagging_common.mako Mon Nov 30 17:53:25 2009 -0500 @@ -1,10 +1,10 @@ ## Render a tagging element if there is a tagged_item. %if tagged_item is not None and elt_id is not None: - ${render_tagging_element(tagged_item, elt_id=elt_id, in_form=in_form, input_size=input_size, tag_click_fn=tag_click_fn)} + ${render_tagging_element(tagged_item=tagged_item, elt_id=elt_id, elt_context=elt_context, in_form=in_form, input_size=input_size, tag_click_fn=tag_click_fn)} %endif ## Render the tags 'tags' as an autocomplete element. -<%def name="render_tagging_element(tagged_item, elt_id, use_toggle_link='true', in_form='false', input_size='15', tag_click_fn='default_tag_click_fn', get_toggle_link_text_fn='default_get_toggle_link_text_fn', editable='true')"> +<%def name="render_tagging_element(tagged_item, elt_id, elt_context, use_toggle_link='true', in_form='false', input_size='15', tag_click_fn='default_tag_click_fn', get_toggle_link_text_fn='default_get_toggle_link_text_fn', editable='true')"> <script type="text/javascript"> // // Set up autocomplete tagger. @@ -75,8 +75,8 @@ tag_click_fn: ${tag_click_fn}, <% tagged_item_id = trans.security.encode_id(tagged_item.id) %> ajax_autocomplete_tag_url: "${h.url_for( controller='tag', action='tag_autocomplete_data', id=tagged_item_id, item_class=tagged_item.__class__.__name__ )}", - ajax_add_tag_url: "${h.url_for( controller='tag', action='add_tag_async', id=tagged_item_id, item_class=tagged_item.__class__.__name__ )}", - ajax_delete_tag_url: "${h.url_for( controller='tag', action='remove_tag_async', id=tagged_item_id, item_class=tagged_item.__class__.__name__ )}", + ajax_add_tag_url: "${h.url_for( controller='tag', action='add_tag_async', id=tagged_item_id, item_class=tagged_item.__class__.__name__, context=elt_context )}", + ajax_delete_tag_url: "${h.url_for( controller='tag', action='remove_tag_async', id=tagged_item_id, item_class=tagged_item.__class__.__name__, context=elt_context )}", delete_tag_img: "${h.url_for('/static/images/delete_tag_icon_gray.png')}", delete_tag_img_rollover: "${h.url_for('/static/images/delete_tag_icon_white.png')}", add_tag_img: "${h.url_for('/static/images/add_icon.png')}", diff -r 9d4945bbdcf5 -r f98643c26eb7 universe_wsgi.ini.sample --- a/universe_wsgi.ini.sample Mon Nov 30 15:33:45 2009 -0500 +++ b/universe_wsgi.ini.sample Mon Nov 30 17:53:25 2009 -0500 @@ -144,6 +144,9 @@ # Log events log_events = True +# Log user actions +log_actions = True + # Configuration for debugging middleware debug = True use_lint = False

1 0

[hg] galaxy 3132: Merge
by Greg Von Kuster 02 Dec '09

02 Dec '09

details: http://www.bx.psu.edu/hg/galaxy/rev/efed28fa6a5e changeset: 3132:efed28fa6a5e user: Enis Afgan <afgane(a)gmail.com> date: Mon Nov 30 16:47:18 2009 -0500 description: Merge diffstat: lib/galaxy/cloud/__init__.py | 100 +++++++++- lib/galaxy/cloud/providers/ec2.py | 300 +++++++++++++++++++++++++++----- lib/galaxy/cloud/providers/eucalyptus.py | 20 +- lib/galaxy/web/controllers/cloud.py | 93 +++++++++- templates/cloud/configure_cloud.mako | 19 +- 5 files changed, 445 insertions(+), 87 deletions(-) diffs (922 lines): diff -r 9d4945bbdcf5 -r efed28fa6a5e lib/galaxy/cloud/__init__.py --- a/lib/galaxy/cloud/__init__.py Mon Nov 30 15:33:45 2009 -0500 +++ b/lib/galaxy/cloud/__init__.py Mon Nov 30 16:47:18 2009 -0500 @@ -29,6 +29,8 @@ SUBMITTED = "submitted", SHUTTING_DOWN_UCI = "shutting-downUCI", SHUTTING_DOWN = "shutting-down", + ADD_STORAGE_UCI = "add-storageUCI", + ADD_STORAGE = "add-storage", AVAILABLE = "available", RUNNING = "running", PENDING = "pending", @@ -40,6 +42,7 @@ TERMINATED = "terminated", SUBMITTED = "submitted", RUNNING = "running", + ADDING = "adding-storage", PENDING = "pending", SHUTTING_DOWN = "shutting-down", ERROR = "error" @@ -48,6 +51,7 @@ store_status = Bunch( WAITING = "waiting", IN_USE = "in-use", + ADDING = "adding", CREATING = "creating", DELETED = 'deleted', ERROR = "error" @@ -161,7 +165,8 @@ model.UCI.table.c.state==uci_states.SUBMITTED_UCI, model.UCI.table.c.state==uci_states.SHUTTING_DOWN_UCI, model.UCI.table.c.state==uci_states.DELETING_UCI, - model.UCI.table.c.state==uci_states.SNAPSHOT_UCI ) ) \ + model.UCI.table.c.state==uci_states.SNAPSHOT_UCI, + model.UCI.table.c.state==uci_states.ADD_STORAGE_UCI ) ) \ .all(): uci_wrapper = UCIwrapper( r, self.app ) new_requests.append( uci_wrapper ) @@ -195,6 +200,12 @@ self.uci_id = uci.id self.app = app self.sa_session = self.app.model.context + base_directory = os.path.join( self.app.config.job_working_directory, "cloud" ) + self.working_directory = os.path.join( base_directory, str( self.uci_id ) ) +# log.debug( "Cloud controller working directory for UCI DB ID '%s': '%s'" % ( self.uci_id, self.working_directory ) ) + if not os.path.exists( base_directory ): + os.mkdir( base_directory ) + # --------- Setter methods ----------------- @@ -364,11 +375,18 @@ self.sa_session.add( uci ) self.sa_session.flush() - def set_store_device( self, store_id, device ): +# def set_store_device( self, store_id, device ): +# uci = self.sa_session.query( model.UCI ).get( self.uci_id ) +# self.sa_session.refresh( uci ) +# uci.store[store_id].device = device +# uci.store[store_id].flush() + + def set_uci_total_size( self, total_size ): uci = self.sa_session.query( model.UCI ).get( self.uci_id ) self.sa_session.refresh( uci ) - uci.store[store_id].device = device - uci.store[store_id].flush() + uci.total_size = total_size + self.sa_session.add( uci ) + self.sa_session.flush() def set_store_error( self, error, store_index=None, store_id=None ): if store_index != None: @@ -405,15 +423,24 @@ def set_store_volume_id( self, store_index, volume_id ): """ - Given store index associated with this UCI in local database, set volume ID as it is registered + Given store index as it is stored in local database, set volume ID as it is registered on the cloud provider (e.g., vol-39890501) """ - uci = self.sa_session.query( model.UCI ).get( self.uci_id ) - self.sa_session.refresh( uci ) - uci.store[store_index].volume_id = volume_id - #uci.store[store_index].flush() - self.sa_session.add( uci ) - self.sa_session.flush() + + if store_index != None: + store = self.sa_session.query( model.CloudStore ).get( store_index ) + store.volume_id = volume_id + self.sa_session.add( store ) + self.sa_session.flush() + else: + return None + +# uci = self.sa_session.query( model.UCI ).get( self.uci_id ) +# self.sa_session.refresh( uci ) +# uci.store[store_index].volume_id = volume_id +# #uci.store[store_index].flush() +# self.sa_session.add( uci ) +# self.sa_session.flush() def set_store_instance( self, vol_id, instance_id ): """ @@ -421,7 +448,29 @@ be given in following format: 'vol-78943248' """ vol = self.sa_session.query( model.CloudStore ).filter( model.CloudStore.table.c.volume_id == vol_id ).first() - vol.inst.instance_id = instance_id + inst = self.sa_session.query( model.CloudInstance ).filter_by( instance_id=instance_id ).first() + vol.inst = inst + self.sa_session.add( vol ) + self.sa_session.flush() + + def set_store_device( self, vol_id, device ): + """ + Stores instance ID that given store volume is attached to. Store volume ID should + be given in following format: 'vol-78943248' + """ + vol = self.sa_session.query( model.CloudStore ).filter( model.CloudStore.table.c.volume_id == vol_id ).first() + vol.device = str( device ) + self.sa_session.add( vol ) + self.sa_session.flush() + + def set_store_deleted( self, vol_id, status=None ): + """ + Set storage volume as deleted in local database. Optionally, set the volume status too. + """ + vol = self.sa_session.query( model.CloudStore ).filter( model.CloudStore.table.c.volume_id == vol_id ).first() + vol.deleted = True + if status != None: + vol.status = status self.sa_session.add( vol ) self.sa_session.flush() @@ -503,6 +552,12 @@ self.sa_session.refresh( uci ) return uci.instance[instance_id].state + def get_instaces_in_state( self, state ): + """ Get database objects of all instances associated with this UCI in given state. """ + return self.sa_session.query( model.CloudInstance ) \ + .filter_by( uci_id=self.uci_id, state = state ) \ + .all() + def get_instances_ids( self ): """ Returns list IDs of all instances' associated with this UCI that are not in 'terminated' or @@ -590,6 +645,11 @@ self.sa_session.refresh( uci ) return uci.store[0].availability_zone + def get_uci_total_size( self ): + uci = self.sa_session.query( model.UCI ).get( self.uci_id ) + self.sa_session.refresh( uci ) + return uci.total_size + def get_store_size( self, store_id=0 ): uci = self.sa_session.query( model.UCI ).get( self.uci_id ) self.sa_session.refresh( uci ) @@ -604,9 +664,18 @@ self.sa_session.refresh( uci ) return uci.store[store_id].volume_id + def get_all_stores_in_status( self, status ): + """ + Return database objects of all stores associated with this UCI that have their + status set to value passed as parameter. + """ + return self.sa_session.query( model.CloudStore ).filter_by( deleted=False, uci_id=self.uci_id, status=status ).all() + def get_all_stores( self ): - """ Returns all storage volumes' database objects associated with this UCI. """ - return self.sa_session.query( model.CloudStore ).filter( model.CloudStore.table.c.uci_id == self.uci_id ).all() + """ Returns all storage volumes' database objects associated with this UCI that have not been marked as 'deleted'. """ + return self.sa_session.query( model.CloudStore ) \ + .filter_by( deleted=False, uci_id=self.uci_id ) \ + .all() def get_snapshots( self, status=None ): """ Returns database objects for all snapshots associated with this UCI and in given status.""" @@ -618,6 +687,9 @@ self.sa_session.refresh( uci ) return uci + def get_uci_working_directory( self ): + return self.working_directory + def uci_launch_time_set( self ): uci = self.sa_session.query( model.UCI ).get( self.uci_id ) self.sa_session.refresh( uci ) diff -r 9d4945bbdcf5 -r efed28fa6a5e lib/galaxy/cloud/providers/ec2.py --- a/lib/galaxy/cloud/providers/ec2.py Mon Nov 30 15:33:45 2009 -0500 +++ b/lib/galaxy/cloud/providers/ec2.py Mon Nov 30 16:47:18 2009 -0500 @@ -1,4 +1,4 @@ -import subprocess, threading, os, errno, time, datetime +import subprocess, threading, os, errno, time, datetime, stat from Queue import Queue, Empty from datetime import datetime @@ -30,6 +30,8 @@ SUBMITTED = "submitted", SHUTTING_DOWN_UCI = "shutting-downUCI", SHUTTING_DOWN = "shutting-down", + ADD_STORAGE_UCI = "add-storageUCI", + ADD_STORAGE = "add-storage", AVAILABLE = "available", RUNNING = "running", PENDING = "pending", @@ -43,6 +45,7 @@ TERMINATED = "terminated", SUBMITTED = "submitted", RUNNING = "running", + ADDING = "adding-storage", PENDING = "pending", SHUTTING_DOWN = "shutting-down", ERROR = "error" @@ -51,6 +54,7 @@ store_status = Bunch( WAITING = "waiting", IN_USE = "in-use", + ADDING = "adding", CREATING = "creating", DELETED = 'deleted', ERROR = "error" @@ -122,6 +126,9 @@ self.stop_uci( uci_wrapper ) elif uci_state==uci_states.SNAPSHOT: self.snapshot_uci( uci_wrapper ) + elif uci_state==uci_states.ADD_STORAGE: + self.add_storage_to_uci( uci_wrapper ) + #self.dummy_start_uci( uci_wrapper ) except: log.exception( "Uncaught exception executing cloud request." ) cnt += 1 @@ -241,25 +248,14 @@ log.info( "Availability zone for UCI (i.e., storage volume) was not selected, using default zone: %s" % self.zone ) uci_wrapper.set_store_availability_zone( self.zone ) - log.info( "Creating volume in zone '%s'..." % uci_wrapper.get_uci_availability_zone() ) + store = uci_wrapper.get_all_stores_in_status( store_status.ADDING )[0] # Because at UCI creation time only 1 storage volume can be created, reference it directly + + log.info( "Creating storage volume in zone '%s' of size '%s'..." % ( uci_wrapper.get_uci_availability_zone(), store.size ) ) # Because only 1 storage volume may be created at UCI config time, index of this storage volume in local Galaxy DB w.r.t # current UCI is 0, so reference it in following methods - vol = conn.create_volume( uci_wrapper.get_store_size( 0 ), uci_wrapper.get_uci_availability_zone(), snapshot=None ) - uci_wrapper.set_store_volume_id( 0, vol.id ) - - # Wait for a while to ensure volume was created -# vol_status = vol.status -# for i in range( 30 ): -# if vol_status is not "available": -# log.debug( 'Updating volume status; current status: %s' % vol_status ) -# vol_status = vol.status -# time.sleep(3) -# if i is 29: -# log.debug( "Error while creating volume '%s'; stuck in state '%s'; deleting volume." % ( vol.id, vol_status ) ) -# conn.delete_volume( vol.id ) -# uci_wrapper.change_state( uci_state='error' ) -# return - + vol = conn.create_volume( store.size, uci_wrapper.get_uci_availability_zone(), snapshot=None ) + uci_wrapper.set_store_volume_id( store.id, vol.id ) + # Retrieve created volume again to get updated status try: vl = conn.get_all_volumes( [vol.id] ) @@ -306,6 +302,7 @@ if conn.delete_volume( v.volume_id ): deletedList.append( v.volume_id ) v.deleted = True + v.status = store_status.DELETED self.sa_session.add( v ) self.sa_session.flush() count += 1 @@ -358,15 +355,180 @@ uci_wrapper.change_state( uci_state=uci_states.AVAILABLE ) - def add_storage_to_uci( self, name ): - """ Adds more storage to specified UCI - TODO""" - + def add_storage_to_uci( self, uci_wrapper ): + """ + Add an additional storage volume to specified UCI by creating the storage volume + on cloud provider, attaching it to currently running instance and adding it to + 'galaxyData' zpool on remote instance. + """ + conn = self.get_connection( uci_wrapper ) + + stores = uci_wrapper.get_all_stores_in_status( store_status.ADDING ) + for store in stores: + vol_size = store.size + availability_zone = uci_wrapper.get_uci_availability_zone() + log.info( "Adding storage volume to UCI '%s' in zone '%s' of size '%s'..." % ( uci_wrapper.get_name(), availability_zone, vol_size ) ) + + try: + vol = conn.create_volume( vol_size, availability_zone, snapshot=None ) + uci_wrapper.set_store_volume_id( store.id, vol.id ) + uci_wrapper.set_store_availability_zone( availability_zone, vol.id ) + log.debug( "New storage volume created: '%s'" % vol.id ) + except boto.exception.EC2ResponseError, e: + err = "EC2 response error while creating storage volume: " + str( e ) + log.error( err ) + uci_wrapper.set_store_error( err, store_id=vol.id ) + uci_wrapper.set_error( err, True ) + return + except Exception, ex: + err = "Error while creating storage volume: " + str( ex ) + log.error( err ) + uci_wrapper.set_error( err, True ) + return + + # Retrieve created volume again to get updated status + try: + vl = conn.get_all_volumes( [vol.id] ) + except boto.exception.EC2ResponseError, e: + err = "EC2 response error while retrieving (i.e., updating status) of just created storage volume '" + vol.id + "': " + str( e ) + log.error( err ) + uci_wrapper.set_store_error( err, store_id=vol.id ) + uci_wrapper.set_error( err, True ) + return + except Exception, ex: + err = "Error while retrieving (i.e., updating status) of just created storage volume '" + vol.id + "': " + str( ex ) + log.error( err ) + uci_wrapper.set_error( err, True ) + return + + # Wait for a while to ensure volume was created + if len( vl ) > 0: + vol_status = vl[0].status # Bc. only single vol is queried, reference it as 0th list element + for i in range( 30 ): + if vol_status != "available": + log.debug( "(%s) Updating volume status; current status: '%s'" % (i, vol_status ) ) + uci_wrapper.change_state( uci_state=vol_status ) + time.sleep(5) + vol_status = vl[0].status + if vol_status == "available": + log.debug( "(%s) New volume status '%s', continuing with file system adjustment." % (i, vol_status ) ) + uci_wrapper.set_store_status( vl[0].id, vol_status ) + break + if i is 29: + err = "Error while creating volume '"+vl[0].id+"'; stuck in state '"+vol_status+"'; deleting volume." + conn.delete_volume( vl[0].id ) + log.error( err ) + uci_wrapper.set_error( err, True ) + uci_wrapper.set_store_error( err, store_id=vol.id ) + conn.delete_volume( vl[0].id ) + uci_wrapper.set_store_deleted( vl[0].id ) + return + else: + err = "Volume '" + vol.id +"' not found by EC2 after being created." + log.error( err ) + uci_wrapper.set_store_error( err, store_id=vol.id ) + uci_wrapper.set_error( err, True ) + return + + # Get private key for given instance + pk = uci_wrapper.get_key_pair_material() + if pk == None: #If pk does not exist, create it + self.check_key_pair( uci_wrapper, conn ) + pk = uci_wrapper.get_key_pair_material() + + # Get working directory for this UCI and store pk into a file + wd = uci_wrapper.get_uci_working_directory() + if not os.path.exists( wd ): + os.mkdir( wd ) + pk_file_path = os.path.join( wd, "pk" ) + + if pk != None: + # Save private key to a file + pk_file = open( pk_file_path, "w" ) + pk_file.write( pk ) + pk_file.close() + else: + err = "ERROR: Private key not available for this UCI." + log.error( err ) + uci_wrapper.set_store_error( err, store_id=vol.id ) + uci_wrapper.set_error( err, True ) + return + + if os.path.exists( pk_file_path ): + # Change permissions of the file - this is required by later used ssh + os.chmod( pk_file_path, stat.S_IRUSR | stat.S_IWUSR ) + + # Get # of storage volumes associated with this UCI to know as which device to connect new volume to the instance + device_num = len( uci_wrapper.get_all_stores_in_status( store_status.IN_USE ) ) + 5 # First device num is 5, so all subsequent ones should follow + + # Get instance that the new storage volume is to be attached to. Although a list is returned, + # only 1 instance can be in 'adding-storage' state (because, for now, only 1 instance is assoc. with + # each UCI) and volume can be attached to only to it + il = uci_wrapper.get_instaces_in_state( instance_states.ADDING ) + if len( il ) > 0: + # Attach new volume to the instance + log.debug( "Attaching new storage volume '%s' to UCI '%s' as device '%s'" % + ( vol.id, uci_wrapper.get_name(), device_num ) ) + try: + vol_status = conn.attach_volume( vol.id, il[0].instance_id, device_num ) + except boto.exception.EC2ResponseError, e: + err = "Attaching just created storage volume '" + vol.id + "'to instance '" + \ + il[0].instance_id + "' as device '" + str( device_num ) + "' failed: " + str( e ) + log.error( err ) + uci_wrapper.set_store_error( err, store_id=vol.id ) + uci_wrapper.set_error( err, True ) + return + # For a while, keep checking attachment status of the new volume + for i in range(30): + log.debug( "Checking attachment status of new volume '%s': '%s'" % ( vol.id, vol_status ) ) + if vol_status == 'attached': + uci_wrapper.set_store_status( vol.id, vol_status ) + uci_wrapper.set_store_device( vol.id, device_num ) + break + if i == 29: + err = "Storage volume '" + vol.id + "' failed to attach to instance '" + il[0].instance_id + \ + "'. Manual check needed." + log.error( err ) + uci_wrapper.set_store_error( err, store_id=vol.id ) + uci_wrapper.set_error( err, False ) + return + + time.sleep(4) + vol_list = conn.get_all_volumes( [vol.id] ) + for v in vol_list: + vol_status = v.attachment_state() + + # Once storage volume is attached, add it to the zpool by issuing system level command + cmd = 'ssh -o StrictHostKeyChecking=no -i '+ pk_file_path +' root@'+il[0].public_dns+' "zpool add galaxyData c7d' + str( device_num )+'"' + log.debug( "Adding new storage volume to zpool cmd: %s" % cmd ) + stdout = os.system( cmd ) + if stdout != 0: + err = "Adding newly created storage volume to zpool on instance '" + il[0].instance_id + \ + "' failed. Error code: " + str( stdout ) + log.error( err ) + uci_wrapper.set_store_error( err, store_id=vol.id ) + uci_wrapper.set_error( err, False ) + return + else: + err = "No instance(s) found in 'adding-storage' state. New disk not added to UCI's zpool." + log.error( err ) + uci_wrapper.set_store_error( err, store_id=vol.id ) + uci_wrapper.set_error( err, True ) + return + + # Update UCI's total storage size + uci_wrapper.set_uci_total_size( uci_wrapper.get_uci_total_size() + vol.size ) + # Reset UCI's and instance's state + uci_wrapper.change_state( uci_state=uci_states.RUNNING, instance_id=il[0].instance_id, i_state=instance_states.RUNNING ) + log.debug( "Successfully added storage volume '%s' to UCI '%s'." % ( vol.id, uci_wrapper.get_name() ) ) + def dummy_start_uci( self, uci_wrapper ): uci = uci_wrapper.get_uci() - log.debug( "Would be starting instance '%s'" % uci.name ) - uci_wrapper.change_state( uci_state.PENDING ) + log.debug( "Dummy start UCI '%s'" % uci.name ) + + +# uci_wrapper.change_state( uci_state.PENDING ) # log.debug( "Sleeping a bit... (%s)" % uci.name ) # time.sleep(20) # log.debug( "Woke up! (%s)" % uci.name ) @@ -418,10 +580,25 @@ if uci_wrapper.get_uci_state() != uci_states.ERROR: # Start an instance log.debug( "Starting instance for UCI '%s'" % uci_wrapper.get_name() ) - #TODO: Once multiple volumes can be attached to a single instance, update 'userdata' composition - userdata = uci_wrapper.get_store_volume_id()+"|"+uci_wrapper.get_access_key()+"|"+uci_wrapper.get_secret_key() + #TODO: Once multiple volumes can be attached to a single instance, update 'userdata' composition + # Compose user data; for storage volumes, separate multiple volumes with a colon (:) ensuring that + # the last volume in the list is not followed by a colon. + stores = uci_wrapper.get_all_stores() + volume_ids = "" + if len( stores ) > 0: + for i, store in enumerate( stores ): + volume_ids += store.volume_id + if i < len( stores )-1: + volume_ids += ":" + else: + err = "No storage volumes found that are associated with UCI '%s'" + uci_wrapper.get_name() + log.error( err ) + uci_wrapper.set_error( err, True ) + return + userdata = volume_ids+"|"+uci_wrapper.get_access_key()+"|"+uci_wrapper.get_secret_key() log.debug( "Using following command: conn.run_instances( image_id='%s', key_name='%s', security_groups=['%s'], user_data=[OMITTED], instance_type='%s', placement='%s' )" % ( mi_id, uci_wrapper.get_key_pair_name(), self.security_group, uci_wrapper.get_instance_type( i_index ), uci_wrapper.get_uci_availability_zone() ) ) + # Start an instance reservation = None try: reservation = conn.run_instances( image_id=mi_id, @@ -454,7 +631,10 @@ uci_wrapper.change_state( s, i_id, s ) uci_wrapper.set_security_group_name( self.security_group, i_id=i_id ) vol_id = uci_wrapper.get_store_volume_id( store_id=0 ) # TODO: Once more that one vol/UCI is allowed, update this! - uci_wrapper.set_store_status( vol_id, store_status.WAITING ) + # Following line is pointless bc. general update updates status of volume to 'available' + # before it actually connects to starting instance... This has been dealt w/ in general update method + #uci_wrapper.set_store_status( vol_id, store_status.WAITING ) + uci_wrapper.set_store_instance( vol_id, i_id ) log.debug( "Instance of UCI '%s' started, current state: '%s'" % ( uci_wrapper.get_name(), uci_wrapper.get_uci_state() ) ) except boto.exception.EC2ResponseError, e: err = "EC2 response error when retrieving instance information for UCI '" + uci_wrapper.get_name() + "': " + str( e ) @@ -571,19 +751,30 @@ .all() for inst in instances: if self.type == inst.uci.credentials.provider.type: - log.debug( "[%s] Running general status update on instance '%s'" % ( inst.uci.credentials.provider.type, inst.instance_id ) ) + log.debug( "[%s] Running general status update on instance '%s'" + % ( inst.uci.credentials.provider.type, inst.instance_id ) ) self.update_instance( inst ) + # Update storage volume(s) associated with current instance + stores = self.sa_session.query( model.CloudStore ) \ + .filter_by( uci_id=inst.uci_id, deleted=False ) \ + .all() + for store in stores: + if self.type == store.uci.credentials.provider.type: # and store.volume_id != None: + log.debug( "[%s] Running general status update on store with local database ID: '%s'" + % ( store.uci.credentials.provider.type, store.id ) ) + self.update_store( store ) # Update storage volume(s) stores = self.sa_session.query( model.CloudStore ) \ - .filter( or_( model.CloudStore.table.c.status==store_status.IN_USE, - model.CloudStore.table.c.status==store_status.CREATING, - model.CloudStore.table.c.status==store_status.WAITING, + .filter( or_( model.CloudStore.table.c.status==store_status.CREATING, +# model.CloudStore.table.c.status==store_status.IN_USE, +# model.CloudStore.table.c.status==store_status.WAITING, model.CloudStore.table.c.status==None ) ) \ .all() for store in stores: if self.type == store.uci.credentials.provider.type: # and store.volume_id != None: - log.debug( "[%s] Running general status update on store with local database ID: '%s'" % ( store.uci.credentials.provider.type, store.id ) ) + log.debug( "[%s] Running general status update on store with local database ID: '%s'" + % ( store.uci.credentials.provider.type, store.id ) ) self.update_store( store ) # else: # log.error( "[%s] There exists an entry for UCI (%s) storage volume without an ID. Storage volume might have been created with " @@ -668,7 +859,8 @@ for i, cInst in enumerate( r.instances ): try: s = cInst.update() - log.debug( "Checking state of cloud instance '%s' associated with UCI '%s' and reservation '%s'. State='%s'" % ( cInst, uci.name, r, s ) ) + log.debug( "Checking state of cloud instance '%s' associated with UCI '%s' " \ + "and reservation '%s'. State='%s'" % ( cInst, uci.name, r, s ) ) if s != inst.state: inst.state = s self.sa_session.add( inst ) @@ -693,7 +885,8 @@ self.sa_session.add( inst ) self.sa_session.flush() except boto.exception.EC2ResponseError, e: - err = "Updating instance status from cloud failed for UCI '"+ uci.name + "' during general status update: " + str( e ) + err = "Updating instance status from cloud failed for UCI '"+ uci.name + \ + "' during general status update: " + str( e ) log.error( err ) uci.error = err uci.state = uci_states.ERROR @@ -715,7 +908,7 @@ # Get reservations handle for given store try: - log.debug( "Updating storage volume command: vl = conn.get_all_volumes( [%s] )" % store.volume_id ) + log.debug( "Retrieving reference to storage volume '%s' during update..." % store.volume_id ) vl = conn.get_all_volumes( [store.volume_id] ) except boto.exception.EC2ResponseError, e: err = "Retrieving volume(s) from cloud failed for UCI '"+ uci.name + "' during general status update: " + str( e ) @@ -729,7 +922,7 @@ # Update store status in local DB with info from cloud provider if len(vl) > 0: try: - log.debug( "Storage volume '%s' current status: '%s'" % (store.volume_id, vl[0].status ) ) + log.debug( "General status update for storage volume '%s'; current status: '%s'" % (store.volume_id, vl[0].status ) ) if store.status != vl[0].status: # In case something failed during creation of UCI but actual storage volume was created and yet # UCI state remained as 'new', try to remedy this by updating UCI state here @@ -746,21 +939,26 @@ store.status = vl[0].status self.sa_session.add( store ) self.sa_session.flush() - if store.inst != None: - if store.inst.instance_id != vl[0].instance_id: - store.inst.instance_id = vl[0].instance_id - self.sa_session.add( store ) - self.sa_session.flush() - if store.attach_time != vl[0].attach_time: - store.attach_time = vl[0].attach_time - self.sa_session.add( store ) - self.sa_session.flush() - if store.device != vl[0].device: - store.device = vl[0].device - self.sa_session.add( store ) - self.sa_session.flush() + # Boto does not seem to be reporting these values although fields exist so comment them out... +# log.debug( "vl[0].instance_id: '%s'" % vl[0].instance_id ) +# if store.inst != None: +# if store.inst.instance_id != vl[0].instance_id: +# store.inst.instance_id = vl[0].instance_id +# self.sa_session.add( store ) +# self.sa_session.flush() +# log.debug( "vl[0].attach_time: '%s'" % vl[0].attach_time ) +# if store.attach_time != vl[0].attach_time: +# store.attach_time = vl[0].attach_time +# self.sa_session.add( store ) +# self.sa_session.flush() +## log.debug( "vl[0].device: '%s'" % vl[0].device ) +# if store.device != vl[0].device: +# store.device = vl[0].device +# self.sa_session.add( store ) +# self.sa_session.flush() except boto.exception.EC2ResponseError, e: - err = "Updating status of volume(s) from cloud failed for UCI '"+ uci.name + "' during general status update: " + str( e ) + err = "Updating status of volume(s) from cloud failed for UCI '"+ uci.name + \ + "' during general status update: " + str( e ) log.error( err ) uci.error = err uci.state = uci_states.ERROR @@ -768,7 +966,7 @@ self.sa_session.flush() return None else: - err = "No storage volumes returned by cloud provider on general update" + err = "No storage volumes returned by cloud provider on general update for volume with id: " + store.volume_id log.error( "%s for UCI '%s'" % ( err, uci.name ) ) store.status = store_status.ERROR store.error = err diff -r 9d4945bbdcf5 -r efed28fa6a5e lib/galaxy/cloud/providers/eucalyptus.py --- a/lib/galaxy/cloud/providers/eucalyptus.py Mon Nov 30 15:33:45 2009 -0500 +++ b/lib/galaxy/cloud/providers/eucalyptus.py Mon Nov 30 16:47:18 2009 -0500 @@ -30,6 +30,8 @@ SUBMITTED = "submitted", SHUTTING_DOWN_UCI = "shutting-downUCI", SHUTTING_DOWN = "shutting-down", + ADD_STORAGE_UCI = "add-storageUCI", + ADD_STORAGE = "add-storage", AVAILABLE = "available", RUNNING = "running", PENDING = "pending", @@ -43,6 +45,7 @@ TERMINATED = "terminated", SUBMITTED = "submitted", RUNNING = "running", + ADDING = "adding-storage", PENDING = "pending", SHUTTING_DOWN = "shutting-down", ERROR = "error" @@ -51,6 +54,7 @@ store_status = Bunch( WAITING = "waiting", IN_USE = "in-use", + ADDING = "adding", CREATING = "creating", DELETED = 'deleted', ERROR = "error" @@ -121,6 +125,8 @@ self.stop_uci( uci_wrapper ) elif uci_state==uci_states.SNAPSHOT: self.snapshot_uci( uci_wrapper ) + elif uci_state==uci_states.ADD_STORAGE: + self.add_storage_to_uci( uci_wrapper ) except: log.exception( "Uncaught exception executing cloud request." ) cnt += 1 @@ -245,9 +251,16 @@ log.info( "Availability zone for UCI (i.e., storage volume) was not selected, using default zone: %s" % self.zone ) uci_wrapper.set_store_availability_zone( self.zone ) - log.debug( "Creating volume; using command: conn.create_volume( %s, '%s', snapshot=None )" % ( uci_wrapper.get_store_size( 0 ), uci_wrapper.get_uci_availability_zone() )) - vol = conn.create_volume( uci_wrapper.get_store_size( 0 ), uci_wrapper.get_uci_availability_zone(), snapshot=None ) - uci_wrapper.set_store_volume_id( 0, vol.id ) +# log.debug( "Creating volume; using command: conn.create_volume( %s, '%s', snapshot=None )" % ( uci_wrapper.get_store_size( 0 ), uci_wrapper.get_uci_availability_zone() )) +# vol = conn.create_volume( uci_wrapper.get_store_size( 0 ), uci_wrapper.get_uci_availability_zone(), snapshot=None ) +# uci_wrapper.set_store_volume_id( 0, vol.id ) + store = uci_wrapper.get_all_stores_in_status( store_status.ADDING )[0] # Because at UCI creation time only 1 storage volume can be created, reference it directly + + log.info( "Creating storage volume in zone '%s' of size '%s'..." % ( uci_wrapper.get_uci_availability_zone(), store.size ) ) + # Because only 1 storage volume may be created at UCI config time, index of this storage volume in local Galaxy DB w.r.t + # current UCI is 0, so reference it in following methods + vol = conn.create_volume( store.size, uci_wrapper.get_uci_availability_zone(), snapshot=None ) + uci_wrapper.set_store_volume_id( store.id, vol.id ) # Retrieve created volume again to get updated status try: @@ -364,6 +377,7 @@ def add_storage_to_uci( self, uci_wrapper ): """ Adds more storage to specified UCI """ + uci_wrapper.set_error( "Adding storage to eucalyptus-based clouds is not yet supported.", True ) def dummy_start_uci( self, uci_wrapper ): diff -r 9d4945bbdcf5 -r efed28fa6a5e lib/galaxy/web/controllers/cloud.py --- a/lib/galaxy/web/controllers/cloud.py Mon Nov 30 15:33:45 2009 -0500 +++ b/lib/galaxy/web/controllers/cloud.py Mon Nov 30 16:47:18 2009 -0500 @@ -41,6 +41,8 @@ SUBMITTED = "submitted", SHUTTING_DOWN_UCI = "shutting-downUCI", SHUTTING_DOWN = "shutting-down", + ADD_STORAGE_UCI = "add-storageUCI", + ADD_STORAGE = "add-storage", AVAILABLE = "available", RUNNING = "running", PENDING = "pending", @@ -54,13 +56,16 @@ TERMINATED = "terminated", SUBMITTED = "submitted", RUNNING = "running", + ADDING = "adding-storage", PENDING = "pending", SHUTTING_DOWN = "shutting-down", ERROR = "error" ) store_status = Bunch( + WAITING = "waiting", IN_USE = "in-use", + ADDING = "adding", CREATING = "creating", DELETED = 'deleted', ERROR = "error" @@ -108,7 +113,9 @@ model.UCI.table.c.state==uci_states.SUBMITTED, model.UCI.table.c.state==uci_states.SUBMITTED_UCI, model.UCI.table.c.state==uci_states.SHUTTING_DOWN, - model.UCI.table.c.state==uci_states.SHUTTING_DOWN_UCI ) ) \ + model.UCI.table.c.state==uci_states.SHUTTING_DOWN_UCI, + model.UCI.table.c.state==uci_states.ADD_STORAGE, + model.UCI.table.c.state==uci_states.ADD_STORAGE_UCI ) ) \ .order_by( desc( model.UCI.table.c.update_time ) ) \ .all() @@ -201,7 +208,8 @@ storage.user = user storage.uci = uci storage.size = volSize - storage.availability_zone = zone + storage.availability_zone = zone + storage.status = store_status.ADDING # Persist session = trans.sa_session session.add( uci ) @@ -306,6 +314,8 @@ ( uci.state != uci_states.ERROR ) and \ ( uci.state != uci_states.SHUTTING_DOWN_UCI ) and \ ( uci.state != uci_states.SHUTTING_DOWN ) and \ + ( uci.state != uci_states.ADD_STORAGE_UCI ) and \ + ( uci.state != uci_states.ADD_STORAGE ) and \ ( uci.state != uci_states.AVAILABLE ): uci.state = uci_states.SHUTTING_DOWN_UCI session = trans.sa_session @@ -496,13 +506,53 @@ @web.expose @web.require_login( "add instance storage" ) - def add_storage( self, trans, id ): - instance = get_uci( trans, id ) + def add_storage( self, trans, id, vol_size=None ): + error = None + uci = get_uci( trans, id ) + stores = get_stores_in_status( trans, uci, store_status.IN_USE ) + # Start adding of storage making sure given UCI is running and that at least one + # storage volume is attached to it (this is needed to by cloud controller to know + # as which device to attach the new storage volume) + if uci.state == uci_states.RUNNING and len( stores ) > 0: + if vol_size is not None: + try: + vol_size = int( vol_size ) + except ValueError: + error = "Volume size must be integer value between 1 and 1000." + + if not error: + user = trans.get_user() + + storage = model.CloudStore() + storage.user = user + storage.uci = uci + storage.size = vol_size + storage.status = store_status.ADDING + + # Set state of instance - NOTE that this code will only work (with code in cloud controller) + # for scenario where a UCI is associated with *1* compute instance!!! + instances = get_instances( trans, uci ) + instances.state = instance_states.ADDING + + uci.state = uci_states.ADD_STORAGE_UCI + # Persist + session = trans.sa_session + session.add( instances ) + session.add( storage ) + session.add( uci ) + session.flush() + # Log and display the management page + trans.log_event( "User added storage volume to UCI: '%s'" % uci.name ) + trans.set_message( "Adding of storage to instance '%s' initiated." % uci.name ) + return self.list( trans ) + else: + error( "Storage can only be added to instances that are in state 'RUNNING' with existing " \ + "storage volume(s) already attached." ) - error( "Adding storage to instance '%s' is not supported yet." % instance.name ) - - return self.list( trans ) + return trans.show_form( + web.FormBuilder( url_for( id=trans.security.encode_id(uci.id) ), "Add storage to an instance", submit_text="Add" ) + .add_text( "vol_size", "Storage size (1-1000 GB)", value='', error=error ) ) # ----- Image methods ----- @web.expose @@ -1034,6 +1084,7 @@ dict = {} dict['id'] = uci.id dict['state'] = uci.state + dict['total_size'] = uci.total_size if uci.error != None: dict['error'] = str( uci.error ) else: @@ -1136,19 +1187,30 @@ def get_stores( trans, uci ): """ - Get stores objects that are connected to uci object + Get stores objects that are associated with given uci and are not in 'error' status """ user = trans.get_user() stores = trans.sa_session.query( model.CloudStore ) \ - .filter_by( user=user, uci_id=uci.id ) \ + .filter_by( user=user, uci_id=uci.id, deleted=False ) \ .filter( model.CloudStore.table.c.status != store_status.ERROR ) \ .all() return stores +def get_stores_in_status( trans, uci, status ): + """ + Get stores objects that are associated with given uci and are not have given status + """ + user = trans.get_user() + stores = trans.sa_session.query( model.CloudStore ) \ + .filter_by( user=user, uci_id=uci.id, status=status ) \ + .all() + + return stores + def get_instances( trans, uci ): """ - Get objects of instances that are pending or running and are connected to uci object + Get objects of instances that are pending or running and are connected to the given uci object """ user = trans.get_user() instances = trans.sa_session.query( model.CloudInstance ) \ @@ -1159,6 +1221,17 @@ return instances +def get_instances_in_state( trans, uci, state ): + """ + Get objects of instances that are in specified state and are connected to the given uci object + """ + user = trans.get_user() + instances = trans.sa_session.query( model.CloudInstance ) \ + .filter_by( user=user, uci_id=uci.id, state=state ) \ + .all() + + return instances + def get_connection( trans, creds ): """ Establishes cloud connection using user's credentials diff -r 9d4945bbdcf5 -r efed28fa6a5e templates/cloud/configure_cloud.mako --- a/templates/cloud/configure_cloud.mako Mon Nov 30 15:33:45 2009 -0500 +++ b/templates/cloud/configure_cloud.mako Mon Nov 30 16:47:18 2009 -0500 @@ -79,7 +79,8 @@ }); } - // Update 'state' and 'time alive' fields + // Update 'size', 'state' and 'time alive' fields + $(elem + "-size").text( data[i].total_size ); $(elem + "-state").text( data[i].state ); if ( new_state != 'error' ) { // Because 'error' state is handled as a JS link, don't include it in update $(elem + "-state-p").text( data[i].state ); @@ -219,7 +220,7 @@ ${liveInstance.name} (${liveInstance.credentials.name}) <a id="li-${i}-popup" class="popup-arrow" style="display: none;">▼</a> </td> - <td>${str(liveInstance.total_size)}</td> + <td id="${ liveInstance.id }-size">${str(liveInstance.total_size)}</td> <td id="${ liveInstance.id }-state">${str(liveInstance.state)}</td> <td id="${ liveInstance.id }-launch_time"> ##${str(liveInstance.launch_time)[:16]} @@ -251,7 +252,8 @@ <a class="action-button" confirm="Are you sure you want to stop instance '${liveInstance.name}'?" href="${h.url_for( action='stop', id=trans.security.encode_id(liveInstance.id) )}">Stop</a> <a class="action-button" href="${h.url_for( action='rename_uci', id=trans.security.encode_id(liveInstance.id) )}">Rename</a> <a class="action-button" href="${h.url_for( action='view_uci_details', id=trans.security.encode_id(liveInstance.id) )}">View details</a> - <a class="action-button" href="${h.url_for( action='uci_usage_report', id=trans.security.encode_id(liveInstance.id) )}">Usage report</a> + <a class="action-button" href="${h.url_for( action='add_storage', id=trans.security.encode_id(liveInstance.id) )}">Add storage</a> + <a class="action-button" href="${h.url_for( action='uci_usage_report', id=trans.security.encode_id(liveInstance.id) )}">Usage report</a> </div> </td> </tr> @@ -315,14 +317,13 @@ </td> <td> <div popupmenu="pi-${i}-popup"> - <a class="action-button" href="${h.url_for( action='start', id=trans.security.encode_id(prevInstance.id), type='m1.small' )}"> Start m1.small</a> + <a class="action-button" href="${h.url_for( action='uci_usage_report', id=trans.security.encode_id(prevInstance.id) )}">Usage report</a> + <a class="action-button" href="${h.url_for( action='rename_uci', id=trans.security.encode_id(prevInstance.id) )}">Rename</a> + <a class="action-button" href="${h.url_for( action='start', id=trans.security.encode_id(prevInstance.id), type='m1.small' )}"> Start m1.small</a> <a class="action-button" href="${h.url_for( action='start', id=trans.security.encode_id(prevInstance.id), type='c1.medium' )}"> Start c1.medium</a> - <a class="action-button" href="${h.url_for( action='rename_uci', id=trans.security.encode_id(prevInstance.id) )}">Rename</a> - <a class="action-button" href="${h.url_for( action='uci_usage_report', id=trans.security.encode_id(prevInstance.id) )}">Usage report</a> - <a class="action-button" href="${h.url_for( action='create_snapshot', id=trans.security.encode_id(prevInstance.id) )}">Create snapshot</a> + <a class="action-button" href="${h.url_for( action='create_snapshot', id=trans.security.encode_id(prevInstance.id) )}">Create snapshot</a> <a class="action-button" href="${h.url_for( action='view_snapshots', id=trans.security.encode_id(prevInstance.id) )}">View snapshots</a> - <a class="action-button" href="${h.url_for( action='add_storage', id=trans.security.encode_id(prevInstance.id) )}" target="_parent">Add storage</a> - <a class="action-button" confirm="Are you sure you want to delete instance '${prevInstance.name}'? This will delete all of your data assocaiated with this instance!" href="${h.url_for( action='delete_uci', id=trans.security.encode_id(prevInstance.id) )}">Delete</a> + <a class="action-button" confirm="Are you sure you want to delete instance '${prevInstance.name}'? This will delete all of your data assocaiated with this instance!" href="${h.url_for( action='delete_uci', id=trans.security.encode_id(prevInstance.id) )}">Delete</a> </div> </td> </tr>

1 0

[hg] galaxy 3136: Added some error checking to pileup_parser too...
by Greg Von Kuster 02 Dec '09

02 Dec '09

details: http://www.bx.psu.edu/hg/galaxy/rev/fe853d8e062d changeset: 3136:fe853d8e062d user: Kelly Vincent <kpvincent(a)bx.psu.edu> date: Tue Dec 01 16:18:25 2009 -0500 description: Added some error checking to pileup_parser tool so that it exits with a meaningful message if the user tries to do some 6-column work on a 10-column file, or vice versa. diffstat: tools/samtools/pileup_interval.py | 16 ++++++++++++++-- 1 files changed, 14 insertions(+), 2 deletions(-) diffs (40 lines): diff -r eb317734e391 -r fe853d8e062d tools/samtools/pileup_interval.py --- a/tools/samtools/pileup_interval.py Tue Dec 01 11:49:07 2009 -0500 +++ b/tools/samtools/pileup_interval.py Tue Dec 01 16:18:25 2009 -0500 @@ -1,7 +1,7 @@ #! /usr/bin/python """ -Creates a pileup file from a bam file and a reference. +Condenses pileup format into ranges of bases. usage: %prog [options] -i, --input=i: Input pileup file @@ -18,6 +18,7 @@ from galaxy import eggs import pkg_resources; pkg_resources.require( "bx-python" ) from bx.cookbook import doc_optparse +import sys def stop_err( msg ): sys.stderr.write( msg ) @@ -56,7 +57,18 @@ bases = [] while inLine.strip() != '': lineParts = inLine.split('\t') - seq, loc, base, cov = lineParts[seqIndex], int(lineParts[locIndex]), lineParts[baseIndex], int(lineParts[covIndex]) + try: + seq, loc, base, cov = lineParts[seqIndex], int(lineParts[locIndex]), lineParts[baseIndex], int(lineParts[covIndex]) + except IndexError, ei: + if options.format == 'ten': + stop_err( 'It appears that you have selected 10 columns while your file has 6. Make sure that the number of columns you specify matches the number in your file.\n' + str( ei ) ) + else: + stop_err( 'There appears to be something wrong with your column index values.\n' + str( ei ) ) + except ValueError, ev: + if options.format == 'six': + stop_err( 'It appears that you have selected 6 columns while your file has 10. Make sure that the number of columns you specify matches the number in your file.\n' + str( ev ) ) + else: + stop_err( 'There appears to be something wrong with your column index values.\n' + str( ev ) ) # strout += str(startLoc) + '\n' # strout += str(bases) + '\n' # strout += '%s\t%s\t%s\t%s\n' % (seq, loc, base, cov)

1 0

[hg] galaxy 3131: Added ability for a user to add additional sto...
by Greg Von Kuster 02 Dec '09

02 Dec '09

details: http://www.bx.psu.edu/hg/galaxy/rev/1d93eb88c93f changeset: 3131:1d93eb88c93f user: Enis Afgan <afgane(a)gmail.com> date: Mon Nov 30 16:36:38 2009 -0500 description: Added ability for a user to add additional storage volume(s) to a running EC2 instance. In this context, a new EBS volume is created, attached to a running instance, and then added to remote zpool. When an instance is shut down and then started back up, all of the associated storage volumes are automatically attached to it and the zpool is imported. diffstat: lib/galaxy/cloud/__init__.py | 100 +++++++++- lib/galaxy/cloud/providers/ec2.py | 300 +++++++++++++++++++++++++++----- lib/galaxy/cloud/providers/eucalyptus.py | 20 +- lib/galaxy/web/controllers/cloud.py | 93 +++++++++- templates/cloud/configure_cloud.mako | 19 +- 5 files changed, 445 insertions(+), 87 deletions(-) diffs (922 lines): diff -r f4654abcec1e -r 1d93eb88c93f lib/galaxy/cloud/__init__.py --- a/lib/galaxy/cloud/__init__.py Fri Nov 20 11:03:41 2009 -0500 +++ b/lib/galaxy/cloud/__init__.py Mon Nov 30 16:36:38 2009 -0500 @@ -29,6 +29,8 @@ SUBMITTED = "submitted", SHUTTING_DOWN_UCI = "shutting-downUCI", SHUTTING_DOWN = "shutting-down", + ADD_STORAGE_UCI = "add-storageUCI", + ADD_STORAGE = "add-storage", AVAILABLE = "available", RUNNING = "running", PENDING = "pending", @@ -40,6 +42,7 @@ TERMINATED = "terminated", SUBMITTED = "submitted", RUNNING = "running", + ADDING = "adding-storage", PENDING = "pending", SHUTTING_DOWN = "shutting-down", ERROR = "error" @@ -48,6 +51,7 @@ store_status = Bunch( WAITING = "waiting", IN_USE = "in-use", + ADDING = "adding", CREATING = "creating", DELETED = 'deleted', ERROR = "error" @@ -161,7 +165,8 @@ model.UCI.table.c.state==uci_states.SUBMITTED_UCI, model.UCI.table.c.state==uci_states.SHUTTING_DOWN_UCI, model.UCI.table.c.state==uci_states.DELETING_UCI, - model.UCI.table.c.state==uci_states.SNAPSHOT_UCI ) ) \ + model.UCI.table.c.state==uci_states.SNAPSHOT_UCI, + model.UCI.table.c.state==uci_states.ADD_STORAGE_UCI ) ) \ .all(): uci_wrapper = UCIwrapper( r, self.app ) new_requests.append( uci_wrapper ) @@ -195,6 +200,12 @@ self.uci_id = uci.id self.app = app self.sa_session = self.app.model.context + base_directory = os.path.join( self.app.config.job_working_directory, "cloud" ) + self.working_directory = os.path.join( base_directory, str( self.uci_id ) ) +# log.debug( "Cloud controller working directory for UCI DB ID '%s': '%s'" % ( self.uci_id, self.working_directory ) ) + if not os.path.exists( base_directory ): + os.mkdir( base_directory ) + # --------- Setter methods ----------------- @@ -364,11 +375,18 @@ self.sa_session.add( uci ) self.sa_session.flush() - def set_store_device( self, store_id, device ): +# def set_store_device( self, store_id, device ): +# uci = self.sa_session.query( model.UCI ).get( self.uci_id ) +# self.sa_session.refresh( uci ) +# uci.store[store_id].device = device +# uci.store[store_id].flush() + + def set_uci_total_size( self, total_size ): uci = self.sa_session.query( model.UCI ).get( self.uci_id ) self.sa_session.refresh( uci ) - uci.store[store_id].device = device - uci.store[store_id].flush() + uci.total_size = total_size + self.sa_session.add( uci ) + self.sa_session.flush() def set_store_error( self, error, store_index=None, store_id=None ): if store_index != None: @@ -405,15 +423,24 @@ def set_store_volume_id( self, store_index, volume_id ): """ - Given store index associated with this UCI in local database, set volume ID as it is registered + Given store index as it is stored in local database, set volume ID as it is registered on the cloud provider (e.g., vol-39890501) """ - uci = self.sa_session.query( model.UCI ).get( self.uci_id ) - self.sa_session.refresh( uci ) - uci.store[store_index].volume_id = volume_id - #uci.store[store_index].flush() - self.sa_session.add( uci ) - self.sa_session.flush() + + if store_index != None: + store = self.sa_session.query( model.CloudStore ).get( store_index ) + store.volume_id = volume_id + self.sa_session.add( store ) + self.sa_session.flush() + else: + return None + +# uci = self.sa_session.query( model.UCI ).get( self.uci_id ) +# self.sa_session.refresh( uci ) +# uci.store[store_index].volume_id = volume_id +# #uci.store[store_index].flush() +# self.sa_session.add( uci ) +# self.sa_session.flush() def set_store_instance( self, vol_id, instance_id ): """ @@ -421,7 +448,29 @@ be given in following format: 'vol-78943248' """ vol = self.sa_session.query( model.CloudStore ).filter( model.CloudStore.table.c.volume_id == vol_id ).first() - vol.inst.instance_id = instance_id + inst = self.sa_session.query( model.CloudInstance ).filter_by( instance_id=instance_id ).first() + vol.inst = inst + self.sa_session.add( vol ) + self.sa_session.flush() + + def set_store_device( self, vol_id, device ): + """ + Stores instance ID that given store volume is attached to. Store volume ID should + be given in following format: 'vol-78943248' + """ + vol = self.sa_session.query( model.CloudStore ).filter( model.CloudStore.table.c.volume_id == vol_id ).first() + vol.device = str( device ) + self.sa_session.add( vol ) + self.sa_session.flush() + + def set_store_deleted( self, vol_id, status=None ): + """ + Set storage volume as deleted in local database. Optionally, set the volume status too. + """ + vol = self.sa_session.query( model.CloudStore ).filter( model.CloudStore.table.c.volume_id == vol_id ).first() + vol.deleted = True + if status != None: + vol.status = status self.sa_session.add( vol ) self.sa_session.flush() @@ -503,6 +552,12 @@ self.sa_session.refresh( uci ) return uci.instance[instance_id].state + def get_instaces_in_state( self, state ): + """ Get database objects of all instances associated with this UCI in given state. """ + return self.sa_session.query( model.CloudInstance ) \ + .filter_by( uci_id=self.uci_id, state = state ) \ + .all() + def get_instances_ids( self ): """ Returns list IDs of all instances' associated with this UCI that are not in 'terminated' or @@ -590,6 +645,11 @@ self.sa_session.refresh( uci ) return uci.store[0].availability_zone + def get_uci_total_size( self ): + uci = self.sa_session.query( model.UCI ).get( self.uci_id ) + self.sa_session.refresh( uci ) + return uci.total_size + def get_store_size( self, store_id=0 ): uci = self.sa_session.query( model.UCI ).get( self.uci_id ) self.sa_session.refresh( uci ) @@ -604,9 +664,18 @@ self.sa_session.refresh( uci ) return uci.store[store_id].volume_id + def get_all_stores_in_status( self, status ): + """ + Return database objects of all stores associated with this UCI that have their + status set to value passed as parameter. + """ + return self.sa_session.query( model.CloudStore ).filter_by( deleted=False, uci_id=self.uci_id, status=status ).all() + def get_all_stores( self ): - """ Returns all storage volumes' database objects associated with this UCI. """ - return self.sa_session.query( model.CloudStore ).filter( model.CloudStore.table.c.uci_id == self.uci_id ).all() + """ Returns all storage volumes' database objects associated with this UCI that have not been marked as 'deleted'. """ + return self.sa_session.query( model.CloudStore ) \ + .filter_by( deleted=False, uci_id=self.uci_id ) \ + .all() def get_snapshots( self, status=None ): """ Returns database objects for all snapshots associated with this UCI and in given status.""" @@ -618,6 +687,9 @@ self.sa_session.refresh( uci ) return uci + def get_uci_working_directory( self ): + return self.working_directory + def uci_launch_time_set( self ): uci = self.sa_session.query( model.UCI ).get( self.uci_id ) self.sa_session.refresh( uci ) diff -r f4654abcec1e -r 1d93eb88c93f lib/galaxy/cloud/providers/ec2.py --- a/lib/galaxy/cloud/providers/ec2.py Fri Nov 20 11:03:41 2009 -0500 +++ b/lib/galaxy/cloud/providers/ec2.py Mon Nov 30 16:36:38 2009 -0500 @@ -1,4 +1,4 @@ -import subprocess, threading, os, errno, time, datetime +import subprocess, threading, os, errno, time, datetime, stat from Queue import Queue, Empty from datetime import datetime @@ -30,6 +30,8 @@ SUBMITTED = "submitted", SHUTTING_DOWN_UCI = "shutting-downUCI", SHUTTING_DOWN = "shutting-down", + ADD_STORAGE_UCI = "add-storageUCI", + ADD_STORAGE = "add-storage", AVAILABLE = "available", RUNNING = "running", PENDING = "pending", @@ -43,6 +45,7 @@ TERMINATED = "terminated", SUBMITTED = "submitted", RUNNING = "running", + ADDING = "adding-storage", PENDING = "pending", SHUTTING_DOWN = "shutting-down", ERROR = "error" @@ -51,6 +54,7 @@ store_status = Bunch( WAITING = "waiting", IN_USE = "in-use", + ADDING = "adding", CREATING = "creating", DELETED = 'deleted', ERROR = "error" @@ -122,6 +126,9 @@ self.stop_uci( uci_wrapper ) elif uci_state==uci_states.SNAPSHOT: self.snapshot_uci( uci_wrapper ) + elif uci_state==uci_states.ADD_STORAGE: + self.add_storage_to_uci( uci_wrapper ) + #self.dummy_start_uci( uci_wrapper ) except: log.exception( "Uncaught exception executing cloud request." ) cnt += 1 @@ -241,25 +248,14 @@ log.info( "Availability zone for UCI (i.e., storage volume) was not selected, using default zone: %s" % self.zone ) uci_wrapper.set_store_availability_zone( self.zone ) - log.info( "Creating volume in zone '%s'..." % uci_wrapper.get_uci_availability_zone() ) + store = uci_wrapper.get_all_stores_in_status( store_status.ADDING )[0] # Because at UCI creation time only 1 storage volume can be created, reference it directly + + log.info( "Creating storage volume in zone '%s' of size '%s'..." % ( uci_wrapper.get_uci_availability_zone(), store.size ) ) # Because only 1 storage volume may be created at UCI config time, index of this storage volume in local Galaxy DB w.r.t # current UCI is 0, so reference it in following methods - vol = conn.create_volume( uci_wrapper.get_store_size( 0 ), uci_wrapper.get_uci_availability_zone(), snapshot=None ) - uci_wrapper.set_store_volume_id( 0, vol.id ) - - # Wait for a while to ensure volume was created -# vol_status = vol.status -# for i in range( 30 ): -# if vol_status is not "available": -# log.debug( 'Updating volume status; current status: %s' % vol_status ) -# vol_status = vol.status -# time.sleep(3) -# if i is 29: -# log.debug( "Error while creating volume '%s'; stuck in state '%s'; deleting volume." % ( vol.id, vol_status ) ) -# conn.delete_volume( vol.id ) -# uci_wrapper.change_state( uci_state='error' ) -# return - + vol = conn.create_volume( store.size, uci_wrapper.get_uci_availability_zone(), snapshot=None ) + uci_wrapper.set_store_volume_id( store.id, vol.id ) + # Retrieve created volume again to get updated status try: vl = conn.get_all_volumes( [vol.id] ) @@ -306,6 +302,7 @@ if conn.delete_volume( v.volume_id ): deletedList.append( v.volume_id ) v.deleted = True + v.status = store_status.DELETED self.sa_session.add( v ) self.sa_session.flush() count += 1 @@ -358,15 +355,180 @@ uci_wrapper.change_state( uci_state=uci_states.AVAILABLE ) - def add_storage_to_uci( self, name ): - """ Adds more storage to specified UCI - TODO""" - + def add_storage_to_uci( self, uci_wrapper ): + """ + Add an additional storage volume to specified UCI by creating the storage volume + on cloud provider, attaching it to currently running instance and adding it to + 'galaxyData' zpool on remote instance. + """ + conn = self.get_connection( uci_wrapper ) + + stores = uci_wrapper.get_all_stores_in_status( store_status.ADDING ) + for store in stores: + vol_size = store.size + availability_zone = uci_wrapper.get_uci_availability_zone() + log.info( "Adding storage volume to UCI '%s' in zone '%s' of size '%s'..." % ( uci_wrapper.get_name(), availability_zone, vol_size ) ) + + try: + vol = conn.create_volume( vol_size, availability_zone, snapshot=None ) + uci_wrapper.set_store_volume_id( store.id, vol.id ) + uci_wrapper.set_store_availability_zone( availability_zone, vol.id ) + log.debug( "New storage volume created: '%s'" % vol.id ) + except boto.exception.EC2ResponseError, e: + err = "EC2 response error while creating storage volume: " + str( e ) + log.error( err ) + uci_wrapper.set_store_error( err, store_id=vol.id ) + uci_wrapper.set_error( err, True ) + return + except Exception, ex: + err = "Error while creating storage volume: " + str( ex ) + log.error( err ) + uci_wrapper.set_error( err, True ) + return + + # Retrieve created volume again to get updated status + try: + vl = conn.get_all_volumes( [vol.id] ) + except boto.exception.EC2ResponseError, e: + err = "EC2 response error while retrieving (i.e., updating status) of just created storage volume '" + vol.id + "': " + str( e ) + log.error( err ) + uci_wrapper.set_store_error( err, store_id=vol.id ) + uci_wrapper.set_error( err, True ) + return + except Exception, ex: + err = "Error while retrieving (i.e., updating status) of just created storage volume '" + vol.id + "': " + str( ex ) + log.error( err ) + uci_wrapper.set_error( err, True ) + return + + # Wait for a while to ensure volume was created + if len( vl ) > 0: + vol_status = vl[0].status # Bc. only single vol is queried, reference it as 0th list element + for i in range( 30 ): + if vol_status != "available": + log.debug( "(%s) Updating volume status; current status: '%s'" % (i, vol_status ) ) + uci_wrapper.change_state( uci_state=vol_status ) + time.sleep(5) + vol_status = vl[0].status + if vol_status == "available": + log.debug( "(%s) New volume status '%s', continuing with file system adjustment." % (i, vol_status ) ) + uci_wrapper.set_store_status( vl[0].id, vol_status ) + break + if i is 29: + err = "Error while creating volume '"+vl[0].id+"'; stuck in state '"+vol_status+"'; deleting volume." + conn.delete_volume( vl[0].id ) + log.error( err ) + uci_wrapper.set_error( err, True ) + uci_wrapper.set_store_error( err, store_id=vol.id ) + conn.delete_volume( vl[0].id ) + uci_wrapper.set_store_deleted( vl[0].id ) + return + else: + err = "Volume '" + vol.id +"' not found by EC2 after being created." + log.error( err ) + uci_wrapper.set_store_error( err, store_id=vol.id ) + uci_wrapper.set_error( err, True ) + return + + # Get private key for given instance + pk = uci_wrapper.get_key_pair_material() + if pk == None: #If pk does not exist, create it + self.check_key_pair( uci_wrapper, conn ) + pk = uci_wrapper.get_key_pair_material() + + # Get working directory for this UCI and store pk into a file + wd = uci_wrapper.get_uci_working_directory() + if not os.path.exists( wd ): + os.mkdir( wd ) + pk_file_path = os.path.join( wd, "pk" ) + + if pk != None: + # Save private key to a file + pk_file = open( pk_file_path, "w" ) + pk_file.write( pk ) + pk_file.close() + else: + err = "ERROR: Private key not available for this UCI." + log.error( err ) + uci_wrapper.set_store_error( err, store_id=vol.id ) + uci_wrapper.set_error( err, True ) + return + + if os.path.exists( pk_file_path ): + # Change permissions of the file - this is required by later used ssh + os.chmod( pk_file_path, stat.S_IRUSR | stat.S_IWUSR ) + + # Get # of storage volumes associated with this UCI to know as which device to connect new volume to the instance + device_num = len( uci_wrapper.get_all_stores_in_status( store_status.IN_USE ) ) + 5 # First device num is 5, so all subsequent ones should follow + + # Get instance that the new storage volume is to be attached to. Although a list is returned, + # only 1 instance can be in 'adding-storage' state (because, for now, only 1 instance is assoc. with + # each UCI) and volume can be attached to only to it + il = uci_wrapper.get_instaces_in_state( instance_states.ADDING ) + if len( il ) > 0: + # Attach new volume to the instance + log.debug( "Attaching new storage volume '%s' to UCI '%s' as device '%s'" % + ( vol.id, uci_wrapper.get_name(), device_num ) ) + try: + vol_status = conn.attach_volume( vol.id, il[0].instance_id, device_num ) + except boto.exception.EC2ResponseError, e: + err = "Attaching just created storage volume '" + vol.id + "'to instance '" + \ + il[0].instance_id + "' as device '" + str( device_num ) + "' failed: " + str( e ) + log.error( err ) + uci_wrapper.set_store_error( err, store_id=vol.id ) + uci_wrapper.set_error( err, True ) + return + # For a while, keep checking attachment status of the new volume + for i in range(30): + log.debug( "Checking attachment status of new volume '%s': '%s'" % ( vol.id, vol_status ) ) + if vol_status == 'attached': + uci_wrapper.set_store_status( vol.id, vol_status ) + uci_wrapper.set_store_device( vol.id, device_num ) + break + if i == 29: + err = "Storage volume '" + vol.id + "' failed to attach to instance '" + il[0].instance_id + \ + "'. Manual check needed." + log.error( err ) + uci_wrapper.set_store_error( err, store_id=vol.id ) + uci_wrapper.set_error( err, False ) + return + + time.sleep(4) + vol_list = conn.get_all_volumes( [vol.id] ) + for v in vol_list: + vol_status = v.attachment_state() + + # Once storage volume is attached, add it to the zpool by issuing system level command + cmd = 'ssh -o StrictHostKeyChecking=no -i '+ pk_file_path +' root@'+il[0].public_dns+' "zpool add galaxyData c7d' + str( device_num )+'"' + log.debug( "Adding new storage volume to zpool cmd: %s" % cmd ) + stdout = os.system( cmd ) + if stdout != 0: + err = "Adding newly created storage volume to zpool on instance '" + il[0].instance_id + \ + "' failed. Error code: " + str( stdout ) + log.error( err ) + uci_wrapper.set_store_error( err, store_id=vol.id ) + uci_wrapper.set_error( err, False ) + return + else: + err = "No instance(s) found in 'adding-storage' state. New disk not added to UCI's zpool." + log.error( err ) + uci_wrapper.set_store_error( err, store_id=vol.id ) + uci_wrapper.set_error( err, True ) + return + + # Update UCI's total storage size + uci_wrapper.set_uci_total_size( uci_wrapper.get_uci_total_size() + vol.size ) + # Reset UCI's and instance's state + uci_wrapper.change_state( uci_state=uci_states.RUNNING, instance_id=il[0].instance_id, i_state=instance_states.RUNNING ) + log.debug( "Successfully added storage volume '%s' to UCI '%s'." % ( vol.id, uci_wrapper.get_name() ) ) + def dummy_start_uci( self, uci_wrapper ): uci = uci_wrapper.get_uci() - log.debug( "Would be starting instance '%s'" % uci.name ) - uci_wrapper.change_state( uci_state.PENDING ) + log.debug( "Dummy start UCI '%s'" % uci.name ) + + +# uci_wrapper.change_state( uci_state.PENDING ) # log.debug( "Sleeping a bit... (%s)" % uci.name ) # time.sleep(20) # log.debug( "Woke up! (%s)" % uci.name ) @@ -418,10 +580,25 @@ if uci_wrapper.get_uci_state() != uci_states.ERROR: # Start an instance log.debug( "Starting instance for UCI '%s'" % uci_wrapper.get_name() ) - #TODO: Once multiple volumes can be attached to a single instance, update 'userdata' composition - userdata = uci_wrapper.get_store_volume_id()+"|"+uci_wrapper.get_access_key()+"|"+uci_wrapper.get_secret_key() + #TODO: Once multiple volumes can be attached to a single instance, update 'userdata' composition + # Compose user data; for storage volumes, separate multiple volumes with a colon (:) ensuring that + # the last volume in the list is not followed by a colon. + stores = uci_wrapper.get_all_stores() + volume_ids = "" + if len( stores ) > 0: + for i, store in enumerate( stores ): + volume_ids += store.volume_id + if i < len( stores )-1: + volume_ids += ":" + else: + err = "No storage volumes found that are associated with UCI '%s'" + uci_wrapper.get_name() + log.error( err ) + uci_wrapper.set_error( err, True ) + return + userdata = volume_ids+"|"+uci_wrapper.get_access_key()+"|"+uci_wrapper.get_secret_key() log.debug( "Using following command: conn.run_instances( image_id='%s', key_name='%s', security_groups=['%s'], user_data=[OMITTED], instance_type='%s', placement='%s' )" % ( mi_id, uci_wrapper.get_key_pair_name(), self.security_group, uci_wrapper.get_instance_type( i_index ), uci_wrapper.get_uci_availability_zone() ) ) + # Start an instance reservation = None try: reservation = conn.run_instances( image_id=mi_id, @@ -454,7 +631,10 @@ uci_wrapper.change_state( s, i_id, s ) uci_wrapper.set_security_group_name( self.security_group, i_id=i_id ) vol_id = uci_wrapper.get_store_volume_id( store_id=0 ) # TODO: Once more that one vol/UCI is allowed, update this! - uci_wrapper.set_store_status( vol_id, store_status.WAITING ) + # Following line is pointless bc. general update updates status of volume to 'available' + # before it actually connects to starting instance... This has been dealt w/ in general update method + #uci_wrapper.set_store_status( vol_id, store_status.WAITING ) + uci_wrapper.set_store_instance( vol_id, i_id ) log.debug( "Instance of UCI '%s' started, current state: '%s'" % ( uci_wrapper.get_name(), uci_wrapper.get_uci_state() ) ) except boto.exception.EC2ResponseError, e: err = "EC2 response error when retrieving instance information for UCI '" + uci_wrapper.get_name() + "': " + str( e ) @@ -571,19 +751,30 @@ .all() for inst in instances: if self.type == inst.uci.credentials.provider.type: - log.debug( "[%s] Running general status update on instance '%s'" % ( inst.uci.credentials.provider.type, inst.instance_id ) ) + log.debug( "[%s] Running general status update on instance '%s'" + % ( inst.uci.credentials.provider.type, inst.instance_id ) ) self.update_instance( inst ) + # Update storage volume(s) associated with current instance + stores = self.sa_session.query( model.CloudStore ) \ + .filter_by( uci_id=inst.uci_id, deleted=False ) \ + .all() + for store in stores: + if self.type == store.uci.credentials.provider.type: # and store.volume_id != None: + log.debug( "[%s] Running general status update on store with local database ID: '%s'" + % ( store.uci.credentials.provider.type, store.id ) ) + self.update_store( store ) # Update storage volume(s) stores = self.sa_session.query( model.CloudStore ) \ - .filter( or_( model.CloudStore.table.c.status==store_status.IN_USE, - model.CloudStore.table.c.status==store_status.CREATING, - model.CloudStore.table.c.status==store_status.WAITING, + .filter( or_( model.CloudStore.table.c.status==store_status.CREATING, +# model.CloudStore.table.c.status==store_status.IN_USE, +# model.CloudStore.table.c.status==store_status.WAITING, model.CloudStore.table.c.status==None ) ) \ .all() for store in stores: if self.type == store.uci.credentials.provider.type: # and store.volume_id != None: - log.debug( "[%s] Running general status update on store with local database ID: '%s'" % ( store.uci.credentials.provider.type, store.id ) ) + log.debug( "[%s] Running general status update on store with local database ID: '%s'" + % ( store.uci.credentials.provider.type, store.id ) ) self.update_store( store ) # else: # log.error( "[%s] There exists an entry for UCI (%s) storage volume without an ID. Storage volume might have been created with " @@ -668,7 +859,8 @@ for i, cInst in enumerate( r.instances ): try: s = cInst.update() - log.debug( "Checking state of cloud instance '%s' associated with UCI '%s' and reservation '%s'. State='%s'" % ( cInst, uci.name, r, s ) ) + log.debug( "Checking state of cloud instance '%s' associated with UCI '%s' " \ + "and reservation '%s'. State='%s'" % ( cInst, uci.name, r, s ) ) if s != inst.state: inst.state = s self.sa_session.add( inst ) @@ -693,7 +885,8 @@ self.sa_session.add( inst ) self.sa_session.flush() except boto.exception.EC2ResponseError, e: - err = "Updating instance status from cloud failed for UCI '"+ uci.name + "' during general status update: " + str( e ) + err = "Updating instance status from cloud failed for UCI '"+ uci.name + \ + "' during general status update: " + str( e ) log.error( err ) uci.error = err uci.state = uci_states.ERROR @@ -715,7 +908,7 @@ # Get reservations handle for given store try: - log.debug( "Updating storage volume command: vl = conn.get_all_volumes( [%s] )" % store.volume_id ) + log.debug( "Retrieving reference to storage volume '%s' during update..." % store.volume_id ) vl = conn.get_all_volumes( [store.volume_id] ) except boto.exception.EC2ResponseError, e: err = "Retrieving volume(s) from cloud failed for UCI '"+ uci.name + "' during general status update: " + str( e ) @@ -729,7 +922,7 @@ # Update store status in local DB with info from cloud provider if len(vl) > 0: try: - log.debug( "Storage volume '%s' current status: '%s'" % (store.volume_id, vl[0].status ) ) + log.debug( "General status update for storage volume '%s'; current status: '%s'" % (store.volume_id, vl[0].status ) ) if store.status != vl[0].status: # In case something failed during creation of UCI but actual storage volume was created and yet # UCI state remained as 'new', try to remedy this by updating UCI state here @@ -746,21 +939,26 @@ store.status = vl[0].status self.sa_session.add( store ) self.sa_session.flush() - if store.inst != None: - if store.inst.instance_id != vl[0].instance_id: - store.inst.instance_id = vl[0].instance_id - self.sa_session.add( store ) - self.sa_session.flush() - if store.attach_time != vl[0].attach_time: - store.attach_time = vl[0].attach_time - self.sa_session.add( store ) - self.sa_session.flush() - if store.device != vl[0].device: - store.device = vl[0].device - self.sa_session.add( store ) - self.sa_session.flush() + # Boto does not seem to be reporting these values although fields exist so comment them out... +# log.debug( "vl[0].instance_id: '%s'" % vl[0].instance_id ) +# if store.inst != None: +# if store.inst.instance_id != vl[0].instance_id: +# store.inst.instance_id = vl[0].instance_id +# self.sa_session.add( store ) +# self.sa_session.flush() +# log.debug( "vl[0].attach_time: '%s'" % vl[0].attach_time ) +# if store.attach_time != vl[0].attach_time: +# store.attach_time = vl[0].attach_time +# self.sa_session.add( store ) +# self.sa_session.flush() +## log.debug( "vl[0].device: '%s'" % vl[0].device ) +# if store.device != vl[0].device: +# store.device = vl[0].device +# self.sa_session.add( store ) +# self.sa_session.flush() except boto.exception.EC2ResponseError, e: - err = "Updating status of volume(s) from cloud failed for UCI '"+ uci.name + "' during general status update: " + str( e ) + err = "Updating status of volume(s) from cloud failed for UCI '"+ uci.name + \ + "' during general status update: " + str( e ) log.error( err ) uci.error = err uci.state = uci_states.ERROR @@ -768,7 +966,7 @@ self.sa_session.flush() return None else: - err = "No storage volumes returned by cloud provider on general update" + err = "No storage volumes returned by cloud provider on general update for volume with id: " + store.volume_id log.error( "%s for UCI '%s'" % ( err, uci.name ) ) store.status = store_status.ERROR store.error = err diff -r f4654abcec1e -r 1d93eb88c93f lib/galaxy/cloud/providers/eucalyptus.py --- a/lib/galaxy/cloud/providers/eucalyptus.py Fri Nov 20 11:03:41 2009 -0500 +++ b/lib/galaxy/cloud/providers/eucalyptus.py Mon Nov 30 16:36:38 2009 -0500 @@ -30,6 +30,8 @@ SUBMITTED = "submitted", SHUTTING_DOWN_UCI = "shutting-downUCI", SHUTTING_DOWN = "shutting-down", + ADD_STORAGE_UCI = "add-storageUCI", + ADD_STORAGE = "add-storage", AVAILABLE = "available", RUNNING = "running", PENDING = "pending", @@ -43,6 +45,7 @@ TERMINATED = "terminated", SUBMITTED = "submitted", RUNNING = "running", + ADDING = "adding-storage", PENDING = "pending", SHUTTING_DOWN = "shutting-down", ERROR = "error" @@ -51,6 +54,7 @@ store_status = Bunch( WAITING = "waiting", IN_USE = "in-use", + ADDING = "adding", CREATING = "creating", DELETED = 'deleted', ERROR = "error" @@ -121,6 +125,8 @@ self.stop_uci( uci_wrapper ) elif uci_state==uci_states.SNAPSHOT: self.snapshot_uci( uci_wrapper ) + elif uci_state==uci_states.ADD_STORAGE: + self.add_storage_to_uci( uci_wrapper ) except: log.exception( "Uncaught exception executing cloud request." ) cnt += 1 @@ -245,9 +251,16 @@ log.info( "Availability zone for UCI (i.e., storage volume) was not selected, using default zone: %s" % self.zone ) uci_wrapper.set_store_availability_zone( self.zone ) - log.debug( "Creating volume; using command: conn.create_volume( %s, '%s', snapshot=None )" % ( uci_wrapper.get_store_size( 0 ), uci_wrapper.get_uci_availability_zone() )) - vol = conn.create_volume( uci_wrapper.get_store_size( 0 ), uci_wrapper.get_uci_availability_zone(), snapshot=None ) - uci_wrapper.set_store_volume_id( 0, vol.id ) +# log.debug( "Creating volume; using command: conn.create_volume( %s, '%s', snapshot=None )" % ( uci_wrapper.get_store_size( 0 ), uci_wrapper.get_uci_availability_zone() )) +# vol = conn.create_volume( uci_wrapper.get_store_size( 0 ), uci_wrapper.get_uci_availability_zone(), snapshot=None ) +# uci_wrapper.set_store_volume_id( 0, vol.id ) + store = uci_wrapper.get_all_stores_in_status( store_status.ADDING )[0] # Because at UCI creation time only 1 storage volume can be created, reference it directly + + log.info( "Creating storage volume in zone '%s' of size '%s'..." % ( uci_wrapper.get_uci_availability_zone(), store.size ) ) + # Because only 1 storage volume may be created at UCI config time, index of this storage volume in local Galaxy DB w.r.t + # current UCI is 0, so reference it in following methods + vol = conn.create_volume( store.size, uci_wrapper.get_uci_availability_zone(), snapshot=None ) + uci_wrapper.set_store_volume_id( store.id, vol.id ) # Retrieve created volume again to get updated status try: @@ -364,6 +377,7 @@ def add_storage_to_uci( self, uci_wrapper ): """ Adds more storage to specified UCI """ + uci_wrapper.set_error( "Adding storage to eucalyptus-based clouds is not yet supported.", True ) def dummy_start_uci( self, uci_wrapper ): diff -r f4654abcec1e -r 1d93eb88c93f lib/galaxy/web/controllers/cloud.py --- a/lib/galaxy/web/controllers/cloud.py Fri Nov 20 11:03:41 2009 -0500 +++ b/lib/galaxy/web/controllers/cloud.py Mon Nov 30 16:36:38 2009 -0500 @@ -41,6 +41,8 @@ SUBMITTED = "submitted", SHUTTING_DOWN_UCI = "shutting-downUCI", SHUTTING_DOWN = "shutting-down", + ADD_STORAGE_UCI = "add-storageUCI", + ADD_STORAGE = "add-storage", AVAILABLE = "available", RUNNING = "running", PENDING = "pending", @@ -54,13 +56,16 @@ TERMINATED = "terminated", SUBMITTED = "submitted", RUNNING = "running", + ADDING = "adding-storage", PENDING = "pending", SHUTTING_DOWN = "shutting-down", ERROR = "error" ) store_status = Bunch( + WAITING = "waiting", IN_USE = "in-use", + ADDING = "adding", CREATING = "creating", DELETED = 'deleted', ERROR = "error" @@ -108,7 +113,9 @@ model.UCI.table.c.state==uci_states.SUBMITTED, model.UCI.table.c.state==uci_states.SUBMITTED_UCI, model.UCI.table.c.state==uci_states.SHUTTING_DOWN, - model.UCI.table.c.state==uci_states.SHUTTING_DOWN_UCI ) ) \ + model.UCI.table.c.state==uci_states.SHUTTING_DOWN_UCI, + model.UCI.table.c.state==uci_states.ADD_STORAGE, + model.UCI.table.c.state==uci_states.ADD_STORAGE_UCI ) ) \ .order_by( desc( model.UCI.table.c.update_time ) ) \ .all() @@ -201,7 +208,8 @@ storage.user = user storage.uci = uci storage.size = volSize - storage.availability_zone = zone + storage.availability_zone = zone + storage.status = store_status.ADDING # Persist session = trans.sa_session session.add( uci ) @@ -306,6 +314,8 @@ ( uci.state != uci_states.ERROR ) and \ ( uci.state != uci_states.SHUTTING_DOWN_UCI ) and \ ( uci.state != uci_states.SHUTTING_DOWN ) and \ + ( uci.state != uci_states.ADD_STORAGE_UCI ) and \ + ( uci.state != uci_states.ADD_STORAGE ) and \ ( uci.state != uci_states.AVAILABLE ): uci.state = uci_states.SHUTTING_DOWN_UCI session = trans.sa_session @@ -496,13 +506,53 @@ @web.expose @web.require_login( "add instance storage" ) - def add_storage( self, trans, id ): - instance = get_uci( trans, id ) + def add_storage( self, trans, id, vol_size=None ): + error = None + uci = get_uci( trans, id ) + stores = get_stores_in_status( trans, uci, store_status.IN_USE ) + # Start adding of storage making sure given UCI is running and that at least one + # storage volume is attached to it (this is needed to by cloud controller to know + # as which device to attach the new storage volume) + if uci.state == uci_states.RUNNING and len( stores ) > 0: + if vol_size is not None: + try: + vol_size = int( vol_size ) + except ValueError: + error = "Volume size must be integer value between 1 and 1000." + + if not error: + user = trans.get_user() + + storage = model.CloudStore() + storage.user = user + storage.uci = uci + storage.size = vol_size + storage.status = store_status.ADDING + + # Set state of instance - NOTE that this code will only work (with code in cloud controller) + # for scenario where a UCI is associated with *1* compute instance!!! + instances = get_instances( trans, uci ) + instances.state = instance_states.ADDING + + uci.state = uci_states.ADD_STORAGE_UCI + # Persist + session = trans.sa_session + session.add( instances ) + session.add( storage ) + session.add( uci ) + session.flush() + # Log and display the management page + trans.log_event( "User added storage volume to UCI: '%s'" % uci.name ) + trans.set_message( "Adding of storage to instance '%s' initiated." % uci.name ) + return self.list( trans ) + else: + error( "Storage can only be added to instances that are in state 'RUNNING' with existing " \ + "storage volume(s) already attached." ) - error( "Adding storage to instance '%s' is not supported yet." % instance.name ) - - return self.list( trans ) + return trans.show_form( + web.FormBuilder( url_for( id=trans.security.encode_id(uci.id) ), "Add storage to an instance", submit_text="Add" ) + .add_text( "vol_size", "Storage size (1-1000 GB)", value='', error=error ) ) # ----- Image methods ----- @web.expose @@ -1034,6 +1084,7 @@ dict = {} dict['id'] = uci.id dict['state'] = uci.state + dict['total_size'] = uci.total_size if uci.error != None: dict['error'] = str( uci.error ) else: @@ -1136,19 +1187,30 @@ def get_stores( trans, uci ): """ - Get stores objects that are connected to uci object + Get stores objects that are associated with given uci and are not in 'error' status """ user = trans.get_user() stores = trans.sa_session.query( model.CloudStore ) \ - .filter_by( user=user, uci_id=uci.id ) \ + .filter_by( user=user, uci_id=uci.id, deleted=False ) \ .filter( model.CloudStore.table.c.status != store_status.ERROR ) \ .all() return stores +def get_stores_in_status( trans, uci, status ): + """ + Get stores objects that are associated with given uci and are not have given status + """ + user = trans.get_user() + stores = trans.sa_session.query( model.CloudStore ) \ + .filter_by( user=user, uci_id=uci.id, status=status ) \ + .all() + + return stores + def get_instances( trans, uci ): """ - Get objects of instances that are pending or running and are connected to uci object + Get objects of instances that are pending or running and are connected to the given uci object """ user = trans.get_user() instances = trans.sa_session.query( model.CloudInstance ) \ @@ -1159,6 +1221,17 @@ return instances +def get_instances_in_state( trans, uci, state ): + """ + Get objects of instances that are in specified state and are connected to the given uci object + """ + user = trans.get_user() + instances = trans.sa_session.query( model.CloudInstance ) \ + .filter_by( user=user, uci_id=uci.id, state=state ) \ + .all() + + return instances + def get_connection( trans, creds ): """ Establishes cloud connection using user's credentials diff -r f4654abcec1e -r 1d93eb88c93f templates/cloud/configure_cloud.mako --- a/templates/cloud/configure_cloud.mako Fri Nov 20 11:03:41 2009 -0500 +++ b/templates/cloud/configure_cloud.mako Mon Nov 30 16:36:38 2009 -0500 @@ -79,7 +79,8 @@ }); } - // Update 'state' and 'time alive' fields + // Update 'size', 'state' and 'time alive' fields + $(elem + "-size").text( data[i].total_size ); $(elem + "-state").text( data[i].state ); if ( new_state != 'error' ) { // Because 'error' state is handled as a JS link, don't include it in update $(elem + "-state-p").text( data[i].state ); @@ -219,7 +220,7 @@ ${liveInstance.name} (${liveInstance.credentials.name}) <a id="li-${i}-popup" class="popup-arrow" style="display: none;">▼</a> </td> - <td>${str(liveInstance.total_size)}</td> + <td id="${ liveInstance.id }-size">${str(liveInstance.total_size)}</td> <td id="${ liveInstance.id }-state">${str(liveInstance.state)}</td> <td id="${ liveInstance.id }-launch_time"> ##${str(liveInstance.launch_time)[:16]} @@ -251,7 +252,8 @@ <a class="action-button" confirm="Are you sure you want to stop instance '${liveInstance.name}'?" href="${h.url_for( action='stop', id=trans.security.encode_id(liveInstance.id) )}">Stop</a> <a class="action-button" href="${h.url_for( action='rename_uci', id=trans.security.encode_id(liveInstance.id) )}">Rename</a> <a class="action-button" href="${h.url_for( action='view_uci_details', id=trans.security.encode_id(liveInstance.id) )}">View details</a> - <a class="action-button" href="${h.url_for( action='uci_usage_report', id=trans.security.encode_id(liveInstance.id) )}">Usage report</a> + <a class="action-button" href="${h.url_for( action='add_storage', id=trans.security.encode_id(liveInstance.id) )}">Add storage</a> + <a class="action-button" href="${h.url_for( action='uci_usage_report', id=trans.security.encode_id(liveInstance.id) )}">Usage report</a> </div> </td> </tr> @@ -315,14 +317,13 @@ </td> <td> <div popupmenu="pi-${i}-popup"> - <a class="action-button" href="${h.url_for( action='start', id=trans.security.encode_id(prevInstance.id), type='m1.small' )}"> Start m1.small</a> + <a class="action-button" href="${h.url_for( action='uci_usage_report', id=trans.security.encode_id(prevInstance.id) )}">Usage report</a> + <a class="action-button" href="${h.url_for( action='rename_uci', id=trans.security.encode_id(prevInstance.id) )}">Rename</a> + <a class="action-button" href="${h.url_for( action='start', id=trans.security.encode_id(prevInstance.id), type='m1.small' )}"> Start m1.small</a> <a class="action-button" href="${h.url_for( action='start', id=trans.security.encode_id(prevInstance.id), type='c1.medium' )}"> Start c1.medium</a> - <a class="action-button" href="${h.url_for( action='rename_uci', id=trans.security.encode_id(prevInstance.id) )}">Rename</a> - <a class="action-button" href="${h.url_for( action='uci_usage_report', id=trans.security.encode_id(prevInstance.id) )}">Usage report</a> - <a class="action-button" href="${h.url_for( action='create_snapshot', id=trans.security.encode_id(prevInstance.id) )}">Create snapshot</a> + <a class="action-button" href="${h.url_for( action='create_snapshot', id=trans.security.encode_id(prevInstance.id) )}">Create snapshot</a> <a class="action-button" href="${h.url_for( action='view_snapshots', id=trans.security.encode_id(prevInstance.id) )}">View snapshots</a> - <a class="action-button" href="${h.url_for( action='add_storage', id=trans.security.encode_id(prevInstance.id) )}" target="_parent">Add storage</a> - <a class="action-button" confirm="Are you sure you want to delete instance '${prevInstance.name}'? This will delete all of your data assocaiated with this instance!" href="${h.url_for( action='delete_uci', id=trans.security.encode_id(prevInstance.id) )}">Delete</a> + <a class="action-button" confirm="Are you sure you want to delete instance '${prevInstance.name}'? This will delete all of your data assocaiated with this instance!" href="${h.url_for( action='delete_uci', id=trans.security.encode_id(prevInstance.id) )}">Delete</a> </div> </td> </tr>

1 0

[hg] galaxy 3128: When setting metadata externally, substitute a...
by Greg Von Kuster 02 Dec '09

02 Dec '09

details: http://www.bx.psu.edu/hg/galaxy/rev/92d6a9796b24 changeset: 3128:92d6a9796b24 user: Dan Blankenberg <dan(a)bx.psu.edu> date: Mon Nov 30 14:55:30 2009 -0500 description: When setting metadata externally, substitute a MetadataTempFile object for all populated metadata FileParameters. Fixes the problem with writting to metadata FileParameter files on cluster nodes with read-only database/files directory. diffstat: lib/galaxy/datatypes/metadata.py | 21 +++- lib/galaxy/model/mapping.py | 1 + lib/galaxy/model/migrate/versions/0028_external_metadata_file_override.py | 47 +++++++++ scripts/set_metadata.py | 23 ++++- tools/regVariation/quality_filter.xml | 2 +- 5 files changed, 87 insertions(+), 7 deletions(-) diffs (172 lines): diff -r 9f7a2a3be0e7 -r 92d6a9796b24 lib/galaxy/datatypes/metadata.py --- a/lib/galaxy/datatypes/metadata.py Mon Nov 30 11:59:54 2009 -0500 +++ b/lib/galaxy/datatypes/metadata.py Mon Nov 30 14:55:30 2009 -0500 @@ -429,7 +429,9 @@ if MetadataTempFile.is_JSONified_value( value ): value = MetadataTempFile.from_JSON( value ) if isinstance( value, MetadataTempFile ): - mf = self.new_file( dataset = parent, **value.kwds ) + mf = parent.metadata.get( self.spec.name, None) + if mf is None: + mf = self.new_file( dataset = parent, **value.kwds ) shutil.move( value.file_name, mf.file_name ) value = mf.id return value @@ -521,7 +523,7 @@ if dataset_path.false_path and dataset_path.real_path == metadata_files.dataset.file_name: return dataset_path.false_path return "" - return "%s,%s,%s,%s,%s" % ( metadata_files.filename_in, metadata_files.filename_kwds, metadata_files.filename_out, metadata_files.filename_results_code, __get_filename_override() ) + return "%s,%s,%s,%s,%s,%s" % ( metadata_files.filename_in, metadata_files.filename_kwds, metadata_files.filename_out, metadata_files.filename_results_code, __get_filename_override(), metadata_files.filename_override_metadata ) if not isinstance( datasets, list ): datasets = [ datasets ] if exec_dir is None: @@ -558,11 +560,22 @@ open( metadata_files.filename_out, 'wb+' ) # create the file on disk, so it cannot be reused by tempfile (unlikely, but possible) #file to store a 'return code' indicating the results of the set_meta() call #results code is like (True/False - if setting metadata was successful/failed , exception or string of reason of success/failure ) - metadata_files.filename_results_code = relpath( tempfile.NamedTemporaryFile( dir = tmp_dir, prefix = "metadata_out_%s_" % key ).name ) + metadata_files.filename_results_code = relpath( tempfile.NamedTemporaryFile( dir = tmp_dir, prefix = "metadata_results_%s_" % key ).name ) simplejson.dump( ( False, 'External set_meta() not called' ), open( metadata_files.filename_results_code, 'wb+' ) ) # create the file on disk, so it cannot be reused by tempfile (unlikely, but possible) #file to store kwds passed to set_meta() metadata_files.filename_kwds = relpath( tempfile.NamedTemporaryFile( dir = tmp_dir, prefix = "metadata_kwds_%s_" % key ).name ) simplejson.dump( kwds, open( metadata_files.filename_kwds, 'wb+' ), ensure_ascii=True ) + #existing metadata file parameters need to be overridden with cluster-writable file locations + metadata_files.filename_override_metadata = relpath( tempfile.NamedTemporaryFile( dir = tmp_dir, prefix = "metadata_override_%s_" % key ).name ) + open( metadata_files.filename_override_metadata, 'wb+' ) # create the file on disk, so it cannot be reused by tempfile (unlikely, but possible) + override_metadata = [] + for meta_key, spec_value in dataset.metadata.spec.iteritems(): + if isinstance( spec_value.param, FileParameter ) and dataset.metadata.get( meta_key, None ) is not None: + metadata_temp = MetadataTempFile() + shutil.copy( dataset.metadata.get( meta_key, None ).file_name, metadata_temp.file_name ) + override_metadata.append( ( meta_key, metadata_temp.to_JSON() ) ) + simplejson.dump( override_metadata, open( metadata_files.filename_override_metadata, 'wb+' ) ) + #add to session and flush sa_session.add( metadata_files ) sa_session.flush() metadata_files_list.append( metadata_files ) @@ -585,7 +598,7 @@ #can occur if the job was stopped before completion, but a MetadataTempFile is used in the set_meta MetadataTempFile.cleanup_from_JSON_dict_filename( metadata_files.filename_out ) dataset_key = self.get_dataset_metadata_key( metadata_files.dataset ) - for key, fname in [ ( 'filename_in', metadata_files.filename_in ), ( 'filename_out', metadata_files.filename_out ), ( 'filename_results_code', metadata_files.filename_results_code ), ( 'filename_kwds', metadata_files.filename_kwds ) ]: + for key, fname in [ ( 'filename_in', metadata_files.filename_in ), ( 'filename_out', metadata_files.filename_out ), ( 'filename_results_code', metadata_files.filename_results_code ), ( 'filename_kwds', metadata_files.filename_kwds ), ( 'filename_override_metadata', metadata_files.filename_override_metadata ) ]: try: os.remove( fname ) except Exception, e: diff -r 9f7a2a3be0e7 -r 92d6a9796b24 lib/galaxy/model/mapping.py --- a/lib/galaxy/model/mapping.py Mon Nov 30 11:59:54 2009 -0500 +++ b/lib/galaxy/model/mapping.py Mon Nov 30 14:55:30 2009 -0500 @@ -358,6 +358,7 @@ Column( "filename_out", String( 255 ) ), Column( "filename_results_code", String( 255 ) ), Column( "filename_kwds", String( 255 ) ), + Column( "filename_override_metadata", String( 255 ) ), Column( "job_runner_external_pid", String( 255 ) ) ) Event.table = Table( "event", metadata, diff -r 9f7a2a3be0e7 -r 92d6a9796b24 lib/galaxy/model/migrate/versions/0028_external_metadata_file_override.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/lib/galaxy/model/migrate/versions/0028_external_metadata_file_override.py Mon Nov 30 14:55:30 2009 -0500 @@ -0,0 +1,47 @@ +""" +This script adds the filename_override_metadata column to the JobExternalOutputMetadata table, +allowing existing metadata files to be written when using external metadata and a cluster +set up with read-only access to database/files +""" +from sqlalchemy import * +from sqlalchemy.orm import * +from sqlalchemy.exceptions import * +from migrate import * +from migrate.changeset import * +import datetime +now = datetime.datetime.utcnow +import sys, logging +# Need our custom types, but don't import anything else from model +from galaxy.model.custom_types import * + +log = logging.getLogger( __name__ ) +log.setLevel(logging.DEBUG) +handler = logging.StreamHandler( sys.stdout ) +format = "%(name)s %(levelname)s %(asctime)s %(message)s" +formatter = logging.Formatter( format ) +handler.setFormatter( formatter ) +log.addHandler( handler ) + +metadata = MetaData( migrate_engine ) +db_session = scoped_session( sessionmaker( bind=migrate_engine, autoflush=False, autocommit=True ) ) + +def display_migration_details(): + print "========================================" + print "This script adds the filename_override_metadata column to the JobExternalOutputMetadata table," + print" allowing existing metadata files to be written when using external metadata and a cluster" + print "set up with read-only access to database/files" + print "========================================" +def upgrade(): + display_migration_details() + # Load existing tables + metadata.reflect() + try: + job_external_output_metadata = Table( "job_external_output_metadata", metadata, autoload=True ) + col = Column( "filename_override_metadata", String( 255 ) ) + col.create( job_external_output_metadata ) + assert col is job_external_output_metadata.c.filename_override_metadata + except Exception, e: + log.debug( "Adding column 'filename_override_metadata' to job_external_output_metadata table failed: %s" % ( str( e ) ) ) + +def downgrade(): + pass diff -r 9f7a2a3be0e7 -r 92d6a9796b24 scripts/set_metadata.py --- a/scripts/set_metadata.py Mon Nov 30 11:59:54 2009 -0500 +++ b/scripts/set_metadata.py Mon Nov 30 14:55:30 2009 -0500 @@ -50,17 +50,36 @@ except: continue for filenames in sys.argv[1:]: - filename_in, filename_kwds, filename_out, filename_results_code, dataset_filename_override = filenames.split( ',' ) + fields = filenames.split( ',' ) + filename_in = fields.pop( 0 ) + filename_kwds = fields.pop( 0 ) + filename_out = fields.pop( 0 ) + filename_results_code = fields.pop( 0 ) + dataset_filename_override = fields.pop( 0 ) + #Need to be careful with the way that these parameters are populated from the filename splitting, + #because if a job is running when the server is updated, any existing external metadata command-lines + #will not have info about the newly added override_metadata file + if fields: + override_metadata = fields.pop( 0 ) + else: + override_metadata = None try: dataset = cPickle.load( open( filename_in ) ) #load DatasetInstance if dataset_filename_override: dataset.dataset.external_filename = dataset_filename_override if ext_override.get( dataset.dataset.id, None ): dataset.extension = ext_override[ dataset.dataset.id ] + #Metadata FileParameter types may not be writable on a cluster node, and are therefore temporarily substituted with MetadataTempFiles + if override_metadata: + override_metadata = simplejson.load( open( override_metadata ) ) + for metadata_name, metadata_file_override in override_metadata: + if galaxy.datatypes.metadata.MetadataTempFile.is_JSONified_value( metadata_file_override ): + metadata_file_override = galaxy.datatypes.metadata.MetadataTempFile.from_JSON( metadata_file_override ) + setattr( dataset.metadata, metadata_name, metadata_file_override ) kwds = stringify_dictionary_keys( simplejson.load( open( filename_kwds ) ) )#load kwds; need to ensure our keywords are not unicode dataset.datatype.set_meta( dataset, **kwds ) dataset.metadata.to_JSON_dict( filename_out ) # write out results of set_meta - simplejson.dump( ( True, 'Metadata has been set successfully' ), open( filename_results_code, 'wb+' ) ) #setting metadata has suceeded + simplejson.dump( ( True, 'Metadata has been set successfully' ), open( filename_results_code, 'wb+' ) ) #setting metadata has succeeded except Exception, e: simplejson.dump( ( False, str( e ) ), open( filename_results_code, 'wb+' ) ) #setting metadata has failed somehow clear_mappers() diff -r 9f7a2a3be0e7 -r 92d6a9796b24 tools/regVariation/quality_filter.xml --- a/tools/regVariation/quality_filter.xml Mon Nov 30 11:59:54 2009 -0500 +++ b/tools/regVariation/quality_filter.xml Mon Nov 30 14:55:30 2009 -0500 @@ -61,7 +61,7 @@ </conditional> </inputs> <outputs> - <data format="maf" name="out_file1"/> + <data format="maf" name="out_file1" metadata_source="input"/> </outputs> <requirements> <requirement type="python-module">numpy</requirement>

1 0

[hg] galaxy 3129: Fix MAF to BED tool. This tool should be rewri...
by Greg Von Kuster 02 Dec '09

02 Dec '09

details: http://www.bx.psu.edu/hg/galaxy/rev/9d4945bbdcf5 changeset: 3129:9d4945bbdcf5 user: Dan Blankenberg <dan(a)bx.psu.edu> date: Mon Nov 30 15:33:45 2009 -0500 description: Fix MAF to BED tool. This tool should be rewritten to not use a code file, but instead use one of the standard ways to create additional outputs, similar to how maf_to_interval functions. diffstat: tools/maf/maf_to_bed.py | 3 +-- tools/maf/maf_to_bed.xml | 4 ++-- tools/maf/maf_to_bed_code.py | 1 + 3 files changed, 4 insertions(+), 4 deletions(-) diffs (36 lines): diff -r 92d6a9796b24 -r 9d4945bbdcf5 tools/maf/maf_to_bed.py --- a/tools/maf/maf_to_bed.py Mon Nov 30 14:55:30 2009 -0500 +++ b/tools/maf/maf_to_bed.py Mon Nov 30 15:33:45 2009 -0500 @@ -15,8 +15,7 @@ input_filename = sys.argv[1] output_filename = sys.argv[2] #where to store files that become additional output - database_tmp_dir = './database/tmp' #os.path.join(os.path.split(os.path.split(os.path.realpath(output_filename))[0])[0],'tmp') #database/tmp - + database_tmp_dir = sys.argv[5] species = sys.argv[3].split(',') partial = sys.argv[4] diff -r 92d6a9796b24 -r 9d4945bbdcf5 tools/maf/maf_to_bed.xml --- a/tools/maf/maf_to_bed.xml Mon Nov 30 14:55:30 2009 -0500 +++ b/tools/maf/maf_to_bed.xml Mon Nov 30 15:33:45 2009 -0500 @@ -1,6 +1,6 @@ -<tool id="MAF_To_BED1" name="Maf to BED"> +<tool id="MAF_To_BED1" name="Maf to BED" force_history_refresh="True"> <description>Converts a MAF formated file to the BED format</description> - <command interpreter="python">maf_to_bed.py $input1 $out_file1 $species $complete_blocks</command> + <command interpreter="python">maf_to_bed.py $input1 $out_file1 $species $complete_blocks $__new_file_path__</command> <inputs> <param format="maf" name="input1" type="data" label="MAF file to convert"/> <param name="species" type="select" label="Select species" display="checkboxes" multiple="true" help="a separate history item will be created for each checked species"> diff -r 92d6a9796b24 -r 9d4945bbdcf5 tools/maf/maf_to_bed_code.py --- a/tools/maf/maf_to_bed_code.py Mon Nov 30 14:55:30 2009 -0500 +++ b/tools/maf/maf_to_bed_code.py Mon Nov 30 15:33:45 2009 -0500 @@ -46,6 +46,7 @@ newdata.state = newdata.states.ERROR newdata.dbkey = dbkey newdata.init_meta() + newdata.set_meta() newdata.set_peek() app.model.context.flush() output_data_list.append(newdata)

1 0

[hg] galaxy 3127: Fixed a bug in 'fetch closest features' tool.
by Greg Von Kuster 02 Dec '09

02 Dec '09

details: http://www.bx.psu.edu/hg/galaxy/rev/9f7a2a3be0e7 changeset: 3127:9f7a2a3be0e7 user: guru date: Mon Nov 30 11:59:54 2009 -0500 description: Fixed a bug in 'fetch closest features' tool. diffstat: tools/new_operations/flanking_features.py | 5 ++--- 1 files changed, 2 insertions(+), 3 deletions(-) diffs (22 lines): diff -r 71b779fde315 -r 9f7a2a3be0e7 tools/new_operations/flanking_features.py --- a/tools/new_operations/flanking_features.py Mon Nov 30 11:56:58 2009 -0500 +++ b/tools/new_operations/flanking_features.py Mon Nov 30 11:59:54 2009 -0500 @@ -121,7 +121,7 @@ map(outfields.append, result_down[-1].other) #The last element of result_down will be the closest element to the given interval yield outfields - if either: + if either and (result_up or result_down): if result_up and result_down: if abs(start - int(result_up[res_ind].end)) <= abs(end - int(result_down[-1].start)): map(outfields.append, result_up[res_ind].other) @@ -131,8 +131,7 @@ map(outfields.append, result_up[res_ind].other) elif result_down: map(outfields.append, result_down[-1].other) #The last element of result_down will be the closest element to the given interval - yield outfields - + yield outfields def main(): options, args = doc_optparse.parse( __doc__ )

1 0

[hg] galaxy 3125: Fix for request_events table migration script ...
by Greg Von Kuster 02 Dec '09

02 Dec '09

details: http://www.bx.psu.edu/hg/galaxy/rev/a8234f4d7012 changeset: 3125:a8234f4d7012 user: rc date: Mon Nov 30 09:35:21 2009 -0500 description: Fix for request_events table migration script for MySQL & sqlite diffstat: lib/galaxy/model/migrate/versions/0027_request_events.py | 5 +++-- 1 files changed, 3 insertions(+), 2 deletions(-) diffs (16 lines): diff -r 341d7c1c5c9a -r a8234f4d7012 lib/galaxy/model/migrate/versions/0027_request_events.py --- a/lib/galaxy/model/migrate/versions/0027_request_events.py Mon Nov 30 09:05:28 2009 -0500 +++ b/lib/galaxy/model/migrate/versions/0027_request_events.py Mon Nov 30 09:35:21 2009 -0500 @@ -73,9 +73,10 @@ "%s AS create_time," + \ "%s AS update_time," + \ "request.id AS request_id," + \ - "request.state AS state " + \ + "request.state AS state," + \ + "'%s' AS comment " + \ "FROM request;" - cmd = cmd % ( nextval('request_event'), localtimestamp(), localtimestamp() ) + cmd = cmd % ( nextval('request_event'), localtimestamp(), localtimestamp(), 'Imported from request table') db_session.execute( cmd ) # Delete the state column

1 0

[hg] galaxy 3126: Added 'N' to the list of mask characters, as p...
by Greg Von Kuster 02 Dec '09

02 Dec '09

details: http://www.bx.psu.edu/hg/galaxy/rev/71b779fde315 changeset: 3126:71b779fde315 user: guru date: Mon Nov 30 11:56:58 2009 -0500 description: Added 'N' to the list of mask characters, as per Melissa's request. diffstat: tools/regVariation/maf_cpg_filter.py | 2 +- tools/regVariation/maf_cpg_filter.xml | 1 + tools/regVariation/quality_filter.py | 2 +- tools/regVariation/quality_filter.xml | 1 + 4 files changed, 4 insertions(+), 2 deletions(-) diffs (46 lines): diff -r a8234f4d7012 -r 71b779fde315 tools/regVariation/maf_cpg_filter.py --- a/tools/regVariation/maf_cpg_filter.py Mon Nov 30 09:35:21 2009 -0500 +++ b/tools/regVariation/maf_cpg_filter.py Mon Nov 30 11:56:58 2009 -0500 @@ -39,7 +39,7 @@ reader = bx.align.maf.Reader( open(inp_file, 'r') ) writer = bx.align.maf.Writer( open(out_file,'w') ) - mask_chr_dict = {0:'#', 1:'$', 2:'^', 3:'*', 4:'?'} + mask_chr_dict = {0:'#', 1:'$', 2:'^', 3:'*', 4:'?', 5:'N'} mask = mask_chr_dict[mask] if sitetype == "CpG": diff -r a8234f4d7012 -r 71b779fde315 tools/regVariation/maf_cpg_filter.xml --- a/tools/regVariation/maf_cpg_filter.xml Mon Nov 30 09:35:21 2009 -0500 +++ b/tools/regVariation/maf_cpg_filter.xml Mon Nov 30 11:56:58 2009 -0500 @@ -21,6 +21,7 @@ <option value="2">^</option> <option value="3">*</option> <option value="4">?</option> + <option value="5">N</option> </param> <conditional name="masksite"> <param name="type" size="5" type="select" label="Sites to be masked"> diff -r a8234f4d7012 -r 71b779fde315 tools/regVariation/quality_filter.py --- a/tools/regVariation/quality_filter.py Mon Nov 30 09:35:21 2009 -0500 +++ b/tools/regVariation/quality_filter.py Mon Nov 30 11:56:58 2009 -0500 @@ -100,7 +100,7 @@ stop_err( "No mask species selected, try again by selecting at least one species to mask." ) mask_chr_count = 0 - mask_chr_dict = {0:'#', 1:'$', 2:'^', 3:'*', 4:'?'} + mask_chr_dict = {0:'#', 1:'$', 2:'^', 3:'*', 4:'?', 5:'N'} mask_reg_dict = {0:'Current pos', 1:'Current+Downstream', 2:'Current+Upstream', 3:'Current+Both sides'} #ensure dbkey is present in the twobit loc file diff -r a8234f4d7012 -r 71b779fde315 tools/regVariation/quality_filter.xml --- a/tools/regVariation/quality_filter.xml Mon Nov 30 09:35:21 2009 -0500 +++ b/tools/regVariation/quality_filter.xml Mon Nov 30 11:56:58 2009 -0500 @@ -37,6 +37,7 @@ <option value="2">^</option> <option value="3">*</option> <option value="4">?</option> + <option value="5">N</option> </param> <conditional name="mask_region"> <param name="region" type="select" label="Mask region">

1 0