- galaxy-commits - lists.galaxyproject.org

commit/galaxy-central: greg: Fix tool shed functional test I just broke.
by commits-noreply＠bitbucket.org 10 Jan '14

10 Jan '14

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/ad9661041105/ Changeset: ad9661041105 User: greg Date: 2014-01-10 21:05:57 Summary: Fix tool shed functional test I just broke. Affected #: 1 file diff -r 5f6c670dca9f42d52480f66afeb91b63710e7ffb -r ad9661041105bbffe4772ee9bedbcf174c53eac3 test/tool_shed/functional/test_0000_basic_repository_features.py --- a/test/tool_shed/functional/test_0000_basic_repository_features.py +++ b/test/tool_shed/functional/test_0000_basic_repository_features.py @@ -349,6 +349,6 @@ break # Check for the changeset revision, repository name, owner username, 'repos' in the clone url, and the captured # unicode decoding error message. - strings_displayed = [ 'Changeset %d:%s' % ( revision_number, revision_hash ), 'filtering_0000', 'user1', 'repos', 'added:', + strings_displayed = [ '%d:%s' % ( revision_number, revision_hash ), 'filtering_0000', 'user1', 'repos', 'added:', '+These characters should not' ] - self.load_changeset_in_tool_shed( repository_id, changeset_revision, strings_displayed=strings_displayed ) \ No newline at end of file + self.load_changeset_in_tool_shed( repository_id, changeset_revision, strings_displayed=strings_displayed ) Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: inithello: Use class instead of substrings to identify integrated tool panel tools, sections, workflows, and section labels when saving and updating the panel.
by commits-noreply＠bitbucket.org 10 Jan '14

10 Jan '14

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/5f6c670dca9f/ Changeset: 5f6c670dca9f User: inithello Date: 2014-01-10 21:02:20 Summary: Use class instead of substrings to identify integrated tool panel tools, sections, workflows, and section labels when saving and updating the panel. Affected #: 2 files diff -r afa0a32a883dde3dee88e8cfeb2f84ba16b27f1c -r 5f6c670dca9f42d52480f66afeb91b63710e7ffb lib/galaxy/tools/__init__.py --- a/lib/galaxy/tools/__init__.py +++ b/lib/galaxy/tools/__init__.py @@ -62,6 +62,7 @@ from galaxy.web import url_for from galaxy.web.form_builder import SelectField from galaxy.model.item_attrs import Dictifiable +from galaxy.model import Workflow from tool_shed.util import shed_util_common from .loader import load_tool, template_macro_params @@ -142,13 +143,13 @@ # 1) adjusting init_tools() and called methods to get this right # 2) redesigning the code and/or data structure used to read/write integrated_tool_panel.xml for key, value in self.integrated_tool_panel.iteritems(): - if key.startswith( 'section_' ): + if isinstance( value, ToolSection ): for section_key, section_value in value.elems.iteritems(): if section_value is None: - if section_key.startswith( 'tool_' ): + if isinstance( section_value, Tool ): tool_id = section_key[5:] value.elems[section_key] = self.tools_by_id.get( tool_id ) - elif section_key.startswith( 'workflow_' ): + elif isinstance( section_value, Workflow ): workflow_id = section_key[9:] value.elems[section_key] = self.workflows_by_id.get( workflow_id ) @@ -252,19 +253,19 @@ def load_tool_panel( self ): for key, val in self.integrated_tool_panel.items(): - if key.startswith( 'tool_' ): + if isinstance( val, Tool ): tool_id = key.replace( 'tool_', '', 1 ) if tool_id in self.tools_by_id: self.__add_tool_to_tool_panel( tool_id, self.tool_panel, section=False ) - elif key.startswith( 'workflow_' ): + elif isinstance( val, Workflow ): workflow_id = key.replace( 'workflow_', '', 1 ) if workflow_id in self.workflows_by_id: workflow = self.workflows_by_id[ workflow_id ] self.tool_panel[ key ] = workflow log.debug( "Loaded workflow: %s %s" % ( workflow_id, workflow.name ) ) - elif key.startswith( 'label_' ): + elif isinstance( val, ToolSectionLabel ): self.tool_panel[ key ] = val - elif key.startswith( 'section_' ): + elif isinstance( val, ToolSection ): elem = ElementTree.Element( 'section' ) elem.attrib[ 'id' ] = val.id or '' elem.attrib[ 'name' ] = val.name or '' @@ -276,13 +277,13 @@ tool_id = section_key.replace( 'tool_', '', 1 ) if tool_id in self.tools_by_id: self.__add_tool_to_tool_panel( tool_id, section, section=True ) - elif section_key.startswith( 'workflow_' ): + elif isinstance( section_val, Workflow ): workflow_id = section_key.replace( 'workflow_', '', 1 ) if workflow_id in self.workflows_by_id: workflow = self.workflows_by_id[ workflow_id ] section.elems[ section_key ] = workflow log.debug( "Loaded workflow: %s %s" % ( workflow_id, workflow.name ) ) - elif section_key.startswith( 'label_' ): + elif isinstance( val, ToolSectionLabel ): if section_val: section.elems[ section_key ] = section_val log.debug( "Loaded label: %s" % ( section_val.text ) ) @@ -315,7 +316,7 @@ elif section_elem.tag == 'label': key = 'label_%s' % section_elem.get( 'id' ) section.elems[ key ] = None - key = 'section_%s' % elem.get( 'id' ) + key = elem.get( 'id' ) self.integrated_tool_panel[ key ] = section elif elem.tag == 'label': key = 'label_%s' % elem.get( 'id' ) @@ -331,28 +332,28 @@ os.write( fd, '<toolbox>\n' ) for key, item in self.integrated_tool_panel.items(): if item: - if key.startswith( 'tool_' ): + if isinstance( item, Tool ): os.write( fd, ' <tool id="%s" />\n' % item.id ) - elif key.startswith( 'workflow_' ): + elif isinstance( item, Workflow ): os.write( fd, ' <workflow id="%s" />\n' % item.id ) - elif key.startswith( 'label_' ): + elif isinstance( item, ToolSectionLabel ): label_id = item.id or '' label_text = item.text or '' label_version = item.version or '' os.write( fd, ' <label id="%s" text="%s" version="%s" />\n' % ( label_id, label_text, label_version ) ) - elif key.startswith( 'section_' ): + elif isinstance( item, ToolSection ): section_id = item.id or '' section_name = item.name or '' section_version = item.version or '' os.write( fd, ' <section id="%s" name="%s" version="%s">\n' % ( section_id, section_name, section_version ) ) for section_key, section_item in item.elems.items(): - if section_key.startswith( 'tool_' ): + if isinstance( section_item, Tool ): if section_item: os.write( fd, ' <tool id="%s" />\n' % section_item.id ) - elif section_key.startswith( 'workflow_' ): + elif isinstance( section_item, Workflow ): if section_item: os.write( fd, ' <workflow id="%s" />\n' % section_item.id ) - elif section_key.startswith( 'label_' ): + elif isinstance( section_item, ToolSectionLabel ): if section_item: label_id = section_item.id or '' label_text = section_item.text or '' @@ -576,7 +577,7 @@ integrated_panel_dict.insert( index, key, label ) def load_section_tag_set( self, elem, tool_path, load_panel_dict, index=None ): - key = 'section_' + elem.get( "id" ) + key = elem.get( "id" ) if key in self.tool_panel: section = self.tool_panel[ key ] elems = section.elems @@ -1078,14 +1079,14 @@ def get_panel_section( self ): for key, item in self.app.toolbox.integrated_tool_panel.items(): if item: - if key.startswith( 'tool_' ): + if isinstance( item, Tool ): if item.id == self.id: return '', '' - if key.startswith( 'section_' ): + if isinstance( item, ToolSection ): section_id = item.id or '' section_name = item.name or '' for section_key, section_item in item.elems.items(): - if section_key.startswith( 'tool_' ): + if isinstance( section_item, Tool ): if section_item: if section_item.id == self.id: return section_id, section_name diff -r afa0a32a883dde3dee88e8cfeb2f84ba16b27f1c -r 5f6c670dca9f42d52480f66afeb91b63710e7ffb lib/tool_shed/util/tool_util.py --- a/lib/tool_shed/util/tool_util.py +++ b/lib/tool_shed/util/tool_util.py @@ -466,11 +466,7 @@ return sample_files, deleted_sample_files def get_or_create_tool_section( trans, tool_panel_section_id, new_tool_panel_section_label=None ): - if tool_panel_section_id.startswith( 'section_' ): - tool_panel_section_key = tool_panel_section_id - tool_panel_section_id.lstrip( 'section_' ) - else: - tool_panel_section_key = 'section_%s' % str( tool_panel_section_id ) + tool_panel_section_key = str( tool_panel_section_id ) if tool_panel_section_key in trans.app.toolbox.tool_panel: # Appending a tool to an existing section in trans.app.toolbox.tool_panel tool_section = trans.app.toolbox.tool_panel[ tool_panel_section_key ] @@ -680,16 +676,13 @@ # received tool_panel_section_id must be the id retrieved from a tool panel config (e.g., tool_conf.xml, which # may have getext). If new_tool_panel_section_label is received, a new section will be added to the tool panel. if new_tool_panel_section_label: - section_id = 'section_%s' % str( new_tool_panel_section_label.lower().replace( ' ', '_' ) ) + section_id = str( new_tool_panel_section_label.lower().replace( ' ', '_' ) ) tool_panel_section_key, tool_section = \ get_or_create_tool_section( trans, tool_panel_section_id=section_id, new_tool_panel_section_label=new_tool_panel_section_label ) elif tool_panel_section_id: - if tool_panel_section_id.startswith( 'section_' ): - tool_panel_section_key = str( tool_panel_section_id ) - else: - tool_panel_section_key = 'section_%s' % str( tool_panel_section_id ) + tool_panel_section_key = str( tool_panel_section_id ) tool_section = trans.app.toolbox.tool_panel[ tool_panel_section_key ] else: return None, None @@ -1000,7 +993,7 @@ for config_elem in config_elems: if config_elem.tag == 'section': # Get the section key for the in-memory tool panel. - section_key = 'section_%s' % str( config_elem.get( "id" ) ) + section_key = str( config_elem.get( "id" ) ) # Generate the list of tool elements to remove. tool_elems_to_remove = [] for tool_elem in config_elem: Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: greg: Include the changeset revision date whenever displaying the changeset revision for a repository in the tool shed per request by Ross Lazarus.
by commits-noreply＠bitbucket.org 10 Jan '14

10 Jan '14

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/afa0a32a883d/ Changeset: afa0a32a883d User: greg Date: 2014-01-10 20:30:18 Summary: Include the changeset revision date whenever displaying the changeset revision for a repository in the tool shed per request by Ross Lazarus. Affected #: 18 files diff -r c8d2e455f0e865662028d831d20a83316f2ab33f -r afa0a32a883dde3dee88e8cfeb2f84ba16b27f1c lib/galaxy/webapps/tool_shed/controllers/repository.py --- a/lib/galaxy/webapps/tool_shed/controllers/repository.py +++ b/lib/galaxy/webapps/tool_shed/controllers/repository.py @@ -2432,6 +2432,7 @@ message='Select a repository to rate', status='error' ) ) repository = suc.get_repository_in_tool_shed( trans, id ) + changeset_revision = repository.tip( trans.app ) repo = hg.repository( suc.get_configured_ui(), repository.repo_path( trans.app ) ) if repository.user == trans.user: return trans.response.send_redirect( web.url_for( controller='repository', @@ -2447,12 +2448,14 @@ rra = self.get_user_item_rating( trans.sa_session, trans.user, repository, webapp_model=trans.model ) metadata = metadata_util.get_repository_metadata_by_repository_id_changeset_revision( trans, id, - repository.tip( trans.app ), + changeset_revision, metadata_only=True ) repository_type_select_field = rt_util.build_repository_type_select_field( trans, repository=repository ) + revision_label = suc.get_revision_label( trans, repository, changeset_revision ) return trans.fill_template( '/webapps/tool_shed/repository/rate_repository.mako', repository=repository, metadata=metadata, + revision_label=revision_label, avg_rating=avg_rating, display_reviews=display_reviews, num_ratings=num_ratings, @@ -2862,13 +2865,10 @@ has_metadata = True else: has_metadata = False - t, tz = ctx.date() - date = datetime( *gmtime( float( t ) - tz )[:6] ) - display_date = date.strftime( "%Y-%m-%d" ) change_dict = { 'ctx' : ctx, 'rev' : str( ctx.rev() ), 'date' : date, - 'display_date' : display_date, + 'display_date' : suc.get_readable_ctx_date( ctx ), 'description' : ctx.description(), 'files' : ctx.files(), 'user' : ctx.user(), @@ -2923,16 +2923,18 @@ metadata = metadata_util.get_repository_metadata_by_repository_id_changeset_revision( trans, id, ctx_str, metadata_only=True ) # For rendering the prev button. if ctx_parent: + ctx_parent_date = suc.get_readable_ctx_date( ctx_parent ) ctx_parent_rev = ctx_parent.rev() if ctx_parent_rev < 0: prev = None else: - prev = "%s:%s" % ( ctx_parent_rev, ctx_parent ) + prev = "<b>%s:%s</b><i>(%s)</i>" % ( ctx_parent_rev, ctx_parent, ctx_parent_date ) else: prev = None if ctx_child: + ctx_child_date = suc.get_readable_ctx_date( ctx_child ) ctx_child_rev = ctx_child.rev() - next = "%s:%s" % ( ctx_child_rev, ctx_child ) + next = "<b>%s:%s</b><i>(%s)</i>" % ( ctx_child_rev, ctx_child, ctx_child_date ) else: next = None return trans.fill_template( '/webapps/tool_shed/repository/view_changeset.mako', diff -r c8d2e455f0e865662028d831d20a83316f2ab33f -r afa0a32a883dde3dee88e8cfeb2f84ba16b27f1c lib/tool_shed/grids/admin_grids.py --- a/lib/tool_shed/grids/admin_grids.py +++ b/lib/tool_shed/grids/admin_grids.py @@ -19,48 +19,72 @@ class UserGrid( grids.Grid ): - # TODO: move this to an admin_common controller since it is virtually the same in the galaxy webapp. + + class UserLoginColumn( grids.TextColumn ): + def get_value( self, trans, grid, user ): return user.email + + class UserNameColumn( grids.TextColumn ): + def get_value( self, trans, grid, user ): if user.username: return user.username return 'not set' + + class GroupsColumn( grids.GridColumn ): + def get_value( self, trans, grid, user ): if user.groups: return len( user.groups ) return 0 + + class RolesColumn( grids.GridColumn ): + def get_value( self, trans, grid, user ): if user.roles: return len( user.roles ) return 0 + + class ExternalColumn( grids.GridColumn ): + def get_value( self, trans, grid, user ): if user.external: return 'yes' return 'no' + + class LastLoginColumn( grids.GridColumn ): + def get_value( self, trans, grid, user ): if user.galaxy_sessions: return self.format( user.galaxy_sessions[ 0 ].update_time ) return 'never' + + class StatusColumn( grids.GridColumn ): + def get_value( self, trans, grid, user ): if user.purged: return "purged" elif user.deleted: return "deleted" return "" + + class EmailColumn( grids.GridColumn ): + def filter( self, trans, user, query, column_filter ): if column_filter == 'All': return query return query.filter( and_( model.Tool.table.c.user_id == model.User.table.c.id, model.User.table.c.email == column_filter ) ) + title = "Users" model_class = model.User template='/admin/user/grid.mako' @@ -114,37 +138,56 @@ num_rows_per_page = 50 preserve_state = False use_paging = True + def get_current_item( self, trans, **kwargs ): return trans.user + class RoleGrid( grids.Grid ): - # TODO: move this to an admin_common controller since it is virtually the same in the galaxy webapp. + + class NameColumn( grids.TextColumn ): + def get_value( self, trans, grid, role ): return role.name + + class DescriptionColumn( grids.TextColumn ): + def get_value( self, trans, grid, role ): if role.description: return role.description return '' + + class TypeColumn( grids.TextColumn ): def get_value( self, trans, grid, role ): return role.type + + class StatusColumn( grids.GridColumn ): + def get_value( self, trans, grid, role ): if role.deleted: return "deleted" return "" + + class GroupsColumn( grids.GridColumn ): + def get_value( self, trans, grid, role ): if role.groups: return len( role.groups ) return 0 + + class UsersColumn( grids.GridColumn ): + def get_value( self, trans, grid, role ): if role.users: return len( role.users ) return 0 + title = "Roles" model_class = model.Role template='/admin/dataset_security/role/grid.mako' @@ -205,29 +248,43 @@ num_rows_per_page = 50 preserve_state = False use_paging = True + def apply_query_filter( self, trans, query, **kwd ): return query.filter( model.Role.type != model.Role.types.PRIVATE ) + class GroupGrid( grids.Grid ): - # TODO: move this to an admin_common controller since it is virtually the same in the galaxy webapp. + + class NameColumn( grids.TextColumn ): + def get_value( self, trans, grid, group ): return group.name + + class StatusColumn( grids.GridColumn ): + def get_value( self, trans, grid, group ): if group.deleted: return "deleted" return "" + + class RolesColumn( grids.GridColumn ): + def get_value( self, trans, grid, group ): if group.roles: return len( group.roles ) return 0 + + class UsersColumn( grids.GridColumn ): + def get_value( self, trans, grid, group ): if group.members: return len( group.members ) return 0 + title = "Groups" model_class = model.Group template='/admin/dataset_security/group/grid.mako' @@ -279,6 +336,7 @@ preserve_state = False use_paging = True + class ManageCategoryGrid( CategoryGrid ): columns = [ col for col in CategoryGrid.columns ] # Override the NameColumn to include an Edit link @@ -292,12 +350,17 @@ dict( controller='admin', action='manage_categories', operation='create' ) ) ] + class AdminRepositoryGrid( RepositoryGrid ): + + class DeletedColumn( grids.BooleanColumn ): + def get_value( self, trans, grid, repository ): if repository.deleted: return 'yes' return '' + columns = [ RepositoryGrid.NameColumn( "Name", key="name", link=( lambda item: dict( operation="view_or_manage_repository", id=item.id ) ), @@ -326,27 +389,43 @@ async_compatible=False ) ) standard_filters = [] default_filter = {} + def build_initial_query( self, trans, **kwd ): return trans.sa_session.query( model.Repository ) \ .join( model.User.table ) + class RepositoryMetadataGrid( grids.Grid ): + + class IdColumn( grids.IntegerColumn ): + def get_value( self, trans, grid, repository_metadata ): return repository_metadata.id + + class NameColumn( grids.TextColumn ): + def get_value( self, trans, grid, repository_metadata ): return repository_metadata.repository.name + + class OwnerColumn( grids.TextColumn ): + def get_value( self, trans, grid, repository_metadata ): return repository_metadata.repository.user.username + class RevisionColumn( grids.TextColumn ): + def get_value( self, trans, grid, repository_metadata ): repository = repository_metadata.repository repo = hg.repository( suc.get_configured_ui(), repository.repo_path( trans.app ) ) ctx = suc.get_changectx_for_changeset( repo, repository_metadata.changeset_revision ) - return "%s:%s" % ( str( ctx.rev() ), repository_metadata.changeset_revision ) + return suc.get_revision_label( trans, repository, repository_metadata.changeset_revision ) + + class ToolsColumn( grids.TextColumn ): + def get_value( self, trans, grid, repository_metadata ): tools_str = '0' if repository_metadata: @@ -358,7 +437,10 @@ # tools_str += '%s <b>%s</b><br/>' % ( tool_metadata_dict[ 'id' ], tool_metadata_dict[ 'version' ] ) return '%d' % len( metadata[ 'tools' ] ) return tools_str + + class DatatypesColumn( grids.TextColumn ): + def get_value( self, trans, grid, repository_metadata ): datatypes_str = '0' if repository_metadata: @@ -370,7 +452,10 @@ # datatypes_str += '%s<br/>' % datatype_metadata_dict[ 'extension' ] return '%d' % len( metadata[ 'datatypes' ] ) return datatypes_str + + class WorkflowsColumn( grids.TextColumn ): + def get_value( self, trans, grid, repository_metadata ): workflows_str = '0' if repository_metadata: @@ -387,21 +472,31 @@ # workflows_str += '%s<br/>' % workflow_metadata_dict[ 'name' ] return '%d' % len( metadata[ 'workflows' ] ) return workflows_str + + class DeletedColumn( grids.BooleanColumn ): + def get_value( self, trans, grid, repository_metadata ): if repository_metadata.repository.deleted: return 'yes' return '' + + class DeprecatedColumn( grids.BooleanColumn ): + def get_value( self, trans, grid, repository_metadata ): if repository_metadata.repository.deprecated: return 'yes' return '' + + class MaliciousColumn( grids.BooleanColumn ): + def get_value( self, trans, grid, repository_metadata ): if repository_metadata.malicious: return 'yes' return '' + # Grid definition title = "Repository Metadata" model_class = model.RepositoryMetadata @@ -440,6 +535,7 @@ num_rows_per_page = 50 preserve_state = False use_paging = True + def build_initial_query( self, trans, **kwd ): return trans.sa_session.query( model.RepositoryMetadata ) \ .join( model.Repository.table ) diff -r c8d2e455f0e865662028d831d20a83316f2ab33f -r afa0a32a883dde3dee88e8cfeb2f84ba16b27f1c lib/tool_shed/grids/repository_grids.py --- a/lib/tool_shed/grids/repository_grids.py +++ b/lib/tool_shed/grids/repository_grids.py @@ -101,6 +101,10 @@ if len( select_field.options ) > 1: return select_field.get_html() elif len( select_field.options ) == 1: + option_items = select_field.options[ 0 ][ 0 ] + rev_label, rev_date = option_items.split( ' ' ) + rev_date = '<i><font color="#666666">%s</font></i>' % rev_date + return '%s %s' % ( rev_label, rev_date ) return select_field.options[ 0 ][ 0 ] return '' @@ -1134,7 +1138,7 @@ repository = repository_metadata.repository changeset_revision = repository_metadata.changeset_revision changeset_revision_label = suc.get_revision_label( trans, repository, changeset_revision ) - return escape_html( changeset_revision_label ) + return changeset_revision_label class MaliciousColumn( grids.BooleanColumn ): diff -r c8d2e455f0e865662028d831d20a83316f2ab33f -r afa0a32a883dde3dee88e8cfeb2f84ba16b27f1c lib/tool_shed/grids/repository_review_grids.py --- a/lib/tool_shed/grids/repository_review_grids.py +++ b/lib/tool_shed/grids/repository_review_grids.py @@ -83,9 +83,10 @@ if repository_metadata_revisions: rval = '' for repository_metadata in repository_metadata_revisions: - rev, label, changeset_revision = metadata_util.get_rev_label_changeset_revision_from_repository_metadata( trans, - repository_metadata, - repository=repository ) + rev, label, changeset_revision = \ + metadata_util.get_rev_label_changeset_revision_from_repository_metadata( trans, + repository_metadata, + repository=repository ) rval += '<a href="manage_repository_reviews_of_revision?id=%s&changeset_revision=%s">%s</a><br/>' % \ ( trans.security.encode_id( repository.id ), changeset_revision, label ) return rval diff -r c8d2e455f0e865662028d831d20a83316f2ab33f -r afa0a32a883dde3dee88e8cfeb2f84ba16b27f1c lib/tool_shed/grids/util.py --- a/lib/tool_shed/grids/util.py +++ b/lib/tool_shed/grids/util.py @@ -16,7 +16,8 @@ select_field.add_option( option_tup[ 0 ], option_tup[ 1 ], selected=selected ) return select_field -def build_changeset_revision_select_field( trans, repository, selected_value=None, add_id_to_name=True, downloadable=False, reviewed=False, not_reviewed=False ): +def build_changeset_revision_select_field( trans, repository, selected_value=None, add_id_to_name=True, + downloadable=False, reviewed=False, not_reviewed=False ): """Build a SelectField whose options are the changeset_rev strings of certain revisions of the received repository.""" options = [] changeset_tups = [] @@ -46,7 +47,10 @@ # Restrict the options to all revisions that have associated metadata. repository_metadata_revisions = repository.metadata_revisions for repository_metadata in repository_metadata_revisions: - rev, label, changeset_revision = metadata_util.get_rev_label_changeset_revision_from_repository_metadata( trans, repository_metadata, repository=repository ) + rev, label, changeset_revision = \ + metadata_util.get_rev_label_changeset_revision_from_repository_metadata( trans, + repository_metadata, + repository=repository ) changeset_tups.append( ( rev, label, changeset_revision ) ) refresh_on_change_values.append( changeset_revision ) # Sort options by the revision label. Even though the downloadable_revisions query sorts by update_time, diff -r c8d2e455f0e865662028d831d20a83316f2ab33f -r afa0a32a883dde3dee88e8cfeb2f84ba16b27f1c lib/tool_shed/util/metadata_util.py --- a/lib/tool_shed/util/metadata_util.py +++ b/lib/tool_shed/util/metadata_util.py @@ -1108,16 +1108,19 @@ changeset_revision = repository_metadata.changeset_revision ctx = suc.get_changectx_for_changeset( repo, changeset_revision ) if ctx: + changeset_revision_date = suc.get_readable_ctx_date( ctx ) rev = '%04d' % ctx.rev() - label = "%s:%s" % ( str( ctx.rev() ), changeset_revision ) + label = "%s:%s (%s)" % ( str( ctx.rev() ), changeset_revision, changeset_revision_date ) else: rev = '-1' label = "-1:%s" % changeset_revision return rev, label, changeset_revision -def get_sample_files_from_disk( repository_files_dir, tool_path=None, relative_install_dir=None, resetting_all_metadata_on_repository=False ): +def get_sample_files_from_disk( repository_files_dir, tool_path=None, relative_install_dir=None, + resetting_all_metadata_on_repository=False ): if resetting_all_metadata_on_repository: - # Keep track of the location where the repository is temporarily cloned so that we can strip it when setting metadata. + # Keep track of the location where the repository is temporarily cloned so that we can strip + # it when setting metadata. work_dir = repository_files_dir sample_file_metadata_paths = [] sample_file_copy_paths = [] diff -r c8d2e455f0e865662028d831d20a83316f2ab33f -r afa0a32a883dde3dee88e8cfeb2f84ba16b27f1c lib/tool_shed/util/shed_util_common.py --- a/lib/tool_shed/util/shed_util_common.py +++ b/lib/tool_shed/util/shed_util_common.py @@ -924,6 +924,13 @@ return trans.install_model.context.query( trans.install_model.ToolShedRepository ) \ .filter( trans.install_model.ToolShedRepository.table.c.uninstalled == False ) +def get_readable_ctx_date( ctx ): + """Convert the date of the changeset (the received ctx) to a human-readable date.""" + t, tz = ctx.date() + date = datetime( *gmtime( float( t ) - tz )[ :6 ] ) + ctx_date = date.strftime( "%Y-%m-%d" ) + return ctx_date + def get_repo_info_tuple_contents( repo_info_tuple ): """Take care in handling the repo_info_tuple as it evolves over time as new tool shed features are introduced.""" if len( repo_info_tuple ) == 6: @@ -1206,7 +1213,8 @@ repo = hg.repository( get_configured_ui(), repository.repo_path( trans.app ) ) ctx = get_changectx_for_changeset( repo, changeset_revision ) if ctx: - return "%s:%s" % ( str( ctx.rev() ), changeset_revision ) + return '%s:%s <i><font color="#666666">(%s)</font></i>' % \ + ( str( ctx.rev() ), changeset_revision, str( get_readable_ctx_date( ctx ) ) ) else: return "-1:%s" % changeset_revision @@ -1215,7 +1223,8 @@ ctx = get_changectx_for_changeset( repo, changeset_revision ) if ctx: rev = '%04d' % ctx.rev() - label = "%s:%s" % ( str( ctx.rev() ), changeset_revision ) + label = '%s:%s <i><font color="#666666">(%s)</font></i>' % \ + ( str( ctx.rev() ), changeset_revision, str( get_readable_ctx_date( ctx ) ) ) else: rev = '-1' label = "-1:%s" % changeset_revision @@ -1477,9 +1486,6 @@ email_from = 'galaxy-no-reply@' + trans.request.host.split( ':' )[0] tip_changeset = repo.changelog.tip() ctx = repo.changectx( tip_changeset ) - t, tz = ctx.date() - date = datetime( *gmtime( float( t ) - tz )[:6] ) - display_date = date.strftime( "%Y-%m-%d" ) try: username = ctx.user().split()[0] except: @@ -1490,6 +1496,7 @@ template = new_repo_email_alert_template else: template = email_alert_template + display_date = get_readable_ctx_date( ctx ) admin_body = string.Template( template ).safe_substitute( host=trans.request.host, sharable_link=sharable_link, repository_name=repository.name, diff -r c8d2e455f0e865662028d831d20a83316f2ab33f -r afa0a32a883dde3dee88e8cfeb2f84ba16b27f1c templates/webapps/tool_shed/common/repository_actions_menu.mako --- a/templates/webapps/tool_shed/common/repository_actions_menu.mako +++ b/templates/webapps/tool_shed/common/repository_actions_menu.mako @@ -174,7 +174,9 @@ <a class="action-button" target="galaxy_main" href="${h.url_for( controller='repository', action='deprecate', id=trans.security.encode_id( repository.id ), mark_deprecated=False )}">Mark repository as not deprecated</a> %endif %if can_download: - <a class="action-button" href="${h.url_for( controller='repository', action='export', repository_id=trans.app.security.encode_id( repository.id ), changeset_revision=changeset_revision )}">Export this revision</a> + %if changeset_revision is not None: + <a class="action-button" href="${h.url_for( controller='repository', action='export', repository_id=trans.app.security.encode_id( repository.id ), changeset_revision=changeset_revision )}">Export this revision</a> + %endif <a class="action-button" href="${h.url_for( controller='repository', action='download', repository_id=trans.app.security.encode_id( repository.id ), changeset_revision=repository.tip( trans.app ), file_type='gz' )}">Download as a .tar.gz file</a><a class="action-button" href="${h.url_for( controller='repository', action='download', repository_id=trans.app.security.encode_id( repository.id ), changeset_revision=repository.tip( trans.app ), file_type='bz2' )}">Download as a .tar.bz2 file</a><a class="action-button" href="${h.url_for( controller='repository', action='download', repository_id=trans.app.security.encode_id( repository.id ), changeset_revision=repository.tip( trans.app ), file_type='zip' )}">Download as a zip file</a> diff -r c8d2e455f0e865662028d831d20a83316f2ab33f -r afa0a32a883dde3dee88e8cfeb2f84ba16b27f1c templates/webapps/tool_shed/repository/manage_repository.mako --- a/templates/webapps/tool_shed/repository/manage_repository.mako +++ b/templates/webapps/tool_shed/repository/manage_repository.mako @@ -181,9 +181,9 @@ <div class="form-row"><label>Revision:</label> %if can_view_change_log: - <a href="${h.url_for( controller='repository', action='view_changelog', id=trans.app.security.encode_id( repository.id ) )}">${revision_label | h}</a> + <a href="${h.url_for( controller='repository', action='view_changelog', id=trans.app.security.encode_id( repository.id ) )}">${revision_label}</a> %else: - ${revision_label | h} + ${revision_label} %endif </div><div class="form-row"> diff -r c8d2e455f0e865662028d831d20a83316f2ab33f -r afa0a32a883dde3dee88e8cfeb2f84ba16b27f1c templates/webapps/tool_shed/repository/preview_tools_in_changeset.mako --- a/templates/webapps/tool_shed/repository/preview_tools_in_changeset.mako +++ b/templates/webapps/tool_shed/repository/preview_tools_in_changeset.mako @@ -51,7 +51,7 @@ %else: <div class="form-row"><label>Revision:</label> - ${revision_label | h} + ${revision_label} </div> %endif </div> diff -r c8d2e455f0e865662028d831d20a83316f2ab33f -r afa0a32a883dde3dee88e8cfeb2f84ba16b27f1c templates/webapps/tool_shed/repository/rate_repository.mako --- a/templates/webapps/tool_shed/repository/rate_repository.mako +++ b/templates/webapps/tool_shed/repository/rate_repository.mako @@ -44,16 +44,20 @@ ${render_clone_str( repository )} </div> %endif - ${render_repository_type_select_field( repository_type_select_field, render_help=False )} <div class="toolFormBody"><div class="form-row"> + <label>Type:</label> + ${repository.type | h} + <div style="clear: both"></div> + </div> + <div class="form-row"><label>Description:</label> ${repository.description | h} <div style="clear: both"></div></div><div class="form-row"> - <label>Version:</label> - ${repository.revision( trans.app ) | h} + <label>Revision:</label> + ${revision_label} <div style="clear: both"></div></div><div class="form-row"> diff -r c8d2e455f0e865662028d831d20a83316f2ab33f -r afa0a32a883dde3dee88e8cfeb2f84ba16b27f1c templates/webapps/tool_shed/repository/view_changelog.mako --- a/templates/webapps/tool_shed/repository/view_changelog.mako +++ b/templates/webapps/tool_shed/repository/view_changelog.mako @@ -5,7 +5,6 @@ <%namespace file="/webapps/tool_shed/common/repository_actions_menu.mako" import="render_tool_shed_repository_actions" /><% - from galaxy.web.framework.helpers import time_ago is_new = repository.is_new( trans.app ) can_push = trans.app.security_agent.can_push( trans.app, trans.user, repository ) can_download = not is_new and ( not is_malicious or can_push ) @@ -108,7 +107,7 @@ </div><div class="form-row"><label>Pushed:</label> - ${time_ago( changeset[ 'date' ] )} + ${changeset[ 'display_date' ]} </div></td></tr> diff -r c8d2e455f0e865662028d831d20a83316f2ab33f -r afa0a32a883dde3dee88e8cfeb2f84ba16b27f1c templates/webapps/tool_shed/repository/view_changeset.mako --- a/templates/webapps/tool_shed/repository/view_changeset.mako +++ b/templates/webapps/tool_shed/repository/view_changeset.mako @@ -49,13 +49,15 @@ %endif <div class="toolForm"><% + from tool_shed.util.shed_util_common import get_readable_ctx_date + changeset_revision_date = get_readable_ctx_date( ctx ) if can_download: - title_str = 'Changeset %s:%s' % ( ctx.rev(), ctx ) + title_str = 'Changeset <b>%s:%s</b><i>(%s)</i>' % ( ctx.rev(), ctx, changeset_revision_date ) else: - title_str = '%s changeset %s:%s' % ( repository.name, ctx.rev(), ctx ) + title_str = '%s changeset <b>%s:%s</b><i>(%s)</i>' % ( repository.name, ctx.rev(), ctx, changeset_revision_date ) %><div class="toolFormTitle"> - ${title_str | h} + ${title_str} </div><div class="toolFormBody"><table class="grid"> @@ -63,10 +65,10 @@ <tr><td> %if prev: - <a class="action-button" href="${h.url_for( controller='repository', action='view_changeset', id=trans.security.encode_id( repository.id ), ctx_str=ctx_parent )}">Previous changeset ${prev | h}</a> + <a class="action-button" href="${h.url_for( controller='repository', action='view_changeset', id=trans.security.encode_id( repository.id ), ctx_str=ctx_parent )}">Previous changeset ${prev}</a> %endif %if next: - <a class="action-button" href="${h.url_for( controller='repository', action='view_changeset', id=trans.security.encode_id( repository.id ), ctx_str=ctx_child )}">Next changeset ${next | h}</a> + <a class="action-button" href="${h.url_for( controller='repository', action='view_changeset', id=trans.security.encode_id( repository.id ), ctx_str=ctx_child )}">Next changeset ${next}</a> %endif </td></tr> diff -r c8d2e455f0e865662028d831d20a83316f2ab33f -r afa0a32a883dde3dee88e8cfeb2f84ba16b27f1c templates/webapps/tool_shed/repository/view_repository.mako --- a/templates/webapps/tool_shed/repository/view_repository.mako +++ b/templates/webapps/tool_shed/repository/view_repository.mako @@ -100,7 +100,11 @@ ${repository.name | h} %endif </div> - ${render_repository_type_select_field( repository_type_select_field, render_help=False )} + <div class="form-row"> + <label>Type:</label> + ${repository.type | h} + <div style="clear: both"></div> + </div><div class="form-row"><label>Synopsis:</label> ${repository.description | h} @@ -113,7 +117,7 @@ %if can_view_change_log: <a href="${h.url_for( controller='repository', action='view_changelog', id=trans.app.security.encode_id( repository.id ) )}">${revision_label}</a> %else: - ${revision_label | h} + ${revision_label} %endif </div><div class="form-row"> diff -r c8d2e455f0e865662028d831d20a83316f2ab33f -r afa0a32a883dde3dee88e8cfeb2f84ba16b27f1c templates/webapps/tool_shed/repository_review/browse_review.mako --- a/templates/webapps/tool_shed/repository_review/browse_review.mako +++ b/templates/webapps/tool_shed/repository_review/browse_review.mako @@ -34,7 +34,7 @@ </div><div class="form-row"><label>Repository revision:</label> - <a class="action-button" href="${h.url_for( controller='repository_review', action='view_or_manage_repository', id=trans.security.encode_id( repository.id ), changeset_revision=review.changeset_revision )}">${changeset_revision_label | h}</a> + <a class="action-button" href="${h.url_for( controller='repository_review', action='view_or_manage_repository', id=trans.security.encode_id( repository.id ), changeset_revision=review.changeset_revision )}">${changeset_revision_label}</a><div style="clear: both"></div></div><div class="form-row"> diff -r c8d2e455f0e865662028d831d20a83316f2ab33f -r afa0a32a883dde3dee88e8cfeb2f84ba16b27f1c templates/webapps/tool_shed/repository_review/reviews_of_changeset_revision.mako --- a/templates/webapps/tool_shed/repository_review/reviews_of_changeset_revision.mako +++ b/templates/webapps/tool_shed/repository_review/reviews_of_changeset_revision.mako @@ -45,7 +45,7 @@ <div class="toolFormBody"><div class="form-row"><label>Revision:</label> - <a class="action-button" href="${h.url_for( controller='repository_review', action='view_or_manage_repository', id=trans.security.encode_id( repository.id ), changeset_revision=changeset_revision )}">${changeset_revision_label | h}</a> + <a class="action-button" href="${h.url_for( controller='repository_review', action='view_or_manage_repository', id=trans.security.encode_id( repository.id ), changeset_revision=changeset_revision )}">${changeset_revision_label}</a><div style="clear: both"></div></div><div class="form-row"> diff -r c8d2e455f0e865662028d831d20a83316f2ab33f -r afa0a32a883dde3dee88e8cfeb2f84ba16b27f1c templates/webapps/tool_shed/repository_review/reviews_of_repository.mako --- a/templates/webapps/tool_shed/repository_review/reviews_of_repository.mako +++ b/templates/webapps/tool_shed/repository_review/reviews_of_repository.mako @@ -67,7 +67,7 @@ <tr><td><div style="float:left;" class="menubutton split popup" id="${changeset_revision}-popup"> - <a class="view-info" href="${h.url_for( controller='repository_review', action='view_or_manage_repository', id=trans.security.encode_id( repository.id ), changeset_revision=changeset_revision )}">${changeset_revision_label | h}</a> + <a class="view-info" href="${h.url_for( controller='repository_review', action='view_or_manage_repository', id=trans.security.encode_id( repository.id ), changeset_revision=changeset_revision )}">${changeset_revision_label}</a></div><div popupmenu="${changeset_revision}-popup"> %if repository_reviews: diff -r c8d2e455f0e865662028d831d20a83316f2ab33f -r afa0a32a883dde3dee88e8cfeb2f84ba16b27f1c templates/webapps/tool_shed/repository_review/select_previous_review.mako --- a/templates/webapps/tool_shed/repository_review/select_previous_review.mako +++ b/templates/webapps/tool_shed/repository_review/select_previous_review.mako @@ -30,7 +30,7 @@ %endif <div class="warningmessage"> - You have elected to create a new review for revision <b>${changeset_revision_label | h}</b>of this repository. Since previous revisions have been reviewed, + You have elected to create a new review for revision <b>${changeset_revision_label}</b>of this repository. Since previous revisions have been reviewed, you can select a previous review to copy to your new review, or click the <b>Create a review without copying</b> button. </div> @@ -39,7 +39,7 @@ <div class="toolFormBody"><div class="form-row"><label>Revision for new review:</label> - <a class="action-button" href="${h.url_for( controller='repository_review', action='view_or_manage_repository', id=trans.security.encode_id( repository.id ), changeset_revision=changeset_revision )}">${changeset_revision_label | h}</a> + <a class="action-button" href="${h.url_for( controller='repository_review', action='view_or_manage_repository', id=trans.security.encode_id( repository.id ), changeset_revision=changeset_revision )}">${changeset_revision_label}</a><div style="clear: both"></div></div><div class="form-row"> @@ -76,7 +76,7 @@ <a class="action-button" href="${h.url_for( controller='repository_review', action='create_review', id=trans.security.encode_id( repository.id ), changeset_revision=changeset_revision, previous_review_id=encoded_review_id )}">Copy this review</a></div></td> - <td>${previous_changeset_revision_label | h}</td> + <td>${previous_changeset_revision_label}</td><td>${render_star_rating( repository_rating_name, review.rating, disabled=True )}</td><td>${approved_str | h}</td></tr> Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: dannon: Temporarily add simplejson back until webhelpers is updated
by commits-noreply＠bitbucket.org 10 Jan '14

10 Jan '14

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/c8d2e455f0e8/ Changeset: c8d2e455f0e8 User: dannon Date: 2014-01-10 20:22:03 Summary: Temporarily add simplejson back until webhelpers is updated Affected #: 1 file diff -r d7b4edc2d749d9835f61dccaab0e17614a9cd336 -r c8d2e455f0e865662028d831d20a83316f2ab33f eggs.ini --- a/eggs.ini +++ b/eggs.ini @@ -26,6 +26,7 @@ pysam = 0.4.2 pysqlite = 2.5.6 python_lzo = 1.08_2.03_static +simplejson = 2.1.1 threadframe = 0.2 guppy = 0.1.8 SQLAlchemy = 0.7.9 Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: 3 new changesets
by commits-noreply＠bitbucket.org 10 Jan '14

10 Jan '14

3 new commits in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/7a411112a661/ Changeset: 7a411112a661 User: takadonet Date: 2013-11-08 16:38:08 Summary: added API call to delete library dataset Affected #: 1 file diff -r b9e7d751cfb5a263d778786efa4e4b4e965b3992 -r 7a411112a66155050c748c667c47cf7f6dd24d7e lib/galaxy/webapps/galaxy/api/library_contents.py --- a/lib/galaxy/webapps/galaxy/api/library_contents.py +++ b/lib/galaxy/webapps/galaxy/api/library_contents.py @@ -2,8 +2,7 @@ API operations on the contents of a library. """ import logging - -from galaxy import web +from galaxy import web , exceptions from galaxy.model import ExtendedMetadata, ExtendedMetadataIndex from galaxy.web.base.controller import BaseAPIController, UsesLibraryMixin, UsesLibraryMixinItems from galaxy.web.base.controller import UsesHistoryDatasetAssociationMixin @@ -318,3 +317,67 @@ return 'LibraryFolder', content_id[1:] else: raise HTTPBadRequest( 'Malformed library content id ( %s ) specified, unable to decode.' % str( content_id ) ) + + + @web.expose_api + def delete( self, trans, library_id, id, **kwd ): + """ + delete( self, trans, library_id, id, **kwd ) + * DELETE /api/libraries/{library_id}/contents/{id} + delete the HDA with the given ``id`` + + + :type id: str + :param id: the encoded id of the library dataset to delete + :type kwd: dict + :param kwd: (optional) dictionary structure containing: + + * payload: a dictionary itself containing: + * purge: if True, purge the HDA + + :rtype: dict + :returns: an error object if an error occurred or a dictionary containing: + * id: the encoded id of the library dataset, + * deleted: if the library dataset was marked as deleted, + * purged: if the library dataset was purged + """ + # a request body is optional here + purge = False + if kwd.get( 'payload', None ): + purge = util.string_as_bool( kwd['payload'].get( 'purge', False ) ) + rval = { 'id' : id } + try: + hda = self.get_library_dataset( trans, id, check_ownership=False, check_accessible=True ) + hda.deleted = True + if purge: + if not trans.app.config.allow_user_dataset_purge: + raise exceptions.httpexceptions.HTTPForbidden( + detail='This instance does not allow user dataset purging' ) + hda.purged = True + trans.sa_session.add( hda ) + trans.sa_session.flush() + if hda.dataset.user_can_purge: + try: + hda.dataset.full_delete() + trans.sa_session.add( hda.dataset ) + except: + pass + # flush now to preserve deleted state in case of later interruption + trans.sa_session.flush() + rval[ 'purged' ] = True + trans.sa_session.flush() + rval[ 'deleted' ] = True + except exceptions.httpexceptions.HTTPInternalServerError, http_server_err: + log.exception( 'HDA API, delete: uncaught HTTPInternalServerError: %s, %s\n%s', + id, str( kwd ), str( http_server_err ) ) + raise + except exceptions.httpexceptions.HTTPException: + raise + except Exception, exc: + log.exception( 'HDA API, delete: uncaught exception: %s, %s\n%s', + id, str( kwd ), str( exc ) ) + trans.response.status = 500 + rval.update({ 'error': str( exc ) }) + return rval + + https://bitbucket.org/galaxy/galaxy-central/commits/69669b88ac78/ Changeset: 69669b88ac78 User: takadonet Date: 2013-11-08 16:38:37 Summary: Merged galaxy/galaxy-central into default Affected #: 1 file diff -r 7a411112a66155050c748c667c47cf7f6dd24d7e -r 69669b88ac784f8db470772f15bb17e13c314fa7 lib/galaxy/tools/parameters/basic.py --- a/lib/galaxy/tools/parameters/basic.py +++ b/lib/galaxy/tools/parameters/basic.py @@ -174,8 +174,12 @@ def to_dict( self, trans, view='collection', value_mapper=None ): """ to_dict tool parameter. This can be overridden by subclasses. """ tool_dict = super( ToolParameter, self ).to_dict() - #TODO: removing html as it causes a lot of errors on subclasses - needs histories, etc. - #tool_dict[ 'html' ] = urllib.quote( self.get_html( trans ) ) + #TODO: wrapping html as it causes a lot of errors on subclasses - needs histories, etc. + try: + tool_dict[ 'html' ] = urllib.quote( self.get_html( trans ) ) + except AssertionError, e: + pass #HACK for assert trans.history, 'requires a history' + tool_dict[ 'model_class' ] = self.__class__.__name__ if hasattr( self, 'value' ): tool_dict[ 'value' ] = self.value https://bitbucket.org/galaxy/galaxy-central/commits/d7b4edc2d749/ Changeset: d7b4edc2d749 User: carlfeberhard Date: 2014-01-10 19:59:58 Summary: Merged in takadonet/galaxy-central-takadonet (pull request #253) Add API call to allow for deletion/purge of dataset in a data library Affected #: 1 file diff -r c04e79918595e0cefe85edac156ff5a962f4a686 -r d7b4edc2d749d9835f61dccaab0e17614a9cd336 lib/galaxy/webapps/galaxy/api/library_contents.py --- a/lib/galaxy/webapps/galaxy/api/library_contents.py +++ b/lib/galaxy/webapps/galaxy/api/library_contents.py @@ -2,8 +2,7 @@ API operations on the contents of a library. """ import logging - -from galaxy import web +from galaxy import web , exceptions from galaxy.model import ExtendedMetadata, ExtendedMetadataIndex from galaxy.web.base.controller import BaseAPIController, UsesLibraryMixin, UsesLibraryMixinItems from galaxy.web.base.controller import UsesHistoryDatasetAssociationMixin @@ -318,3 +317,67 @@ return 'LibraryFolder', content_id[1:] else: raise HTTPBadRequest( 'Malformed library content id ( %s ) specified, unable to decode.' % str( content_id ) ) + + + @web.expose_api + def delete( self, trans, library_id, id, **kwd ): + """ + delete( self, trans, library_id, id, **kwd ) + * DELETE /api/libraries/{library_id}/contents/{id} + delete the HDA with the given ``id`` + + + :type id: str + :param id: the encoded id of the library dataset to delete + :type kwd: dict + :param kwd: (optional) dictionary structure containing: + + * payload: a dictionary itself containing: + * purge: if True, purge the HDA + + :rtype: dict + :returns: an error object if an error occurred or a dictionary containing: + * id: the encoded id of the library dataset, + * deleted: if the library dataset was marked as deleted, + * purged: if the library dataset was purged + """ + # a request body is optional here + purge = False + if kwd.get( 'payload', None ): + purge = util.string_as_bool( kwd['payload'].get( 'purge', False ) ) + rval = { 'id' : id } + try: + hda = self.get_library_dataset( trans, id, check_ownership=False, check_accessible=True ) + hda.deleted = True + if purge: + if not trans.app.config.allow_user_dataset_purge: + raise exceptions.httpexceptions.HTTPForbidden( + detail='This instance does not allow user dataset purging' ) + hda.purged = True + trans.sa_session.add( hda ) + trans.sa_session.flush() + if hda.dataset.user_can_purge: + try: + hda.dataset.full_delete() + trans.sa_session.add( hda.dataset ) + except: + pass + # flush now to preserve deleted state in case of later interruption + trans.sa_session.flush() + rval[ 'purged' ] = True + trans.sa_session.flush() + rval[ 'deleted' ] = True + except exceptions.httpexceptions.HTTPInternalServerError, http_server_err: + log.exception( 'HDA API, delete: uncaught HTTPInternalServerError: %s, %s\n%s', + id, str( kwd ), str( http_server_err ) ) + raise + except exceptions.httpexceptions.HTTPException: + raise + except Exception, exc: + log.exception( 'HDA API, delete: uncaught exception: %s, %s\n%s', + id, str( kwd ), str( exc ) ) + trans.response.status = 500 + rval.update({ 'error': str( exc ) }) + return rval + + Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: guerler: Fix refresh events for select fields in toolshed grid
by commits-noreply＠bitbucket.org 10 Jan '14

10 Jan '14

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/c04e79918595/ Changeset: c04e79918595 User: guerler Date: 2014-01-10 19:07:37 Summary: Fix refresh events for select fields in toolshed grid Affected #: 4 files diff -r 984377efb491a30e4afd320c9fbff2ff1e1241af -r c04e79918595e0cefe85edac156ff5a962f4a686 static/scripts/galaxy.base.js --- a/static/scripts/galaxy.base.js +++ b/static/scripts/galaxy.base.js @@ -623,8 +623,8 @@ }); }; -$(document).ready( function() { - +// Initialize refresh events. +function init_refresh_on_change () { $("select[refresh_on_change='true']").change( function() { var select_field = $(this), select_val = select_field.val(), @@ -663,7 +663,13 @@ $( "a[confirm]" ).click( function() { return confirm( $(this).attr("confirm") ); }); +}; +$(document).ready( function() { + + // Refresh events for form fields. + init_refresh_on_change(); + // Tooltips if ( $.fn.tooltip ) { // Put tooltips below items in panel header so that they do not overlap masthead. diff -r 984377efb491a30e4afd320c9fbff2ff1e1241af -r c04e79918595e0cefe85edac156ff5a962f4a686 static/scripts/galaxy.grids.js --- a/static/scripts/galaxy.grids.js +++ b/static/scripts/galaxy.grids.js @@ -199,6 +199,9 @@ // configure elements this.init_grid_elements(); + + // attach global event handler + init_refresh_on_change(); }, // Initialize grid controls diff -r 984377efb491a30e4afd320c9fbff2ff1e1241af -r c04e79918595e0cefe85edac156ff5a962f4a686 static/scripts/packed/galaxy.base.js --- a/static/scripts/packed/galaxy.base.js +++ b/static/scripts/packed/galaxy.base.js @@ -1,1 +1,1 @@ -(function(){var b=0;var c=["ms","moz","webkit","o"];for(var a=0;a<c.length&&!window.requestAnimationFrame;++a){window.requestAnimationFrame=window[c[a]+"RequestAnimationFrame"];window.cancelRequestAnimationFrame=window[c[a]+"CancelRequestAnimationFrame"]}if(!window.requestAnimationFrame){window.requestAnimationFrame=function(h,e){var d=new Date().getTime();var f=Math.max(0,16-(d-b));var g=window.setTimeout(function(){h(d+f)},f);b=d+f;return g}}if(!window.cancelAnimationFrame){window.cancelAnimationFrame=function(d){clearTimeout(d)}}}());if(!Array.indexOf){Array.prototype.indexOf=function(c){for(var b=0,a=this.length;b<a;b++){if(this[b]==c){return b}}return -1}}function obj_length(c){if(c.length!==undefined){return c.length}var b=0;for(var a in c){b++}return b}$.fn.makeAbsolute=function(a){return this.each(function(){var b=$(this);var c=b.position();b.css({position:"absolute",marginLeft:0,marginTop:0,top:c.top,left:c.left,right:$(window).width()-(c.left+b.width())});if(a){b.remove().appendTo("body")}})};function make_popupmenu(b,c){var a=(b.data("menu_options"));b.data("menu_options",c);if(a){return}b.bind("click.show_popup",function(d){$(".popmenu-wrapper").remove();setTimeout(function(){var g=$("<ul class='dropdown-menu' id='"+b.attr("id")+"-menu'></ul>");var f=b.data("menu_options");if(obj_length(f)<=0){$("<li>No Options.</li>").appendTo(g)}$.each(f,function(j,i){if(i){var l=i.action||i;g.append($("<li></li>").append($("<a>").attr("href",i.url).html(j).click(l)))}else{g.append($("<li></li>").addClass("head").append($("<a href='#'></a>").html(j)))}});var h=$("<div class='popmenu-wrapper' style='position: absolute;left: 0; top: -1000;'></div>").append(g).appendTo("body");var e=d.pageX-h.width()/2;e=Math.min(e,$(document).scrollLeft()+$(window).width()-$(h).width()-5);e=Math.max(e,$(document).scrollLeft()+5);h.css({top:d.pageY,left:e})},10);setTimeout(function(){var f=function(h){$(h).bind("click.close_popup",function(){$(".popmenu-wrapper").remove();h.unbind("click.close_popup")})};f($(window.document));f($(window.top.document));for(var e=window.top.frames.length;e--;){var g=$(window.top.frames[e].document);f(g)}},50);return false})}function make_popup_menus(a){a=a||document;$(a).find("div[popupmenu]").each(function(){var b={};var d=$(this);d.find("a").each(function(){var g=$(this),i=g.get(0),e=i.getAttribute("confirm"),f=i.getAttribute("href"),h=i.getAttribute("target");if(!f){b[g.text()]=null}else{b[g.text()]={url:f,action:function(){if(!e||confirm(e)){if(h){window.open(f,h)}else{g.click()}}}}}});var c=$(a).find("#"+d.attr("popupmenu"));c.find("a").bind("click",function(f){f.stopPropagation();return true});make_popupmenu(c,b);c.addClass("popup");d.remove()})}function naturalSort(j,h){var p=/(-?[0-9\.]+)/g,k=j.toString().toLowerCase()||"",g=h.toString().toLowerCase()||"",l=String.fromCharCode(0),n=k.replace(p,l+"$1"+l).split(l),e=g.replace(p,l+"$1"+l).split(l),d=(new Date(k)).getTime(),o=d?(new Date(g)).getTime():null;if(o){if(d<o){return -1}else{if(d>o){return 1}}}var m,f;for(var i=0,c=Math.max(n.length,e.length);i<c;i++){m=parseFloat(n[i])||n[i];f=parseFloat(e[i])||e[i];if(m<f){return -1}else{if(m>f){return 1}}}return 0}$.fn.refresh_select2=function(){var b=$(this);var a={width:"resolve",closeOnSelect:!b.is("[MULTIPLE]")};return b.select2(a)};function replace_big_select_inputs(a,c,b){if(!jQuery.fn.select2){return}if(a===undefined){a=20}if(c===undefined){c=3000}b=b||$("select");b.each(function(){var e=$(this);var d=e.find("option").length;if((d<a)||(d>c)){return}if(e.hasClass("no-autocomplete")){return}e.refresh_select2()})}$.fn.make_text_editable=function(g){var d=("num_cols" in g?g.num_cols:30),c=("num_rows" in g?g.num_rows:4),e=("use_textarea" in g?g.use_textarea:false),b=("on_finish" in g?g.on_finish:null),f=("help_text" in g?g.help_text:null);var a=$(this);a.addClass("editable-text").click(function(l){if($(this).children(":input").length>0){return}a.removeClass("editable-text");var i=function(m){a.find(":input").remove();if(m!==""){a.text(m)}else{a.html("<br>")}a.addClass("editable-text");if(b){b(m)}};var h=("cur_text" in g?g.cur_text:a.text()),k,j;if(e){k=$("<textarea/>").attr({rows:c,cols:d}).text($.trim(h)).keyup(function(m){if(m.keyCode===27){i(h)}});j=$("<button/>").text("Done").click(function(){i(k.val());return false})}else{k=$("<input type='text'/>").attr({value:$.trim(h),size:d}).blur(function(){i(h)}).keyup(function(m){if(m.keyCode===27){$(this).trigger("blur")}else{if(m.keyCode===13){i($(this).val())}}})}a.text("");a.append(k);if(j){a.append(j)}k.focus();k.select();l.stopPropagation()});if(f){a.attr("title",f).tooltip()}return a};function async_save_text(d,f,e,a,c,h,i,g,b){if(c===undefined){c=30}if(i===undefined){i=4}$("#"+d).click(function(){if($("#renaming-active").length>0){return}var l=$("#"+f),k=l.text(),j;if(h){j=$("<textarea></textarea>").attr({rows:i,cols:c}).text($.trim(k))}else{j=$("<input type='text'></input>").attr({value:$.trim(k),size:c})}j.attr("id","renaming-active");j.blur(function(){$(this).remove();l.show();if(b){b(j)}});j.keyup(function(n){if(n.keyCode===27){$(this).trigger("blur")}else{if(n.keyCode===13){var m={};m[a]=$(this).val();$(this).trigger("blur");$.ajax({url:e,data:m,error:function(){alert("Text editing for elt "+f+" failed")},success:function(o){if(o!==""){l.text(o)}else{l.html("<em>None</em>")}if(b){b(j)}}})}}});if(g){g(j)}l.hide();j.insertAfter(l);j.focus();j.select();return})}function init_history_items(d,a,c){var b=function(){try{var e=$.jStorage.get("history_expand_state");if(e){for(var g in e){$("#"+g+" div.historyItemBody").show()}}}catch(f){$.jStorage.deleteKey("history_expand_state")}if($.browser.mozilla){$("div.historyItemBody").each(function(){if(!$(this).is(":visible")){$(this).find("pre.peek").css("overflow","hidden")}})}d.each(function(){var j=this.id,h=$(this).children("div.historyItemBody"),i=h.find("pre.peek");$(this).find(".historyItemTitleBar > .historyItemTitle").wrap("<a href='javascript:void(0);'></a>").click(function(){var k;if(h.is(":visible")){if($.browser.mozilla){i.css("overflow","hidden")}h.slideUp("fast");if(!c){k=$.jStorage.get("history_expand_state");if(k){delete k[j];$.jStorage.set("history_expand_state",k)}}}else{h.slideDown("fast",function(){if($.browser.mozilla){i.css("overflow","auto")}});if(!c){k=$.jStorage.get("history_expand_state");if(!k){k={}}k[j]=true;$.jStorage.set("history_expand_state",k)}}return false})});$("#top-links > a.toggle").click(function(){var h=$.jStorage.get("history_expand_state");if(!h){h={}}$("div.historyItemBody:visible").each(function(){if($.browser.mozilla){$(this).find("pre.peek").css("overflow","hidden")}$(this).slideUp("fast");if(h){delete h[$(this).parent().attr("id")]}});$.jStorage.set("history_expand_state",h)}).show()};b()}function commatize(b){b+="";var a=/(\d+)(\d{3})/;while(a.test(b)){b=b.replace(a,"$1,$2")}return b}function reset_tool_search(a){var c=$("#galaxy_tools").contents();if(c.length===0){c=$(document)}$(this).removeClass("search_active");c.find(".toolTitle").removeClass("search_match");c.find(".toolSectionBody").hide();c.find(".toolTitle").show();c.find(".toolPanelLabel").show();c.find(".toolSectionWrapper").each(function(){if($(this).attr("id")!=="recently_used_wrapper"){$(this).show()}else{if($(this).hasClass("user_pref_visible")){$(this).show()}}});c.find("#search-no-results").hide();c.find("#search-spinner").hide();if(a){var b=c.find("#tool-search-query");b.val("search tools")}}var GalaxyAsync=function(a){this.url_dict={};this.log_action=(a===undefined?false:a)};GalaxyAsync.prototype.set_func_url=function(a,b){this.url_dict[a]=b};GalaxyAsync.prototype.set_user_pref=function(a,b){var c=this.url_dict[arguments.callee];if(c===undefined){return false}$.ajax({url:c,data:{pref_name:a,pref_value:b},error:function(){return false},success:function(){return true}})};GalaxyAsync.prototype.log_user_action=function(c,b,d){if(!this.log_action){return}var a=this.url_dict[arguments.callee];if(a===undefined){return false}$.ajax({url:a,data:{action:c,context:b,params:d},error:function(){return false},success:function(){return true}})};$(document).ready(function(){$("select[refresh_on_change='true']").change(function(){var a=$(this),e=a.val(),d=false,c=a.attr("refresh_on_change_values");if(c){c=c.split(",");var b=a.attr("last_selected_value");if($.inArray(e,c)===-1&&$.inArray(b,c)===-1){return}}$(window).trigger("refresh_on_change");$(document).trigger("convert_to_values");a.get(0).form.submit()});$(":checkbox[refresh_on_change='true']").click(function(){var a=$(this),e=a.val(),d=false,c=a.attr("refresh_on_change_values");if(c){c=c.split(",");var b=a.attr("last_selected_value");if($.inArray(e,c)===-1&&$.inArray(b,c)===-1){return}}$(window).trigger("refresh_on_change");a.get(0).form.submit()});$("a[confirm]").click(function(){return confirm($(this).attr("confirm"))});if($.fn.tooltip){$(".unified-panel-header [title]").tooltip({placement:"bottom"});$("[title]").tooltip({placement:"top"})}make_popup_menus();replace_big_select_inputs(20,1500);$("a").click(function(){var b=$(this);var c=(parent.frames&&parent.frames.galaxy_main);if((b.attr("target")=="galaxy_main")&&(!c)){var a=b.attr("href");if(a.indexOf("?")==-1){a+="?"}else{a+="&"}a+="use_panels=True";b.attr("href",a);b.attr("target","_self")}return b})}); \ No newline at end of file +(function(){var b=0;var c=["ms","moz","webkit","o"];for(var a=0;a<c.length&&!window.requestAnimationFrame;++a){window.requestAnimationFrame=window[c[a]+"RequestAnimationFrame"];window.cancelRequestAnimationFrame=window[c[a]+"CancelRequestAnimationFrame"]}if(!window.requestAnimationFrame){window.requestAnimationFrame=function(h,e){var d=new Date().getTime();var f=Math.max(0,16-(d-b));var g=window.setTimeout(function(){h(d+f)},f);b=d+f;return g}}if(!window.cancelAnimationFrame){window.cancelAnimationFrame=function(d){clearTimeout(d)}}}());if(!Array.indexOf){Array.prototype.indexOf=function(c){for(var b=0,a=this.length;b<a;b++){if(this[b]==c){return b}}return -1}}function obj_length(c){if(c.length!==undefined){return c.length}var b=0;for(var a in c){b++}return b}$.fn.makeAbsolute=function(a){return this.each(function(){var b=$(this);var c=b.position();b.css({position:"absolute",marginLeft:0,marginTop:0,top:c.top,left:c.left,right:$(window).width()-(c.left+b.width())});if(a){b.remove().appendTo("body")}})};function make_popupmenu(b,c){var a=(b.data("menu_options"));b.data("menu_options",c);if(a){return}b.bind("click.show_popup",function(d){$(".popmenu-wrapper").remove();setTimeout(function(){var g=$("<ul class='dropdown-menu' id='"+b.attr("id")+"-menu'></ul>");var f=b.data("menu_options");if(obj_length(f)<=0){$("<li>No Options.</li>").appendTo(g)}$.each(f,function(j,i){if(i){var l=i.action||i;g.append($("<li></li>").append($("<a>").attr("href",i.url).html(j).click(l)))}else{g.append($("<li></li>").addClass("head").append($("<a href='#'></a>").html(j)))}});var h=$("<div class='popmenu-wrapper' style='position: absolute;left: 0; top: -1000;'></div>").append(g).appendTo("body");var e=d.pageX-h.width()/2;e=Math.min(e,$(document).scrollLeft()+$(window).width()-$(h).width()-5);e=Math.max(e,$(document).scrollLeft()+5);h.css({top:d.pageY,left:e})},10);setTimeout(function(){var f=function(h){$(h).bind("click.close_popup",function(){$(".popmenu-wrapper").remove();h.unbind("click.close_popup")})};f($(window.document));f($(window.top.document));for(var e=window.top.frames.length;e--;){var g=$(window.top.frames[e].document);f(g)}},50);return false})}function make_popup_menus(a){a=a||document;$(a).find("div[popupmenu]").each(function(){var b={};var d=$(this);d.find("a").each(function(){var g=$(this),i=g.get(0),e=i.getAttribute("confirm"),f=i.getAttribute("href"),h=i.getAttribute("target");if(!f){b[g.text()]=null}else{b[g.text()]={url:f,action:function(){if(!e||confirm(e)){if(h){window.open(f,h)}else{g.click()}}}}}});var c=$(a).find("#"+d.attr("popupmenu"));c.find("a").bind("click",function(f){f.stopPropagation();return true});make_popupmenu(c,b);c.addClass("popup");d.remove()})}function naturalSort(j,h){var p=/(-?[0-9\.]+)/g,k=j.toString().toLowerCase()||"",g=h.toString().toLowerCase()||"",l=String.fromCharCode(0),n=k.replace(p,l+"$1"+l).split(l),e=g.replace(p,l+"$1"+l).split(l),d=(new Date(k)).getTime(),o=d?(new Date(g)).getTime():null;if(o){if(d<o){return -1}else{if(d>o){return 1}}}var m,f;for(var i=0,c=Math.max(n.length,e.length);i<c;i++){m=parseFloat(n[i])||n[i];f=parseFloat(e[i])||e[i];if(m<f){return -1}else{if(m>f){return 1}}}return 0}$.fn.refresh_select2=function(){var b=$(this);var a={width:"resolve",closeOnSelect:!b.is("[MULTIPLE]")};return b.select2(a)};function replace_big_select_inputs(a,c,b){if(!jQuery.fn.select2){return}if(a===undefined){a=20}if(c===undefined){c=3000}b=b||$("select");b.each(function(){var e=$(this);var d=e.find("option").length;if((d<a)||(d>c)){return}if(e.hasClass("no-autocomplete")){return}e.refresh_select2()})}$.fn.make_text_editable=function(g){var d=("num_cols" in g?g.num_cols:30),c=("num_rows" in g?g.num_rows:4),e=("use_textarea" in g?g.use_textarea:false),b=("on_finish" in g?g.on_finish:null),f=("help_text" in g?g.help_text:null);var a=$(this);a.addClass("editable-text").click(function(l){if($(this).children(":input").length>0){return}a.removeClass("editable-text");var i=function(m){a.find(":input").remove();if(m!==""){a.text(m)}else{a.html("<br>")}a.addClass("editable-text");if(b){b(m)}};var h=("cur_text" in g?g.cur_text:a.text()),k,j;if(e){k=$("<textarea/>").attr({rows:c,cols:d}).text($.trim(h)).keyup(function(m){if(m.keyCode===27){i(h)}});j=$("<button/>").text("Done").click(function(){i(k.val());return false})}else{k=$("<input type='text'/>").attr({value:$.trim(h),size:d}).blur(function(){i(h)}).keyup(function(m){if(m.keyCode===27){$(this).trigger("blur")}else{if(m.keyCode===13){i($(this).val())}}})}a.text("");a.append(k);if(j){a.append(j)}k.focus();k.select();l.stopPropagation()});if(f){a.attr("title",f).tooltip()}return a};function async_save_text(d,f,e,a,c,h,i,g,b){if(c===undefined){c=30}if(i===undefined){i=4}$("#"+d).click(function(){if($("#renaming-active").length>0){return}var l=$("#"+f),k=l.text(),j;if(h){j=$("<textarea></textarea>").attr({rows:i,cols:c}).text($.trim(k))}else{j=$("<input type='text'></input>").attr({value:$.trim(k),size:c})}j.attr("id","renaming-active");j.blur(function(){$(this).remove();l.show();if(b){b(j)}});j.keyup(function(n){if(n.keyCode===27){$(this).trigger("blur")}else{if(n.keyCode===13){var m={};m[a]=$(this).val();$(this).trigger("blur");$.ajax({url:e,data:m,error:function(){alert("Text editing for elt "+f+" failed")},success:function(o){if(o!==""){l.text(o)}else{l.html("<em>None</em>")}if(b){b(j)}}})}}});if(g){g(j)}l.hide();j.insertAfter(l);j.focus();j.select();return})}function init_history_items(d,a,c){var b=function(){try{var e=$.jStorage.get("history_expand_state");if(e){for(var g in e){$("#"+g+" div.historyItemBody").show()}}}catch(f){$.jStorage.deleteKey("history_expand_state")}if($.browser.mozilla){$("div.historyItemBody").each(function(){if(!$(this).is(":visible")){$(this).find("pre.peek").css("overflow","hidden")}})}d.each(function(){var j=this.id,h=$(this).children("div.historyItemBody"),i=h.find("pre.peek");$(this).find(".historyItemTitleBar > .historyItemTitle").wrap("<a href='javascript:void(0);'></a>").click(function(){var k;if(h.is(":visible")){if($.browser.mozilla){i.css("overflow","hidden")}h.slideUp("fast");if(!c){k=$.jStorage.get("history_expand_state");if(k){delete k[j];$.jStorage.set("history_expand_state",k)}}}else{h.slideDown("fast",function(){if($.browser.mozilla){i.css("overflow","auto")}});if(!c){k=$.jStorage.get("history_expand_state");if(!k){k={}}k[j]=true;$.jStorage.set("history_expand_state",k)}}return false})});$("#top-links > a.toggle").click(function(){var h=$.jStorage.get("history_expand_state");if(!h){h={}}$("div.historyItemBody:visible").each(function(){if($.browser.mozilla){$(this).find("pre.peek").css("overflow","hidden")}$(this).slideUp("fast");if(h){delete h[$(this).parent().attr("id")]}});$.jStorage.set("history_expand_state",h)}).show()};b()}function commatize(b){b+="";var a=/(\d+)(\d{3})/;while(a.test(b)){b=b.replace(a,"$1,$2")}return b}function reset_tool_search(a){var c=$("#galaxy_tools").contents();if(c.length===0){c=$(document)}$(this).removeClass("search_active");c.find(".toolTitle").removeClass("search_match");c.find(".toolSectionBody").hide();c.find(".toolTitle").show();c.find(".toolPanelLabel").show();c.find(".toolSectionWrapper").each(function(){if($(this).attr("id")!=="recently_used_wrapper"){$(this).show()}else{if($(this).hasClass("user_pref_visible")){$(this).show()}}});c.find("#search-no-results").hide();c.find("#search-spinner").hide();if(a){var b=c.find("#tool-search-query");b.val("search tools")}}var GalaxyAsync=function(a){this.url_dict={};this.log_action=(a===undefined?false:a)};GalaxyAsync.prototype.set_func_url=function(a,b){this.url_dict[a]=b};GalaxyAsync.prototype.set_user_pref=function(a,b){var c=this.url_dict[arguments.callee];if(c===undefined){return false}$.ajax({url:c,data:{pref_name:a,pref_value:b},error:function(){return false},success:function(){return true}})};GalaxyAsync.prototype.log_user_action=function(c,b,d){if(!this.log_action){return}var a=this.url_dict[arguments.callee];if(a===undefined){return false}$.ajax({url:a,data:{action:c,context:b,params:d},error:function(){return false},success:function(){return true}})};function init_refresh_on_change(){$("select[refresh_on_change='true']").change(function(){var a=$(this),e=a.val(),d=false,c=a.attr("refresh_on_change_values");if(c){c=c.split(",");var b=a.attr("last_selected_value");if($.inArray(e,c)===-1&&$.inArray(b,c)===-1){return}}$(window).trigger("refresh_on_change");$(document).trigger("convert_to_values");a.get(0).form.submit()});$(":checkbox[refresh_on_change='true']").click(function(){var a=$(this),e=a.val(),d=false,c=a.attr("refresh_on_change_values");if(c){c=c.split(",");var b=a.attr("last_selected_value");if($.inArray(e,c)===-1&&$.inArray(b,c)===-1){return}}$(window).trigger("refresh_on_change");a.get(0).form.submit()});$("a[confirm]").click(function(){return confirm($(this).attr("confirm"))})}$(document).ready(function(){init_refresh_on_change();if($.fn.tooltip){$(".unified-panel-header [title]").tooltip({placement:"bottom"});$("[title]").tooltip({placement:"top"})}make_popup_menus();replace_big_select_inputs(20,1500);$("a").click(function(){var b=$(this);var c=(parent.frames&&parent.frames.galaxy_main);if((b.attr("target")=="galaxy_main")&&(!c)){var a=b.attr("href");if(a.indexOf("?")==-1){a+="?"}else{a+="&"}a+="use_panels=True";b.attr("href",a);b.attr("target","_self")}return b})}); \ No newline at end of file diff -r 984377efb491a30e4afd320c9fbff2ff1e1241af -r c04e79918595e0cefe85edac156ff5a962f4a686 static/scripts/packed/galaxy.grids.js --- a/static/scripts/packed/galaxy.grids.js +++ b/static/scripts/packed/galaxy.grids.js @@ -1,1 +1,1 @@ -jQuery.ajaxSettings.traditional=true;define(["mvc/ui"],function(){var a=Backbone.Model.extend({defaults:{url_base:"",async:false,async_ops:[],categorical_filters:[],filters:{},sort_key:null,show_item_checkboxes:false,advanced_search:false,cur_page:1,num_pages:1,operation:undefined,item_ids:undefined},can_async_op:function(c){return _.indexOf(this.attributes.async_ops,c)!==-1},add_filter:function(g,h,d){if(d){var e=this.attributes.filters[g],c;if(e===null||e===undefined){c=h}else{if(typeof(e)=="string"){if(e=="All"){c=h}else{var f=[];f[0]=e;f[1]=h;c=f}}else{c=e;c.push(h)}}this.attributes.filters[g]=c}else{this.attributes.filters[g]=h}},remove_filter:function(d,g){var c=this.attributes.filters[d];if(c===null||c===undefined){return false}var f=true;if(typeof(c)==="string"){if(c=="All"){f=false}else{delete this.attributes.filters[d]}}else{var e=_.indexOf(c,g);if(e!==-1){c.splice(e,1)}else{f=false}}return f},get_url_data:function(){var c={async:this.attributes.async,sort:this.attributes.sort_key,page:this.attributes.cur_page,show_item_checkboxes:this.attributes.show_item_checkboxes,advanced_search:this.attributes.advanced_search};if(this.attributes.operation){c.operation=this.attributes.operation}if(this.attributes.item_ids){c.id=this.attributes.item_ids}var d=this;_.each(_.pairs(d.attributes.filters),function(e){c["f-"+e[0]]=e[1]});return c},get_url:function(c){return this.get("url_base")+"?"+$.param(this.get_url_data())+"&"+$.param(c)}});var b=Backbone.View.extend({grid:null,initialize:function(c){this.init_grid(c);this.init_grid_controls();$("input[type=text]").each(function(){$(this).click(function(){$(this).select()}).keyup(function(){$(this).css("font-style","normal")})})},handle_refresh:function(c){if(c){if($.inArray("history",c)>-1){if(top.Galaxy&&top.Galaxy.currHistoryPanel){top.Galaxy.currHistoryPanel.loadCurrentHistory()}}}},init_grid:function(e){this.grid=new a(e);var d=this.grid.attributes;this.handle_refresh(d.refresh_frames);var c=this.grid.get("url_base");c=c.replace(/^.*\/\/[^\/]+/,"");this.grid.set("url_base",c);$("#grid-table-body").html(this.template_body(d));$("#grid-table-footer").html(this.template_footer(d));if(d.message){$("#grid-message").html(this.template_message(d));setTimeout(function(){$("#grid-message").html("")},5000)}this.init_grid_elements()},init_grid_controls:function(){$(".submit-image").each(function(){$(this).mousedown(function(){$(this).addClass("gray-background")});$(this).mouseup(function(){$(this).removeClass("gray-background")})});var c=this;$(".sort-link").each(function(){$(this).click(function(){c.set_sort_condition($(this).attr("sort_key"));return false})});$(".categorical-filter > a").each(function(){$(this).click(function(){c.set_categorical_filter($(this).attr("filter_key"),$(this).attr("filter_val"));return false})});$(".text-filter-form").each(function(){$(this).submit(function(){var g=$(this).attr("column_key");var f=$("#input-"+g+"-filter");var h=f.val();f.val("");c.add_filter_condition(g,h);return false})});var d=$("#input-tags-filter");if(d.length){d.autocomplete(this.grid.history_tag_autocomplete_url,{selectFirst:false,autoFill:false,highlight:false,mustMatch:false})}var e=$("#input-name-filter");if(e.length){e.autocomplete(this.grid.history_name_autocomplete_url,{selectFirst:false,autoFill:false,highlight:false,mustMatch:false})}$(".advanced-search-toggle").each(function(){$(this).click(function(){$("#standard-search").slideToggle("fast");$("#advanced-search").slideToggle("fast");return false})})},init_grid_elements:function(){$(".grid").each(function(){var s=$(this).find("input.grid-row-select-checkbox");var r=$(this).find("span.grid-selected-count");var t=function(){r.text($(s).filter(":checked").length)};$(s).each(function(){$(this).change(t)});t()});if($(".community_rating_star").length!==0){$(".community_rating_star").rating({})}var q=this.grid.attributes;var p=this;$(".page-link > a").each(function(){$(this).click(function(){p.set_page($(this).attr("page_num"));return false})});$(".use-inbound").each(function(){$(this).click(function(r){p.execute({href:$(this).attr("href"),inbound:true});return false})});$(".use-outbound").each(function(){$(this).click(function(r){p.execute({href:$(this).attr("href")});return false})});var f=q.items.length;if(f==0){return}for(var k in q.items){var o=q.items[k];var l=$("#grid-"+k+"-popup");l.off();var d=new PopupMenu(l);for(var h in q.operations){var e=q.operations[h];var m=e.label;var c=o.operation_config[m];var g=o.encode_id;if(c.allowed&&e.allow_popup){var n={html:e.label,href:c.url_args,target:c.target,confirmation_text:e.confirm,inbound:e.inbound};n.func=function(t){t.preventDefault();var s=$(t.target).html();var r=this.findItemByHtml(s);p.execute(r)};d.addItem(n)}}}},add_filter_condition:function(e,g){if(g===""){return false}this.grid.add_filter(e,g,true);var f=$("<span>"+g+"<a href='javascript:void(0);'><span class='delete-search-icon' /></span></a>");f.addClass("text-filter-val");var d=this;f.click(function(){d.grid.remove_filter(e,g);$(this).remove();d.go_page_one();d.execute()});var c=$("#"+e+"-filtering-criteria");c.append(f);this.go_page_one();this.execute()},set_sort_condition:function(h){var g=this.grid.get("sort_key");var f=h;if(g.indexOf(h)!==-1){if(g.substring(0,1)!=="-"){f="-"+h}else{}}$(".sort-arrow").remove();var e=(f.substring(0,1)=="-")?"↑":"↓";var c=$("<span>"+e+"</span>").addClass("sort-arrow");var d=$("#"+h+"-header");d.append(c);this.grid.set("sort_key",f);this.go_page_one();this.execute()},set_categorical_filter:function(e,g){var d=this.grid.get("categorical_filters")[e],f=this.grid.get("filters")[e];var c=this;$("."+e+"-filter").each(function(){var m=$.trim($(this).text());var k=d[m];var l=k[e];if(l==g){$(this).empty();$(this).addClass("current-filter");$(this).append(m)}else{if(l==f){$(this).empty();var h=$("<a href='#'>"+m+"</a>");h.click(function(){c.set_categorical_filter(e,l)});$(this).removeClass("current-filter");$(this).append(h)}}});this.grid.add_filter(e,g);this.go_page_one();this.execute()},set_page:function(c){var d=this;$(".page-link").each(function(){var k=$(this).attr("id"),g=parseInt(k.split("-")[2],10),e=d.grid.get("cur_page"),h;if(g===c){h=$(this).children().text();$(this).empty();$(this).addClass("inactive-link");$(this).text(h)}else{if(g===e){h=$(this).text();$(this).empty();$(this).removeClass("inactive-link");var f=$("<a href='#'>"+h+"</a>");f.click(function(){d.set_page(g)});$(this).append(f)}}});if(c==="all"){this.grid.set("cur_page",c)}else{this.grid.set("cur_page",parseInt(c,10))}this.execute()},submit_operation:function(f,g){var e=$('input[name="id"]:checked').length;if(!e>0){return false}var d=$(f).val();var c=[];$("input[name=id]:checked").each(function(){c.push($(this).val())});this.execute({operation:d,id:c,confirmation_text:g});return true},execute:function(n){var f=null;var e=null;var g=null;var c=null;var m=null;if(n){e=n.href;g=n.operation;f=n.id;c=n.confirmation_text;m=n.inbound;if(e!==undefined&&e.indexOf("operation=")!=-1){var l=e.split("?");if(l.length>1){var k=l[1];var d=k.split("&");for(var h=0;h<d.length;h++){if(d[h].indexOf("operation")!=-1){g=d[h].split("=")[1];g=g.replace(/\+/g," ")}else{if(d[h].indexOf("id")!=-1){f=d[h].split("=")[1]}}}}}}if(g&&f){if(c&&c!=""&&c!="None"&&c!="null"){if(!confirm(c)){return false}}g=g.toLowerCase();this.grid.set({operation:g,item_ids:f});if(this.grid.can_async_op(g)){this.update_grid()}else{this.go_to(m,"")}return false}if(e){this.go_to(m,e);return false}if(this.grid.get("async")){this.update_grid()}else{this.go_to(m,"")}return false},go_to:function(f,d){var e=this.grid.get("async");this.grid.set("async",false);advanced_search=$("#advanced-search").is(":visible");this.grid.set("advanced_search",advanced_search);if(!d){d=this.grid.get("url_base")+"?"+$.param(this.grid.get_url_data())}this.grid.set({operation:undefined,item_ids:undefined,async:e});if(f){var c=$(".grid-header").closest(".inbound");if(c.length!==0){c.load(d);return}}window.location=d},update_grid:function(){var d=(this.grid.get("operation")?"POST":"GET");$(".loading-elt-overlay").show();var c=this;$.ajax({type:d,url:c.grid.get("url_base"),data:c.grid.get_url_data(),error:function(e){alert("Grid refresh failed")},success:function(e){c.init_grid($.parseJSON(e));$(".loading-elt-overlay").hide()},complete:function(){c.grid.set({operation:undefined,item_ids:undefined})}})},check_all_items:function(){var c=document.getElementById("check_all"),d=document.getElementsByTagName("input"),f=0,e;if(c.checked===true){for(e=0;e<d.length;e++){if(d[e].name.indexOf("id")!==-1){d[e].checked=true;f++}}}else{for(e=0;e<d.length;e++){if(d[e].name.indexOf("id")!==-1){d[e].checked=false}}}this.init_grid_elements()},go_page_one:function(){var c=this.grid.get("cur_page");if(c!==null&&c!==undefined&&c!=="all"){this.grid.set("cur_page",1)}},template_body:function(t){var m="";var u=0;var g=t.items.length;if(g==0){m+='<tr><td colspan="100"><em>No Items</em></td></tr>';u=1}for(var h in t.items){var r=t.items[h];var c=r.encode_id;var k="grid-"+h+"-popup";m+="<tr ";if(t.current_item_id==r.id){m+='class="current"'}m+=">";if(t.show_item_checkboxes){m+='<td style="width: 1.5em;"><input type="checkbox" name="id" value="'+c+'" id="'+c+'" class="grid-row-select-checkbox" /></td>'}for(j in t.columns){var f=t.columns[j];if(f.visible){var e="";if(f.nowrap){e='style="white-space:nowrap;"'}var s=r.column_config[f.label];var l=s.link;var n=s.value;var q=s.inbound;if(jQuery.type(n)==="string"){n=n.replace(/\/\//g,"/")}var d="";var p="";if(f.attach_popup){d="grid-"+h+"-popup";p="menubutton";if(l!=""){p+=" split"}p+=" popup"}m+="<td "+e+">";if(l){if(t.operations.length!=0){m+='<div id="'+d+'" class="'+p+'" style="float: left;">'}var o="";if(q){o="use-inbound"}else{o="use-outbound"}m+='<a class="label '+o+'" href="'+l+'" onclick="return false;">'+n+"</a>";if(t.operations.length!=0){m+="</div>"}}else{m+='<div id="'+d+'" class="'+p+'"><label id="'+f.label_id_prefix+c+'" for="'+c+'">'+n+"</label></div>"}m+="</td>"}}m+="</tr>";u++}return m},template_footer:function(q){var m="";if(q.use_paging&&q.num_pages>1){var o=q.num_page_links;var c=q.cur_page_num;var p=q.num_pages;var l=o/2;var k=c-l;var g=0;if(k==0){k=1;g=l-(c-k)}var f=l+g;var e=c+f;if(e<=p){max_offset=0}else{e=p;max_offset=f-(e+1-c)}if(max_offset!=0){k-=max_offset;if(k<1){k=1}}m+='<tr id="page-links-row">';if(q.show_item_checkboxes){m+="<td></td>"}m+='<td colspan="100"><span id="page-link-container">Page:';if(k>1){m+='<span class="page-link" id="page-link-1"><a href="'+this.grid.get_url({page:n})+'" page_num="1" onclick="return false;">1</a></span> ...'}for(var n=k;n<e+1;n++){if(n==q.cur_page_num){m+='<span class="page-link inactive-link" id="page-link-'+n+'">'+n+"</span>"}else{m+='<span class="page-link" id="page-link-'+n+'"><a href="'+this.grid.get_url({page:n})+'" onclick="return false;" page_num="'+n+'">'+n+"</a></span>"}}if(e<p){m+='...<span class="page-link" id="page-link-'+p+'"><a href="'+this.grid.get_url({page:p})+'" onclick="return false;" page_num="'+p+'">'+p+"</a></span>"}m+="</span>";m+='<span class="page-link" id="show-all-link-span"> | <a href="'+this.grid.get_url({page:"all"})+'" onclick="return false;" page_num="all">Show All</a></span></td></tr>'}if(q.show_item_checkboxes){m+='<tr><input type="hidden" id="operation" name="operation" value=""><td></td><td colspan="100">For <span class="grid-selected-count"></span> selected '+q.get_class_plural+": ";for(i in q.operations){var d=q.operations[i];if(d.allow_multiple){m+='<input type="button" value="'+d.label+'" class="action-button" onclick="gridView.submit_operation(this, \''+d.confirm+"')\"> "}}m+="</td></tr>"}var h=false;for(i in q.operations){if(q.operations[i].global_operation){h=true;break}}if(h){m+='<tr><td colspan="100">';for(i in q.operations){var d=q.operations[i];if(d.global_operation){m+='<a class="action-button" href="'+d.global_operation+'">'+d.label+"</a>"}}m+="</td></tr>"}if(q.legend){m+='<tr><td colspan="100">'+q.legend+"</td></tr>"}return m},template_message:function(c){return'<p><div class="'+c.status+'message transient-message">'+c.message+'</div><div style="clear: both"></div></p>'}});return{Grid:a,GridView:b}}); \ No newline at end of file +jQuery.ajaxSettings.traditional=true;define(["mvc/ui"],function(){var a=Backbone.Model.extend({defaults:{url_base:"",async:false,async_ops:[],categorical_filters:[],filters:{},sort_key:null,show_item_checkboxes:false,advanced_search:false,cur_page:1,num_pages:1,operation:undefined,item_ids:undefined},can_async_op:function(c){return _.indexOf(this.attributes.async_ops,c)!==-1},add_filter:function(g,h,d){if(d){var e=this.attributes.filters[g],c;if(e===null||e===undefined){c=h}else{if(typeof(e)=="string"){if(e=="All"){c=h}else{var f=[];f[0]=e;f[1]=h;c=f}}else{c=e;c.push(h)}}this.attributes.filters[g]=c}else{this.attributes.filters[g]=h}},remove_filter:function(d,g){var c=this.attributes.filters[d];if(c===null||c===undefined){return false}var f=true;if(typeof(c)==="string"){if(c=="All"){f=false}else{delete this.attributes.filters[d]}}else{var e=_.indexOf(c,g);if(e!==-1){c.splice(e,1)}else{f=false}}return f},get_url_data:function(){var c={async:this.attributes.async,sort:this.attributes.sort_key,page:this.attributes.cur_page,show_item_checkboxes:this.attributes.show_item_checkboxes,advanced_search:this.attributes.advanced_search};if(this.attributes.operation){c.operation=this.attributes.operation}if(this.attributes.item_ids){c.id=this.attributes.item_ids}var d=this;_.each(_.pairs(d.attributes.filters),function(e){c["f-"+e[0]]=e[1]});return c},get_url:function(c){return this.get("url_base")+"?"+$.param(this.get_url_data())+"&"+$.param(c)}});var b=Backbone.View.extend({grid:null,initialize:function(c){this.init_grid(c);this.init_grid_controls();$("input[type=text]").each(function(){$(this).click(function(){$(this).select()}).keyup(function(){$(this).css("font-style","normal")})})},handle_refresh:function(c){if(c){if($.inArray("history",c)>-1){if(top.Galaxy&&top.Galaxy.currHistoryPanel){top.Galaxy.currHistoryPanel.loadCurrentHistory()}}}},init_grid:function(e){this.grid=new a(e);var d=this.grid.attributes;this.handle_refresh(d.refresh_frames);var c=this.grid.get("url_base");c=c.replace(/^.*\/\/[^\/]+/,"");this.grid.set("url_base",c);$("#grid-table-body").html(this.template_body(d));$("#grid-table-footer").html(this.template_footer(d));if(d.message){$("#grid-message").html(this.template_message(d));setTimeout(function(){$("#grid-message").html("")},5000)}this.init_grid_elements();init_refresh_on_change()},init_grid_controls:function(){$(".submit-image").each(function(){$(this).mousedown(function(){$(this).addClass("gray-background")});$(this).mouseup(function(){$(this).removeClass("gray-background")})});var c=this;$(".sort-link").each(function(){$(this).click(function(){c.set_sort_condition($(this).attr("sort_key"));return false})});$(".categorical-filter > a").each(function(){$(this).click(function(){c.set_categorical_filter($(this).attr("filter_key"),$(this).attr("filter_val"));return false})});$(".text-filter-form").each(function(){$(this).submit(function(){var g=$(this).attr("column_key");var f=$("#input-"+g+"-filter");var h=f.val();f.val("");c.add_filter_condition(g,h);return false})});var d=$("#input-tags-filter");if(d.length){d.autocomplete(this.grid.history_tag_autocomplete_url,{selectFirst:false,autoFill:false,highlight:false,mustMatch:false})}var e=$("#input-name-filter");if(e.length){e.autocomplete(this.grid.history_name_autocomplete_url,{selectFirst:false,autoFill:false,highlight:false,mustMatch:false})}$(".advanced-search-toggle").each(function(){$(this).click(function(){$("#standard-search").slideToggle("fast");$("#advanced-search").slideToggle("fast");return false})})},init_grid_elements:function(){$(".grid").each(function(){var s=$(this).find("input.grid-row-select-checkbox");var r=$(this).find("span.grid-selected-count");var t=function(){r.text($(s).filter(":checked").length)};$(s).each(function(){$(this).change(t)});t()});if($(".community_rating_star").length!==0){$(".community_rating_star").rating({})}var q=this.grid.attributes;var p=this;$(".page-link > a").each(function(){$(this).click(function(){p.set_page($(this).attr("page_num"));return false})});$(".use-inbound").each(function(){$(this).click(function(r){p.execute({href:$(this).attr("href"),inbound:true});return false})});$(".use-outbound").each(function(){$(this).click(function(r){p.execute({href:$(this).attr("href")});return false})});var f=q.items.length;if(f==0){return}for(var k in q.items){var o=q.items[k];var l=$("#grid-"+k+"-popup");l.off();var d=new PopupMenu(l);for(var h in q.operations){var e=q.operations[h];var m=e.label;var c=o.operation_config[m];var g=o.encode_id;if(c.allowed&&e.allow_popup){var n={html:e.label,href:c.url_args,target:c.target,confirmation_text:e.confirm,inbound:e.inbound};n.func=function(t){t.preventDefault();var s=$(t.target).html();var r=this.findItemByHtml(s);p.execute(r)};d.addItem(n)}}}},add_filter_condition:function(e,g){if(g===""){return false}this.grid.add_filter(e,g,true);var f=$("<span>"+g+"<a href='javascript:void(0);'><span class='delete-search-icon' /></span></a>");f.addClass("text-filter-val");var d=this;f.click(function(){d.grid.remove_filter(e,g);$(this).remove();d.go_page_one();d.execute()});var c=$("#"+e+"-filtering-criteria");c.append(f);this.go_page_one();this.execute()},set_sort_condition:function(h){var g=this.grid.get("sort_key");var f=h;if(g.indexOf(h)!==-1){if(g.substring(0,1)!=="-"){f="-"+h}else{}}$(".sort-arrow").remove();var e=(f.substring(0,1)=="-")?"↑":"↓";var c=$("<span>"+e+"</span>").addClass("sort-arrow");var d=$("#"+h+"-header");d.append(c);this.grid.set("sort_key",f);this.go_page_one();this.execute()},set_categorical_filter:function(e,g){var d=this.grid.get("categorical_filters")[e],f=this.grid.get("filters")[e];var c=this;$("."+e+"-filter").each(function(){var m=$.trim($(this).text());var k=d[m];var l=k[e];if(l==g){$(this).empty();$(this).addClass("current-filter");$(this).append(m)}else{if(l==f){$(this).empty();var h=$("<a href='#'>"+m+"</a>");h.click(function(){c.set_categorical_filter(e,l)});$(this).removeClass("current-filter");$(this).append(h)}}});this.grid.add_filter(e,g);this.go_page_one();this.execute()},set_page:function(c){var d=this;$(".page-link").each(function(){var k=$(this).attr("id"),g=parseInt(k.split("-")[2],10),e=d.grid.get("cur_page"),h;if(g===c){h=$(this).children().text();$(this).empty();$(this).addClass("inactive-link");$(this).text(h)}else{if(g===e){h=$(this).text();$(this).empty();$(this).removeClass("inactive-link");var f=$("<a href='#'>"+h+"</a>");f.click(function(){d.set_page(g)});$(this).append(f)}}});if(c==="all"){this.grid.set("cur_page",c)}else{this.grid.set("cur_page",parseInt(c,10))}this.execute()},submit_operation:function(f,g){var e=$('input[name="id"]:checked').length;if(!e>0){return false}var d=$(f).val();var c=[];$("input[name=id]:checked").each(function(){c.push($(this).val())});this.execute({operation:d,id:c,confirmation_text:g});return true},execute:function(n){var f=null;var e=null;var g=null;var c=null;var m=null;if(n){e=n.href;g=n.operation;f=n.id;c=n.confirmation_text;m=n.inbound;if(e!==undefined&&e.indexOf("operation=")!=-1){var l=e.split("?");if(l.length>1){var k=l[1];var d=k.split("&");for(var h=0;h<d.length;h++){if(d[h].indexOf("operation")!=-1){g=d[h].split("=")[1];g=g.replace(/\+/g," ")}else{if(d[h].indexOf("id")!=-1){f=d[h].split("=")[1]}}}}}}if(g&&f){if(c&&c!=""&&c!="None"&&c!="null"){if(!confirm(c)){return false}}g=g.toLowerCase();this.grid.set({operation:g,item_ids:f});if(this.grid.can_async_op(g)){this.update_grid()}else{this.go_to(m,"")}return false}if(e){this.go_to(m,e);return false}if(this.grid.get("async")){this.update_grid()}else{this.go_to(m,"")}return false},go_to:function(f,d){var e=this.grid.get("async");this.grid.set("async",false);advanced_search=$("#advanced-search").is(":visible");this.grid.set("advanced_search",advanced_search);if(!d){d=this.grid.get("url_base")+"?"+$.param(this.grid.get_url_data())}this.grid.set({operation:undefined,item_ids:undefined,async:e});if(f){var c=$(".grid-header").closest(".inbound");if(c.length!==0){c.load(d);return}}window.location=d},update_grid:function(){var d=(this.grid.get("operation")?"POST":"GET");$(".loading-elt-overlay").show();var c=this;$.ajax({type:d,url:c.grid.get("url_base"),data:c.grid.get_url_data(),error:function(e){alert("Grid refresh failed")},success:function(e){c.init_grid($.parseJSON(e));$(".loading-elt-overlay").hide()},complete:function(){c.grid.set({operation:undefined,item_ids:undefined})}})},check_all_items:function(){var c=document.getElementById("check_all"),d=document.getElementsByTagName("input"),f=0,e;if(c.checked===true){for(e=0;e<d.length;e++){if(d[e].name.indexOf("id")!==-1){d[e].checked=true;f++}}}else{for(e=0;e<d.length;e++){if(d[e].name.indexOf("id")!==-1){d[e].checked=false}}}this.init_grid_elements()},go_page_one:function(){var c=this.grid.get("cur_page");if(c!==null&&c!==undefined&&c!=="all"){this.grid.set("cur_page",1)}},template_body:function(t){var m="";var u=0;var g=t.items.length;if(g==0){m+='<tr><td colspan="100"><em>No Items</em></td></tr>';u=1}for(var h in t.items){var r=t.items[h];var c=r.encode_id;var k="grid-"+h+"-popup";m+="<tr ";if(t.current_item_id==r.id){m+='class="current"'}m+=">";if(t.show_item_checkboxes){m+='<td style="width: 1.5em;"><input type="checkbox" name="id" value="'+c+'" id="'+c+'" class="grid-row-select-checkbox" /></td>'}for(j in t.columns){var f=t.columns[j];if(f.visible){var e="";if(f.nowrap){e='style="white-space:nowrap;"'}var s=r.column_config[f.label];var l=s.link;var n=s.value;var q=s.inbound;if(jQuery.type(n)==="string"){n=n.replace(/\/\//g,"/")}var d="";var p="";if(f.attach_popup){d="grid-"+h+"-popup";p="menubutton";if(l!=""){p+=" split"}p+=" popup"}m+="<td "+e+">";if(l){if(t.operations.length!=0){m+='<div id="'+d+'" class="'+p+'" style="float: left;">'}var o="";if(q){o="use-inbound"}else{o="use-outbound"}m+='<a class="label '+o+'" href="'+l+'" onclick="return false;">'+n+"</a>";if(t.operations.length!=0){m+="</div>"}}else{m+='<div id="'+d+'" class="'+p+'"><label id="'+f.label_id_prefix+c+'" for="'+c+'">'+n+"</label></div>"}m+="</td>"}}m+="</tr>";u++}return m},template_footer:function(q){var m="";if(q.use_paging&&q.num_pages>1){var o=q.num_page_links;var c=q.cur_page_num;var p=q.num_pages;var l=o/2;var k=c-l;var g=0;if(k==0){k=1;g=l-(c-k)}var f=l+g;var e=c+f;if(e<=p){max_offset=0}else{e=p;max_offset=f-(e+1-c)}if(max_offset!=0){k-=max_offset;if(k<1){k=1}}m+='<tr id="page-links-row">';if(q.show_item_checkboxes){m+="<td></td>"}m+='<td colspan="100"><span id="page-link-container">Page:';if(k>1){m+='<span class="page-link" id="page-link-1"><a href="'+this.grid.get_url({page:n})+'" page_num="1" onclick="return false;">1</a></span> ...'}for(var n=k;n<e+1;n++){if(n==q.cur_page_num){m+='<span class="page-link inactive-link" id="page-link-'+n+'">'+n+"</span>"}else{m+='<span class="page-link" id="page-link-'+n+'"><a href="'+this.grid.get_url({page:n})+'" onclick="return false;" page_num="'+n+'">'+n+"</a></span>"}}if(e<p){m+='...<span class="page-link" id="page-link-'+p+'"><a href="'+this.grid.get_url({page:p})+'" onclick="return false;" page_num="'+p+'">'+p+"</a></span>"}m+="</span>";m+='<span class="page-link" id="show-all-link-span"> | <a href="'+this.grid.get_url({page:"all"})+'" onclick="return false;" page_num="all">Show All</a></span></td></tr>'}if(q.show_item_checkboxes){m+='<tr><input type="hidden" id="operation" name="operation" value=""><td></td><td colspan="100">For <span class="grid-selected-count"></span> selected '+q.get_class_plural+": ";for(i in q.operations){var d=q.operations[i];if(d.allow_multiple){m+='<input type="button" value="'+d.label+'" class="action-button" onclick="gridView.submit_operation(this, \''+d.confirm+"')\"> "}}m+="</td></tr>"}var h=false;for(i in q.operations){if(q.operations[i].global_operation){h=true;break}}if(h){m+='<tr><td colspan="100">';for(i in q.operations){var d=q.operations[i];if(d.global_operation){m+='<a class="action-button" href="'+d.global_operation+'">'+d.label+"</a>"}}m+="</td></tr>"}if(q.legend){m+='<tr><td colspan="100">'+q.legend+"</td></tr>"}return m},template_message:function(c){return'<p><div class="'+c.status+'message transient-message">'+c.message+'</div><div style="clear: both"></div></p>'}});return{Grid:a,GridView:b}}); \ No newline at end of file Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: carlfeberhard: History & HDA API: fix unicode support for tags in update
by commits-noreply＠bitbucket.org 10 Jan '14

10 Jan '14

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/984377efb491/ Changeset: 984377efb491 User: carlfeberhard Date: 2014-01-10 19:02:26 Summary: History & HDA API: fix unicode support for tags in update Affected #: 1 file diff -r b142bc80747e5079b7f406259dc346cdba86a044 -r 984377efb491a30e4afd320c9fbff2ff1e1241af lib/galaxy/web/base/controller.py --- a/lib/galaxy/web/base/controller.py +++ b/lib/galaxy/web/base/controller.py @@ -2469,7 +2469,7 @@ # based on controllers/tag retag_async: delete all old, reset to entire new trans.app.tag_handler.delete_item_tags( trans, user, item ) new_tags_str = ','.join( new_tags_list ) - trans.app.tag_handler.apply_item_tags( trans, user, item, new_tags_str.encode( 'utf-8' ) ) + trans.app.tag_handler.apply_item_tags( trans, user, item, unicode( new_tags_str.encode( 'utf-8' ), 'utf-8' ) ) trans.sa_session.flush() return item.tags Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: carlfeberhard: History & HDA API: fix unicode support in update
by commits-noreply＠bitbucket.org 10 Jan '14

10 Jan '14

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/b142bc80747e/ Changeset: b142bc80747e User: carlfeberhard Date: 2014-01-10 18:55:43 Summary: History & HDA API: fix unicode support in update Affected #: 5 files diff -r e41c77b8ba69351cb243fa4ba651e9b270b4b91c -r b142bc80747e5079b7f406259dc346cdba86a044 lib/galaxy/web/base/controller.py --- a/lib/galaxy/web/base/controller.py +++ b/lib/galaxy/web/base/controller.py @@ -179,6 +179,29 @@ rval[k] = self.encode_all_ids(trans, v, recursive) return rval + # incoming param validation + # should probably be in sep. serializer class/object _used_ by controller + def validate_and_sanitize_basestring( self, key, val ): + if not isinstance( val, basestring ): + raise ValueError( '%s must be a string or unicode: %s' %( key, str( type( val ) ) ) ) + return unicode( sanitize_html( val, 'utf-8', 'text/html' ), 'utf-8' ) + + def validate_and_sanitize_basestring_list( self, key, val ): + if not isinstance( val, list ): + raise ValueError( '%s must be a list: %s' %( key, str( type( val ) ) ) ) + return [ unicode( sanitize_html( t, 'utf-8', 'text/html' ), 'utf-8' ) for t in val ] + + def validate_boolean( self, key, val ): + if not isinstance( val, bool ): + raise ValueError( '%s must be a boolean: %s' %( key, str( type( val ) ) ) ) + return val + + #TODO: + #def validate_integer( self, key, val, min, max ): + #def validate_float( self, key, val, min, max ): + #def validate_number( self, key, val, min, max ): + #def validate_genome_build( self, key, val ): + Root = BaseController diff -r e41c77b8ba69351cb243fa4ba651e9b270b4b91c -r b142bc80747e5079b7f406259dc346cdba86a044 lib/galaxy/webapps/galaxy/api/histories.py --- a/lib/galaxy/webapps/galaxy/api/histories.py +++ b/lib/galaxy/webapps/galaxy/api/histories.py @@ -342,34 +342,14 @@ 'id', 'model_class', 'nice_size', 'contents_url', 'purged', 'tags', 'state', 'state_details', 'state_ids' ) - validated_payload = {} for key, val in payload.items(): - # TODO: lots of boilerplate here, but overhead on abstraction is equally onerous - if key == 'name': - if not ( isinstance( val, str ) or isinstance( val, unicode ) ): - raise ValueError( 'name must be a string or unicode: %s' %( str( type( val ) ) ) ) - validated_payload[ 'name' ] = sanitize_html( val, 'utf-8' ) - #TODO:?? if sanitized != val: log.warn( 'script kiddie' ) - elif key == 'deleted': - if not isinstance( val, bool ): - raise ValueError( 'deleted must be a boolean: %s' %( str( type( val ) ) ) ) - validated_payload[ 'deleted' ] = val - elif key == 'published': - if not isinstance( val, bool ): - raise ValueError( 'published must be a boolean: %s' %( str( type( val ) ) ) ) - validated_payload[ 'published' ] = val - elif key == 'genome_build' and val is not None: - if not ( isinstance( val, str ) or isinstance( val, unicode ) ): - raise ValueError( 'genome_build must be a string: %s' %( str( type( val ) ) ) ) - validated_payload[ 'genome_build' ] = sanitize_html( val, 'utf-8' ) - elif key == 'annotation': - if not ( isinstance( val, str ) or isinstance( val, unicode ) ): - raise ValueError( 'annotation must be a string or unicode: %s' %( str( type( val ) ) ) ) - validated_payload[ 'annotation' ] = sanitize_html( val, 'utf-8' ) + if key in ( 'name', 'genome_build', 'annotation' ): + validated_payload[ key ] = self.validate_and_sanitize_basestring( key, val ) + if key in ( 'deleted', 'published' ): + validated_payload[ key ] = self.validate_boolean( key, val ) elif key == 'tags': - if isinstance( val, list ): - validated_payload[ 'tags' ] = [ sanitize_html( t, 'utf-8' ) for t in val ] + validated_payload[ key ] = self.validate_and_sanitize_basestring_list( key, val ) elif key not in valid_but_uneditable_keys: pass #log.warn( 'unknown key: %s', str( key ) ) diff -r e41c77b8ba69351cb243fa4ba651e9b270b4b91c -r b142bc80747e5079b7f406259dc346cdba86a044 lib/galaxy/webapps/galaxy/api/history_contents.py --- a/lib/galaxy/webapps/galaxy/api/history_contents.py +++ b/lib/galaxy/webapps/galaxy/api/history_contents.py @@ -385,38 +385,17 @@ ) validated_payload = {} for key, val in payload.items(): - # TODO: lots of boilerplate here, but overhead on abstraction is equally onerous - # typecheck, parse, remap key - if key == 'name': - if not ( isinstance( val, str ) or isinstance( val, unicode ) ): - raise ValueError( 'name must be a string or unicode: %s' %( str( type( val ) ) ) ) - validated_payload[ 'name' ] = util.sanitize_html.sanitize_html( val, 'utf-8' ) - #TODO:?? if sanitized != val: log.warn( 'script kiddie' ) - elif key == 'deleted': - if not isinstance( val, bool ): - raise ValueError( 'deleted must be a boolean: %s' %( str( type( val ) ) ) ) - validated_payload[ 'deleted' ] = val - elif key == 'visible': - if not isinstance( val, bool ): - raise ValueError( 'visible must be a boolean: %s' %( str( type( val ) ) ) ) - validated_payload[ 'visible' ] = val - elif key == 'genome_build': - if not ( isinstance( val, str ) or isinstance( val, unicode ) ): - raise ValueError( 'genome_build must be a string: %s' %( str( type( val ) ) ) ) - validated_payload[ 'dbkey' ] = util.sanitize_html.sanitize_html( val, 'utf-8' ) - elif key == 'annotation' and val is not None: - if not ( isinstance( val, str ) or isinstance( val, unicode ) ): - raise ValueError( 'annotation must be a string or unicode: %s' %( str( type( val ) ) ) ) - validated_payload[ 'annotation' ] = util.sanitize_html.sanitize_html( val, 'utf-8' ) - elif key == 'misc_info': - if not ( isinstance( val, str ) or isinstance( val, unicode ) ): - raise ValueError( 'misc_info must be a string or unicode: %s' %( str( type( val ) ) ) ) - validated_payload[ 'info' ] = util.sanitize_html.sanitize_html( val, 'utf-8' ) + if key in ( 'name', 'genome_build', 'misc_info', 'annotation' ): + val = self.validate_and_sanitize_basestring( key, val ) + #TODO: need better remap system or eliminate the need altogether + key = 'dbkey' if key == 'genome_build' else key + key = 'info' if key == 'misc_info' else key + validated_payload[ key ] = val + if key in ( 'deleted', 'visible' ): + validated_payload[ key ] = self.validate_boolean( key, val ) elif key == 'tags': - if isinstance( val, list ): - validated_payload[ 'tags' ] = [ sanitize_html( t, 'utf-8' ) for t in val ] + validated_payload[ key ] = self.validate_and_sanitize_basestring_list( key, val ) elif key not in valid_but_uneditable_keys: pass #log.warn( 'unknown key: %s', str( key ) ) return validated_payload - diff -r e41c77b8ba69351cb243fa4ba651e9b270b4b91c -r b142bc80747e5079b7f406259dc346cdba86a044 test/casperjs/api-hda-tests.js --- a/test/casperjs/api-hda-tests.js +++ b/test/casperjs/api-hda-tests.js @@ -174,24 +174,14 @@ hdaShow = this.api.hdas.show( lastHistory.id, firstHda.id ); this.test.assert( hdaShow.name === 'New name', "Update sanitized name: " + hdaShow.name ); - //NOTE!: this fails on sqlite3 (with default setup) - try { - this.test.comment( 'update should allow unicode in names' ); - var unicodeName = 'Ржевский сапоги'; - returned = this.api.hdas.update( lastHistory.id, firstHda.id, { - name : unicodeName - }); - //this.debug( 'returned:\n' + this.jsonStr( returned ) ); - hdaShow = this.api.hdas.show( lastHistory.id, firstHda.id ); - this.test.assert( hdaShow.name === unicodeName, "Update accepted unicode name: " + hdaShow.name ); - } catch( err ){ - //this.debug( this.jsonStr( err ) ); - if( ( err instanceof this.api.APIError ) - && ( err.status === 500 ) - && ( err.message.indexOf( '(ProgrammingError) You must not use 8-bit bytestrings' ) !== -1 ) ){ - this.skipTest( 'Unicode update failed. Are you using sqlite3 as the db?' ); - } - } + this.test.comment( 'update should allow unicode in names' ); + var unicodeName = 'Ржевский сапоги'; + returned = this.api.hdas.update( lastHistory.id, firstHda.id, { + name : unicodeName + }); + //this.debug( 'returned:\n' + this.jsonStr( returned ) ); + hdaShow = this.api.hdas.show( lastHistory.id, firstHda.id ); + this.test.assert( hdaShow.name === unicodeName, "Update accepted unicode name: " + hdaShow.name ); this.test.comment( 'update should allow escaped quotations in names' ); var quotedName = '"Bler"'; @@ -239,11 +229,11 @@ }); //this.debug( 'returned:\n' + this.jsonStr( returned ) ); hdaShow = this.api.hdas.show( lastHistory.id, firstHda.id ); + //this.debug( 'hdaShow:\n' + this.jsonStr( hdaShow ) ); this.test.assert( hdaShow.genome_build === 'hg18', "genome_build successfully set via update: " + hdaShow.genome_build ); this.test.assert( hdaShow.metadata_dbkey === 'hg18', "metadata_dbkey successfully set via the same update: " + hdaShow.metadata_dbkey ); - this.test.comment( 'update should sanitize any genome_build' ); returned = this.api.hdas.update( lastHistory.id, firstHda.id, { genome_build : 'hg18<script type="text/javascript" src="bler">alert("blah");</script>' @@ -257,23 +247,14 @@ this.test.comment( 'update should allow unicode in genome builds' ); var unicodeBuild = 'Ржевский18'; - //NOTE!: this fails on sqlite3 (with default setup) - try { - returned = this.api.hdas.update( lastHistory.id, firstHda.id, { - name : unicodeBuild - }); - //this.debug( 'returned:\n' + this.jsonStr( returned ) ); - hdaShow = this.api.hdas.show( lastHistory.id, firstHda.id ); - this.test.assert( hdaShow.genome_build === unicodeBuild, - "Update accepted unicode genome_build: " + hdaShow.name ); - } catch( err ){ - //this.debug( this.jsonStr( err ) ); - if( ( err instanceof this.api.APIError ) - && ( err.status === 500 ) - && ( err.message.indexOf( '(ProgrammingError) You must not use 8-bit bytestrings' ) !== -1 ) ){ - this.skipTest( 'Unicode update failed. Are you using sqlite3 as the db?' ); - } - } + returned = this.api.hdas.update( lastHistory.id, firstHda.id, { + genome_build : unicodeBuild + }); + this.debug( 'returned:\n' + this.jsonStr( returned ) ); + hdaShow = this.api.hdas.show( lastHistory.id, firstHda.id ); + this.debug( 'hdaShow:\n' + this.jsonStr( hdaShow ) ); + this.test.assert( hdaShow.genome_build === unicodeBuild, + "Update accepted unicode genome_build: " + hdaShow.genome_build ); // ........................................................................................... misc_info/info this.test.comment( 'update should allow changing the misc_info' ); @@ -298,25 +279,14 @@ this.test.comment( 'update should allow unicode in misc_info' ); var unicodeInfo = '여보!'; - //NOTE!: this fails on sqlite3 (with default setup) - try { - returned = this.api.hdas.update( lastHistory.id, firstHda.id, { - misc_info : unicodeInfo - }); - //this.debug( 'returned:\n' + this.jsonStr( returned ) ); - hdaShow = this.api.hdas.show( lastHistory.id, firstHda.id ); - this.test.assert( hdaShow.misc_info === unicodeInfo, - "Update accepted unicode misc_info: " + hdaShow.misc_info ); - } catch( err ){ - //this.debug( this.jsonStr( err ) ); - if( ( err instanceof this.api.APIError ) - && ( err.status === 500 ) - && ( err.message.indexOf( '(ProgrammingError) You must not use 8-bit bytestrings' ) !== -1 ) ){ - this.skipTest( 'Unicode update failed. Are you using sqlite3 as the db?' ); - } - } + returned = this.api.hdas.update( lastHistory.id, firstHda.id, { + misc_info : unicodeInfo + }); + //this.debug( 'returned:\n' + this.jsonStr( returned ) ); + hdaShow = this.api.hdas.show( lastHistory.id, firstHda.id ); + this.test.assert( hdaShow.misc_info === unicodeInfo, + "Update accepted unicode misc_info: " + hdaShow.misc_info ); -/* // ........................................................................................... annotation // currently fails because no annotation is returned in details this.test.comment( 'update should allow changing the annotation' ); @@ -338,25 +308,15 @@ this.test.assert( hdaShow.annotation === 'New annotation', "Update sanitized annotation: " + hdaShow.annotation ); - //NOTE!: this fails on sqlite3 (with default setup) - try { - this.test.comment( 'update should allow unicode in annotations' ); - var unicodeAnnotation = 'お願いは、それが落下させない'; - returned = this.api.hdas.update( lastHistory.id, firstHda.id, { - annotation : unicodeAnnotation - }); - //this.debug( 'returned:\n' + this.jsonStr( returned ) ); - hdaShow = this.api.hdas.show( lastHistory.id, firstHda.id ); - this.test.assert( hdaShow.annotation === unicodeAnnotation, - "Update accepted unicode annotation: " + hdaShow.annotation ); - } catch( err ){ - //this.debug( this.jsonStr( err ) ); - if( ( err instanceof this.api.APIError ) - && ( err.status === 500 ) - && ( err.message.indexOf( '(ProgrammingError) You must not use 8-bit bytestrings' ) !== -1 ) ){ - this.skipTest( 'Unicode update failed. Are you using sqlite3 as the db?' ); - } - } + this.test.comment( 'update should allow unicode in annotations' ); + var unicodeAnnotation = 'お願いは、それが落下させない'; + returned = this.api.hdas.update( lastHistory.id, firstHda.id, { + annotation : unicodeAnnotation + }); + //this.debug( 'returned:\n' + this.jsonStr( returned ) ); + hdaShow = this.api.hdas.show( lastHistory.id, firstHda.id ); + this.test.assert( hdaShow.annotation === unicodeAnnotation, + "Update accepted unicode annotation: " + hdaShow.annotation ); this.test.comment( 'update should allow escaped quotations in annotations' ); var quotedAnnotation = '"Bler"'; @@ -367,8 +327,6 @@ hdaShow = this.api.hdas.show( lastHistory.id, firstHda.id ); this.test.assert( hdaShow.annotation === quotedAnnotation, "Update accepted escaped quotations in annotation: " + hdaShow.annotation ); -*/ - // ------------------------------------------------------------------------------------------- ERRORS this.test.comment( 'create should error with "Please define the source" when the param "from_ld_id" is not used' ); @@ -391,6 +349,8 @@ //var returned = this.api.hdas.update( lastHistory.id, { deleted: true, blerp: 'blerp' }); //this.debug( 'returned:' + this.jsonStr( returned ) ); //this.debug( 'page:' + this.jsonStr( this.page ) ); +/* +*/ }); // =================================================================== diff -r e41c77b8ba69351cb243fa4ba651e9b270b4b91c -r b142bc80747e5079b7f406259dc346cdba86a044 test/casperjs/api-history-tests.js --- a/test/casperjs/api-history-tests.js +++ b/test/casperjs/api-history-tests.js @@ -218,23 +218,14 @@ this.test.assert( historyShow.name === 'New name', "Update sanitized name: " + historyShow.name ); //NOTE!: this fails on sqlite3 (with default setup) - try { - this.test.comment( 'update should allow unicode in names' ); - var unicodeName = '桜ゲノム'; - returned = this.api.histories.update( newFirstHistory.id, { - name : unicodeName - }); - //this.debug( 'returned:\n' + this.jsonStr( returned ) ); - historyShow = this.api.histories.show( newFirstHistory.id ); - this.test.assert( historyShow.name === unicodeName, "Update accepted unicode name: " + historyShow.name ); - } catch( err ){ - //this.debug( this.jsonStr( err ) ); - if( ( err instanceof this.api.APIError ) - && ( err.status === 500 ) - && ( err.message.indexOf( '(ProgrammingError) You must not use 8-bit bytestrings' ) !== -1 ) ){ - this.skipTest( 'Unicode update failed. Are you using sqlite3 as the db?' ); - } - } + this.test.comment( 'update should allow unicode in names' ); + var unicodeName = '桜ゲノム'; + returned = this.api.histories.update( newFirstHistory.id, { + name : unicodeName + }); + //this.debug( 'returned:\n' + this.jsonStr( returned ) ); + historyShow = this.api.histories.show( newFirstHistory.id ); + this.test.assert( historyShow.name === unicodeName, "Update accepted unicode name: " + historyShow.name ); this.test.comment( 'update should allow escaped quotations in names' ); var quotedName = '"Bler"'; @@ -294,27 +285,16 @@ this.test.assert( historyShow.genome_build === 'hg18', "Update sanitized genome_build: " + historyShow.genome_build ); + // removing for now until I can determine the relationship between unicode and genome_builds this.test.comment( 'update should allow unicode in genome builds' ); var unicodeBuild = '桜12'; - //NOTE!: this fails on sqlite3 (with default setup) - try { - returned = this.api.histories.update( newFirstHistory.id, { - name : unicodeBuild - }); - //this.debug( 'returned:\n' + this.jsonStr( returned ) ); - historyShow = this.api.histories.show( newFirstHistory.id ); - this.test.assert( historyShow.genome_build === unicodeBuild, - "Update accepted unicode genome_build: " + historyShow.genome_build ); - } catch( err ){ - //this.debug( this.jsonStr( err ) ); - if( ( err instanceof this.api.APIError ) - && ( err.status === 500 ) - && ( err.message.indexOf( '(ProgrammingError) You must not use 8-bit bytestrings' ) !== -1 ) ){ - this.skipTest( 'Unicode update failed. Are you using sqlite3 as the db?' ); - } else { - throw err; - } - } + returned = this.api.histories.update( newFirstHistory.id, { + genome_build : unicodeBuild + }); + //this.debug( 'returned:\n' + this.jsonStr( returned ) ); + historyShow = this.api.histories.show( newFirstHistory.id ); + this.test.assert( historyShow.genome_build === unicodeBuild, + "Update accepted unicode genome_build: " + historyShow.genome_build ); // ........................................................................................... annotation @@ -337,25 +317,15 @@ this.test.assert( historyShow.annotation === 'New annotation', "Update sanitized annotation: " + historyShow.annotation ); - //NOTE!: this fails on sqlite3 (with default setup) - try { - this.test.comment( 'update should allow unicode in annotations' ); - var unicodeAnnotation = 'お願いは、それが落下させない'; - returned = this.api.histories.update( newFirstHistory.id, { - annotation : unicodeAnnotation - }); - //this.debug( 'returned:\n' + this.jsonStr( returned ) ); - historyShow = this.api.histories.show( newFirstHistory.id ); - this.test.assert( historyShow.annotation === unicodeAnnotation, - "Update accepted unicode annotation: " + historyShow.annotation ); - } catch( err ){ - //this.debug( this.jsonStr( err ) ); - if( ( err instanceof this.api.APIError ) - && ( err.status === 500 ) - && ( err.message.indexOf( '(ProgrammingError) You must not use 8-bit bytestrings' ) !== -1 ) ){ - this.skipTest( 'Unicode update failed. Are you using sqlite3 as the db?' ); - } - } + this.test.comment( 'update should allow unicode in annotations' ); + var unicodeAnnotation = 'お願いは、それが落下させない'; + returned = this.api.histories.update( newFirstHistory.id, { + annotation : unicodeAnnotation + }); + //this.debug( 'returned:\n' + this.jsonStr( returned ) ); + historyShow = this.api.histories.show( newFirstHistory.id ); + this.test.assert( historyShow.annotation === unicodeAnnotation, + "Update accepted unicode annotation: " + historyShow.annotation ); this.test.comment( 'update should allow escaped quotations in annotations' ); var quotedAnnotation = '"Bler"'; @@ -371,7 +341,6 @@ // ------------------------------------------------------------------------------------------- ERRORS //TODO: make sure expected errors are being passed back (but no permissions checks here - different suite) // bad ids: index, show, update, delete, undelete - /* */ //this.debug( this.jsonStr( historyShow ) ); Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: 2 new changesets
by commits-noreply＠bitbucket.org 10 Jan '14

10 Jan '14

2 new commits in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/d96d5309b5c5/ Changeset: d96d5309b5c5 Branch: stable User: inithello Date: 2014-01-08 18:27:14 Summary: When a push is received in the hg controller, update the repository on disk after the push is complete. Affected #: 1 file diff -r 3e76aceca9d0ab563e285d630dd7575140236b0d -r d96d5309b5c52bb43c410c7a943808294bee12c0 lib/galaxy/webapps/tool_shed/controllers/hg.py --- a/lib/galaxy/webapps/tool_shed/controllers/hg.py +++ b/lib/galaxy/webapps/tool_shed/controllers/hg.py @@ -2,6 +2,7 @@ from galaxy import web from galaxy.web.base.controller import BaseUIController from tool_shed.util.shed_util_common import get_repository_by_name_and_owner +from tool_shed.util.shed_util_common import update_repository from tool_shed.util.metadata_util import set_repository_metadata from galaxy import eggs @@ -9,6 +10,8 @@ import mercurial.__version__ from mercurial.hgweb.hgwebdir_mod import hgwebdir from mercurial.hgweb.request import wsgiapplication +from mercurial import hg +from mercurial import ui log = logging.getLogger(__name__) @@ -36,6 +39,11 @@ repository = get_repository_by_name_and_owner( trans.app, name, owner ) if repository: if hg_version >= '2.2.3': + # Update the repository on disk to the tip revision, because the web upload form uses the on-disk working + # directory. If the repository is not updated on disk, pushing from the command line and then uploading + # via the web interface will result in a new head being created. + repo = hg.repository( ui.ui(), repository.repo_path( trans.app ) ) + update_repository( repo, ctx_rev=None ) # Set metadata using the repository files on disk. error_message, status = set_repository_metadata( trans, repository ) if status == 'ok' and error_message: https://bitbucket.org/galaxy/galaxy-central/commits/e41c77b8ba69/ Changeset: e41c77b8ba69 User: inithello Date: 2014-01-10 15:52:10 Summary: Merge stable. Affected #: 0 files Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: 4 new changesets
by commits-noreply＠bitbucket.org 10 Jan '14

10 Jan '14

4 new commits in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/f238040cc9a2/ Changeset: f238040cc9a2 User: jmchilton Date: 2014-01-10 15:45:36 Summary: Rework key injection logic in test framework. Pull logic for how master and user API keys are determinined for testing out of functional_tests.py for reuse elsewhere. This refactoring will help the creation of an API test framework. Affected #: 2 files diff -r 66df8b51245cd141fb7180e401e725ef3b1a0e37 -r f238040cc9a261db98edfa316384b33851f22486 scripts/functional_tests.py --- a/scripts/functional_tests.py +++ b/scripts/functional_tests.py @@ -51,6 +51,8 @@ from galaxy.util.json import to_json_string from functional import database_contexts +from base.api_util import get_master_api_key +from base.api_util import get_user_api_key import nose.core import nose.config @@ -64,8 +66,6 @@ default_galaxy_test_port_max = 9999 default_galaxy_locales = 'en' default_galaxy_test_file_dir = "test-data" -default_galaxy_master_key = "TEST123" -default_galaxy_user_key = None migrated_tool_panel_config = 'migrated_tools_conf.xml' installed_tool_panel_configs = [ 'shed_tool_conf.xml' ] @@ -342,7 +342,7 @@ galaxy_data_manager_data_path = tempfile.mkdtemp( prefix='data_manager_tool-data', dir=data_manager_test_tmp_path ) # ---- Build Application -------------------------------------------------- - master_api_key = os.environ.get( "GALAXY_TEST_MASTER_API_KEY", default_galaxy_master_key ) + master_api_key = get_master_api_key() app = None if start_server: kwargs = dict( admin_users='test(a)bx.psu.edu', @@ -477,7 +477,7 @@ import functional.test_workflow functional.test_workflow.WorkflowTestCase.workflow_test_file = workflow_test functional.test_workflow.WorkflowTestCase.master_api_key = master_api_key - functional.test_workflow.WorkflowTestCase.user_api_key = os.environ.get( "GALAXY_TEST_USER_API_KEY", default_galaxy_user_key ) + functional.test_workflow.WorkflowTestCase.user_api_key = get_user_api_key() data_manager_test = __check_arg( '-data_managers', param=False ) if data_manager_test: import functional.test_data_managers @@ -486,7 +486,7 @@ tmp_dir=data_manager_test_tmp_path, testing_shed_tools=testing_shed_tools, master_api_key=master_api_key, - user_api_key=os.environ.get( "GALAXY_TEST_USER_API_KEY", default_galaxy_user_key ), + user_api_key=get_user_api_key(), ) else: # We must make sure that functional.test_toolbox is always imported after @@ -499,7 +499,7 @@ functional.test_toolbox.build_tests( testing_shed_tools=testing_shed_tools, master_api_key=master_api_key, - user_api_key=os.environ.get( "GALAXY_TEST_USER_API_KEY", default_galaxy_user_key ), + user_api_key=get_user_api_key(), ) test_config = nose.config.Config( env=os.environ, ignoreFiles=ignore_files, plugins=nose.plugins.manager.DefaultPluginManager() ) test_config.configure( sys.argv ) diff -r 66df8b51245cd141fb7180e401e725ef3b1a0e37 -r f238040cc9a261db98edfa316384b33851f22486 test/base/api_util.py --- /dev/null +++ b/test/base/api_util.py @@ -0,0 +1,20 @@ +import os + +DEFAULT_GALAXY_MASTER_API_KEY = "TEST123" +DEFAULT_GALAXY_USER_API_KEY = None + + +def get_master_api_key(): + """ Test master API key to use for functional test. This key should be + configured as a master API key and should be able to create additional + users and keys. + """ + return os.environ.get( "GALAXY_TEST_MASTER_API_KEY", DEFAULT_GALAXY_MASTER_API_KEY ) + + +def get_user_api_key(): + """ Test user API key to use for functional tests. If set, this should drive + API based testing - if not set master API key should be used to create a new + user and API key for tests. + """ + return os.environ.get( "GALAXY_TEST_USER_API_KEY", DEFAULT_GALAXY_USER_API_KEY ) https://bitbucket.org/galaxy/galaxy-central/commits/efa607692e17/ Changeset: efa607692e17 User: jmchilton Date: 2014-01-10 15:45:36 Summary: Flush out API interactor... ... this will enable functional testing of API. Affected #: 1 file diff -r f238040cc9a261db98edfa316384b33851f22486 -r efa607692e171a5aa820d0a4b5ac9c8471431886 test/base/interactor.py --- a/test/base/interactor.py +++ b/test/base/interactor.py @@ -31,10 +31,11 @@ class GalaxyInteractorApi( object ): - def __init__( self, twill_test_case ): + def __init__( self, twill_test_case, test_user=None ): self.twill_test_case = twill_test_case self.api_url = "%s/api" % twill_test_case.url.rstrip("/") - self.api_key = self.__get_user_key( twill_test_case.user_api_key, twill_test_case.master_api_key ) + self.master_api_key = twill_test_case.master_api_key + self.api_key = self.__get_user_key( twill_test_case.user_api_key, twill_test_case.master_api_key, test_user=test_user ) self.uploads = {} def verify_output( self, history_id, output_data, outfile, attributes, shed_tool_id, maxseconds ): @@ -251,19 +252,26 @@ ) return self._post( "tools", files=files, data=data ) - def __get_user_key( self, user_key, admin_key ): - if user_key: - return user_key + def ensure_user_with_email( self, email ): + admin_key = self.master_api_key all_users = self._get( 'users', key=admin_key ).json() try: - test_user = [ user for user in all_users if user["email"] == 'test(a)bx.psu.edu' ][0] + test_user = [ user for user in all_users if user["email"] == email ][0] except IndexError: data = dict( - email='test(a)bx.psu.edu', + email=email, password='testuser', username='admin-user', ) test_user = self._post( 'users', data, key=admin_key ).json() + return test_user + + def __get_user_key( self, user_key, admin_key, test_user=None ): + if not test_user: + test_user = "test(a)bx.psu.edu" + if user_key: + return user_key + test_user = self.ensure_user_with_email(test_user) return self._post( "users/%s/api_key" % test_user['id'], key=admin_key ).json() def __dataset_fetcher( self, history_id ): @@ -275,16 +283,16 @@ return fetcher - def _post( self, path, data={}, files=None, key=None): + def _post( self, path, data={}, files=None, key=None, admin=False): if not key: - key = self.api_key + key = self.api_key if not admin else self.master_api_key data = data.copy() data['key'] = key return post_request( "%s/%s" % (self.api_url, path), data=data, files=files ) - def _get( self, path, data={}, key=None ): + def _get( self, path, data={}, key=None, admin=False ): if not key: - key = self.api_key + key = self.api_key if not admin else self.master_api_key data = data.copy() data['key'] = key if path.startswith("/api"): @@ -413,14 +421,17 @@ try: from requests import get as get_request from requests import post as post_request + from requests import put as put_request + from requests import delete as delete_request except ImportError: import urllib2 import httplib class RequestsLikeResponse( object ): - def __init__( self, content ): + def __init__( self, content, status_code ): self.content = content + self.status_code = status_code def json( self ): return loads( self.content ) @@ -431,23 +442,44 @@ argsep = '?' url = url + argsep + '&'.join( [ '%s=%s' % (k, v) for k, v in params.iteritems() ] ) #req = urllib2.Request( url, headers = { 'Content-Type': 'application/json' } ) - return RequestsLikeResponse(urllib2.urlopen( url ).read() ) + try: + response = urllib2.urlopen( url ) + return RequestsLikeResponse( response.read(), status_code=response.getcode() ) + except urllib2.HTTPError as e: + return RequestsLikeResponse( e.read(), status_code=e.code ) - def post_request( url, data, files ): + def post_request( url, data, files={} ): + return __multipart_request( url, data, files, verb="POST" ) + + def put_request( url, data, files={} ): + return __multipart_request( url, data, files, verb="PUT" ) + + def delete_request( url ): + opener = urllib2.build_opener(urllib2.HTTPHandler) + request = urllib2.Request(url) + request.get_method = lambda: 'DELETE' + try: + response = opener.open(request) + return RequestsLikeResponse( response.read(), status_code=response.getcode() ) + except urllib2.HTTPError as e: + return RequestsLikeResponse( e.read(), status_code=e.code ) + + + def __multipart_request( url, data, files={}, verb="POST" ): parsed_url = urllib2.urlparse.urlparse( url ) - return __post_multipart( host=parsed_url.netloc, selector=parsed_url.path, fields=data.iteritems(), files=(files or {}).iteritems() ) + return __multipart( host=parsed_url.netloc, selector=parsed_url.path, fields=data.iteritems(), files=(files or {}).iteritems(), verb=verb ) # http://stackoverflow.com/a/681182 - def __post_multipart(host, selector, fields, files): + def __multipart(host, selector, fields, files, verb="POST"): + h = httplib.HTTP(host) + h.putrequest(verb, selector) content_type, body = __encode_multipart_formdata(fields, files) - h = httplib.HTTP(host) - h.putrequest('POST', selector) h.putheader('content-type', content_type) h.putheader('content-length', str(len(body))) h.endheaders() h.send(body) errcode, errmsg, headers = h.getreply() - return RequestsLikeResponse(h.file.read()) + return RequestsLikeResponse(h.file.read(), status_code=errcode) def __encode_multipart_formdata(fields, files): LIMIT = '----------lImIt_of_THE_fIle_eW_$' https://bitbucket.org/galaxy/galaxy-central/commits/1cd7fd029a01/ Changeset: 1cd7fd029a01 User: jmchilton Date: 2014-01-10 15:45:36 Summary: Eliminate interactor dependency on deprecated global variable. Lessens the need to import things in a specific order - as long as database context is run before interactor methods everything should be fine now. TODO: Eliminate rest of dependencies on this variable. Affected #: 1 file diff -r efa607692e171a5aa820d0a4b5ac9c8471431886 -r 1cd7fd029a0195fe8ee4c3e5d423591fb4403a8f test/base/interactor.py --- a/test/base/interactor.py +++ b/test/base/interactor.py @@ -4,7 +4,7 @@ from galaxy.util.odict import odict import galaxy.model from galaxy.model.orm import and_, desc -from base.test_db_util import sa_session +from functional import database_contexts from json import dumps, loads from logging import getLogger @@ -391,9 +391,9 @@ # Start with a new history self.twill_test_case.logout() self.twill_test_case.login( email='test(a)bx.psu.edu' ) - admin_user = sa_session.query( galaxy.model.User ).filter( galaxy.model.User.table.c.email == 'test(a)bx.psu.edu' ).one() + admin_user = database_contexts.galaxy_context.query( galaxy.model.User ).filter( galaxy.model.User.table.c.email == 'test(a)bx.psu.edu' ).one() self.twill_test_case.new_history() - latest_history = sa_session.query( galaxy.model.History ) \ + latest_history = database_contexts.galaxy_context.query( galaxy.model.History ) \ .filter( and_( galaxy.model.History.table.c.deleted == False, galaxy.model.History.table.c.user_id == admin_user.id ) ) \ .order_by( desc( galaxy.model.History.table.c.create_time ) ) \ @@ -417,7 +417,7 @@ # Lets just try to use requests if it is available, but if not provide fallback -# on custom implementations of limited requests get/post functionality. +# on custom implementations of limited requests get, put, etc... functionality. try: from requests import get as get_request from requests import post as post_request @@ -464,7 +464,6 @@ except urllib2.HTTPError as e: return RequestsLikeResponse( e.read(), status_code=e.code ) - def __multipart_request( url, data, files={}, verb="POST" ): parsed_url = urllib2.urlparse.urlparse( url ) return __multipart( host=parsed_url.netloc, selector=parsed_url.path, fields=data.iteritems(), files=(files or {}).iteritems(), verb=verb ) https://bitbucket.org/galaxy/galaxy-central/commits/95314eb284fa/ Changeset: 95314eb284fa User: jmchilton Date: 2014-01-10 15:45:36 Summary: Implement framework for testing API... ... if you can call 1 new class a framework. Includes a few test cases to exercise/drive it. These examples include a histories API test (a typical API test) and a general test of the API framework itself (mostly just the run_as functionality). This includes changes to interactor.py to make it more useful outside the context of tool/workflow testing as well as a tweak to test Galaxy that gets started to allow testing of the run_as feature. Affected #: 5 files diff -r 1cd7fd029a0195fe8ee4c3e5d423591fb4403a8f -r 95314eb284faf3024207f70ef0dafe63c285e379 scripts/functional_tests.py --- a/scripts/functional_tests.py +++ b/scripts/functional_tests.py @@ -346,6 +346,7 @@ app = None if start_server: kwargs = dict( admin_users='test(a)bx.psu.edu', + api_allow_run_as='test(a)bx.psu.edu', allow_library_path_paste=True, allow_user_creation=True, allow_user_deletion=True, diff -r 1cd7fd029a0195fe8ee4c3e5d423591fb4403a8f -r 95314eb284faf3024207f70ef0dafe63c285e379 test/base/api.py --- /dev/null +++ b/test/base/api.py @@ -0,0 +1,85 @@ +# TODO: We don't need all of TwillTestCase, strip down to a common super class +# shared by API and Twill test cases. +from .twilltestcase import TwillTestCase + +from base.interactor import GalaxyInteractorApi as BaseInteractor + +from .api_util import get_master_api_key +from .api_util import get_user_api_key + +from urllib import urlencode + + +TEST_USER = "user(a)bx.psu.edu" + + +# TODO: Allow these to point at existing Galaxy instances. +class ApiTestCase( TwillTestCase ): + + def setUp( self ): + super( ApiTestCase, self ).setUp( ) + self.user_api_key = get_user_api_key() + self.master_api_key = get_master_api_key() + self.galaxy_interactor = ApiTestInteractor( self ) + + def _api_url( self, path, params=None, use_key=None ): + if not params: + params = {} + url = "%s/api/%s" % ( self.url, path ) + if use_key: + params[ "key" ] = self.galaxy_interactor.api_key + query = urlencode( params ) + if query: + url = "%s?%s" % ( url, query ) + return url + + def _setup_user( self, email ): + self.galaxy_interactor.ensure_user_with_email(email) + users = self._get( "users", admin=True ).json() + user = [ user for user in users if user["email"] == email ][0] + return user + + def _get( self, *args, **kwds ): + return self.galaxy_interactor.get( *args, **kwds ) + + def _post( self, *args, **kwds ): + return self.galaxy_interactor.post( *args, **kwds ) + + def _assert_status_code_is( self, response, expected_status_code ): + response_status_code = response.status_code + if expected_status_code != response_status_code: + try: + body = response.json() + except Exception: + body = "INVALID JSON RESPONSE" + assertion_message_template = "Request status code (%d) was not expected value %d. Body was %s" + assertion_message = assertion_message_template % ( response_status_code, expected_status_code, body ) + raise AssertionError( assertion_message ) + + def _assert_has_keys( self, response, *keys ): + for key in keys: + assert key in response, "Response [%s] does not contain key [%s]" % ( response, key ) + + def _random_key( self ): # Used for invalid request testing... + return "1234567890123456" + + _assert_has_key = _assert_has_keys + + +class ApiTestInteractor( BaseInteractor ): + """ Specialized variant of the API interactor (originally developed for + tool functional tests) for testing the API generally. + """ + + def __init__( self, test_case ): + super( ApiTestInteractor, self ).__init__( test_case, test_user=TEST_USER ) + + # This variant the lower level get and post methods are meant to be used + # directly to test API - instead of relying on higher-level constructs for + # specific pieces of the API (the way it is done with the variant for tool) + # testing. + def get( self, *args, **kwds ): + return self._get( *args, **kwds ) + + def post( self, *args, **kwds ): + return self._post( *args, **kwds ) diff -r 1cd7fd029a0195fe8ee4c3e5d423591fb4403a8f -r 95314eb284faf3024207f70ef0dafe63c285e379 test/functional/api/test_framework.py --- /dev/null +++ b/test/functional/api/test_framework.py @@ -0,0 +1,27 @@ +# This file doesn't test any API in particular but is meant to functionally +# test the API framework itself. +from base import api + + +class ApiFrameworkTestCase( api.ApiTestCase ): + + # Next several tests test the API's run_as functionality. + def test_user_cannont_run_as( self ): + post_data = dict( name="TestHistory1", run_as="another_user" ) + # Normal user cannot run_as... + create_response = self._post( "histories", data=post_data ) + self._assert_status_code_is( create_response, 403 ) + + def test_run_as_invalid_user( self ): + post_data = dict( name="TestHistory1", run_as="another_user" ) + # admin user can run_as, but this user doesn't exist, expect 400. + create_response = self._post( "histories", data=post_data, admin=True ) + self._assert_status_code_is( create_response, 400 ) + + def test_run_as_valid_user( self ): + run_as_user = self._setup_user( "for_run_as(a)bx.psu.edu" ) + post_data = dict( name="TestHistory1", run_as=run_as_user[ "id" ] ) + # Use run_as with admin user and for another user just created, this + # should work. + create_response = self._post( "histories", data=post_data, admin=True ) + self._assert_status_code_is( create_response, 200 ) diff -r 1cd7fd029a0195fe8ee4c3e5d423591fb4403a8f -r 95314eb284faf3024207f70ef0dafe63c285e379 test/functional/api/test_histories.py --- /dev/null +++ b/test/functional/api/test_histories.py @@ -0,0 +1,26 @@ +from base import api +# requests.post or something like it if unavailable +from base.interactor import post_request + + +class HistoriesApiTestCase( api.ApiTestCase ): + + def test_create_history( self ): + # Create a history. + post_data = dict( name="TestHistory1" ) + create_response = self._post( "histories", data=post_data ).json() + self._assert_has_keys( create_response, "name", "id" ) + self.assertEquals( create_response[ "name" ], "TestHistory1" ) + created_id = create_response[ "id" ] + + # Make sure new history appears in index of user's histories. + index_response = self._get( "histories" ).json() + indexed_history = [ h for h in index_response if h[ "id" ] == created_id ][0] + self.assertEquals(indexed_history[ "name" ], "TestHistory1") + + def test_create_anonymous_fails( self ): + post_data = dict( name="CannotCreate" ) + # Using lower-level _api_url will cause key to not be injected. + histories_url = self._api_url( "histories" ) + create_response = post_request( url=histories_url, data=post_data ) + self._assert_status_code_is( create_response, 403 ) Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: carlfeberhard: Remove history panel prefs from session storage on logout; possible fix for duplicating tags_used behavior
by commits-noreply＠bitbucket.org 10 Jan '14

10 Jan '14

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/66df8b51245c/ Changeset: 66df8b51245c User: carlfeberhard Date: 2014-01-10 15:32:39 Summary: Remove history panel prefs from session storage on logout; possible fix for duplicating tags_used behavior Affected #: 4 files diff -r 3dde0f4765fbff33789badeaa17503349559ff72 -r 66df8b51245cd141fb7180e401e725ef3b1a0e37 static/scripts/mvc/tags.js --- a/static/scripts/mvc/tags.js +++ b/static/scripts/mvc/tags.js @@ -56,7 +56,7 @@ /** @returns {jQuery} the input for this view */ $input : function(){ - return this.$el.find( '.tags-input' ); + return this.$el.find( 'input.tags-input' ); }, /** @returns {String[]} all tags used by the current user */ @@ -73,7 +73,9 @@ view.model.save({ tags: event.val }, { silent: true }); // if it's new, add the tag to the users tags if( event.added ){ - view._addNewTagToTagsUsed( event.added.text ); + //??: solve weird behavior in FF on test.galaxyproject.org where + // event.added.text is string object: 'String{ 0="o", 1="n", 2="e" }' + view._addNewTagToTagsUsed( event.added.text + '' ); } }); }, diff -r 3dde0f4765fbff33789badeaa17503349559ff72 -r 66df8b51245cd141fb7180e401e725ef3b1a0e37 static/scripts/mvc/user/user-model.js --- a/static/scripts/mvc/user/user-model.js +++ b/static/scripts/mvc/user/user-model.js @@ -74,8 +74,12 @@ */ clearSessionStorage : function(){ for( var key in sessionStorage ){ - //TODO: currently only history - if( key.indexOf( 'HistoryView.' ) === 0 ){ + //TODO: store these under the user key so we don't have to do this + // currently only history + if( key.indexOf( 'history:' ) === 0 ){ + sessionStorage.removeItem( key ); + + } else if( key === 'history-panel' ){ sessionStorage.removeItem( key ); } } diff -r 3dde0f4765fbff33789badeaa17503349559ff72 -r 66df8b51245cd141fb7180e401e725ef3b1a0e37 static/scripts/packed/mvc/tags.js --- a/static/scripts/packed/mvc/tags.js +++ b/static/scripts/packed/mvc/tags.js @@ -1,1 +1,1 @@ -var TagsEditor=Backbone.View.extend(LoggableMixin).extend(HiddenUntilActivatedViewMixin).extend({tagName:"div",className:"tags-display",initialize:function(a){this.listenTo(this.model,"change:tags",function(){this.render()});this.hiddenUntilActivated(a.$activator,a)},render:function(){var a=this;this.$el.html(this._template());this.$input().select2({placeholder:"Add tags",width:"100%",tags:function(){return a._getTagsUsed()}});this._setUpBehaviors();return this},_template:function(){return['<label class="prompt">',_l("Tags"),"</label>",'<input class="tags-input" value="',this.tagsToCSV(),'" />'].join("")},tagsToCSV:function(){var a=this.model.get("tags");if(!_.isArray(a)||_.isEmpty(a)){return""}return a.sort().join(",")},$input:function(){return this.$el.find(".tags-input")},_getTagsUsed:function(){return Galaxy.currUser.get("tags_used")},_setUpBehaviors:function(){var a=this;this.$input().on("change",function(b){a.model.save({tags:b.val},{silent:true});if(b.added){a._addNewTagToTagsUsed(b.added.text)}})},_addNewTagToTagsUsed:function(a){var b=Galaxy.currUser.get("tags_used");if(!_.contains(b,a)){b.push(a);b.sort();Galaxy.currUser.set("tags_used",b)}},remove:function(){this.$input.off();this.stopListening(this.model);Backbone.View.prototype.remove.call(this)},toString:function(){return["TagsEditor(",this.model+"",")"].join("")}}); \ No newline at end of file +var TagsEditor=Backbone.View.extend(LoggableMixin).extend(HiddenUntilActivatedViewMixin).extend({tagName:"div",className:"tags-display",initialize:function(a){this.listenTo(this.model,"change:tags",function(){this.render()});this.hiddenUntilActivated(a.$activator,a)},render:function(){var a=this;this.$el.html(this._template());this.$input().select2({placeholder:"Add tags",width:"100%",tags:function(){return a._getTagsUsed()}});this._setUpBehaviors();return this},_template:function(){return['<label class="prompt">',_l("Tags"),"</label>",'<input class="tags-input" value="',this.tagsToCSV(),'" />'].join("")},tagsToCSV:function(){var a=this.model.get("tags");if(!_.isArray(a)||_.isEmpty(a)){return""}return a.sort().join(",")},$input:function(){return this.$el.find("input.tags-input")},_getTagsUsed:function(){return Galaxy.currUser.get("tags_used")},_setUpBehaviors:function(){var a=this;this.$input().on("change",function(b){a.model.save({tags:b.val},{silent:true});if(b.added){a._addNewTagToTagsUsed(b.added.text+"")}})},_addNewTagToTagsUsed:function(a){var b=Galaxy.currUser.get("tags_used");if(!_.contains(b,a)){b.push(a);b.sort();Galaxy.currUser.set("tags_used",b)}},remove:function(){this.$input.off();this.stopListening(this.model);Backbone.View.prototype.remove.call(this)},toString:function(){return["TagsEditor(",this.model+"",")"].join("")}}); \ No newline at end of file diff -r 3dde0f4765fbff33789badeaa17503349559ff72 -r 66df8b51245cd141fb7180e401e725ef3b1a0e37 static/scripts/packed/mvc/user/user-model.js --- a/static/scripts/packed/mvc/user/user-model.js +++ b/static/scripts/packed/mvc/user/user-model.js @@ -1,1 +1,1 @@ -var User=Backbone.Model.extend(LoggableMixin).extend({urlRoot:galaxy_config.root+"api/users",defaults:{id:null,username:"("+_l("anonymous user")+")",email:"",total_disk_usage:0,nice_total_disk_usage:"",quota_percent:null},initialize:function(a){this.log("User.initialize:",a);this.on("loaded",function(b,c){this.log(this+" has loaded:",b,c)});this.on("change",function(b,c){this.log(this+" has changed:",b,c.changes)})},isAnonymous:function(){return(!this.get("email"))},loadFromApi:function(d,b){d=d||User.CURRENT_ID_STR;b=b||{};var a=this,c=b.success;b.success=function(f,e){a.trigger("loaded",f,e);if(c){c(f,e)}};if(d===User.CURRENT_ID_STR){b.url=this.urlRoot+"/"+User.CURRENT_ID_STR}return Backbone.Model.prototype.fetch.call(this,b)},clearSessionStorage:function(){for(var a in sessionStorage){if(a.indexOf("HistoryView.")===0){sessionStorage.removeItem(a)}}},toString:function(){var a=[this.get("username")];if(this.get("id")){a.unshift(this.get("id"));a.push(this.get("email"))}return"User("+a.join(":")+")"}});User.CURRENT_ID_STR="current";User.getCurrentUserFromApi=function(b){var a=new User();a.loadFromApi(User.CURRENT_ID_STR,b);return a};var UserCollection=Backbone.Collection.extend(LoggableMixin).extend({model:User,urlRoot:galaxy_config.root+"api/users"}); \ No newline at end of file +var User=Backbone.Model.extend(LoggableMixin).extend({urlRoot:galaxy_config.root+"api/users",defaults:{id:null,username:"("+_l("anonymous user")+")",email:"",total_disk_usage:0,nice_total_disk_usage:"",quota_percent:null},initialize:function(a){this.log("User.initialize:",a);this.on("loaded",function(b,c){this.log(this+" has loaded:",b,c)});this.on("change",function(b,c){this.log(this+" has changed:",b,c.changes)})},isAnonymous:function(){return(!this.get("email"))},loadFromApi:function(d,b){d=d||User.CURRENT_ID_STR;b=b||{};var a=this,c=b.success;b.success=function(f,e){a.trigger("loaded",f,e);if(c){c(f,e)}};if(d===User.CURRENT_ID_STR){b.url=this.urlRoot+"/"+User.CURRENT_ID_STR}return Backbone.Model.prototype.fetch.call(this,b)},clearSessionStorage:function(){for(var a in sessionStorage){if(a.indexOf("history:")===0){sessionStorage.removeItem(a)}else{if(a==="history-panel"){sessionStorage.removeItem(a)}}}},toString:function(){var a=[this.get("username")];if(this.get("id")){a.unshift(this.get("id"));a.push(this.get("email"))}return"User("+a.join(":")+")"}});User.CURRENT_ID_STR="current";User.getCurrentUserFromApi=function(b){var a=new User();a.loadFromApi(User.CURRENT_ID_STR,b);return a};var UserCollection=Backbone.Collection.extend(LoggableMixin).extend({model:User,urlRoot:galaxy_config.root+"api/users"}); \ No newline at end of file Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: 3 new changesets
by commits-noreply＠bitbucket.org 10 Jan '14

10 Jan '14

3 new commits in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/995ca7707640/ Changeset: 995ca7707640 User: dannon Date: 2014-01-10 13:29:05 Summary: Remove dependency on simplejson; convert everything over to using the json module in the python standard library. Affected #: 44 files diff -r 0c5e20558f6bb8d690dd9c9d25c172f08e837b1b -r 995ca770764013bcf0c61092de8436c9278357f7 eggs.ini --- a/eggs.ini +++ b/eggs.ini @@ -26,7 +26,6 @@ pysam = 0.4.2 pysqlite = 2.5.6 python_lzo = 1.08_2.03_static -simplejson = 2.1.1 threadframe = 0.2 guppy = 0.1.8 SQLAlchemy = 0.7.9 diff -r 0c5e20558f6bb8d690dd9c9d25c172f08e837b1b -r 995ca770764013bcf0c61092de8436c9278357f7 lib/galaxy/datatypes/metadata.py --- a/lib/galaxy/datatypes/metadata.py +++ b/lib/galaxy/datatypes/metadata.py @@ -2,15 +2,13 @@ Galaxy Metadata """ -from galaxy import eggs -eggs.require("simplejson") import copy import cPickle +import json import logging import os import shutil -import simplejson import sys import tempfile import weakref @@ -130,7 +128,7 @@ def from_JSON_dict( self, filename ): dataset = self.parent log.debug( 'loading metadata from file for: %s %s' % ( dataset.__class__.__name__, dataset.id ) ) - JSONified_dict = simplejson.load( open( filename ) ) + JSONified_dict = json.load( open( filename ) ) for name, spec in self.spec.items(): if name in JSONified_dict: dataset._metadata[ name ] = spec.param.from_external_value( JSONified_dict[ name ], dataset ) @@ -146,7 +144,7 @@ for name, spec in self.spec.items(): if name in dataset_meta_dict: meta_dict[ name ] = spec.param.to_external_value( dataset_meta_dict[ name ] ) - simplejson.dump( meta_dict, open( filename, 'wb+' ) ) + json.dump( meta_dict, open( filename, 'wb+' ) ) def __getstate__( self ): return None #cannot pickle a weakref item (self._parent), when data._metadata_collection is None, it will be recreated on demand @@ -456,7 +454,7 @@ class DictParameter( MetadataParameter ): def to_string( self, value ): - return simplejson.dumps( value ) + return json.dumps( value ) class PythonObjectParameter( MetadataParameter ): @@ -594,7 +592,7 @@ @classmethod def cleanup_from_JSON_dict_filename( cls, filename ): try: - for key, value in simplejson.load( open( filename ) ).items(): + for key, value in json.load( open( filename ) ).items(): if cls.is_JSONified_value( value ): value = cls.from_JSON( value ) if isinstance( value, cls ) and os.path.exists( value.file_name ): @@ -686,10 +684,10 @@ #file to store a 'return code' indicating the results of the set_meta() call #results code is like (True/False - if setting metadata was successful/failed , exception or string of reason of success/failure ) metadata_files.filename_results_code = abspath( tempfile.NamedTemporaryFile( dir = tmp_dir, prefix = "metadata_results_%s_" % key ).name ) - simplejson.dump( ( False, 'External set_meta() not called' ), open( metadata_files.filename_results_code, 'wb+' ) ) # create the file on disk, so it cannot be reused by tempfile (unlikely, but possible) + json.dump( ( False, 'External set_meta() not called' ), open( metadata_files.filename_results_code, 'wb+' ) ) # create the file on disk, so it cannot be reused by tempfile (unlikely, but possible) #file to store kwds passed to set_meta() metadata_files.filename_kwds = abspath( tempfile.NamedTemporaryFile( dir = tmp_dir, prefix = "metadata_kwds_%s_" % key ).name ) - simplejson.dump( kwds, open( metadata_files.filename_kwds, 'wb+' ), ensure_ascii=True ) + json.dump( kwds, open( metadata_files.filename_kwds, 'wb+' ), ensure_ascii=True ) #existing metadata file parameters need to be overridden with cluster-writable file locations metadata_files.filename_override_metadata = abspath( tempfile.NamedTemporaryFile( dir = tmp_dir, prefix = "metadata_override_%s_" % key ).name ) open( metadata_files.filename_override_metadata, 'wb+' ) # create the file on disk, so it cannot be reused by tempfile (unlikely, but possible) @@ -699,7 +697,7 @@ metadata_temp = MetadataTempFile() shutil.copy( dataset.metadata.get( meta_key, None ).file_name, metadata_temp.file_name ) override_metadata.append( ( meta_key, metadata_temp.to_JSON() ) ) - simplejson.dump( override_metadata, open( metadata_files.filename_override_metadata, 'wb+' ) ) + json.dump( override_metadata, open( metadata_files.filename_override_metadata, 'wb+' ) ) #add to session and flush sa_session.add( metadata_files ) sa_session.flush() @@ -711,7 +709,7 @@ metadata_files = self.get_output_filenames_by_dataset( dataset, sa_session ) if not metadata_files: return False # this file doesn't exist - rval, rstring = simplejson.load( open( metadata_files.filename_results_code ) ) + rval, rstring = json.load( open( metadata_files.filename_results_code ) ) if not rval: log.debug( 'setting metadata externally failed for %s %s: %s' % ( dataset.__class__.__name__, dataset.id, rstring ) ) return rval diff -r 0c5e20558f6bb8d690dd9c9d25c172f08e837b1b -r 995ca770764013bcf0c61092de8436c9278357f7 lib/galaxy/datatypes/sequence.py --- a/lib/galaxy/datatypes/sequence.py +++ b/lib/galaxy/datatypes/sequence.py @@ -2,12 +2,14 @@ Sequence classes """ -import data +from . import data import gzip +import json import logging import os import re import string + from cgi import escape from galaxy import eggs, util @@ -16,8 +18,6 @@ from galaxy.datatypes.sniff import get_test_fname, get_headers from galaxy.datatypes.metadata import MetadataElement -eggs.require("simplejson") -import simplejson try: eggs.require( "bx-python" ) @@ -44,8 +44,8 @@ def set_peek( self, dataset, is_multi_byte=False ): if not dataset.dataset.purged: try: - parsed_data = simplejson.load(open(dataset.file_name)) - # dataset.peek = simplejson.dumps(data, sort_keys=True, indent=4) + parsed_data = json.load(open(dataset.file_name)) + # dataset.peek = json.dumps(data, sort_keys=True, indent=4) dataset.peek = data.get_file_peek( dataset.file_name, is_multi_byte=is_multi_byte ) dataset.blurb = '%d sections' % len(parsed_data['sections']) except Exception, e: @@ -60,7 +60,7 @@ def sniff( self, filename ): if os.path.getsize(filename) < 50000: try: - data = simplejson.load(open(filename)) + data = json.load(open(filename)) sections = data['sections'] for section in sections: if 'start' not in section or 'end' not in section or 'sequences' not in section: @@ -155,7 +155,7 @@ do_slow_split = classmethod(do_slow_split) def do_fast_split( cls, input_datasets, toc_file_datasets, subdir_generator_function, split_params): - data = simplejson.load(open(toc_file_datasets[0].file_name)) + data = json.load(open(toc_file_datasets[0].file_name)) sections = data['sections'] total_sequences = long(0) for section in sections: @@ -191,7 +191,7 @@ toc = toc_file_datasets[ds_no] split_data['args']['toc_file'] = toc.file_name f = open(os.path.join(dir, 'split_info_%s.json' % base_name), 'w') - simplejson.dump(split_data, f) + json.dump(split_data, f) f.close() start_sequence += sequences_per_file[part_no] return directories @@ -557,7 +557,7 @@ sequence_count = long(args['num_sequences']) if 'toc_file' in args: - toc_file = simplejson.load(open(args['toc_file'], 'r')) + toc_file = json.load(open(args['toc_file'], 'r')) commands = Sequence.get_split_commands_with_toc(input_name, output_name, toc_file, start_sequence, sequence_count) else: commands = Sequence.get_split_commands_sequential(is_gzip(input_name), input_name, output_name, start_sequence, sequence_count) diff -r 0c5e20558f6bb8d690dd9c9d25c172f08e837b1b -r 995ca770764013bcf0c61092de8436c9278357f7 lib/galaxy/jobs/runners/drmaa.py --- a/lib/galaxy/jobs/runners/drmaa.py +++ b/lib/galaxy/jobs/runners/drmaa.py @@ -2,13 +2,13 @@ Job control via the DRMAA API. """ +import json +import logging import os +import string +import subprocess import sys import time -import string -import logging -import subprocess -import simplejson as json from galaxy import eggs from galaxy import model diff -r 0c5e20558f6bb8d690dd9c9d25c172f08e837b1b -r 995ca770764013bcf0c61092de8436c9278357f7 lib/galaxy/jobs/runners/lwr_client/action_mapper.py --- a/lib/galaxy/jobs/runners/lwr_client/action_mapper.py +++ b/lib/galaxy/jobs/runners/lwr_client/action_mapper.py @@ -1,4 +1,4 @@ -from simplejson import load +from json import load from os.path import abspath from os.path import dirname from os.path import join diff -r 0c5e20558f6bb8d690dd9c9d25c172f08e837b1b -r 995ca770764013bcf0c61092de8436c9278357f7 lib/galaxy/jobs/runners/lwr_client/client.py --- a/lib/galaxy/jobs/runners/lwr_client/client.py +++ b/lib/galaxy/jobs/runners/lwr_client/client.py @@ -1,7 +1,6 @@ import os import shutil -import simplejson -from simplejson import dumps +from json import dumps, loads from time import sleep from .destination import submit_params @@ -16,7 +15,7 @@ def __call__(self, func): def replacement(*args, **kwargs): response = func(*args, **kwargs) - return simplejson.loads(response) + return loads(response) return replacement diff -r 0c5e20558f6bb8d690dd9c9d25c172f08e837b1b -r 995ca770764013bcf0c61092de8436c9278357f7 lib/galaxy/model/__init__.py --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -6,7 +6,6 @@ """ from galaxy import eggs -eggs.require("simplejson") eggs.require("pexpect") import codecs @@ -15,7 +14,7 @@ import operator import os import pexpect -import simplejson +import json import socket import time from string import Template @@ -2250,7 +2249,7 @@ template_data[template.name] = tmp_dict return template_data def templates_json( self, use_name=False ): - return simplejson.dumps( self.templates_dict( use_name=use_name ) ) + return json.dumps( self.templates_dict( use_name=use_name ) ) def get_display_name( self ): """ diff -r 0c5e20558f6bb8d690dd9c9d25c172f08e837b1b -r 995ca770764013bcf0c61092de8436c9278357f7 lib/galaxy/model/custom_types.py --- a/lib/galaxy/model/custom_types.py +++ b/lib/galaxy/model/custom_types.py @@ -1,8 +1,6 @@ from sqlalchemy.types import * -import pkg_resources -pkg_resources.require("simplejson") -import simplejson +import json import pickle import copy import uuid @@ -19,8 +17,8 @@ log = logging.getLogger( __name__ ) # Default JSON encoder and decoder -json_encoder = simplejson.JSONEncoder( sort_keys=True ) -json_decoder = simplejson.JSONDecoder( ) +json_encoder = json.JSONEncoder( sort_keys=True ) +json_decoder = json.JSONDecoder( ) def _sniffnfix_pg9_hex(value): """ diff -r 0c5e20558f6bb8d690dd9c9d25c172f08e837b1b -r 995ca770764013bcf0c61092de8436c9278357f7 lib/galaxy/tools/__init__.py --- a/lib/galaxy/tools/__init__.py +++ b/lib/galaxy/tools/__init__.py @@ -4,6 +4,7 @@ import binascii import glob +import json import logging import os import pipes @@ -19,14 +20,12 @@ from math import isinf from galaxy import eggs -eggs.require( "simplejson" ) eggs.require( "MarkupSafe" ) #MarkupSafe must load before mako eggs.require( "Mako" ) eggs.require( "elementtree" ) eggs.require( "Paste" ) eggs.require( "SQLAlchemy >= 0.4" ) -import simplejson from cgi import FieldStorage from elementtree import ElementTree from mako.template import Template @@ -869,7 +868,7 @@ value = params_to_strings( tool.inputs, self.inputs, app ) value["__page__"] = self.page value["__rerun_remap_job_id__"] = self.rerun_remap_job_id - value = simplejson.dumps( value ) + value = json.dumps( value ) # Make it secure if secure: a = hmac_new( app.config.tool_secret, value ) @@ -888,7 +887,7 @@ test = hmac_new( app.config.tool_secret, value ) assert a == test # Restore from string - values = json_fix( simplejson.loads( value ) ) + values = json_fix( json.loads( value ) ) self.page = values.pop( "__page__" ) if '__rerun_remap_job_id__' in values: self.rerun_remap_job_id = values.pop( "__rerun_remap_job_id__" ) @@ -2921,7 +2920,7 @@ try: json_file = open( os.path.join( job_working_directory, jobs.TOOL_PROVIDED_JOB_METADATA_FILE ), 'r' ) for line in json_file: - line = simplejson.loads( line ) + line = json.loads( line ) if line.get( 'type' ) == 'new_primary_dataset': new_primary_datasets[ os.path.split( line.get( 'filename' ) )[-1] ] = line except Exception: @@ -3085,7 +3084,7 @@ if json_filename is None: json_filename = file_name out = open( json_filename, 'w' ) - out.write( simplejson.dumps( json_params ) ) + out.write( json.dumps( json_params ) ) out.close() class DataSourceTool( OutputParameterJSONTool ): @@ -3145,7 +3144,7 @@ if json_filename is None: json_filename = file_name out = open( json_filename, 'w' ) - out.write( simplejson.dumps( json_params ) ) + out.write( json.dumps( json_params ) ) out.close() class AsyncDataSourceTool( DataSourceTool ): diff -r 0c5e20558f6bb8d690dd9c9d25c172f08e837b1b -r 995ca770764013bcf0c61092de8436c9278357f7 lib/galaxy/tools/data_manager/manager.py --- a/lib/galaxy/tools/data_manager/manager.py +++ b/lib/galaxy/tools/data_manager/manager.py @@ -1,9 +1,6 @@ -import pkg_resources - -pkg_resources.require( "simplejson" ) - -import os, errno -import simplejson +import errno +import json +import os from galaxy import util from galaxy.util.odict import odict @@ -226,7 +223,7 @@ #TODO: fix this merging below for output_name, output_dataset in out_data.iteritems(): try: - output_dict = simplejson.loads( open( output_dataset.file_name ).read() ) + output_dict = json.loads( open( output_dataset.file_name ).read() ) except Exception, e: log.warning( 'Error reading DataManagerTool json for "%s": %s' % ( output_name, e ) ) continue diff -r 0c5e20558f6bb8d690dd9c9d25c172f08e837b1b -r 995ca770764013bcf0c61092de8436c9278357f7 lib/galaxy/tools/genome_index/__init__.py --- a/lib/galaxy/tools/genome_index/__init__.py +++ b/lib/galaxy/tools/genome_index/__init__.py @@ -1,6 +1,11 @@ from __future__ import with_statement -import os, shutil, logging, tempfile, tarfile +import json +import logging +import os +import shutil +import tarfile +import tempfile from galaxy import model, util from galaxy.web.framework.helpers import to_unicode @@ -9,9 +14,6 @@ from galaxy.web.base.controller import UsesHistoryMixin from galaxy.tools.data import ToolDataTableManager -import pkg_resources -pkg_resources.require("simplejson") -import simplejson log = logging.getLogger(__name__) @@ -74,7 +76,7 @@ fp = open( gitd.dataset.get_file_name(), 'r' ) deferred = sa_session.query( model.DeferredJob ).filter_by( id=gitd.deferred_job_id ).first() try: - logloc = simplejson.load( fp ) + logloc = json.load( fp ) except ValueError: deferred.state = app.model.DeferredJob.states.ERROR sa_session.add( deferred ) diff -r 0c5e20558f6bb8d690dd9c9d25c172f08e837b1b -r 995ca770764013bcf0c61092de8436c9278357f7 lib/galaxy/tools/genome_index/index_genome.py --- a/lib/galaxy/tools/genome_index/index_genome.py +++ b/lib/galaxy/tools/genome_index/index_genome.py @@ -7,11 +7,17 @@ """ from __future__ import with_statement -import optparse, sys, os, tempfile, time, subprocess, shlex, tarfile, shutil +import json +import optparse +import os +import shlex +import shutil +import subprocess +import sys +import tarfile +import tempfile +import time -import pkg_resources -pkg_resources.require("simplejson") -import simplejson class ManagedIndexer(): def __init__( self, output_file, infile, workingdir, rsync_url, tooldata ): @@ -76,7 +82,7 @@ return result def _flush_files( self ): - simplejson.dump( self.locations, self.outfile ) + json.dump( self.locations, self.outfile ) self.outfile.close() self.logfile.close() @@ -318,4 +324,4 @@ returncode = idxobj.run_indexer( indexer ) if not returncode: exit(1) - exit(0) \ No newline at end of file + exit(0) diff -r 0c5e20558f6bb8d690dd9c9d25c172f08e837b1b -r 995ca770764013bcf0c61092de8436c9278357f7 lib/galaxy/tools/imp_exp/__init__.py --- a/lib/galaxy/tools/imp_exp/__init__.py +++ b/lib/galaxy/tools/imp_exp/__init__.py @@ -1,4 +1,4 @@ -import os, shutil, logging, tempfile, simplejson +import os, shutil, logging, tempfile, json from galaxy import model from galaxy.tools.parameters.basic import UnvalidatedValue from galaxy.web.framework.helpers import to_unicode @@ -226,13 +226,13 @@ self.sa_session.add( imported_job ) self.sa_session.flush() - class HistoryDatasetAssociationIDEncoder( simplejson.JSONEncoder ): + class HistoryDatasetAssociationIDEncoder( json.JSONEncoder ): """ Custom JSONEncoder for a HistoryDatasetAssociation that encodes an HDA as its ID. """ def default( self, obj ): """ Encode an HDA, default encoding for everything else. """ if isinstance( obj, model.HistoryDatasetAssociation ): return obj.id - return simplejson.JSONEncoder.default( self, obj ) + return json.JSONEncoder.default( self, obj ) # Set parameters. May be useful to look at metadata.py for creating parameters. # TODO: there may be a better way to set parameters, e.g.: @@ -311,7 +311,7 @@ del metadata[ name ] return metadata - class HistoryDatasetAssociationEncoder( simplejson.JSONEncoder ): + class HistoryDatasetAssociationEncoder( json.JSONEncoder ): """ Custom JSONEncoder for a HistoryDatasetAssociation. """ def default( self, obj ): """ Encode an HDA, default encoding for everything else. """ @@ -337,7 +337,7 @@ } if isinstance( obj, UnvalidatedValue ): return obj.__str__() - return simplejson.JSONEncoder.default( self, obj ) + return json.JSONEncoder.default( self, obj ) # # Create attributes/metadata files for export. diff -r 0c5e20558f6bb8d690dd9c9d25c172f08e837b1b -r 995ca770764013bcf0c61092de8436c9278357f7 lib/galaxy/util/__init__.py --- a/lib/galaxy/util/__init__.py +++ b/lib/galaxy/util/__init__.py @@ -2,7 +2,26 @@ Utility functions used systemwide. """ -import binascii, errno, grp, logging, os, pickle, random, re, shutil, smtplib, stat, string, sys, tempfile, threading + +from __future__ import absolute_import + +import binascii +import errno +import grp +import json +import logging +import os +import pickle +import random +import re +import shutil +import smtplib +import stat +import string +import sys +import tempfile +import threading + from email.MIMEText import MIMEText from os.path import relpath @@ -21,12 +40,9 @@ eggs.require( "wchartype" ) import wchartype -from inflection import Inflector, English +from .inflection import Inflector, English inflector = Inflector(English) -eggs.require( "simplejson" ) -import simplejson - log = logging.getLogger(__name__) _lock = threading.RLock() @@ -292,8 +308,8 @@ def pretty_print_json(json_data, is_json_string=False): if is_json_string: - json_data = simplejson.loads(json_data) - return simplejson.dumps(json_data, sort_keys=True, indent=4 * ' ') + json_data = json.loads(json_data) + return json.dumps(json_data, sort_keys=True, indent=4 * ' ') # characters that are valid valid_chars = set(string.letters + string.digits + " -=_.()/+*^,:?!") diff -r 0c5e20558f6bb8d690dd9c9d25c172f08e837b1b -r 995ca770764013bcf0c61092de8436c9278357f7 lib/galaxy/util/json.py --- a/lib/galaxy/util/json.py +++ b/lib/galaxy/util/json.py @@ -1,16 +1,15 @@ +from __future__ import absolute_import __all__ = [ "to_json_string", "from_json_string", "json_fix", "validate_jsonrpc_request", "validate_jsonrpc_response", "jsonrpc_request", "jsonrpc_response" ] -import random, string, logging +import json +import logging +import random import socket +import string -import pkg_resources -pkg_resources.require( "simplejson" ) - -import simplejson - -to_json_string = simplejson.dumps -from_json_string = simplejson.loads +to_json_string = json.dumps +from_json_string = json.loads log = logging.getLogger( __name__ ) diff -r 0c5e20558f6bb8d690dd9c9d25c172f08e837b1b -r 995ca770764013bcf0c61092de8436c9278357f7 lib/galaxy/visualization/data_providers/phyloviz/baseparser.py --- a/lib/galaxy/visualization/data_providers/phyloviz/baseparser.py +++ b/lib/galaxy/visualization/data_providers/phyloviz/baseparser.py @@ -1,6 +1,4 @@ -import pkg_resources -pkg_resources.require("simplejson") -import simplejson +import json class Node(object): """Node class of PhyloTree, which represents a CLAUDE in a phylogenetic tree""" @@ -118,7 +116,7 @@ def toJson(self, jsonDict): """Convenience method to get a json string from a python json dict""" - return simplejson.dumps(jsonDict) + return json.dumps(jsonDict) def _writeJsonToFile(self, filepath, json): """Writes the file out to the system""" diff -r 0c5e20558f6bb8d690dd9c9d25c172f08e837b1b -r 995ca770764013bcf0c61092de8436c9278357f7 lib/galaxy/web/framework/__init__.py --- a/lib/galaxy/web/framework/__init__.py +++ b/lib/galaxy/web/framework/__init__.py @@ -2,6 +2,7 @@ Galaxy web application framework """ +import hashlib import inspect import os import pkg_resources @@ -9,27 +10,24 @@ import socket import string import time -import hashlib + +from functools import wraps from Cookie import CookieError - pkg_resources.require( "Cheetah" ) from Cheetah.Template import Template + +#TODO: Relative imports to be removed import base -from functools import wraps +import helpers + from galaxy import util from galaxy.exceptions import MessageException -from galaxy.util.json import to_json_string, from_json_string +from galaxy.util import asbool +from galaxy.util import safe_str_cmp from galaxy.util.backports.importlib import import_module +from galaxy.util.json import from_json_string, to_json_string from galaxy.util.sanitize_html import sanitize_html -from galaxy.util import safe_str_cmp - -pkg_resources.require( "simplejson" ) -import simplejson - -import helpers - -from galaxy.util import asbool import paste.httpexceptions @@ -76,7 +74,7 @@ @wraps(func) def decorator( self, trans, *args, **kwargs ): trans.response.set_content_type( "text/javascript" ) - return simplejson.dumps( func( self, trans, *args, **kwargs ) ) + return to_json_string( func( self, trans, *args, **kwargs ) ) if not hasattr(func, '_orig'): decorator._orig = func decorator.exposed = True @@ -86,7 +84,7 @@ @wraps(func) def decorator( self, trans, *args, **kwargs ): trans.response.set_content_type( "text/javascript" ) - return simplejson.dumps( func( self, trans, *args, **kwargs ), indent=4, sort_keys=True ) + return to_json_string( func( self, trans, *args, **kwargs ), indent=4, sort_keys=True ) if not hasattr(func, '_orig'): decorator._orig = func decorator.exposed = True @@ -158,7 +156,7 @@ for k, v in payload.iteritems(): if isinstance(v, (str, unicode)): try: - payload[k] = simplejson.loads(v) + payload[k] = from_json_string(v) except: # may not actually be json, just continue pass @@ -167,7 +165,7 @@ # Assume application/json content type and parse request body manually, since wsgi won't do it. However, the order of this check # should ideally be in reverse, with the if clause being a check for application/json and the else clause assuming a standard encoding # such as multipart/form-data. Leaving it as is for backward compatibility, just in case. - payload = util.recursively_stringify_dictionary_keys( simplejson.loads( trans.request.body ) ) + payload = util.recursively_stringify_dictionary_keys( from_json_string( trans.request.body ) ) return payload try: kwargs['payload'] = extract_payload_from_request(trans, func, kwargs) @@ -198,9 +196,9 @@ try: rval = func( self, trans, *args, **kwargs) if to_json and trans.debug: - rval = simplejson.dumps( rval, indent=4, sort_keys=True ) + rval = to_json_string( rval, indent=4, sort_keys=True ) elif to_json: - rval = simplejson.dumps( rval ) + rval = to_json_string( rval ) return rval except paste.httpexceptions.HTTPException: raise # handled diff -r 0c5e20558f6bb8d690dd9c9d25c172f08e837b1b -r 995ca770764013bcf0c61092de8436c9278357f7 lib/galaxy/webapps/demo_sequencer/framework/__init__.py --- a/lib/galaxy/webapps/demo_sequencer/framework/__init__.py +++ b/lib/galaxy/webapps/demo_sequencer/framework/__init__.py @@ -2,9 +2,15 @@ Demo sequencer web application framework """ +import json +import os import pkg_resources +import random +import socket +import string +import sys +import time -import os, sys, time, socket, random, string pkg_resources.require( "Cheetah" ) from Cheetah.Template import Template @@ -19,9 +25,6 @@ from galaxy.util import asbool -pkg_resources.require( "simplejson" ) -import simplejson - pkg_resources.require( "Mako" ) import mako.template import mako.lookup diff -r 0c5e20558f6bb8d690dd9c9d25c172f08e837b1b -r 995ca770764013bcf0c61092de8436c9278357f7 lib/galaxy/webapps/galaxy/controllers/data_admin.py --- a/lib/galaxy/webapps/galaxy/controllers/data_admin.py +++ b/lib/galaxy/webapps/galaxy/controllers/data_admin.py @@ -1,14 +1,13 @@ -import sys, ftplib +import ftplib +import json +import sys from galaxy import model, util from galaxy.jobs import transfer_manager +from galaxy.model.orm import * from galaxy.web.base.controller import * -from galaxy.web.framework.helpers import time_ago, iff, grids -from galaxy.model.orm import * +from galaxy.web.framework.helpers import grids, iff, time_ago from library_common import get_comptypes, lucene_search, whoosh_search -import pkg_resources -pkg_resources.require("simplejson") -import simplejson # Older py compatibility try: @@ -161,7 +160,7 @@ gname = deferred.params[ 'intname' ] indexers = ', '.join( deferred.params[ 'indexes' ] ) jobs = self._get_jobs( deferred, trans ) - jsonjobs = simplejson.dumps( jobs ) + jsonjobs = json.dumps( jobs ) return trans.fill_template( '/admin/data_admin/download_status.mako', name=gname, indexers=indexers, mainjob=jobid, jobs=jobs, jsonjobs=jsonjobs ) @web.expose @@ -173,7 +172,7 @@ jobid = params.get( 'jobid', '' ) job = sa_session.query( model.DeferredJob ).filter_by( id=jobid ).first() jobs = self._get_jobs( job, trans ) - return trans.fill_template( '/admin/data_admin/ajax_status.mako', json=simplejson.dumps( jobs ) ) + return trans.fill_template( '/admin/data_admin/ajax_status.mako', json=json.dumps( jobs ) ) def _get_job( self, jobid, jobtype, trans ): sa = trans.app.model.context.current @@ -297,4 +296,4 @@ params = dict( status='ok', dbkey=dbkey, datatype='fasta', url=url, user=trans.user.id, liftover=newlift, longname=longname, indexers=indexers ) - return params \ No newline at end of file + return params diff -r 0c5e20558f6bb8d690dd9c9d25c172f08e837b1b -r 995ca770764013bcf0c61092de8436c9278357f7 lib/galaxy/webapps/galaxy/controllers/root.py --- a/lib/galaxy/webapps/galaxy/controllers/root.py +++ b/lib/galaxy/webapps/galaxy/controllers/root.py @@ -508,7 +508,7 @@ Attempts to parse values passed as boolean, float, then int. Defaults to string. Non-recursive (will not parse lists). """ - #TODO: use simplejson or json + #TODO: use json rval = {} for k in kwd: rval[ k ] = kwd[k] diff -r 0c5e20558f6bb8d690dd9c9d25c172f08e837b1b -r 995ca770764013bcf0c61092de8436c9278357f7 lib/galaxy/webapps/galaxy/controllers/workflow.py --- a/lib/galaxy/webapps/galaxy/controllers/workflow.py +++ b/lib/galaxy/webapps/galaxy/controllers/workflow.py @@ -1,13 +1,12 @@ import pkg_resources -pkg_resources.require( "simplejson" ) pkg_resources.require( "SVGFig" ) import base64 import httplib +import json import math import os import sgmllib -import simplejson import svgfig import urllib2 @@ -20,17 +19,17 @@ from galaxy import web from galaxy.datatypes.data import Data from galaxy.jobs.actions.post import ActionBox +from galaxy.model.item_attrs import UsesAnnotations, UsesItemRatings from galaxy.model.mapping import desc from galaxy.tools.parameters import RuntimeValue, visit_input_values from galaxy.tools.parameters.basic import DataToolParameter, DrillDownSelectToolParameter, SelectToolParameter, UnvalidatedValue from galaxy.tools.parameters.grouping import Conditional, Repeat +from galaxy.util.json import to_json_string from galaxy.util.odict import odict -from galaxy.util.json import to_json_string from galaxy.util.sanitize_html import sanitize_html from galaxy.util.topsort import CycleError, topsort, topsort_levels from galaxy.web import error, url_for from galaxy.web.base.controller import BaseUIController, SharableMixin, UsesStoredWorkflowMixin -from galaxy.model.item_attrs import UsesAnnotations, UsesItemRatings from galaxy.web.framework import form from galaxy.web.framework.helpers import grids, time_ago from galaxy.web.framework.helpers import to_unicode @@ -812,7 +811,7 @@ # Put parameters in workflow mode trans.workflow_building_mode = True # Convert incoming workflow data from json - data = simplejson.loads( workflow_data ) + data = json.loads( workflow_data ) # Create new workflow from incoming data workflow = model.Workflow() # Just keep the last name (user can rename later) @@ -920,7 +919,7 @@ # # Create workflow content JSON. - workflow_content = simplejson.dumps( workflow_dict, indent=4, sort_keys=True ) + workflow_content = json.dumps( workflow_dict, indent=4, sort_keys=True ) # Create myExperiment request. request_raw = trans.fill_template( "workflow/myexp_export.mako", \ @@ -1073,7 +1072,7 @@ if workflow_data: # Convert incoming workflow data from json try: - data = simplejson.loads( workflow_data ) + data = json.loads( workflow_data ) except Exception, e: data = None message = "The data content does not appear to be a Galaxy workflow.<br/>Exception: %s" % str( e ) @@ -1294,7 +1293,7 @@ # It is possible for a workflow to have 0 steps if len( workflow.steps ) == 0: error( "Workflow cannot be run because it does not have any steps" ) - #workflow = Workflow.from_simple( simplejson.loads( stored.encoded_value ), trans.app ) + #workflow = Workflow.from_simple( json.loads( stored.encoded_value ), trans.app ) if workflow.has_cycles: error( "Workflow cannot be run because it contains cycles" ) if workflow.has_errors: diff -r 0c5e20558f6bb8d690dd9c9d25c172f08e837b1b -r 995ca770764013bcf0c61092de8436c9278357f7 lib/tool_shed/scripts/api/common.py --- a/lib/tool_shed/scripts/api/common.py +++ b/lib/tool_shed/scripts/api/common.py @@ -1,4 +1,8 @@ -import os, sys, urllib, urllib2 +import json +import os +import sys +import urllib +import urllib2 new_path = [ os.path.join( os.path.dirname( __file__ ), '..', '..', '..', '..', 'lib' ) ] new_path.extend( sys.path[ 1: ] ) @@ -7,9 +11,6 @@ from galaxy import eggs import pkg_resources -pkg_resources.require( "simplejson" ) -import simplejson - pkg_resources.require( "pycrypto" ) from Crypto.Cipher import Blowfish from Crypto.Util.randpool import RandomPool @@ -29,9 +30,9 @@ # Sends an API DELETE request and acts as a generic formatter for the JSON response - 'data' will become the JSON payload read by Galaxy. try: url = make_url( api_key, url ) - req = urllib2.Request( url, headers = { 'Content-Type': 'application/json' }, data = simplejson.dumps( data )) + req = urllib2.Request( url, headers = { 'Content-Type': 'application/json' }, data = json.dumps( data )) req.get_method = lambda: 'DELETE' - r = simplejson.loads( urllib2.urlopen( req ).read() ) + r = json.loads( urllib2.urlopen( req ).read() ) except urllib2.HTTPError, e: if return_formatted: print e @@ -86,8 +87,8 @@ # Do the actual GET. url = make_url( url, api_key=api_key ) try: - return simplejson.loads( urllib2.urlopen( url ).read() ) - except simplejson.decoder.JSONDecodeError, e: + return json.loads( urllib2.urlopen( url ).read() ) + except ValueError, e: print "URL did not return JSON data" sys.exit(1) @@ -106,15 +107,15 @@ def post( url, data, api_key=None ): # Do the actual POST. url = make_url( url, api_key=api_key ) - req = urllib2.Request( url, headers = { 'Content-Type': 'application/json' }, data = simplejson.dumps( data ) ) - return simplejson.loads( urllib2.urlopen( req ).read() ) + req = urllib2.Request( url, headers = { 'Content-Type': 'application/json' }, data = json.dumps( data ) ) + return json.loads( urllib2.urlopen( req ).read() ) def put( url, data, api_key=None ): # Do the actual PUT. url = make_url( url, api_key=api_key ) - req = urllib2.Request( url, headers = { 'Content-Type': 'application/json' }, data = simplejson.dumps( data )) + req = urllib2.Request( url, headers = { 'Content-Type': 'application/json' }, data = json.dumps( data )) req.get_method = lambda: 'PUT' - return simplejson.loads( urllib2.urlopen( req ).read() ) + return json.loads( urllib2.urlopen( req ).read() ) def submit( url, data, api_key=None, return_formatted=True ): # Sends an API POST request and acts as a generic formatter for the JSON response - 'data' will become the JSON payload read by Galaxy. diff -r 0c5e20558f6bb8d690dd9c9d25c172f08e837b1b -r 995ca770764013bcf0c61092de8436c9278357f7 lib/tool_shed/scripts/api/tool_shed_repository_revision_update.py --- a/lib/tool_shed/scripts/api/tool_shed_repository_revision_update.py +++ b/lib/tool_shed/scripts/api/tool_shed_repository_revision_update.py @@ -5,17 +5,16 @@ usage: tool_shed_repository_revision_update.py key url key1=value1 key2=value2 ... """ -import os, sys +import json +import os +import sys + sys.path.insert( 0, os.path.dirname( __file__ ) ) from common import update -import pkg_resources -pkg_resources.require( "simplejson" ) -import simplejson - -to_json_string = simplejson.dumps -from_json_string = simplejson.loads +to_json_string = json.dumps +from_json_string = json.loads data = {} for key, value in [ kwarg.split( '=', 1 ) for kwarg in sys.argv[ 3: ] ]: diff -r 0c5e20558f6bb8d690dd9c9d25c172f08e837b1b -r 995ca770764013bcf0c61092de8436c9278357f7 lib/tool_shed/util/encoding_util.py --- a/lib/tool_shed/util/encoding_util.py +++ b/lib/tool_shed/util/encoding_util.py @@ -1,14 +1,10 @@ import binascii +import json import logging from galaxy import eggs from galaxy.util.hash_util import hmac_new from galaxy.util.json import json_fix -import pkg_resources - -pkg_resources.require( "simplejson" ) -import simplejson - log = logging.getLogger( __name__ ) encoding_sep = '__esep__' @@ -23,7 +19,7 @@ # Restore from string values = None try: - values = simplejson.loads( value ) + values = json.loads( value ) except Exception, e: #log.debug( "Decoding json value from tool shed for value '%s' threw exception: %s" % ( str( value ), str( e ) ) ) pass @@ -39,7 +35,7 @@ def tool_shed_encode( val ): if isinstance( val, dict ): - value = simplejson.dumps( val ) + value = json.dumps( val ) else: value = val a = hmac_new( 'ToolShedAndGalaxyMustHaveThisSameKey', value ) diff -r 0c5e20558f6bb8d690dd9c9d25c172f08e837b1b -r 995ca770764013bcf0c61092de8436c9278357f7 scripts/api/common.py --- a/scripts/api/common.py +++ b/scripts/api/common.py @@ -1,3 +1,4 @@ +import json import logging import os import sys @@ -10,9 +11,6 @@ from galaxy import eggs import pkg_resources -pkg_resources.require( "simplejson" ) -import simplejson - pkg_resources.require( "pycrypto" ) from Crypto.Cipher import Blowfish from Crypto.Util.randpool import RandomPool @@ -35,30 +33,30 @@ # Do the actual GET. url = make_url( api_key, url ) try: - return simplejson.loads( urllib2.urlopen( url ).read() ) - except simplejson.decoder.JSONDecodeError, e: + return json.loads( urllib2.urlopen( url ).read() ) + except json.decoder.JSONDecodeError, e: print "URL did not return JSON data" sys.exit(1) def post( api_key, url, data ): # Do the actual POST. url = make_url( api_key, url ) - req = urllib2.Request( url, headers = { 'Content-Type': 'application/json' }, data = simplejson.dumps( data ) ) - return simplejson.loads( urllib2.urlopen( req ).read() ) + req = urllib2.Request( url, headers = { 'Content-Type': 'application/json' }, data = json.dumps( data ) ) + return json.loads( urllib2.urlopen( req ).read() ) def put( api_key, url, data ): # Do the actual PUT url = make_url( api_key, url ) - req = urllib2.Request( url, headers = { 'Content-Type': 'application/json' }, data = simplejson.dumps( data )) + req = urllib2.Request( url, headers = { 'Content-Type': 'application/json' }, data = json.dumps( data )) req.get_method = lambda: 'PUT' - return simplejson.loads( urllib2.urlopen( req ).read() ) + return json.loads( urllib2.urlopen( req ).read() ) def __del( api_key, url, data ): # Do the actual DELETE url = make_url( api_key, url ) - req = urllib2.Request( url, headers = { 'Content-Type': 'application/json' }, data = simplejson.dumps( data )) + req = urllib2.Request( url, headers = { 'Content-Type': 'application/json' }, data = json.dumps( data )) req.get_method = lambda: 'DELETE' - return simplejson.loads( urllib2.urlopen( req ).read() ) + return json.loads( urllib2.urlopen( req ).read() ) def display( api_key, url, return_formatted=True ): diff -r 0c5e20558f6bb8d690dd9c9d25c172f08e837b1b -r 995ca770764013bcf0c61092de8436c9278357f7 scripts/api/workflow_import_from_file_rpark.py --- a/scripts/api/workflow_import_from_file_rpark.py +++ b/scripts/api/workflow_import_from_file_rpark.py @@ -11,11 +11,11 @@ from common import submit ### Rpark edit ### -import simplejson +import json def openWorkflow(in_file): with open(in_file) as f: - temp_data = simplejson.load(f) + temp_data = json.load(f) return temp_data; diff -r 0c5e20558f6bb8d690dd9c9d25c172f08e837b1b -r 995ca770764013bcf0c61092de8436c9278357f7 scripts/drmaa_external_killer.py --- a/scripts/drmaa_external_killer.py +++ b/scripts/drmaa_external_killer.py @@ -4,10 +4,11 @@ Terminates a DRMAA job if given a job id and (appropriate) user id. """ +import errno +import json import os +import pwd import sys -import errno -import pwd #import drmaa new_path = [ os.path.join( os.getcwd(), "lib" ) ] new_path.extend( sys.path[1:] ) # remove scripts/ from the path @@ -15,8 +16,6 @@ from galaxy import eggs import pkg_resources -pkg_resources.require("simplejson") -import simplejson as json pkg_resources.require("drmaa") import drmaa diff -r 0c5e20558f6bb8d690dd9c9d25c172f08e837b1b -r 995ca770764013bcf0c61092de8436c9278357f7 scripts/drmaa_external_runner.py --- a/scripts/drmaa_external_runner.py +++ b/scripts/drmaa_external_runner.py @@ -10,8 +10,8 @@ import sys import errno import pwd +import json -#import simplejson as json #import drmaa new_path = [ os.path.join( os.getcwd(), "lib" ) ] new_path.extend( sys.path[1:] ) # remove scripts/ from the path @@ -19,8 +19,6 @@ from galaxy import eggs import pkg_resources -pkg_resources.require("simplejson") -import simplejson as json pkg_resources.require("drmaa") import drmaa diff -r 0c5e20558f6bb8d690dd9c9d25c172f08e837b1b -r 995ca770764013bcf0c61092de8436c9278357f7 scripts/external_chown_script.py --- a/scripts/external_chown_script.py +++ b/scripts/external_chown_script.py @@ -1,17 +1,17 @@ #!/usr/bin/env python +import errno +import json import os +import pwd import sys -import errno -import pwd #import drmaa + new_path = [ os.path.join( os.getcwd(), "lib" ) ] new_path.extend( sys.path[1:] ) # remove scripts/ from the path sys.path = new_path from galaxy import eggs import pkg_resources -pkg_resources.require("simplejson") -import simplejson as json pkg_resources.require("drmaa") import drmaa diff -r 0c5e20558f6bb8d690dd9c9d25c172f08e837b1b -r 995ca770764013bcf0c61092de8436c9278357f7 scripts/extract_dataset_part.py --- a/scripts/extract_dataset_part.py +++ b/scripts/extract_dataset_part.py @@ -6,9 +6,10 @@ on a gid in a scatter-gather mode. This does part of the scatter. """ +import json +import logging import os import sys -import logging logging.basicConfig() log = logging.getLogger( __name__ ) @@ -16,11 +17,6 @@ new_path.extend( sys.path[1:] ) # remove scripts/ from the path sys.path = new_path -from galaxy import eggs -import pkg_resources -pkg_resources.require("simplejson") -import simplejson - # This junk is here to prevent loading errors import galaxy.model.mapping #need to load this before we unpickle, in order to setup properties assigned by the mappers galaxy.model.Job() #this looks REAL stupid, but it is REQUIRED in order for SA to insert parameters into the classes defined by the mappers --> it appears that instantiating ANY mapper'ed class would suffice here @@ -33,7 +29,7 @@ if not os.path.isfile(file_path): #Nothing to do - some splitters don't write a JSON file sys.exit(0) - data = simplejson.load(open(file_path, 'r')) + data = json.load(open(file_path, 'r')) try: class_name_parts = data['class_name'].split('.') module_name = '.'.join(class_name_parts[:-1]) diff -r 0c5e20558f6bb8d690dd9c9d25c172f08e837b1b -r 995ca770764013bcf0c61092de8436c9278357f7 scripts/galaxy_messaging/server/data_transfer.py --- a/scripts/galaxy_messaging/server/data_transfer.py +++ b/scripts/galaxy_messaging/server/data_transfer.py @@ -13,12 +13,21 @@ """ import ConfigParser -import sys, os, time, traceback +import cookielib +import datetime +import logging import optparse -import urllib,urllib2, cookielib, shutil -import logging, time, datetime +import os +import shutil +import sys +import time +import time +import traceback +import urllib +import urllib2 import xml.dom.minidom + from xml_helper import get_value, get_value_index log = logging.getLogger( "datatx_" + str( os.getpid() ) ) @@ -39,14 +48,11 @@ sys.path = new_path from galaxy import eggs -from galaxy.util.json import from_json_string, to_json_string from galaxy.model import SampleDataset from galaxy.web.api.samples import SamplesAPIController import pkg_resources pkg_resources.require( "pexpect" ) import pexpect -pkg_resources.require( "simplejson" ) -import simplejson log.debug(str(dir(api))) diff -r 0c5e20558f6bb8d690dd9c9d25c172f08e837b1b -r 995ca770764013bcf0c61092de8436c9278357f7 scripts/set_metadata.py --- a/scripts/set_metadata.py +++ b/scripts/set_metadata.py @@ -10,9 +10,11 @@ logging.basicConfig() log = logging.getLogger( __name__ ) +import cPickle +import json import os import sys -import cPickle + # ensure supported version from check_python import check_python try: @@ -26,8 +28,6 @@ from galaxy import eggs import pkg_resources -pkg_resources.require("simplejson") -import simplejson import galaxy.model.mapping # need to load this before we unpickle, in order to setup properties assigned by the mappers galaxy.model.Job() # this looks REAL stupid, but it is REQUIRED in order for SA to insert parameters into the classes defined by the mappers --> it appears that instantiating ANY mapper'ed class would suffice here from galaxy.util import stringify_dictionary_keys @@ -107,17 +107,17 @@ dataset.extension = ext_override[ dataset.dataset.id ] # Metadata FileParameter types may not be writable on a cluster node, and are therefore temporarily substituted with MetadataTempFiles if override_metadata: - override_metadata = simplejson.load( open( override_metadata ) ) + override_metadata = json.load( open( override_metadata ) ) for metadata_name, metadata_file_override in override_metadata: if galaxy.datatypes.metadata.MetadataTempFile.is_JSONified_value( metadata_file_override ): metadata_file_override = galaxy.datatypes.metadata.MetadataTempFile.from_JSON( metadata_file_override ) setattr( dataset.metadata, metadata_name, metadata_file_override ) - kwds = stringify_dictionary_keys( simplejson.load( open( filename_kwds ) ) ) # load kwds; need to ensure our keywords are not unicode + kwds = stringify_dictionary_keys( json.load( open( filename_kwds ) ) ) # load kwds; need to ensure our keywords are not unicode dataset.datatype.set_meta( dataset, **kwds ) dataset.metadata.to_JSON_dict( filename_out ) # write out results of set_meta - simplejson.dump( ( True, 'Metadata has been set successfully' ), open( filename_results_code, 'wb+' ) ) # setting metadata has succeeded + json.dump( ( True, 'Metadata has been set successfully' ), open( filename_results_code, 'wb+' ) ) # setting metadata has succeeded except Exception, e: - simplejson.dump( ( False, str( e ) ), open( filename_results_code, 'wb+' ) ) # setting metadata has failed somehow + json.dump( ( False, str( e ) ), open( filename_results_code, 'wb+' ) ) # setting metadata has failed somehow clear_mappers() # Shut down any additional threads that might have been created via the ObjectStore object_store.shutdown() diff -r 0c5e20558f6bb8d690dd9c9d25c172f08e837b1b -r 995ca770764013bcf0c61092de8436c9278357f7 test/base/interactor.py --- a/test/base/interactor.py +++ b/test/base/interactor.py @@ -5,7 +5,7 @@ import galaxy.model from galaxy.model.orm import and_, desc from base.test_db_util import sa_session -from simplejson import dumps, loads +from json import dumps, loads from logging import getLogger log = getLogger( __name__ ) diff -r 0c5e20558f6bb8d690dd9c9d25c172f08e837b1b -r 995ca770764013bcf0c61092de8436c9278357f7 test/functional/test_workflow.py --- a/test/functional/test_workflow.py +++ b/test/functional/test_workflow.py @@ -5,7 +5,7 @@ from galaxy.util import parse_xml from galaxy.tools.test import parse_param_elem, require_file, test_data_iter, parse_output_elems -from simplejson import load, dumps +from json import load, dumps from logging import getLogger log = getLogger( __name__ ) diff -r 0c5e20558f6bb8d690dd9c9d25c172f08e837b1b -r 995ca770764013bcf0c61092de8436c9278357f7 test/tool_shed/base/twilltestcase.py --- a/test/tool_shed/base/twilltestcase.py +++ b/test/tool_shed/base/twilltestcase.py @@ -3,7 +3,6 @@ import os import re import test_db_util -import simplejson import shutil import logging import time diff -r 0c5e20558f6bb8d690dd9c9d25c172f08e837b1b -r 995ca770764013bcf0c61092de8436c9278357f7 tools/filters/join.py --- a/tools/filters/join.py +++ b/tools/filters/join.py @@ -8,20 +8,15 @@ """ -import optparse, os, sys, tempfile, struct +import json +import optparse +import os import psyco_full - -try: - simple_json_exception = None - from galaxy import eggs - from galaxy.util.bunch import Bunch - from galaxy.util import stringify_dictionary_keys - import pkg_resources - pkg_resources.require("simplejson") - import simplejson -except Exception, e: - simplejson_exception = e - simplejson = None +import struct +import sys +import tempfile +from galaxy.util.bunch import Bunch +from galaxy.util import stringify_dictionary_keys class OffsetList: @@ -337,11 +332,9 @@ fill_options = None if options.fill_options_file is not None: try: - if simplejson is None: - raise simplejson_exception - fill_options = Bunch( **stringify_dictionary_keys( simplejson.load( open( options.fill_options_file ) ) ) ) #simplejson.load( open( options.fill_options_file ) ) + fill_options = Bunch( **stringify_dictionary_keys( json.load( open( options.fill_options_file ) ) ) ) #json.load( open( options.fill_options_file ) ) except Exception, e: - print "Warning: Ignoring fill options due to simplejson error (%s)." % e + print "Warning: Ignoring fill options due to json error (%s)." % e if fill_options is None: fill_options = Bunch() if 'fill_unjoined_only' not in fill_options: diff -r 0c5e20558f6bb8d690dd9c9d25c172f08e837b1b -r 995ca770764013bcf0c61092de8436c9278357f7 tools/filters/joiner.xml --- a/tools/filters/joiner.xml +++ b/tools/filters/joiner.xml @@ -51,7 +51,7 @@ </inputs><configfiles><configfile name="fill_options_file"><% -import simplejson +import json %> #set $__fill_options = {} #if $fill_empty_columns['fill_empty_columns_switch'] == 'fill_empty': @@ -72,7 +72,7 @@ #end for #end if #end if -${simplejson.dumps( __fill_options )} +${json.dumps( __fill_options )} </configfile></configfiles><outputs> diff -r 0c5e20558f6bb8d690dd9c9d25c172f08e837b1b -r 995ca770764013bcf0c61092de8436c9278357f7 tools/genomespace/genomespace_exporter.py --- a/tools/genomespace/genomespace_exporter.py +++ b/tools/genomespace/genomespace_exporter.py @@ -1,12 +1,16 @@ #Dan Blankenberg -import optparse, os, urllib2, urllib, cookielib, hashlib, base64, cgi, binascii, logging - -from galaxy import eggs -import pkg_resources - -pkg_resources.require( "simplejson" ) -import simplejson +import base64 +import binascii +import cgi +import cookielib +import hashlib +import json +import logging +import optparse +import os +import urllib +import urllib2 log = logging.getLogger( "tools.genomespace.genomespace_exporter" )#( __name__ ) @@ -58,7 +62,7 @@ dir_request = urllib2.Request( url, headers = { 'Content-Type': 'application/json', 'Accept': 'application/json' } ) dir_request.get_method = lambda: 'GET' try: - dir_dict = simplejson.loads( url_opener.open( dir_request ).read() ) + dir_dict = json.loads( url_opener.open( dir_request ).read() ) except urllib2.HTTPError, e: #print "e", e, url #punting, assuming lack of permissions at this low of a level... continue @@ -81,16 +85,16 @@ if dir_slice in ( '', '/', None ): continue url = '/'.join( ( directory_dict['url'], urllib.quote( dir_slice.replace( '/', '_' ), safe='' ) ) ) - new_dir_request = urllib2.Request( url, headers = { 'Content-Type': 'application/json', 'Accept': 'application/json' }, data = simplejson.dumps( payload ) ) + new_dir_request = urllib2.Request( url, headers = { 'Content-Type': 'application/json', 'Accept': 'application/json' }, data = json.dumps( payload ) ) new_dir_request.get_method = lambda: 'PUT' - directory_dict = simplejson.loads( url_opener.open( new_dir_request ).read() ) + directory_dict = json.loads( url_opener.open( new_dir_request ).read() ) return directory_dict def get_genome_space_launch_apps( atm_url, url_opener, file_url, file_type ): gs_request = urllib2.Request( "%s/%s/webtool/descriptor" % ( atm_url, GENOMESPACE_API_VERSION_STRING ) ) gs_request.get_method = lambda: 'GET' opened_gs_request = url_opener.open( gs_request ) - webtool_descriptors = simplejson.loads( opened_gs_request.read() ) + webtool_descriptors = json.loads( opened_gs_request.read() ) webtools = [] for webtool in webtool_descriptors: webtool_name = webtool.get( 'name' ) @@ -125,7 +129,7 @@ except urllib2.HTTPError, e: log.debug( 'GenomeSpace export tool failed reading a directory "%s": %s' % ( url, e ) ) return #bad url, go to next - cur_directory = simplejson.loads( cur_directory ) + cur_directory = json.loads( cur_directory ) directory = cur_directory.get( 'directory', {} ) contents = cur_directory.get( 'contents', [] ) if directory.get( 'isDirectory', False ): diff -r 0c5e20558f6bb8d690dd9c9d25c172f08e837b1b -r 995ca770764013bcf0c61092de8436c9278357f7 tools/genomespace/genomespace_file_browser.py --- a/tools/genomespace/genomespace_file_browser.py +++ b/tools/genomespace/genomespace_file_browser.py @@ -1,12 +1,13 @@ #Dan Blankenberg -import optparse, os, urllib, urllib2, urlparse, cookielib +import cookielib +import json +import optparse +import os +import urllib +import urllib2 +import urlparse -from galaxy import eggs -import pkg_resources - -pkg_resources.require( "simplejson" ) -import simplejson GENOMESPACE_API_VERSION_STRING = "v1.0" GENOMESPACE_SERVER_URL_PROPERTIES = "https://dm.genomespace.org/config/%s/serverurl.properties" % ( GENOMESPACE_API_VERSION_STRING ) @@ -87,12 +88,12 @@ gs_request = urllib2.Request( "%s/%s/dataformat/list" % ( dm_site, GENOMESPACE_API_VERSION_STRING ) ) gs_request.get_method = lambda: 'GET' opened_gs_request = url_opener.open( gs_request ) - genomespace_formats = simplejson.loads( opened_gs_request.read() ) + genomespace_formats = json.loads( opened_gs_request.read() ) for format in genomespace_formats: GENOMESPACE_FORMAT_IDENTIFIER_TO_GENOMESPACE_EXT[ format['url'] ] = format['name'] def download_from_genomespace_file_browser( json_parameter_file, genomespace_site ): - json_params = simplejson.loads( open( json_parameter_file, 'r' ).read() ) + json_params = json.loads( open( json_parameter_file, 'r' ).read() ) datasource_params = json_params.get( 'param_dict' ) username = datasource_params.get( "gs-username", None ) token = datasource_params.get( "gs-token", None ) @@ -150,14 +151,14 @@ filename = "-%s" % filename used_filenames.append( filename ) output_filename = os.path.join( datasource_params['__new_file_path__'], 'primary_%i_%s_visible_%s' % ( hda_id, filename, galaxy_ext ) ) - metadata_parameter_file.write( "%s\n" % simplejson.dumps( dict( type = 'new_primary_dataset', + metadata_parameter_file.write( "%s\n" % json.dumps( dict( type = 'new_primary_dataset', base_dataset_id = dataset_id, ext = galaxy_ext, filename = output_filename, name = "GenomeSpace import on %s" % ( original_filename ) ) ) ) else: if dataset_id is not None: - metadata_parameter_file.write( "%s\n" % simplejson.dumps( dict( type = 'dataset', + metadata_parameter_file.write( "%s\n" % json.dumps( dict( type = 'dataset', dataset_id = dataset_id, ext = galaxy_ext, name = "GenomeSpace import on %s" % ( filename ) ) ) ) diff -r 0c5e20558f6bb8d690dd9c9d25c172f08e837b1b -r 995ca770764013bcf0c61092de8436c9278357f7 tools/genomespace/genomespace_importer.py --- a/tools/genomespace/genomespace_importer.py +++ b/tools/genomespace/genomespace_importer.py @@ -1,12 +1,14 @@ #Dan Blankenberg -import optparse, os, urllib2, urllib, cookielib, urlparse, tempfile, shutil - -from galaxy import eggs -import pkg_resources - -pkg_resources.require( "simplejson" ) -import simplejson +import cookielib +import json +import optparse +import os +import shutil +import tempfile +import urllib +import urllib2 +import urlparse import galaxy.model # need to import model before sniff to resolve a circular import dependency from galaxy.datatypes import sniff @@ -91,12 +93,12 @@ gs_request = urllib2.Request( "%s/%s/dataformat/list" % ( dm_site, GENOMESPACE_API_VERSION_STRING ) ) gs_request.get_method = lambda: 'GET' opened_gs_request = url_opener.open( gs_request ) - genomespace_formats = simplejson.loads( opened_gs_request.read() ) + genomespace_formats = json.loads( opened_gs_request.read() ) for format in genomespace_formats: GENOMESPACE_FORMAT_IDENTIFIER_TO_GENOMESPACE_EXT[ format['url'] ] = format['name'] def download_from_genomespace_importer( username, token, json_parameter_file, genomespace_site ): - json_params = simplejson.loads( open( json_parameter_file, 'r' ).read() ) + json_params = json.loads( open( json_parameter_file, 'r' ).read() ) datasource_params = json_params.get( 'param_dict' ) assert None not in [ username, token ], "Missing GenomeSpace username or token." output_filename = datasource_params.get( "output_file1", None ) @@ -152,7 +154,7 @@ metadata_request = urllib2.Request( "%s/%s/filemetadata/%s" % ( genomespace_site_dict['dmServer'], GENOMESPACE_API_VERSION_STRING, download_file_path ) ) metadata_request.get_method = lambda: 'GET' metadata_url = url_opener.open( metadata_request ) - file_metadata_dict = simplejson.loads( metadata_url.read() ) + file_metadata_dict = json.loads( metadata_url.read() ) metadata_url.close() file_type = file_metadata_dict.get( 'dataFormat', None ) if file_type and file_type.get( 'url' ): @@ -176,7 +178,7 @@ #save json info for single primary dataset if dataset_id is not None: - metadata_parameter_file.write( "%s\n" % simplejson.dumps( dict( type = 'dataset', + metadata_parameter_file.write( "%s\n" % json.dumps( dict( type = 'dataset', dataset_id = dataset_id, ext = file_type, name = "GenomeSpace importer on %s" % ( filename ) ) ) ) @@ -189,7 +191,7 @@ used_filenames.append( filename ) target_output_filename = os.path.join( datasource_params['__new_file_path__'], 'primary_%i_%s_visible_%s' % ( hda_id, filename, file_type ) ) shutil.move( output_filename, target_output_filename ) - metadata_parameter_file.write( "%s\n" % simplejson.dumps( dict( type = 'new_primary_dataset', + metadata_parameter_file.write( "%s\n" % json.dumps( dict( type = 'new_primary_dataset', base_dataset_id = base_dataset_id, ext = file_type, filename = target_output_filename, diff -r 0c5e20558f6bb8d690dd9c9d25c172f08e837b1b -r 995ca770764013bcf0c61092de8436c9278357f7 tools/new_operations/column_join.py --- a/tools/new_operations/column_join.py +++ b/tools/new_operations/column_join.py @@ -13,19 +13,16 @@ other_inputs: the other input files to join """ -import optparse, os, re, struct, sys, tempfile +import json +import optparse +import os +import re +import struct +import sys +import tempfile -try: - simple_json_exception = None - from galaxy import eggs - from galaxy.util.bunch import Bunch - from galaxy.util import stringify_dictionary_keys - import pkg_resources - pkg_resources.require("simplejson") - import simplejson -except Exception, e: - simplejson_exception = e - simplejson = None +from galaxy.util.bunch import Bunch +from galaxy.util import stringify_dictionary_keys def stop_err( msg ): sys.stderr.write( msg ) @@ -162,11 +159,9 @@ fill_options = None if options.fill_options_file != 'None' and options.fill_options_file is not None: try: - if simplejson is None: - raise simplejson_exception - fill_options = Bunch( **stringify_dictionary_keys( simplejson.load( open( options.fill_options_file ) ) ) ) + fill_options = Bunch( **stringify_dictionary_keys( json.load( open( options.fill_options_file ) ) ) ) except Exception, e: - print 'Warning: Ignoring fill options due to simplejson error (%s).' % e + print 'Warning: Ignoring fill options due to json error (%s).' % e if fill_options is None: fill_options = Bunch() if 'file1_columns' not in fill_options: diff -r 0c5e20558f6bb8d690dd9c9d25c172f08e837b1b -r 995ca770764013bcf0c61092de8436c9278357f7 tools/new_operations/column_join.xml --- a/tools/new_operations/column_join.xml +++ b/tools/new_operations/column_join.xml @@ -49,7 +49,7 @@ </inputs><configfiles><configfile name="fill_options_file"><% -import simplejson +import json %> #set $__fill_options = {} #if $fill_empty_columns['fill_empty_columns_switch'] == 'fill_empty': @@ -65,7 +65,7 @@ #end for #end if #end if -${simplejson.dumps( __fill_options )} +${json.dumps( __fill_options )} </configfile></configfiles><outputs> diff -r 0c5e20558f6bb8d690dd9c9d25c172f08e837b1b -r 995ca770764013bcf0c61092de8436c9278357f7 tools/peak_calling/macs_wrapper.py --- a/tools/peak_calling/macs_wrapper.py +++ b/tools/peak_calling/macs_wrapper.py @@ -1,8 +1,12 @@ -import sys, subprocess, tempfile, shutil, glob, os, os.path, gzip -from galaxy import eggs -import pkg_resources -pkg_resources.require( "simplejson" ) -import simplejson +import glob +import gzip +import json +import os +import os.path +import shutil +import subprocess +import sys +import tempfile CHUNK_SIZE = 1024 @@ -42,7 +46,7 @@ out.close() def main(): - options = simplejson.load( open( sys.argv[1] ) ) + options = json.load( open( sys.argv[1] ) ) output_bed = sys.argv[2] output_extra_html = sys.argv[3] output_extra_path = sys.argv[4] diff -r 0c5e20558f6bb8d690dd9c9d25c172f08e837b1b -r 995ca770764013bcf0c61092de8436c9278357f7 tools/peak_calling/macs_wrapper.xml --- a/tools/peak_calling/macs_wrapper.xml +++ b/tools/peak_calling/macs_wrapper.xml @@ -93,7 +93,7 @@ </outputs><configfiles><configfile name="options_file"><% -import simplejson +import json %> #set $__options = { 'experiment_name':str( $experiment_name ), 'gsize':int( float( str( $gsize ) ) ), 'tsize':str( $tsize ), 'bw':str( $bw ), 'pvalue':str( $pvalue ), 'mfold':str( $mfold ), 'nolambda':str( $nolambda ), 'lambdaset': str( $lambdaset ), 'futurefdr':str( $futurefdr ) } #if str( $xls_to_interval ) == 'create': @@ -135,7 +135,7 @@ #if $diag_type['diag_type_selector'] == 'diag': #set $__options['diag'] = { 'fe-min':str( $diag_type['fe-min'] ), 'fe-max':str( $diag_type['fe-max'] ), 'fe-step':str( $diag_type['fe-step'] ) } #end if -${ simplejson.dumps( __options ) } +${ json.dumps( __options ) } </configfile></configfiles><tests> https://bitbucket.org/galaxy/galaxy-central/commits/2483cc9e597d/ Changeset: 2483cc9e597d User: dannon Date: 2014-01-10 13:40:02 Summary: Merge. Affected #: 172 files diff -r 995ca770764013bcf0c61092de8436c9278357f7 -r 2483cc9e597d37ab75c5bb46d467622f33b48bc7 job_conf.xml.sample_advanced --- a/job_conf.xml.sample_advanced +++ b/job_conf.xml.sample_advanced @@ -6,7 +6,19 @@ --><plugin id="local" type="runner" load="galaxy.jobs.runners.local:LocalJobRunner"/><plugin id="pbs" type="runner" load="galaxy.jobs.runners.pbs:PBSJobRunner" workers="2"/> - <plugin id="drmaa" type="runner" load="galaxy.jobs.runners.drmaa:DRMAAJobRunner"/> + <plugin id="drmaa" type="runner" load="galaxy.jobs.runners.drmaa:DRMAAJobRunner"> +  + <param id="invalidjobexception_state">ok</param> + <param id="invalidjobexception_retries">0</param> + <param id="internalexception_state">ok</param> + <param id="internalexception_retries">0</param> + </plugin> + <plugin id="sge" type="runner" load="galaxy.jobs.runners.drmaa:DRMAAJobRunner"> +  + <param id="drmaa_library_path">/sge/lib/libdrmaa.so</param> + </plugin><plugin id="lwr" type="runner" load="galaxy.jobs.runners.lwr:LwrJobRunner"> @@ -14,6 +26,7 @@ </plugin><plugin id="cli" type="runner" load="galaxy.jobs.runners.cli:ShellJobRunner" /><plugin id="condor" type="runner" load="galaxy.jobs.runners.condor:CondorJobRunner" /> + <plugin id="slurm" type="runner" load="galaxy.jobs.runners.slurm:SlurmJobRunner" /></plugins><handlers default="handlers"><handler id="handler0" tags="handlers"/><handler id="handler1" tags="handlers"/> +  + <handler id="sge_handler"> + <plugin id="sge"/> + </handler><handler id="special_handler0" tags="special_handlers"/><handler id="special_handler1" tags="special_handlers"/><handler id="trackster_handler"/> diff -r 995ca770764013bcf0c61092de8436c9278357f7 -r 2483cc9e597d37ab75c5bb46d467622f33b48bc7 lib/galaxy/jobs/__init__.py --- a/lib/galaxy/jobs/__init__.py +++ b/lib/galaxy/jobs/__init__.py @@ -86,6 +86,7 @@ self.app = app self.runner_plugins = [] self.handlers = {} + self.handler_runner_plugins = {} self.default_handler_id = None self.destinations = {} self.destination_tags = {} @@ -138,6 +139,10 @@ else: log.debug("Read definition for handler '%s'" % id) self.handlers[id] = (id,) + for plugin in handler.findall('plugin'): + if id not in self.handler_runner_plugins: + self.handler_runner_plugins[id] = [] + self.handler_runner_plugins[id].append( plugin.get('id') ) if handler.get('tags', None) is not None: for tag in [ x.strip() for x in handler.get('tags').split(',') ]: if tag in self.handlers: @@ -420,13 +425,19 @@ """ return self.destinations.get(id_or_tag, None) - def get_job_runner_plugins(self): + def get_job_runner_plugins(self, handler_id): """Load all configured job runner plugins :returns: list of job runner plugins """ rval = {} - for runner in self.runner_plugins: + if handler_id in self.handler_runner_plugins: + plugins_to_load = [ rp for rp in self.runner_plugins if rp['id'] in self.handler_runner_plugins[handler_id] ] + log.info( "Handler '%s' will load specified runner plugins: %s", handler_id, ', '.join( [ rp['id'] for rp in plugins_to_load ] ) ) + else: + plugins_to_load = self.runner_plugins + log.info( "Handler '%s' will load all configured runner plugins", handler_id ) + for runner in plugins_to_load: class_names = [] module = None id = runner['id'] @@ -477,7 +488,7 @@ try: rval[id] = runner_class( self.app, runner[ 'workers' ], **runner.get( 'kwds', {} ) ) except TypeError: - log.warning( "Job runner '%s:%s' has not been converted to a new-style runner" % ( module_name, class_name ) ) + log.exception( "Job runner '%s:%s' has not been converted to a new-style runner or encountered TypeError on load" % ( module_name, class_name ) ) rval[id] = runner_class( self.app ) log.debug( "Loaded job runner '%s:%s' as '%s'" % ( module_name, class_name, id ) ) return rval diff -r 995ca770764013bcf0c61092de8436c9278357f7 -r 2483cc9e597d37ab75c5bb46d467622f33b48bc7 lib/galaxy/jobs/handler.py --- a/lib/galaxy/jobs/handler.py +++ b/lib/galaxy/jobs/handler.py @@ -565,7 +565,7 @@ class DefaultJobDispatcher( object ): def __init__( self, app ): self.app = app - self.job_runners = self.app.job_config.get_job_runner_plugins() + self.job_runners = self.app.job_config.get_job_runner_plugins( self.app.config.server_name ) # Once plugins are loaded, all job destinations that were created from # URLs can have their URL params converted to the destination's param # dict by the plugin. diff -r 995ca770764013bcf0c61092de8436c9278357f7 -r 2483cc9e597d37ab75c5bb46d467622f33b48bc7 lib/galaxy/jobs/runners/__init__.py --- a/lib/galaxy/jobs/runners/__init__.py +++ b/lib/galaxy/jobs/runners/__init__.py @@ -22,13 +22,39 @@ STOP_SIGNAL = object() + +class RunnerParams( object ): + + def __init__( self, specs = None, params = None ): + self.specs = specs or dict() + self.params = params or dict() + for name, value in self.params.items(): + assert name in self.specs, 'Invalid job runner parameter for this plugin: %s' % name + if 'map' in self.specs[ name ]: + try: + self.params[ name ] = self.specs[ name ][ 'map' ]( value ) + except Exception, e: + raise Exception( 'Job runner parameter "%s" value "%s" could not be converted to the correct type: %s' % ( name, value, e ) ) + if 'valid' in self.specs[ name ]: + assert self.specs[ name ][ 'valid' ]( value ), 'Job runner parameter %s failed validation' % name + + def __getattr__( self, name ): + return self.params.get( name, self.specs[ name ][ 'default' ] ) + + class BaseJobRunner( object ): - def __init__( self, app, nworkers ): + def __init__( self, app, nworkers, **kwargs ): """Start the job runner """ self.app = app self.sa_session = app.model.context self.nworkers = nworkers + runner_param_specs = dict( recheck_missing_job_retries = dict( map = int, valid = lambda x: x >= 0, default = 0 ) ) + if 'runner_param_specs' in kwargs: + runner_param_specs.update( kwargs.pop( 'runner_param_specs' ) ) + if kwargs: + log.debug( 'Loading %s with params: %s', self.runner_name, kwargs ) + self.runner_params = RunnerParams( specs = runner_param_specs, params = kwargs ) def _init_worker_threads(self): """Start ``nworkers`` worker threads. @@ -115,7 +141,7 @@ job_wrapper.cleanup() return False elif job_state != model.Job.states.QUEUED: - log.info( "(%d) Job is in state %s, skipping execution" % ( job_id, job_state ) ) + log.info( "(%s) Job is in state %s, skipping execution" % ( job_id, job_state ) ) # cleanup may not be safe in all states return False @@ -226,6 +252,10 @@ options.update(**kwds) return job_script(**options) + def _complete_terminal_job( self, ajs, **kwargs ): + if ajs.job_wrapper.get_state() != model.Job.states.DELETED: + self.work_queue.put( ( self.finish_job, ajs ) ) + class AsynchronousJobState( object ): """ @@ -287,8 +317,8 @@ to the correct methods (queue, finish, cleanup) at appropriate times.. """ - def __init__( self, app, nworkers ): - super( AsynchronousJobRunner, self ).__init__( app, nworkers ) + def __init__( self, app, nworkers, **kwargs ): + super( AsynchronousJobRunner, self ).__init__( app, nworkers, **kwargs ) # 'watched' and 'queue' are both used to keep track of jobs to watch. # 'queue' is used to add new watched jobs, and can be called from # any thread (usually by the 'queue_job' method). 'watched' must only diff -r 995ca770764013bcf0c61092de8436c9278357f7 -r 2483cc9e597d37ab75c5bb46d467622f33b48bc7 lib/galaxy/jobs/runners/drmaa.py --- a/lib/galaxy/jobs/runners/drmaa.py +++ b/lib/galaxy/jobs/runners/drmaa.py @@ -16,27 +16,12 @@ from galaxy.jobs.runners import AsynchronousJobState, AsynchronousJobRunner eggs.require( "drmaa" ) -# We foolishly named this file the same as the name exported by the drmaa -# library... 'import drmaa' imports itself. -drmaa = __import__( "drmaa" ) log = logging.getLogger( __name__ ) __all__ = [ 'DRMAAJobRunner' ] -drmaa_state = { - drmaa.JobState.UNDETERMINED: 'process status cannot be determined', - drmaa.JobState.QUEUED_ACTIVE: 'job is queued and active', - drmaa.JobState.SYSTEM_ON_HOLD: 'job is queued and in system hold', - drmaa.JobState.USER_ON_HOLD: 'job is queued and in user hold', - drmaa.JobState.USER_SYSTEM_ON_HOLD: 'job is queued and in user and system hold', - drmaa.JobState.RUNNING: 'job is running', - drmaa.JobState.SYSTEM_SUSPENDED: 'job is system suspended', - drmaa.JobState.USER_SUSPENDED: 'job is user suspended', - drmaa.JobState.DONE: 'job finished normally', - drmaa.JobState.FAILED: 'job finished, but failed', -} - +drmaa = None DRMAA_jobTemplate_attributes = [ 'args', 'remoteCommand', 'outputPath', 'errorPath', 'nativeSpecification', 'jobName', 'email', 'project' ] @@ -48,8 +33,50 @@ """ runner_name = "DRMAARunner" - def __init__( self, app, nworkers ): + def __init__( self, app, nworkers, **kwargs ): """Start the job runner""" + + global drmaa + + runner_param_specs = dict( + drmaa_library_path = dict( map = str, default = os.environ.get( 'DRMAA_LIBRARY_PATH', None ) ), + invalidjobexception_state = dict( map = str, valid = lambda x: x in ( model.Job.states.OK, model.Job.states.ERROR ), default = model.Job.states.OK ), + invalidjobexception_retries = dict( map = int, valid = lambda x: int >= 0, default = 0 ), + internalexception_state = dict( map = str, valid = lambda x: x in ( model.Job.states.OK, model.Job.states.ERROR ), default = model.Job.states.OK ), + internalexception_retries = dict( map = int, valid = lambda x: int >= 0, default = 0 ) ) + + if 'runner_param_specs' not in kwargs: + kwargs[ 'runner_param_specs' ] = dict() + kwargs[ 'runner_param_specs' ].update( runner_param_specs ) + + super( DRMAAJobRunner, self ).__init__( app, nworkers, **kwargs ) + + # This allows multiple drmaa runners (although only one per handler) in the same job config file + if 'drmaa_library_path' in kwargs: + log.info( 'Overriding DRMAA_LIBRARY_PATH due to runner plugin parameter: %s', self.runner_params.drmaa_library_path ) + os.environ['DRMAA_LIBRARY_PATH'] = self.runner_params.drmaa_library_path + + # We foolishly named this file the same as the name exported by the drmaa + # library... 'import drmaa' imports itself. + drmaa = __import__( "drmaa" ) + + # Subclasses may need access to state constants + self.drmaa_job_states = drmaa.JobState + + # Descriptive state strings pulled from the drmaa lib itself + self.drmaa_job_state_strings = { + drmaa.JobState.UNDETERMINED: 'process status cannot be determined', + drmaa.JobState.QUEUED_ACTIVE: 'job is queued and active', + drmaa.JobState.SYSTEM_ON_HOLD: 'job is queued and in system hold', + drmaa.JobState.USER_ON_HOLD: 'job is queued and in user hold', + drmaa.JobState.USER_SYSTEM_ON_HOLD: 'job is queued and in user and system hold', + drmaa.JobState.RUNNING: 'job is running', + drmaa.JobState.SYSTEM_SUSPENDED: 'job is system suspended', + drmaa.JobState.USER_SUSPENDED: 'job is user suspended', + drmaa.JobState.DONE: 'job finished normally', + drmaa.JobState.FAILED: 'job finished, but failed', + } + self.ds = drmaa.Session() self.ds.initialize() @@ -58,7 +85,6 @@ self.external_killJob_script = app.config.drmaa_external_killjob_script self.userid = None - super( DRMAAJobRunner, self ).__init__( app, nworkers ) self._init_monitor_thread() self._init_worker_threads() @@ -137,8 +163,10 @@ job_wrapper.cleanup() return - log.debug( "(%s) submitting file %s" % ( galaxy_id_tag, ajs.job_file ) ) - log.debug( "(%s) command is: %s" % ( galaxy_id_tag, command_line ) ) + log.debug( "(%s) submitting file %s", galaxy_id_tag, ajs.job_file ) + log.debug( "(%s) command is: %s", galaxy_id_tag, command_line ) + if native_spec: + log.debug( "(%s) native specification is: %s", galaxy_id_tag, native_spec ) # runJob will raise if there's a submit problem if self.external_runJob_script is None: @@ -175,6 +203,20 @@ # Add to our 'queue' of jobs to monitor self.monitor_queue.put( ajs ) + def _complete_terminal_job( self, ajs, drmaa_state, **kwargs ): + """ + Handle a job upon its termination in the DRM. This method is meant to + be overridden by subclasses to improve post-mortem and reporting of + failures. + """ + if drmaa_state == drmaa.JobState.FAILED: + if ajs.job_wrapper.get_state() != model.Job.states.DELETED: + ajs.stop_job = False + ajs.fail_message = "The cluster DRM system terminated this job" + self.work_queue.put( ( self.fail_job, ajs ) ) + elif drmaa_state == drmaa.JobState.DONE: + super( DRMAAJobRunner, self )._complete_terminal_job( ajs ) + def check_watched_items( self ): """ Called by the monitor thread to look at each watched job and deal @@ -188,16 +230,27 @@ try: assert external_job_id not in ( None, 'None' ), '(%s/%s) Invalid job id' % ( galaxy_id_tag, external_job_id ) state = self.ds.jobStatus( external_job_id ) - # InternalException was reported to be necessary on some DRMs, but - # this could cause failures to be detected as completion! Please - # report if you experience problems with this. - except ( drmaa.InvalidJobException, drmaa.InternalException ), e: - # we should only get here if an orphaned job was put into the queue at app startup - log.info( "(%s/%s) job left DRM queue with following message: %s" % ( galaxy_id_tag, external_job_id, e ) ) - self.work_queue.put( ( self.finish_job, ajs ) ) + except ( drmaa.InternalException, drmaa.InvalidJobException ), e: + ecn = e.__class__.__name__ + retry_param = ecn.lower() + '_retries' + state_param = ecn.lower() + '_state' + retries = getattr( ajs, retry_param, 0 ) + if self.runner_params[ retry_param ] > 0: + if retries < self.runner_params[ retry_param ]: + # will retry check on next iteration + setattr( ajs, retry_param, retries + 1 ) + continue + if self.runner_params[ state_param ] == model.Job.states.OK: + log.info( "(%s/%s) job left DRM queue with following message: %s", galaxy_id_tag, external_job_id, e ) + self.work_queue.put( ( self.finish_job, ajs ) ) + elif self.runner_params[ state_param ] == model.Job.states.ERROR: + log.info( "(%s/%s) job check resulted in %s after %s tries: %s", galaxy_id_tag, external_job_id, ecn, retries, e ) + self.work_queue.put( ( self.fail_job, ajs ) ) + else: + raise Exception( "%s is set to an invalid value (%s), this should not be possible. See galaxy.jobs.drmaa.__init__()", state_param, self.runner_params[ state_param ] ) continue except drmaa.DrmCommunicationException, e: - log.warning( "(%s/%s) unable to communicate with DRM: %s" % ( galaxy_id_tag, external_job_id, e )) + log.warning( "(%s/%s) unable to communicate with DRM: %s", galaxy_id_tag, external_job_id, e ) new_watched.append( ajs ) continue except Exception, e: @@ -208,19 +261,12 @@ self.work_queue.put( ( self.fail_job, ajs ) ) continue if state != old_state: - log.debug( "(%s/%s) state change: %s" % ( galaxy_id_tag, external_job_id, drmaa_state[state] ) ) + log.debug( "(%s/%s) state change: %s" % ( galaxy_id_tag, external_job_id, self.drmaa_job_state_strings[state] ) ) if state == drmaa.JobState.RUNNING and not ajs.running: ajs.running = True ajs.job_wrapper.change_state( model.Job.states.RUNNING ) - if state == drmaa.JobState.FAILED: - if ajs.job_wrapper.get_state() != model.Job.states.DELETED: - ajs.stop_job = False - ajs.fail_message = "The cluster DRM system terminated this job" - self.work_queue.put( ( self.fail_job, ajs ) ) - continue - if state == drmaa.JobState.DONE: - if ajs.job_wrapper.get_state() != model.Job.states.DELETED: - self.work_queue.put( ( self.finish_job, ajs ) ) + if state in ( drmaa.JobState.FAILED, drmaa.JobState.DONE ): + self._complete_terminal_job( ajs, drmaa_state = state ) continue ajs.old_state = state new_watched.append( ajs ) diff -r 995ca770764013bcf0c61092de8436c9278357f7 -r 2483cc9e597d37ab75c5bb46d467622f33b48bc7 lib/galaxy/jobs/runners/slurm.py --- /dev/null +++ b/lib/galaxy/jobs/runners/slurm.py @@ -0,0 +1,57 @@ +""" +SLURM job control via the DRMAA API. +""" + +import time +import logging +import subprocess + +from galaxy import model +from galaxy.jobs.runners.drmaa import DRMAAJobRunner + +log = logging.getLogger( __name__ ) + +__all__ = [ 'SlurmJobRunner' ] + + +class SlurmJobRunner( DRMAAJobRunner ): + runner_name = "SlurmRunner" + + def _complete_terminal_job( self, ajs, drmaa_state, **kwargs ): + def __get_jobinfo(): + scontrol_out = subprocess.check_output( ( 'scontrol', '-o', 'show', 'job', ajs.job_id ) ) + return dict( [ out_param.split( '=', 1 ) for out_param in scontrol_out.split() ] ) + if drmaa_state == self.drmaa_job_states.FAILED: + try: + job_info = __get_jobinfo() + sleep = 1 + while job_info['JobState'] == 'COMPLETING': + log.debug( '(%s/%s) Waiting %s seconds for failed job to exit COMPLETING state for post-mortem', ajs.job_wrapper.get_id_tag(), ajs.job_id, sleep ) + time.sleep( sleep ) + sleep *= 2 + if sleep > 64: + ajs.fail_message = "This job failed and the system timed out while trying to determine the cause of the failure." + break + job_info = __get_jobinfo() + if job_info['JobState'] == 'TIMEOUT': + ajs.fail_message = "This job was terminated because it ran longer than the maximum allowed job run time." + elif job_info['JobState'] == 'NODE_FAIL': + log.warning( '(%s/%s) Job failed due to node failure, attempting resubmission', ajs.job_wrapper.get_id_tag(), ajs.job_id ) + ajs.job_wrapper.change_state( model.Job.states.QUEUED, info = 'Job was resubmitted due to node failure' ) + try: + self.queue_job( ajs.job_wrapper ) + return + except: + ajs.fail_message = "This job failed due to a cluster node failure, and an attempt to resubmit the job failed." + elif job_info['JobState'] == 'CANCELLED': + ajs.fail_message = "This job failed because it was cancelled by an administrator." + else: + ajs.fail_message = "This job failed for reasons that could not be determined." + ajs.fail_message += '\nPlease click the bug icon to report this problem if you need help.' + ajs.stop_job = False + self.work_queue.put( ( self.fail_job, ajs ) ) + except Exception, e: + log.exception( '(%s/%s) Unable to inspect failed slurm job using scontrol, job will be unconditionally failed: %s', ajs.job_wrapper.get_id_tag(), ajs.job_id, e ) + super( SlurmJobRunner, self )._complete_terminal_job( ajs, drmaa_state = drmaa_state ) + elif drmaa_state == self.drmaa_job_states.DONE: + super( SlurmJobRunner, self )._complete_terminal_job( ajs, drmaa_state = drmaa_state ) diff -r 995ca770764013bcf0c61092de8436c9278357f7 -r 2483cc9e597d37ab75c5bb46d467622f33b48bc7 lib/galaxy/model/__init__.py --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -18,6 +18,7 @@ import socket import time from string import Template +from itertools import ifilter import galaxy.datatypes import galaxy.datatypes.registry @@ -42,6 +43,11 @@ # Default Value Required for unit tests datatypes_registry.load_datatypes() +# When constructing filters with in for a fixed set of ids, maximum +# number of items to place in the IN statement. Different databases +# are going to have different limits so it is likely best to not let +# this be unlimited - filter in Python if over this limit. +MAX_IN_FILTER_LENGTH = 100 class NoConverterException(Exception): def __init__(self, value): @@ -892,6 +898,33 @@ rval = galaxy.datatypes.data.nice_size( rval ) return rval + def contents_iter( self, **kwds ): + """ + Fetch filtered list of contents of history. + """ + python_filter = None + db_session = object_session( self ) + assert db_session != None + query = db_session.query( HistoryDatasetAssociation ).filter( HistoryDatasetAssociation.table.c.history_id == self.id ) + query = query.order_by( HistoryDatasetAssociation.table.c.hid.asc() ) + deleted = galaxy.util.string_as_bool_or_none( kwds.get( 'deleted', None ) ) + if deleted is not None: + query = query.filter( HistoryDatasetAssociation.deleted == bool( kwds['deleted'] ) ) + visible = galaxy.util.string_as_bool_or_none( kwds.get( 'visible', None ) ) + if visible is not None: + query = query.filter( HistoryDatasetAssociation.visible == bool( kwds['visible'] ) ) + if 'ids' in kwds: + ids = kwds['ids'] + max_in_filter_length = kwds.get('max_in_filter_length', MAX_IN_FILTER_LENGTH) + if len(ids) < max_in_filter_length: + query = query.filter( HistoryDatasetAssociation.id.in_(ids) ) + else: + python_filter = lambda hda: hda.id in ids + if python_filter: + return ifilter(python_filter, query) + else: + return query + def copy_tags_from(self,target_user,source_history): for src_shta in source_history.tags: new_shta = src_shta.copy() diff -r 995ca770764013bcf0c61092de8436c9278357f7 -r 2483cc9e597d37ab75c5bb46d467622f33b48bc7 lib/galaxy/webapps/galaxy/api/history_contents.py --- a/lib/galaxy/webapps/galaxy/api/history_contents.py +++ b/lib/galaxy/webapps/galaxy/api/history_contents.py @@ -51,47 +51,28 @@ else: history = self.get_history( trans, history_id, check_ownership=True, check_accessible=True ) - # if ids, return _FULL_ data (as show) for each id passed + contents_kwds = {} if ids: - ids = ids.split( ',' ) - for index, hda in enumerate( history.datasets ): - encoded_hda_id = trans.security.encode_id( hda.id ) - if encoded_hda_id in ids: - #TODO: share code with show - rval.append( self._detailed_hda_dict( trans, hda ) ) - - # if no ids passed, return a _SUMMARY_ of _all_ datasets in the history + ids = map( lambda id: trans.security.decode_id( id ), ids.split( ',' ) ) + contents_kwds[ 'ids' ] = ids + # If explicit ids given, always used detailed result. + details = 'all' else: + contents_kwds[ 'deleted' ] = kwd.get( 'deleted', None ) + contents_kwds[ 'visible' ] = kwd.get( 'visible', None ) # details param allows a mixed set of summary and detailed hdas #TODO: this is getting convoluted due to backwards compat details = kwd.get( 'details', None ) or [] if details and details != 'all': details = util.listify( details ) - # by default return all datasets - even if deleted or hidden (defaulting the next switches to None) - # if specified return those datasets that match the setting - # backwards compat - return_deleted = util.string_as_bool_or_none( kwd.get( 'deleted', None ) ) - return_visible = util.string_as_bool_or_none( kwd.get( 'visible', None ) ) - - for hda in history.datasets: - # if either return_ setting has been requested (!= None), skip hdas that don't match the request - if return_deleted is not None: - if( ( return_deleted and not hda.deleted ) - or ( not return_deleted and hda.deleted ) ): - continue - if return_visible is not None: - if( ( return_visible and not hda.visible ) - or ( not return_visible and hda.visible ) ): - continue - - encoded_hda_id = trans.security.encode_id( hda.id ) - if( ( encoded_hda_id in details ) - or ( details == 'all' ) ): - rval.append( self._detailed_hda_dict( trans, hda ) ) - else: - rval.append( self._summary_hda_dict( trans, history_id, hda ) ) - + for hda in history.contents_iter( **contents_kwds ): + encoded_hda_id = trans.security.encode_id( hda.id ) + detailed = details == 'all' or ( encoded_hda_id in details ) + if detailed: + rval.append( self._detailed_hda_dict( trans, hda ) ) + else: + rval.append( self._summary_hda_dict( trans, history_id, hda ) ) except Exception, e: # for errors that are not specific to one hda (history lookup or summary list) rval = "Error in history API at listing contents: " + str( e ) diff -r 995ca770764013bcf0c61092de8436c9278357f7 -r 2483cc9e597d37ab75c5bb46d467622f33b48bc7 lib/galaxy/webapps/galaxy/api/tool_shed_repositories.py --- a/lib/galaxy/webapps/galaxy/api/tool_shed_repositories.py +++ b/lib/galaxy/webapps/galaxy/api/tool_shed_repositories.py @@ -19,11 +19,6 @@ log = logging.getLogger( __name__ ) -def default_tool_shed_repository_value_mapper( trans, tool_shed_repository ): - value_mapper={ 'id' : trans.security.encode_id( tool_shed_repository.id ), - 'error_message' : tool_shed_repository.error_message or '' } - return value_mapper - def get_message_for_no_shed_tool_config(): # This Galaxy instance is not configured with a shed-related tool panel configuration file. message = 'The tool_config_file setting in universe_wsgi.ini must include at least one shed tool configuration file name with a <toolbox> ' @@ -48,8 +43,8 @@ :param id: the encoded id of the ToolShedRepository object """ # Example URL: http://localhost:8763/api/tool_shed_repositories/f2db41e1fa331b3e/exported_… - # Since exported workflows are dictionaries with very few attributes that differentiate them from each other, we'll build the - # list based on the following dictionary of those few attributes. + # Since exported workflows are dictionaries with very few attributes that differentiate them from each + # other, we'll build the list based on the following dictionary of those few attributes. exported_workflows = [] repository = suc.get_tool_shed_repository_by_id( trans, id ) metadata = repository.metadata @@ -58,17 +53,23 @@ else: exported_workflow_tups = [] for index, exported_workflow_tup in enumerate( exported_workflow_tups ): - # The exported_workflow_tup looks like ( relative_path, exported_workflow_dict ), where the value of relative_path is the location - # on disk (relative to the root of the installed repository) where the exported_workflow_dict file (.ga file) is located. + # The exported_workflow_tup looks like ( relative_path, exported_workflow_dict ), where the value of + # relative_path is the location on disk (relative to the root of the installed repository) where the + # exported_workflow_dict file (.ga file) is located. exported_workflow_dict = exported_workflow_tup[ 1 ] annotation = exported_workflow_dict.get( 'annotation', '' ) format_version = exported_workflow_dict.get( 'format-version', '' ) workflow_name = exported_workflow_dict.get( 'name', '' ) - # Since we don't have an in-memory object with an id, we'll identify the exported workflow via it's location (i.e., index) in the list. + # Since we don't have an in-memory object with an id, we'll identify the exported workflow via it's + # location (i.e., index) in the list. display_dict = dict( index=index, annotation=annotation, format_version=format_version, workflow_name=workflow_name ) exported_workflows.append( display_dict ) return exported_workflows + def __get_value_mapper( self, trans ): + value_mapper = { 'id' : trans.security.encode_id } + return value_mapper + @web.expose_api def import_workflow( self, trans, payload, **kwd ): """ @@ -96,13 +97,11 @@ # Since we don't have an in-memory object with an id, we'll identify the exported workflow via it's location (i.e., index) in the list. exported_workflow = exported_workflows[ int( index ) ] workflow_name = exported_workflow[ 'workflow_name' ] - workflow, status, message = workflow_util.import_workflow( trans, repository, workflow_name ) + workflow, status, error_message = workflow_util.import_workflow( trans, repository, workflow_name ) if status == 'error': - log.error( message, exc_info=True ) - trans.response.status = 500 - return message - else: - return workflow.to_dict( view='element' ) + log.debug( error_message ) + return {} + return workflow.to_dict( view='element' ) @web.expose_api def import_workflows( self, trans, **kwd ): @@ -125,11 +124,9 @@ imported_workflow_dicts = [] for exported_workflow_dict in exported_workflows: workflow_name = exported_workflow_dict[ 'workflow_name' ] - workflow, status, message = workflow_util.import_workflow( trans, repository, workflow_name ) + workflow, status, error_message = workflow_util.import_workflow( trans, repository, workflow_name ) if status == 'error': - log.error( message, exc_info=True ) - trans.response.status = 500 - return message + log.debug( error_message ) else: imported_workflow_dicts.append( workflow.to_dict( view='element' ) ) return imported_workflow_dicts @@ -142,22 +139,15 @@ """ # Example URL: http://localhost:8763/api/tool_shed_repositories tool_shed_repository_dicts = [] - try: - query = trans.install_model.context.query( trans.app.install_model.ToolShedRepository ) \ - .order_by( trans.app.install_model.ToolShedRepository.table.c.name ) \ - .all() - for tool_shed_repository in query: - tool_shed_repository_dict = tool_shed_repository.to_dict( value_mapper=default_tool_shed_repository_value_mapper( trans, tool_shed_repository ) ) - tool_shed_repository_dict[ 'url' ] = web.url_for( controller='tool_shed_repositories', - action='show', - id=trans.security.encode_id( tool_shed_repository.id ) ) - tool_shed_repository_dicts.append( tool_shed_repository_dict ) - return tool_shed_repository_dicts - except Exception, e: - message = "Error in the tool_shed_repositories API in index: %s" % str( e ) - log.error( message, exc_info=True ) - trans.response.status = 500 - return message + for tool_shed_repository in trans.install_model.context.query( trans.app.install_model.ToolShedRepository ) \ + .order_by( trans.app.install_model.ToolShedRepository.table.c.name ): + tool_shed_repository_dict = \ + tool_shed_repository.to_dict( value_mapper=self.__get_value_mapper( trans ) ) + tool_shed_repository_dict[ 'url' ] = web.url_for( controller='tool_shed_repositories', + action='show', + id=trans.security.encode_id( tool_shed_repository.id ) ) + tool_shed_repository_dicts.append( tool_shed_repository_dict ) + return tool_shed_repository_dicts @web.expose_api def install_repository_revision( self, trans, payload, **kwd ): @@ -208,8 +198,7 @@ # Make sure this Galaxy instance is configured with a shed-related tool panel configuration file. if not suc.have_shed_tool_conf_for_install( trans ): message = get_message_for_no_shed_tool_config() - log.error( message, exc_info=True ) - trans.response.status = 500 + log.debug( message ) return dict( status='error', error=message ) # Make sure the current user's API key proves he is an admin user in this Galaxy instance. if not trans.user_is_admin(): @@ -225,18 +214,20 @@ except Exception, e: message = "Error attempting to retrieve installation information from tool shed %s for revision %s of repository %s owned by %s: %s" % \ ( str( tool_shed_url ), str( changeset_revision ), str( name ), str( owner ), str( e ) ) - log.error( message, exc_info=True ) - trans.response.status = 500 + log.debug( message ) return dict( status='error', error=message ) if raw_text: + # If successful, the response from get_repository_revision_install_info will be 3 + # dictionaries, a dictionary defining the Repository, a dictionary defining the + # Repository revision (RepositoryMetadata), and a dictionary including the additional + # information required to install the repository. items = json.from_json_string( raw_text ) repository_revision_dict = items[ 1 ] repo_info_dict = items[ 2 ] else: message = "Unable to retrieve installation information from tool shed %s for revision %s of repository %s owned by %s: %s" % \ ( str( tool_shed_url ), str( changeset_revision ), str( name ), str( owner ), str( e ) ) - log.error( message, exc_info=True ) - trans.response.status = 500 + log.debug( message ) return dict( status='error', error=message ) repo_info_dicts = [ repo_info_dict ] # Make sure the tool shed returned everything we need for installing the repository. @@ -345,7 +336,7 @@ tool_path, install_tool_dependencies, reinstalling=False ) - tool_shed_repository_dict = tool_shed_repository.as_dict( value_mapper=default_tool_shed_repository_value_mapper( trans, tool_shed_repository ) ) + tool_shed_repository_dict = tool_shed_repository.as_dict( value_mapper=self.__get_value_mapper( trans ) ) tool_shed_repository_dict[ 'url' ] = web.url_for( controller='tool_shed_repositories', action='show', id=trans.security.encode_id( tool_shed_repository.id ) ) @@ -394,8 +385,7 @@ if not suc.have_shed_tool_conf_for_install( trans ): # This Galaxy instance is not configured with a shed-related tool panel configuration file. message = get_message_for_no_shed_tool_config() - log.error( message, exc_info=True ) - trans.response.status = 500 + log.debug( message ) return dict( status='error', error=message ) if not trans.user_is_admin(): raise HTTPForbidden( detail='You are not authorized to install a tool shed repository into this Galaxy instance.' ) @@ -410,8 +400,7 @@ len( changeset_revisions ) != num_specified_repositories: message = 'Error in tool_shed_repositories API in install_repository_revisions: the received parameters must be ordered ' message += 'lists so that positional values in tool_shed_urls, names, owners and changeset_revisions are associated.' - log.error( message, exc_info=True ) - trans.response.status = 500 + log.debug( message ) return dict( status='error', error=message ) # Get the information about the Galaxy components (e.g., tool pane section, tool config file, etc) that will contain information # about each of the repositories being installed. @@ -482,7 +471,7 @@ repair_dict = repository_util.repair_tool_shed_repository( trans, repository, encoding_util.tool_shed_encode( repo_info_dict ) ) - repository_dict = repository.to_dict( value_mapper=default_tool_shed_repository_value_mapper( trans, repository ) ) + repository_dict = repository.to_dict( value_mapper=self.__get_value_mapper( trans ) ) repository_dict[ 'url' ] = web.url_for( controller='tool_shed_repositories', action='show', id=trans.security.encode_id( repository.id ) ) @@ -502,39 +491,39 @@ :param key: the API key of the Galaxy admin user. """ - try: - start_time = strftime( "%Y-%m-%d %H:%M:%S" ) - results = dict( start_time=start_time, - successful_count=0, - unsuccessful_count=0, - repository_status=[] ) - # Make sure the current user's API key proves he is an admin user in this Galaxy instance. - if not trans.user_is_admin(): - raise HTTPForbidden( detail='You are not authorized to reset metadata on repositories installed into this Galaxy instance.' ) - query = suc.get_query_for_setting_metadata_on_repositories( trans, my_writable=False, order=False ) - # Now reset metadata on all remaining repositories. - for repository in query: - repository_id = trans.security.encode_id( repository.id ) - try: - invalid_file_tups, metadata_dict = metadata_util.reset_all_metadata_on_installed_repository( trans, repository_id ) - if invalid_file_tups: - message = tool_util.generate_message_for_invalid_tools( trans, invalid_file_tups, repository, None, as_html=False ) - results[ 'unsuccessful_count' ] += 1 - else: - message = "Successfully reset metadata on repository %s owned by %s" % ( str( repository.name ), str( repository.owner ) ) - results[ 'successful_count' ] += 1 - except Exception, e: - message = "Error resetting metadata on repository %s owned by %s: %s" % ( str( repository.name ), str( repository.owner ), str( e ) ) + start_time = strftime( "%Y-%m-%d %H:%M:%S" ) + results = dict( start_time=start_time, + successful_count=0, + unsuccessful_count=0, + repository_status=[] ) + # Make sure the current user's API key proves he is an admin user in this Galaxy instance. + if not trans.user_is_admin(): + raise HTTPForbidden( detail='You are not authorized to reset metadata on repositories installed into this Galaxy instance.' ) + query = suc.get_query_for_setting_metadata_on_repositories( trans, my_writable=False, order=False ) + # Now reset metadata on all remaining repositories. + for repository in query: + repository_id = trans.security.encode_id( repository.id ) + try: + invalid_file_tups, metadata_dict = metadata_util.reset_all_metadata_on_installed_repository( trans, repository_id ) + if invalid_file_tups: + message = tool_util.generate_message_for_invalid_tools( trans, + invalid_file_tups, + repository, + None, + as_html=False ) results[ 'unsuccessful_count' ] += 1 - results[ 'repository_status' ].append( message ) - stop_time = strftime( "%Y-%m-%d %H:%M:%S" ) - results[ 'stop_time' ] = stop_time - return json.to_json_string( results, sort_keys=True, indent=4 * ' ' ) - except Exception, e: - message = "Error in the Galaxy tool_shed_repositories API in reset_metadata_on_installed_repositories: %s" % str( e ) - log.error( message, exc_info=True ) - trans.response.status = 500 - return message + else: + message = "Successfully reset metadata on repository %s owned by %s" % \ + ( str( repository.name ), str( repository.owner ) ) + results[ 'successful_count' ] += 1 + except Exception, e: + message = "Error resetting metadata on repository %s owned by %s: %s" % \ + ( str( repository.name ), str( repository.owner ), str( e ) ) + results[ 'unsuccessful_count' ] += 1 + results[ 'repository_status' ].append( message ) + stop_time = strftime( "%Y-%m-%d %H:%M:%S" ) + results[ 'stop_time' ] = stop_time + return json.to_json_string( results, sort_keys=True, indent=4 * ' ' ) @web.expose_api def show( self, trans, id, **kwd ): @@ -545,15 +534,12 @@ :param id: the encoded id of the ToolShedRepository object """ # Example URL: http://localhost:8763/api/tool_shed_repositories/df7a1f0c02a5b08e - try: - tool_shed_repository = suc.get_tool_shed_repository_by_id( trans, id ) - tool_shed_repository_dict = tool_shed_repository.as_dict( value_mapper=default_tool_shed_repository_value_mapper( trans, tool_shed_repository ) ) - tool_shed_repository_dict[ 'url' ] = web.url_for( controller='tool_shed_repositories', - action='show', - id=trans.security.encode_id( tool_shed_repository.id ) ) - return tool_shed_repository_dict - except Exception, e: - message = "Error in tool_shed_repositories API in index: " + str( e ) - log.error( message, exc_info=True ) - trans.response.status = 500 - return message + tool_shed_repository = suc.get_tool_shed_repository_by_id( trans, id ) + if tool_shed_repository is None: + log.debug( "Unable to locate tool_shed_repository record for id %s." % ( str( id ) ) ) + return {} + tool_shed_repository_dict = tool_shed_repository.as_dict( value_mapper=self.__get_value_mapper( trans ) ) + tool_shed_repository_dict[ 'url' ] = web.url_for( controller='tool_shed_repositories', + action='show', + id=trans.security.encode_id( tool_shed_repository.id ) ) + return tool_shed_repository_dict diff -r 995ca770764013bcf0c61092de8436c9278357f7 -r 2483cc9e597d37ab75c5bb46d467622f33b48bc7 lib/galaxy/webapps/tool_shed/api/repositories.py --- a/lib/galaxy/webapps/tool_shed/api/repositories.py +++ b/lib/galaxy/webapps/tool_shed/api/repositories.py @@ -36,22 +36,27 @@ :param name: the name of the Repository :param owner: the owner of the Repository - Returns the ordered list of changeset revision hash strings that are associated with installable revisions. As in the changelog, the - list is ordered oldest to newest. + Returns the ordered list of changeset revision hash strings that are associated with installable revisions. + As in the changelog, the list is ordered oldest to newest. """ # Example URL: http://localhost:9009/api/repositories/get_installable_revisions?name=add_c… - try: + if name and owner: # Get the repository information. repository = suc.get_repository_by_name_and_owner( trans.app, name, owner ) + if repository is None: + error_message = "Error in the Tool Shed repositories API in get_ordered_installable_revisions: " + error_message += "cannot locate repository %s owned by %s." % ( str( name ), str( owner ) ) + log.debug( error_message ) + return [] repo_dir = repository.repo_path( trans.app ) repo = hg.repository( suc.get_configured_ui(), repo_dir ) ordered_installable_revisions = suc.get_ordered_metadata_changeset_revisions( repository, repo, downloadable=True ) return ordered_installable_revisions - except Exception, e: - message = "Error in the Tool Shed repositories API in get_ordered_installable_revisions: %s" % str( e ) - log.error( message, exc_info=True ) - trans.response.status = 500 - return message + else: + error_message = "Error in the Tool Shed repositories API in get_ordered_installable_revisions: " + error_message += "invalid name %s or owner %s received." % ( str( name ), str( owner ) ) + log.debug( error_message ) + return [] @web.expose_api_anonymous def get_repository_revision_install_info( self, trans, name, owner, changeset_revision, **kwd ): @@ -106,49 +111,65 @@ ] } """ - repository_value_mapper = { 'id' : trans.security.encode_id, - 'user_id' : trans.security.encode_id } - # Example URL: http://localhost:9009/api/repositories/get_repository_revision_install_info… - try: + # Example URL: + # http://<xyz>/api/repositories/get_repository_revision_install_info?name=<n>&owner=<o>&changeset_revision=<cr> + if name and owner and changeset_revision: # Get the repository information. repository = suc.get_repository_by_name_and_owner( trans.app, name, owner ) + if repository is None: + log.debug( 'Cannot locate repository %s owned by %s' % ( str( name ), str( owner ) ) ) + return {}, {}, {} encoded_repository_id = trans.security.encode_id( repository.id ) - repository_dict = repository.to_dict( view='element', value_mapper=repository_value_mapper ) + repository_dict = repository.to_dict( view='element', + value_mapper=self.__get_value_mapper( trans ) ) repository_dict[ 'url' ] = web.url_for( controller='repositories', action='show', id=encoded_repository_id ) # Get the repository_metadata information. - repository_metadata = suc.get_repository_metadata_by_changeset_revision( trans, encoded_repository_id, changeset_revision ) + repository_metadata = suc.get_repository_metadata_by_changeset_revision( trans, + encoded_repository_id, + changeset_revision ) if not repository_metadata: - # The changeset_revision column in the repository_metadata table has been updated with a new value value, so find the - # changeset_revision to which we need to update. + # The changeset_revision column in the repository_metadata table has been updated with a new + # value value, so find the changeset_revision to which we need to update. repo_dir = repository.repo_path( trans.app ) repo = hg.repository( suc.get_configured_ui(), repo_dir ) new_changeset_revision = suc.get_next_downloadable_changeset_revision( repository, repo, changeset_revision ) - repository_metadata = suc.get_repository_metadata_by_changeset_revision( trans, encoded_repository_id, new_changeset_revision ) + repository_metadata = suc.get_repository_metadata_by_changeset_revision( trans, + encoded_repository_id, + new_changeset_revision ) changeset_revision = new_changeset_revision if repository_metadata: encoded_repository_metadata_id = trans.security.encode_id( repository_metadata.id ) repository_metadata_dict = repository_metadata.to_dict( view='collection', - value_mapper=self.__get_value_mapper( trans, repository_metadata ) ) + value_mapper=self.__get_value_mapper( trans ) ) repository_metadata_dict[ 'url' ] = web.url_for( controller='repository_revisions', action='show', id=encoded_repository_metadata_id ) # Get the repo_info_dict for installing the repository. - repo_info_dict, includes_tools, includes_tool_dependencies, includes_tools_for_display_in_tool_panel, \ - has_repository_dependencies, has_repository_dependencies_only_if_compiling_contained_td = \ + repo_info_dict, \ + includes_tools, \ + includes_tool_dependencies, \ + includes_tools_for_display_in_tool_panel, \ + has_repository_dependencies, \ + has_repository_dependencies_only_if_compiling_contained_td = \ repository_util.get_repo_info_dict( trans, encoded_repository_id, changeset_revision ) return repository_dict, repository_metadata_dict, repo_info_dict else: - message = "Unable to locate repository_metadata record for repository id %d and changeset_revision %s" % ( repository.id, changeset_revision ) - log.error( message, exc_info=True ) - trans.response.status = 500 + log.debug( "Unable to locate repository_metadata record for repository id %s and changeset_revision %s" % \ + ( str( repository.id ), str( changeset_revision ) ) ) return repository_dict, {}, {} - except Exception, e: - message = "Error in the Tool Shed repositories API in get_repository_revision_install_info: %s" % str( e ) - log.error( message, exc_info=True ) - trans.response.status = 500 - return message + else: + debug_msg = "Error in the Tool Shed repositories API in get_repository_revision_install_info: " + debug_msg += "Invalid name %s or owner %s or changeset_revision %s received." % \ + ( str( name ), str( owner ), str( changeset_revision ) ) + log.debug( debug_msg ) + return {}, {}, {} + + def __get_value_mapper( self, trans ): + value_mapper = { 'id' : trans.security.encode_id, + 'repository_id' : trans.security.encode_id } + return value_mapper @web.expose_api def import_capsule( self, trans, payload, **kwd ): @@ -177,29 +198,27 @@ uploaded_file=None, capsule_file_name=None ) if os.path.getsize( os.path.abspath( capsule_file_name ) ) == 0: - message = 'Your capsule file is empty.' - log.error( message, exc_info=True ) - trans.response.status = 500 - return message + log.debug( 'Your capsule file %s is empty.' % str( capsule_file_name ) ) + return {} try: # Open for reading with transparent compression. tar_archive = tarfile.open( capsule_file_path, 'r:*' ) except tarfile.ReadError, e: - message = 'Error opening file %s: %s' % ( str( capsule_file_name ), str( e ) ) - log.error( message, exc_info=True ) - trans.response.status = 500 - return message + log.debug( 'Error opening capsule file %s: %s' % ( str( capsule_file_name ), str( e ) ) ) + return {} capsule_dict[ 'tar_archive' ] = tar_archive capsule_dict[ 'capsule_file_name' ] = capsule_file_name capsule_dict = import_util.extract_capsule_files( trans, **capsule_dict ) capsule_dict = import_util.validate_capsule( trans, **capsule_dict ) status = capsule_dict.get( 'status', 'error' ) if status == 'error': - message = 'The capsule contents are invalid and cannpt be imported:<br/>%s' % str( capsule_dict.get( 'error_message', '' ) ) - log.error( message, exc_info=True ) - trans.response.status = 500 - return message + log.debug( 'The capsule contents are invalid and cannpt be imported:<br/>%s' % \ + str( capsule_dict.get( 'error_message', '' ) ) ) + return {} encoded_file_path = capsule_dict.get( 'encoded_file_path', None ) + if encoded_file_path is None: + log.debug( 'The capsule_dict %s is missing the required encoded_file_path entry.' % str( capsule_dict ) ) + return {} file_path = encoding_util.tool_shed_decode( encoded_file_path ) export_info_file_path = os.path.join( file_path, 'export_info.xml' ) export_info_dict = import_util.get_export_info_dict( export_info_file_path ) @@ -216,12 +235,14 @@ # Add the capsule_file_name and encoded_file_path to the repository_status_info_dict. repository_status_info_dict[ 'capsule_file_name' ] = capsule_file_name repository_status_info_dict[ 'encoded_file_path' ] = encoded_file_path - import_results_tups = repository_maintenance_util.create_repository_and_import_archive( trans, - repository_status_info_dict, - import_results_tups ) + import_results_tups = \ + repository_maintenance_util.create_repository_and_import_archive( trans, + repository_status_info_dict, + import_results_tups ) import_util.check_status_and_reset_downloadable( trans, import_results_tups ) suc.remove_dir( file_path ) - # NOTE: the order of installation is defined in import_results_tups, but order will be lost when transferred to return_dict. + # NOTE: the order of installation is defined in import_results_tups, but order will be lost + # when transferred to return_dict. return_dict = {} for import_results_tup in import_results_tups: ok, name_owner, message = import_results_tup @@ -237,28 +258,19 @@ GET /api/repositories Displays a collection (list) of repositories. """ - value_mapper = { 'id' : trans.security.encode_id, - 'user_id' : trans.security.encode_id } # Example URL: http://localhost:9009/api/repositories repository_dicts = [] - deleted = util.string_as_bool( deleted ) - try: - query = trans.sa_session.query( trans.app.model.Repository ) \ - .filter( trans.app.model.Repository.table.c.deleted == deleted ) \ - .order_by( trans.app.model.Repository.table.c.name ) \ - .all() - for repository in query: - repository_dict = repository.to_dict( view='collection', value_mapper=value_mapper ) - repository_dict[ 'url' ] = web.url_for( controller='repositories', - action='show', - id=trans.security.encode_id( repository.id ) ) - repository_dicts.append( repository_dict ) - return repository_dicts - except Exception, e: - message = "Error in the Tool Shed repositories API in index: %s" % str( e ) - log.error( message, exc_info=True ) - trans.response.status = 500 - return message + deleted = util.asbool( deleted ) + for repository in trans.sa_session.query( trans.app.model.Repository ) \ + .filter( trans.app.model.Repository.table.c.deleted == deleted ) \ + .order_by( trans.app.model.Repository.table.c.name ): + repository_dict = repository.to_dict( view='collection', + value_mapper=self.__get_value_mapper( trans ) ) + repository_dict[ 'url' ] = web.url_for( controller='repositories', + action='show', + id=trans.security.encode_id( repository.id ) ) + repository_dicts.append( repository_dict ) + return repository_dicts @web.expose_api def repository_ids_for_setting_metadata( self, trans, my_writable=False, **kwd ): @@ -273,28 +285,22 @@ in addition to those repositories of type tool_dependency_definition. This param is ignored if the current user is not an admin user, in which case this same restriction is automatic. """ - try: - if trans.user_is_admin(): - my_writable = util.asbool( my_writable ) - else: - my_writable = True - handled_repository_ids = [] - repository_ids = [] - query = suc.get_query_for_setting_metadata_on_repositories( trans, my_writable=my_writable, order=False ) - # Make sure repositories of type tool_dependency_definition are first in the list. - for repository in query: - if repository.type == rt_util.TOOL_DEPENDENCY_DEFINITION and repository.id not in handled_repository_ids: - repository_ids.append( trans.security.encode_id( repository.id ) ) - # Now add all remaining repositories to the list. - for repository in query: - if repository.type != rt_util.TOOL_DEPENDENCY_DEFINITION and repository.id not in handled_repository_ids: - repository_ids.append( trans.security.encode_id( repository.id ) ) - return repository_ids - except Exception, e: - message = "Error in the Tool Shed repositories API in repository_ids_for_setting_metadata: %s" % str( e ) - log.error( message, exc_info=True ) - trans.response.status = 500 - return message + if trans.user_is_admin(): + my_writable = util.asbool( my_writable ) + else: + my_writable = True + handled_repository_ids = [] + repository_ids = [] + query = suc.get_query_for_setting_metadata_on_repositories( trans, my_writable=my_writable, order=False ) + # Make sure repositories of type tool_dependency_definition are first in the list. + for repository in query: + if repository.type == rt_util.TOOL_DEPENDENCY_DEFINITION and repository.id not in handled_repository_ids: + repository_ids.append( trans.security.encode_id( repository.id ) ) + # Now add all remaining repositories to the list. + for repository in query: + if repository.type != rt_util.TOOL_DEPENDENCY_DEFINITION and repository.id not in handled_repository_ids: + repository_ids.append( trans.security.encode_id( repository.id ) ) + return repository_ids @web.expose_api def reset_metadata_on_repositories( self, trans, payload, **kwd ): @@ -318,6 +324,7 @@ :param skip_file (optional): A local file name that contains the encoded repository ids associated with repositories to skip. This param can be used as an alternative to the above encoded_ids_to_skip. """ + def handle_repository( trans, repository, results ): log.debug( "Resetting metadata on repository %s" % str( repository.name ) ) repository_id = trans.security.encode_id( repository.id ) @@ -335,53 +342,48 @@ status = '%s : %s' % ( str( repository.name ), message ) results[ 'repository_status' ].append( status ) return results - try: - start_time = strftime( "%Y-%m-%d %H:%M:%S" ) - results = dict( start_time=start_time, - repository_status=[], - successful_count=0, - unsuccessful_count=0 ) - handled_repository_ids = [] - encoded_ids_to_skip = payload.get( 'encoded_ids_to_skip', [] ) - skip_file = payload.get( 'skip_file', None ) - if skip_file and os.path.exists( skip_file ) and not encoded_ids_to_skip: - # Load the list of encoded_ids_to_skip from the skip_file. - # Contents of file must be 1 encoded repository id per line. - lines = open( skip_file, 'rb' ).readlines() - for line in lines: - if line.startswith( '#' ): - # Skip comments. - continue - encoded_ids_to_skip.append( line.rstrip( '\n' ) ) - if trans.user_is_admin(): - my_writable = util.asbool( payload.get( 'my_writable', False ) ) - else: - my_writable = True - query = suc.get_query_for_setting_metadata_on_repositories( trans, my_writable=my_writable, order=False ) - # First reset metadata on all repositories of type repository_dependency_definition. - for repository in query: - encoded_id = trans.security.encode_id( repository.id ) - if encoded_id in encoded_ids_to_skip: - log.debug( "Skipping repository with id %s because it is in encoded_ids_to_skip %s" % \ - ( str( repository.id ), str( encoded_ids_to_skip ) ) ) - elif repository.type == rt_util.TOOL_DEPENDENCY_DEFINITION and repository.id not in handled_repository_ids: - results = handle_repository( trans, repository, results ) - # Now reset metadata on all remaining repositories. - for repository in query: - encoded_id = trans.security.encode_id( repository.id ) - if encoded_id in encoded_ids_to_skip: - log.debug( "Skipping repository with id %s because it is in encoded_ids_to_skip %s" % \ - ( str( repository.id ), str( encoded_ids_to_skip ) ) ) - elif repository.type != rt_util.TOOL_DEPENDENCY_DEFINITION and repository.id not in handled_repository_ids: - results = handle_repository( trans, repository, results ) - stop_time = strftime( "%Y-%m-%d %H:%M:%S" ) - results[ 'stop_time' ] = stop_time - return json.to_json_string( results, sort_keys=True, indent=4 * ' ' ) - except Exception, e: - message = "Error in the Tool Shed repositories API in reset_metadata_on_repositories: %s" % str( e ) - log.error( message, exc_info=True ) - trans.response.status = 500 - return message + + start_time = strftime( "%Y-%m-%d %H:%M:%S" ) + results = dict( start_time=start_time, + repository_status=[], + successful_count=0, + unsuccessful_count=0 ) + handled_repository_ids = [] + encoded_ids_to_skip = payload.get( 'encoded_ids_to_skip', [] ) + skip_file = payload.get( 'skip_file', None ) + if skip_file and os.path.exists( skip_file ) and not encoded_ids_to_skip: + # Load the list of encoded_ids_to_skip from the skip_file. + # Contents of file must be 1 encoded repository id per line. + lines = open( skip_file, 'rb' ).readlines() + for line in lines: + if line.startswith( '#' ): + # Skip comments. + continue + encoded_ids_to_skip.append( line.rstrip( '\n' ) ) + if trans.user_is_admin(): + my_writable = util.asbool( payload.get( 'my_writable', False ) ) + else: + my_writable = True + query = suc.get_query_for_setting_metadata_on_repositories( trans, my_writable=my_writable, order=False ) + # First reset metadata on all repositories of type repository_dependency_definition. + for repository in query: + encoded_id = trans.security.encode_id( repository.id ) + if encoded_id in encoded_ids_to_skip: + log.debug( "Skipping repository with id %s because it is in encoded_ids_to_skip %s" % \ + ( str( repository.id ), str( encoded_ids_to_skip ) ) ) + elif repository.type == rt_util.TOOL_DEPENDENCY_DEFINITION and repository.id not in handled_repository_ids: + results = handle_repository( trans, repository, results ) + # Now reset metadata on all remaining repositories. + for repository in query: + encoded_id = trans.security.encode_id( repository.id ) + if encoded_id in encoded_ids_to_skip: + log.debug( "Skipping repository with id %s because it is in encoded_ids_to_skip %s" % \ + ( str( repository.id ), str( encoded_ids_to_skip ) ) ) + elif repository.type != rt_util.TOOL_DEPENDENCY_DEFINITION and repository.id not in handled_repository_ids: + results = handle_repository( trans, repository, results ) + stop_time = strftime( "%Y-%m-%d %H:%M:%S" ) + results[ 'stop_time' ] = stop_time + return json.to_json_string( results, sort_keys=True, indent=4 * ' ' ) @web.expose_api def reset_metadata_on_repository( self, trans, payload, **kwd ): @@ -395,6 +397,7 @@ The following parameters must be included in the payload. :param repository_id: the encoded id of the repository on which metadata is to be reset. """ + def handle_repository( trans, start_time, repository ): results = dict( start_time=start_time, repository_status=[] ) @@ -410,21 +413,16 @@ status = '%s : %s' % ( str( repository.name ), message ) results[ 'repository_status' ].append( status ) return results - try: - repository_id = payload.get( 'repository_id', None ) - if repository_id is not None: - repository = suc.get_repository_in_tool_shed( trans, repository_id ) - start_time = strftime( "%Y-%m-%d %H:%M:%S" ) - log.debug( "%s...resetting metadata on repository %s" % ( start_time, str( repository.name ) ) ) - results = handle_repository( trans, start_time, repository ) - stop_time = strftime( "%Y-%m-%d %H:%M:%S" ) - results[ 'stop_time' ] = stop_time - return json.to_json_string( results, sort_keys=True, indent=4 * ' ' ) - except Exception, e: - message = "Error in the Tool Shed repositories API in reset_metadata_on_repositories: %s" % str( e ) - log.error( message, exc_info=True ) - trans.response.status = 500 - return message + + repository_id = payload.get( 'repository_id', None ) + if repository_id is not None: + repository = suc.get_repository_in_tool_shed( trans, repository_id ) + start_time = strftime( "%Y-%m-%d %H:%M:%S" ) + log.debug( "%s...resetting metadata on repository %s" % ( start_time, str( repository.name ) ) ) + results = handle_repository( trans, start_time, repository ) + stop_time = strftime( "%Y-%m-%d %H:%M:%S" ) + results[ 'stop_time' ] = stop_time + return json.to_json_string( results, sort_keys=True, indent=4 * ' ' ) @web.expose_api_anonymous def show( self, trans, id, **kwd ): @@ -434,27 +432,14 @@ :param id: the encoded id of the Repository object """ - value_mapper = { 'id' : trans.security.encode_id, - 'user_id' : trans.security.encode_id } # Example URL: http://localhost:9009/api/repositories/f9cad7b01a472135 - try: - repository = suc.get_repository_in_tool_shed( trans, id ) - repository_dict = repository.to_dict( view='element', value_mapper=value_mapper ) - repository_dict[ 'url' ] = web.url_for( controller='repositories', - action='show', - id=trans.security.encode_id( repository.id ) ) - return repository_dict - except Exception, e: - message = "Error in the Tool Shed repositories API in show: %s" % str( e ) - log.error( message, exc_info=True ) - trans.response.status = 500 - return message - - def __get_value_mapper( self, trans, repository_metadata ): - value_mapper = { 'id' : trans.security.encode_id, - 'repository_id' : trans.security.encode_id } - if repository_metadata.time_last_tested is not None: - # For some reason the Dictifiable.to_dict() method in ~/galaxy/model/item_attrs.py requires - # a function rather than a mapped value, so just pass the time_ago function here. - value_mapper[ 'time_last_tested' ] = time_ago - return value_mapper + repository = suc.get_repository_in_tool_shed( trans, id ) + if repository is None: + log.debug( "Unable to locate repository record for id %s." % ( str( id ) ) ) + return {} + repository_dict = repository.to_dict( view='element', + value_mapper=self.__get_value_mapper( trans ) ) + repository_dict[ 'url' ] = web.url_for( controller='repositories', + action='show', + id=trans.security.encode_id( repository.id ) ) + return repository_dict diff -r 995ca770764013bcf0c61092de8436c9278357f7 -r 2483cc9e597d37ab75c5bb46d467622f33b48bc7 lib/galaxy/webapps/tool_shed/api/repository_revisions.py --- a/lib/galaxy/webapps/tool_shed/api/repository_revisions.py +++ b/lib/galaxy/webapps/tool_shed/api/repository_revisions.py @@ -1,6 +1,5 @@ import datetime import logging -from galaxy.web.framework.helpers import time_ago from tool_shed.util import metadata_util from galaxy import web from galaxy import util @@ -48,61 +47,29 @@ if not changeset_revision: raise HTTPBadRequest( detail="Missing required parameter 'changeset_revision'." ) export_repository_dependencies = payload.get( 'export_repository_dependencies', False ) - try: - # We'll currently support only gzip-compressed tar archives. - file_type = 'gz' - export_repository_dependencies = util.string_as_bool( export_repository_dependencies ) - # Get the repository information. - repository = suc.get_repository_by_name_and_owner( trans.app, name, owner ) - repository_id = trans.security.encode_id( repository.id ) - response = export_util.export_repository( trans, - tool_shed_url, - repository_id, - str( repository.name ), - changeset_revision, - file_type, - export_repository_dependencies, - api=True ) - return response - except Exception, e: - message = "Error in the Tool Shed repository_revisions API in export: %s" % str( e ) - log.error( message, exc_info=True ) - trans.response.status = 500 - return message + # We'll currently support only gzip-compressed tar archives. + file_type = 'gz' + export_repository_dependencies = util.asbool( export_repository_dependencies ) + # Get the repository information. + repository = suc.get_repository_by_name_and_owner( trans.app, name, owner ) + if repository is None: + error_message = 'Cannot locate repository with name %s and owner %s,' % ( str( name ), str( owner ) ) + log.debug( error_message ) + return None, error_message + repository_id = trans.security.encode_id( repository.id ) + return export_util.export_repository( trans, + tool_shed_url, + repository_id, + str( repository.name ), + changeset_revision, + file_type, + export_repository_dependencies, + api=True ) - @web.expose_api_anonymous - def repository_dependencies( self, trans, id, **kwd ): - """ - GET /api/repository_revisions/{encoded repository_metadata id}/repository_dependencies - Displays information about a repository_metadata record in the Tool Shed. - - :param id: the encoded id of the `RepositoryMetadata` object - """ - # Example URL: http://localhost:9009/api/repository_revisions/repository_dependencies/bb12… + def __get_value_mapper( self, trans ): value_mapper = { 'id' : trans.security.encode_id, - 'user_id' : trans.security.encode_id } - repository_dependencies_dicts = [] - try: - repository_metadata = metadata_util.get_repository_metadata_by_id( trans, id ) - metadata = repository_metadata.metadata - if metadata and 'repository_dependencies' in metadata: - rd_tups = metadata[ 'repository_dependencies' ][ 'repository_dependencies' ] - for rd_tup in rd_tups: - tool_shed, name, owner, changeset_revision = rd_tup[ 0:4 ] - repository_dependency = suc.get_repository_by_name_and_owner( trans.app, name, owner ) - repository_dependency_dict = repository_dependency.to_dict( view='element', value_mapper=value_mapper ) - # We have to add the changeset_revision of of the repository dependency. - repository_dependency_dict[ 'changeset_revision' ] = changeset_revision - repository_dependency_dict[ 'url' ] = web.url_for( controller='repositories', - action='show', - id=trans.security.encode_id( repository_dependency.id ) ) - repository_dependencies_dicts.append( repository_dependency_dict ) - return repository_dependencies_dicts - except Exception, e: - message = "Error in the Tool Shed repository_revisions API in repository_dependencies: %s" % str( e ) - log.error( message, exc_info=True ) - trans.response.status = 500 - return message + 'repository_id' : trans.security.encode_id } + return value_mapper @web.expose_api_anonymous def index( self, trans, **kwd ): @@ -117,59 +84,94 @@ # Filter by downloadable if received. downloadable = kwd.get( 'downloadable', None ) if downloadable is not None: - clause_list.append( trans.model.RepositoryMetadata.table.c.downloadable == util.string_as_bool( downloadable ) ) + clause_list.append( trans.model.RepositoryMetadata.table.c.downloadable == util.asbool( downloadable ) ) # Filter by malicious if received. malicious = kwd.get( 'malicious', None ) if malicious is not None: - clause_list.append( trans.model.RepositoryMetadata.table.c.malicious == util.string_as_bool( malicious ) ) + clause_list.append( trans.model.RepositoryMetadata.table.c.malicious == util.asbool( malicious ) ) # Filter by tools_functionally_correct if received. tools_functionally_correct = kwd.get( 'tools_functionally_correct', None ) if tools_functionally_correct is not None: - clause_list.append( trans.model.RepositoryMetadata.table.c.tools_functionally_correct == util.string_as_bool( tools_functionally_correct ) ) + clause_list.append( trans.model.RepositoryMetadata.table.c.tools_functionally_correct == util.asbool( tools_functionally_correct ) ) # Filter by missing_test_components if received. missing_test_components = kwd.get( 'missing_test_components', None ) if missing_test_components is not None: - clause_list.append( trans.model.RepositoryMetadata.table.c.missing_test_components == util.string_as_bool( missing_test_components ) ) + clause_list.append( trans.model.RepositoryMetadata.table.c.missing_test_components == util.asbool( missing_test_components ) ) # Filter by do_not_test if received. do_not_test = kwd.get( 'do_not_test', None ) if do_not_test is not None: - clause_list.append( trans.model.RepositoryMetadata.table.c.do_not_test == util.string_as_bool( do_not_test ) ) + clause_list.append( trans.model.RepositoryMetadata.table.c.do_not_test == util.asbool( do_not_test ) ) # Filter by includes_tools if received. includes_tools = kwd.get( 'includes_tools', None ) if includes_tools is not None: - clause_list.append( trans.model.RepositoryMetadata.table.c.includes_tools == util.string_as_bool( includes_tools ) ) + clause_list.append( trans.model.RepositoryMetadata.table.c.includes_tools == util.asbool( includes_tools ) ) # Filter by test_install_error if received. test_install_error = kwd.get( 'test_install_error', None ) if test_install_error is not None: - clause_list.append( trans.model.RepositoryMetadata.table.c.test_install_error == util.string_as_bool( test_install_error ) ) + clause_list.append( trans.model.RepositoryMetadata.table.c.test_install_error == util.asbool( test_install_error ) ) # Filter by skip_tool_test if received. skip_tool_test = kwd.get( 'skip_tool_test', None ) if skip_tool_test is not None: - skip_tool_test = util.string_as_bool( skip_tool_test ) + skip_tool_test = util.asbool( skip_tool_test ) skipped_metadata_ids_subquery = select( [ trans.app.model.SkipToolTest.table.c.repository_metadata_id ] ) if skip_tool_test: clause_list.append( trans.model.RepositoryMetadata.id.in_( skipped_metadata_ids_subquery ) ) else: clause_list.append( not_( trans.model.RepositoryMetadata.id.in_( skipped_metadata_ids_subquery ) ) ) - # Generate and execute the query. - try: - query = trans.sa_session.query( trans.app.model.RepositoryMetadata ) \ - .filter( and_( *clause_list ) ) \ - .order_by( trans.app.model.RepositoryMetadata.table.c.repository_id.desc() ) \ - .all() - for repository_metadata in query: - repository_metadata_dict = repository_metadata.to_dict( view='collection', - value_mapper=self.__get_value_mapper( trans, repository_metadata ) ) - repository_metadata_dict[ 'url' ] = web.url_for( controller='repository_revisions', - action='show', - id=trans.security.encode_id( repository_metadata.id ) ) - repository_metadata_dicts.append( repository_metadata_dict ) - return repository_metadata_dicts - except Exception, e: - message = "Error in the Tool Shed repository_revisions API in index: " + str( e ) - log.error( message, exc_info=True ) - trans.response.status = 500 - return message + for repository_metadata in trans.sa_session.query( trans.app.model.RepositoryMetadata ) \ + .filter( and_( *clause_list ) ) \ + .order_by( trans.app.model.RepositoryMetadata.table.c.repository_id.desc() ): + repository_metadata_dict = repository_metadata.to_dict( view='collection', + value_mapper=self.__get_value_mapper( trans ) ) + repository_metadata_dict[ 'url' ] = web.url_for( controller='repository_revisions', + action='show', + id=trans.security.encode_id( repository_metadata.id ) ) + repository_metadata_dicts.append( repository_metadata_dict ) + return repository_metadata_dicts + + @web.expose_api_anonymous + def repository_dependencies( self, trans, id, **kwd ): + """ + GET /api/repository_revisions/{encoded repository_metadata id}/repository_dependencies + Displays information about a repository_metadata record in the Tool Shed. + + :param id: the encoded id of the `RepositoryMetadata` object + """ + # Example URL: http://localhost:9009/api/repository_revisions/repository_dependencies/bb12… + repository_dependencies_dicts = [] + repository_metadata = metadata_util.get_repository_metadata_by_id( trans, id ) + if repository_metadata is None: + log.debug( 'Invalid repository_metadata id received: %s' % str( id ) ) + return repository_dependencies_dicts + metadata = repository_metadata.metadata + if metadata is None: + log.debug( 'The repository_metadata record with id %s has no metadata.' % str ( id ) ) + return repository_dependencies_dicts + if 'repository_dependencies' in metadata: + rd_tups = metadata[ 'repository_dependencies' ][ 'repository_dependencies' ] + for rd_tup in rd_tups: + tool_shed, name, owner, changeset_revision = rd_tup[ 0:4 ] + repository_dependency = suc.get_repository_by_name_and_owner( trans.app, name, owner ) + if repository_dependency is None: + log.dbug( 'Cannot locate repository dependency %s owned by %s.' % ( name, owner ) ) + continue + repository_dependency_id = trans.security.encode_id( repository_dependency.id ) + repository_dependency_repository_metadata = \ + suc.get_repository_metadata_by_changeset_revision( trans, repository_dependency_id, changeset_revision ) + if repository_dependency_repository_metadata is None: + log.debug( 'Cannot locate repository_metadata with id %s for repository dependency %s owned by %s.' % \ + ( str( repository_dependency_id ), str( name ), str( owner ) ) ) + continue + repository_dependency_repository_metadata_id = trans.security.encode_id( repository_dependency_repository_metadata.id ) + repository_dependency_dict = repository_dependency.to_dict( view='element', + value_mapper=self.__get_value_mapper( trans ) ) + # We have to add the changeset_revision of of the repository dependency. + repository_dependency_dict[ 'changeset_revision' ] = changeset_revision + repository_dependency_dict[ 'url' ] = web.url_for( controller='repositories', + action='show', + id=repository_dependency_repository_metadata_id ) + repository_dependencies_dicts.append( repository_dependency_dict ) + return repository_dependencies_dicts @web.expose_api_anonymous def show( self, trans, id, **kwd ): @@ -180,19 +182,16 @@ :param id: the encoded id of the `RepositoryMetadata` object """ # Example URL: http://localhost:9009/api/repository_revisions/bb125606ff9ea620 - try: - repository_metadata = metadata_util.get_repository_metadata_by_id( trans, id ) - repository_metadata_dict = repository_metadata.to_dict( view='element', - value_mapper=self.__get_value_mapper( trans, repository_metadata ) ) - repository_metadata_dict[ 'url' ] = web.url_for( controller='repository_revisions', - action='show', - id=trans.security.encode_id( repository_metadata.id ) ) - return repository_metadata_dict - except Exception, e: - message = "Error in the Tool Shed repository_revisions API in show: %s" % str( e ) - log.error( message, exc_info=True ) - trans.response.status = 500 - return message + repository_metadata = metadata_util.get_repository_metadata_by_id( trans, id ) + if repository_metadata is None: + log.debug( 'Cannot locate repository_metadata with id %s' % str( id ) ) + return {} + repository_metadata_dict = repository_metadata.to_dict( view='element', + value_mapper=self.__get_value_mapper( trans ) ) + repository_metadata_dict[ 'url' ] = web.url_for( controller='repository_revisions', + action='show', + id=trans.security.encode_id( repository_metadata.id ) ) + return repository_metadata_dict @web.expose_api def update( self, trans, payload, **kwd ): @@ -201,41 +200,32 @@ Updates the value of specified columns of the repository_metadata table based on the key / value pairs in payload. """ repository_metadata_id = kwd.get( 'id', None ) - try: - repository_metadata = metadata_util.get_repository_metadata_by_id( trans, repository_metadata_id ) - flush_needed = False - for key, new_value in payload.items(): - if key == 'time_last_tested': - repository_metadata.time_last_tested = datetime.datetime.utcnow() - flush_needed = True - elif hasattr( repository_metadata, key ): - # log information when setting attributes associated with the Tool Shed's install and test framework. - if key in [ 'do_not_test', 'includes_tools', 'missing_test_components', 'test_install_error', - 'tools_functionally_correct' ]: - log.debug( 'Setting repository_metadata table column %s to value %s for changeset_revision %s via the Tool Shed API.' % \ - ( str( key ), str( new_value ), str( repository_metadata.changeset_revision ) ) ) - setattr( repository_metadata, key, new_value ) - flush_needed = True - if flush_needed: - trans.sa_session.add( repository_metadata ) - trans.sa_session.flush() - except Exception, e: - message = "Error in the Tool Shed repository_revisions API in update: %s" % str( e ) - log.error( message, exc_info=True ) - trans.response.status = 500 - return message + if repository_metadata_id is None: + raise HTTPBadRequest( detail="Missing required parameter 'id'." ) + repository_metadata = metadata_util.get_repository_metadata_by_id( trans, repository_metadata_id ) + if repository_metadata is None: + log.debug( 'Cannot locate repository_metadata with id %s' % str( repository_metadata_id ) ) + return {} + flush_needed = False + for key, new_value in payload.items(): + if key == 'time_last_tested': + repository_metadata.time_last_tested = datetime.datetime.utcnow() + flush_needed = True + elif hasattr( repository_metadata, key ): + # log information when setting attributes associated with the Tool Shed's install and test framework. + if key in [ 'do_not_test', 'includes_tools', 'missing_test_components', 'test_install_error', + 'tools_functionally_correct' ]: + log.debug( 'Setting repository_metadata column %s to value %s for changeset_revision %s via the Tool Shed API.' % \ + ( str( key ), str( new_value ), str( repository_metadata.changeset_revision ) ) ) + setattr( repository_metadata, key, new_value ) + flush_needed = True + if flush_needed: + trans.sa_session.add( repository_metadata ) + trans.sa_session.flush() + trans.sa_session.refresh( repository_metadata ) repository_metadata_dict = repository_metadata.to_dict( view='element', - value_mapper=self.__get_value_mapper( trans, repository_metadata ) ) + value_mapper=self.__get_value_mapper( trans ) ) repository_metadata_dict[ 'url' ] = web.url_for( controller='repository_revisions', action='show', id=trans.security.encode_id( repository_metadata.id ) ) return repository_metadata_dict - - def __get_value_mapper( self, trans, repository_metadata ): - value_mapper = { 'id' : trans.security.encode_id, - 'repository_id' : trans.security.encode_id } - if repository_metadata.time_last_tested is not None: - # For some reason the Dictifiable.to_dict() method in ~/galaxy/model/item_attrs.py requires - # a function rather than a mapped value, so just pass the time_ago function here. - value_mapper[ 'time_last_tested' ] = time_ago - return value_mapper diff -r 995ca770764013bcf0c61092de8436c9278357f7 -r 2483cc9e597d37ab75c5bb46d467622f33b48bc7 lib/galaxy/webapps/tool_shed/controllers/hg.py --- a/lib/galaxy/webapps/tool_shed/controllers/hg.py +++ b/lib/galaxy/webapps/tool_shed/controllers/hg.py @@ -2,6 +2,7 @@ from galaxy import web from galaxy.web.base.controller import BaseUIController from tool_shed.util.shed_util_common import get_repository_by_name_and_owner +from tool_shed.util.shed_util_common import update_repository from tool_shed.util.metadata_util import set_repository_metadata from galaxy import eggs @@ -9,6 +10,8 @@ import mercurial.__version__ from mercurial.hgweb.hgwebdir_mod import hgwebdir from mercurial.hgweb.request import wsgiapplication +from mercurial import hg +from mercurial import ui log = logging.getLogger(__name__) @@ -36,6 +39,11 @@ repository = get_repository_by_name_and_owner( trans.app, name, owner ) if repository: if hg_version >= '2.2.3': + # Update the repository on disk to the tip revision, because the web upload form uses the on-disk working + # directory. If the repository is not updated on disk, pushing from the command line and then uploading + # via the web interface will result in a new head being created. + repo = hg.repository( ui.ui(), repository.repo_path( trans.app ) ) + update_repository( repo, ctx_rev=None ) # Set metadata using the repository files on disk. error_message, status = set_repository_metadata( trans, repository ) if status == 'ok' and error_message: diff -r 995ca770764013bcf0c61092de8436c9278357f7 -r 2483cc9e597d37ab75c5bb46d467622f33b48bc7 lib/tool_shed/galaxy_install/install_manager.py --- a/lib/tool_shed/galaxy_install/install_manager.py +++ b/lib/tool_shed/galaxy_install/install_manager.py @@ -567,14 +567,17 @@ def order_repositories_for_installation( self, tool_shed_repositories, repository_dependencies_dict ): """ - Some repositories may have repository dependencies that are required to be installed before the dependent repository. This method will - inspect the list of repositories about to be installed and make sure to order them appropriately. For each repository about to be installed, - if required repositories are not contained in the list of repositories about to be installed, then they are not considered. Repository - dependency definitions that contain circular dependencies should not result in an infinite loop, but obviously prior installation will not be - handled for one or more of the repositories that require prior installation. This process is similar to the process used when installing tool - shed repositories (i.e., the order_components_for_installation() method in ~/lib/tool_shed/galaxy_install/repository_util), but does not handle - managing tool panel sections and other components since repository dependency definitions contained in tool shed repositories with migrated - tools must never define a relationship to a repository dependency that contains a tool. + Some repositories may have repository dependencies that are required to be installed before the dependent + repository. This method will inspect the list of repositories about to be installed and make sure to order + them appropriately. For each repository about to be installed, if required repositories are not contained + in the list of repositories about to be installed, then they are not considered. Repository dependency + definitions that contain circular dependencies should not result in an infinite loop, but obviously prior + installation will not be handled for one or more of the repositories that require prior installation. This + process is similar to the process used when installing tool shed repositories (i.e., the + order_components_for_installation() method in ~/lib/tool_shed/galaxy_install/repository_util), but does not + handle managing tool panel sections and other components since repository dependency definitions contained + in tool shed repositories with migrated tools must never define a relationship to a repository dependency + that contains a tool. """ ordered_tool_shed_repositories = [] ordered_tsr_ids = [] diff -r 995ca770764013bcf0c61092de8436c9278357f7 -r 2483cc9e597d37ab75c5bb46d467622f33b48bc7 lib/tool_shed/scripts/api/common.py --- a/lib/tool_shed/scripts/api/common.py +++ b/lib/tool_shed/scripts/api/common.py @@ -8,28 +8,18 @@ new_path.extend( sys.path[ 1: ] ) sys.path = new_path +import tool_shed.util.shed_util_common as suc + from galaxy import eggs import pkg_resources -pkg_resources.require( "pycrypto" ) -from Crypto.Cipher import Blowfish -from Crypto.Util.randpool import RandomPool -from Crypto.Util import number - -def encode_id( config_id_secret, obj_id ): - # Utility method to encode ID's - id_cipher = Blowfish.new( config_id_secret ) - # Convert to string - s = str( obj_id ) - # Pad to a multiple of 8 with leading "!" - s = ( "!" * ( 8 - len(s) % 8 ) ) + s - # Encrypt - return id_cipher.encrypt( s ).encode( 'hex' ) - def delete( api_key, url, data, return_formatted=True ): - # Sends an API DELETE request and acts as a generic formatter for the JSON response - 'data' will become the JSON payload read by Galaxy. + """ + Sends an API DELETE request and acts as a generic formatter for the JSON response. The + 'data' will become the JSON payload read by the Tool Shed. + """ try: - url = make_url( api_key, url ) + url = make_url( url, api_key=api_key, args=None ) req = urllib2.Request( url, headers = { 'Content-Type': 'application/json' }, data = json.dumps( data )) req.get_method = lambda: 'DELETE' r = json.loads( urllib2.urlopen( req ).read() ) @@ -47,12 +37,13 @@ print r def display( url, api_key=None, return_formatted=True ): - # Sends an API GET request and acts as a generic formatter for the JSON response. + """Sends an API GET request and acts as a generic formatter for the JSON response.""" try: r = get( url, api_key=api_key ) except urllib2.HTTPError, e: print e - print e.read( 1024 ) # Only return the first 1K of errors. + # Only return the first 1K of errors. + print e.read( 1024 ) sys.exit( 1 ) if type( r ) == unicode: print 'error: %s' % r @@ -84,16 +75,94 @@ print 'response is unknown type: %s' % type( r ) def get( url, api_key=None ): - # Do the actual GET. - url = make_url( url, api_key=api_key ) + """Do the GET.""" + url = make_url( url, api_key=api_key, args=None ) try: return json.loads( urllib2.urlopen( url ).read() ) except ValueError, e: print "URL did not return JSON data" - sys.exit(1) + sys.exit( 1 ) + +def get_api_url( base, parts=[], params=None ): + """Compose and return a URL for the Tool Shed API.""" + if 'api' in parts and parts.index( 'api' ) != 0: + parts.pop( parts.index( 'api' ) ) + parts.insert( 0, 'api' ) + elif 'api' not in parts: + parts.insert( 0, 'api' ) + url = suc.url_join( base, *parts ) + if params is not None: + try: + query_string = urllib.urlencode( params ) + except Exception, e: + # The value of params must be a string. + query_string = params + url += '?%s' % query_string + return url + +def get_latest_downloadable_changeset_revision_via_api( url, name, owner ): + """ + Return the latest downloadable changeset revision for the repository defined by the received + name and owner. + """ + error_message = '' + parts = [ 'api', 'repositories', 'get_ordered_installable_revisions' ] + params = dict( name=name, owner=owner ) + api_url = get_api_url( base=url, parts=parts, params=params ) + changeset_revisions, error_message = json_from_url( api_url ) + if changeset_revisions is None or error_message: + return None, error_message + if len( changeset_revisions ) >= 1: + return changeset_revisions[ -1 ], error_message + return suc.INITIAL_CHANGELOG_HASH, error_message + +def get_repository_dict( url, repository_dict ): + """ + Send a request to the Tool Shed to get additional information about the repository defined + by the received repository_dict. Add the information to the repository_dict and return it. + """ + error_message = '' + if not isinstance( repository_dict, dict ): + error_message = 'Invalid repository_dict received: %s' % str( repository_dict ) + return None, error_message + repository_id = repository_dict.get( 'repository_id', None ) + if repository_id is None: + error_message = 'Invalid repository_dict does not contain a repository_id entry: %s' % str( repository_dict ) + return None, error_message + parts = [ 'api', 'repositories', repository_id ] + api_url = get_api_url( base=url, parts=parts ) + extended_dict, error_message = json_from_url( api_url ) + if extended_dict is None or error_message: + return None, error_message + name = extended_dict.get( 'name', None ) + owner = extended_dict.get( 'owner', None ) + if name is not None and owner is not None: + name = str( name ) + owner = str( owner ) + latest_changeset_revision, error_message = get_latest_downloadable_changeset_revision_via_api( url, name, owner ) + if latest_changeset_revision is None or error_message: + return None, error_message + extended_dict[ 'latest_revision' ] = str( latest_changeset_revision ) + return extended_dict, error_message + else: + error_message = 'Invalid extended_dict does not contain name or woner entries: %s' % str( extended_dict ) + return None, error_message + +def json_from_url( url ): + """Send a request to the Tool Shed via the Tool Shed API and handle the response.""" + error_message = '' + url_handle = urllib.urlopen( url ) + url_contents = url_handle.read() + try: + parsed_json = simplejson.loads( url_contents ) + except Exception, e: + error_message = str( url_contents ) + print 'Error parsing JSON data in json_from_url(): ', str( e ) + return None, error_message + return parsed_json, error_message def make_url( url, api_key=None, args=None ): - # Adds the API Key to the URL if it's not already there. + """Adds the API Key to the URL if it's not already there.""" if args is None: args = [] argsep = '&' @@ -105,20 +174,23 @@ return url + argsep + '&'.join( [ '='.join( t ) for t in args ] ) def post( url, data, api_key=None ): - # Do the actual POST. - url = make_url( url, api_key=api_key ) + """Do the POST.""" + url = make_url( url, api_key=api_key, args=None ) req = urllib2.Request( url, headers = { 'Content-Type': 'application/json' }, data = json.dumps( data ) ) return json.loads( urllib2.urlopen( req ).read() ) def put( url, data, api_key=None ): - # Do the actual PUT. - url = make_url( url, api_key=api_key ) + """Do the PUT.""" + url = make_url( url, api_key=api_key, args=None ) req = urllib2.Request( url, headers = { 'Content-Type': 'application/json' }, data = json.dumps( data )) req.get_method = lambda: 'PUT' return json.loads( urllib2.urlopen( req ).read() ) def submit( url, data, api_key=None, return_formatted=True ): - # Sends an API POST request and acts as a generic formatter for the JSON response - 'data' will become the JSON payload read by Galaxy. + """ + Sends an API POST request and acts as a generic formatter for the JSON response. The + 'data' will become the JSON payload read by the Tool Shed. + """ try: r = post( url, data, api_key=api_key ) except urllib2.HTTPError, e: @@ -133,7 +205,8 @@ print 'Response' print '--------' if type( r ) == list: - # Currently the only implemented responses are lists of dicts, because submission creates some number of collection elements. + # Currently the only implemented responses are lists of dicts, because submission creates + # some number of collection elements. for i in r: if type( i ) == dict: if 'url' in i: @@ -150,9 +223,12 @@ print r def update( api_key, url, data, return_formatted=True ): - # Sends an API PUT request and acts as a generic formatter for the JSON response - 'data' will become the JSON payload read by Galaxy. + """ + Sends an API PUT request and acts as a generic formatter for the JSON response. The + 'data' will become the JSON payload read by the Tool Shed. + """ try: - r = put( api_key, url, data ) + r = put( url, data, api_key=api_key ) except urllib2.HTTPError, e: if return_formatted: print e This diff is so big that we needed to truncate the remainder. https://bitbucket.org/galaxy/galaxy-central/commits/3dde0f4765fb/ Changeset: 3dde0f4765fb User: dannon Date: 2014-01-10 13:42:58 Summary: One more simplejson/json fix that slipped through the merge. Affected #: 1 file diff -r 2483cc9e597d37ab75c5bb46d467622f33b48bc7 -r 3dde0f4765fbff33789badeaa17503349559ff72 lib/tool_shed/scripts/api/common.py --- a/lib/tool_shed/scripts/api/common.py +++ b/lib/tool_shed/scripts/api/common.py @@ -154,7 +154,7 @@ url_handle = urllib.urlopen( url ) url_contents = url_handle.read() try: - parsed_json = simplejson.loads( url_contents ) + parsed_json = json.loads( url_contents ) except Exception, e: error_message = str( url_contents ) print 'Error parsing JSON data in json_from_url(): ', str( e ) Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: 2 new changesets
by commits-noreply＠bitbucket.org 09 Jan '14

09 Jan '14

2 new commits in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/e72b8fd2d28b/ Changeset: e72b8fd2d28b User: carlfeberhard Date: 2014-01-09 22:00:07 Summary: Functional twill tests (history): remove check box assertion on history/list (as it's rendered in js now), fix history json scanning errors Affected #: 2 files diff -r 17e9e3ffb571e8f200ee8bf195a3708b2be4792b -r e72b8fd2d28b0a5371d13a77469746d0786496de test/base/twilltestcase.py --- a/test/base/twilltestcase.py +++ b/test/base/twilltestcase.py @@ -335,6 +335,7 @@ self.visit_page( "history?show_deleted=False" ) else: self.visit_page( "history" ) + json_data = {} try: tc.find( pattern, flags=( 'm' if multiline else '' ) ) # twill stores the regex match in a special stack variable @@ -541,7 +542,6 @@ self.home() self.visit_page( "history/list" ) self.check_page_for_string( 'Saved Histories' ) - self.check_page_for_string( '<input type="checkbox" name="id" value=' ) self.check_page_for_string( 'operation=Rename' ) self.check_page_for_string( 'operation=Switch' ) self.check_page_for_string( 'operation=Delete' ) @@ -553,7 +553,6 @@ self.home() self.visit_page( "history/list?f-deleted=True" ) self.check_page_for_string( 'Saved Histories' ) - self.check_page_for_string( '<input type="checkbox" name="id" value=' ) self.check_page_for_string( 'operation=Undelete' ) for check_str in strings_displayed: self.check_page_for_string( check_str ) diff -r 17e9e3ffb571e8f200ee8bf195a3708b2be4792b -r e72b8fd2d28b0a5371d13a77469746d0786496de test/functional/test_history_functions.py --- a/test/functional/test_history_functions.py +++ b/test/functional/test_history_functions.py @@ -82,6 +82,7 @@ .first() assert historyB is not None, "Problem retrieving historyB from database" assert historyA.id == historyB.id, "After the same user logged out and back in, their last used history was not associated with their new session" + def test_005_deleting_histories( self ): """Testing deleting histories""" # Logged in as admin_user @@ -144,6 +145,7 @@ raise AssertionError, "Default permissions were incorrectly deleted from the db for history id %d when it was deleted" % history2.id # Current history is empty self.history_options( user=True ) + def test_010_history_rename( self ): """Testing renaming a history""" # Logged in as admin_user @@ -157,10 +159,12 @@ raise AssertionError, "History id %d deleted when it should not be" % latest_history.id self.rename_history( self.security.encode_id( history3.id ), history3.name, new_name=urllib.quote( 'history 3' ) ) sa_session.refresh( history3 ) + def test_015_history_list( self ): """Testing viewing previously stored active histories""" # Logged in as admin_user self.view_stored_active_histories() + def test_020_share_current_history( self ): """Testing sharing the current history which contains only public datasets""" # Logged in as admin_user @@ -210,10 +214,13 @@ # Test sharing history3 with an invalid user self.share_current_history( 'jack(a)jill.com', strings_displayed_after_submit=[ 'jack(a)jill.com is not a valid Galaxy user.' ] ) + def test_025_delete_shared_current_history( self ): """Testing deleting the current history after it was shared""" # Logged in as admin_user - self.delete_current_history( strings_displayed=[ "History (%s) has been shared with others, unshare it before deleting it." % history3.name ] ) + self.delete_current_history( + strings_displayed=[ "History (%s) has been shared with others, unshare it before deleting it." % history3.name ] ) + def test_030_clone_shared_history( self ): """Testing copying a shared history""" # logged in as admin user @@ -656,7 +663,7 @@ if hda[ 'id' ] == self.security.encode_id( hda_2_bed.id ): return ( not hda[ 'accessible' ] ) return False - self.check_history_json( r'\bhdaJson\s*=\s*(.*);', hda_2_bed_is_inaccessible ) + self.check_history_json( r'\bhdaJSON\s*=\s*(.*);', hda_2_bed_is_inaccessible ) # Admin users can view all datasets ( using the history/view feature ), so make sure 2.bed is accessible to the admin self.logout() @@ -734,7 +741,7 @@ log.info( 'deleting last hda' ) self.delete_history_item( str( latest_hda.id ) ) # check the historyPanel settings.show_deleted for a null json value (no show_deleted in query string) - self.check_history_json( r'\bpage_show_deleted\s*=\s*(.*),', lambda x: x == None ) + self.check_history_json( r'\bshow_deleted\s*:\s*(.*),', lambda x: x == None ) # reload this history with the show_deleted flag set in the query string # the deleted dataset should be there with the proper 'deleted' text @@ -742,7 +749,7 @@ log.info( 'turning show_deleted on' ) #self.visit_url( "%s/history/?show_deleted=True" % self.url ) # check the historyPanel settings.show_deleted for a true json value - self.check_history_json( r'\bpage_show_deleted\s*=\s*(.*),', lambda x: x == True, show_deleted=True ) + self.check_history_json( r'\bshow_deleted\s*:\s*(.*),', lambda x: x == True, show_deleted=True ) # reload this history again with the show_deleted flag set TO FALSE in the query string # make sure the 'history empty' message shows @@ -750,7 +757,7 @@ log.info( 'turning show_deleted off' ) #self.visit_url( "%s/history/?show_deleted=False" % self.url ) # check the historyPanel settings.show_deleted for a false json value - self.check_history_json( r'\bpage_show_deleted\s*=\s*(.*),', lambda x: x == False, show_deleted=False ) + self.check_history_json( r'\bshow_deleted\s*:\s*(.*),', lambda x: x == False, show_deleted=False ) # delete this history self.delete_history( self.security.encode_id( latest_history.id ) ) https://bitbucket.org/galaxy/galaxy-central/commits/d02540589348/ Changeset: d02540589348 User: carlfeberhard Date: 2014-01-09 22:01:23 Summary: Merge Affected #: 7 files diff -r e72b8fd2d28b0a5371d13a77469746d0786496de -r d02540589348a2840f0492c91dbabe441fa2fb26 lib/galaxy/webapps/galaxy/api/tool_shed_repositories.py --- a/lib/galaxy/webapps/galaxy/api/tool_shed_repositories.py +++ b/lib/galaxy/webapps/galaxy/api/tool_shed_repositories.py @@ -19,11 +19,6 @@ log = logging.getLogger( __name__ ) -def default_tool_shed_repository_value_mapper( trans, tool_shed_repository ): - value_mapper={ 'id' : trans.security.encode_id( tool_shed_repository.id ), - 'error_message' : tool_shed_repository.error_message or '' } - return value_mapper - def get_message_for_no_shed_tool_config(): # This Galaxy instance is not configured with a shed-related tool panel configuration file. message = 'The tool_config_file setting in universe_wsgi.ini must include at least one shed tool configuration file name with a <toolbox> ' @@ -48,8 +43,8 @@ :param id: the encoded id of the ToolShedRepository object """ # Example URL: http://localhost:8763/api/tool_shed_repositories/f2db41e1fa331b3e/exported_… - # Since exported workflows are dictionaries with very few attributes that differentiate them from each other, we'll build the - # list based on the following dictionary of those few attributes. + # Since exported workflows are dictionaries with very few attributes that differentiate them from each + # other, we'll build the list based on the following dictionary of those few attributes. exported_workflows = [] repository = suc.get_tool_shed_repository_by_id( trans, id ) metadata = repository.metadata @@ -58,17 +53,23 @@ else: exported_workflow_tups = [] for index, exported_workflow_tup in enumerate( exported_workflow_tups ): - # The exported_workflow_tup looks like ( relative_path, exported_workflow_dict ), where the value of relative_path is the location - # on disk (relative to the root of the installed repository) where the exported_workflow_dict file (.ga file) is located. + # The exported_workflow_tup looks like ( relative_path, exported_workflow_dict ), where the value of + # relative_path is the location on disk (relative to the root of the installed repository) where the + # exported_workflow_dict file (.ga file) is located. exported_workflow_dict = exported_workflow_tup[ 1 ] annotation = exported_workflow_dict.get( 'annotation', '' ) format_version = exported_workflow_dict.get( 'format-version', '' ) workflow_name = exported_workflow_dict.get( 'name', '' ) - # Since we don't have an in-memory object with an id, we'll identify the exported workflow via it's location (i.e., index) in the list. + # Since we don't have an in-memory object with an id, we'll identify the exported workflow via it's + # location (i.e., index) in the list. display_dict = dict( index=index, annotation=annotation, format_version=format_version, workflow_name=workflow_name ) exported_workflows.append( display_dict ) return exported_workflows + def __get_value_mapper( self, trans ): + value_mapper = { 'id' : trans.security.encode_id } + return value_mapper + @web.expose_api def import_workflow( self, trans, payload, **kwd ): """ @@ -96,13 +97,11 @@ # Since we don't have an in-memory object with an id, we'll identify the exported workflow via it's location (i.e., index) in the list. exported_workflow = exported_workflows[ int( index ) ] workflow_name = exported_workflow[ 'workflow_name' ] - workflow, status, message = workflow_util.import_workflow( trans, repository, workflow_name ) + workflow, status, error_message = workflow_util.import_workflow( trans, repository, workflow_name ) if status == 'error': - log.error( message, exc_info=True ) - trans.response.status = 500 - return message - else: - return workflow.to_dict( view='element' ) + log.debug( error_message ) + return {} + return workflow.to_dict( view='element' ) @web.expose_api def import_workflows( self, trans, **kwd ): @@ -125,11 +124,9 @@ imported_workflow_dicts = [] for exported_workflow_dict in exported_workflows: workflow_name = exported_workflow_dict[ 'workflow_name' ] - workflow, status, message = workflow_util.import_workflow( trans, repository, workflow_name ) + workflow, status, error_message = workflow_util.import_workflow( trans, repository, workflow_name ) if status == 'error': - log.error( message, exc_info=True ) - trans.response.status = 500 - return message + log.debug( error_message ) else: imported_workflow_dicts.append( workflow.to_dict( view='element' ) ) return imported_workflow_dicts @@ -142,22 +139,15 @@ """ # Example URL: http://localhost:8763/api/tool_shed_repositories tool_shed_repository_dicts = [] - try: - query = trans.install_model.context.query( trans.app.install_model.ToolShedRepository ) \ - .order_by( trans.app.install_model.ToolShedRepository.table.c.name ) \ - .all() - for tool_shed_repository in query: - tool_shed_repository_dict = tool_shed_repository.to_dict( value_mapper=default_tool_shed_repository_value_mapper( trans, tool_shed_repository ) ) - tool_shed_repository_dict[ 'url' ] = web.url_for( controller='tool_shed_repositories', - action='show', - id=trans.security.encode_id( tool_shed_repository.id ) ) - tool_shed_repository_dicts.append( tool_shed_repository_dict ) - return tool_shed_repository_dicts - except Exception, e: - message = "Error in the tool_shed_repositories API in index: %s" % str( e ) - log.error( message, exc_info=True ) - trans.response.status = 500 - return message + for tool_shed_repository in trans.install_model.context.query( trans.app.install_model.ToolShedRepository ) \ + .order_by( trans.app.install_model.ToolShedRepository.table.c.name ): + tool_shed_repository_dict = \ + tool_shed_repository.to_dict( value_mapper=self.__get_value_mapper( trans ) ) + tool_shed_repository_dict[ 'url' ] = web.url_for( controller='tool_shed_repositories', + action='show', + id=trans.security.encode_id( tool_shed_repository.id ) ) + tool_shed_repository_dicts.append( tool_shed_repository_dict ) + return tool_shed_repository_dicts @web.expose_api def install_repository_revision( self, trans, payload, **kwd ): @@ -208,8 +198,7 @@ # Make sure this Galaxy instance is configured with a shed-related tool panel configuration file. if not suc.have_shed_tool_conf_for_install( trans ): message = get_message_for_no_shed_tool_config() - log.error( message, exc_info=True ) - trans.response.status = 500 + log.debug( message ) return dict( status='error', error=message ) # Make sure the current user's API key proves he is an admin user in this Galaxy instance. if not trans.user_is_admin(): @@ -225,18 +214,20 @@ except Exception, e: message = "Error attempting to retrieve installation information from tool shed %s for revision %s of repository %s owned by %s: %s" % \ ( str( tool_shed_url ), str( changeset_revision ), str( name ), str( owner ), str( e ) ) - log.error( message, exc_info=True ) - trans.response.status = 500 + log.debug( message ) return dict( status='error', error=message ) if raw_text: + # If successful, the response from get_repository_revision_install_info will be 3 + # dictionaries, a dictionary defining the Repository, a dictionary defining the + # Repository revision (RepositoryMetadata), and a dictionary including the additional + # information required to install the repository. items = json.from_json_string( raw_text ) repository_revision_dict = items[ 1 ] repo_info_dict = items[ 2 ] else: message = "Unable to retrieve installation information from tool shed %s for revision %s of repository %s owned by %s: %s" % \ ( str( tool_shed_url ), str( changeset_revision ), str( name ), str( owner ), str( e ) ) - log.error( message, exc_info=True ) - trans.response.status = 500 + log.debug( message ) return dict( status='error', error=message ) repo_info_dicts = [ repo_info_dict ] # Make sure the tool shed returned everything we need for installing the repository. @@ -345,7 +336,7 @@ tool_path, install_tool_dependencies, reinstalling=False ) - tool_shed_repository_dict = tool_shed_repository.as_dict( value_mapper=default_tool_shed_repository_value_mapper( trans, tool_shed_repository ) ) + tool_shed_repository_dict = tool_shed_repository.as_dict( value_mapper=self.__get_value_mapper( trans ) ) tool_shed_repository_dict[ 'url' ] = web.url_for( controller='tool_shed_repositories', action='show', id=trans.security.encode_id( tool_shed_repository.id ) ) @@ -394,8 +385,7 @@ if not suc.have_shed_tool_conf_for_install( trans ): # This Galaxy instance is not configured with a shed-related tool panel configuration file. message = get_message_for_no_shed_tool_config() - log.error( message, exc_info=True ) - trans.response.status = 500 + log.debug( message ) return dict( status='error', error=message ) if not trans.user_is_admin(): raise HTTPForbidden( detail='You are not authorized to install a tool shed repository into this Galaxy instance.' ) @@ -410,8 +400,7 @@ len( changeset_revisions ) != num_specified_repositories: message = 'Error in tool_shed_repositories API in install_repository_revisions: the received parameters must be ordered ' message += 'lists so that positional values in tool_shed_urls, names, owners and changeset_revisions are associated.' - log.error( message, exc_info=True ) - trans.response.status = 500 + log.debug( message ) return dict( status='error', error=message ) # Get the information about the Galaxy components (e.g., tool pane section, tool config file, etc) that will contain information # about each of the repositories being installed. @@ -482,7 +471,7 @@ repair_dict = repository_util.repair_tool_shed_repository( trans, repository, encoding_util.tool_shed_encode( repo_info_dict ) ) - repository_dict = repository.to_dict( value_mapper=default_tool_shed_repository_value_mapper( trans, repository ) ) + repository_dict = repository.to_dict( value_mapper=self.__get_value_mapper( trans ) ) repository_dict[ 'url' ] = web.url_for( controller='tool_shed_repositories', action='show', id=trans.security.encode_id( repository.id ) ) @@ -502,39 +491,39 @@ :param key: the API key of the Galaxy admin user. """ - try: - start_time = strftime( "%Y-%m-%d %H:%M:%S" ) - results = dict( start_time=start_time, - successful_count=0, - unsuccessful_count=0, - repository_status=[] ) - # Make sure the current user's API key proves he is an admin user in this Galaxy instance. - if not trans.user_is_admin(): - raise HTTPForbidden( detail='You are not authorized to reset metadata on repositories installed into this Galaxy instance.' ) - query = suc.get_query_for_setting_metadata_on_repositories( trans, my_writable=False, order=False ) - # Now reset metadata on all remaining repositories. - for repository in query: - repository_id = trans.security.encode_id( repository.id ) - try: - invalid_file_tups, metadata_dict = metadata_util.reset_all_metadata_on_installed_repository( trans, repository_id ) - if invalid_file_tups: - message = tool_util.generate_message_for_invalid_tools( trans, invalid_file_tups, repository, None, as_html=False ) - results[ 'unsuccessful_count' ] += 1 - else: - message = "Successfully reset metadata on repository %s owned by %s" % ( str( repository.name ), str( repository.owner ) ) - results[ 'successful_count' ] += 1 - except Exception, e: - message = "Error resetting metadata on repository %s owned by %s: %s" % ( str( repository.name ), str( repository.owner ), str( e ) ) + start_time = strftime( "%Y-%m-%d %H:%M:%S" ) + results = dict( start_time=start_time, + successful_count=0, + unsuccessful_count=0, + repository_status=[] ) + # Make sure the current user's API key proves he is an admin user in this Galaxy instance. + if not trans.user_is_admin(): + raise HTTPForbidden( detail='You are not authorized to reset metadata on repositories installed into this Galaxy instance.' ) + query = suc.get_query_for_setting_metadata_on_repositories( trans, my_writable=False, order=False ) + # Now reset metadata on all remaining repositories. + for repository in query: + repository_id = trans.security.encode_id( repository.id ) + try: + invalid_file_tups, metadata_dict = metadata_util.reset_all_metadata_on_installed_repository( trans, repository_id ) + if invalid_file_tups: + message = tool_util.generate_message_for_invalid_tools( trans, + invalid_file_tups, + repository, + None, + as_html=False ) results[ 'unsuccessful_count' ] += 1 - results[ 'repository_status' ].append( message ) - stop_time = strftime( "%Y-%m-%d %H:%M:%S" ) - results[ 'stop_time' ] = stop_time - return json.to_json_string( results, sort_keys=True, indent=4 * ' ' ) - except Exception, e: - message = "Error in the Galaxy tool_shed_repositories API in reset_metadata_on_installed_repositories: %s" % str( e ) - log.error( message, exc_info=True ) - trans.response.status = 500 - return message + else: + message = "Successfully reset metadata on repository %s owned by %s" % \ + ( str( repository.name ), str( repository.owner ) ) + results[ 'successful_count' ] += 1 + except Exception, e: + message = "Error resetting metadata on repository %s owned by %s: %s" % \ + ( str( repository.name ), str( repository.owner ), str( e ) ) + results[ 'unsuccessful_count' ] += 1 + results[ 'repository_status' ].append( message ) + stop_time = strftime( "%Y-%m-%d %H:%M:%S" ) + results[ 'stop_time' ] = stop_time + return json.to_json_string( results, sort_keys=True, indent=4 * ' ' ) @web.expose_api def show( self, trans, id, **kwd ): @@ -545,15 +534,12 @@ :param id: the encoded id of the ToolShedRepository object """ # Example URL: http://localhost:8763/api/tool_shed_repositories/df7a1f0c02a5b08e - try: - tool_shed_repository = suc.get_tool_shed_repository_by_id( trans, id ) - tool_shed_repository_dict = tool_shed_repository.as_dict( value_mapper=default_tool_shed_repository_value_mapper( trans, tool_shed_repository ) ) - tool_shed_repository_dict[ 'url' ] = web.url_for( controller='tool_shed_repositories', - action='show', - id=trans.security.encode_id( tool_shed_repository.id ) ) - return tool_shed_repository_dict - except Exception, e: - message = "Error in tool_shed_repositories API in index: " + str( e ) - log.error( message, exc_info=True ) - trans.response.status = 500 - return message + tool_shed_repository = suc.get_tool_shed_repository_by_id( trans, id ) + if tool_shed_repository is None: + log.debug( "Unable to locate tool_shed_repository record for id %s." % ( str( id ) ) ) + return {} + tool_shed_repository_dict = tool_shed_repository.as_dict( value_mapper=self.__get_value_mapper( trans ) ) + tool_shed_repository_dict[ 'url' ] = web.url_for( controller='tool_shed_repositories', + action='show', + id=trans.security.encode_id( tool_shed_repository.id ) ) + return tool_shed_repository_dict diff -r e72b8fd2d28b0a5371d13a77469746d0786496de -r d02540589348a2840f0492c91dbabe441fa2fb26 lib/galaxy/webapps/tool_shed/api/repositories.py --- a/lib/galaxy/webapps/tool_shed/api/repositories.py +++ b/lib/galaxy/webapps/tool_shed/api/repositories.py @@ -36,22 +36,27 @@ :param name: the name of the Repository :param owner: the owner of the Repository - Returns the ordered list of changeset revision hash strings that are associated with installable revisions. As in the changelog, the - list is ordered oldest to newest. + Returns the ordered list of changeset revision hash strings that are associated with installable revisions. + As in the changelog, the list is ordered oldest to newest. """ # Example URL: http://localhost:9009/api/repositories/get_installable_revisions?name=add_c… - try: + if name and owner: # Get the repository information. repository = suc.get_repository_by_name_and_owner( trans.app, name, owner ) + if repository is None: + error_message = "Error in the Tool Shed repositories API in get_ordered_installable_revisions: " + error_message += "cannot locate repository %s owned by %s." % ( str( name ), str( owner ) ) + log.debug( error_message ) + return [] repo_dir = repository.repo_path( trans.app ) repo = hg.repository( suc.get_configured_ui(), repo_dir ) ordered_installable_revisions = suc.get_ordered_metadata_changeset_revisions( repository, repo, downloadable=True ) return ordered_installable_revisions - except Exception, e: - message = "Error in the Tool Shed repositories API in get_ordered_installable_revisions: %s" % str( e ) - log.error( message, exc_info=True ) - trans.response.status = 500 - return message + else: + error_message = "Error in the Tool Shed repositories API in get_ordered_installable_revisions: " + error_message += "invalid name %s or owner %s received." % ( str( name ), str( owner ) ) + log.debug( error_message ) + return [] @web.expose_api_anonymous def get_repository_revision_install_info( self, trans, name, owner, changeset_revision, **kwd ): @@ -106,49 +111,65 @@ ] } """ - repository_value_mapper = { 'id' : trans.security.encode_id, - 'user_id' : trans.security.encode_id } - # Example URL: http://localhost:9009/api/repositories/get_repository_revision_install_info… - try: + # Example URL: + # http://<xyz>/api/repositories/get_repository_revision_install_info?name=<n>&owner=<o>&changeset_revision=<cr> + if name and owner and changeset_revision: # Get the repository information. repository = suc.get_repository_by_name_and_owner( trans.app, name, owner ) + if repository is None: + log.debug( 'Cannot locate repository %s owned by %s' % ( str( name ), str( owner ) ) ) + return {}, {}, {} encoded_repository_id = trans.security.encode_id( repository.id ) - repository_dict = repository.to_dict( view='element', value_mapper=repository_value_mapper ) + repository_dict = repository.to_dict( view='element', + value_mapper=self.__get_value_mapper( trans ) ) repository_dict[ 'url' ] = web.url_for( controller='repositories', action='show', id=encoded_repository_id ) # Get the repository_metadata information. - repository_metadata = suc.get_repository_metadata_by_changeset_revision( trans, encoded_repository_id, changeset_revision ) + repository_metadata = suc.get_repository_metadata_by_changeset_revision( trans, + encoded_repository_id, + changeset_revision ) if not repository_metadata: - # The changeset_revision column in the repository_metadata table has been updated with a new value value, so find the - # changeset_revision to which we need to update. + # The changeset_revision column in the repository_metadata table has been updated with a new + # value value, so find the changeset_revision to which we need to update. repo_dir = repository.repo_path( trans.app ) repo = hg.repository( suc.get_configured_ui(), repo_dir ) new_changeset_revision = suc.get_next_downloadable_changeset_revision( repository, repo, changeset_revision ) - repository_metadata = suc.get_repository_metadata_by_changeset_revision( trans, encoded_repository_id, new_changeset_revision ) + repository_metadata = suc.get_repository_metadata_by_changeset_revision( trans, + encoded_repository_id, + new_changeset_revision ) changeset_revision = new_changeset_revision if repository_metadata: encoded_repository_metadata_id = trans.security.encode_id( repository_metadata.id ) repository_metadata_dict = repository_metadata.to_dict( view='collection', - value_mapper=self.__get_value_mapper( trans, repository_metadata ) ) + value_mapper=self.__get_value_mapper( trans ) ) repository_metadata_dict[ 'url' ] = web.url_for( controller='repository_revisions', action='show', id=encoded_repository_metadata_id ) # Get the repo_info_dict for installing the repository. - repo_info_dict, includes_tools, includes_tool_dependencies, includes_tools_for_display_in_tool_panel, \ - has_repository_dependencies, has_repository_dependencies_only_if_compiling_contained_td = \ + repo_info_dict, \ + includes_tools, \ + includes_tool_dependencies, \ + includes_tools_for_display_in_tool_panel, \ + has_repository_dependencies, \ + has_repository_dependencies_only_if_compiling_contained_td = \ repository_util.get_repo_info_dict( trans, encoded_repository_id, changeset_revision ) return repository_dict, repository_metadata_dict, repo_info_dict else: - message = "Unable to locate repository_metadata record for repository id %d and changeset_revision %s" % ( repository.id, changeset_revision ) - log.error( message, exc_info=True ) - trans.response.status = 500 + log.debug( "Unable to locate repository_metadata record for repository id %s and changeset_revision %s" % \ + ( str( repository.id ), str( changeset_revision ) ) ) return repository_dict, {}, {} - except Exception, e: - message = "Error in the Tool Shed repositories API in get_repository_revision_install_info: %s" % str( e ) - log.error( message, exc_info=True ) - trans.response.status = 500 - return message + else: + debug_msg = "Error in the Tool Shed repositories API in get_repository_revision_install_info: " + debug_msg += "Invalid name %s or owner %s or changeset_revision %s received." % \ + ( str( name ), str( owner ), str( changeset_revision ) ) + log.debug( debug_msg ) + return {}, {}, {} + + def __get_value_mapper( self, trans ): + value_mapper = { 'id' : trans.security.encode_id, + 'repository_id' : trans.security.encode_id } + return value_mapper @web.expose_api def import_capsule( self, trans, payload, **kwd ): @@ -177,29 +198,27 @@ uploaded_file=None, capsule_file_name=None ) if os.path.getsize( os.path.abspath( capsule_file_name ) ) == 0: - message = 'Your capsule file is empty.' - log.error( message, exc_info=True ) - trans.response.status = 500 - return message + log.debug( 'Your capsule file %s is empty.' % str( capsule_file_name ) ) + return {} try: # Open for reading with transparent compression. tar_archive = tarfile.open( capsule_file_path, 'r:*' ) except tarfile.ReadError, e: - message = 'Error opening file %s: %s' % ( str( capsule_file_name ), str( e ) ) - log.error( message, exc_info=True ) - trans.response.status = 500 - return message + log.debug( 'Error opening capsule file %s: %s' % ( str( capsule_file_name ), str( e ) ) ) + return {} capsule_dict[ 'tar_archive' ] = tar_archive capsule_dict[ 'capsule_file_name' ] = capsule_file_name capsule_dict = import_util.extract_capsule_files( trans, **capsule_dict ) capsule_dict = import_util.validate_capsule( trans, **capsule_dict ) status = capsule_dict.get( 'status', 'error' ) if status == 'error': - message = 'The capsule contents are invalid and cannpt be imported:<br/>%s' % str( capsule_dict.get( 'error_message', '' ) ) - log.error( message, exc_info=True ) - trans.response.status = 500 - return message + log.debug( 'The capsule contents are invalid and cannpt be imported:<br/>%s' % \ + str( capsule_dict.get( 'error_message', '' ) ) ) + return {} encoded_file_path = capsule_dict.get( 'encoded_file_path', None ) + if encoded_file_path is None: + log.debug( 'The capsule_dict %s is missing the required encoded_file_path entry.' % str( capsule_dict ) ) + return {} file_path = encoding_util.tool_shed_decode( encoded_file_path ) export_info_file_path = os.path.join( file_path, 'export_info.xml' ) export_info_dict = import_util.get_export_info_dict( export_info_file_path ) @@ -216,12 +235,14 @@ # Add the capsule_file_name and encoded_file_path to the repository_status_info_dict. repository_status_info_dict[ 'capsule_file_name' ] = capsule_file_name repository_status_info_dict[ 'encoded_file_path' ] = encoded_file_path - import_results_tups = repository_maintenance_util.create_repository_and_import_archive( trans, - repository_status_info_dict, - import_results_tups ) + import_results_tups = \ + repository_maintenance_util.create_repository_and_import_archive( trans, + repository_status_info_dict, + import_results_tups ) import_util.check_status_and_reset_downloadable( trans, import_results_tups ) suc.remove_dir( file_path ) - # NOTE: the order of installation is defined in import_results_tups, but order will be lost when transferred to return_dict. + # NOTE: the order of installation is defined in import_results_tups, but order will be lost + # when transferred to return_dict. return_dict = {} for import_results_tup in import_results_tups: ok, name_owner, message = import_results_tup @@ -237,28 +258,19 @@ GET /api/repositories Displays a collection (list) of repositories. """ - value_mapper = { 'id' : trans.security.encode_id, - 'user_id' : trans.security.encode_id } # Example URL: http://localhost:9009/api/repositories repository_dicts = [] - deleted = util.string_as_bool( deleted ) - try: - query = trans.sa_session.query( trans.app.model.Repository ) \ - .filter( trans.app.model.Repository.table.c.deleted == deleted ) \ - .order_by( trans.app.model.Repository.table.c.name ) \ - .all() - for repository in query: - repository_dict = repository.to_dict( view='collection', value_mapper=value_mapper ) - repository_dict[ 'url' ] = web.url_for( controller='repositories', - action='show', - id=trans.security.encode_id( repository.id ) ) - repository_dicts.append( repository_dict ) - return repository_dicts - except Exception, e: - message = "Error in the Tool Shed repositories API in index: %s" % str( e ) - log.error( message, exc_info=True ) - trans.response.status = 500 - return message + deleted = util.asbool( deleted ) + for repository in trans.sa_session.query( trans.app.model.Repository ) \ + .filter( trans.app.model.Repository.table.c.deleted == deleted ) \ + .order_by( trans.app.model.Repository.table.c.name ): + repository_dict = repository.to_dict( view='collection', + value_mapper=self.__get_value_mapper( trans ) ) + repository_dict[ 'url' ] = web.url_for( controller='repositories', + action='show', + id=trans.security.encode_id( repository.id ) ) + repository_dicts.append( repository_dict ) + return repository_dicts @web.expose_api def repository_ids_for_setting_metadata( self, trans, my_writable=False, **kwd ): @@ -273,28 +285,22 @@ in addition to those repositories of type tool_dependency_definition. This param is ignored if the current user is not an admin user, in which case this same restriction is automatic. """ - try: - if trans.user_is_admin(): - my_writable = util.asbool( my_writable ) - else: - my_writable = True - handled_repository_ids = [] - repository_ids = [] - query = suc.get_query_for_setting_metadata_on_repositories( trans, my_writable=my_writable, order=False ) - # Make sure repositories of type tool_dependency_definition are first in the list. - for repository in query: - if repository.type == rt_util.TOOL_DEPENDENCY_DEFINITION and repository.id not in handled_repository_ids: - repository_ids.append( trans.security.encode_id( repository.id ) ) - # Now add all remaining repositories to the list. - for repository in query: - if repository.type != rt_util.TOOL_DEPENDENCY_DEFINITION and repository.id not in handled_repository_ids: - repository_ids.append( trans.security.encode_id( repository.id ) ) - return repository_ids - except Exception, e: - message = "Error in the Tool Shed repositories API in repository_ids_for_setting_metadata: %s" % str( e ) - log.error( message, exc_info=True ) - trans.response.status = 500 - return message + if trans.user_is_admin(): + my_writable = util.asbool( my_writable ) + else: + my_writable = True + handled_repository_ids = [] + repository_ids = [] + query = suc.get_query_for_setting_metadata_on_repositories( trans, my_writable=my_writable, order=False ) + # Make sure repositories of type tool_dependency_definition are first in the list. + for repository in query: + if repository.type == rt_util.TOOL_DEPENDENCY_DEFINITION and repository.id not in handled_repository_ids: + repository_ids.append( trans.security.encode_id( repository.id ) ) + # Now add all remaining repositories to the list. + for repository in query: + if repository.type != rt_util.TOOL_DEPENDENCY_DEFINITION and repository.id not in handled_repository_ids: + repository_ids.append( trans.security.encode_id( repository.id ) ) + return repository_ids @web.expose_api def reset_metadata_on_repositories( self, trans, payload, **kwd ): @@ -318,6 +324,7 @@ :param skip_file (optional): A local file name that contains the encoded repository ids associated with repositories to skip. This param can be used as an alternative to the above encoded_ids_to_skip. """ + def handle_repository( trans, repository, results ): log.debug( "Resetting metadata on repository %s" % str( repository.name ) ) repository_id = trans.security.encode_id( repository.id ) @@ -335,53 +342,48 @@ status = '%s : %s' % ( str( repository.name ), message ) results[ 'repository_status' ].append( status ) return results - try: - start_time = strftime( "%Y-%m-%d %H:%M:%S" ) - results = dict( start_time=start_time, - repository_status=[], - successful_count=0, - unsuccessful_count=0 ) - handled_repository_ids = [] - encoded_ids_to_skip = payload.get( 'encoded_ids_to_skip', [] ) - skip_file = payload.get( 'skip_file', None ) - if skip_file and os.path.exists( skip_file ) and not encoded_ids_to_skip: - # Load the list of encoded_ids_to_skip from the skip_file. - # Contents of file must be 1 encoded repository id per line. - lines = open( skip_file, 'rb' ).readlines() - for line in lines: - if line.startswith( '#' ): - # Skip comments. - continue - encoded_ids_to_skip.append( line.rstrip( '\n' ) ) - if trans.user_is_admin(): - my_writable = util.asbool( payload.get( 'my_writable', False ) ) - else: - my_writable = True - query = suc.get_query_for_setting_metadata_on_repositories( trans, my_writable=my_writable, order=False ) - # First reset metadata on all repositories of type repository_dependency_definition. - for repository in query: - encoded_id = trans.security.encode_id( repository.id ) - if encoded_id in encoded_ids_to_skip: - log.debug( "Skipping repository with id %s because it is in encoded_ids_to_skip %s" % \ - ( str( repository.id ), str( encoded_ids_to_skip ) ) ) - elif repository.type == rt_util.TOOL_DEPENDENCY_DEFINITION and repository.id not in handled_repository_ids: - results = handle_repository( trans, repository, results ) - # Now reset metadata on all remaining repositories. - for repository in query: - encoded_id = trans.security.encode_id( repository.id ) - if encoded_id in encoded_ids_to_skip: - log.debug( "Skipping repository with id %s because it is in encoded_ids_to_skip %s" % \ - ( str( repository.id ), str( encoded_ids_to_skip ) ) ) - elif repository.type != rt_util.TOOL_DEPENDENCY_DEFINITION and repository.id not in handled_repository_ids: - results = handle_repository( trans, repository, results ) - stop_time = strftime( "%Y-%m-%d %H:%M:%S" ) - results[ 'stop_time' ] = stop_time - return json.to_json_string( results, sort_keys=True, indent=4 * ' ' ) - except Exception, e: - message = "Error in the Tool Shed repositories API in reset_metadata_on_repositories: %s" % str( e ) - log.error( message, exc_info=True ) - trans.response.status = 500 - return message + + start_time = strftime( "%Y-%m-%d %H:%M:%S" ) + results = dict( start_time=start_time, + repository_status=[], + successful_count=0, + unsuccessful_count=0 ) + handled_repository_ids = [] + encoded_ids_to_skip = payload.get( 'encoded_ids_to_skip', [] ) + skip_file = payload.get( 'skip_file', None ) + if skip_file and os.path.exists( skip_file ) and not encoded_ids_to_skip: + # Load the list of encoded_ids_to_skip from the skip_file. + # Contents of file must be 1 encoded repository id per line. + lines = open( skip_file, 'rb' ).readlines() + for line in lines: + if line.startswith( '#' ): + # Skip comments. + continue + encoded_ids_to_skip.append( line.rstrip( '\n' ) ) + if trans.user_is_admin(): + my_writable = util.asbool( payload.get( 'my_writable', False ) ) + else: + my_writable = True + query = suc.get_query_for_setting_metadata_on_repositories( trans, my_writable=my_writable, order=False ) + # First reset metadata on all repositories of type repository_dependency_definition. + for repository in query: + encoded_id = trans.security.encode_id( repository.id ) + if encoded_id in encoded_ids_to_skip: + log.debug( "Skipping repository with id %s because it is in encoded_ids_to_skip %s" % \ + ( str( repository.id ), str( encoded_ids_to_skip ) ) ) + elif repository.type == rt_util.TOOL_DEPENDENCY_DEFINITION and repository.id not in handled_repository_ids: + results = handle_repository( trans, repository, results ) + # Now reset metadata on all remaining repositories. + for repository in query: + encoded_id = trans.security.encode_id( repository.id ) + if encoded_id in encoded_ids_to_skip: + log.debug( "Skipping repository with id %s because it is in encoded_ids_to_skip %s" % \ + ( str( repository.id ), str( encoded_ids_to_skip ) ) ) + elif repository.type != rt_util.TOOL_DEPENDENCY_DEFINITION and repository.id not in handled_repository_ids: + results = handle_repository( trans, repository, results ) + stop_time = strftime( "%Y-%m-%d %H:%M:%S" ) + results[ 'stop_time' ] = stop_time + return json.to_json_string( results, sort_keys=True, indent=4 * ' ' ) @web.expose_api def reset_metadata_on_repository( self, trans, payload, **kwd ): @@ -395,6 +397,7 @@ The following parameters must be included in the payload. :param repository_id: the encoded id of the repository on which metadata is to be reset. """ + def handle_repository( trans, start_time, repository ): results = dict( start_time=start_time, repository_status=[] ) @@ -410,21 +413,16 @@ status = '%s : %s' % ( str( repository.name ), message ) results[ 'repository_status' ].append( status ) return results - try: - repository_id = payload.get( 'repository_id', None ) - if repository_id is not None: - repository = suc.get_repository_in_tool_shed( trans, repository_id ) - start_time = strftime( "%Y-%m-%d %H:%M:%S" ) - log.debug( "%s...resetting metadata on repository %s" % ( start_time, str( repository.name ) ) ) - results = handle_repository( trans, start_time, repository ) - stop_time = strftime( "%Y-%m-%d %H:%M:%S" ) - results[ 'stop_time' ] = stop_time - return json.to_json_string( results, sort_keys=True, indent=4 * ' ' ) - except Exception, e: - message = "Error in the Tool Shed repositories API in reset_metadata_on_repositories: %s" % str( e ) - log.error( message, exc_info=True ) - trans.response.status = 500 - return message + + repository_id = payload.get( 'repository_id', None ) + if repository_id is not None: + repository = suc.get_repository_in_tool_shed( trans, repository_id ) + start_time = strftime( "%Y-%m-%d %H:%M:%S" ) + log.debug( "%s...resetting metadata on repository %s" % ( start_time, str( repository.name ) ) ) + results = handle_repository( trans, start_time, repository ) + stop_time = strftime( "%Y-%m-%d %H:%M:%S" ) + results[ 'stop_time' ] = stop_time + return json.to_json_string( results, sort_keys=True, indent=4 * ' ' ) @web.expose_api_anonymous def show( self, trans, id, **kwd ): @@ -434,27 +432,14 @@ :param id: the encoded id of the Repository object """ - value_mapper = { 'id' : trans.security.encode_id, - 'user_id' : trans.security.encode_id } # Example URL: http://localhost:9009/api/repositories/f9cad7b01a472135 - try: - repository = suc.get_repository_in_tool_shed( trans, id ) - repository_dict = repository.to_dict( view='element', value_mapper=value_mapper ) - repository_dict[ 'url' ] = web.url_for( controller='repositories', - action='show', - id=trans.security.encode_id( repository.id ) ) - return repository_dict - except Exception, e: - message = "Error in the Tool Shed repositories API in show: %s" % str( e ) - log.error( message, exc_info=True ) - trans.response.status = 500 - return message - - def __get_value_mapper( self, trans, repository_metadata ): - value_mapper = { 'id' : trans.security.encode_id, - 'repository_id' : trans.security.encode_id } - if repository_metadata.time_last_tested is not None: - # For some reason the Dictifiable.to_dict() method in ~/galaxy/model/item_attrs.py requires - # a function rather than a mapped value, so just pass the time_ago function here. - value_mapper[ 'time_last_tested' ] = time_ago - return value_mapper + repository = suc.get_repository_in_tool_shed( trans, id ) + if repository is None: + log.debug( "Unable to locate repository record for id %s." % ( str( id ) ) ) + return {} + repository_dict = repository.to_dict( view='element', + value_mapper=self.__get_value_mapper( trans ) ) + repository_dict[ 'url' ] = web.url_for( controller='repositories', + action='show', + id=trans.security.encode_id( repository.id ) ) + return repository_dict diff -r e72b8fd2d28b0a5371d13a77469746d0786496de -r d02540589348a2840f0492c91dbabe441fa2fb26 lib/galaxy/webapps/tool_shed/api/repository_revisions.py --- a/lib/galaxy/webapps/tool_shed/api/repository_revisions.py +++ b/lib/galaxy/webapps/tool_shed/api/repository_revisions.py @@ -47,65 +47,29 @@ if not changeset_revision: raise HTTPBadRequest( detail="Missing required parameter 'changeset_revision'." ) export_repository_dependencies = payload.get( 'export_repository_dependencies', False ) - try: - # We'll currently support only gzip-compressed tar archives. - file_type = 'gz' - export_repository_dependencies = util.string_as_bool( export_repository_dependencies ) - # Get the repository information. - repository = suc.get_repository_by_name_and_owner( trans.app, name, owner ) - repository_id = trans.security.encode_id( repository.id ) - response = export_util.export_repository( trans, - tool_shed_url, - repository_id, - str( repository.name ), - changeset_revision, - file_type, - export_repository_dependencies, - api=True ) - return response - except Exception, e: - message = "Error in the Tool Shed repository_revisions API in export: %s" % str( e ) - log.error( message, exc_info=True ) - trans.response.status = 500 - return message + # We'll currently support only gzip-compressed tar archives. + file_type = 'gz' + export_repository_dependencies = util.asbool( export_repository_dependencies ) + # Get the repository information. + repository = suc.get_repository_by_name_and_owner( trans.app, name, owner ) + if repository is None: + error_message = 'Cannot locate repository with name %s and owner %s,' % ( str( name ), str( owner ) ) + log.debug( error_message ) + return None, error_message + repository_id = trans.security.encode_id( repository.id ) + return export_util.export_repository( trans, + tool_shed_url, + repository_id, + str( repository.name ), + changeset_revision, + file_type, + export_repository_dependencies, + api=True ) - @web.expose_api_anonymous - def repository_dependencies( self, trans, id, **kwd ): - """ - GET /api/repository_revisions/{encoded repository_metadata id}/repository_dependencies - Displays information about a repository_metadata record in the Tool Shed. - - :param id: the encoded id of the `RepositoryMetadata` object - """ - # Example URL: http://localhost:9009/api/repository_revisions/repository_dependencies/bb12… + def __get_value_mapper( self, trans ): value_mapper = { 'id' : trans.security.encode_id, - 'user_id' : trans.security.encode_id } - repository_dependencies_dicts = [] - try: - repository_metadata = metadata_util.get_repository_metadata_by_id( trans, id ) - metadata = repository_metadata.metadata - if metadata and 'repository_dependencies' in metadata: - rd_tups = metadata[ 'repository_dependencies' ][ 'repository_dependencies' ] - for rd_tup in rd_tups: - tool_shed, name, owner, changeset_revision = rd_tup[ 0:4 ] - repository_dependency = suc.get_repository_by_name_and_owner( trans.app, name, owner ) - repository_dependency_id = trans.security.encode_id( repository_dependency.id ) - repository_dependency_repository_metadata = \ - suc.get_repository_metadata_by_changeset_revision( trans, repository_dependency_id, changeset_revision ) - repository_dependency_repository_metadata_id = trans.security.encode_id( repository_dependency_repository_metadata.id ) - repository_dependency_dict = repository_dependency.to_dict( view='element', value_mapper=value_mapper ) - # We have to add the changeset_revision of of the repository dependency. - repository_dependency_dict[ 'changeset_revision' ] = changeset_revision - repository_dependency_dict[ 'url' ] = web.url_for( controller='repositories', - action='show', - id=repository_dependency_repository_metadata_id ) - repository_dependencies_dicts.append( repository_dependency_dict ) - return repository_dependencies_dicts - except Exception, e: - message = "Error in the Tool Shed repository_revisions API in repository_dependencies: %s" % str( e ) - log.error( message, exc_info=True ) - trans.response.status = 500 - return message + 'repository_id' : trans.security.encode_id } + return value_mapper @web.expose_api_anonymous def index( self, trans, **kwd ): @@ -120,59 +84,94 @@ # Filter by downloadable if received. downloadable = kwd.get( 'downloadable', None ) if downloadable is not None: - clause_list.append( trans.model.RepositoryMetadata.table.c.downloadable == util.string_as_bool( downloadable ) ) + clause_list.append( trans.model.RepositoryMetadata.table.c.downloadable == util.asbool( downloadable ) ) # Filter by malicious if received. malicious = kwd.get( 'malicious', None ) if malicious is not None: - clause_list.append( trans.model.RepositoryMetadata.table.c.malicious == util.string_as_bool( malicious ) ) + clause_list.append( trans.model.RepositoryMetadata.table.c.malicious == util.asbool( malicious ) ) # Filter by tools_functionally_correct if received. tools_functionally_correct = kwd.get( 'tools_functionally_correct', None ) if tools_functionally_correct is not None: - clause_list.append( trans.model.RepositoryMetadata.table.c.tools_functionally_correct == util.string_as_bool( tools_functionally_correct ) ) + clause_list.append( trans.model.RepositoryMetadata.table.c.tools_functionally_correct == util.asbool( tools_functionally_correct ) ) # Filter by missing_test_components if received. missing_test_components = kwd.get( 'missing_test_components', None ) if missing_test_components is not None: - clause_list.append( trans.model.RepositoryMetadata.table.c.missing_test_components == util.string_as_bool( missing_test_components ) ) + clause_list.append( trans.model.RepositoryMetadata.table.c.missing_test_components == util.asbool( missing_test_components ) ) # Filter by do_not_test if received. do_not_test = kwd.get( 'do_not_test', None ) if do_not_test is not None: - clause_list.append( trans.model.RepositoryMetadata.table.c.do_not_test == util.string_as_bool( do_not_test ) ) + clause_list.append( trans.model.RepositoryMetadata.table.c.do_not_test == util.asbool( do_not_test ) ) # Filter by includes_tools if received. includes_tools = kwd.get( 'includes_tools', None ) if includes_tools is not None: - clause_list.append( trans.model.RepositoryMetadata.table.c.includes_tools == util.string_as_bool( includes_tools ) ) + clause_list.append( trans.model.RepositoryMetadata.table.c.includes_tools == util.asbool( includes_tools ) ) # Filter by test_install_error if received. test_install_error = kwd.get( 'test_install_error', None ) if test_install_error is not None: - clause_list.append( trans.model.RepositoryMetadata.table.c.test_install_error == util.string_as_bool( test_install_error ) ) + clause_list.append( trans.model.RepositoryMetadata.table.c.test_install_error == util.asbool( test_install_error ) ) # Filter by skip_tool_test if received. skip_tool_test = kwd.get( 'skip_tool_test', None ) if skip_tool_test is not None: - skip_tool_test = util.string_as_bool( skip_tool_test ) + skip_tool_test = util.asbool( skip_tool_test ) skipped_metadata_ids_subquery = select( [ trans.app.model.SkipToolTest.table.c.repository_metadata_id ] ) if skip_tool_test: clause_list.append( trans.model.RepositoryMetadata.id.in_( skipped_metadata_ids_subquery ) ) else: clause_list.append( not_( trans.model.RepositoryMetadata.id.in_( skipped_metadata_ids_subquery ) ) ) - # Generate and execute the query. - try: - query = trans.sa_session.query( trans.app.model.RepositoryMetadata ) \ - .filter( and_( *clause_list ) ) \ - .order_by( trans.app.model.RepositoryMetadata.table.c.repository_id.desc() ) \ - .all() - for repository_metadata in query: - repository_metadata_dict = repository_metadata.to_dict( view='collection', - value_mapper=self.__get_value_mapper( trans, repository_metadata ) ) - repository_metadata_dict[ 'url' ] = web.url_for( controller='repository_revisions', - action='show', - id=trans.security.encode_id( repository_metadata.id ) ) - repository_metadata_dicts.append( repository_metadata_dict ) - return repository_metadata_dicts - except Exception, e: - message = "Error in the Tool Shed repository_revisions API in index: " + str( e ) - log.error( message, exc_info=True ) - trans.response.status = 500 - return message + for repository_metadata in trans.sa_session.query( trans.app.model.RepositoryMetadata ) \ + .filter( and_( *clause_list ) ) \ + .order_by( trans.app.model.RepositoryMetadata.table.c.repository_id.desc() ): + repository_metadata_dict = repository_metadata.to_dict( view='collection', + value_mapper=self.__get_value_mapper( trans ) ) + repository_metadata_dict[ 'url' ] = web.url_for( controller='repository_revisions', + action='show', + id=trans.security.encode_id( repository_metadata.id ) ) + repository_metadata_dicts.append( repository_metadata_dict ) + return repository_metadata_dicts + + @web.expose_api_anonymous + def repository_dependencies( self, trans, id, **kwd ): + """ + GET /api/repository_revisions/{encoded repository_metadata id}/repository_dependencies + Displays information about a repository_metadata record in the Tool Shed. + + :param id: the encoded id of the `RepositoryMetadata` object + """ + # Example URL: http://localhost:9009/api/repository_revisions/repository_dependencies/bb12… + repository_dependencies_dicts = [] + repository_metadata = metadata_util.get_repository_metadata_by_id( trans, id ) + if repository_metadata is None: + log.debug( 'Invalid repository_metadata id received: %s' % str( id ) ) + return repository_dependencies_dicts + metadata = repository_metadata.metadata + if metadata is None: + log.debug( 'The repository_metadata record with id %s has no metadata.' % str ( id ) ) + return repository_dependencies_dicts + if 'repository_dependencies' in metadata: + rd_tups = metadata[ 'repository_dependencies' ][ 'repository_dependencies' ] + for rd_tup in rd_tups: + tool_shed, name, owner, changeset_revision = rd_tup[ 0:4 ] + repository_dependency = suc.get_repository_by_name_and_owner( trans.app, name, owner ) + if repository_dependency is None: + log.dbug( 'Cannot locate repository dependency %s owned by %s.' % ( name, owner ) ) + continue + repository_dependency_id = trans.security.encode_id( repository_dependency.id ) + repository_dependency_repository_metadata = \ + suc.get_repository_metadata_by_changeset_revision( trans, repository_dependency_id, changeset_revision ) + if repository_dependency_repository_metadata is None: + log.debug( 'Cannot locate repository_metadata with id %s for repository dependency %s owned by %s.' % \ + ( str( repository_dependency_id ), str( name ), str( owner ) ) ) + continue + repository_dependency_repository_metadata_id = trans.security.encode_id( repository_dependency_repository_metadata.id ) + repository_dependency_dict = repository_dependency.to_dict( view='element', + value_mapper=self.__get_value_mapper( trans ) ) + # We have to add the changeset_revision of of the repository dependency. + repository_dependency_dict[ 'changeset_revision' ] = changeset_revision + repository_dependency_dict[ 'url' ] = web.url_for( controller='repositories', + action='show', + id=repository_dependency_repository_metadata_id ) + repository_dependencies_dicts.append( repository_dependency_dict ) + return repository_dependencies_dicts @web.expose_api_anonymous def show( self, trans, id, **kwd ): @@ -183,19 +182,16 @@ :param id: the encoded id of the `RepositoryMetadata` object """ # Example URL: http://localhost:9009/api/repository_revisions/bb125606ff9ea620 - try: - repository_metadata = metadata_util.get_repository_metadata_by_id( trans, id ) - repository_metadata_dict = repository_metadata.to_dict( view='element', - value_mapper=self.__get_value_mapper( trans, repository_metadata ) ) - repository_metadata_dict[ 'url' ] = web.url_for( controller='repository_revisions', - action='show', - id=trans.security.encode_id( repository_metadata.id ) ) - return repository_metadata_dict - except Exception, e: - message = "Error in the Tool Shed repository_revisions API in show: %s" % str( e ) - log.error( message, exc_info=True ) - trans.response.status = 500 - return message + repository_metadata = metadata_util.get_repository_metadata_by_id( trans, id ) + if repository_metadata is None: + log.debug( 'Cannot locate repository_metadata with id %s' % str( id ) ) + return {} + repository_metadata_dict = repository_metadata.to_dict( view='element', + value_mapper=self.__get_value_mapper( trans ) ) + repository_metadata_dict[ 'url' ] = web.url_for( controller='repository_revisions', + action='show', + id=trans.security.encode_id( repository_metadata.id ) ) + return repository_metadata_dict @web.expose_api def update( self, trans, payload, **kwd ): @@ -204,37 +200,32 @@ Updates the value of specified columns of the repository_metadata table based on the key / value pairs in payload. """ repository_metadata_id = kwd.get( 'id', None ) - try: - repository_metadata = metadata_util.get_repository_metadata_by_id( trans, repository_metadata_id ) - flush_needed = False - for key, new_value in payload.items(): - if key == 'time_last_tested': - repository_metadata.time_last_tested = datetime.datetime.utcnow() - flush_needed = True - elif hasattr( repository_metadata, key ): - # log information when setting attributes associated with the Tool Shed's install and test framework. - if key in [ 'do_not_test', 'includes_tools', 'missing_test_components', 'test_install_error', - 'tools_functionally_correct' ]: - log.debug( 'Setting repository_metadata table column %s to value %s for changeset_revision %s via the Tool Shed API.' % \ - ( str( key ), str( new_value ), str( repository_metadata.changeset_revision ) ) ) - setattr( repository_metadata, key, new_value ) - flush_needed = True - if flush_needed: - trans.sa_session.add( repository_metadata ) - trans.sa_session.flush() - except Exception, e: - message = "Error in the Tool Shed repository_revisions API in update: %s" % str( e ) - log.error( message, exc_info=True ) - trans.response.status = 500 - return message + if repository_metadata_id is None: + raise HTTPBadRequest( detail="Missing required parameter 'id'." ) + repository_metadata = metadata_util.get_repository_metadata_by_id( trans, repository_metadata_id ) + if repository_metadata is None: + log.debug( 'Cannot locate repository_metadata with id %s' % str( repository_metadata_id ) ) + return {} + flush_needed = False + for key, new_value in payload.items(): + if key == 'time_last_tested': + repository_metadata.time_last_tested = datetime.datetime.utcnow() + flush_needed = True + elif hasattr( repository_metadata, key ): + # log information when setting attributes associated with the Tool Shed's install and test framework. + if key in [ 'do_not_test', 'includes_tools', 'missing_test_components', 'test_install_error', + 'tools_functionally_correct' ]: + log.debug( 'Setting repository_metadata column %s to value %s for changeset_revision %s via the Tool Shed API.' % \ + ( str( key ), str( new_value ), str( repository_metadata.changeset_revision ) ) ) + setattr( repository_metadata, key, new_value ) + flush_needed = True + if flush_needed: + trans.sa_session.add( repository_metadata ) + trans.sa_session.flush() + trans.sa_session.refresh( repository_metadata ) repository_metadata_dict = repository_metadata.to_dict( view='element', - value_mapper=self.__get_value_mapper( trans, repository_metadata ) ) + value_mapper=self.__get_value_mapper( trans ) ) repository_metadata_dict[ 'url' ] = web.url_for( controller='repository_revisions', action='show', id=trans.security.encode_id( repository_metadata.id ) ) return repository_metadata_dict - - def __get_value_mapper( self, trans, repository_metadata ): - value_mapper = { 'id' : trans.security.encode_id, - 'repository_id' : trans.security.encode_id } - return value_mapper diff -r e72b8fd2d28b0a5371d13a77469746d0786496de -r d02540589348a2840f0492c91dbabe441fa2fb26 lib/tool_shed/scripts/api/common.py --- a/lib/tool_shed/scripts/api/common.py +++ b/lib/tool_shed/scripts/api/common.py @@ -1,34 +1,27 @@ -import os, sys, urllib, urllib2 +import os +import sys +import urllib +import urllib2 new_path = [ os.path.join( os.path.dirname( __file__ ), '..', '..', '..', '..', 'lib' ) ] new_path.extend( sys.path[ 1: ] ) sys.path = new_path +import tool_shed.util.shed_util_common as suc + from galaxy import eggs import pkg_resources pkg_resources.require( "simplejson" ) import simplejson -pkg_resources.require( "pycrypto" ) -from Crypto.Cipher import Blowfish -from Crypto.Util.randpool import RandomPool -from Crypto.Util import number - -def encode_id( config_id_secret, obj_id ): - # Utility method to encode ID's - id_cipher = Blowfish.new( config_id_secret ) - # Convert to string - s = str( obj_id ) - # Pad to a multiple of 8 with leading "!" - s = ( "!" * ( 8 - len(s) % 8 ) ) + s - # Encrypt - return id_cipher.encrypt( s ).encode( 'hex' ) - def delete( api_key, url, data, return_formatted=True ): - # Sends an API DELETE request and acts as a generic formatter for the JSON response - 'data' will become the JSON payload read by Galaxy. + """ + Sends an API DELETE request and acts as a generic formatter for the JSON response. The + 'data' will become the JSON payload read by the Tool Shed. + """ try: - url = make_url( api_key, url ) + url = make_url( url, api_key=api_key, args=None ) req = urllib2.Request( url, headers = { 'Content-Type': 'application/json' }, data = simplejson.dumps( data )) req.get_method = lambda: 'DELETE' r = simplejson.loads( urllib2.urlopen( req ).read() ) @@ -46,12 +39,13 @@ print r def display( url, api_key=None, return_formatted=True ): - # Sends an API GET request and acts as a generic formatter for the JSON response. + """Sends an API GET request and acts as a generic formatter for the JSON response.""" try: r = get( url, api_key=api_key ) except urllib2.HTTPError, e: print e - print e.read( 1024 ) # Only return the first 1K of errors. + # Only return the first 1K of errors. + print e.read( 1024 ) sys.exit( 1 ) if type( r ) == unicode: print 'error: %s' % r @@ -83,16 +77,94 @@ print 'response is unknown type: %s' % type( r ) def get( url, api_key=None ): - # Do the actual GET. - url = make_url( url, api_key=api_key ) + """Do the GET.""" + url = make_url( url, api_key=api_key, args=None ) try: return simplejson.loads( urllib2.urlopen( url ).read() ) except simplejson.decoder.JSONDecodeError, e: print "URL did not return JSON data" - sys.exit(1) + sys.exit( 1 ) + +def get_api_url( base, parts=[], params=None ): + """Compose and return a URL for the Tool Shed API.""" + if 'api' in parts and parts.index( 'api' ) != 0: + parts.pop( parts.index( 'api' ) ) + parts.insert( 0, 'api' ) + elif 'api' not in parts: + parts.insert( 0, 'api' ) + url = suc.url_join( base, *parts ) + if params is not None: + try: + query_string = urllib.urlencode( params ) + except Exception, e: + # The value of params must be a string. + query_string = params + url += '?%s' % query_string + return url + +def get_latest_downloadable_changeset_revision_via_api( url, name, owner ): + """ + Return the latest downloadable changeset revision for the repository defined by the received + name and owner. + """ + error_message = '' + parts = [ 'api', 'repositories', 'get_ordered_installable_revisions' ] + params = dict( name=name, owner=owner ) + api_url = get_api_url( base=url, parts=parts, params=params ) + changeset_revisions, error_message = json_from_url( api_url ) + if changeset_revisions is None or error_message: + return None, error_message + if len( changeset_revisions ) >= 1: + return changeset_revisions[ -1 ], error_message + return suc.INITIAL_CHANGELOG_HASH, error_message + +def get_repository_dict( url, repository_dict ): + """ + Send a request to the Tool Shed to get additional information about the repository defined + by the received repository_dict. Add the information to the repository_dict and return it. + """ + error_message = '' + if not isinstance( repository_dict, dict ): + error_message = 'Invalid repository_dict received: %s' % str( repository_dict ) + return None, error_message + repository_id = repository_dict.get( 'repository_id', None ) + if repository_id is None: + error_message = 'Invalid repository_dict does not contain a repository_id entry: %s' % str( repository_dict ) + return None, error_message + parts = [ 'api', 'repositories', repository_id ] + api_url = get_api_url( base=url, parts=parts ) + extended_dict, error_message = json_from_url( api_url ) + if extended_dict is None or error_message: + return None, error_message + name = extended_dict.get( 'name', None ) + owner = extended_dict.get( 'owner', None ) + if name is not None and owner is not None: + name = str( name ) + owner = str( owner ) + latest_changeset_revision, error_message = get_latest_downloadable_changeset_revision_via_api( url, name, owner ) + if latest_changeset_revision is None or error_message: + return None, error_message + extended_dict[ 'latest_revision' ] = str( latest_changeset_revision ) + return extended_dict, error_message + else: + error_message = 'Invalid extended_dict does not contain name or woner entries: %s' % str( extended_dict ) + return None, error_message + +def json_from_url( url ): + """Send a request to the Tool Shed via the Tool Shed API and handle the response.""" + error_message = '' + url_handle = urllib.urlopen( url ) + url_contents = url_handle.read() + try: + parsed_json = simplejson.loads( url_contents ) + except Exception, e: + error_message = str( url_contents ) + print 'Error parsing JSON data in json_from_url(): ', str( e ) + return None, error_message + return parsed_json, error_message def make_url( url, api_key=None, args=None ): - # Adds the API Key to the URL if it's not already there. + """Adds the API Key to the URL if it's not already there.""" if args is None: args = [] argsep = '&' @@ -104,20 +176,23 @@ return url + argsep + '&'.join( [ '='.join( t ) for t in args ] ) def post( url, data, api_key=None ): - # Do the actual POST. - url = make_url( url, api_key=api_key ) + """Do the POST.""" + url = make_url( url, api_key=api_key, args=None ) req = urllib2.Request( url, headers = { 'Content-Type': 'application/json' }, data = simplejson.dumps( data ) ) return simplejson.loads( urllib2.urlopen( req ).read() ) def put( url, data, api_key=None ): - # Do the actual PUT. - url = make_url( url, api_key=api_key ) + """Do the PUT.""" + url = make_url( url, api_key=api_key, args=None ) req = urllib2.Request( url, headers = { 'Content-Type': 'application/json' }, data = simplejson.dumps( data )) req.get_method = lambda: 'PUT' return simplejson.loads( urllib2.urlopen( req ).read() ) def submit( url, data, api_key=None, return_formatted=True ): - # Sends an API POST request and acts as a generic formatter for the JSON response - 'data' will become the JSON payload read by Galaxy. + """ + Sends an API POST request and acts as a generic formatter for the JSON response. The + 'data' will become the JSON payload read by the Tool Shed. + """ try: r = post( url, data, api_key=api_key ) except urllib2.HTTPError, e: @@ -132,7 +207,8 @@ print 'Response' print '--------' if type( r ) == list: - # Currently the only implemented responses are lists of dicts, because submission creates some number of collection elements. + # Currently the only implemented responses are lists of dicts, because submission creates + # some number of collection elements. for i in r: if type( i ) == dict: if 'url' in i: @@ -149,9 +225,12 @@ print r def update( api_key, url, data, return_formatted=True ): - # Sends an API PUT request and acts as a generic formatter for the JSON response - 'data' will become the JSON payload read by Galaxy. + """ + Sends an API PUT request and acts as a generic formatter for the JSON response. The + 'data' will become the JSON payload read by the Tool Shed. + """ try: - r = put( api_key, url, data ) + r = put( url, data, api_key=api_key ) except urllib2.HTTPError, e: if return_formatted: print e diff -r e72b8fd2d28b0a5371d13a77469746d0786496de -r d02540589348a2840f0492c91dbabe441fa2fb26 lib/tool_shed/scripts/api/get_filtered_repository_revisions.py --- a/lib/tool_shed/scripts/api/get_filtered_repository_revisions.py +++ b/lib/tool_shed/scripts/api/get_filtered_repository_revisions.py @@ -26,69 +26,19 @@ import urllib sys.path.insert( 0, os.path.dirname( __file__ ) ) + from common import get +from common import get_api_url +from common import get_repository_dict +from common import json_from_url + +from galaxy.util import asbool from galaxy.util.json import from_json_string import tool_shed.util.shed_util_common as suc -def get_api_url( base, parts=[], params=None ): - if 'api' in parts and parts.index( 'api' ) != 0: - parts.pop( parts.index( 'api' ) ) - parts.insert( 0, 'api' ) - elif 'api' not in parts: - parts.insert( 0, 'api' ) - url = suc.url_join( base, *parts ) - if params: - url += '?%s' % params - return url - -def get_latest_downloadable_changeset_revision( url, name, owner ): - error_message = '' - parts = [ 'api', 'repositories', 'get_ordered_installable_revisions' ] - params = urllib.urlencode( dict( name=name, owner=owner ) ) - api_url = get_api_url( base=url, parts=parts, params=params ) - changeset_revisions, error_message = json_from_url( api_url ) - if error_message: - return None, error_message - if changeset_revisions: - return changeset_revisions[ -1 ], error_message - else: - return suc.INITIAL_CHANGELOG_HASH, error_message - -def get_repository_dict( url, repository_dict ): - error_message = '' - parts = [ 'api', 'repositories', repository_dict[ 'repository_id' ] ] - api_url = get_api_url( base=url, parts=parts ) - extended_dict, error_message = json_from_url( api_url ) - if error_message: - return None, error_message - name = str( extended_dict[ 'name' ] ) - owner = str( extended_dict[ 'owner' ] ) - latest_changeset_revision, error_message = get_latest_downloadable_changeset_revision( url, name, owner ) - if error_message: - print error_message - extended_dict[ 'latest_revision' ] = str( latest_changeset_revision ) - return extended_dict, error_message - -def json_from_url( url ): - error_message = '' - url_handle = urllib.urlopen( url ) - url_contents = url_handle.read() - try: - parsed_json = from_json_string( url_contents ) - except Exception, e: - error_message = str( url_contents ) - return None, error_message - return parsed_json, error_message - -def string_as_bool( string ): - if str( string ).lower() in [ 'true' ]: - return True - else: - return False - def main( options ): base_tool_shed_url = options.tool_shed_url.rstrip( '/' ) - latest_revision_only = string_as_bool( options.latest_revision_only ) + latest_revision_only = asbool( options.latest_revision_only ) do_not_test = str( options.do_not_test ) downloadable = str( options.downloadable ) includes_tools = str( options.includes_tools ) @@ -108,30 +58,31 @@ tools_functionally_correct=tools_functionally_correct ) ) api_url = get_api_url( base=base_tool_shed_url, parts=parts, params=params ) baseline_repository_dicts, error_message = json_from_url( api_url ) - if error_message: + if baseline_repository_dicts is None or error_message: print error_message - repository_dicts = [] - for baseline_repository_dict in baseline_repository_dicts: - # We need to get some details from the tool shed API, such as repository name and owner, to pass on to the - # module that will generate the install methods. - repository_dict, error_message = get_repository_dict( base_tool_shed_url, baseline_repository_dict ) - if error_message: - print 'Error getting additional details from the API: ', error_message - repository_dicts.append( baseline_repository_dict ) - else: - # Don't test empty repositories. - changeset_revision = baseline_repository_dict[ 'changeset_revision' ] - if changeset_revision != suc.INITIAL_CHANGELOG_HASH: - # Merge the dictionary returned from /api/repository_revisions with the detailed repository_dict and - # append it to the list of repository_dicts to install and test. - if latest_revision_only: - latest_revision = repository_dict[ 'latest_revision' ] - if changeset_revision == latest_revision: + else: + repository_dicts = [] + for baseline_repository_dict in baseline_repository_dicts: + # We need to get additional details from the tool shed API to pass on to the + # module that will generate the install methods. + repository_dict, error_message = get_repository_dict( base_tool_shed_url, baseline_repository_dict ) + if error_message: + print 'Error getting additional details from the API: ', error_message + repository_dicts.append( baseline_repository_dict ) + else: + # Don't test empty repositories. + changeset_revision = baseline_repository_dict.get( 'changeset_revision', suc.INITIAL_CHANGELOG_HASH ) + if changeset_revision != suc.INITIAL_CHANGELOG_HASH: + # Merge the dictionary returned from /api/repository_revisions with the detailed repository_dict and + # append it to the list of repository_dicts to install and test. + if latest_revision_only: + latest_revision = repository_dict.get( 'latest_revision', suc.INITIAL_CHANGELOG_HASH ) + if changeset_revision == latest_revision: + repository_dicts.append( dict( repository_dict.items() + baseline_repository_dict.items() ) ) + else: repository_dicts.append( dict( repository_dict.items() + baseline_repository_dict.items() ) ) - else: - repository_dicts.append( dict( repository_dict.items() + baseline_repository_dict.items() ) ) - print '\n\n', repository_dicts - print '\nThe url:\n\n', api_url, '\n\nreturned ', len( repository_dicts ), ' repository dictionaries...' + print '\n\n', repository_dicts + print '\nThe url:\n\n', api_url, '\n\nreturned ', len( repository_dicts ), ' repository dictionaries...' if __name__ == '__main__': parser = argparse.ArgumentParser( description='Get a filtered list of repository dictionaries.' ) diff -r e72b8fd2d28b0a5371d13a77469746d0786496de -r d02540589348a2840f0492c91dbabe441fa2fb26 test/install_and_test_tool_shed_repositories/base/util.py --- a/test/install_and_test_tool_shed_repositories/base/util.py +++ b/test/install_and_test_tool_shed_repositories/base/util.py @@ -3,10 +3,9 @@ cwd = os.getcwd() sys.path.append( cwd ) -new_path = [ os.path.join( cwd, "scripts" ), - os.path.join( cwd, "lib" ), +new_path = [ os.path.join( cwd, "lib" ), os.path.join( cwd, 'test' ), - os.path.join( cwd, 'scripts', 'api' ) ] + os.path.join( cwd, 'lib', 'tool_shed', 'scripts', 'api' ) ] new_path.extend( sys.path ) sys.path = new_path @@ -27,6 +26,10 @@ from datetime import datetime from datetime import timedelta +from common import get_api_url +from common import get_latest_downloadable_changeset_revision_via_api +from common import get_repository_dict +from common import json_from_url from common import update from galaxy.util import asbool @@ -267,18 +270,6 @@ version = str( tool_dependency_dict[ 'version' ] ) print "# %s %s version %s" % ( type, name, version ) -def get_api_url( base, parts=[], params=None ): - if 'api' in parts and parts.index( 'api' ) != 0: - parts.pop( parts.index( 'api' ) ) - parts.insert( 0, 'api' ) - elif 'api' not in parts: - parts.insert( 0, 'api' ) - url = suc.url_join( base, *parts ) - if params is not None: - query_string = urllib.urlencode( params ) - url += '?%s' % query_string - return url - def get_database_version( app ): ''' This method returns the value of the version column from the migrate_version table, using the provided app's SQLAlchemy session to determine @@ -296,19 +287,6 @@ break return version -def get_latest_downloadable_changeset_revision( url, name, owner ): - error_message = '' - parts = [ 'api', 'repositories', 'get_ordered_installable_revisions' ] - params = dict( name=name, owner=owner ) - api_url = get_api_url( base=url, parts=parts, params=params ) - changeset_revisions, error_message = json_from_url( api_url ) - if error_message: - return None, error_message - if changeset_revisions: - return changeset_revisions[ -1 ], error_message - else: - return suc.INITIAL_CHANGELOG_HASH, error_message - def get_missing_repository_dependencies( repository ): """ Return the entire list of missing repository dependencies for the received repository. The entire @@ -467,28 +445,6 @@ return None, error_message return repository_dependency_dicts, error_message -def get_repository_dict( url, repository_dict ): - error_message = '' - if not isinstance( repository_dict, dict ): - error_message = 'Invalid repository_dict received: %s' % str( repository_dict ) - return None, error_message - repository_id = repository_dict.get( 'repository_id', None ) - if repository_id is None: - error_message = 'Invalid repository_dict does not contain a repository_id entry: %s' % str( repository_dict ) - return None, error_message - parts = [ 'api', 'repositories', repository_id ] - api_url = get_api_url( base=url, parts=parts ) - extended_dict, error_message = json_from_url( api_url ) - if error_message: - return None, error_message - name = str( extended_dict[ 'name' ] ) - owner = str( extended_dict[ 'owner' ] ) - latest_changeset_revision, error_message = get_latest_downloadable_changeset_revision( url, name, owner ) - if error_message: - return None, error_message - extended_dict[ 'latest_revision' ] = str( latest_changeset_revision ) - return extended_dict, error_message - def get_repository_dependencies_dicts( url, encoded_repository_metadata_id ): """ Return a list if dictionaries that define the repository dependencies of the repository defined by the @@ -709,23 +665,23 @@ return False, None def is_latest_downloadable_revision( url, repository_dict ): - name = str( repository_dict[ 'name' ] ) - owner = str( repository_dict[ 'owner' ] ) - changeset_revision = str( repository_dict[ 'changeset_revision' ] ) - latest_revision = get_latest_downloadable_changeset_revision( url, name=name, owner=owner ) - return changeset_revision == str( latest_revision ) - -def json_from_url( url ): + """ + Return True if the changeset_revision defined in the received repository_dict is the latest + installable revision for the repository. + """ error_message = '' - url_handle = urllib.urlopen( url ) - url_contents = url_handle.read() - try: - parsed_json = from_json_string( url_contents ) - except Exception, e: - error_message = str( url_contents ) - log.exception( 'Error parsing JSON data in json_from_url(): %s.' % str( e ) ) - return None, error_message - return parsed_json, error_message + name = repository_dict.get( 'name', None ) + owner = repository_dict.get( 'owner', None ) + changeset_revision = repository_dict.get( 'changeset_revision', None ) + if name is not None and owner is not None and changeset_revision is not None: + name = str( name ) + owner = str( owner ) + changeset_revision = str( changeset_revision ) + latest_revision, error_message = get_latest_downloadable_changeset_revision_via_api( url, name=name, owner=owner ) + if latest_revision is None or error_message: + return None, error_message + is_latest_downloadable = changeset_revision == str( latest_revision ) + return is_latest_downloadable, error_message def parse_exclude_list( xml_filename ): """Return a list of repositories to exclude from testing.""" @@ -1024,9 +980,15 @@ if can_update_tool_shed: metadata_revision_id = repository_dict.get( 'id', None ) if metadata_revision_id is not None: - name = str( repository_dict[ 'name' ] ) - owner = str( repository_dict[ 'owner' ] ) - changeset_revision = str( repository_dict[ 'changeset_revision' ] ) + name = repository_dict.get( 'name', None ) + owner = repository_dict.get( 'owner', None ) + changeset_revision = repository_dict.get( 'changeset_revision', None ) + if name is None or owner is None or changeset_revision is None: + log.debug( 'Entries for name, owner or changeset_revision missing from repository_dict %s' % str( repository_dict ) ) + return {} + name = str( name ) + owner = str( owner ) + changeset_revision = str( changeset_revision ) log.debug('\n=============================================================\n' ) log.debug( 'Inserting the following into tool_test_results for revision %s of repository %s owned by %s:\n%s' % \ ( changeset_revision, name, owner, str( tool_test_results_dict ) ) ) diff -r e72b8fd2d28b0a5371d13a77469746d0786496de -r d02540589348a2840f0492c91dbabe441fa2fb26 test/install_and_test_tool_shed_repositories/repositories_with_tools/functional_tests.py --- a/test/install_and_test_tool_shed_repositories/repositories_with_tools/functional_tests.py +++ b/test/install_and_test_tool_shed_repositories/repositories_with_tools/functional_tests.py @@ -685,9 +685,16 @@ tool_test_results_dict[ 'failed_tests' ] = failed_test_dicts failed_repository_dict = repository_identifier_dict install_and_test_statistics_dict[ 'at_least_one_test_failed' ].append( failed_repository_dict ) - set_do_not_test = \ - not install_and_test_base_util.is_latest_downloadable_revision( install_and_test_base_util.galaxy_tool_shed_url, - repository_dict ) + is_latest_downloadable_revision, error_message = \ + install_and_test_base_util.is_latest_downloadable_revision( install_and_test_base_util.galaxy_tool_shed_url, + repository_dict ) + if is_latest_downloadable_revision is None or error_message: + log.debug( 'Error attempting to determine if revision %s of repository %s owned by %s ' % \ + ( changeset_revision, name, owner ) ) + log.debug( 'is the latest downloadable revision: %s' % str( error_message ) ) + set_do_not_test = False + else: + set_do_not_test = not is_latest_downloadable_revision params = dict( tools_functionally_correct=False, test_install_error=False, do_not_test=str( set_do_not_test ) ) Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: greg: Tool Shed API fixes and code cleanup.
by commits-noreply＠bitbucket.org 09 Jan '14

09 Jan '14

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/0643bbc23cc1/ Changeset: 0643bbc23cc1 User: greg Date: 2014-01-09 21:59:18 Summary: Tool Shed API fixes and code cleanup. Affected #: 7 files diff -r 17e9e3ffb571e8f200ee8bf195a3708b2be4792b -r 0643bbc23cc1c845516469e3fa07270318c0c1a0 lib/galaxy/webapps/galaxy/api/tool_shed_repositories.py --- a/lib/galaxy/webapps/galaxy/api/tool_shed_repositories.py +++ b/lib/galaxy/webapps/galaxy/api/tool_shed_repositories.py @@ -19,11 +19,6 @@ log = logging.getLogger( __name__ ) -def default_tool_shed_repository_value_mapper( trans, tool_shed_repository ): - value_mapper={ 'id' : trans.security.encode_id( tool_shed_repository.id ), - 'error_message' : tool_shed_repository.error_message or '' } - return value_mapper - def get_message_for_no_shed_tool_config(): # This Galaxy instance is not configured with a shed-related tool panel configuration file. message = 'The tool_config_file setting in universe_wsgi.ini must include at least one shed tool configuration file name with a <toolbox> ' @@ -48,8 +43,8 @@ :param id: the encoded id of the ToolShedRepository object """ # Example URL: http://localhost:8763/api/tool_shed_repositories/f2db41e1fa331b3e/exported_… - # Since exported workflows are dictionaries with very few attributes that differentiate them from each other, we'll build the - # list based on the following dictionary of those few attributes. + # Since exported workflows are dictionaries with very few attributes that differentiate them from each + # other, we'll build the list based on the following dictionary of those few attributes. exported_workflows = [] repository = suc.get_tool_shed_repository_by_id( trans, id ) metadata = repository.metadata @@ -58,17 +53,23 @@ else: exported_workflow_tups = [] for index, exported_workflow_tup in enumerate( exported_workflow_tups ): - # The exported_workflow_tup looks like ( relative_path, exported_workflow_dict ), where the value of relative_path is the location - # on disk (relative to the root of the installed repository) where the exported_workflow_dict file (.ga file) is located. + # The exported_workflow_tup looks like ( relative_path, exported_workflow_dict ), where the value of + # relative_path is the location on disk (relative to the root of the installed repository) where the + # exported_workflow_dict file (.ga file) is located. exported_workflow_dict = exported_workflow_tup[ 1 ] annotation = exported_workflow_dict.get( 'annotation', '' ) format_version = exported_workflow_dict.get( 'format-version', '' ) workflow_name = exported_workflow_dict.get( 'name', '' ) - # Since we don't have an in-memory object with an id, we'll identify the exported workflow via it's location (i.e., index) in the list. + # Since we don't have an in-memory object with an id, we'll identify the exported workflow via it's + # location (i.e., index) in the list. display_dict = dict( index=index, annotation=annotation, format_version=format_version, workflow_name=workflow_name ) exported_workflows.append( display_dict ) return exported_workflows + def __get_value_mapper( self, trans ): + value_mapper = { 'id' : trans.security.encode_id } + return value_mapper + @web.expose_api def import_workflow( self, trans, payload, **kwd ): """ @@ -96,13 +97,11 @@ # Since we don't have an in-memory object with an id, we'll identify the exported workflow via it's location (i.e., index) in the list. exported_workflow = exported_workflows[ int( index ) ] workflow_name = exported_workflow[ 'workflow_name' ] - workflow, status, message = workflow_util.import_workflow( trans, repository, workflow_name ) + workflow, status, error_message = workflow_util.import_workflow( trans, repository, workflow_name ) if status == 'error': - log.error( message, exc_info=True ) - trans.response.status = 500 - return message - else: - return workflow.to_dict( view='element' ) + log.debug( error_message ) + return {} + return workflow.to_dict( view='element' ) @web.expose_api def import_workflows( self, trans, **kwd ): @@ -125,11 +124,9 @@ imported_workflow_dicts = [] for exported_workflow_dict in exported_workflows: workflow_name = exported_workflow_dict[ 'workflow_name' ] - workflow, status, message = workflow_util.import_workflow( trans, repository, workflow_name ) + workflow, status, error_message = workflow_util.import_workflow( trans, repository, workflow_name ) if status == 'error': - log.error( message, exc_info=True ) - trans.response.status = 500 - return message + log.debug( error_message ) else: imported_workflow_dicts.append( workflow.to_dict( view='element' ) ) return imported_workflow_dicts @@ -142,22 +139,15 @@ """ # Example URL: http://localhost:8763/api/tool_shed_repositories tool_shed_repository_dicts = [] - try: - query = trans.install_model.context.query( trans.app.install_model.ToolShedRepository ) \ - .order_by( trans.app.install_model.ToolShedRepository.table.c.name ) \ - .all() - for tool_shed_repository in query: - tool_shed_repository_dict = tool_shed_repository.to_dict( value_mapper=default_tool_shed_repository_value_mapper( trans, tool_shed_repository ) ) - tool_shed_repository_dict[ 'url' ] = web.url_for( controller='tool_shed_repositories', - action='show', - id=trans.security.encode_id( tool_shed_repository.id ) ) - tool_shed_repository_dicts.append( tool_shed_repository_dict ) - return tool_shed_repository_dicts - except Exception, e: - message = "Error in the tool_shed_repositories API in index: %s" % str( e ) - log.error( message, exc_info=True ) - trans.response.status = 500 - return message + for tool_shed_repository in trans.install_model.context.query( trans.app.install_model.ToolShedRepository ) \ + .order_by( trans.app.install_model.ToolShedRepository.table.c.name ): + tool_shed_repository_dict = \ + tool_shed_repository.to_dict( value_mapper=self.__get_value_mapper( trans ) ) + tool_shed_repository_dict[ 'url' ] = web.url_for( controller='tool_shed_repositories', + action='show', + id=trans.security.encode_id( tool_shed_repository.id ) ) + tool_shed_repository_dicts.append( tool_shed_repository_dict ) + return tool_shed_repository_dicts @web.expose_api def install_repository_revision( self, trans, payload, **kwd ): @@ -208,8 +198,7 @@ # Make sure this Galaxy instance is configured with a shed-related tool panel configuration file. if not suc.have_shed_tool_conf_for_install( trans ): message = get_message_for_no_shed_tool_config() - log.error( message, exc_info=True ) - trans.response.status = 500 + log.debug( message ) return dict( status='error', error=message ) # Make sure the current user's API key proves he is an admin user in this Galaxy instance. if not trans.user_is_admin(): @@ -225,18 +214,20 @@ except Exception, e: message = "Error attempting to retrieve installation information from tool shed %s for revision %s of repository %s owned by %s: %s" % \ ( str( tool_shed_url ), str( changeset_revision ), str( name ), str( owner ), str( e ) ) - log.error( message, exc_info=True ) - trans.response.status = 500 + log.debug( message ) return dict( status='error', error=message ) if raw_text: + # If successful, the response from get_repository_revision_install_info will be 3 + # dictionaries, a dictionary defining the Repository, a dictionary defining the + # Repository revision (RepositoryMetadata), and a dictionary including the additional + # information required to install the repository. items = json.from_json_string( raw_text ) repository_revision_dict = items[ 1 ] repo_info_dict = items[ 2 ] else: message = "Unable to retrieve installation information from tool shed %s for revision %s of repository %s owned by %s: %s" % \ ( str( tool_shed_url ), str( changeset_revision ), str( name ), str( owner ), str( e ) ) - log.error( message, exc_info=True ) - trans.response.status = 500 + log.debug( message ) return dict( status='error', error=message ) repo_info_dicts = [ repo_info_dict ] # Make sure the tool shed returned everything we need for installing the repository. @@ -345,7 +336,7 @@ tool_path, install_tool_dependencies, reinstalling=False ) - tool_shed_repository_dict = tool_shed_repository.as_dict( value_mapper=default_tool_shed_repository_value_mapper( trans, tool_shed_repository ) ) + tool_shed_repository_dict = tool_shed_repository.as_dict( value_mapper=self.__get_value_mapper( trans ) ) tool_shed_repository_dict[ 'url' ] = web.url_for( controller='tool_shed_repositories', action='show', id=trans.security.encode_id( tool_shed_repository.id ) ) @@ -394,8 +385,7 @@ if not suc.have_shed_tool_conf_for_install( trans ): # This Galaxy instance is not configured with a shed-related tool panel configuration file. message = get_message_for_no_shed_tool_config() - log.error( message, exc_info=True ) - trans.response.status = 500 + log.debug( message ) return dict( status='error', error=message ) if not trans.user_is_admin(): raise HTTPForbidden( detail='You are not authorized to install a tool shed repository into this Galaxy instance.' ) @@ -410,8 +400,7 @@ len( changeset_revisions ) != num_specified_repositories: message = 'Error in tool_shed_repositories API in install_repository_revisions: the received parameters must be ordered ' message += 'lists so that positional values in tool_shed_urls, names, owners and changeset_revisions are associated.' - log.error( message, exc_info=True ) - trans.response.status = 500 + log.debug( message ) return dict( status='error', error=message ) # Get the information about the Galaxy components (e.g., tool pane section, tool config file, etc) that will contain information # about each of the repositories being installed. @@ -482,7 +471,7 @@ repair_dict = repository_util.repair_tool_shed_repository( trans, repository, encoding_util.tool_shed_encode( repo_info_dict ) ) - repository_dict = repository.to_dict( value_mapper=default_tool_shed_repository_value_mapper( trans, repository ) ) + repository_dict = repository.to_dict( value_mapper=self.__get_value_mapper( trans ) ) repository_dict[ 'url' ] = web.url_for( controller='tool_shed_repositories', action='show', id=trans.security.encode_id( repository.id ) ) @@ -502,39 +491,39 @@ :param key: the API key of the Galaxy admin user. """ - try: - start_time = strftime( "%Y-%m-%d %H:%M:%S" ) - results = dict( start_time=start_time, - successful_count=0, - unsuccessful_count=0, - repository_status=[] ) - # Make sure the current user's API key proves he is an admin user in this Galaxy instance. - if not trans.user_is_admin(): - raise HTTPForbidden( detail='You are not authorized to reset metadata on repositories installed into this Galaxy instance.' ) - query = suc.get_query_for_setting_metadata_on_repositories( trans, my_writable=False, order=False ) - # Now reset metadata on all remaining repositories. - for repository in query: - repository_id = trans.security.encode_id( repository.id ) - try: - invalid_file_tups, metadata_dict = metadata_util.reset_all_metadata_on_installed_repository( trans, repository_id ) - if invalid_file_tups: - message = tool_util.generate_message_for_invalid_tools( trans, invalid_file_tups, repository, None, as_html=False ) - results[ 'unsuccessful_count' ] += 1 - else: - message = "Successfully reset metadata on repository %s owned by %s" % ( str( repository.name ), str( repository.owner ) ) - results[ 'successful_count' ] += 1 - except Exception, e: - message = "Error resetting metadata on repository %s owned by %s: %s" % ( str( repository.name ), str( repository.owner ), str( e ) ) + start_time = strftime( "%Y-%m-%d %H:%M:%S" ) + results = dict( start_time=start_time, + successful_count=0, + unsuccessful_count=0, + repository_status=[] ) + # Make sure the current user's API key proves he is an admin user in this Galaxy instance. + if not trans.user_is_admin(): + raise HTTPForbidden( detail='You are not authorized to reset metadata on repositories installed into this Galaxy instance.' ) + query = suc.get_query_for_setting_metadata_on_repositories( trans, my_writable=False, order=False ) + # Now reset metadata on all remaining repositories. + for repository in query: + repository_id = trans.security.encode_id( repository.id ) + try: + invalid_file_tups, metadata_dict = metadata_util.reset_all_metadata_on_installed_repository( trans, repository_id ) + if invalid_file_tups: + message = tool_util.generate_message_for_invalid_tools( trans, + invalid_file_tups, + repository, + None, + as_html=False ) results[ 'unsuccessful_count' ] += 1 - results[ 'repository_status' ].append( message ) - stop_time = strftime( "%Y-%m-%d %H:%M:%S" ) - results[ 'stop_time' ] = stop_time - return json.to_json_string( results, sort_keys=True, indent=4 * ' ' ) - except Exception, e: - message = "Error in the Galaxy tool_shed_repositories API in reset_metadata_on_installed_repositories: %s" % str( e ) - log.error( message, exc_info=True ) - trans.response.status = 500 - return message + else: + message = "Successfully reset metadata on repository %s owned by %s" % \ + ( str( repository.name ), str( repository.owner ) ) + results[ 'successful_count' ] += 1 + except Exception, e: + message = "Error resetting metadata on repository %s owned by %s: %s" % \ + ( str( repository.name ), str( repository.owner ), str( e ) ) + results[ 'unsuccessful_count' ] += 1 + results[ 'repository_status' ].append( message ) + stop_time = strftime( "%Y-%m-%d %H:%M:%S" ) + results[ 'stop_time' ] = stop_time + return json.to_json_string( results, sort_keys=True, indent=4 * ' ' ) @web.expose_api def show( self, trans, id, **kwd ): @@ -545,15 +534,12 @@ :param id: the encoded id of the ToolShedRepository object """ # Example URL: http://localhost:8763/api/tool_shed_repositories/df7a1f0c02a5b08e - try: - tool_shed_repository = suc.get_tool_shed_repository_by_id( trans, id ) - tool_shed_repository_dict = tool_shed_repository.as_dict( value_mapper=default_tool_shed_repository_value_mapper( trans, tool_shed_repository ) ) - tool_shed_repository_dict[ 'url' ] = web.url_for( controller='tool_shed_repositories', - action='show', - id=trans.security.encode_id( tool_shed_repository.id ) ) - return tool_shed_repository_dict - except Exception, e: - message = "Error in tool_shed_repositories API in index: " + str( e ) - log.error( message, exc_info=True ) - trans.response.status = 500 - return message + tool_shed_repository = suc.get_tool_shed_repository_by_id( trans, id ) + if tool_shed_repository is None: + log.debug( "Unable to locate tool_shed_repository record for id %s." % ( str( id ) ) ) + return {} + tool_shed_repository_dict = tool_shed_repository.as_dict( value_mapper=self.__get_value_mapper( trans ) ) + tool_shed_repository_dict[ 'url' ] = web.url_for( controller='tool_shed_repositories', + action='show', + id=trans.security.encode_id( tool_shed_repository.id ) ) + return tool_shed_repository_dict diff -r 17e9e3ffb571e8f200ee8bf195a3708b2be4792b -r 0643bbc23cc1c845516469e3fa07270318c0c1a0 lib/galaxy/webapps/tool_shed/api/repositories.py --- a/lib/galaxy/webapps/tool_shed/api/repositories.py +++ b/lib/galaxy/webapps/tool_shed/api/repositories.py @@ -36,22 +36,27 @@ :param name: the name of the Repository :param owner: the owner of the Repository - Returns the ordered list of changeset revision hash strings that are associated with installable revisions. As in the changelog, the - list is ordered oldest to newest. + Returns the ordered list of changeset revision hash strings that are associated with installable revisions. + As in the changelog, the list is ordered oldest to newest. """ # Example URL: http://localhost:9009/api/repositories/get_installable_revisions?name=add_c… - try: + if name and owner: # Get the repository information. repository = suc.get_repository_by_name_and_owner( trans.app, name, owner ) + if repository is None: + error_message = "Error in the Tool Shed repositories API in get_ordered_installable_revisions: " + error_message += "cannot locate repository %s owned by %s." % ( str( name ), str( owner ) ) + log.debug( error_message ) + return [] repo_dir = repository.repo_path( trans.app ) repo = hg.repository( suc.get_configured_ui(), repo_dir ) ordered_installable_revisions = suc.get_ordered_metadata_changeset_revisions( repository, repo, downloadable=True ) return ordered_installable_revisions - except Exception, e: - message = "Error in the Tool Shed repositories API in get_ordered_installable_revisions: %s" % str( e ) - log.error( message, exc_info=True ) - trans.response.status = 500 - return message + else: + error_message = "Error in the Tool Shed repositories API in get_ordered_installable_revisions: " + error_message += "invalid name %s or owner %s received." % ( str( name ), str( owner ) ) + log.debug( error_message ) + return [] @web.expose_api_anonymous def get_repository_revision_install_info( self, trans, name, owner, changeset_revision, **kwd ): @@ -106,49 +111,65 @@ ] } """ - repository_value_mapper = { 'id' : trans.security.encode_id, - 'user_id' : trans.security.encode_id } - # Example URL: http://localhost:9009/api/repositories/get_repository_revision_install_info… - try: + # Example URL: + # http://<xyz>/api/repositories/get_repository_revision_install_info?name=<n>&owner=<o>&changeset_revision=<cr> + if name and owner and changeset_revision: # Get the repository information. repository = suc.get_repository_by_name_and_owner( trans.app, name, owner ) + if repository is None: + log.debug( 'Cannot locate repository %s owned by %s' % ( str( name ), str( owner ) ) ) + return {}, {}, {} encoded_repository_id = trans.security.encode_id( repository.id ) - repository_dict = repository.to_dict( view='element', value_mapper=repository_value_mapper ) + repository_dict = repository.to_dict( view='element', + value_mapper=self.__get_value_mapper( trans ) ) repository_dict[ 'url' ] = web.url_for( controller='repositories', action='show', id=encoded_repository_id ) # Get the repository_metadata information. - repository_metadata = suc.get_repository_metadata_by_changeset_revision( trans, encoded_repository_id, changeset_revision ) + repository_metadata = suc.get_repository_metadata_by_changeset_revision( trans, + encoded_repository_id, + changeset_revision ) if not repository_metadata: - # The changeset_revision column in the repository_metadata table has been updated with a new value value, so find the - # changeset_revision to which we need to update. + # The changeset_revision column in the repository_metadata table has been updated with a new + # value value, so find the changeset_revision to which we need to update. repo_dir = repository.repo_path( trans.app ) repo = hg.repository( suc.get_configured_ui(), repo_dir ) new_changeset_revision = suc.get_next_downloadable_changeset_revision( repository, repo, changeset_revision ) - repository_metadata = suc.get_repository_metadata_by_changeset_revision( trans, encoded_repository_id, new_changeset_revision ) + repository_metadata = suc.get_repository_metadata_by_changeset_revision( trans, + encoded_repository_id, + new_changeset_revision ) changeset_revision = new_changeset_revision if repository_metadata: encoded_repository_metadata_id = trans.security.encode_id( repository_metadata.id ) repository_metadata_dict = repository_metadata.to_dict( view='collection', - value_mapper=self.__get_value_mapper( trans, repository_metadata ) ) + value_mapper=self.__get_value_mapper( trans ) ) repository_metadata_dict[ 'url' ] = web.url_for( controller='repository_revisions', action='show', id=encoded_repository_metadata_id ) # Get the repo_info_dict for installing the repository. - repo_info_dict, includes_tools, includes_tool_dependencies, includes_tools_for_display_in_tool_panel, \ - has_repository_dependencies, has_repository_dependencies_only_if_compiling_contained_td = \ + repo_info_dict, \ + includes_tools, \ + includes_tool_dependencies, \ + includes_tools_for_display_in_tool_panel, \ + has_repository_dependencies, \ + has_repository_dependencies_only_if_compiling_contained_td = \ repository_util.get_repo_info_dict( trans, encoded_repository_id, changeset_revision ) return repository_dict, repository_metadata_dict, repo_info_dict else: - message = "Unable to locate repository_metadata record for repository id %d and changeset_revision %s" % ( repository.id, changeset_revision ) - log.error( message, exc_info=True ) - trans.response.status = 500 + log.debug( "Unable to locate repository_metadata record for repository id %s and changeset_revision %s" % \ + ( str( repository.id ), str( changeset_revision ) ) ) return repository_dict, {}, {} - except Exception, e: - message = "Error in the Tool Shed repositories API in get_repository_revision_install_info: %s" % str( e ) - log.error( message, exc_info=True ) - trans.response.status = 500 - return message + else: + debug_msg = "Error in the Tool Shed repositories API in get_repository_revision_install_info: " + debug_msg += "Invalid name %s or owner %s or changeset_revision %s received." % \ + ( str( name ), str( owner ), str( changeset_revision ) ) + log.debug( debug_msg ) + return {}, {}, {} + + def __get_value_mapper( self, trans ): + value_mapper = { 'id' : trans.security.encode_id, + 'repository_id' : trans.security.encode_id } + return value_mapper @web.expose_api def import_capsule( self, trans, payload, **kwd ): @@ -177,29 +198,27 @@ uploaded_file=None, capsule_file_name=None ) if os.path.getsize( os.path.abspath( capsule_file_name ) ) == 0: - message = 'Your capsule file is empty.' - log.error( message, exc_info=True ) - trans.response.status = 500 - return message + log.debug( 'Your capsule file %s is empty.' % str( capsule_file_name ) ) + return {} try: # Open for reading with transparent compression. tar_archive = tarfile.open( capsule_file_path, 'r:*' ) except tarfile.ReadError, e: - message = 'Error opening file %s: %s' % ( str( capsule_file_name ), str( e ) ) - log.error( message, exc_info=True ) - trans.response.status = 500 - return message + log.debug( 'Error opening capsule file %s: %s' % ( str( capsule_file_name ), str( e ) ) ) + return {} capsule_dict[ 'tar_archive' ] = tar_archive capsule_dict[ 'capsule_file_name' ] = capsule_file_name capsule_dict = import_util.extract_capsule_files( trans, **capsule_dict ) capsule_dict = import_util.validate_capsule( trans, **capsule_dict ) status = capsule_dict.get( 'status', 'error' ) if status == 'error': - message = 'The capsule contents are invalid and cannpt be imported:<br/>%s' % str( capsule_dict.get( 'error_message', '' ) ) - log.error( message, exc_info=True ) - trans.response.status = 500 - return message + log.debug( 'The capsule contents are invalid and cannpt be imported:<br/>%s' % \ + str( capsule_dict.get( 'error_message', '' ) ) ) + return {} encoded_file_path = capsule_dict.get( 'encoded_file_path', None ) + if encoded_file_path is None: + log.debug( 'The capsule_dict %s is missing the required encoded_file_path entry.' % str( capsule_dict ) ) + return {} file_path = encoding_util.tool_shed_decode( encoded_file_path ) export_info_file_path = os.path.join( file_path, 'export_info.xml' ) export_info_dict = import_util.get_export_info_dict( export_info_file_path ) @@ -216,12 +235,14 @@ # Add the capsule_file_name and encoded_file_path to the repository_status_info_dict. repository_status_info_dict[ 'capsule_file_name' ] = capsule_file_name repository_status_info_dict[ 'encoded_file_path' ] = encoded_file_path - import_results_tups = repository_maintenance_util.create_repository_and_import_archive( trans, - repository_status_info_dict, - import_results_tups ) + import_results_tups = \ + repository_maintenance_util.create_repository_and_import_archive( trans, + repository_status_info_dict, + import_results_tups ) import_util.check_status_and_reset_downloadable( trans, import_results_tups ) suc.remove_dir( file_path ) - # NOTE: the order of installation is defined in import_results_tups, but order will be lost when transferred to return_dict. + # NOTE: the order of installation is defined in import_results_tups, but order will be lost + # when transferred to return_dict. return_dict = {} for import_results_tup in import_results_tups: ok, name_owner, message = import_results_tup @@ -237,28 +258,19 @@ GET /api/repositories Displays a collection (list) of repositories. """ - value_mapper = { 'id' : trans.security.encode_id, - 'user_id' : trans.security.encode_id } # Example URL: http://localhost:9009/api/repositories repository_dicts = [] - deleted = util.string_as_bool( deleted ) - try: - query = trans.sa_session.query( trans.app.model.Repository ) \ - .filter( trans.app.model.Repository.table.c.deleted == deleted ) \ - .order_by( trans.app.model.Repository.table.c.name ) \ - .all() - for repository in query: - repository_dict = repository.to_dict( view='collection', value_mapper=value_mapper ) - repository_dict[ 'url' ] = web.url_for( controller='repositories', - action='show', - id=trans.security.encode_id( repository.id ) ) - repository_dicts.append( repository_dict ) - return repository_dicts - except Exception, e: - message = "Error in the Tool Shed repositories API in index: %s" % str( e ) - log.error( message, exc_info=True ) - trans.response.status = 500 - return message + deleted = util.asbool( deleted ) + for repository in trans.sa_session.query( trans.app.model.Repository ) \ + .filter( trans.app.model.Repository.table.c.deleted == deleted ) \ + .order_by( trans.app.model.Repository.table.c.name ): + repository_dict = repository.to_dict( view='collection', + value_mapper=self.__get_value_mapper( trans ) ) + repository_dict[ 'url' ] = web.url_for( controller='repositories', + action='show', + id=trans.security.encode_id( repository.id ) ) + repository_dicts.append( repository_dict ) + return repository_dicts @web.expose_api def repository_ids_for_setting_metadata( self, trans, my_writable=False, **kwd ): @@ -273,28 +285,22 @@ in addition to those repositories of type tool_dependency_definition. This param is ignored if the current user is not an admin user, in which case this same restriction is automatic. """ - try: - if trans.user_is_admin(): - my_writable = util.asbool( my_writable ) - else: - my_writable = True - handled_repository_ids = [] - repository_ids = [] - query = suc.get_query_for_setting_metadata_on_repositories( trans, my_writable=my_writable, order=False ) - # Make sure repositories of type tool_dependency_definition are first in the list. - for repository in query: - if repository.type == rt_util.TOOL_DEPENDENCY_DEFINITION and repository.id not in handled_repository_ids: - repository_ids.append( trans.security.encode_id( repository.id ) ) - # Now add all remaining repositories to the list. - for repository in query: - if repository.type != rt_util.TOOL_DEPENDENCY_DEFINITION and repository.id not in handled_repository_ids: - repository_ids.append( trans.security.encode_id( repository.id ) ) - return repository_ids - except Exception, e: - message = "Error in the Tool Shed repositories API in repository_ids_for_setting_metadata: %s" % str( e ) - log.error( message, exc_info=True ) - trans.response.status = 500 - return message + if trans.user_is_admin(): + my_writable = util.asbool( my_writable ) + else: + my_writable = True + handled_repository_ids = [] + repository_ids = [] + query = suc.get_query_for_setting_metadata_on_repositories( trans, my_writable=my_writable, order=False ) + # Make sure repositories of type tool_dependency_definition are first in the list. + for repository in query: + if repository.type == rt_util.TOOL_DEPENDENCY_DEFINITION and repository.id not in handled_repository_ids: + repository_ids.append( trans.security.encode_id( repository.id ) ) + # Now add all remaining repositories to the list. + for repository in query: + if repository.type != rt_util.TOOL_DEPENDENCY_DEFINITION and repository.id not in handled_repository_ids: + repository_ids.append( trans.security.encode_id( repository.id ) ) + return repository_ids @web.expose_api def reset_metadata_on_repositories( self, trans, payload, **kwd ): @@ -318,6 +324,7 @@ :param skip_file (optional): A local file name that contains the encoded repository ids associated with repositories to skip. This param can be used as an alternative to the above encoded_ids_to_skip. """ + def handle_repository( trans, repository, results ): log.debug( "Resetting metadata on repository %s" % str( repository.name ) ) repository_id = trans.security.encode_id( repository.id ) @@ -335,53 +342,48 @@ status = '%s : %s' % ( str( repository.name ), message ) results[ 'repository_status' ].append( status ) return results - try: - start_time = strftime( "%Y-%m-%d %H:%M:%S" ) - results = dict( start_time=start_time, - repository_status=[], - successful_count=0, - unsuccessful_count=0 ) - handled_repository_ids = [] - encoded_ids_to_skip = payload.get( 'encoded_ids_to_skip', [] ) - skip_file = payload.get( 'skip_file', None ) - if skip_file and os.path.exists( skip_file ) and not encoded_ids_to_skip: - # Load the list of encoded_ids_to_skip from the skip_file. - # Contents of file must be 1 encoded repository id per line. - lines = open( skip_file, 'rb' ).readlines() - for line in lines: - if line.startswith( '#' ): - # Skip comments. - continue - encoded_ids_to_skip.append( line.rstrip( '\n' ) ) - if trans.user_is_admin(): - my_writable = util.asbool( payload.get( 'my_writable', False ) ) - else: - my_writable = True - query = suc.get_query_for_setting_metadata_on_repositories( trans, my_writable=my_writable, order=False ) - # First reset metadata on all repositories of type repository_dependency_definition. - for repository in query: - encoded_id = trans.security.encode_id( repository.id ) - if encoded_id in encoded_ids_to_skip: - log.debug( "Skipping repository with id %s because it is in encoded_ids_to_skip %s" % \ - ( str( repository.id ), str( encoded_ids_to_skip ) ) ) - elif repository.type == rt_util.TOOL_DEPENDENCY_DEFINITION and repository.id not in handled_repository_ids: - results = handle_repository( trans, repository, results ) - # Now reset metadata on all remaining repositories. - for repository in query: - encoded_id = trans.security.encode_id( repository.id ) - if encoded_id in encoded_ids_to_skip: - log.debug( "Skipping repository with id %s because it is in encoded_ids_to_skip %s" % \ - ( str( repository.id ), str( encoded_ids_to_skip ) ) ) - elif repository.type != rt_util.TOOL_DEPENDENCY_DEFINITION and repository.id not in handled_repository_ids: - results = handle_repository( trans, repository, results ) - stop_time = strftime( "%Y-%m-%d %H:%M:%S" ) - results[ 'stop_time' ] = stop_time - return json.to_json_string( results, sort_keys=True, indent=4 * ' ' ) - except Exception, e: - message = "Error in the Tool Shed repositories API in reset_metadata_on_repositories: %s" % str( e ) - log.error( message, exc_info=True ) - trans.response.status = 500 - return message + + start_time = strftime( "%Y-%m-%d %H:%M:%S" ) + results = dict( start_time=start_time, + repository_status=[], + successful_count=0, + unsuccessful_count=0 ) + handled_repository_ids = [] + encoded_ids_to_skip = payload.get( 'encoded_ids_to_skip', [] ) + skip_file = payload.get( 'skip_file', None ) + if skip_file and os.path.exists( skip_file ) and not encoded_ids_to_skip: + # Load the list of encoded_ids_to_skip from the skip_file. + # Contents of file must be 1 encoded repository id per line. + lines = open( skip_file, 'rb' ).readlines() + for line in lines: + if line.startswith( '#' ): + # Skip comments. + continue + encoded_ids_to_skip.append( line.rstrip( '\n' ) ) + if trans.user_is_admin(): + my_writable = util.asbool( payload.get( 'my_writable', False ) ) + else: + my_writable = True + query = suc.get_query_for_setting_metadata_on_repositories( trans, my_writable=my_writable, order=False ) + # First reset metadata on all repositories of type repository_dependency_definition. + for repository in query: + encoded_id = trans.security.encode_id( repository.id ) + if encoded_id in encoded_ids_to_skip: + log.debug( "Skipping repository with id %s because it is in encoded_ids_to_skip %s" % \ + ( str( repository.id ), str( encoded_ids_to_skip ) ) ) + elif repository.type == rt_util.TOOL_DEPENDENCY_DEFINITION and repository.id not in handled_repository_ids: + results = handle_repository( trans, repository, results ) + # Now reset metadata on all remaining repositories. + for repository in query: + encoded_id = trans.security.encode_id( repository.id ) + if encoded_id in encoded_ids_to_skip: + log.debug( "Skipping repository with id %s because it is in encoded_ids_to_skip %s" % \ + ( str( repository.id ), str( encoded_ids_to_skip ) ) ) + elif repository.type != rt_util.TOOL_DEPENDENCY_DEFINITION and repository.id not in handled_repository_ids: + results = handle_repository( trans, repository, results ) + stop_time = strftime( "%Y-%m-%d %H:%M:%S" ) + results[ 'stop_time' ] = stop_time + return json.to_json_string( results, sort_keys=True, indent=4 * ' ' ) @web.expose_api def reset_metadata_on_repository( self, trans, payload, **kwd ): @@ -395,6 +397,7 @@ The following parameters must be included in the payload. :param repository_id: the encoded id of the repository on which metadata is to be reset. """ + def handle_repository( trans, start_time, repository ): results = dict( start_time=start_time, repository_status=[] ) @@ -410,21 +413,16 @@ status = '%s : %s' % ( str( repository.name ), message ) results[ 'repository_status' ].append( status ) return results - try: - repository_id = payload.get( 'repository_id', None ) - if repository_id is not None: - repository = suc.get_repository_in_tool_shed( trans, repository_id ) - start_time = strftime( "%Y-%m-%d %H:%M:%S" ) - log.debug( "%s...resetting metadata on repository %s" % ( start_time, str( repository.name ) ) ) - results = handle_repository( trans, start_time, repository ) - stop_time = strftime( "%Y-%m-%d %H:%M:%S" ) - results[ 'stop_time' ] = stop_time - return json.to_json_string( results, sort_keys=True, indent=4 * ' ' ) - except Exception, e: - message = "Error in the Tool Shed repositories API in reset_metadata_on_repositories: %s" % str( e ) - log.error( message, exc_info=True ) - trans.response.status = 500 - return message + + repository_id = payload.get( 'repository_id', None ) + if repository_id is not None: + repository = suc.get_repository_in_tool_shed( trans, repository_id ) + start_time = strftime( "%Y-%m-%d %H:%M:%S" ) + log.debug( "%s...resetting metadata on repository %s" % ( start_time, str( repository.name ) ) ) + results = handle_repository( trans, start_time, repository ) + stop_time = strftime( "%Y-%m-%d %H:%M:%S" ) + results[ 'stop_time' ] = stop_time + return json.to_json_string( results, sort_keys=True, indent=4 * ' ' ) @web.expose_api_anonymous def show( self, trans, id, **kwd ): @@ -434,27 +432,14 @@ :param id: the encoded id of the Repository object """ - value_mapper = { 'id' : trans.security.encode_id, - 'user_id' : trans.security.encode_id } # Example URL: http://localhost:9009/api/repositories/f9cad7b01a472135 - try: - repository = suc.get_repository_in_tool_shed( trans, id ) - repository_dict = repository.to_dict( view='element', value_mapper=value_mapper ) - repository_dict[ 'url' ] = web.url_for( controller='repositories', - action='show', - id=trans.security.encode_id( repository.id ) ) - return repository_dict - except Exception, e: - message = "Error in the Tool Shed repositories API in show: %s" % str( e ) - log.error( message, exc_info=True ) - trans.response.status = 500 - return message - - def __get_value_mapper( self, trans, repository_metadata ): - value_mapper = { 'id' : trans.security.encode_id, - 'repository_id' : trans.security.encode_id } - if repository_metadata.time_last_tested is not None: - # For some reason the Dictifiable.to_dict() method in ~/galaxy/model/item_attrs.py requires - # a function rather than a mapped value, so just pass the time_ago function here. - value_mapper[ 'time_last_tested' ] = time_ago - return value_mapper + repository = suc.get_repository_in_tool_shed( trans, id ) + if repository is None: + log.debug( "Unable to locate repository record for id %s." % ( str( id ) ) ) + return {} + repository_dict = repository.to_dict( view='element', + value_mapper=self.__get_value_mapper( trans ) ) + repository_dict[ 'url' ] = web.url_for( controller='repositories', + action='show', + id=trans.security.encode_id( repository.id ) ) + return repository_dict diff -r 17e9e3ffb571e8f200ee8bf195a3708b2be4792b -r 0643bbc23cc1c845516469e3fa07270318c0c1a0 lib/galaxy/webapps/tool_shed/api/repository_revisions.py --- a/lib/galaxy/webapps/tool_shed/api/repository_revisions.py +++ b/lib/galaxy/webapps/tool_shed/api/repository_revisions.py @@ -47,65 +47,29 @@ if not changeset_revision: raise HTTPBadRequest( detail="Missing required parameter 'changeset_revision'." ) export_repository_dependencies = payload.get( 'export_repository_dependencies', False ) - try: - # We'll currently support only gzip-compressed tar archives. - file_type = 'gz' - export_repository_dependencies = util.string_as_bool( export_repository_dependencies ) - # Get the repository information. - repository = suc.get_repository_by_name_and_owner( trans.app, name, owner ) - repository_id = trans.security.encode_id( repository.id ) - response = export_util.export_repository( trans, - tool_shed_url, - repository_id, - str( repository.name ), - changeset_revision, - file_type, - export_repository_dependencies, - api=True ) - return response - except Exception, e: - message = "Error in the Tool Shed repository_revisions API in export: %s" % str( e ) - log.error( message, exc_info=True ) - trans.response.status = 500 - return message + # We'll currently support only gzip-compressed tar archives. + file_type = 'gz' + export_repository_dependencies = util.asbool( export_repository_dependencies ) + # Get the repository information. + repository = suc.get_repository_by_name_and_owner( trans.app, name, owner ) + if repository is None: + error_message = 'Cannot locate repository with name %s and owner %s,' % ( str( name ), str( owner ) ) + log.debug( error_message ) + return None, error_message + repository_id = trans.security.encode_id( repository.id ) + return export_util.export_repository( trans, + tool_shed_url, + repository_id, + str( repository.name ), + changeset_revision, + file_type, + export_repository_dependencies, + api=True ) - @web.expose_api_anonymous - def repository_dependencies( self, trans, id, **kwd ): - """ - GET /api/repository_revisions/{encoded repository_metadata id}/repository_dependencies - Displays information about a repository_metadata record in the Tool Shed. - - :param id: the encoded id of the `RepositoryMetadata` object - """ - # Example URL: http://localhost:9009/api/repository_revisions/repository_dependencies/bb12… + def __get_value_mapper( self, trans ): value_mapper = { 'id' : trans.security.encode_id, - 'user_id' : trans.security.encode_id } - repository_dependencies_dicts = [] - try: - repository_metadata = metadata_util.get_repository_metadata_by_id( trans, id ) - metadata = repository_metadata.metadata - if metadata and 'repository_dependencies' in metadata: - rd_tups = metadata[ 'repository_dependencies' ][ 'repository_dependencies' ] - for rd_tup in rd_tups: - tool_shed, name, owner, changeset_revision = rd_tup[ 0:4 ] - repository_dependency = suc.get_repository_by_name_and_owner( trans.app, name, owner ) - repository_dependency_id = trans.security.encode_id( repository_dependency.id ) - repository_dependency_repository_metadata = \ - suc.get_repository_metadata_by_changeset_revision( trans, repository_dependency_id, changeset_revision ) - repository_dependency_repository_metadata_id = trans.security.encode_id( repository_dependency_repository_metadata.id ) - repository_dependency_dict = repository_dependency.to_dict( view='element', value_mapper=value_mapper ) - # We have to add the changeset_revision of of the repository dependency. - repository_dependency_dict[ 'changeset_revision' ] = changeset_revision - repository_dependency_dict[ 'url' ] = web.url_for( controller='repositories', - action='show', - id=repository_dependency_repository_metadata_id ) - repository_dependencies_dicts.append( repository_dependency_dict ) - return repository_dependencies_dicts - except Exception, e: - message = "Error in the Tool Shed repository_revisions API in repository_dependencies: %s" % str( e ) - log.error( message, exc_info=True ) - trans.response.status = 500 - return message + 'repository_id' : trans.security.encode_id } + return value_mapper @web.expose_api_anonymous def index( self, trans, **kwd ): @@ -120,59 +84,94 @@ # Filter by downloadable if received. downloadable = kwd.get( 'downloadable', None ) if downloadable is not None: - clause_list.append( trans.model.RepositoryMetadata.table.c.downloadable == util.string_as_bool( downloadable ) ) + clause_list.append( trans.model.RepositoryMetadata.table.c.downloadable == util.asbool( downloadable ) ) # Filter by malicious if received. malicious = kwd.get( 'malicious', None ) if malicious is not None: - clause_list.append( trans.model.RepositoryMetadata.table.c.malicious == util.string_as_bool( malicious ) ) + clause_list.append( trans.model.RepositoryMetadata.table.c.malicious == util.asbool( malicious ) ) # Filter by tools_functionally_correct if received. tools_functionally_correct = kwd.get( 'tools_functionally_correct', None ) if tools_functionally_correct is not None: - clause_list.append( trans.model.RepositoryMetadata.table.c.tools_functionally_correct == util.string_as_bool( tools_functionally_correct ) ) + clause_list.append( trans.model.RepositoryMetadata.table.c.tools_functionally_correct == util.asbool( tools_functionally_correct ) ) # Filter by missing_test_components if received. missing_test_components = kwd.get( 'missing_test_components', None ) if missing_test_components is not None: - clause_list.append( trans.model.RepositoryMetadata.table.c.missing_test_components == util.string_as_bool( missing_test_components ) ) + clause_list.append( trans.model.RepositoryMetadata.table.c.missing_test_components == util.asbool( missing_test_components ) ) # Filter by do_not_test if received. do_not_test = kwd.get( 'do_not_test', None ) if do_not_test is not None: - clause_list.append( trans.model.RepositoryMetadata.table.c.do_not_test == util.string_as_bool( do_not_test ) ) + clause_list.append( trans.model.RepositoryMetadata.table.c.do_not_test == util.asbool( do_not_test ) ) # Filter by includes_tools if received. includes_tools = kwd.get( 'includes_tools', None ) if includes_tools is not None: - clause_list.append( trans.model.RepositoryMetadata.table.c.includes_tools == util.string_as_bool( includes_tools ) ) + clause_list.append( trans.model.RepositoryMetadata.table.c.includes_tools == util.asbool( includes_tools ) ) # Filter by test_install_error if received. test_install_error = kwd.get( 'test_install_error', None ) if test_install_error is not None: - clause_list.append( trans.model.RepositoryMetadata.table.c.test_install_error == util.string_as_bool( test_install_error ) ) + clause_list.append( trans.model.RepositoryMetadata.table.c.test_install_error == util.asbool( test_install_error ) ) # Filter by skip_tool_test if received. skip_tool_test = kwd.get( 'skip_tool_test', None ) if skip_tool_test is not None: - skip_tool_test = util.string_as_bool( skip_tool_test ) + skip_tool_test = util.asbool( skip_tool_test ) skipped_metadata_ids_subquery = select( [ trans.app.model.SkipToolTest.table.c.repository_metadata_id ] ) if skip_tool_test: clause_list.append( trans.model.RepositoryMetadata.id.in_( skipped_metadata_ids_subquery ) ) else: clause_list.append( not_( trans.model.RepositoryMetadata.id.in_( skipped_metadata_ids_subquery ) ) ) - # Generate and execute the query. - try: - query = trans.sa_session.query( trans.app.model.RepositoryMetadata ) \ - .filter( and_( *clause_list ) ) \ - .order_by( trans.app.model.RepositoryMetadata.table.c.repository_id.desc() ) \ - .all() - for repository_metadata in query: - repository_metadata_dict = repository_metadata.to_dict( view='collection', - value_mapper=self.__get_value_mapper( trans, repository_metadata ) ) - repository_metadata_dict[ 'url' ] = web.url_for( controller='repository_revisions', - action='show', - id=trans.security.encode_id( repository_metadata.id ) ) - repository_metadata_dicts.append( repository_metadata_dict ) - return repository_metadata_dicts - except Exception, e: - message = "Error in the Tool Shed repository_revisions API in index: " + str( e ) - log.error( message, exc_info=True ) - trans.response.status = 500 - return message + for repository_metadata in trans.sa_session.query( trans.app.model.RepositoryMetadata ) \ + .filter( and_( *clause_list ) ) \ + .order_by( trans.app.model.RepositoryMetadata.table.c.repository_id.desc() ): + repository_metadata_dict = repository_metadata.to_dict( view='collection', + value_mapper=self.__get_value_mapper( trans ) ) + repository_metadata_dict[ 'url' ] = web.url_for( controller='repository_revisions', + action='show', + id=trans.security.encode_id( repository_metadata.id ) ) + repository_metadata_dicts.append( repository_metadata_dict ) + return repository_metadata_dicts + + @web.expose_api_anonymous + def repository_dependencies( self, trans, id, **kwd ): + """ + GET /api/repository_revisions/{encoded repository_metadata id}/repository_dependencies + Displays information about a repository_metadata record in the Tool Shed. + + :param id: the encoded id of the `RepositoryMetadata` object + """ + # Example URL: http://localhost:9009/api/repository_revisions/repository_dependencies/bb12… + repository_dependencies_dicts = [] + repository_metadata = metadata_util.get_repository_metadata_by_id( trans, id ) + if repository_metadata is None: + log.debug( 'Invalid repository_metadata id received: %s' % str( id ) ) + return repository_dependencies_dicts + metadata = repository_metadata.metadata + if metadata is None: + log.debug( 'The repository_metadata record with id %s has no metadata.' % str ( id ) ) + return repository_dependencies_dicts + if 'repository_dependencies' in metadata: + rd_tups = metadata[ 'repository_dependencies' ][ 'repository_dependencies' ] + for rd_tup in rd_tups: + tool_shed, name, owner, changeset_revision = rd_tup[ 0:4 ] + repository_dependency = suc.get_repository_by_name_and_owner( trans.app, name, owner ) + if repository_dependency is None: + log.dbug( 'Cannot locate repository dependency %s owned by %s.' % ( name, owner ) ) + continue + repository_dependency_id = trans.security.encode_id( repository_dependency.id ) + repository_dependency_repository_metadata = \ + suc.get_repository_metadata_by_changeset_revision( trans, repository_dependency_id, changeset_revision ) + if repository_dependency_repository_metadata is None: + log.debug( 'Cannot locate repository_metadata with id %s for repository dependency %s owned by %s.' % \ + ( str( repository_dependency_id ), str( name ), str( owner ) ) ) + continue + repository_dependency_repository_metadata_id = trans.security.encode_id( repository_dependency_repository_metadata.id ) + repository_dependency_dict = repository_dependency.to_dict( view='element', + value_mapper=self.__get_value_mapper( trans ) ) + # We have to add the changeset_revision of of the repository dependency. + repository_dependency_dict[ 'changeset_revision' ] = changeset_revision + repository_dependency_dict[ 'url' ] = web.url_for( controller='repositories', + action='show', + id=repository_dependency_repository_metadata_id ) + repository_dependencies_dicts.append( repository_dependency_dict ) + return repository_dependencies_dicts @web.expose_api_anonymous def show( self, trans, id, **kwd ): @@ -183,19 +182,16 @@ :param id: the encoded id of the `RepositoryMetadata` object """ # Example URL: http://localhost:9009/api/repository_revisions/bb125606ff9ea620 - try: - repository_metadata = metadata_util.get_repository_metadata_by_id( trans, id ) - repository_metadata_dict = repository_metadata.to_dict( view='element', - value_mapper=self.__get_value_mapper( trans, repository_metadata ) ) - repository_metadata_dict[ 'url' ] = web.url_for( controller='repository_revisions', - action='show', - id=trans.security.encode_id( repository_metadata.id ) ) - return repository_metadata_dict - except Exception, e: - message = "Error in the Tool Shed repository_revisions API in show: %s" % str( e ) - log.error( message, exc_info=True ) - trans.response.status = 500 - return message + repository_metadata = metadata_util.get_repository_metadata_by_id( trans, id ) + if repository_metadata is None: + log.debug( 'Cannot locate repository_metadata with id %s' % str( id ) ) + return {} + repository_metadata_dict = repository_metadata.to_dict( view='element', + value_mapper=self.__get_value_mapper( trans ) ) + repository_metadata_dict[ 'url' ] = web.url_for( controller='repository_revisions', + action='show', + id=trans.security.encode_id( repository_metadata.id ) ) + return repository_metadata_dict @web.expose_api def update( self, trans, payload, **kwd ): @@ -204,37 +200,32 @@ Updates the value of specified columns of the repository_metadata table based on the key / value pairs in payload. """ repository_metadata_id = kwd.get( 'id', None ) - try: - repository_metadata = metadata_util.get_repository_metadata_by_id( trans, repository_metadata_id ) - flush_needed = False - for key, new_value in payload.items(): - if key == 'time_last_tested': - repository_metadata.time_last_tested = datetime.datetime.utcnow() - flush_needed = True - elif hasattr( repository_metadata, key ): - # log information when setting attributes associated with the Tool Shed's install and test framework. - if key in [ 'do_not_test', 'includes_tools', 'missing_test_components', 'test_install_error', - 'tools_functionally_correct' ]: - log.debug( 'Setting repository_metadata table column %s to value %s for changeset_revision %s via the Tool Shed API.' % \ - ( str( key ), str( new_value ), str( repository_metadata.changeset_revision ) ) ) - setattr( repository_metadata, key, new_value ) - flush_needed = True - if flush_needed: - trans.sa_session.add( repository_metadata ) - trans.sa_session.flush() - except Exception, e: - message = "Error in the Tool Shed repository_revisions API in update: %s" % str( e ) - log.error( message, exc_info=True ) - trans.response.status = 500 - return message + if repository_metadata_id is None: + raise HTTPBadRequest( detail="Missing required parameter 'id'." ) + repository_metadata = metadata_util.get_repository_metadata_by_id( trans, repository_metadata_id ) + if repository_metadata is None: + log.debug( 'Cannot locate repository_metadata with id %s' % str( repository_metadata_id ) ) + return {} + flush_needed = False + for key, new_value in payload.items(): + if key == 'time_last_tested': + repository_metadata.time_last_tested = datetime.datetime.utcnow() + flush_needed = True + elif hasattr( repository_metadata, key ): + # log information when setting attributes associated with the Tool Shed's install and test framework. + if key in [ 'do_not_test', 'includes_tools', 'missing_test_components', 'test_install_error', + 'tools_functionally_correct' ]: + log.debug( 'Setting repository_metadata column %s to value %s for changeset_revision %s via the Tool Shed API.' % \ + ( str( key ), str( new_value ), str( repository_metadata.changeset_revision ) ) ) + setattr( repository_metadata, key, new_value ) + flush_needed = True + if flush_needed: + trans.sa_session.add( repository_metadata ) + trans.sa_session.flush() + trans.sa_session.refresh( repository_metadata ) repository_metadata_dict = repository_metadata.to_dict( view='element', - value_mapper=self.__get_value_mapper( trans, repository_metadata ) ) + value_mapper=self.__get_value_mapper( trans ) ) repository_metadata_dict[ 'url' ] = web.url_for( controller='repository_revisions', action='show', id=trans.security.encode_id( repository_metadata.id ) ) return repository_metadata_dict - - def __get_value_mapper( self, trans, repository_metadata ): - value_mapper = { 'id' : trans.security.encode_id, - 'repository_id' : trans.security.encode_id } - return value_mapper diff -r 17e9e3ffb571e8f200ee8bf195a3708b2be4792b -r 0643bbc23cc1c845516469e3fa07270318c0c1a0 lib/tool_shed/scripts/api/common.py --- a/lib/tool_shed/scripts/api/common.py +++ b/lib/tool_shed/scripts/api/common.py @@ -1,34 +1,27 @@ -import os, sys, urllib, urllib2 +import os +import sys +import urllib +import urllib2 new_path = [ os.path.join( os.path.dirname( __file__ ), '..', '..', '..', '..', 'lib' ) ] new_path.extend( sys.path[ 1: ] ) sys.path = new_path +import tool_shed.util.shed_util_common as suc + from galaxy import eggs import pkg_resources pkg_resources.require( "simplejson" ) import simplejson -pkg_resources.require( "pycrypto" ) -from Crypto.Cipher import Blowfish -from Crypto.Util.randpool import RandomPool -from Crypto.Util import number - -def encode_id( config_id_secret, obj_id ): - # Utility method to encode ID's - id_cipher = Blowfish.new( config_id_secret ) - # Convert to string - s = str( obj_id ) - # Pad to a multiple of 8 with leading "!" - s = ( "!" * ( 8 - len(s) % 8 ) ) + s - # Encrypt - return id_cipher.encrypt( s ).encode( 'hex' ) - def delete( api_key, url, data, return_formatted=True ): - # Sends an API DELETE request and acts as a generic formatter for the JSON response - 'data' will become the JSON payload read by Galaxy. + """ + Sends an API DELETE request and acts as a generic formatter for the JSON response. The + 'data' will become the JSON payload read by the Tool Shed. + """ try: - url = make_url( api_key, url ) + url = make_url( url, api_key=api_key, args=None ) req = urllib2.Request( url, headers = { 'Content-Type': 'application/json' }, data = simplejson.dumps( data )) req.get_method = lambda: 'DELETE' r = simplejson.loads( urllib2.urlopen( req ).read() ) @@ -46,12 +39,13 @@ print r def display( url, api_key=None, return_formatted=True ): - # Sends an API GET request and acts as a generic formatter for the JSON response. + """Sends an API GET request and acts as a generic formatter for the JSON response.""" try: r = get( url, api_key=api_key ) except urllib2.HTTPError, e: print e - print e.read( 1024 ) # Only return the first 1K of errors. + # Only return the first 1K of errors. + print e.read( 1024 ) sys.exit( 1 ) if type( r ) == unicode: print 'error: %s' % r @@ -83,16 +77,94 @@ print 'response is unknown type: %s' % type( r ) def get( url, api_key=None ): - # Do the actual GET. - url = make_url( url, api_key=api_key ) + """Do the GET.""" + url = make_url( url, api_key=api_key, args=None ) try: return simplejson.loads( urllib2.urlopen( url ).read() ) except simplejson.decoder.JSONDecodeError, e: print "URL did not return JSON data" - sys.exit(1) + sys.exit( 1 ) + +def get_api_url( base, parts=[], params=None ): + """Compose and return a URL for the Tool Shed API.""" + if 'api' in parts and parts.index( 'api' ) != 0: + parts.pop( parts.index( 'api' ) ) + parts.insert( 0, 'api' ) + elif 'api' not in parts: + parts.insert( 0, 'api' ) + url = suc.url_join( base, *parts ) + if params is not None: + try: + query_string = urllib.urlencode( params ) + except Exception, e: + # The value of params must be a string. + query_string = params + url += '?%s' % query_string + return url + +def get_latest_downloadable_changeset_revision_via_api( url, name, owner ): + """ + Return the latest downloadable changeset revision for the repository defined by the received + name and owner. + """ + error_message = '' + parts = [ 'api', 'repositories', 'get_ordered_installable_revisions' ] + params = dict( name=name, owner=owner ) + api_url = get_api_url( base=url, parts=parts, params=params ) + changeset_revisions, error_message = json_from_url( api_url ) + if changeset_revisions is None or error_message: + return None, error_message + if len( changeset_revisions ) >= 1: + return changeset_revisions[ -1 ], error_message + return suc.INITIAL_CHANGELOG_HASH, error_message + +def get_repository_dict( url, repository_dict ): + """ + Send a request to the Tool Shed to get additional information about the repository defined + by the received repository_dict. Add the information to the repository_dict and return it. + """ + error_message = '' + if not isinstance( repository_dict, dict ): + error_message = 'Invalid repository_dict received: %s' % str( repository_dict ) + return None, error_message + repository_id = repository_dict.get( 'repository_id', None ) + if repository_id is None: + error_message = 'Invalid repository_dict does not contain a repository_id entry: %s' % str( repository_dict ) + return None, error_message + parts = [ 'api', 'repositories', repository_id ] + api_url = get_api_url( base=url, parts=parts ) + extended_dict, error_message = json_from_url( api_url ) + if extended_dict is None or error_message: + return None, error_message + name = extended_dict.get( 'name', None ) + owner = extended_dict.get( 'owner', None ) + if name is not None and owner is not None: + name = str( name ) + owner = str( owner ) + latest_changeset_revision, error_message = get_latest_downloadable_changeset_revision_via_api( url, name, owner ) + if latest_changeset_revision is None or error_message: + return None, error_message + extended_dict[ 'latest_revision' ] = str( latest_changeset_revision ) + return extended_dict, error_message + else: + error_message = 'Invalid extended_dict does not contain name or woner entries: %s' % str( extended_dict ) + return None, error_message + +def json_from_url( url ): + """Send a request to the Tool Shed via the Tool Shed API and handle the response.""" + error_message = '' + url_handle = urllib.urlopen( url ) + url_contents = url_handle.read() + try: + parsed_json = simplejson.loads( url_contents ) + except Exception, e: + error_message = str( url_contents ) + print 'Error parsing JSON data in json_from_url(): ', str( e ) + return None, error_message + return parsed_json, error_message def make_url( url, api_key=None, args=None ): - # Adds the API Key to the URL if it's not already there. + """Adds the API Key to the URL if it's not already there.""" if args is None: args = [] argsep = '&' @@ -104,20 +176,23 @@ return url + argsep + '&'.join( [ '='.join( t ) for t in args ] ) def post( url, data, api_key=None ): - # Do the actual POST. - url = make_url( url, api_key=api_key ) + """Do the POST.""" + url = make_url( url, api_key=api_key, args=None ) req = urllib2.Request( url, headers = { 'Content-Type': 'application/json' }, data = simplejson.dumps( data ) ) return simplejson.loads( urllib2.urlopen( req ).read() ) def put( url, data, api_key=None ): - # Do the actual PUT. - url = make_url( url, api_key=api_key ) + """Do the PUT.""" + url = make_url( url, api_key=api_key, args=None ) req = urllib2.Request( url, headers = { 'Content-Type': 'application/json' }, data = simplejson.dumps( data )) req.get_method = lambda: 'PUT' return simplejson.loads( urllib2.urlopen( req ).read() ) def submit( url, data, api_key=None, return_formatted=True ): - # Sends an API POST request and acts as a generic formatter for the JSON response - 'data' will become the JSON payload read by Galaxy. + """ + Sends an API POST request and acts as a generic formatter for the JSON response. The + 'data' will become the JSON payload read by the Tool Shed. + """ try: r = post( url, data, api_key=api_key ) except urllib2.HTTPError, e: @@ -132,7 +207,8 @@ print 'Response' print '--------' if type( r ) == list: - # Currently the only implemented responses are lists of dicts, because submission creates some number of collection elements. + # Currently the only implemented responses are lists of dicts, because submission creates + # some number of collection elements. for i in r: if type( i ) == dict: if 'url' in i: @@ -149,9 +225,12 @@ print r def update( api_key, url, data, return_formatted=True ): - # Sends an API PUT request and acts as a generic formatter for the JSON response - 'data' will become the JSON payload read by Galaxy. + """ + Sends an API PUT request and acts as a generic formatter for the JSON response. The + 'data' will become the JSON payload read by the Tool Shed. + """ try: - r = put( api_key, url, data ) + r = put( url, data, api_key=api_key ) except urllib2.HTTPError, e: if return_formatted: print e diff -r 17e9e3ffb571e8f200ee8bf195a3708b2be4792b -r 0643bbc23cc1c845516469e3fa07270318c0c1a0 lib/tool_shed/scripts/api/get_filtered_repository_revisions.py --- a/lib/tool_shed/scripts/api/get_filtered_repository_revisions.py +++ b/lib/tool_shed/scripts/api/get_filtered_repository_revisions.py @@ -26,69 +26,19 @@ import urllib sys.path.insert( 0, os.path.dirname( __file__ ) ) + from common import get +from common import get_api_url +from common import get_repository_dict +from common import json_from_url + +from galaxy.util import asbool from galaxy.util.json import from_json_string import tool_shed.util.shed_util_common as suc -def get_api_url( base, parts=[], params=None ): - if 'api' in parts and parts.index( 'api' ) != 0: - parts.pop( parts.index( 'api' ) ) - parts.insert( 0, 'api' ) - elif 'api' not in parts: - parts.insert( 0, 'api' ) - url = suc.url_join( base, *parts ) - if params: - url += '?%s' % params - return url - -def get_latest_downloadable_changeset_revision( url, name, owner ): - error_message = '' - parts = [ 'api', 'repositories', 'get_ordered_installable_revisions' ] - params = urllib.urlencode( dict( name=name, owner=owner ) ) - api_url = get_api_url( base=url, parts=parts, params=params ) - changeset_revisions, error_message = json_from_url( api_url ) - if error_message: - return None, error_message - if changeset_revisions: - return changeset_revisions[ -1 ], error_message - else: - return suc.INITIAL_CHANGELOG_HASH, error_message - -def get_repository_dict( url, repository_dict ): - error_message = '' - parts = [ 'api', 'repositories', repository_dict[ 'repository_id' ] ] - api_url = get_api_url( base=url, parts=parts ) - extended_dict, error_message = json_from_url( api_url ) - if error_message: - return None, error_message - name = str( extended_dict[ 'name' ] ) - owner = str( extended_dict[ 'owner' ] ) - latest_changeset_revision, error_message = get_latest_downloadable_changeset_revision( url, name, owner ) - if error_message: - print error_message - extended_dict[ 'latest_revision' ] = str( latest_changeset_revision ) - return extended_dict, error_message - -def json_from_url( url ): - error_message = '' - url_handle = urllib.urlopen( url ) - url_contents = url_handle.read() - try: - parsed_json = from_json_string( url_contents ) - except Exception, e: - error_message = str( url_contents ) - return None, error_message - return parsed_json, error_message - -def string_as_bool( string ): - if str( string ).lower() in [ 'true' ]: - return True - else: - return False - def main( options ): base_tool_shed_url = options.tool_shed_url.rstrip( '/' ) - latest_revision_only = string_as_bool( options.latest_revision_only ) + latest_revision_only = asbool( options.latest_revision_only ) do_not_test = str( options.do_not_test ) downloadable = str( options.downloadable ) includes_tools = str( options.includes_tools ) @@ -108,30 +58,31 @@ tools_functionally_correct=tools_functionally_correct ) ) api_url = get_api_url( base=base_tool_shed_url, parts=parts, params=params ) baseline_repository_dicts, error_message = json_from_url( api_url ) - if error_message: + if baseline_repository_dicts is None or error_message: print error_message - repository_dicts = [] - for baseline_repository_dict in baseline_repository_dicts: - # We need to get some details from the tool shed API, such as repository name and owner, to pass on to the - # module that will generate the install methods. - repository_dict, error_message = get_repository_dict( base_tool_shed_url, baseline_repository_dict ) - if error_message: - print 'Error getting additional details from the API: ', error_message - repository_dicts.append( baseline_repository_dict ) - else: - # Don't test empty repositories. - changeset_revision = baseline_repository_dict[ 'changeset_revision' ] - if changeset_revision != suc.INITIAL_CHANGELOG_HASH: - # Merge the dictionary returned from /api/repository_revisions with the detailed repository_dict and - # append it to the list of repository_dicts to install and test. - if latest_revision_only: - latest_revision = repository_dict[ 'latest_revision' ] - if changeset_revision == latest_revision: + else: + repository_dicts = [] + for baseline_repository_dict in baseline_repository_dicts: + # We need to get additional details from the tool shed API to pass on to the + # module that will generate the install methods. + repository_dict, error_message = get_repository_dict( base_tool_shed_url, baseline_repository_dict ) + if error_message: + print 'Error getting additional details from the API: ', error_message + repository_dicts.append( baseline_repository_dict ) + else: + # Don't test empty repositories. + changeset_revision = baseline_repository_dict.get( 'changeset_revision', suc.INITIAL_CHANGELOG_HASH ) + if changeset_revision != suc.INITIAL_CHANGELOG_HASH: + # Merge the dictionary returned from /api/repository_revisions with the detailed repository_dict and + # append it to the list of repository_dicts to install and test. + if latest_revision_only: + latest_revision = repository_dict.get( 'latest_revision', suc.INITIAL_CHANGELOG_HASH ) + if changeset_revision == latest_revision: + repository_dicts.append( dict( repository_dict.items() + baseline_repository_dict.items() ) ) + else: repository_dicts.append( dict( repository_dict.items() + baseline_repository_dict.items() ) ) - else: - repository_dicts.append( dict( repository_dict.items() + baseline_repository_dict.items() ) ) - print '\n\n', repository_dicts - print '\nThe url:\n\n', api_url, '\n\nreturned ', len( repository_dicts ), ' repository dictionaries...' + print '\n\n', repository_dicts + print '\nThe url:\n\n', api_url, '\n\nreturned ', len( repository_dicts ), ' repository dictionaries...' if __name__ == '__main__': parser = argparse.ArgumentParser( description='Get a filtered list of repository dictionaries.' ) diff -r 17e9e3ffb571e8f200ee8bf195a3708b2be4792b -r 0643bbc23cc1c845516469e3fa07270318c0c1a0 test/install_and_test_tool_shed_repositories/base/util.py --- a/test/install_and_test_tool_shed_repositories/base/util.py +++ b/test/install_and_test_tool_shed_repositories/base/util.py @@ -3,10 +3,9 @@ cwd = os.getcwd() sys.path.append( cwd ) -new_path = [ os.path.join( cwd, "scripts" ), - os.path.join( cwd, "lib" ), +new_path = [ os.path.join( cwd, "lib" ), os.path.join( cwd, 'test' ), - os.path.join( cwd, 'scripts', 'api' ) ] + os.path.join( cwd, 'lib', 'tool_shed', 'scripts', 'api' ) ] new_path.extend( sys.path ) sys.path = new_path @@ -27,6 +26,10 @@ from datetime import datetime from datetime import timedelta +from common import get_api_url +from common import get_latest_downloadable_changeset_revision_via_api +from common import get_repository_dict +from common import json_from_url from common import update from galaxy.util import asbool @@ -267,18 +270,6 @@ version = str( tool_dependency_dict[ 'version' ] ) print "# %s %s version %s" % ( type, name, version ) -def get_api_url( base, parts=[], params=None ): - if 'api' in parts and parts.index( 'api' ) != 0: - parts.pop( parts.index( 'api' ) ) - parts.insert( 0, 'api' ) - elif 'api' not in parts: - parts.insert( 0, 'api' ) - url = suc.url_join( base, *parts ) - if params is not None: - query_string = urllib.urlencode( params ) - url += '?%s' % query_string - return url - def get_database_version( app ): ''' This method returns the value of the version column from the migrate_version table, using the provided app's SQLAlchemy session to determine @@ -296,19 +287,6 @@ break return version -def get_latest_downloadable_changeset_revision( url, name, owner ): - error_message = '' - parts = [ 'api', 'repositories', 'get_ordered_installable_revisions' ] - params = dict( name=name, owner=owner ) - api_url = get_api_url( base=url, parts=parts, params=params ) - changeset_revisions, error_message = json_from_url( api_url ) - if error_message: - return None, error_message - if changeset_revisions: - return changeset_revisions[ -1 ], error_message - else: - return suc.INITIAL_CHANGELOG_HASH, error_message - def get_missing_repository_dependencies( repository ): """ Return the entire list of missing repository dependencies for the received repository. The entire @@ -467,28 +445,6 @@ return None, error_message return repository_dependency_dicts, error_message -def get_repository_dict( url, repository_dict ): - error_message = '' - if not isinstance( repository_dict, dict ): - error_message = 'Invalid repository_dict received: %s' % str( repository_dict ) - return None, error_message - repository_id = repository_dict.get( 'repository_id', None ) - if repository_id is None: - error_message = 'Invalid repository_dict does not contain a repository_id entry: %s' % str( repository_dict ) - return None, error_message - parts = [ 'api', 'repositories', repository_id ] - api_url = get_api_url( base=url, parts=parts ) - extended_dict, error_message = json_from_url( api_url ) - if error_message: - return None, error_message - name = str( extended_dict[ 'name' ] ) - owner = str( extended_dict[ 'owner' ] ) - latest_changeset_revision, error_message = get_latest_downloadable_changeset_revision( url, name, owner ) - if error_message: - return None, error_message - extended_dict[ 'latest_revision' ] = str( latest_changeset_revision ) - return extended_dict, error_message - def get_repository_dependencies_dicts( url, encoded_repository_metadata_id ): """ Return a list if dictionaries that define the repository dependencies of the repository defined by the @@ -709,23 +665,23 @@ return False, None def is_latest_downloadable_revision( url, repository_dict ): - name = str( repository_dict[ 'name' ] ) - owner = str( repository_dict[ 'owner' ] ) - changeset_revision = str( repository_dict[ 'changeset_revision' ] ) - latest_revision = get_latest_downloadable_changeset_revision( url, name=name, owner=owner ) - return changeset_revision == str( latest_revision ) - -def json_from_url( url ): + """ + Return True if the changeset_revision defined in the received repository_dict is the latest + installable revision for the repository. + """ error_message = '' - url_handle = urllib.urlopen( url ) - url_contents = url_handle.read() - try: - parsed_json = from_json_string( url_contents ) - except Exception, e: - error_message = str( url_contents ) - log.exception( 'Error parsing JSON data in json_from_url(): %s.' % str( e ) ) - return None, error_message - return parsed_json, error_message + name = repository_dict.get( 'name', None ) + owner = repository_dict.get( 'owner', None ) + changeset_revision = repository_dict.get( 'changeset_revision', None ) + if name is not None and owner is not None and changeset_revision is not None: + name = str( name ) + owner = str( owner ) + changeset_revision = str( changeset_revision ) + latest_revision, error_message = get_latest_downloadable_changeset_revision_via_api( url, name=name, owner=owner ) + if latest_revision is None or error_message: + return None, error_message + is_latest_downloadable = changeset_revision == str( latest_revision ) + return is_latest_downloadable, error_message def parse_exclude_list( xml_filename ): """Return a list of repositories to exclude from testing.""" @@ -1024,9 +980,15 @@ if can_update_tool_shed: metadata_revision_id = repository_dict.get( 'id', None ) if metadata_revision_id is not None: - name = str( repository_dict[ 'name' ] ) - owner = str( repository_dict[ 'owner' ] ) - changeset_revision = str( repository_dict[ 'changeset_revision' ] ) + name = repository_dict.get( 'name', None ) + owner = repository_dict.get( 'owner', None ) + changeset_revision = repository_dict.get( 'changeset_revision', None ) + if name is None or owner is None or changeset_revision is None: + log.debug( 'Entries for name, owner or changeset_revision missing from repository_dict %s' % str( repository_dict ) ) + return {} + name = str( name ) + owner = str( owner ) + changeset_revision = str( changeset_revision ) log.debug('\n=============================================================\n' ) log.debug( 'Inserting the following into tool_test_results for revision %s of repository %s owned by %s:\n%s' % \ ( changeset_revision, name, owner, str( tool_test_results_dict ) ) ) diff -r 17e9e3ffb571e8f200ee8bf195a3708b2be4792b -r 0643bbc23cc1c845516469e3fa07270318c0c1a0 test/install_and_test_tool_shed_repositories/repositories_with_tools/functional_tests.py --- a/test/install_and_test_tool_shed_repositories/repositories_with_tools/functional_tests.py +++ b/test/install_and_test_tool_shed_repositories/repositories_with_tools/functional_tests.py @@ -685,9 +685,16 @@ tool_test_results_dict[ 'failed_tests' ] = failed_test_dicts failed_repository_dict = repository_identifier_dict install_and_test_statistics_dict[ 'at_least_one_test_failed' ].append( failed_repository_dict ) - set_do_not_test = \ - not install_and_test_base_util.is_latest_downloadable_revision( install_and_test_base_util.galaxy_tool_shed_url, - repository_dict ) + is_latest_downloadable_revision, error_message = \ + install_and_test_base_util.is_latest_downloadable_revision( install_and_test_base_util.galaxy_tool_shed_url, + repository_dict ) + if is_latest_downloadable_revision is None or error_message: + log.debug( 'Error attempting to determine if revision %s of repository %s owned by %s ' % \ + ( changeset_revision, name, owner ) ) + log.debug( 'is the latest downloadable revision: %s' % str( error_message ) ) + set_do_not_test = False + else: + set_do_not_test = not is_latest_downloadable_revision params = dict( tools_functionally_correct=False, test_install_error=False, do_not_test=str( set_do_not_test ) ) Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: carlfeberhard: Remove extra includes in index.mako; Functional browser tests: create 'is loaded' fn for home page to simplify, improve loading in user module, misc. fixes
by commits-noreply＠bitbucket.org 09 Jan '14

09 Jan '14

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/17e9e3ffb571/ Changeset: 17e9e3ffb571 User: carlfeberhard Date: 2014-01-09 20:34:25 Summary: Remove extra includes in index.mako; Functional browser tests: create 'is loaded' fn for home page to simplify, improve loading in user module, misc. fixes Affected #: 13 files diff -r 4e6b003daaa44a06ac13c74c5ce8c75fa7bbb7ee -r 17e9e3ffb571e8f200ee8bf195a3708b2be4792b templates/webapps/galaxy/root/index.mako --- a/templates/webapps/galaxy/root/index.mako +++ b/templates/webapps/galaxy/root/index.mako @@ -1,9 +1,7 @@ <%inherit file="/webapps/galaxy/base_panels.mako"/><%namespace file="/root/tool_menu.mako" import="*" /> -<%namespace file="/history/history_panel.mako" import="history_panel_javascripts" /><%namespace file="/history/history_panel.mako" import="current_history_panel" /> -<%namespace file="/history/history_panel.mako" import="history_panel" /><%def name="stylesheets()"> ${parent.stylesheets()} diff -r 4e6b003daaa44a06ac13c74c5ce8c75fa7bbb7ee -r 17e9e3ffb571e8f200ee8bf195a3708b2be4792b test/casperjs/anon-history-tests.js --- a/test/casperjs/anon-history-tests.js +++ b/test/casperjs/anon-history-tests.js @@ -60,16 +60,8 @@ // =================================================================== TESTS -// ------------------------------------------------------------------- anonymous new, history -// open galaxy - ensure not logged in -spaceghost.thenOpen( spaceghost.baseUrl, function(){ - var loggedInAs = spaceghost.user.loggedInAs(); - this.debug( 'loggedInAs: ' + loggedInAs ); - if( loggedInAs ){ this.logout(); } -}); - -// ------------------------------------------------------------------- check the empty history for well formedness -spaceghost.historypanel.waitForHdas( function testPanelStructure(){ +// ------------------------------------------------------------------- check the anonymous new, history for form +spaceghost.openHomePage().historypanel.waitForHdas( function testPanelStructure(){ this.test.comment( 'history panel for anonymous user, new history' ); this.test.comment( "history name should exist, be visible, and have text " + unnamedName ); @@ -121,8 +113,7 @@ // ------------------------------------------------------------------- anon user can run tool on file // ------------------------------------------------------------------- anon user registers/logs in -> same history -spaceghost.user.loginOrRegisterUser( email, password ); -spaceghost.thenOpen( spaceghost.baseUrl, function(){ +spaceghost.user.loginOrRegisterUser( email, password ).openHomePage( function(){ this.test.comment( 'anon-user should login and be associated with previous history' ); var loggedInAs = spaceghost.user.loggedInAs(); @@ -139,8 +130,7 @@ }); // ------------------------------------------------------------------- logs out -> new history -spaceghost.user.logout(); -spaceghost.thenOpen( spaceghost.baseUrl, function(){ +spaceghost.user.logout().openHomePage( function(){ this.test.comment( 'logging out should create a new, anonymous history' ); this.historypanel.waitForHdas( function(){ diff -r 4e6b003daaa44a06ac13c74c5ce8c75fa7bbb7ee -r 17e9e3ffb571e8f200ee8bf195a3708b2be4792b test/casperjs/api-history-tests.js --- a/test/casperjs/api-history-tests.js +++ b/test/casperjs/api-history-tests.js @@ -304,13 +304,15 @@ //this.debug( 'returned:\n' + this.jsonStr( returned ) ); historyShow = this.api.histories.show( newFirstHistory.id ); this.test.assert( historyShow.genome_build === unicodeBuild, - "Update accepted unicode genome_build: " + historyShow.name ); + "Update accepted unicode genome_build: " + historyShow.genome_build ); } catch( err ){ //this.debug( this.jsonStr( err ) ); if( ( err instanceof this.api.APIError ) && ( err.status === 500 ) && ( err.message.indexOf( '(ProgrammingError) You must not use 8-bit bytestrings' ) !== -1 ) ){ this.skipTest( 'Unicode update failed. Are you using sqlite3 as the db?' ); + } else { + throw err; } } diff -r 4e6b003daaa44a06ac13c74c5ce8c75fa7bbb7ee -r 17e9e3ffb571e8f200ee8bf195a3708b2be4792b test/casperjs/hda-state-tests.js --- a/test/casperjs/hda-state-tests.js +++ b/test/casperjs/hda-state-tests.js @@ -62,17 +62,14 @@ peekShouldBeArray = []; // ------------------------------------------------------------------- set up -// start a new user +// start a new user and upload a file spaceghost.user.loginOrRegisterUser( email, password ); - -// upload a file spaceghost.then( function upload(){ spaceghost.tools.uploadFile( filepathToUpload, function uploadCallback( _uploadInfo ){ testUploadInfo = _uploadInfo; }); }); - // =================================================================== TEST HELPERS //NOTE: to be called with fn.call( spaceghost, ... ) diff -r 4e6b003daaa44a06ac13c74c5ce8c75fa7bbb7ee -r 17e9e3ffb571e8f200ee8bf195a3708b2be4792b test/casperjs/history-options-tests.js --- a/test/casperjs/history-options-tests.js +++ b/test/casperjs/history-options-tests.js @@ -47,9 +47,8 @@ // =================================================================== TESTS // ------------------------------------------------------------------- set up -// start a new user +// start a new user and upload a file spaceghost.user.loginOrRegisterUser( email, password ); - spaceghost.tools.uploadFile( filepathToUpload, function uploadCallback( _uploadInfo ){ testUploadInfo = _uploadInfo; }); diff -r 4e6b003daaa44a06ac13c74c5ce8c75fa7bbb7ee -r 17e9e3ffb571e8f200ee8bf195a3708b2be4792b test/casperjs/history-panel-tests.js --- a/test/casperjs/history-panel-tests.js +++ b/test/casperjs/history-panel-tests.js @@ -73,7 +73,7 @@ spaceghost.user.loginOrRegisterUser( email, password ); // ------------------------------------------------------------------- check structure of empty history -spaceghost.thenOpen( spaceghost.baseUrl ).historypanel.waitForHdas( function(){ +spaceghost.openHomePage().historypanel.waitForHdas( function(){ this.test.comment( 'history panel with a new, empty history should be well formed' ); this.test.comment( "history name should exist, be visible, and have text " + unnamedName ); @@ -173,9 +173,9 @@ this.test.assertVisible( nameSelector, 'History name is visible' ); this.test.assertSelectorHasText( nameSelector, newHistoryName, 'History name is ' + newHistoryName ); - this.test.comment( "history subtitle should display size and size should be " + onetxtFilesize + " bytes" ); var onetxtFilesize = require( 'fs' ).size( this.options.scriptDir + filepathToUpload ), expectedSubtitle = onetxtFilesize + ' bytes'; + this.test.comment( "history subtitle should display size and size should be " + onetxtFilesize + " bytes" ); this.test.assertExists( subtitleSelector, 'Found ' + subtitleSelector ); this.test.assertVisible( subtitleSelector, 'History subtitle is visible' ); this.test.assertSelectorHasText( subtitleSelector, expectedSubtitle, diff -r 4e6b003daaa44a06ac13c74c5ce8c75fa7bbb7ee -r 17e9e3ffb571e8f200ee8bf195a3708b2be4792b test/casperjs/login-tests.js --- a/test/casperjs/login-tests.js +++ b/test/casperjs/login-tests.js @@ -1,5 +1,3 @@ -// have to handle errors here - or phantom/casper won't bail but _HANG_ -//TODO: global error handler? try { var utils = require( 'utils' ), xpath = require( 'casper' ).selectXPath, @@ -25,12 +23,6 @@ phantom.exit( 1 ); } -// =================================================================== -/* TODO: - move selectors and assertText strings into global object for easier editing - - -*/ // =================================================================== globals and helpers var email = spaceghost.user.getRandomEmail(), password = '123456'; @@ -43,21 +35,14 @@ // =================================================================== TESTS // register a user (again...) -spaceghost.thenOpen( spaceghost.baseUrl, function(){ - this.test.comment( 'registering: ' + email ); - spaceghost.user.registerUser( email, password ); -}); +spaceghost.openHomePage() + .user.registerUser( email, password ) + .user.logout(); -// log them out - check for empty logged in text -spaceghost.then( function(){ - this.test.comment( 'logging out: ' + email ); - spaceghost.user.logout(); -}); -spaceghost.thenOpen( spaceghost.baseUrl, function(){ - spaceghost.waitForMasthead( function() { - this.test.assertDoesntExist( xpath( userEmailSelector ) ); - this.test.assert( spaceghost.user.loggedInAs() === '', 'loggedInAs() is empty string' ); - }); +spaceghost.openHomePage( function(){ + this.test.comment( 'log out should be reflected in user menu' ); + this.test.assertDoesntExist( xpath( userEmailSelector ) ); + this.test.assert( spaceghost.user.loggedInAs() === '', 'loggedInAs() is empty string' ); }); // log them back in - check for email in logged in text @@ -65,16 +50,13 @@ this.test.comment( 'logging back in: ' + email ); spaceghost.user._submitLogin( email, password ); //No such user }); -spaceghost.thenOpen( spaceghost.baseUrl, function(){ +spaceghost.openHomePage( function(){ this.test.assertSelectorHasText( xpath( userEmailSelector ), email ); this.test.assert( spaceghost.user.loggedInAs() === email, 'loggedInAs() matches email' ); }); // finally log back out for next tests -spaceghost.then( function(){ - this.test.comment( 'logging out: ' + email ); - spaceghost.user.logout(); -}); +spaceghost.user.logout(); // ------------------------------------------------------------------- shouldn't work // can't log in: users that don't exist, bad emails, sql injection (hurhur) @@ -120,10 +102,6 @@ }); }); }); - -spaceghost.then( function(){ - this.user.logout(); -}); /* */ // =================================================================== diff -r 4e6b003daaa44a06ac13c74c5ce8c75fa7bbb7ee -r 17e9e3ffb571e8f200ee8bf195a3708b2be4792b test/casperjs/modules/api.js --- a/test/casperjs/modules/api.js +++ b/test/casperjs/modules/api.js @@ -1,12 +1,4 @@ // =================================================================== module object, exports -/** Creates a new api module object. - * @param {SpaceGhost} spaceghost a spaceghost instance - * @exported - */ -exports.create = function createAPI( spaceghost, apikey ){ - return new API( spaceghost ); -}; - /** User object constructor. * @param {SpaceGhost} spaceghost a spaceghost instance * @param {String} apikey apikey for use when not using session authentication @@ -28,14 +20,21 @@ }; exports.API = API; +/** Creates a new api module object. + * @param {SpaceGhost} spaceghost a spaceghost instance + * @exported + */ +exports.create = function createAPI( spaceghost, apikey ){ + return new API( spaceghost ); +}; + + API.prototype.toString = function toString(){ return ( this.spaceghost + '.API:' + (( this.apikey )?( this.apikey ):( '(session)' )) ); }; // ------------------------------------------------------------------- APIError -APIError.prototype = new Error(); -APIError.prototype.constructor = Error; /** @class Thrown when Galaxy the API returns an error from a request */ function APIError( msg, status ){ Error.apply( this, arguments ); @@ -43,6 +42,8 @@ this.message = msg; this.status = status; } +APIError.prototype = new Error(); +APIError.prototype.constructor = Error; API.prototype.APIError = APIError; exports.APIError = APIError; diff -r 4e6b003daaa44a06ac13c74c5ce8c75fa7bbb7ee -r 17e9e3ffb571e8f200ee8bf195a3708b2be4792b test/casperjs/modules/tools.js --- a/test/casperjs/modules/tools.js +++ b/test/casperjs/modules/tools.js @@ -86,14 +86,12 @@ uploadInfo = {}; uploadInfo[ spaceghost.data.selectors.tools.upload.fileInput ] = filepath; - // click the upload tool - spaceghost.thenOpen( spaceghost.baseUrl, function(){ - // we can apprently click a tool label without expanding the tool container for it - this.waitForSelector( '.toolMenu', function(){ - this.click( xpath( '//a[contains(text(),"Upload File")]' ) ); - this.jumpToMain( function(){ - this.waitForSelector( 'body' ); - }); + spaceghost.openHomePage( function(){ + // load the upload tool form + // (we can apprently click a tool label without expanding the tool container for it) + this.click( xpath( '//a[contains(text(),"Upload File")]' ) ); + this.jumpToMain( function(){ + this.waitForSelector( 'body' ); }); }); diff -r 4e6b003daaa44a06ac13c74c5ce8c75fa7bbb7ee -r 17e9e3ffb571e8f200ee8bf195a3708b2be4792b test/casperjs/modules/user.js --- a/test/casperjs/modules/user.js +++ b/test/casperjs/modules/user.js @@ -43,11 +43,11 @@ confirm : ( confirm !== undefined )?( confirm ):( password ) }; - spaceghost.thenOpen( spaceghost.baseUrl, function(){ - this.waitForMasthead( function() { - this.clickLabel( spaceghost.data.labels.masthead.menus.user ); - this.clickLabel( spaceghost.data.labels.masthead.userMenu.register ); + spaceghost.openHomePage( function(){ + this.clickLabel( spaceghost.data.labels.masthead.menus.user ); + this.clickLabel( spaceghost.data.labels.masthead.userMenu.register ); + this.waitForNavigation( 'user/create', function beforeRegister(){ this.withMainPanel( function mainBeforeRegister(){ spaceghost.debug( '(' + spaceghost.getCurrentUrl() + ') registering user:\n' + spaceghost.jsonStr( userInfo ) ); @@ -56,11 +56,13 @@ this.click( xpath( spaceghost.data.selectors.registrationPage.submit_xpath ) ); }); }); - //// debugging - //spaceghost.withFrame( spaceghost.data.selectors.frames.main, function mainAfterRegister(){ - // var messageInfo = spaceghost.getElementInfo( spaceghost.data.selectors.messages.all ); - // spaceghost.debug( 'post registration message:\n' + spaceghost.jsonStr( messageInfo ) ); - //}); + + this.waitForNavigation( 'user/create', function afterRegister(){ + // this.withMainPanel( function mainAfterRegister(){ + // var messageInfo = spaceghost.getElementInfo( spaceghost.data.selectors.messages.all ); + // spaceghost.debug( 'post registration message:\n' + spaceghost.jsonStr( messageInfo ) ); + // }); + }); }); }; @@ -79,24 +81,23 @@ password: password }; - spaceghost.thenOpen( spaceghost.baseUrl, function(){ - spaceghost.waitForMasthead( function() { - spaceghost.clickLabel( spaceghost.data.labels.masthead.userMenu.login ); + spaceghost.openHomePage( function(){ + this.clickLabel( spaceghost.data.labels.masthead.menus.user ); + this.clickLabel( spaceghost.data.labels.masthead.userMenu.login ); - spaceghost.withMainPanel( function mainBeforeLogin(){ + this.waitForNavigation( 'user/login', function beforeLogin(){ + this.withMainPanel( function mainBeforeLogin(){ spaceghost.debug( '(' + spaceghost.getCurrentUrl() + ') logging in user:\n' + spaceghost.jsonStr( loginInfo ) ); spaceghost.fill( spaceghost.data.selectors.loginPage.form, loginInfo, false ); spaceghost.click( xpath( spaceghost.data.selectors.loginPage.submit_xpath ) ); }); + }); - //// debugging - //spaceghost.withFrame( spaceghost.data.selectors.frames.main, function mainAfterLogin(){ - // //TODO: prob. could use a more generalized form of this for url breakdown/checking - // if( spaceghost.getCurrentUrl().search( spaceghost.data.selectors.loginPage.url_regex ) != -1 ){ - // var messageInfo = spaceghost.getElementInfo( spaceghost.data.selectors.messages.all ); - // spaceghost.debug( 'post login message:\n' + spaceghost.jsonStr( messageInfo ) ); - // } + this.waitForNavigation( 'user/login', function afterLogin(){ + //this.withMainPanel( function mainAfterLogin(){ + // var messageInfo = spaceghost.getElementInfo( spaceghost.data.selectors.messages.all ); + // spaceghost.debug( 'post login message:\n' + spaceghost.jsonStr( messageInfo ) ); //}); }); }); @@ -122,6 +123,9 @@ this.warning( 'Registration failed: ' + messageInfo.text ); throw new spaceghost.GalaxyError( 'RegistrationError: ' + messageInfo.text ); } + + this.clickLabel( 'Return to the home page.' ); + this.waitForNavigation( '' ); }); return spaceghost; }; @@ -135,19 +139,21 @@ var spaceghost = this.spaceghost; this._submitLogin( email, password ); - spaceghost.waitForMasthead( function() { - spaceghost.withMainPanel( function mainAfterLogin(){ - if( spaceghost.getCurrentUrl().search( spaceghost.data.selectors.loginPage.url_regex ) !== -1 ){ - var messageInfo = spaceghost.getElementInfo( spaceghost.data.selectors.messages.all ); - if( messageInfo && messageInfo.attributes[ 'class' ] === 'errormessage' ){ - this.warning( 'Login failed: ' + messageInfo.text ); - throw new spaceghost.GalaxyError( 'LoginError: ' + messageInfo.text ); - } - } - if( spaceghost.user.loggedInAs() === email ){ - spaceghost.info( 'logged in as ' + email ); - } - }); + //spaceghost.withMainPanel( function mainAfterLogin(){ + // if( spaceghost.getCurrentUrl().search( spaceghost.data.selectors.loginPage.url_regex ) !== -1 ){ + // var messageInfo = spaceghost.getElementInfo( spaceghost.data.selectors.messages.all ); + // if( messageInfo && messageInfo.attributes[ 'class' ] === 'errormessage' ){ + // this.warning( 'Login failed: ' + messageInfo.text ); + // throw new spaceghost.GalaxyError( 'LoginError: ' + messageInfo.text ); + // } + // } + //}); + this.spaceghost.then( function checkLogin(){ + if( spaceghost.user.loggedInAs() !== email ){ + throw new spaceghost.GalaxyError( 'LoginError' ); + } else { + spaceghost.info( 'logged in as ' + email ); + } }); return spaceghost; }; @@ -156,19 +162,19 @@ * @returns {String} email of currently logged in user or '' if no one logged in */ User.prototype.loggedInAs = function loggedInAs(){ - var spaceghost = this.spaceghost, - userEmail = ''; - try { - var emailSelector = xpath( spaceghost.data.selectors.masthead.userMenu.userEmail_xpath ), - loggedInInfo = spaceghost.elementInfoOrNull( emailSelector ); - if( loggedInInfo !== null ){ - userEmail = loggedInInfo.text.replace( 'Logged in as ', '' ); + return this.spaceghost.jumpToTop( function(){ + var userEmail = ''; + try { + var emailSelector = xpath( this.data.selectors.masthead.userMenu.userEmail_xpath ), + loggedInInfo = this.elementInfoOrNull( emailSelector ); + if( loggedInInfo !== null ){ + userEmail = loggedInInfo.text.replace( 'Logged in as ', '' ); + } + } catch( err ){ + this.warn( err ); } - } catch( err ){ - spaceghost.warn( err ); - } - //console.debug( 'loggedInInfo:', spaceghost.jsonStr( loggedInInfo ) ); - return userEmail; + return userEmail; + }); }; /** Log out the current user @@ -176,14 +182,15 @@ */ User.prototype.logout = function logout(){ var spaceghost = this.spaceghost; - spaceghost.thenOpen( spaceghost.baseUrl, function(){ - this.info( 'user logging out' ); - spaceghost.waitForMasthead( function _logout() { - if( spaceghost.user.loggedInAs() ){ - spaceghost.clickLabel( spaceghost.data.labels.masthead.menus.user ); - spaceghost.clickLabel( spaceghost.data.labels.masthead.userMenu.logout ); - } - }); + this.spaceghost.openHomePage( function(){ + if( spaceghost.user.loggedInAs() ){ + spaceghost.clickLabel( spaceghost.data.labels.masthead.menus.user ); + spaceghost.clickLabel( spaceghost.data.labels.masthead.userMenu.logout ); + spaceghost.waitForNavigation( 'user/logout', function _toLogoutPage() { + spaceghost.clickLabel( 'go to the home page' ); + spaceghost.waitForNavigation( '' ); + }); + } }); return spaceghost; }; @@ -198,10 +205,10 @@ var spaceghost = this.spaceghost; // attempt a login, if that fails - register spaceghost.tryStepsCatch( function tryToLogin(){ - spaceghost.open( spaceghost.baseUrl ).user.login( email, password ); + spaceghost.openHomePage().user.login( email, password ); }, function failedLoginRegister(){ - spaceghost.open( spaceghost.baseUrl ).user.registerUser( email, password, username ); + spaceghost.openHomePage().user.registerUser( email, password, username ); }); return spaceghost; }; @@ -216,7 +223,8 @@ // check for the setting in sg and the universe_wsgi.ini file var adminData = this.spaceghost.options.adminUser, iniAdminEmails = this.spaceghost.getUniverseSetting( 'admin_users' ); - iniAdminEmails = ( iniAdminEmails )?( iniAdminEmails.split( ',' ).map( function( email ) { return email.trim(); } ) ):( null ); + iniAdminEmails = ( iniAdminEmails )? + ( iniAdminEmails.split( ',' ).map( function( email ) { return email.trim(); } ) ):( null ); //TODO: seems like we only need the wsgi setting - that's the only thing we can't change if( adminData ){ diff -r 4e6b003daaa44a06ac13c74c5ce8c75fa7bbb7ee -r 17e9e3ffb571e8f200ee8bf195a3708b2be4792b test/casperjs/registration-tests.js --- a/test/casperjs/registration-tests.js +++ b/test/casperjs/registration-tests.js @@ -1,4 +1,3 @@ -// have to handle errors here - or phantom/casper won't bail but _HANG_ try { var utils = require( 'utils' ), xpath = require( 'casper' ).selectXPath, @@ -24,13 +23,6 @@ phantom.exit( 1 ); } -// =================================================================== -/* TODO: - move selectors and assertText strings into global object for easier editing - pass email, etc. for first (successful) registration (for use with other tests) - - -*/ // =================================================================== globals and helpers var email = spaceghost.user.getRandomEmail(), password = '123456', @@ -38,50 +30,27 @@ username = 'test' + Date.now(); // =================================================================== TESTS -spaceghost.thenOpen( spaceghost.baseUrl, function(){ +spaceghost.openHomePage( function(){ this.test.comment( 'loading galaxy homepage' ); - // can we load galaxy? this.test.assertTitle( 'Galaxy' ); - // xpath selector use: this.test.assertExists( xpath( "//div[@id='masthead']" ), 'found masthead' ); }); -// failing tests for...testing...the tests -//spaceghost.thenOpen( spaceghost.baseUrl, function(){ -// this.test.comment( 'loading galaxy homepage' ); -// // can we load galaxy? -// this.test.assertTitle( 'Blorgo' ); -// // xpath selector use: -// this.test.assertExists( xpath( "//div[@id='facebook']" ), 'found facebook' ); -//}); - - // ------------------------------------------------------------------- register a new user spaceghost.then( function(){ this.test.comment( 'registering user: ' + email ); this.user._submitRegistration( email, password, username, confirm ); }); -spaceghost.thenOpen( spaceghost.baseUrl, function(){ - this.waitForMasthead( function() { - this.clickLabel( 'User' ); - this.test.assertSelectorHasText( xpath( spaceghost.data.selectors.masthead.userMenu.userEmail_xpath ), - email, '#user-email === ' + email ); - }); +spaceghost.openHomePage( function(){ + this.clickLabel( 'User' ); + var loggedInAs = this.fetchText( xpath( spaceghost.data.selectors.masthead.userMenu.userEmail_xpath ) ); + this.test.assert( loggedInAs.indexOf( email ) !== -1, 'found proper email in user menu: ' + loggedInAs ); }); // ------------------------------------------------------------------- log out that user -spaceghost.then( function(){ - this.waitForMasthead( function() { - this.test.comment( 'logging out user: ' + email ); - this.user.logout(); - }); -}); -spaceghost.then( function(){ - this.waitForMasthead( function() { - var emailSelector = xpath( this.data.selectors.masthead.userMenu.userEmail_xpath ); - this.debug( 'email:' + this.elementInfoOrNull( emailSelector ) ); - this.test.assert( !this.elementInfoOrNull( emailSelector ), 'user email not found' ); - }); +spaceghost.user.logout().openHomePage( function(){ + var emailSelector = xpath( this.data.selectors.masthead.userMenu.userEmail_xpath ); + this.test.assert( !this.elementInfoOrNull( emailSelector ), 'user email not found' ); }); // ------------------------------------------------------------------- bad user registrations @@ -165,7 +134,6 @@ this.assertErrorMessage( 'Public name is taken; please choose another' ); }); - // ------------------------------------------------------------------- test the convenience fns // these versions are for conv. use in other tests, they should throw errors if used improperly spaceghost.then( function(){ @@ -177,11 +145,6 @@ }); }); -spaceghost.then( function(){ - //??: necessary? - this.user.logout(); -}); - // =================================================================== spaceghost.run( function(){ this.test.done(); diff -r 4e6b003daaa44a06ac13c74c5ce8c75fa7bbb7ee -r 17e9e3ffb571e8f200ee8bf195a3708b2be4792b test/casperjs/spaceghost.js --- a/test/casperjs/spaceghost.js +++ b/test/casperjs/spaceghost.js @@ -507,6 +507,38 @@ Casper.prototype.run.call( this, new_onComplete, time ); }; +// ------------------------------------------------------------------- home page +/** Wait for the homepage/index/Analyze Data to load fully. + */ +SpaceGhost.prototype.openHomePage = function openHomePage( then, delay ){ +//TODO: delay doesn't seem to work + this.thenOpen( this.baseUrl, function _openHomePage(){ + this.waitFor( + function waitForCheck(){ + return this.homePageIsLoaded(); + }, + then, + function openHomePageTimeout(){ + throw new GalaxyError( 'Homepage timed out' ); + }, + delay + ); + }); + return this; +}; + +/** Check for visibility of main home page elements: masthead, tool menu, history panel. + */ +SpaceGhost.prototype.homePageIsLoaded = function homePageIsLoaded(){ + //this.debug( 'homePageIsLoaded: ' + [ + // this.visible( '#masthead' ), + // this.visible( this.data.selectors.toolMenu.container ), + // this.visible( '#current-history-panel' )].join( ', ' ) ); + return ( this.visible( '#masthead' ) + && this.visible( this.data.selectors.toolMenu.container ) + && this.visible( '#current-history-panel' ) ); +}; + // ------------------------------------------------------------------- try step /** Install a function as an error handler temporarily, run a function with steps, then remove the handler. * A rough stand-in for try catch with steps. @@ -549,6 +581,7 @@ }); }; + // ------------------------------------------------------------------- misc /** Hover over an element. * NOTE: not for use with iframes (main, tool, history) - they need to re-calc @@ -662,15 +695,6 @@ return returned; }; -///** Jumps into history frame, exectutes fn, and jumps back to original frame. -// * @param {Selector} frame the selector for the frame to jump to -// * @param {Function} fn function called when in the frame -// * @returns {Any} the return value of fn -// */ -//SpaceGhost.prototype.jumpToHistory = function jumpToHistory( fn ){ -// return this.jumpToFrame( this.data.selectors.frames.history, fn ); -//}; - /** Jumps into main frame, exectutes fn, and jumps back to original frame. * @param {Selector} frame the selector for the frame to jump to * @param {Function} fn function called when in the frame @@ -862,6 +886,7 @@ this.test.assert( classes.indexOf( className ) === -1, msg ); }; + // =================================================================== CONVENIENCE /** Wraps casper.getElementInfo in try, returning null if element not found instead of erroring. * @param {String} selector css or xpath selector for the element to find @@ -1096,7 +1121,7 @@ SpaceGhost.prototype.waitForMasthead = function wait( then ) { return this.waitForText( this.data.labels.masthead.menus.user, then ); -} +}; // =================================================================== TEST DATA @@ -1118,16 +1143,23 @@ }, frames : { - main : 'galaxy_main', - history : 'galaxy_history' + main : 'galaxy_main' }, masthead : { - adminLink : '#masthead a[href="/admin/index"]', - userMenu : { + id : '#masthead', + adminLink : '#masthead a[href="/admin/index"]', + userMenu : { userEmail_xpath : '//a[contains(text(),"Logged in as")]' } }, + toolMenu : { + container : '.toolMenuContainer' + }, + historyPanel : { + current : '#current-history-panel' + }, + loginPage : { form : 'form#login', submit_xpath : "//input[@value='Login']", @@ -1135,7 +1167,8 @@ }, registrationPage : { form : 'form#registration', - submit_xpath : "//input[@value='Submit']" + submit_xpath : "//input[@value='Submit']", + returnLink : '//a[contains(text(),"Return to the home page")]' }, tools : { general : { diff -r 4e6b003daaa44a06ac13c74c5ce8c75fa7bbb7ee -r 17e9e3ffb571e8f200ee8bf195a3708b2be4792b test/casperjs/upload-tests.js --- a/test/casperjs/upload-tests.js +++ b/test/casperjs/upload-tests.js @@ -43,16 +43,12 @@ // =================================================================== TESTS // ------------------------------------------------------------------- start a new user -spaceghost.user.loginOrRegisterUser( email, password ); -//??: why is a reload needed here? If we don't, loggedInAs === '' ... -spaceghost.thenOpen( spaceghost.baseUrl, function(){ +spaceghost.user.loginOrRegisterUser( email, password ).openHomePage( function(){ var loggedInAs = spaceghost.user.loggedInAs(); this.test.assert( loggedInAs === email, 'loggedInAs() matches email: "' + loggedInAs + '"' ); }); - // ------------------------------------------------------------------- long form - // upload a file... spaceghost.then( function(){ this.test.comment( 'Test uploading a file' ); Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: guerler: Merge
by commits-noreply＠bitbucket.org 09 Jan '14

09 Jan '14

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/4e6b003daaa4/ Changeset: 4e6b003daaa4 User: guerler Date: 2014-01-09 20:06:19 Summary: Merge Affected #: 5 files diff -r b3a603f522174fa1fc4623580cce39e4069b189c -r 4e6b003daaa44a06ac13c74c5ce8c75fa7bbb7ee scripts/cleanup_datasets/pgcleanup.py --- a/scripts/cleanup_datasets/pgcleanup.py +++ b/scripts/cleanup_datasets/pgcleanup.py @@ -105,7 +105,7 @@ url = make_url(self.config.database_connection) log.info('Connecting to database with URL: %s' % url) - args = url.translate_connect_args() + args = url.translate_connect_args( username='user' ) args.update(url.query) assert url.get_dialect().name == 'postgresql', 'This script can only be used with PostgreSQL.' Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: natefoo: Log native spec on drmaa job submission.
by commits-noreply＠bitbucket.org 09 Jan '14

09 Jan '14

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/b3a603f52217/ Changeset: b3a603f52217 User: natefoo Date: 2014-01-09 19:27:47 Summary: Log native spec on drmaa job submission. Affected #: 1 file diff -r b0cb955a9b284bea1423e613be7365ef5df8a071 -r b3a603f522174fa1fc4623580cce39e4069b189c lib/galaxy/jobs/runners/drmaa.py --- a/lib/galaxy/jobs/runners/drmaa.py +++ b/lib/galaxy/jobs/runners/drmaa.py @@ -163,8 +163,10 @@ job_wrapper.cleanup() return - log.debug( "(%s) submitting file %s" % ( galaxy_id_tag, ajs.job_file ) ) - log.debug( "(%s) command is: %s" % ( galaxy_id_tag, command_line ) ) + log.debug( "(%s) submitting file %s", galaxy_id_tag, ajs.job_file ) + log.debug( "(%s) command is: %s", galaxy_id_tag, command_line ) + if native_spec: + log.debug( "(%s) native specification is: %s", galaxy_id_tag, native_spec ) # runJob will raise if there's a submit problem if self.external_runJob_script is None: Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: natefoo: Merge 26da838b778c from stable.
by commits-noreply＠bitbucket.org 09 Jan '14

09 Jan '14

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/b0cb955a9b28/ Changeset: b0cb955a9b28 User: natefoo Date: 2014-01-09 19:09:52 Summary: Merge 26da838b778c from stable. Affected #: 1 file Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: inithello: Merged in 11734:e059c6607f34 from stable.
by commits-noreply＠bitbucket.org 09 Jan '14

09 Jan '14

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/d8f02450982c/ Changeset: d8f02450982c User: inithello Date: 2014-01-09 18:58:55 Summary: Merged in 11734:e059c6607f34 from stable. Affected #: 3 files Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: 6 new changesets
by commits-noreply＠bitbucket.org 09 Jan '14

09 Jan '14

6 new commits in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/229064ed6add/ Changeset: 229064ed6add User: natefoo Date: 2014-01-07 21:13:01 Summary: Allow handlers to control which runner plugins they will load. Affected #: 2 files diff -r 968389346b3491bf1a90cf359ff258075816b1e8 -r 229064ed6addac469460190415360e553ae91a0f lib/galaxy/jobs/__init__.py --- a/lib/galaxy/jobs/__init__.py +++ b/lib/galaxy/jobs/__init__.py @@ -86,6 +86,7 @@ self.app = app self.runner_plugins = [] self.handlers = {} + self.handler_runner_plugins = {} self.default_handler_id = None self.destinations = {} self.destination_tags = {} @@ -138,6 +139,10 @@ else: log.debug("Read definition for handler '%s'" % id) self.handlers[id] = (id,) + for plugin in handler.findall('plugin'): + if id not in self.handler_runner_plugins: + self.handler_runner_plugins[id] = [] + self.handler_runner_plugins[id].append( plugin.get('id') ) if handler.get('tags', None) is not None: for tag in [ x.strip() for x in handler.get('tags').split(',') ]: if tag in self.handlers: @@ -420,13 +425,19 @@ """ return self.destinations.get(id_or_tag, None) - def get_job_runner_plugins(self): + def get_job_runner_plugins(self, handler_id): """Load all configured job runner plugins :returns: list of job runner plugins """ rval = {} - for runner in self.runner_plugins: + if handler_id in self.handler_runner_plugins: + plugins_to_load = [ rp for rp in self.runner_plugins if rp['id'] in self.handler_runner_plugins[handler_id] ] + log.info( "Handler '%s' will load specified runner plugins: %s", handler_id, ', '.join( [ rp['id'] for rp in plugins_to_load ] ) ) + else: + plugins_to_load = self.runner_plugins + log.info( "Handler '%s' will load all configured runner plugins", handler_id ) + for runner in plugins_to_load: class_names = [] module = None id = runner['id'] @@ -477,7 +488,7 @@ try: rval[id] = runner_class( self.app, runner[ 'workers' ], **runner.get( 'kwds', {} ) ) except TypeError: - log.warning( "Job runner '%s:%s' has not been converted to a new-style runner" % ( module_name, class_name ) ) + log.exception( "Job runner '%s:%s' has not been converted to a new-style runner or encountered TypeError on load" % ( module_name, class_name ) ) rval[id] = runner_class( self.app ) log.debug( "Loaded job runner '%s:%s' as '%s'" % ( module_name, class_name, id ) ) return rval diff -r 968389346b3491bf1a90cf359ff258075816b1e8 -r 229064ed6addac469460190415360e553ae91a0f lib/galaxy/jobs/handler.py --- a/lib/galaxy/jobs/handler.py +++ b/lib/galaxy/jobs/handler.py @@ -565,7 +565,7 @@ class DefaultJobDispatcher( object ): def __init__( self, app ): self.app = app - self.job_runners = self.app.job_config.get_job_runner_plugins() + self.job_runners = self.app.job_config.get_job_runner_plugins( self.app.config.server_name ) # Once plugins are loaded, all job destinations that were created from # URLs can have their URL params converted to the destination's param # dict by the plugin. https://bitbucket.org/galaxy/galaxy-central/commits/681643c22fed/ Changeset: 681643c22fed User: natefoo Date: 2014-01-07 21:18:11 Summary: Allow runners plugins to accept params, provide some simple load-time validation logic for params. Affected #: 1 file diff -r 229064ed6addac469460190415360e553ae91a0f -r 681643c22fedeac632567ae6bcf204616d871f07 lib/galaxy/jobs/runners/__init__.py --- a/lib/galaxy/jobs/runners/__init__.py +++ b/lib/galaxy/jobs/runners/__init__.py @@ -22,13 +22,39 @@ STOP_SIGNAL = object() + +class RunnerParams( object ): + + def __init__( self, specs = None, params = None ): + self.specs = specs or dict() + self.params = params or dict() + for name, value in self.params.items(): + assert name in self.specs, 'Invalid job runner parameter for this plugin: %s' % name + if 'map' in self.specs[ name ]: + try: + self.params[ name ] = self.specs[ name ][ 'map' ]( value ) + except Exception, e: + raise Exception( 'Job runner parameter "%s" value "%s" could not be converted to the correct type: %s' % ( name, value, e ) ) + if 'valid' in self.specs[ name ]: + assert self.specs[ name ][ 'valid' ]( value ), 'Job runner parameter %s failed validation' % name + + def __getattr__( self, name ): + return self.params.get( name, self.specs[ name ][ 'default' ] ) + + class BaseJobRunner( object ): - def __init__( self, app, nworkers ): + def __init__( self, app, nworkers, **kwargs ): """Start the job runner """ self.app = app self.sa_session = app.model.context self.nworkers = nworkers + runner_param_specs = dict( recheck_missing_job_retries = dict( map = int, valid = lambda x: x >= 0, default = 0 ) ) + if 'runner_param_specs' in kwargs: + runner_param_specs.update( kwargs.pop( 'runner_param_specs' ) ) + if kwargs: + log.debug( 'Loading %s with params: %s', self.runner_name, kwargs ) + self.runner_params = RunnerParams( specs = runner_param_specs, params = kwargs ) def _init_worker_threads(self): """Start ``nworkers`` worker threads. @@ -115,7 +141,7 @@ job_wrapper.cleanup() return False elif job_state != model.Job.states.QUEUED: - log.info( "(%d) Job is in state %s, skipping execution" % ( job_id, job_state ) ) + log.info( "(%s) Job is in state %s, skipping execution" % ( job_id, job_state ) ) # cleanup may not be safe in all states return False @@ -287,8 +313,8 @@ to the correct methods (queue, finish, cleanup) at appropriate times.. """ - def __init__( self, app, nworkers ): - super( AsynchronousJobRunner, self ).__init__( app, nworkers ) + def __init__( self, app, nworkers, **kwargs ): + super( AsynchronousJobRunner, self ).__init__( app, nworkers, **kwargs ) # 'watched' and 'queue' are both used to keep track of jobs to watch. # 'queue' is used to add new watched jobs, and can be called from # any thread (usually by the 'queue_job' method). 'watched' must only https://bitbucket.org/galaxy/galaxy-central/commits/e67776961e48/ Changeset: e67776961e48 User: natefoo Date: 2014-01-07 21:21:50 Summary: Make job terminal state logic somewhat configurable via runner params, update DRMAA runner plugin changes for configurable terminal state handling. Also allow the drmaa plugin to read the drmaa library path from job_conf.xml rather than the $DRMAA_LIBRARY_PATH environment variable. Affected #: 2 files diff -r 681643c22fedeac632567ae6bcf204616d871f07 -r e67776961e48dca45850c95b0f053178fd0e61b6 lib/galaxy/jobs/runners/__init__.py --- a/lib/galaxy/jobs/runners/__init__.py +++ b/lib/galaxy/jobs/runners/__init__.py @@ -252,6 +252,10 @@ options.update(**kwds) return job_script(**options) + def _complete_terminal_job( self, ajs, **kwargs ): + if ajs.job_wrapper.get_state() != model.Job.states.DELETED: + self.work_queue.put( ( self.finish_job, ajs ) ) + class AsynchronousJobState( object ): """ diff -r 681643c22fedeac632567ae6bcf204616d871f07 -r e67776961e48dca45850c95b0f053178fd0e61b6 lib/galaxy/jobs/runners/drmaa.py --- a/lib/galaxy/jobs/runners/drmaa.py +++ b/lib/galaxy/jobs/runners/drmaa.py @@ -16,27 +16,12 @@ from galaxy.jobs.runners import AsynchronousJobState, AsynchronousJobRunner eggs.require( "drmaa" ) -# We foolishly named this file the same as the name exported by the drmaa -# library... 'import drmaa' imports itself. -drmaa = __import__( "drmaa" ) log = logging.getLogger( __name__ ) __all__ = [ 'DRMAAJobRunner' ] -drmaa_state = { - drmaa.JobState.UNDETERMINED: 'process status cannot be determined', - drmaa.JobState.QUEUED_ACTIVE: 'job is queued and active', - drmaa.JobState.SYSTEM_ON_HOLD: 'job is queued and in system hold', - drmaa.JobState.USER_ON_HOLD: 'job is queued and in user hold', - drmaa.JobState.USER_SYSTEM_ON_HOLD: 'job is queued and in user and system hold', - drmaa.JobState.RUNNING: 'job is running', - drmaa.JobState.SYSTEM_SUSPENDED: 'job is system suspended', - drmaa.JobState.USER_SUSPENDED: 'job is user suspended', - drmaa.JobState.DONE: 'job finished normally', - drmaa.JobState.FAILED: 'job finished, but failed', -} - +drmaa = None DRMAA_jobTemplate_attributes = [ 'args', 'remoteCommand', 'outputPath', 'errorPath', 'nativeSpecification', 'jobName', 'email', 'project' ] @@ -48,8 +33,50 @@ """ runner_name = "DRMAARunner" - def __init__( self, app, nworkers ): + def __init__( self, app, nworkers, **kwargs ): """Start the job runner""" + + global drmaa + + runner_param_specs = dict( + drmaa_library_path = dict( map = str, default = os.environ.get( 'DRMAA_LIBRARY_PATH', None ) ), + invalidjobexception_state = dict( map = str, valid = lambda x: x in ( model.Job.states.OK, model.Job.states.ERROR ), default = model.Job.states.OK ), + invalidjobexception_retries = dict( map = int, valid = lambda x: int >= 0, default = 0 ), + internalexception_state = dict( map = str, valid = lambda x: x in ( model.Job.states.OK, model.Job.states.ERROR ), default = model.Job.states.OK ), + internalexception_retries = dict( map = int, valid = lambda x: int >= 0, default = 0 ) ) + + if 'runner_param_specs' not in kwargs: + kwargs[ 'runner_param_specs' ] = dict() + kwargs[ 'runner_param_specs' ].update( runner_param_specs ) + + super( DRMAAJobRunner, self ).__init__( app, nworkers, **kwargs ) + + # This allows multiple drmaa runners (although only one per handler) in the same job config file + if 'drmaa_library_path' in kwargs: + log.info( 'Overriding DRMAA_LIBRARY_PATH due to runner plugin parameter: %s', self.runner_params.drmaa_library_path ) + os.environ['DRMAA_LIBRARY_PATH'] = self.runner_params.drmaa_library_path + + # We foolishly named this file the same as the name exported by the drmaa + # library... 'import drmaa' imports itself. + drmaa = __import__( "drmaa" ) + + # Subclasses may need access to state constants + self.drmaa_job_states = drmaa.JobState + + # Descriptive state strings pulled from the drmaa lib itself + self.drmaa_job_state_strings = { + drmaa.JobState.UNDETERMINED: 'process status cannot be determined', + drmaa.JobState.QUEUED_ACTIVE: 'job is queued and active', + drmaa.JobState.SYSTEM_ON_HOLD: 'job is queued and in system hold', + drmaa.JobState.USER_ON_HOLD: 'job is queued and in user hold', + drmaa.JobState.USER_SYSTEM_ON_HOLD: 'job is queued and in user and system hold', + drmaa.JobState.RUNNING: 'job is running', + drmaa.JobState.SYSTEM_SUSPENDED: 'job is system suspended', + drmaa.JobState.USER_SUSPENDED: 'job is user suspended', + drmaa.JobState.DONE: 'job finished normally', + drmaa.JobState.FAILED: 'job finished, but failed', + } + self.ds = drmaa.Session() self.ds.initialize() @@ -58,7 +85,6 @@ self.external_killJob_script = app.config.drmaa_external_killjob_script self.userid = None - super( DRMAAJobRunner, self ).__init__( app, nworkers ) self._init_monitor_thread() self._init_worker_threads() @@ -175,6 +201,20 @@ # Add to our 'queue' of jobs to monitor self.monitor_queue.put( ajs ) + def _complete_terminal_job( self, ajs, drmaa_state, **kwargs ): + """ + Handle a job upon its termination in the DRM. This method is meant to + be overridden by subclasses to improve post-mortem and reporting of + failures. + """ + if drmaa_state == drmaa.JobState.FAILED: + if ajs.job_wrapper.get_state() != model.Job.states.DELETED: + ajs.stop_job = False + ajs.fail_message = "The cluster DRM system terminated this job" + self.work_queue.put( ( self.fail_job, ajs ) ) + elif drmaa_state == drmaa.JobState.DONE: + super( DRMAAJobRunner, self )._complete_terminal_job( ajs ) + def check_watched_items( self ): """ Called by the monitor thread to look at each watched job and deal @@ -188,16 +228,27 @@ try: assert external_job_id not in ( None, 'None' ), '(%s/%s) Invalid job id' % ( galaxy_id_tag, external_job_id ) state = self.ds.jobStatus( external_job_id ) - # InternalException was reported to be necessary on some DRMs, but - # this could cause failures to be detected as completion! Please - # report if you experience problems with this. - except ( drmaa.InvalidJobException, drmaa.InternalException ), e: - # we should only get here if an orphaned job was put into the queue at app startup - log.info( "(%s/%s) job left DRM queue with following message: %s" % ( galaxy_id_tag, external_job_id, e ) ) - self.work_queue.put( ( self.finish_job, ajs ) ) + except ( drmaa.InternalException, drmaa.InvalidJobException ), e: + ecn = e.__class__.__name__ + retry_param = ecn.lower() + '_retries' + state_param = ecn.lower() + '_state' + retries = getattr( ajs, retry_param, 0 ) + if self.runner_params[ retry_param ] > 0: + if retries < self.runner_params[ retry_param ]: + # will retry check on next iteration + setattr( ajs, retry_param, retries + 1 ) + continue + if self.runner_params[ state_param ] == model.Job.states.OK: + log.info( "(%s/%s) job left DRM queue with following message: %s", galaxy_id_tag, external_job_id, e ) + self.work_queue.put( ( self.finish_job, ajs ) ) + elif self.runner_params[ state_param ] == model.Job.states.ERROR: + log.info( "(%s/%s) job check resulted in %s after %s tries: %s", galaxy_id_tag, external_job_id, ecn, retries, e ) + self.work_queue.put( ( self.fail_job, ajs ) ) + else: + raise Exception( "%s is set to an invalid value (%s), this should not be possible. See galaxy.jobs.drmaa.__init__()", state_param, self.runner_params[ state_param ] ) continue except drmaa.DrmCommunicationException, e: - log.warning( "(%s/%s) unable to communicate with DRM: %s" % ( galaxy_id_tag, external_job_id, e )) + log.warning( "(%s/%s) unable to communicate with DRM: %s", galaxy_id_tag, external_job_id, e ) new_watched.append( ajs ) continue except Exception, e: @@ -208,19 +259,12 @@ self.work_queue.put( ( self.fail_job, ajs ) ) continue if state != old_state: - log.debug( "(%s/%s) state change: %s" % ( galaxy_id_tag, external_job_id, drmaa_state[state] ) ) + log.debug( "(%s/%s) state change: %s" % ( galaxy_id_tag, external_job_id, self.drmaa_job_state_strings[state] ) ) if state == drmaa.JobState.RUNNING and not ajs.running: ajs.running = True ajs.job_wrapper.change_state( model.Job.states.RUNNING ) - if state == drmaa.JobState.FAILED: - if ajs.job_wrapper.get_state() != model.Job.states.DELETED: - ajs.stop_job = False - ajs.fail_message = "The cluster DRM system terminated this job" - self.work_queue.put( ( self.fail_job, ajs ) ) - continue - if state == drmaa.JobState.DONE: - if ajs.job_wrapper.get_state() != model.Job.states.DELETED: - self.work_queue.put( ( self.finish_job, ajs ) ) + if state in ( drmaa.JobState.FAILED, drmaa.JobState.DONE ): + self._complete_terminal_job( ajs, drmaa_state = state ) continue ajs.old_state = state new_watched.append( ajs ) https://bitbucket.org/galaxy/galaxy-central/commits/d3df5b5e91f3/ Changeset: d3df5b5e91f3 User: natefoo Date: 2014-01-07 21:23:11 Summary: Add a Slurm runner (subclassed from the drmaa runner), adds some logic for determining/reporting the reason of job failure and requeuing jobs that failed due to node failure. Affected #: 1 file diff -r e67776961e48dca45850c95b0f053178fd0e61b6 -r d3df5b5e91f3c42cffdba83f0ea9300b5caa172a lib/galaxy/jobs/runners/slurm.py --- /dev/null +++ b/lib/galaxy/jobs/runners/slurm.py @@ -0,0 +1,57 @@ +""" +SLURM job control via the DRMAA API. +""" + +import time +import logging +import subprocess + +from galaxy import model +from galaxy.jobs.runners.drmaa import DRMAAJobRunner + +log = logging.getLogger( __name__ ) + +__all__ = [ 'SlurmJobRunner' ] + + +class SlurmJobRunner( DRMAAJobRunner ): + runner_name = "SlurmRunner" + + def _complete_terminal_job( self, ajs, drmaa_state, **kwargs ): + def __get_jobinfo(): + scontrol_out = subprocess.check_output( ( 'scontrol', '-o', 'show', 'job', ajs.job_id ) ) + return dict( [ out_param.split( '=', 1 ) for out_param in scontrol_out.split() ] ) + if drmaa_state == self.drmaa_job_states.FAILED: + try: + job_info = __get_jobinfo() + sleep = 1 + while job_info['JobState'] == 'COMPLETING': + log.debug( '(%s/%s) Waiting %s seconds for failed job to exit COMPLETING state for post-mortem', ajs.job_wrapper.get_id_tag(), ajs.job_id, sleep ) + time.sleep( sleep ) + sleep *= 2 + if sleep > 64: + ajs.fail_message = "This job failed and the system timed out while trying to determine the cause of the failure." + break + job_info = __get_jobinfo() + if job_info['JobState'] == 'TIMEOUT': + ajs.fail_message = "This job was terminated because it ran longer than the maximum allowed job run time." + elif job_info['JobState'] == 'NODE_FAIL': + log.warning( '(%s/%s) Job failed due to node failure, attempting resubmission', ajs.job_wrapper.get_id_tag(), ajs.job_id ) + ajs.job_wrapper.change_state( model.Job.states.QUEUED, info = 'Job was resubmitted due to node failure' ) + try: + self.queue_job( ajs.job_wrapper ) + return + except: + ajs.fail_message = "This job failed due to a cluster node failure, and an attempt to resubmit the job failed." + elif job_info['JobState'] == 'CANCELLED': + ajs.fail_message = "This job failed because it was cancelled by an administrator." + else: + ajs.fail_message = "This job failed for reasons that could not be determined." + ajs.fail_message += '\nPlease click the bug icon to report this problem if you need help.' + ajs.stop_job = False + self.work_queue.put( ( self.fail_job, ajs ) ) + except Exception, e: + log.exception( '(%s/%s) Unable to inspect failed slurm job using scontrol, job will be unconditionally failed: %s', ajs.job_wrapper.get_id_tag(), ajs.job_id, e ) + super( SlurmJobRunner, self )._complete_terminal_job( ajs, drmaa_state = drmaa_state ) + elif drmaa_state == self.drmaa_job_states.DONE: + super( SlurmJobRunner, self )._complete_terminal_job( ajs, drmaa_state = drmaa_state ) https://bitbucket.org/galaxy/galaxy-central/commits/aac6bd9c589b/ Changeset: aac6bd9c589b User: natefoo Date: 2014-01-07 21:23:38 Summary: Sample job_conf.xml for new features introduced. Affected #: 1 file diff -r d3df5b5e91f3c42cffdba83f0ea9300b5caa172a -r aac6bd9c589bf43ec5835c901911704336932d1d job_conf.xml.sample_advanced --- a/job_conf.xml.sample_advanced +++ b/job_conf.xml.sample_advanced @@ -6,7 +6,19 @@ --><plugin id="local" type="runner" load="galaxy.jobs.runners.local:LocalJobRunner"/><plugin id="pbs" type="runner" load="galaxy.jobs.runners.pbs:PBSJobRunner" workers="2"/> - <plugin id="drmaa" type="runner" load="galaxy.jobs.runners.drmaa:DRMAAJobRunner"/> + <plugin id="drmaa" type="runner" load="galaxy.jobs.runners.drmaa:DRMAAJobRunner"> +  + <param id="invalidjobexception_state">ok</param> + <param id="invalidjobexception_retries">0</param> + <param id="internalexception_state">ok</param> + <param id="internalexception_retries">0</param> + </plugin> + <plugin id="sge" type="runner" load="galaxy.jobs.runners.drmaa:DRMAAJobRunner"> +  + <param id="drmaa_library_path">/sge/lib/libdrmaa.so</param> + </plugin><plugin id="lwr" type="runner" load="galaxy.jobs.runners.lwr:LwrJobRunner"> @@ -14,6 +26,7 @@ </plugin><plugin id="cli" type="runner" load="galaxy.jobs.runners.cli:ShellJobRunner" /><plugin id="condor" type="runner" load="galaxy.jobs.runners.condor:CondorJobRunner" /> + <plugin id="slurm" type="runner" load="galaxy.jobs.runners.slurm:SlurmJobRunner" /></plugins><handlers default="handlers"><handler id="handler0" tags="handlers"/><handler id="handler1" tags="handlers"/> +  + <handler id="sge_handler"> + <plugin id="sge"/> + </handler><handler id="special_handler0" tags="special_handlers"/><handler id="special_handler1" tags="special_handlers"/><handler id="trackster_handler"/> https://bitbucket.org/galaxy/galaxy-central/commits/2d525856226f/ Changeset: 2d525856226f User: natefoo Date: 2014-01-09 18:01:19 Summary: Merged in natefoo/galaxy-central (pull request #290) Configurable terminal job state, Slurm enhancements Affected #: 6 files diff -r 10ac38a448152b9a927c34f36334724dfd173bfc -r 2d525856226f0115a5f8dea5e7fedfae2fcbcf8a job_conf.xml.sample_advanced --- a/job_conf.xml.sample_advanced +++ b/job_conf.xml.sample_advanced @@ -6,7 +6,19 @@ --><plugin id="local" type="runner" load="galaxy.jobs.runners.local:LocalJobRunner"/><plugin id="pbs" type="runner" load="galaxy.jobs.runners.pbs:PBSJobRunner" workers="2"/> - <plugin id="drmaa" type="runner" load="galaxy.jobs.runners.drmaa:DRMAAJobRunner"/> + <plugin id="drmaa" type="runner" load="galaxy.jobs.runners.drmaa:DRMAAJobRunner"> +  + <param id="invalidjobexception_state">ok</param> + <param id="invalidjobexception_retries">0</param> + <param id="internalexception_state">ok</param> + <param id="internalexception_retries">0</param> + </plugin> + <plugin id="sge" type="runner" load="galaxy.jobs.runners.drmaa:DRMAAJobRunner"> +  + <param id="drmaa_library_path">/sge/lib/libdrmaa.so</param> + </plugin><plugin id="lwr" type="runner" load="galaxy.jobs.runners.lwr:LwrJobRunner"> @@ -14,6 +26,7 @@ </plugin><plugin id="cli" type="runner" load="galaxy.jobs.runners.cli:ShellJobRunner" /><plugin id="condor" type="runner" load="galaxy.jobs.runners.condor:CondorJobRunner" /> + <plugin id="slurm" type="runner" load="galaxy.jobs.runners.slurm:SlurmJobRunner" /></plugins><handlers default="handlers"><handler id="handler0" tags="handlers"/><handler id="handler1" tags="handlers"/> +  + <handler id="sge_handler"> + <plugin id="sge"/> + </handler><handler id="special_handler0" tags="special_handlers"/><handler id="special_handler1" tags="special_handlers"/><handler id="trackster_handler"/> diff -r 10ac38a448152b9a927c34f36334724dfd173bfc -r 2d525856226f0115a5f8dea5e7fedfae2fcbcf8a lib/galaxy/jobs/__init__.py --- a/lib/galaxy/jobs/__init__.py +++ b/lib/galaxy/jobs/__init__.py @@ -86,6 +86,7 @@ self.app = app self.runner_plugins = [] self.handlers = {} + self.handler_runner_plugins = {} self.default_handler_id = None self.destinations = {} self.destination_tags = {} @@ -138,6 +139,10 @@ else: log.debug("Read definition for handler '%s'" % id) self.handlers[id] = (id,) + for plugin in handler.findall('plugin'): + if id not in self.handler_runner_plugins: + self.handler_runner_plugins[id] = [] + self.handler_runner_plugins[id].append( plugin.get('id') ) if handler.get('tags', None) is not None: for tag in [ x.strip() for x in handler.get('tags').split(',') ]: if tag in self.handlers: @@ -420,13 +425,19 @@ """ return self.destinations.get(id_or_tag, None) - def get_job_runner_plugins(self): + def get_job_runner_plugins(self, handler_id): """Load all configured job runner plugins :returns: list of job runner plugins """ rval = {} - for runner in self.runner_plugins: + if handler_id in self.handler_runner_plugins: + plugins_to_load = [ rp for rp in self.runner_plugins if rp['id'] in self.handler_runner_plugins[handler_id] ] + log.info( "Handler '%s' will load specified runner plugins: %s", handler_id, ', '.join( [ rp['id'] for rp in plugins_to_load ] ) ) + else: + plugins_to_load = self.runner_plugins + log.info( "Handler '%s' will load all configured runner plugins", handler_id ) + for runner in plugins_to_load: class_names = [] module = None id = runner['id'] @@ -477,7 +488,7 @@ try: rval[id] = runner_class( self.app, runner[ 'workers' ], **runner.get( 'kwds', {} ) ) except TypeError: - log.warning( "Job runner '%s:%s' has not been converted to a new-style runner" % ( module_name, class_name ) ) + log.exception( "Job runner '%s:%s' has not been converted to a new-style runner or encountered TypeError on load" % ( module_name, class_name ) ) rval[id] = runner_class( self.app ) log.debug( "Loaded job runner '%s:%s' as '%s'" % ( module_name, class_name, id ) ) return rval diff -r 10ac38a448152b9a927c34f36334724dfd173bfc -r 2d525856226f0115a5f8dea5e7fedfae2fcbcf8a lib/galaxy/jobs/handler.py --- a/lib/galaxy/jobs/handler.py +++ b/lib/galaxy/jobs/handler.py @@ -565,7 +565,7 @@ class DefaultJobDispatcher( object ): def __init__( self, app ): self.app = app - self.job_runners = self.app.job_config.get_job_runner_plugins() + self.job_runners = self.app.job_config.get_job_runner_plugins( self.app.config.server_name ) # Once plugins are loaded, all job destinations that were created from # URLs can have their URL params converted to the destination's param # dict by the plugin. diff -r 10ac38a448152b9a927c34f36334724dfd173bfc -r 2d525856226f0115a5f8dea5e7fedfae2fcbcf8a lib/galaxy/jobs/runners/__init__.py --- a/lib/galaxy/jobs/runners/__init__.py +++ b/lib/galaxy/jobs/runners/__init__.py @@ -22,13 +22,39 @@ STOP_SIGNAL = object() + +class RunnerParams( object ): + + def __init__( self, specs = None, params = None ): + self.specs = specs or dict() + self.params = params or dict() + for name, value in self.params.items(): + assert name in self.specs, 'Invalid job runner parameter for this plugin: %s' % name + if 'map' in self.specs[ name ]: + try: + self.params[ name ] = self.specs[ name ][ 'map' ]( value ) + except Exception, e: + raise Exception( 'Job runner parameter "%s" value "%s" could not be converted to the correct type: %s' % ( name, value, e ) ) + if 'valid' in self.specs[ name ]: + assert self.specs[ name ][ 'valid' ]( value ), 'Job runner parameter %s failed validation' % name + + def __getattr__( self, name ): + return self.params.get( name, self.specs[ name ][ 'default' ] ) + + class BaseJobRunner( object ): - def __init__( self, app, nworkers ): + def __init__( self, app, nworkers, **kwargs ): """Start the job runner """ self.app = app self.sa_session = app.model.context self.nworkers = nworkers + runner_param_specs = dict( recheck_missing_job_retries = dict( map = int, valid = lambda x: x >= 0, default = 0 ) ) + if 'runner_param_specs' in kwargs: + runner_param_specs.update( kwargs.pop( 'runner_param_specs' ) ) + if kwargs: + log.debug( 'Loading %s with params: %s', self.runner_name, kwargs ) + self.runner_params = RunnerParams( specs = runner_param_specs, params = kwargs ) def _init_worker_threads(self): """Start ``nworkers`` worker threads. @@ -115,7 +141,7 @@ job_wrapper.cleanup() return False elif job_state != model.Job.states.QUEUED: - log.info( "(%d) Job is in state %s, skipping execution" % ( job_id, job_state ) ) + log.info( "(%s) Job is in state %s, skipping execution" % ( job_id, job_state ) ) # cleanup may not be safe in all states return False @@ -226,6 +252,10 @@ options.update(**kwds) return job_script(**options) + def _complete_terminal_job( self, ajs, **kwargs ): + if ajs.job_wrapper.get_state() != model.Job.states.DELETED: + self.work_queue.put( ( self.finish_job, ajs ) ) + class AsynchronousJobState( object ): """ @@ -287,8 +317,8 @@ to the correct methods (queue, finish, cleanup) at appropriate times.. """ - def __init__( self, app, nworkers ): - super( AsynchronousJobRunner, self ).__init__( app, nworkers ) + def __init__( self, app, nworkers, **kwargs ): + super( AsynchronousJobRunner, self ).__init__( app, nworkers, **kwargs ) # 'watched' and 'queue' are both used to keep track of jobs to watch. # 'queue' is used to add new watched jobs, and can be called from # any thread (usually by the 'queue_job' method). 'watched' must only diff -r 10ac38a448152b9a927c34f36334724dfd173bfc -r 2d525856226f0115a5f8dea5e7fedfae2fcbcf8a lib/galaxy/jobs/runners/drmaa.py --- a/lib/galaxy/jobs/runners/drmaa.py +++ b/lib/galaxy/jobs/runners/drmaa.py @@ -16,27 +16,12 @@ from galaxy.jobs.runners import AsynchronousJobState, AsynchronousJobRunner eggs.require( "drmaa" ) -# We foolishly named this file the same as the name exported by the drmaa -# library... 'import drmaa' imports itself. -drmaa = __import__( "drmaa" ) log = logging.getLogger( __name__ ) __all__ = [ 'DRMAAJobRunner' ] -drmaa_state = { - drmaa.JobState.UNDETERMINED: 'process status cannot be determined', - drmaa.JobState.QUEUED_ACTIVE: 'job is queued and active', - drmaa.JobState.SYSTEM_ON_HOLD: 'job is queued and in system hold', - drmaa.JobState.USER_ON_HOLD: 'job is queued and in user hold', - drmaa.JobState.USER_SYSTEM_ON_HOLD: 'job is queued and in user and system hold', - drmaa.JobState.RUNNING: 'job is running', - drmaa.JobState.SYSTEM_SUSPENDED: 'job is system suspended', - drmaa.JobState.USER_SUSPENDED: 'job is user suspended', - drmaa.JobState.DONE: 'job finished normally', - drmaa.JobState.FAILED: 'job finished, but failed', -} - +drmaa = None DRMAA_jobTemplate_attributes = [ 'args', 'remoteCommand', 'outputPath', 'errorPath', 'nativeSpecification', 'jobName', 'email', 'project' ] @@ -48,8 +33,50 @@ """ runner_name = "DRMAARunner" - def __init__( self, app, nworkers ): + def __init__( self, app, nworkers, **kwargs ): """Start the job runner""" + + global drmaa + + runner_param_specs = dict( + drmaa_library_path = dict( map = str, default = os.environ.get( 'DRMAA_LIBRARY_PATH', None ) ), + invalidjobexception_state = dict( map = str, valid = lambda x: x in ( model.Job.states.OK, model.Job.states.ERROR ), default = model.Job.states.OK ), + invalidjobexception_retries = dict( map = int, valid = lambda x: int >= 0, default = 0 ), + internalexception_state = dict( map = str, valid = lambda x: x in ( model.Job.states.OK, model.Job.states.ERROR ), default = model.Job.states.OK ), + internalexception_retries = dict( map = int, valid = lambda x: int >= 0, default = 0 ) ) + + if 'runner_param_specs' not in kwargs: + kwargs[ 'runner_param_specs' ] = dict() + kwargs[ 'runner_param_specs' ].update( runner_param_specs ) + + super( DRMAAJobRunner, self ).__init__( app, nworkers, **kwargs ) + + # This allows multiple drmaa runners (although only one per handler) in the same job config file + if 'drmaa_library_path' in kwargs: + log.info( 'Overriding DRMAA_LIBRARY_PATH due to runner plugin parameter: %s', self.runner_params.drmaa_library_path ) + os.environ['DRMAA_LIBRARY_PATH'] = self.runner_params.drmaa_library_path + + # We foolishly named this file the same as the name exported by the drmaa + # library... 'import drmaa' imports itself. + drmaa = __import__( "drmaa" ) + + # Subclasses may need access to state constants + self.drmaa_job_states = drmaa.JobState + + # Descriptive state strings pulled from the drmaa lib itself + self.drmaa_job_state_strings = { + drmaa.JobState.UNDETERMINED: 'process status cannot be determined', + drmaa.JobState.QUEUED_ACTIVE: 'job is queued and active', + drmaa.JobState.SYSTEM_ON_HOLD: 'job is queued and in system hold', + drmaa.JobState.USER_ON_HOLD: 'job is queued and in user hold', + drmaa.JobState.USER_SYSTEM_ON_HOLD: 'job is queued and in user and system hold', + drmaa.JobState.RUNNING: 'job is running', + drmaa.JobState.SYSTEM_SUSPENDED: 'job is system suspended', + drmaa.JobState.USER_SUSPENDED: 'job is user suspended', + drmaa.JobState.DONE: 'job finished normally', + drmaa.JobState.FAILED: 'job finished, but failed', + } + self.ds = drmaa.Session() self.ds.initialize() @@ -58,7 +85,6 @@ self.external_killJob_script = app.config.drmaa_external_killjob_script self.userid = None - super( DRMAAJobRunner, self ).__init__( app, nworkers ) self._init_monitor_thread() self._init_worker_threads() @@ -175,6 +201,20 @@ # Add to our 'queue' of jobs to monitor self.monitor_queue.put( ajs ) + def _complete_terminal_job( self, ajs, drmaa_state, **kwargs ): + """ + Handle a job upon its termination in the DRM. This method is meant to + be overridden by subclasses to improve post-mortem and reporting of + failures. + """ + if drmaa_state == drmaa.JobState.FAILED: + if ajs.job_wrapper.get_state() != model.Job.states.DELETED: + ajs.stop_job = False + ajs.fail_message = "The cluster DRM system terminated this job" + self.work_queue.put( ( self.fail_job, ajs ) ) + elif drmaa_state == drmaa.JobState.DONE: + super( DRMAAJobRunner, self )._complete_terminal_job( ajs ) + def check_watched_items( self ): """ Called by the monitor thread to look at each watched job and deal @@ -188,16 +228,27 @@ try: assert external_job_id not in ( None, 'None' ), '(%s/%s) Invalid job id' % ( galaxy_id_tag, external_job_id ) state = self.ds.jobStatus( external_job_id ) - # InternalException was reported to be necessary on some DRMs, but - # this could cause failures to be detected as completion! Please - # report if you experience problems with this. - except ( drmaa.InvalidJobException, drmaa.InternalException ), e: - # we should only get here if an orphaned job was put into the queue at app startup - log.info( "(%s/%s) job left DRM queue with following message: %s" % ( galaxy_id_tag, external_job_id, e ) ) - self.work_queue.put( ( self.finish_job, ajs ) ) + except ( drmaa.InternalException, drmaa.InvalidJobException ), e: + ecn = e.__class__.__name__ + retry_param = ecn.lower() + '_retries' + state_param = ecn.lower() + '_state' + retries = getattr( ajs, retry_param, 0 ) + if self.runner_params[ retry_param ] > 0: + if retries < self.runner_params[ retry_param ]: + # will retry check on next iteration + setattr( ajs, retry_param, retries + 1 ) + continue + if self.runner_params[ state_param ] == model.Job.states.OK: + log.info( "(%s/%s) job left DRM queue with following message: %s", galaxy_id_tag, external_job_id, e ) + self.work_queue.put( ( self.finish_job, ajs ) ) + elif self.runner_params[ state_param ] == model.Job.states.ERROR: + log.info( "(%s/%s) job check resulted in %s after %s tries: %s", galaxy_id_tag, external_job_id, ecn, retries, e ) + self.work_queue.put( ( self.fail_job, ajs ) ) + else: + raise Exception( "%s is set to an invalid value (%s), this should not be possible. See galaxy.jobs.drmaa.__init__()", state_param, self.runner_params[ state_param ] ) continue except drmaa.DrmCommunicationException, e: - log.warning( "(%s/%s) unable to communicate with DRM: %s" % ( galaxy_id_tag, external_job_id, e )) + log.warning( "(%s/%s) unable to communicate with DRM: %s", galaxy_id_tag, external_job_id, e ) new_watched.append( ajs ) continue except Exception, e: @@ -208,19 +259,12 @@ self.work_queue.put( ( self.fail_job, ajs ) ) continue if state != old_state: - log.debug( "(%s/%s) state change: %s" % ( galaxy_id_tag, external_job_id, drmaa_state[state] ) ) + log.debug( "(%s/%s) state change: %s" % ( galaxy_id_tag, external_job_id, self.drmaa_job_state_strings[state] ) ) if state == drmaa.JobState.RUNNING and not ajs.running: ajs.running = True ajs.job_wrapper.change_state( model.Job.states.RUNNING ) - if state == drmaa.JobState.FAILED: - if ajs.job_wrapper.get_state() != model.Job.states.DELETED: - ajs.stop_job = False - ajs.fail_message = "The cluster DRM system terminated this job" - self.work_queue.put( ( self.fail_job, ajs ) ) - continue - if state == drmaa.JobState.DONE: - if ajs.job_wrapper.get_state() != model.Job.states.DELETED: - self.work_queue.put( ( self.finish_job, ajs ) ) + if state in ( drmaa.JobState.FAILED, drmaa.JobState.DONE ): + self._complete_terminal_job( ajs, drmaa_state = state ) continue ajs.old_state = state new_watched.append( ajs ) diff -r 10ac38a448152b9a927c34f36334724dfd173bfc -r 2d525856226f0115a5f8dea5e7fedfae2fcbcf8a lib/galaxy/jobs/runners/slurm.py --- /dev/null +++ b/lib/galaxy/jobs/runners/slurm.py @@ -0,0 +1,57 @@ +""" +SLURM job control via the DRMAA API. +""" + +import time +import logging +import subprocess + +from galaxy import model +from galaxy.jobs.runners.drmaa import DRMAAJobRunner + +log = logging.getLogger( __name__ ) + +__all__ = [ 'SlurmJobRunner' ] + + +class SlurmJobRunner( DRMAAJobRunner ): + runner_name = "SlurmRunner" + + def _complete_terminal_job( self, ajs, drmaa_state, **kwargs ): + def __get_jobinfo(): + scontrol_out = subprocess.check_output( ( 'scontrol', '-o', 'show', 'job', ajs.job_id ) ) + return dict( [ out_param.split( '=', 1 ) for out_param in scontrol_out.split() ] ) + if drmaa_state == self.drmaa_job_states.FAILED: + try: + job_info = __get_jobinfo() + sleep = 1 + while job_info['JobState'] == 'COMPLETING': + log.debug( '(%s/%s) Waiting %s seconds for failed job to exit COMPLETING state for post-mortem', ajs.job_wrapper.get_id_tag(), ajs.job_id, sleep ) + time.sleep( sleep ) + sleep *= 2 + if sleep > 64: + ajs.fail_message = "This job failed and the system timed out while trying to determine the cause of the failure." + break + job_info = __get_jobinfo() + if job_info['JobState'] == 'TIMEOUT': + ajs.fail_message = "This job was terminated because it ran longer than the maximum allowed job run time." + elif job_info['JobState'] == 'NODE_FAIL': + log.warning( '(%s/%s) Job failed due to node failure, attempting resubmission', ajs.job_wrapper.get_id_tag(), ajs.job_id ) + ajs.job_wrapper.change_state( model.Job.states.QUEUED, info = 'Job was resubmitted due to node failure' ) + try: + self.queue_job( ajs.job_wrapper ) + return + except: + ajs.fail_message = "This job failed due to a cluster node failure, and an attempt to resubmit the job failed." + elif job_info['JobState'] == 'CANCELLED': + ajs.fail_message = "This job failed because it was cancelled by an administrator." + else: + ajs.fail_message = "This job failed for reasons that could not be determined." + ajs.fail_message += '\nPlease click the bug icon to report this problem if you need help.' + ajs.stop_job = False + self.work_queue.put( ( self.fail_job, ajs ) ) + except Exception, e: + log.exception( '(%s/%s) Unable to inspect failed slurm job using scontrol, job will be unconditionally failed: %s', ajs.job_wrapper.get_id_tag(), ajs.job_id, e ) + super( SlurmJobRunner, self )._complete_terminal_job( ajs, drmaa_state = drmaa_state ) + elif drmaa_state == self.drmaa_job_states.DONE: + super( SlurmJobRunner, self )._complete_terminal_job( ajs, drmaa_state = drmaa_state ) Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: 2 new changesets
by commits-noreply＠bitbucket.org 09 Jan '14

09 Jan '14

2 new commits in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/cfdf54c48ecd/ Changeset: cfdf54c48ecd Branch: stable User: dan Date: 2013-12-02 20:48:30 Summary: Fix for missing eggs.require in lib/galaxy/exceptions/__init__.py. Affected #: 1 file diff -r a0f7e0e5f05af4112ded852cf61c138582c99c04 -r cfdf54c48ecdcbe5bad4a9af14822a88d7fd6082 lib/galaxy/exceptions/__init__.py --- a/lib/galaxy/exceptions/__init__.py +++ b/lib/galaxy/exceptions/__init__.py @@ -1,6 +1,10 @@ """ Custom exceptions for Galaxy """ + +from galaxy import eggs +eggs.require( "Paste" ) + from paste import httpexceptions class MessageException( Exception ): https://bitbucket.org/galaxy/galaxy-central/commits/3e76aceca9d0/ Changeset: 3e76aceca9d0 Branch: stable User: natefoo Date: 2014-01-09 16:46:01 Summary: Fix username argument to PostgreSQL connection in pgcleanup.py Affected #: 1 file diff -r cfdf54c48ecdcbe5bad4a9af14822a88d7fd6082 -r 3e76aceca9d0ab563e285d630dd7575140236b0d scripts/cleanup_datasets/pgcleanup.py --- a/scripts/cleanup_datasets/pgcleanup.py +++ b/scripts/cleanup_datasets/pgcleanup.py @@ -105,7 +105,7 @@ url = make_url(self.config.database_connection) log.info('Connecting to database with URL: %s' % url) - args = url.translate_connect_args() + args = url.translate_connect_args( username='user' ) args.update(url.query) assert url.get_dialect().name == 'postgresql', 'This script can only be used with PostgreSQL.' Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: dan: Special case genomespace file browser import tool's handling of GenomeSpace format identifier 'http://www.genomespace.org/datamanager/dataformat/unknown' to cause datatype auto-dection via sniff with a fallback to using filename extention.
by commits-noreply＠bitbucket.org 09 Jan '14

09 Jan '14

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/10ac38a44815/ Changeset: 10ac38a44815 User: dan Date: 2014-01-09 16:54:58 Summary: Special case genomespace file browser import tool's handling of GenomeSpace format identifier 'http://www.genomespace.org/datamanager/dataformat/unknown' to cause datatype auto-dection via sniff with a fallback to using filename extention. Affected #: 1 file diff -r 110a0aabd293fd1393d1296cb220a0b153d69123 -r 10ac38a448152b9a927c34f36334724dfd173bfc tools/genomespace/genomespace_file_browser.py --- a/tools/genomespace/genomespace_file_browser.py +++ b/tools/genomespace/genomespace_file_browser.py @@ -8,11 +8,16 @@ pkg_resources.require( "simplejson" ) import simplejson +import galaxy.model # need to import model before sniff to resolve a circular import dependency +from galaxy.datatypes import sniff +from galaxy.datatypes.registry import Registry + GENOMESPACE_API_VERSION_STRING = "v1.0" GENOMESPACE_SERVER_URL_PROPERTIES = "https://dm.genomespace.org/config/%s/serverurl.properties" % ( GENOMESPACE_API_VERSION_STRING ) CHUNK_SIZE = 2**20 #1mb +AUTO_GALAXY_EXT = "auto" DEFAULT_GALAXY_EXT = "data" #genomespace format identifier is the URL @@ -39,6 +44,9 @@ 'gmt': 'gmt', 'gct': 'gct'} +GENOMESPACE_UNKNOWN_FORMAT_KEY = 'unknown' +GENOMESPACE_FORMAT_IDENTIFIER_UNKNOWN = None + VALID_CHARS = '.-()[]0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ ' def chunk_write( source_stream, target_stream, source_method = "read", target_method="write" ): @@ -67,7 +75,7 @@ ext = GENOMESPACE_EXT_TO_GALAXY_EXT.get( ext, None ) if ext is None: #could check content type, etc here - ext = DEFAULT_GALAXY_EXT + ext = AUTO_GALAXY_EXT return ext def get_genomespace_site_urls(): @@ -90,6 +98,8 @@ genomespace_formats = simplejson.loads( opened_gs_request.read() ) for format in genomespace_formats: GENOMESPACE_FORMAT_IDENTIFIER_TO_GENOMESPACE_EXT[ format['url'] ] = format['name'] + global GENOMESPACE_FORMAT_IDENTIFIER_UNKNOWN + GENOMESPACE_FORMAT_IDENTIFIER_UNKNOWN = dict( map( lambda x: ( x[1], x[0] ) , GENOMESPACE_FORMAT_IDENTIFIER_TO_GENOMESPACE_EXT.iteritems() ) ).get( GENOMESPACE_UNKNOWN_FORMAT_KEY, GENOMESPACE_FORMAT_IDENTIFIER_UNKNOWN ) def download_from_genomespace_file_browser( json_parameter_file, genomespace_site ): json_params = simplejson.loads( open( json_parameter_file, 'r' ).read() ) @@ -108,6 +118,11 @@ file_url_prefix = "fileUrl" file_type_prefix = "fileFormat" metadata_parameter_file = open( json_params['job_config']['TOOL_PROVIDED_JOB_METADATA_FILE'], 'wb' ) + + #setup datatypes registry for sniffing + datatypes_registry = Registry() + datatypes_registry.load_datatypes( root_dir = json_params[ 'job_config' ][ 'GALAXY_ROOT_DIR' ], config = json_params[ 'job_config' ][ 'GALAXY_DATATYPES_CONF_FILE' ] ) + file_numbers = [] for name in datasource_params.keys(): if name.startswith( file_url_prefix ): @@ -143,28 +158,47 @@ filename = urllib.unquote_plus( parsed_url[2].split( '/' )[-1] ) if not filename: filename = download_url + metadata_dict = None + original_filename = filename if output_filename is None: - original_filename = filename filename = ''.join( c in VALID_CHARS and c or '-' for c in filename ) while filename in used_filenames: filename = "-%s" % filename used_filenames.append( filename ) output_filename = os.path.join( datasource_params['__new_file_path__'], 'primary_%i_%s_visible_%s' % ( hda_id, filename, galaxy_ext ) ) - metadata_parameter_file.write( "%s\n" % simplejson.dumps( dict( type = 'new_primary_dataset', - base_dataset_id = dataset_id, - ext = galaxy_ext, - filename = output_filename, - name = "GenomeSpace import on %s" % ( original_filename ) ) ) ) + + metadata_dict = dict( type = 'new_primary_dataset', + base_dataset_id = dataset_id, + ext = galaxy_ext, + filename = output_filename, + name = "GenomeSpace import on %s" % ( original_filename ) ) else: if dataset_id is not None: - metadata_parameter_file.write( "%s\n" % simplejson.dumps( dict( type = 'dataset', - dataset_id = dataset_id, - ext = galaxy_ext, - name = "GenomeSpace import on %s" % ( filename ) ) ) ) + metadata_dict = dict( type = 'dataset', + dataset_id = dataset_id, + ext = galaxy_ext, + name = "GenomeSpace import on %s" % ( filename ) ) output_file = open( output_filename, 'wb' ) chunk_write( target_download_url, output_file ) output_file.close() + + if ( galaxy_ext == AUTO_GALAXY_EXT or filetype_url == GENOMESPACE_FORMAT_IDENTIFIER_UNKNOWN ) and metadata_dict: + #try to sniff datatype + try: + galaxy_ext = sniff.handle_uploaded_dataset_file( output_filename, datatypes_registry ) + except: + #sniff failed + galaxy_ext = original_filename.rsplit( '.', 1 )[-1] + if galaxy_ext not in datatypes_registry.datatypes_by_extension: + galaxy_ext = DEFAULT_GALAXY_EXT + metadata_dict[ 'ext' ] = galaxy_ext + output_filename = None #only have one filename available + + #write out metadata info + if metadata_dict: + metadata_parameter_file.write( "%s\n" % simplejson.dumps( metadata_dict ) ) + metadata_parameter_file.close() return True Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: inithello: Use separate methods to deactivate and uninstall repositories.
by commits-noreply＠bitbucket.org 09 Jan '14

09 Jan '14

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/110a0aabd293/ Changeset: 110a0aabd293 User: inithello Date: 2014-01-09 14:49:22 Summary: Use separate methods to deactivate and uninstall repositories. Affected #: 18 files diff -r ca11e10aa1a5b7318d3727816bf0c2fe6137c522 -r 110a0aabd293fd1393d1296cb220a0b153d69123 test/tool_shed/base/twilltestcase.py --- a/test/tool_shed/base/twilltestcase.py +++ b/test/tool_shed/base/twilltestcase.py @@ -383,7 +383,17 @@ except: pass return previously_created, username_taken, invalid_username - + + def deactivate_repository( self, installed_repository, strings_displayed=[], strings_not_displayed=[] ): + url = '/admin_toolshed/deactivate_or_uninstall_repository?id=%s' % self.security.encode_id( installed_repository.id ) + self.visit_galaxy_url( url ) + self.check_for_strings( strings_displayed, strings_not_displayed ) + form = tc.browser.get_form( 'deactivate_or_uninstall_repository' ) + kwd = self.set_form_value( form, {}, 'remove_from_disk', False ) + tc.submit( 'deactivate_or_uninstall_repository_button' ) + strings_displayed = [ 'The repository named', 'has been deactivated' ] + self.check_for_strings( strings_displayed, strings_not_displayed=[] ) + def delete_files_from_repository( self, repository, filenames=[], strings_displayed=[ 'were deleted from the repository' ], strings_not_displayed=[] ): files_to_delete = [] basepath = self.get_repo_path( repository ) @@ -1204,18 +1214,14 @@ self.visit_url( url ) self.check_for_strings( strings_displayed, strings_not_displayed ) - def uninstall_repository( self, installed_repository, remove_from_disk=True, is_required=False, strings_displayed=[], strings_not_displayed=[] ): + def uninstall_repository( self, installed_repository, strings_displayed=[], strings_not_displayed=[] ): url = '/admin_toolshed/deactivate_or_uninstall_repository?id=%s' % self.security.encode_id( installed_repository.id ) self.visit_galaxy_url( url ) self.check_for_strings( strings_displayed, strings_not_displayed ) form = tc.browser.get_form( 'deactivate_or_uninstall_repository' ) - kwd = self.set_form_value( form, {}, 'remove_from_disk', remove_from_disk ) + kwd = self.set_form_value( form, {}, 'remove_from_disk', True ) tc.submit( 'deactivate_or_uninstall_repository_button' ) - strings_displayed = [ 'The repository named' ] - if remove_from_disk and not is_required: - strings_displayed.append( 'has been uninstalled' ) - else: - strings_displayed.append( 'has been deactivated' ) + strings_displayed = [ 'The repository named', 'has been uninstalled' ] self.check_for_strings( strings_displayed, strings_not_displayed=[] ) def update_installed_repository( self, installed_repository, strings_displayed=[], strings_not_displayed=[] ): diff -r ca11e10aa1a5b7318d3727816bf0c2fe6137c522 -r 110a0aabd293fd1393d1296cb220a0b153d69123 test/tool_shed/functional/test_0490_export_import_repositories.py --- a/test/tool_shed/functional/test_0490_export_import_repositories.py +++ b/test/tool_shed/functional/test_0490_export_import_repositories.py @@ -65,7 +65,7 @@ global capsule_filepath repository = self.test_db_util.get_repository_by_name_and_owner( repository_name, common.test_user_1_name ) capsule_filepath = self.export_capsule( repository ) - log.debug( os.path.exists( capsule_filepath ) ) + assert os.path.exists( capsule_filepath ), 'Exported repository capsule file path %s not found.' % capsule_filepath def test_0015_verify_exported_capsule( self ): '''Verify the exported capsule contents.''' diff -r ca11e10aa1a5b7318d3727816bf0c2fe6137c522 -r 110a0aabd293fd1393d1296cb220a0b153d69123 test/tool_shed/functional/test_1020_install_repository_with_repository_dependencies.py --- a/test/tool_shed/functional/test_1020_install_repository_with_repository_dependencies.py +++ b/test/tool_shed/functional/test_1020_install_repository_with_repository_dependencies.py @@ -121,7 +121,7 @@ global base_datatypes_count installed_repository = self.test_db_util.get_installed_repository_by_name_owner( datatypes_repository_name, common.test_user_1_name ) old_datatypes_count = int( self.get_datatypes_count() ) - self.uninstall_repository( installed_repository, remove_from_disk=False ) + self.deactivate_repository( installed_repository ) current_datatypes = int( self.get_datatypes_count() ) assert current_datatypes < old_datatypes_count, 'Uninstalling emboss did not remove datatypes.' diff -r ca11e10aa1a5b7318d3727816bf0c2fe6137c522 -r 110a0aabd293fd1393d1296cb220a0b153d69123 test/tool_shed/functional/test_1040_install_repository_basic_circular_dependencies.py --- a/test/tool_shed/functional/test_1040_install_repository_basic_circular_dependencies.py +++ b/test/tool_shed/functional/test_1040_install_repository_basic_circular_dependencies.py @@ -109,7 +109,7 @@ installed_filtering_repository = self.test_db_util.get_installed_repository_by_name_owner( filtering_repository_name, common.test_user_1_name ) self.display_installed_repository_manage_page( installed_freebayes_repository ) self.display_installed_repository_manage_page( installed_filtering_repository ) - self.uninstall_repository( installed_filtering_repository, remove_from_disk=False ) + self.deactivate_repository( installed_filtering_repository ) self.test_db_util.ga_refresh( installed_filtering_repository ) self.check_galaxy_repository_tool_panel_section( installed_filtering_repository, '' ) strings_displayed = [ 'Missing repository', 'filtering', 'freebayes_0040', 'user1', "Galaxy's freebayes tool for test 0040" ] @@ -134,7 +134,7 @@ installed_filtering_repository = self.test_db_util.get_installed_repository_by_name_owner( filtering_repository_name, common.test_user_1_name ) self.display_installed_repository_manage_page( installed_freebayes_repository ) self.display_installed_repository_manage_page( installed_filtering_repository ) - self.uninstall_repository( installed_freebayes_repository, remove_from_disk=False ) + self.deactivate_repository( installed_freebayes_repository ) self.test_db_util.ga_refresh( installed_freebayes_repository ) self.check_galaxy_repository_tool_panel_section( installed_freebayes_repository, '' ) strings_displayed = [ 'Missing repository', 'freebayes', 'filtering_0040', 'user1', "Galaxy's filtering tool for test 0040" ] @@ -149,7 +149,7 @@ installed_freebayes_repository = self.test_db_util.get_installed_repository_by_name_owner( freebayes_repository_name, common.test_user_1_name ) self.display_installed_repository_manage_page( installed_freebayes_repository ) self.display_installed_repository_manage_page( installed_filtering_repository ) - self.uninstall_repository( installed_filtering_repository, remove_from_disk=False ) + self.deactivate_repository( installed_filtering_repository ) self.test_db_util.ga_refresh( installed_filtering_repository ) self.check_galaxy_repository_tool_panel_section( installed_filtering_repository, '' ) strings_displayed = [ 'Missing repository', 'filtering', 'freebayes_0040', 'user1', "Galaxy's freebayes tool for test 0040" ] diff -r ca11e10aa1a5b7318d3727816bf0c2fe6137c522 -r 110a0aabd293fd1393d1296cb220a0b153d69123 test/tool_shed/functional/test_1050_circular_dependencies_4_levels.py --- a/test/tool_shed/functional/test_1050_circular_dependencies_4_levels.py +++ b/test/tool_shed/functional/test_1050_circular_dependencies_4_levels.py @@ -315,7 +315,7 @@ '''Deactivate emboss_datatypes and verify that the datatypes count is reduced.''' original_datatypes = self.get_datatypes_count() repository = self.test_db_util.get_installed_repository_by_name_owner( emboss_datatypes_repository_name, common.test_user_1_name ) - self.uninstall_repository( repository, remove_from_disk=False ) + self.deactivate_repository( repository ) assert original_datatypes > self.get_datatypes_count(), 'Deactivating emboss_datatypes did not remove datatypes.' # Now we have emboss, bismark, column_maker, and convert_chars installed, filtering and freebayes never installed, and emboss_datatypes deactivated. installed_repositories = [ ( column_repository_name, common.test_user_1_name ), @@ -330,7 +330,7 @@ def test_0070_uninstall_emboss_repository( self ): '''Uninstall the emboss_5 repository.''' repository = self.test_db_util.get_installed_repository_by_name_owner( emboss_repository_name, common.test_user_1_name ) - self.uninstall_repository( repository, remove_from_disk=True ) + self.uninstall_repository( repository ) strings_not_displayed = [ repository.name, repository.installed_changeset_revision ] self.display_galaxy_browse_repositories_page( strings_not_displayed=strings_not_displayed ) self.test_db_util.ga_refresh( repository ) diff -r ca11e10aa1a5b7318d3727816bf0c2fe6137c522 -r 110a0aabd293fd1393d1296cb220a0b153d69123 test/tool_shed/functional/test_1080_advanced_circular_dependency_installation.py --- a/test/tool_shed/functional/test_1080_advanced_circular_dependency_installation.py +++ b/test/tool_shed/functional/test_1080_advanced_circular_dependency_installation.py @@ -162,7 +162,7 @@ common.test_user_1_name ) installed_column_repository = self.test_db_util.get_installed_repository_by_name_owner( column_repository_name, common.test_user_1_name ) - self.uninstall_repository( installed_convert_repository, remove_from_disk=False ) + self.deactivate_repository( installed_convert_repository ) strings_displayed = [ 'column_maker_0080', 'Add column', self.url.replace( 'http://', '' ), @@ -199,7 +199,7 @@ common.test_user_1_name ) installed_column_repository = self.test_db_util.get_installed_repository_by_name_owner( column_repository_name, common.test_user_1_name ) - self.uninstall_repository( installed_column_repository, remove_from_disk=False ) + self.deactivate_repository( installed_column_repository ) strings_displayed = [ 'convert_chars_0080', 'Convert delimiters', self.url.replace( 'http://', '' ), @@ -217,7 +217,7 @@ common.test_user_1_name ) installed_column_repository = self.test_db_util.get_installed_repository_by_name_owner( column_repository_name, common.test_user_1_name ) - self.uninstall_repository( installed_convert_repository, remove_from_disk=False ) + self.deactivate_repository( installed_convert_repository ) strings_not_displayed = [ 'column_maker_0080', installed_column_repository.installed_changeset_revision, 'convert_chars_0080', @@ -274,7 +274,7 @@ common.test_user_1_name ) installed_column_repository = self.test_db_util.get_installed_repository_by_name_owner( column_repository_name, common.test_user_1_name ) - self.uninstall_repository( installed_column_repository, remove_from_disk=True ) + self.uninstall_repository( installed_column_repository ) strings_displayed = [ 'convert_chars_0080', 'Convert delimiters', self.url.replace( 'http://', '' ), @@ -320,7 +320,7 @@ common.test_user_1_name ) installed_column_repository = self.test_db_util.get_installed_repository_by_name_owner( column_repository_name, common.test_user_1_name ) - self.uninstall_repository( installed_convert_repository, remove_from_disk=False ) + self.deactivate_repository( installed_convert_repository ) strings_displayed = [ 'column_maker_0080', 'Add column', self.url.replace( 'http://', '' ), @@ -340,7 +340,7 @@ common.test_user_1_name ) installed_column_repository = self.test_db_util.get_installed_repository_by_name_owner( column_repository_name, common.test_user_1_name ) - self.uninstall_repository( installed_column_repository, remove_from_disk=False ) + self.deactivate_repository( installed_column_repository ) strings_displayed = [ 'convert_chars_0080', 'Convert delimiters', self.url.replace( 'http://', '' ), @@ -387,8 +387,8 @@ common.test_user_1_name ) installed_column_repository = self.test_db_util.get_installed_repository_by_name_owner( column_repository_name, common.test_user_1_name ) - self.uninstall_repository( installed_column_repository, remove_from_disk=False ) - self.uninstall_repository( installed_convert_repository, remove_from_disk=False ) + self.deactivate_repository( installed_column_repository ) + self.deactivate_repository( installed_convert_repository ) self.test_db_util.install_session.refresh( installed_column_repository ) self.test_db_util.install_session.refresh( installed_convert_repository ) self.check_galaxy_repository_tool_panel_section( installed_column_repository, '' ) diff -r ca11e10aa1a5b7318d3727816bf0c2fe6137c522 -r 110a0aabd293fd1393d1296cb220a0b153d69123 test/tool_shed/functional/test_1085_repository_dependency_handling.py --- a/test/tool_shed/functional/test_1085_repository_dependency_handling.py +++ b/test/tool_shed/functional/test_1085_repository_dependency_handling.py @@ -124,14 +124,14 @@ '''uninstall column_maker, verify same section''' installed_column_repository = self.test_db_util.get_installed_repository_by_name_owner( column_repository_name, common.test_user_1_name ) - self.uninstall_repository( installed_column_repository, remove_from_disk=True ) + self.uninstall_repository( installed_column_repository ) self.test_db_util.ga_refresh( installed_column_repository ) self.check_galaxy_repository_tool_panel_section( installed_column_repository, 'column_maker' ) def test_0030_uninstall_convert_repository( self ): installed_convert_repository = self.test_db_util.get_installed_repository_by_name_owner( convert_repository_name, common.test_user_1_name ) - self.uninstall_repository( installed_convert_repository, remove_from_disk=True ) + self.uninstall_repository( installed_convert_repository ) self.test_db_util.ga_refresh( installed_convert_repository ) self.check_galaxy_repository_tool_panel_section( installed_convert_repository, 'column_maker' ) @@ -173,8 +173,8 @@ common.test_user_1_name ) installed_column_repository = self.test_db_util.get_installed_repository_by_name_owner( column_repository_name, common.test_user_1_name ) - self.uninstall_repository( installed_convert_repository, remove_from_disk=True ) - self.uninstall_repository( installed_column_repository, remove_from_disk=True ) + self.uninstall_repository( installed_convert_repository ) + self.uninstall_repository( installed_column_repository ) self.test_db_util.ga_refresh( installed_convert_repository ) self.test_db_util.ga_refresh( installed_column_repository ) self.check_galaxy_repository_tool_panel_section( installed_column_repository, 'new_column_maker' ) diff -r ca11e10aa1a5b7318d3727816bf0c2fe6137c522 -r 110a0aabd293fd1393d1296cb220a0b153d69123 test/tool_shed/functional/test_1087_install_updated_repository_dependencies.py --- a/test/tool_shed/functional/test_1087_install_updated_repository_dependencies.py +++ b/test/tool_shed/functional/test_1087_install_updated_repository_dependencies.py @@ -89,7 +89,7 @@ new_tool_panel_section_label='column_maker', strings_not_displayed=[ 'install_repository_dependencies' ] ) installed_column_repository = self.test_db_util.get_installed_repository_by_name_owner( column_repository_name, common.test_user_1_name ) - self.uninstall_repository( installed_column_repository, remove_from_disk=True ) + self.uninstall_repository( installed_column_repository ) def test_0020_upload_dependency_xml( self ): '''Upload a repository_dependencies.xml file to column_maker that specifies convert_chars.''' diff -r ca11e10aa1a5b7318d3727816bf0c2fe6137c522 -r 110a0aabd293fd1393d1296cb220a0b153d69123 test/tool_shed/functional/test_1090_install_tool_from_tool_search.py --- a/test/tool_shed/functional/test_1090_install_tool_from_tool_search.py +++ b/test/tool_shed/functional/test_1090_install_tool_from_tool_search.py @@ -251,7 +251,7 @@ self.display_galaxy_browse_repositories_page( strings_displayed=strings_displayed, strings_not_displayed=strings_not_displayed ) self.verify_installed_repositories( installed_repositories ) filtering_repository = self.test_db_util.get_installed_repository_by_name_owner( filtering_repository_name, common.test_user_1_name ) - self.uninstall_repository( filtering_repository, remove_from_disk=False ) + self.deactivate_repository( filtering_repository ) strings_displayed = [ 'freebayes_0090' ] strings_not_displayed = [ 'filtering_0090', 'emboss_0090', 'emboss_datatypes_0090', 'bwa_color_0090', 'bwa_base_0090' ] self.display_galaxy_browse_repositories_page( strings_displayed=strings_displayed, strings_not_displayed=strings_not_displayed ) @@ -278,7 +278,7 @@ self.display_galaxy_browse_repositories_page( strings_displayed=strings_displayed, strings_not_displayed=strings_not_displayed ) self.verify_installed_repositories( installed_repositories ) datatypes_repository = self.test_db_util.get_installed_repository_by_name_owner( emboss_datatypes_repository_name, common.test_user_1_name ) - self.uninstall_repository( datatypes_repository, remove_from_disk=True ) + self.uninstall_repository( datatypes_repository ) strings_displayed = [ 'freebayes_0090' ] strings_not_displayed = [ 'emboss_datatypes_0090', 'filtering_0090', 'emboss_0090', 'bwa_color_0090', 'bwa_base_0090' ] self.display_galaxy_browse_repositories_page( strings_displayed=strings_displayed, strings_not_displayed=strings_not_displayed ) diff -r ca11e10aa1a5b7318d3727816bf0c2fe6137c522 -r 110a0aabd293fd1393d1296cb220a0b153d69123 test/tool_shed/functional/test_1100_install_repository_with_complex_dependencies.py --- a/test/tool_shed/functional/test_1100_install_repository_with_complex_dependencies.py +++ b/test/tool_shed/functional/test_1100_install_repository_with_complex_dependencies.py @@ -278,7 +278,7 @@ ''' base_repository = self.test_db_util.get_installed_repository_by_name_owner( bwa_base_repository_name, common.test_user_1_name ) tool_repository = self.test_db_util.get_installed_repository_by_name_owner( bwa_package_repository_name, common.test_user_1_name ) - self.uninstall_repository( tool_repository, remove_from_disk=False ) + self.deactivate_repository( tool_repository ) env_sh_path = os.path.join( self.galaxy_tool_dependency_dir, 'bwa', '0.5.9', diff -r ca11e10aa1a5b7318d3727816bf0c2fe6137c522 -r 110a0aabd293fd1393d1296cb220a0b153d69123 test/tool_shed/functional/test_1130_datatype_converters.py --- a/test/tool_shed/functional/test_1130_datatype_converters.py +++ b/test/tool_shed/functional/test_1130_datatype_converters.py @@ -87,5 +87,5 @@ so we have to uninstall it first, then verify that it was not assigned a tool panel section. ''' repository = self.test_db_util.get_installed_repository_by_name_owner( repository_name, common.test_user_1_name ) - self.uninstall_repository( repository, remove_from_disk=True ) + self.uninstall_repository( repository ) self.verify_installed_repository_no_tool_panel_section( repository ) diff -r ca11e10aa1a5b7318d3727816bf0c2fe6137c522 -r 110a0aabd293fd1393d1296cb220a0b153d69123 test/tool_shed/functional/test_1160_circular_prior_installation_required.py --- a/test/tool_shed/functional/test_1160_circular_prior_installation_required.py +++ b/test/tool_shed/functional/test_1160_circular_prior_installation_required.py @@ -217,18 +217,16 @@ assert filter_repository.update_time > convert_repository.update_time, 'Error: convert_chars_0160 shows a later update time than filtering_0160' assert filter_repository.update_time > column_repository.update_time, 'Error: column_maker_0160 shows a later update time than filtering_0160' - def test_0040_uninstall_all_repositories( self ): + def test_0040_deactivate_all_repositories( self ): '''Uninstall convert_chars_0160, column_maker_0160, and filtering_0160.''' filter_repository = self.test_db_util.get_installed_repository_by_name_owner( filter_repository_name, common.test_user_1_name ) column_repository = self.test_db_util.get_installed_repository_by_name_owner( column_repository_name, common.test_user_1_name ) convert_repository = self.test_db_util.get_installed_repository_by_name_owner( convert_repository_name, common.test_user_1_name ) - # TODO: Tweak the uninstall_repository method by pulling out the stuff that deactivates a repository - # instead of uninstalling it and create a new deactivate_repository method. - self.uninstall_repository( filter_repository, remove_from_disk=False ) - self.uninstall_repository( column_repository, remove_from_disk=False ) - self.uninstall_repository( convert_repository, remove_from_disk=False ) + self.deactivate_repository( filter_repository ) + self.deactivate_repository( column_repository ) + self.deactivate_repository( convert_repository ) - def test_0045_reactivatel_filter_repository( self ): + def test_0045_reactivate_filter_repository( self ): '''Reinstall the filtering_0160 repository.''' filter_repository = self.test_db_util.get_installed_repository_by_name_owner( filter_repository_name, common.test_user_1_name ) self.reactivate_repository( filter_repository ) diff -r ca11e10aa1a5b7318d3727816bf0c2fe6137c522 -r 110a0aabd293fd1393d1296cb220a0b153d69123 test/tool_shed/functional/test_1200_uninstall_and_reinstall_basic_repository.py --- a/test/tool_shed/functional/test_1200_uninstall_and_reinstall_basic_repository.py +++ b/test/tool_shed/functional/test_1200_uninstall_and_reinstall_basic_repository.py @@ -1,7 +1,9 @@ from tool_shed.base.twilltestcase import ShedTwillTestCase, common, os + class UninstallingAndReinstallingRepositories( ShedTwillTestCase ): '''Test uninstalling and reinstalling a basic repository.''' + def test_0000_initiate_users( self ): """Create necessary user accounts.""" self.logout() @@ -19,6 +21,7 @@ galaxy_admin_user = self.test_db_util.get_galaxy_user( common.admin_email ) assert galaxy_admin_user is not None, 'Problem retrieving user with email %s from the database' % common.admin_email galaxy_admin_user_private_role = self.test_db_util.get_galaxy_private_role( galaxy_admin_user ) + def test_0005_ensure_repositories_and_categories_exist( self ): '''Create the 0000 category and upload the filtering repository to the tool shed, if necessary.''' category = self.create_category( name='Test 0000 Basic Repository Features 1', description='Test 0000 Basic Repository Features 1' ) @@ -67,6 +70,7 @@ commit_message='Uploaded readme for 2.2.0', strings_displayed=[], strings_not_displayed=[] ) + def test_0010_install_filtering_repository( self ): '''Install the filtering repository into the Galaxy instance.''' self.galaxy_logout() @@ -82,14 +86,16 @@ self.url.replace( 'http://', '' ), installed_repository.installed_changeset_revision ] self.display_galaxy_browse_repositories_page( strings_displayed=strings_displayed ) + def test_0015_uninstall_filtering_repository( self ): '''Uninstall the filtering repository.''' installed_repository = self.test_db_util.get_installed_repository_by_name_owner( 'filtering_0000', common.test_user_1_name ) - self.uninstall_repository( installed_repository, remove_from_disk=True ) + self.uninstall_repository( installed_repository ) strings_not_displayed = [ 'filtering_0000', "Galaxy's filtering tool for test 0000", installed_repository.installed_changeset_revision ] self.display_galaxy_browse_repositories_page( strings_not_displayed=strings_not_displayed ) + def test_0020_reinstall_filtering_repository( self ): '''Reinstall the filtering repository.''' installed_repository = self.test_db_util.get_installed_repository_by_name_owner( 'filtering_0000', common.test_user_1_name ) @@ -103,14 +109,16 @@ strings_displayed.extend( [ 'Installed tool shed repository', 'Valid tools', 'Filter1' ] ) self.display_installed_repository_manage_page( installed_repository, strings_displayed=strings_displayed ) self.verify_tool_metadata_for_installed_repository( installed_repository ) + def test_0025_deactivate_filtering_repository( self ): '''Deactivate the filtering repository without removing it from disk.''' installed_repository = self.test_db_util.get_installed_repository_by_name_owner( 'filtering_0000', common.test_user_1_name ) - self.uninstall_repository( installed_repository, remove_from_disk=False ) + self.deactivate_repository( installed_repository ) strings_not_displayed = [ 'filtering_0000', "Galaxy's filtering tool for test 0000", installed_repository.installed_changeset_revision ] self.display_galaxy_browse_repositories_page( strings_not_displayed=strings_not_displayed ) + def test_0030_reactivate_filtering_repository( self ): '''Reactivate the filtering repository and verify that it now shows up in the list of installed repositories.''' installed_repository = self.test_db_util.get_installed_repository_by_name_owner( 'filtering_0000', common.test_user_1_name ) diff -r ca11e10aa1a5b7318d3727816bf0c2fe6137c522 -r 110a0aabd293fd1393d1296cb220a0b153d69123 test/tool_shed/functional/test_1210_uninstall_reinstall_repository_with_tool_dependencies.py --- a/test/tool_shed/functional/test_1210_uninstall_reinstall_repository_with_tool_dependencies.py +++ b/test/tool_shed/functional/test_1210_uninstall_reinstall_repository_with_tool_dependencies.py @@ -1,7 +1,9 @@ from tool_shed.base.twilltestcase import ShedTwillTestCase, common, os + class UninstallingAndReinstallingRepositories( ShedTwillTestCase ): '''Test uninstalling and reinstalling a repository with tool dependencies.''' + def test_0000_initiate_users( self ): """Create necessary user accounts.""" self.galaxy_logout() @@ -19,6 +21,7 @@ admin_user = self.test_db_util.get_user( common.admin_email ) assert admin_user is not None, 'Problem retrieving user with email %s from the database' % common.admin_email admin_user_private_role = self.test_db_util.get_private_role( admin_user ) + def test_0005_ensure_repositories_and_categories_exist( self ): '''Create the 0010 category and upload the freebayes repository to the tool shed, if necessary.''' category = self.create_category( name='Test 0010 Repository With Tool Dependencies', description='Tests for a repository with tool dependencies.' ) @@ -84,6 +87,7 @@ commit_message='Uploaded valid tool dependency XML.', strings_displayed=[], strings_not_displayed=[] ) + def test_0010_install_freebayes_repository( self ): '''Install the freebayes repository into the Galaxy instance.''' self.galaxy_logout() @@ -101,12 +105,14 @@ self.url.replace( 'http://', '' ), installed_repository.installed_changeset_revision ] self.display_galaxy_browse_repositories_page( strings_displayed=strings_displayed ) + def test_0015_uninstall_freebayes_repository( self ): '''Uninstall the freebayes repository.''' installed_repository = self.test_db_util.get_installed_repository_by_name_owner( 'freebayes_0010', common.test_user_1_name ) - self.uninstall_repository( installed_repository, remove_from_disk=True ) + self.uninstall_repository( installed_repository ) strings_not_displayed = [ installed_repository.name, installed_repository.installed_changeset_revision ] self.display_galaxy_browse_repositories_page( strings_not_displayed=strings_not_displayed ) + def test_0020_reinstall_freebayes_repository( self ): '''Reinstall the freebayes repository.''' installed_repository = self.test_db_util.get_installed_repository_by_name_owner( 'freebayes_0010', common.test_user_1_name ) @@ -120,12 +126,14 @@ strings_displayed.extend( [ 'Installed tool shed repository', 'Valid tools', 'FreeBayes' ] ) self.display_installed_repository_manage_page( installed_repository, strings_displayed=strings_displayed ) self.verify_tool_metadata_for_installed_repository( installed_repository ) + def test_0025_deactivate_freebayes_repository( self ): '''Deactivate the freebayes repository without removing it from disk.''' installed_repository = self.test_db_util.get_installed_repository_by_name_owner( 'freebayes_0010', common.test_user_1_name ) - self.uninstall_repository( installed_repository, remove_from_disk=False ) + self.deactivate_repository( installed_repository ) strings_not_displayed = [ installed_repository.name, installed_repository.installed_changeset_revision ] self.display_galaxy_browse_repositories_page( strings_not_displayed=strings_not_displayed ) + def test_0030_reactivate_freebayes_repository( self ): '''Reactivate the freebayes repository and verify that it now shows up in the list of installed repositories.''' installed_repository = self.test_db_util.get_installed_repository_by_name_owner( 'freebayes_0010', common.test_user_1_name ) diff -r ca11e10aa1a5b7318d3727816bf0c2fe6137c522 -r 110a0aabd293fd1393d1296cb220a0b153d69123 test/tool_shed/functional/test_1220_uninstall_reinstall_repository_with_repository_dependencies.py --- a/test/tool_shed/functional/test_1220_uninstall_reinstall_repository_with_repository_dependencies.py +++ b/test/tool_shed/functional/test_1220_uninstall_reinstall_repository_with_repository_dependencies.py @@ -108,7 +108,7 @@ def test_0015_uninstall_emboss_repository( self ): '''Uninstall the emboss repository.''' installed_repository = self.test_db_util.get_installed_repository_by_name_owner( emboss_repository_name, common.test_user_1_name ) - self.uninstall_repository( installed_repository, remove_from_disk=True ) + self.uninstall_repository( installed_repository ) strings_not_displayed = [ installed_repository.name, installed_repository.installed_changeset_revision ] self.display_galaxy_browse_repositories_page( strings_not_displayed=strings_not_displayed ) @@ -129,7 +129,7 @@ def test_0025_deactivate_emboss_repository( self ): '''Deactivate the emboss repository without removing it from disk.''' installed_repository = self.test_db_util.get_installed_repository_by_name_owner( emboss_repository_name, common.test_user_1_name ) - self.uninstall_repository( installed_repository, remove_from_disk=False ) + self.deactivate_repository( installed_repository ) strings_not_displayed = [ 'emboss_0020', installed_repository.installed_changeset_revision ] self.display_galaxy_browse_repositories_page( strings_not_displayed=strings_not_displayed ) diff -r ca11e10aa1a5b7318d3727816bf0c2fe6137c522 -r 110a0aabd293fd1393d1296cb220a0b153d69123 test/tool_shed/functional/test_1230_uninstall_reinstall_repository_with_dependency_revisions.py --- a/test/tool_shed/functional/test_1230_uninstall_reinstall_repository_with_dependency_revisions.py +++ b/test/tool_shed/functional/test_1230_uninstall_reinstall_repository_with_dependency_revisions.py @@ -144,7 +144,7 @@ def test_0015_uninstall_emboss_repository( self ): '''Uninstall the emboss repository.''' installed_repository = self.test_db_util.get_installed_repository_by_name_owner( emboss_repository_name, common.test_user_1_name ) - self.uninstall_repository( installed_repository, remove_from_disk=True ) + self.uninstall_repository( installed_repository ) strings_not_displayed = [ installed_repository.installed_changeset_revision ] self.display_galaxy_browse_repositories_page( strings_not_displayed=strings_not_displayed ) @@ -165,7 +165,7 @@ def test_0025_deactivate_emboss_repository( self ): '''Deactivate the emboss repository without removing it from disk.''' installed_repository = self.test_db_util.get_installed_repository_by_name_owner( emboss_repository_name, common.test_user_1_name ) - self.uninstall_repository( installed_repository, remove_from_disk=False ) + self.deactivate_repository( installed_repository ) strings_not_displayed = [ installed_repository.installed_changeset_revision ] self.display_galaxy_browse_repositories_page( strings_not_displayed=strings_not_displayed ) diff -r ca11e10aa1a5b7318d3727816bf0c2fe6137c522 -r 110a0aabd293fd1393d1296cb220a0b153d69123 test/tool_shed/functional/test_1430_repair_installed_repository.py --- a/test/tool_shed/functional/test_1430_repair_installed_repository.py +++ b/test/tool_shed/functional/test_1430_repair_installed_repository.py @@ -146,10 +146,9 @@ installed_repository = self.test_db_util.get_installed_repository_by_name_owner( 'filter_1430', common.test_user_1_name ) strings_displayed = [ 'This repository cannot be uninstalled' ] strings_not_displayed = [ 'Check to uninstall' ] - self.uninstall_repository( installed_repository, - remove_from_disk=False, - strings_displayed=strings_displayed, - strings_not_displayed=strings_not_displayed ) + self.deactivate_repository( installed_repository, + strings_displayed=strings_displayed, + strings_not_displayed=strings_not_displayed ) strings_not_displayed = [ 'filter_1430', "Galaxy's filter tool for test 1430", installed_repository.installed_changeset_revision ] diff -r ca11e10aa1a5b7318d3727816bf0c2fe6137c522 -r 110a0aabd293fd1393d1296cb220a0b153d69123 test/tool_shed/functional/test_1450_installing_datatypes_sniffers.py --- a/test/tool_shed/functional/test_1450_installing_datatypes_sniffers.py +++ b/test/tool_shed/functional/test_1450_installing_datatypes_sniffers.py @@ -119,7 +119,7 @@ global repository_datatypes_count global base_datatypes_count global base_sniffers_count - self.uninstall_repository( repository, remove_from_disk=False ) + self.deactivate_repository( repository ) new_datatypes_count = self.get_datatypes_count() assert new_datatypes_count == base_datatypes_count, 'Expected %d datatypes, got %d' % ( base_datatypes_count, new_datatypes_count ) current_sniffers = self.get_sniffers_count() @@ -157,7 +157,7 @@ repository = self.test_db_util.get_installed_repository_by_name_owner( repository_name, common.test_user_1_name ) global repository_datatypes_count global base_datatypes_count - self.uninstall_repository( repository, remove_from_disk=True ) + self.uninstall_repository( repository ) new_datatypes_count = self.get_datatypes_count() assert new_datatypes_count == base_datatypes_count, 'Expected %d datatypes, got %d' % ( base_datatypes_count, new_datatypes_count ) current_sniffers = self.get_sniffers_count() Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: greg: Handle invalid information from the tool shed API in the tool shed's install and test framework.
by commits-noreply＠bitbucket.org 08 Jan '14

08 Jan '14

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/ca11e10aa1a5/ Changeset: ca11e10aa1a5 User: greg Date: 2014-01-09 04:29:33 Summary: Handle invalid information from the tool shed API in the tool shed's install and test framework. Affected #: 1 file diff -r ce0d87c3afb0269307640dc64daee3b47095dedf -r ca11e10aa1a5b7318d3727816bf0c2fe6137c522 test/install_and_test_tool_shed_repositories/base/util.py --- a/test/install_and_test_tool_shed_repositories/base/util.py +++ b/test/install_and_test_tool_shed_repositories/base/util.py @@ -889,9 +889,19 @@ log.debug( 'due to the following error getting repository_dependencies_dicts:\n%s' % str( error_message ) ) else: for repository_dependencies_dict in repository_dependencies_dicts: - name = str( repository_dependencies_dict[ 'name' ] ) - owner = str( repository_dependencies_dict[ 'owner' ] ) - changeset_revision = str( repository_dependencies_dict[ 'changeset_revision' ] ) + if not isinstance( repository_dependencies_dict, dict ): + log.debug( 'Skipping invalid repository_dependencies_dict: %s' % str( repository_dependencies_dict ) ) + continue + name = repository_dependencies_dict.get( 'name', None ) + owner = repository_dependencies_dict.get( 'owner', None ) + changeset_revision = repository_dependencies_dict.get( 'changeset_revision', None ) + if name is None or owner is None or changeset_revision is None: + log.debug( 'Skipping invalid repository_dependencies_dict due to missing name,owner or changeset_revision: %s' % \ + str( repository_dependencies_dict ) ) + continue + name = str( name ) + owner = str( owner ) + changeset_revision = str( changeset_revision ) log.debug( 'Checking installation containers for revision %s of repository dependency %s owned by %s' % \ ( changeset_revision, name, owner ) ) required_repository_metadata_id = repository_dependencies_dict[ 'id' ] @@ -907,21 +917,21 @@ # Check the required repository's time_last_tested value to see if its tool_test_results column # has been updated within the past 12 hours. The RepositoryMetadata class's to_dict() method # returns the value of time_last_tested in datetime.isoformat(). - twelve_hours_ago = ( datetime.utcnow() - timedelta( hours=12 ) ).isoformat() + twenty_hours_ago = ( datetime.utcnow() - timedelta( hours=20 ) ).isoformat() time_last_tested, error_message = get_time_last_tested( galaxy_tool_shed_url, repository_metadata_id ) - if time_last_tested is not None and time_last_tested < twelve_hours_ago: + if time_last_tested is not None and time_last_tested < twenty_hours_ago: log.debug( 'The install containers for version %s of repository dependency %s owned by %s have been ' % \ ( changeset_revision, name, owner ) ) - log.debug( 'populated within the past 12 hours (likely in this test run), so skipping this check.' ) + log.debug( 'populated within the past 20 hours (likely in this test run), so skipping this check.' ) continue elif time_last_tested is None: log.debug( 'The time_last_tested column value is None for version %s of repository dependency %s owned by %s.' % \ ( changeset_revision, name, owner ) ) - elif time_last_tested < twelve_hours_ago: - log.debug( 'Version %s of repository dependency %s owned by %s was last tested less than 12 hours ago.' % \ + elif time_last_tested < twenty_hours_ago: + log.debug( 'Version %s of repository dependency %s owned by %s was last tested less than 20 hours ago.' % \ ( changeset_revision, name, owner ) ) else: - log.debug( 'Version %s of repository dependency %s owned by %s was last tested more than 12 hours ago.' % \ + log.debug( 'Version %s of repository dependency %s owned by %s was last tested more than 20 hours ago.' % \ ( changeset_revision, name, owner ) ) # Inspect the tool_test_results_dict for the last test run to see if it has not yet been populated. if len( tool_test_results_dicts ) == 0: Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: greg: Eliminate unnecessary RepositoryMetadata.to_dict() method in the tool shed.
by commits-noreply＠bitbucket.org 08 Jan '14

08 Jan '14

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/ce0d87c3afb0/ Changeset: ce0d87c3afb0 User: greg Date: 2014-01-09 03:25:19 Summary: Eliminate unnecessary RepositoryMetadata.to_dict() method in the tool shed. Affected #: 1 file diff -r 61721f9eda65b1e7444264249815ad993e61fb27 -r ce0d87c3afb0269307640dc64daee3b47095dedf lib/galaxy/webapps/tool_shed/model/__init__.py --- a/lib/galaxy/webapps/tool_shed/model/__init__.py +++ b/lib/galaxy/webapps/tool_shed/model/__init__.py @@ -271,14 +271,6 @@ return True return False - def to_dict( self, view='collection', value_mapper=None ): - rval = super( RepositoryMetadata, self ).to_dict( view=view, value_mapper=value_mapper ) - if 'time_last_tested' in rval: - time_last_tested = rval[ 'time_last_tested' ] - if time_last_tested is not None: - rval[ 'time_last_tested' ] = time_last_tested.isoformat() - return rval - class SkipToolTest( object, Dictifiable ): dict_collection_visible_keys = ( 'id', 'repository_metadata_id', 'initial_changeset_revision' ) Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0