1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/changeset/7dcea1c1c833/ changeset: 7dcea1c1c833 user: greg date: 2012-07-10 17:38:32 summary: Fixes for installing tool shed repositories that include proprietary datatypes. affected #: 6 files diff -r 761a17ad3aad454283abca2bf33866ad954a2d97 -r 7dcea1c1c833dac37844430180db6705d4846aa3 lib/galaxy/tool_shed/install_manager.py --- a/lib/galaxy/tool_shed/install_manager.py +++ b/lib/galaxy/tool_shed/install_manager.py @@ -190,9 +190,11 @@ except: pass if 'datatypes' in metadata_dict: - update_tool_shed_repository_status( self.app, - tool_shed_repository, - self.app.model.ToolShedRepository.installation_status.LOADING_PROPRIETARY_DATATYPES ) + tool_shed_repository.status = self.app.model.ToolShedRepository.installation_status.LOADING_PROPRIETARY_DATATYPES + if not tool_shed_repository.includes_datatypes: + tool_shed_repository.includes_datatypes = True + self.app.sa_session.add( tool_shed_repository ) + self.app.sa_session.flush() work_dir = make_tmp_directory() datatypes_config = get_config_from_repository( self.app, 'datatypes_conf.xml', diff -r 761a17ad3aad454283abca2bf33866ad954a2d97 -r 7dcea1c1c833dac37844430180db6705d4846aa3 lib/galaxy/util/shed_util.py --- a/lib/galaxy/util/shed_util.py +++ b/lib/galaxy/util/shed_util.py @@ -309,6 +309,15 @@ for filename in sample_files: if filename not in sample_files_copied: copy_sample_file( app, filename, dest_path=dest_path ) +def create_repo_info_dict( repository, owner, repository_clone_url, changeset_revision, ctx_rev, metadata ): + repo_info_dict = {} + repo_info_dict[ repository.name ] = ( repository.description, + repository_clone_url, + changeset_revision, + ctx_rev, + owner, + metadata.get( 'tool_dependencies', None ) ) + return repo_info_dict def create_repository_dict_for_proprietary_datatypes( tool_shed, name, owner, installed_changeset_revision, tool_dicts, converter_path=None, display_path=None ): return dict( tool_shed=tool_shed, repository_name=name, diff -r 761a17ad3aad454283abca2bf33866ad954a2d97 -r 7dcea1c1c833dac37844430180db6705d4846aa3 lib/galaxy/web/base/controller.py --- a/lib/galaxy/web/base/controller.py +++ b/lib/galaxy/web/base/controller.py @@ -1,7 +1,7 @@ """ Contains functionality needed in every web interface """ -import os, time, logging, re, string, sys, glob, shutil, tempfile, subprocess +import os, time, logging, re, string, sys, glob, shutil, tempfile, subprocess, operator from datetime import date, datetime, timedelta from time import strftime from galaxy import config, tools, web, util @@ -177,6 +177,14 @@ def not_implemented( self, trans, **kwd ): raise HTTPNotImplemented() +class Datatype( object ): + """Used for storing in-memory list of datatypes currently in the datatypes registry.""" + def __init__( self, extension, dtype, type_extension, mimetype, display_in_upload ): + self.extension = extension + self.dtype = dtype + self.type_extension = type_extension + self.mimetype = mimetype + self.display_in_upload = display_in_upload # # -- Mixins for working with Galaxy objects. -- # @@ -2722,3 +2730,18 @@ return trans.environ[ 'webapp' ] # The default is galaxy. return 'galaxy' +def sort_by_attr( seq, attr ): + """ + Sort the sequence of objects by object's attribute + Arguments: + seq - the list or any sequence (including immutable one) of objects to sort. + attr - the name of attribute to sort by + """ + # Use the "Schwartzian transform" + # Create the auxiliary list of tuples where every i-th tuple has form + # (seq[i].attr, i, seq[i]) and sort it. The second item of tuple is needed not + # only to provide stable sorting, but mainly to eliminate comparison of objects + # (which can be expensive or prohibited) in case of equal attribute values. + intermed = map( None, map( getattr, seq, ( attr, ) * len( seq ) ), xrange( len( seq ) ), seq ) + intermed.sort() + return map( operator.getitem, intermed, ( -1, ) * len( intermed ) ) diff -r 761a17ad3aad454283abca2bf33866ad954a2d97 -r 7dcea1c1c833dac37844430180db6705d4846aa3 lib/galaxy/web/controllers/admin_toolshed.py --- a/lib/galaxy/web/controllers/admin_toolshed.py +++ b/lib/galaxy/web/controllers/admin_toolshed.py @@ -605,7 +605,8 @@ tool_section = trans.app.toolbox.tool_panel[ tool_panel_section_key ] else: tool_section = None - for tool_shed_repository, repo_info_dict in zip( tool_shed_repositories, repo_info_dicts ): + for tup in zip( tool_shed_repositories, repo_info_dicts ): + tool_shed_repository, repo_info_dict = tup # Clone each repository to the configured location. update_tool_shed_repository_status( trans.app, tool_shed_repository, trans.model.ToolShedRepository.installation_status.CLONING ) repo_info_tuple = repo_info_dict[ tool_shed_repository.name ] @@ -670,37 +671,6 @@ shutil.rmtree( work_dir ) except: pass - if 'datatypes' in metadata: - update_tool_shed_repository_status( trans.app, - tool_shed_repository, - trans.model.ToolShedRepository.installation_status.LOADING_PROPRIETARY_DATATYPES ) - work_dir = make_tmp_directory() - datatypes_config = get_config_from_repository( trans.app, - 'datatypes_conf.xml', - tool_shed_repository, - tool_shed_repository.installed_changeset_revision, - work_dir ) - # Load proprietary data types required by tools. - converter_path, display_path = alter_config_and_load_prorietary_datatypes( trans.app, datatypes_config, relative_install_dir, override=False ) - if converter_path or display_path: - # Create a dictionary of tool shed repository related information. - repository_dict = create_repository_dict_for_proprietary_datatypes( tool_shed=self.tool_shed, - name=tool_shed_repository.name, - owner=tool_shed_repository.owner, - installed_changeset_revision=tool_shed_repository.installed_changeset_revision, - tool_dicts=metadata.get( 'tools', [] ), - converter_path=converter_path, - display_path=display_path ) - if converter_path: - # Load proprietary datatype converters - trans.app.datatypes_registry.load_datatype_converters( trans.app.toolbox, installed_repository_dict=repository_dict ) - if display_path: - # Load proprietary datatype display applications - trans.app.datatypes_registry.load_display_applications( installed_repository_dict=repository_dict ) - try: - shutil.rmtree( work_dir ) - except: - pass update_tool_shed_repository_status( trans.app, tool_shed_repository, trans.model.ToolShedRepository.installation_status.INSTALLED ) tsr_ids_for_monitoring = [ trans.security.encode_id( tsr.id ) for tsr in tool_shed_repositories ] return trans.response.send_redirect( web.url_for( controller='admin_toolshed', @@ -749,9 +719,11 @@ except: pass if 'datatypes' in metadata_dict: - update_tool_shed_repository_status( trans.app, - tool_shed_repository, - trans.model.ToolShedRepository.installation_status.LOADING_PROPRIETARY_DATATYPES ) + tool_shed_repository.status = trans.model.ToolShedRepository.installation_status.LOADING_PROPRIETARY_DATATYPES + if not tool_shed_repository.includes_datatypes: + tool_shed_repository.includes_datatypes = True + trans.sa_session.add( tool_shed_repository ) + trans.sa_session.flush() work_dir = make_tmp_directory() datatypes_config = get_config_from_repository( trans.app, 'datatypes_conf.xml', @@ -1255,13 +1227,12 @@ repo_info_dict = kwd.get( 'repo_info_dict', None ) if not repo_info_dict: # This should only happen if the tool_shed_repository does not include any valid tools. - repo_info_dict = {} - repo_info_dict[ tool_shed_repository.name ] = ( tool_shed_repository.description, - repository_clone_url, - tool_shed_repository.installed_changeset_revision, - ctx_rev, - tool_shed_repository.owner, - metadata.get( 'tool_dependencies', None ) ) + repo_info_dict = create_repo_info_dict( tool_shed_repository, + tool_shed_repository.owner, + repository_clone_url, + tool_shed_repository.installed_changeset_revision, + ctx_rev, + metadata ) new_kwd = dict( includes_tool_dependencies=tool_shed_repository.includes_tool_dependencies, includes_tools=tool_shed_repository.includes_tools, install_tool_dependencies=install_tool_dependencies, @@ -1291,7 +1262,8 @@ if ids is not None and status_list is not None: ids = util.listify( ids ) status_list = util.listify( status_list ) - for id, status in zip( ids, status_list ): + for tup in zip( ids, status_list ): + id, status = tup repository = trans.sa_session.query( trans.model.ToolShedRepository ).get( trans.security.decode_id( id ) ) if repository.status != status: rval.append( dict( id=id, @@ -1335,13 +1307,12 @@ tool_shed_url = get_url_from_repository_tool_shed( trans.app, repository ) ctx_rev = get_ctx_rev( tool_shed_url, repository.name, repository.owner, repository.installed_changeset_revision ) repository_clone_url = generate_clone_url( trans, repository ) - repo_info_dict = {} - repo_info_dict[ repository.name ] = ( repository.description, - repository_clone_url, - repository.installed_changeset_revision, - ctx_rev, - repository.owner, - metadata.get( 'tool_dependencies', None ) ) + repo_info_dict = create_repo_info_dict( repository, + repository.owner, + repository_clone_url, + repository.installed_changeset_revision, + ctx_rev, + metadata ) # Get the location in the tool panel in which the tool was originally loaded. if 'tool_panel_section' in metadata: tool_panel_dict = metadata[ 'tool_panel_section' ] @@ -1419,7 +1390,8 @@ if ids is not None and status_list is not None: ids = util.listify( ids ) status_list = util.listify( status_list ) - for id, status in zip( ids, status_list ): + for tup in zip( ids, status_list ): + id, status = tup tool_dependency = trans.sa_session.query( trans.model.ToolDependency ).get( trans.security.decode_id( id ) ) if tool_dependency.status != status: rval.append( dict( id=id, diff -r 761a17ad3aad454283abca2bf33866ad954a2d97 -r 7dcea1c1c833dac37844430180db6705d4846aa3 lib/galaxy/webapps/community/controllers/repository.py --- a/lib/galaxy/webapps/community/controllers/repository.py +++ b/lib/galaxy/webapps/community/controllers/repository.py @@ -9,7 +9,7 @@ from galaxy.web.framework.helpers import time_ago, iff, grids from galaxy.util.json import from_json_string, to_json_string from galaxy.model.orm import * -from galaxy.util.shed_util import get_changectx_for_changeset, get_configured_ui, get_repository_file_contents, make_tmp_directory, NOT_TOOL_CONFIGS +from galaxy.util.shed_util import create_repo_info_dict, get_changectx_for_changeset, get_configured_ui, get_repository_file_contents, make_tmp_directory, NOT_TOOL_CONFIGS from galaxy.util.shed_util import open_repository_files_folder, reversed_lower_upper_bounded_changelog, strip_path from galaxy.tool_shed.encoding_util import * from common import * @@ -931,29 +931,6 @@ trans.sa_session.flush() download_url = '/repos/%s/%s/archive/%s' % ( repository.user.username, repository.name, file_type_str ) return trans.response.send_redirect( download_url ) - def __encode_repo_info_dict( self, trans, repository_metadata_ids ): - repo_info_dict = {} - includes_tools = False - includes_tool_dependencies = False - for repository_metadata_id in util.listify( repository_metadata_ids ): - repository_metadata = get_repository_metadata_by_id( trans, repository_metadata_id ) - if not includes_tools and 'tools' in repository_metadata.metadata: - includes_tools = True - if not includes_tool_dependencies and 'tool_dependencies' in repository_metadata.metadata: - includes_tool_dependencies = True - repository = get_repository( trans, trans.security.encode_id( repository_metadata.repository_id ) ) - # Get the changelog rev for this changeset_revision. - repo_dir = repository.repo_path - repo = hg.repository( get_configured_ui(), repo_dir ) - changeset_revision = repository_metadata.changeset_revision - ctx = get_changectx_for_changeset( repo, changeset_revision ) - repository_id = trans.security.encode_id( repository.id ) - repository_clone_url = generate_clone_url( trans, repository_id ) - if includes_tool_dependencies: - repo_info_dict[ repository.name ] = ( repository.description, repository_clone_url, changeset_revision, str( ctx.rev() ) ) - else: - repo_info_dict[ repository.name ] = ( repository.description, repository_clone_url, changeset_revision, str( ctx.rev() ) ) - return tool_shed_encode( repo_info_dict ), includes_tools, includes_tool_dependencies @web.expose def find_tools( self, trans, **kwd ): params = util.Params( kwd ) @@ -1156,7 +1133,8 @@ includes_tools = False includes_tool_dependencies = False repo_info_dicts = [] - for repository_id, changeset_revision in zip( util.listify( repository_ids ), util.listify( changeset_revisions ) ): + for tup in zip( util.listify( repository_ids ), util.listify( changeset_revisions ) ): + repository_id, changeset_revision = tup repository_clone_url = generate_clone_url( trans, repository_id ) repository = get_repository( trans, repository_id ) repository_metadata = get_repository_metadata_by_changeset_revision( trans, repository_id, changeset_revision ) @@ -1168,15 +1146,8 @@ repo_dir = repository.repo_path repo = hg.repository( get_configured_ui(), repo_dir ) ctx = get_changectx_for_changeset( repo, changeset_revision ) - repo_info_dict = {} - repo_info_dict[ repository.name ] = ( repository.description, - repository_clone_url, - changeset_revision, - str( ctx.rev() ), - repository.user.username, - metadata.get( 'tool_dependencies', None ) ) - encoded_repo_info_dict = tool_shed_encode( repo_info_dict ) - repo_info_dicts.append( encoded_repo_info_dict ) + repo_info_dict = create_repo_info_dict( repository, repository.user.username, repository_clone_url, changeset_revision, str( ctx.rev() ), metadata ) + repo_info_dicts.append( tool_shed_encode( repo_info_dict ) ) return dict( includes_tools=includes_tools, includes_tool_dependencies=includes_tool_dependencies, repo_info_dicts=repo_info_dicts ) @web.expose def get_readme( self, trans, **kwd ): diff -r 761a17ad3aad454283abca2bf33866ad954a2d97 -r 7dcea1c1c833dac37844430180db6705d4846aa3 templates/admin/view_datatypes_registry.mako --- a/templates/admin/view_datatypes_registry.mako +++ b/templates/admin/view_datatypes_registry.mako @@ -5,8 +5,24 @@ ${render_msg( message, status )} %endif +<% + import galaxy.util + from galaxy.web.base.controller import sort_by_attr, Datatype + ctr = 0 + datatypes = [] + for elem in trans.app.datatypes_registry.datatype_elems: + # Build a list of objects that can be sorted. + extension = elem.get( 'extension', None ) + dtype = elem.get( 'type', None ) + type_extension = elem.get( 'type_extension', None ) + mimetype = elem.get( 'mimetype', None ) + display_in_upload = galaxy.util.string_as_bool( elem.get( 'display_in_upload', False ) ) + datatypes.append( Datatype( extension, dtype, type_extension, mimetype, display_in_upload ) ) + sorted_datatypes = sort_by_attr( datatypes, 'extension' ) +%> + <div class="toolForm"> - <div class="toolFormTitle">Current datatypes registry contains ${len( trans.app.datatypes_registry.datatype_elems )} datatypes</div> + <div class="toolFormTitle">Current datatypes registry contains ${len( sorted_datatypes )} datatypes</div><div class="toolFormBody"><table class="manage-table colored" border="0" cellspacing="0" cellpadding="0" width="100%"><tr> @@ -14,33 +30,23 @@ <th>Type</th><th>Mimetype</th><th>Display in upload</th> - <th>Datatype class</th></tr> - <% ctr = 0 %> - %for elem in trans.app.datatypes_registry.datatype_elems: - <% - import galaxy.util - extension = elem.get( 'extension', None ) - dtype = elem.get( 'type', None ) - type_extension = elem.get( 'type_extension', None ) - mimetype = elem.get( 'mimetype', None ) - display_in_upload = galaxy.util.string_as_bool( elem.get( 'display_in_upload', False ) ) - %> + %for datatype in sorted_datatypes: %if ctr % 2 == 1: <tr class="odd_row"> %else: <tr class="tr"> %endif - <td>${extension}</td> - <td>${dtype}</td> + <td>${datatype.extension}</td> + <td>${datatype.dtype}</td><td> - %if mimetype: - ${mimetype} + %if datatype.mimetype: + ${datatype.mimetype} %endif </td><td> - %if display_in_upload: - ${display_in_upload} + %if datatype.display_in_upload: + ${datatype.display_in_upload} %endif </td></tr> Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.