12 new commits in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/8896bf67ff7a/ Changeset: 8896bf67ff7a User: jmchilton Date: 2014-12-29 14:11:54+00:00 Summary: Correct sample value for migrated tool config. Affected #: 1 file diff -r bfd679408c84a4c6f1d2b1a09eec1dab9c2d212b -r 8896bf67ff7ae6f4379ff0e18c30559a20cf6584 config/galaxy.ini.sample --- a/config/galaxy.ini.sample +++ b/config/galaxy.ini.sample @@ -148,7 +148,7 @@ # Tool config maintained by tool migration scripts. If you use the migration # scripts to install tools that have been migrated to the tool shed upon a new # release, they will be added to this tool config file. -#migrated_tools_config = migrated_tools_conf.xml +#migrated_tools_config = config/migrated_tools_conf.xml # File that contains the XML section and tool tags from all tool panel config # files integrated into a single file that defines the tool panel layout. This https://bitbucket.org/galaxy/galaxy-central/commits/d87f3d474c9a/ Changeset: d87f3d474c9a User: jmchilton Date: 2014-12-29 14:11:54+00:00 Summary: Small simplification to tool_panel_manager.py. Affected #: 1 file diff -r 8896bf67ff7ae6f4379ff0e18c30559a20cf6584 -r d87f3d474c9ab798e88286f80c09cef2bc41451a lib/tool_shed/galaxy_install/tools/tool_panel_manager.py --- a/lib/tool_shed/galaxy_install/tools/tool_panel_manager.py +++ b/lib/tool_shed/galaxy_install/tools/tool_panel_manager.py @@ -243,22 +243,13 @@ tool_file_path, tup_guid, tool = repository_tool_tup if tup_guid == guid: break - if inside_section: - tool_elem = self.generate_tool_elem( tool_shed, - repository_name, - changeset_revision, - owner, - tool_file_path, - tool, - tool_section ) - else: - tool_elem = self.generate_tool_elem( tool_shed, - repository_name, - changeset_revision, - owner, - tool_file_path, - tool, - None ) + tool_elem = self.generate_tool_elem( tool_shed, + repository_name, + changeset_revision, + owner, + tool_file_path, + tool, + tool_section if inside_section else None ) if inside_section: if section_in_elem_list: elem_list[ index ] = tool_section https://bitbucket.org/galaxy/galaxy-central/commits/000cd07f16f9/ Changeset: 000cd07f16f9 User: jmchilton Date: 2014-12-29 14:11:54+00:00 Summary: Reused get_section method to hide ToolBox.tool_panel from install_manager.py. Affected #: 4 files diff -r d87f3d474c9ab798e88286f80c09cef2bc41451a -r 000cd07f16f991cb78025e23f0872bafd58b9daf lib/galaxy/tools/__init__.py --- a/lib/galaxy/tools/__init__.py +++ b/lib/galaxy/tools/__init__.py @@ -277,13 +277,13 @@ self.write_integrated_tool_panel_config_file() app.reindex_tool_search() - def get_or_create_section( self, section_id, new_label=None ): + def get_section( self, section_id, new_label=None, create_if_needed=False ): tool_panel_section_key = str( section_id ) if tool_panel_section_key in self.tool_panel: # Appending a tool to an existing section in toolbox.tool_panel tool_section = self.tool_panel[ tool_panel_section_key ] log.debug( "Appending to tool panel section: %s" % str( tool_section.name ) ) - else: + elif create_if_needed: # Appending a new section to toolbox.tool_panel if new_label is None: # This might add an ugly section label to the tool panel, but, oh well... @@ -296,6 +296,8 @@ tool_section = ToolSection( section_dict ) self.tool_panel[ tool_panel_section_key ] = tool_section log.debug( "Loading new tool panel section: %s" % str( tool_section.name ) ) + else: + tool_section = None return tool_panel_section_key, tool_section def __resolve_tool_path(self, tool_path, config_filename): diff -r d87f3d474c9ab798e88286f80c09cef2bc41451a -r 000cd07f16f991cb78025e23f0872bafd58b9daf lib/tool_shed/galaxy_install/install_manager.py --- a/lib/tool_shed/galaxy_install/install_manager.py +++ b/lib/tool_shed/galaxy_install/install_manager.py @@ -809,12 +809,10 @@ def install_tool_shed_repository( self, tool_shed_repository, repo_info_dict, tool_panel_section_key, shed_tool_conf, tool_path, install_tool_dependencies, reinstalling=False ): if tool_panel_section_key: - try: - tool_section = self.app.toolbox.tool_panel[ tool_panel_section_key ] - except KeyError: + _, tool_section = self.app.toolbox.get_section( tool_panel_section_key ) + if tool_section is None: log.debug( 'Invalid tool_panel_section_key "%s" specified. Tools will be loaded outside of sections in the tool panel.', str( tool_panel_section_key ) ) - tool_section = None else: tool_section = None if isinstance( repo_info_dict, basestring ): diff -r d87f3d474c9ab798e88286f80c09cef2bc41451a -r 000cd07f16f991cb78025e23f0872bafd58b9daf lib/tool_shed/galaxy_install/tools/tool_panel_manager.py --- a/lib/tool_shed/galaxy_install/tools/tool_panel_manager.py +++ b/lib/tool_shed/galaxy_install/tools/tool_panel_manager.py @@ -297,7 +297,7 @@ return tool_section def get_or_create_tool_section( self, toolbox, tool_panel_section_id, new_tool_panel_section_label=None ): - return toolbox.get_or_create_section( section_id=tool_panel_section_id, new_label=new_tool_panel_section_label ) + return toolbox.get_section( section_id=tool_panel_section_id, new_label=new_tool_panel_section_label, create_if_needed=True ) def get_shed_tool_conf_dict( self, shed_tool_conf ): """ diff -r d87f3d474c9ab798e88286f80c09cef2bc41451a -r 000cd07f16f991cb78025e23f0872bafd58b9daf test/unit/tool_shed_unit_tests/test_tool_panel_manager.py --- a/test/unit/tool_shed_unit_tests/test_tool_panel_manager.py +++ b/test/unit/tool_shed_unit_tests/test_tool_panel_manager.py @@ -46,7 +46,7 @@ self.tool, ) ] - _, section = self.toolbox.get_or_create_section("tid1") + _, section = self.toolbox.get_section("tid1", create_if_needed=True) tpm = self.tpm tool_panel_dict = tpm.generate_tool_panel_dict_for_new_install( tool_dicts=new_tools, https://bitbucket.org/galaxy/galaxy-central/commits/739ca42b8555/ Changeset: 739ca42b8555 User: jmchilton Date: 2014-12-29 14:11:54+00:00 Summary: PEP-8 fixes for various Tool Shed components. - lib/tool_shed/util/tool_util.py - lib/tool_shed/galaxy_install/tool_migration_manager.py. - lib/tool_shed/util/shed_util_common.py. - lib/galaxy/model/tool_shed_install/__init__.py. Affected #: 4 files diff -r 000cd07f16f991cb78025e23f0872bafd58b9daf -r 739ca42b855530e8293cbb25298e1d4f6ef03d15 lib/galaxy/model/tool_shed_install/__init__.py --- a/lib/galaxy/model/tool_shed_install/__init__.py +++ b/lib/galaxy/model/tool_shed_install/__init__.py @@ -11,9 +11,9 @@ class ToolShedRepository( object ): dict_collection_visible_keys = ( 'id', 'tool_shed', 'name', 'owner', 'installed_changeset_revision', 'changeset_revision', 'ctx_rev', 'includes_datatypes', - 'tool_shed_status', 'deleted', 'uninstalled', 'dist_to_shed', 'status', 'error_message' ) + 'tool_shed_status', 'deleted', 'uninstalled', 'dist_to_shed', 'status', 'error_message' ) dict_element_visible_keys = ( 'id', 'tool_shed', 'name', 'owner', 'installed_changeset_revision', 'changeset_revision', 'ctx_rev', 'includes_datatypes', - 'tool_shed_status', 'deleted', 'uninstalled', 'dist_to_shed', 'status', 'error_message' ) + 'tool_shed_status', 'deleted', 'uninstalled', 'dist_to_shed', 'status', 'error_message' ) installation_status = Bunch( NEW='New', CLONING='Cloning', SETTING_TOOL_VERSIONS='Setting tool versions', @@ -356,12 +356,12 @@ for rd_tup in rd_tups: if len( rd_tup ) == 5: tool_shed, name, owner, changeset_revision, prior_installation_required, only_if_compiling_contained_td = \ - common_util.parse_repository_dependency_tuple( rd_tup, contains_error=False ) + common_util.parse_repository_dependency_tuple( rd_tup, contains_error=False ) if asbool( prior_installation_required ): required_rd_tups_that_must_be_installed.append( ( tool_shed, name, owner, changeset_revision, 'True', 'False' ) ) elif len( rd_tup ) == 6: tool_shed, name, owner, changeset_revision, prior_installation_required, only_if_compiling_contained_td = \ - common_util.parse_repository_dependency_tuple( rd_tup, contains_error=False ) + common_util.parse_repository_dependency_tuple( rd_tup, contains_error=False ) # The repository dependency will only be required to be previously installed if it does not fall into the category of # a repository that must be installed only so that its contained tool dependency can be used for compiling the tool # dependency of the dependent repository. @@ -493,12 +493,14 @@ class RepositoryRepositoryDependencyAssociation( object ): + def __init__( self, tool_shed_repository_id=None, repository_dependency_id=None ): self.tool_shed_repository_id = tool_shed_repository_id self.repository_dependency_id = repository_dependency_id class RepositoryDependency( object ): + def __init__( self, tool_shed_repository_id=None ): self.tool_shed_repository_id = tool_shed_repository_id @@ -647,6 +649,7 @@ class ToolVersionAssociation( object ): + def __init__( self, id=None, tool_id=None, parent_id=None ): self.id = id self.tool_id = tool_id @@ -654,6 +657,7 @@ class MigrateTools( object ): + def __init__( self, repository_id=None, repository_path=None, version=None ): self.repository_id = repository_id self.repository_path = repository_path diff -r 000cd07f16f991cb78025e23f0872bafd58b9daf -r 739ca42b855530e8293cbb25298e1d4f6ef03d15 lib/tool_shed/galaxy_install/tool_migration_manager.py --- a/lib/tool_shed/galaxy_install/tool_migration_manager.py +++ b/lib/tool_shed/galaxy_install/tool_migration_manager.py @@ -171,8 +171,6 @@ clone_dir = os.path.join( self.tool_path, relative_clone_dir ) if not self.__iscloned( clone_dir ): repository_clone_url = os.path.join( self.tool_shed_url, 'repos', owner, name ) - relative_install_dir = os.path.join( relative_clone_dir, name ) - install_dir = os.path.join( clone_dir, name ) ctx_rev = suc.get_ctx_rev( self.app, self.tool_shed_url, name, owner, changeset_revision ) tool_shed_repository = suc.create_or_update_tool_shed_repository( app=self.app, name=name, @@ -406,7 +404,7 @@ tool_configs_to_filter.append( tool_config ) else: print 'The tool "%s" (%s) has not been enabled because it is not defined in a proprietary tool config (%s).' \ - % ( guid, tool_config, ", ".join( self.proprietary_tool_confs or [] ) ) + % ( guid, tool_config, ", ".join( self.proprietary_tool_confs or [] ) ) if tool_configs_to_filter: lock = threading.Lock() lock.acquire( True ) @@ -421,7 +419,7 @@ repository=tool_shed_repository, changeset_revision=tool_shed_repository.changeset_revision, repository_clone_url=repository_clone_url, - shed_config_dict = self.shed_config_dict, + shed_config_dict=self.shed_config_dict, relative_install_dir=relative_install_dir, repository_files_dir=None, resetting_all_metadata_on_repository=False, @@ -504,11 +502,11 @@ datatypes_config = hg_util.get_config_from_disk( suc.DATATYPES_CONFIG_FILENAME, repo_install_dir ) # Load proprietary data types required by tools. The value of override is not # important here since the Galaxy server will be started after this installation - #completes. + # completes. converter_path, display_path = \ cdl.alter_config_and_load_prorietary_datatypes( datatypes_config, repo_install_dir, - override=False ) + override=False ) if converter_path or display_path: # Create a dictionary of tool shed repository related information. repository_dict = \ @@ -583,23 +581,23 @@ tool_version_dicts = json.loads( text ) tvm.handle_tool_versions( tool_version_dicts, tool_shed_repository ) else: - # Set the tool versions since they seem to be missing for this repository in the tool shed. - # CRITICAL NOTE: These default settings may not properly handle all parent/child associations. + # Set the tool versions since they seem to be missing + # for this repository in the tool shed. CRITICAL NOTE: + # These default settings may not properly handle all + # parent/child associations. for tool_dict in metadata_dict[ 'tools' ]: - flush_needed = False tool_id = tool_dict[ 'guid' ] old_tool_id = tool_dict[ 'id' ] - tool_version = tool_dict[ 'version' ] tool_version_using_old_id = tvm.get_tool_version( old_tool_id ) tool_version_using_guid = tvm.get_tool_version( tool_id ) if not tool_version_using_old_id: tool_version_using_old_id = self.app.install_model.ToolVersion( tool_id=old_tool_id, - tool_shed_repository=tool_shed_repository ) + tool_shed_repository=tool_shed_repository ) self.app.install_model.context.add( tool_version_using_old_id ) self.app.install_model.context.flush() if not tool_version_using_guid: tool_version_using_guid = self.app.install_model.ToolVersion( tool_id=tool_id, - tool_shed_repository=tool_shed_repository ) + tool_shed_repository=tool_shed_repository ) self.app.install_model.context.add( tool_version_using_guid ) self.app.install_model.context.flush() # Associate the two versions as parent / child. @@ -640,7 +638,6 @@ ordered_tsr_ids = [] processed_tsr_ids = [] prior_install_required_dict = self.get_prior_install_required_dict( tool_shed_repositories, repository_dependencies_dict ) - tsr_ids = [ tool_shed_repository.id for tool_shed_repository in tool_shed_repositories ] while len( processed_tsr_ids ) != len( prior_install_required_dict.keys() ): tsr_id = suc.get_next_prior_import_or_install_required_dict_entry( prior_install_required_dict, processed_tsr_ids ) processed_tsr_ids.append( tsr_id ) diff -r 000cd07f16f991cb78025e23f0872bafd58b9daf -r 739ca42b855530e8293cbb25298e1d4f6ef03d15 lib/tool_shed/util/shed_util_common.py --- a/lib/tool_shed/util/shed_util_common.py +++ b/lib/tool_shed/util/shed_util_common.py @@ -3,6 +3,7 @@ import os import re import shutil +import socket import string import sqlalchemy.orm.exc @@ -81,6 +82,7 @@ '${host}' """ + def create_or_update_tool_shed_repository( app, name, description, installed_changeset_revision, ctx_rev, repository_clone_url, metadata_dict, status, current_changeset_revision=None, owner='', dist_to_shed=False ): """ @@ -118,8 +120,8 @@ owner, installed_changeset_revision ) if tool_shed_repository: - log.debug( "Updating an existing row for repository '%s' in the tool_shed_repository table, status set to '%s'." % \ - ( str( name ), str( status ) ) ) + log.debug( "Updating an existing row for repository '%s' in the tool_shed_repository table, status set to '%s'." % + ( str( name ), str( status ) ) ) tool_shed_repository.description = description tool_shed_repository.changeset_revision = current_changeset_revision tool_shed_repository.ctx_rev = ctx_rev @@ -129,8 +131,8 @@ tool_shed_repository.uninstalled = uninstalled tool_shed_repository.status = status else: - log.debug( "Adding new row for repository '%s' in the tool_shed_repository table, status set to '%s'." % \ - ( str( name ), str( status ) ) ) + log.debug( "Adding new row for repository '%s' in the tool_shed_repository table, status set to '%s'." % + ( str( name ), str( status ) ) ) tool_shed_repository = \ app.install_model.ToolShedRepository( tool_shed=tool_shed, name=name, @@ -149,6 +151,7 @@ context.flush() return tool_shed_repository + def extract_components_from_tuple( repository_components_tuple ): '''Extract the repository components from the provided tuple in a backward-compatible manner.''' toolshed = repository_components_tuple[ 0 ] @@ -164,6 +167,7 @@ components_list = [ toolshed, name, owner, changeset_revision, prior_installation_required, only_if_compiling_contained_td ] return components_list + def generate_sharable_link_for_repository_in_tool_shed( repository, changeset_revision=None ): """Generate the URL for sharing a repository that is in the tool shed.""" base_url = url_for( '/', qualified=True ).rstrip( '/' ) @@ -173,6 +177,7 @@ sharable_url += '/%s' % changeset_revision return sharable_url + def generate_tool_guid( repository_clone_url, tool ): """ Generate a guid for the installed tool. It is critical that this guid matches the guid for @@ -182,6 +187,7 @@ tmp_url = common_util.remove_protocol_and_user_from_clone_url( repository_clone_url ) return '%s/%s/%s' % ( tmp_url, tool.id, tool.version ) + def generate_tool_shed_repository_install_dir( repository_clone_url, changeset_revision ): """ Generate a repository installation directory that guarantees repositories with the same @@ -196,6 +202,7 @@ tool_shed_url = common_util.remove_port_from_tool_shed_url( tool_shed_url ) return common_util.url_join( tool_shed_url, 'repos', repo_path, changeset_revision ) + def get_absolute_path_to_file_in_repository( repo_files_dir, file_name ): """Return the absolute path to a specified disk file contained in a repository.""" stripped_file_name = basic_util.strip_path( file_name ) @@ -207,19 +214,22 @@ return os.path.abspath( os.path.join( root, name ) ) return file_path + def get_categories( app ): """Get all categories from the database.""" sa_session = app.model.context.current return sa_session.query( app.model.Category ) \ - .filter( app.model.Category.table.c.deleted==False ) \ + .filter( app.model.Category.table.c.deleted == False ) \ .order_by( app.model.Category.table.c.name ) \ .all() + def get_category( app, id ): """Get a category from the database.""" sa_session = app.model.context.current return sa_session.query( app.model.Category ).get( app.security.decode_id( id ) ) + def get_category_by_name( app, name ): """Get a category from the database via name.""" sa_session = app.model.context.current @@ -228,6 +238,7 @@ except sqlalchemy.orm.exc.NoResultFound: return None + def get_ctx_rev( app, tool_shed_url, name, owner, changeset_revision ): """ Send a request to the tool shed to retrieve the ctx_rev for a repository defined by the @@ -240,6 +251,7 @@ ctx_rev = common_util.tool_shed_get( app, tool_shed_url, url ) return ctx_rev + def get_current_repository_metadata_for_changeset_revision( app, repository, changeset_revision ): encoded_repository_id = app.security.encode_id( repository.id ) repository_metadata = get_repository_metadata_by_changeset_revision( app, @@ -261,6 +273,7 @@ return repository_metadata return None + def get_ids_of_tool_shed_repositories_being_installed( app, as_string=False ): installing_repository_ids = [] new_status = app.install_model.ToolShedRepository.installation_status.NEW @@ -280,6 +293,7 @@ return ','.join( installing_repository_ids ) return installing_repository_ids + def get_latest_downloadable_changeset_revision( app, repository, repo ): repository_tip = repository.tip( app ) repository_metadata = get_repository_metadata_by_changeset_revision( app, app.security.encode_id( repository.id ), repository_tip ) @@ -290,6 +304,7 @@ return changeset_revisions[ -1 ] return hg_util.INITIAL_CHANGELOG_HASH + def get_next_downloadable_changeset_revision( repository, repo, after_changeset_revision ): """ Return the installable changeset_revision in the repository changelog after the changeset to which @@ -311,6 +326,7 @@ found_after_changeset_revision = True return None + def get_next_prior_import_or_install_required_dict_entry( prior_required_dict, processed_tsr_ids ): """ This method is used in the Tool Shed when exporting a repository and its dependencies, and in Galaxy @@ -344,6 +360,7 @@ continue return key + def get_ordered_metadata_changeset_revisions( repository, repo, downloadable=True ): """ Return an ordered list of changeset_revisions that are associated with metadata @@ -366,6 +383,7 @@ sorted_changeset_revisions = [ str( changeset_tup[ 1 ] ) for changeset_tup in sorted_changeset_tups ] return sorted_changeset_revisions + def get_prior_import_or_install_required_dict( app, tsr_ids, repo_info_dicts ): """ This method is used in the Tool Shed when exporting a repository and its dependencies, @@ -390,6 +408,7 @@ prior_import_or_install_required_dict[ encoded_repository_id ] = prior_import_or_install_ids return prior_import_or_install_required_dict + def get_repo_info_tuple_contents( repo_info_tuple ): """Take care in handling the repo_info_tuple as it evolves over time as new tool shed features are introduced.""" if len( repo_info_tuple ) == 6: @@ -399,6 +418,7 @@ description, repository_clone_url, changeset_revision, ctx_rev, repository_owner, repository_dependencies, tool_dependencies = repo_info_tuple return description, repository_clone_url, changeset_revision, ctx_rev, repository_owner, repository_dependencies, tool_dependencies + def get_repository_and_repository_dependencies_from_repo_info_dict( app, repo_info_dict ): """Return a tool_shed_repository or repository record defined by the information in the received repo_info_dict.""" repository_name = repo_info_dict.keys()[ 0 ] @@ -414,6 +434,7 @@ repository = get_repository_by_name_and_owner( app, repository_name, repository_owner ) return repository, repository_dependencies + def get_repository_by_id( app, id ): """Get a repository from the database via id.""" if is_tool_shed_client( app ): @@ -422,28 +443,31 @@ sa_session = app.model.context.current return sa_session.query( app.model.Repository ).get( app.security.decode_id( id ) ) + def get_repository_by_name( app, name ): """Get a repository from the database via name.""" repository_query = get_repository_query( app ) return repository_query.filter_by( name=name ).first() + def get_repository_by_name_and_owner( app, name, owner ): """Get a repository from the database via name and owner""" repository_query = get_repository_query( app ) if is_tool_shed_client( app ): return repository_query \ - .filter( and_( app.install_model.ToolShedRepository.table.c.name == name, - app.install_model.ToolShedRepository.table.c.owner == owner ) ) \ - .first() + .filter( and_( app.install_model.ToolShedRepository.table.c.name == name, + app.install_model.ToolShedRepository.table.c.owner == owner ) ) \ + .first() # We're in the tool shed. user = get_user_by_username( app, owner ) if user: return repository_query \ - .filter( and_( app.model.Repository.table.c.name == name, - app.model.Repository.table.c.user_id == user.id ) ) \ - .first() + .filter( and_( app.model.Repository.table.c.name == name, + app.model.Repository.table.c.user_id == user.id ) ) \ + .first() return None + def get_repository_dependency_types( repository_dependencies ): """ Inspect the received list of repository_dependencies tuples and return boolean values @@ -470,6 +494,7 @@ break return has_repository_dependencies, has_repository_dependencies_only_if_compiling_contained_td + def get_repository_for_dependency_relationship( app, tool_shed, name, owner, changeset_revision ): """ Return an installed tool_shed_repository database record that is defined by either the current changeset @@ -509,6 +534,7 @@ changeset_revision=text ) return repository + def get_repository_file_contents( file_path ): """Return the display-safe contents of a repository file for display in a browser.""" if checkers.is_gzip( file_path ): @@ -543,6 +569,7 @@ beginning_on_size_error=True ) return safe_str + def get_repository_files( folder_path ): """Return the file hierarchy of a tool shed repository.""" contents = [] @@ -558,6 +585,7 @@ contents.sort() return contents + def get_repository_from_refresh_on_change( app, **kwd ): # The changeset_revision_select_field in several grids performs a refresh_on_change which sends in request parameters like # changeset_revison_1, changeset_revision_2, etc. One of the many select fields on the grid performed the refresh_on_change, @@ -575,6 +603,7 @@ # This should never be reached - raise an exception? return v, None + def get_repository_ids_requiring_prior_import_or_install( app, tsr_ids, repository_dependencies ): """ This method is used in the Tool Shed when exporting a repository and its dependencies, @@ -593,11 +622,11 @@ continue for rd_tup in rd_tups: tool_shed, \ - name, \ - owner, \ - changeset_revision, \ - prior_installation_required, \ - only_if_compiling_contained_td = \ + name, \ + owner, \ + changeset_revision, \ + prior_installation_required, \ + only_if_compiling_contained_td = \ common_util.parse_repository_dependency_tuple( rd_tup ) # If only_if_compiling_contained_td is False, then the repository dependency # is not required to be installed prior to the dependent repository even if @@ -626,11 +655,13 @@ prior_tsr_ids.append( encoded_repository_id ) return prior_tsr_ids + def get_repository_in_tool_shed( app, id ): """Get a repository on the tool shed side from the database via id.""" sa_session = app.model.context.current return sa_session.query( app.model.Repository ).get( app.security.decode_id( id ) ) + def get_repository_metadata_by_changeset_revision( app, id, changeset_revision ): """Get metadata for a specified repository change set from the database.""" # Make sure there are no duplicate records, and return the single unique record for the changeset_revision. @@ -652,6 +683,7 @@ return all_metadata_records[ 0 ] return None + def get_repository_owner( cleaned_repository_url ): """Gvien a "cleaned" repository clone URL, return the owner of the repository.""" items = cleaned_repository_url.split( '/repos/' ) @@ -660,12 +692,13 @@ repo_path = repo_path.replace( '/', '', 1 ) return repo_path.lstrip( '/' ).split( '/' )[ 0 ] + def get_repository_owner_from_clone_url( repository_clone_url ): """Given a repository clone URL, return the owner of the repository.""" tmp_url = common_util.remove_protocol_and_user_from_clone_url( repository_clone_url ) - tool_shed = tmp_url.split( '/repos/' )[ 0 ].rstrip( '/' ) return get_repository_owner( tmp_url ) + def get_repository_query( app ): if is_tool_shed_client( app ): query = app.install_model.context.query( app.install_model.ToolShedRepository ) @@ -673,6 +706,7 @@ query = app.model.context.query( app.model.Repository ) return query + def get_tool_panel_config_tool_path_install_dir( app, repository ): """ Return shed-related tool panel config, the tool_path configured in it, and the relative path to @@ -690,12 +724,13 @@ # Just pick a semi-random shed config. for shed_config_dict in app.toolbox.shed_tool_confs: if ( repository.dist_to_shed and shed_config_dict[ 'config_filename' ] == app.config.migrated_tools_config ) \ - or ( not repository.dist_to_shed and shed_config_dict[ 'config_filename' ] != app.config.migrated_tools_config ): + or ( not repository.dist_to_shed and shed_config_dict[ 'config_filename' ] != app.config.migrated_tools_config ): break shed_tool_conf = shed_config_dict[ 'config_filename' ] tool_path = shed_config_dict[ 'tool_path' ] return shed_tool_conf, tool_path, relative_install_dir + def get_tool_path_by_shed_tool_conf_filename( app, shed_tool_conf ): """ Return the tool_path config setting for the received shed_tool_conf file by searching the tool box's in-memory list of shed_tool_confs for the @@ -711,10 +746,12 @@ return shed_tool_conf_dict[ 'tool_path' ] return None + def get_tool_shed_from_clone_url( repository_clone_url ): tmp_url = common_util.remove_protocol_and_user_from_clone_url( repository_clone_url ) return tmp_url.split( '/repos/' )[ 0 ].rstrip( '/' ) + def get_tool_shed_repository_by_id( app, repository_id ): """Return a tool shed repository database record defined by the id.""" # This method is used only in Galaxy, not the tool shed. @@ -722,6 +759,7 @@ .filter( app.install_model.ToolShedRepository.table.c.id == app.security.decode_id( repository_id ) ) \ .first() + def get_tool_shed_repository_by_shed_name_owner_changeset_revision( app, tool_shed, name, owner, changeset_revision ): """ Return a tool shed repository database record defined by the combination of a tool_shed, repository name, @@ -733,11 +771,12 @@ tool_shed = common_util.remove_protocol_from_tool_shed_url( tool_shed ) return repository_query \ .filter( and_( app.install_model.ToolShedRepository.table.c.tool_shed == tool_shed, - app.install_model.ToolShedRepository.table.c.name == name, - app.install_model.ToolShedRepository.table.c.owner == owner, - app.install_model.ToolShedRepository.table.c.changeset_revision == changeset_revision ) ) \ + app.install_model.ToolShedRepository.table.c.name == name, + app.install_model.ToolShedRepository.table.c.owner == owner, + app.install_model.ToolShedRepository.table.c.changeset_revision == changeset_revision ) ) \ .first() + def get_tool_shed_repository_by_shed_name_owner_installed_changeset_revision( app, tool_shed, name, owner, installed_changeset_revision ): """ @@ -750,11 +789,12 @@ tool_shed = common_util.remove_protocol_from_tool_shed_url( tool_shed ) return repository_query \ .filter( and_( app.install_model.ToolShedRepository.table.c.tool_shed == tool_shed, - app.install_model.ToolShedRepository.table.c.name == name, - app.install_model.ToolShedRepository.table.c.owner == owner, - app.install_model.ToolShedRepository.table.c.installed_changeset_revision == installed_changeset_revision ) ) \ + app.install_model.ToolShedRepository.table.c.name == name, + app.install_model.ToolShedRepository.table.c.owner == owner, + app.install_model.ToolShedRepository.table.c.installed_changeset_revision == installed_changeset_revision ) ) \ .first() + def get_tool_shed_status_for_installed_repository( app, repository ): """ Send a request to the tool shed to retrieve information about newer installable repository revisions, @@ -775,8 +815,8 @@ except HTTPError, e: # This should handle backward compatility to the Galaxy 12/20/12 release. We used to only handle updates for an installed revision # using a boolean value. - log.debug( "Error attempting to get tool shed status for installed repository %s: %s\nAttempting older 'check_for_updates' method.\n" % \ - ( str( repository.name ), str( e ) ) ) + log.debug( "Error attempting to get tool shed status for installed repository %s: %s\nAttempting older 'check_for_updates' method.\n" % + ( str( repository.name ), str( e ) ) ) params = '?name=%s&owner=%s&changeset_revision=%s&from_update_manager=True' % ( str( repository.name ), str( repository.owner ), str( repository.changeset_revision ) ) @@ -793,6 +833,7 @@ log.exception( "Error attempting to get tool shed status for installed repository %s: %s" % ( str( repository.name ), str( e ) ) ) return {} + def get_tool_shed_repository_status_label( app, tool_shed_repository=None, name=None, owner=None, changeset_revision=None, repository_clone_url=None ): """Return a color-coded label for the status of the received tool-shed_repository installed into Galaxy.""" if tool_shed_repository is None: @@ -842,6 +883,7 @@ status_label = 'unknown status' return '<div class="count-box state-color-%s">%s</div>' % ( bgcolor, status_label ) + def get_updated_changeset_revisions( app, name, owner, changeset_revision ): """ Return a string of comma-separated changeset revision hashes for all available updates to the received changeset @@ -863,6 +905,7 @@ return changeset_hashes_str return '' + def get_updated_changeset_revisions_from_tool_shed( app, tool_shed_url, name, owner, changeset_revision ): """ Get all appropriate newer changeset revisions for the repository defined by @@ -880,11 +923,13 @@ text = common_util.tool_shed_get( app, tool_shed_url, url ) return text + def get_user( app, id ): """Get a user from the database by id.""" sa_session = app.model.context.current return sa_session.query( app.model.User ).get( app.security.decode_id( id ) ) + def get_user_by_username( app, username ): """Get a user from the database by username.""" sa_session = app.model.context.current @@ -893,9 +938,10 @@ .filter( app.model.User.table.c.username == username ) \ .one() return user - except Exception, e: + except Exception: return None + def handle_email_alerts( app, host, repository, content_alert_str='', new_repo_alert=False, admin_only=False ): """ There are 2 complementary features that enable a tool shed user to receive email notification: @@ -947,7 +993,7 @@ admin_body = string.Template( template ).safe_substitute( host=host, sharable_link=sharable_link, repository_name=repository.name, - revision='%s:%s' %( str( ctx.rev() ), ctx ), + revision='%s:%s' % ( str( ctx.rev() ), ctx ), display_date=display_date, description=ctx.description(), username=username, @@ -955,7 +1001,7 @@ body = string.Template( template ).safe_substitute( host=host, sharable_link=sharable_link, repository_name=repository.name, - revision='%s:%s' %( str( ctx.rev() ), ctx ), + revision='%s:%s' % ( str( ctx.rev() ), ctx ), display_date=display_date, description=ctx.description(), username=username, @@ -985,9 +1031,10 @@ util.send_mail( frm, to, subject, admin_body, app.config ) else: util.send_mail( frm, to, subject, body, app.config ) - except Exception, e: + except Exception: log.exception( "An error occurred sending a tool shed repository update alert by email." ) + def have_shed_tool_conf_for_install( app ): if not app.toolbox.shed_tool_confs: return False @@ -999,6 +1046,7 @@ return True return False + def is_tool_shed_client( app ): """ The tool shed and clients to the tool (i.e. Galaxy) require a lot @@ -1008,6 +1056,7 @@ """ return hasattr( app, "install_model" ) + def open_repository_files_folder( folder_path ): """ Return a list of dictionaries, each of which contains information for a file or directory contained @@ -1026,14 +1075,15 @@ is_folder = True if filename: full_path = os.path.join( folder_path, filename ) - node = { "title" : filename, - "isFolder" : is_folder, - "isLazy" : is_folder, - "tooltip" : full_path, - "key" : full_path } + node = { "title": filename, + "isFolder": is_folder, + "isLazy": is_folder, + "tooltip": full_path, + "key": full_path } folder_contents.append( node ) return folder_contents + def repository_was_previously_installed( app, tool_shed_url, repository_name, repo_info_tuple, from_tip=False ): """ Find out if a repository is already installed into Galaxy - there are several scenarios where this @@ -1077,6 +1127,7 @@ return tool_shed_repository, previous_changeset_revision return None, None + def set_image_paths( app, encoded_repository_id, text ): """ Handle tool help image display for tools that are contained in repositories in @@ -1106,6 +1157,7 @@ text = re.sub( r'\.\. image:: (?!https?://)/?(.+)', r'.. image:: %s/\1' % route_to_images, text ) return text + def set_repository_attributes( app, repository, status, error_message, deleted, uninstalled, remove_from_disk=False ): if remove_from_disk: relative_install_dir = repository.repo_path( app ) @@ -1123,6 +1175,7 @@ app.install_model.context.add( repository ) app.install_model.context.flush() + def tool_shed_is_this_tool_shed( toolshed_base_url ): """Determine if a tool shed is the current tool shed.""" cleaned_toolshed_base_url = common_util.remove_protocol_from_tool_shed_url( toolshed_base_url ) diff -r 000cd07f16f991cb78025e23f0872bafd58b9daf -r 739ca42b855530e8293cbb25298e1d4f6ef03d15 lib/tool_shed/util/tool_util.py --- a/lib/tool_shed/util/tool_util.py +++ b/lib/tool_shed/util/tool_util.py @@ -5,15 +5,14 @@ import galaxy.tools from galaxy import util from galaxy.datatypes import checkers -from galaxy.tools import parameters from galaxy.util.expressions import ExpressionContext from galaxy.web.form_builder import SelectField -from galaxy.tools.actions.upload import UploadToolAction from tool_shed.util import basic_util log = logging.getLogger( __name__ ) + def build_shed_tool_conf_select_field( app ): """Build a SelectField whose options are the keys in app.toolbox.shed_tool_confs.""" options = [] @@ -30,6 +29,7 @@ select_field.add_option( option_tup[ 0 ], option_tup[ 1 ] ) return select_field + def build_tool_panel_section_select_field( app ): """Build a SelectField whose options are the sections of the current in-memory toolbox.""" options = [] @@ -41,6 +41,7 @@ select_field.add_option( option_tup[ 0 ], option_tup[ 1 ] ) return select_field + def copy_sample_file( app, filename, dest_path=None ): """ Copy xxx.sample to dest_path/xxx.sample and dest_path/xxx. The default value for dest_path @@ -61,6 +62,7 @@ if not os.path.exists( os.path.join( dest_path, copied_file ) ): shutil.copy( full_source_path, os.path.join( dest_path, copied_file ) ) + def copy_sample_files( app, sample_files, tool_path=None, sample_files_copied=None, dest_path=None ): """ Copy all appropriate files to dest_path in the local Galaxy environment that have not @@ -75,11 +77,12 @@ filename_sans_path = os.path.split( filename )[ 1 ] if filename_sans_path not in filenames_not_to_copy and filename not in sample_files_copied: if tool_path: - filename=os.path.join( tool_path, filename ) + filename = os.path.join( tool_path, filename ) # Attempt to ensure we're copying an appropriate file. if is_data_index_sample_file( filename ): copy_sample_file( app, filename, dest_path=dest_path ) + def generate_message_for_invalid_tools( app, invalid_file_tups, repository, metadata_dict, as_html=True, displaying_invalid_tool=False ): if as_html: @@ -124,6 +127,7 @@ message += "%s%s%s - %s%s" % ( bold_start, tool_file, bold_end, correction_msg, new_line ) return message + def get_headers( fname, sep, count=60, is_multi_byte=False ): """Returns a list with the first 'count' lines split by 'sep'.""" headers = [] @@ -137,6 +141,7 @@ break return headers + def get_tool_path_install_dir( partial_install_dir, shed_tool_conf_dict, tool_dict, config_elems ): for elem in config_elems: if elem.tag == 'tool': @@ -153,6 +158,7 @@ return tool_path, relative_install_dir return None, None + def handle_missing_index_file( app, tool_path, sample_files, repository_tools_tups, sample_files_copied ): """ Inspect each tool to see if it has any input parameters that are dynamically @@ -180,6 +186,7 @@ repository_tools_tups[ index ] = ( tup_path, guid, repository_tool ) return repository_tools_tups, sample_files_copied + def is_column_based( fname, sep='\t', skip=0, is_multi_byte=False ): """See if the file is column based with respect to a separator.""" headers = get_headers( fname, sep, is_multi_byte=is_multi_byte ) @@ -199,6 +206,7 @@ return False return True + def is_data_index_sample_file( file_path ): """ Attempt to determine if a .sample file is appropriate for copying to ~/tool-data when @@ -224,6 +232,7 @@ # Default to copying the file if none of the above are true. return True + def new_state( trans, tool, invalid=False ): """Create a new `DefaultToolState` for the received tool. Only inputs on the first page will be initialized.""" state = galaxy.tools.DefaultToolState() @@ -240,9 +249,10 @@ state.inputs[ input.name ] = [] return state + def panel_entry_per_tool( tool_section_dict ): # Return True if tool_section_dict looks like this. - # {<Tool guid> : + # {<Tool guid> : # [{ tool_config : <tool_config_file>, # id: <ToolSection id>, # version : <ToolSection version>, @@ -258,6 +268,7 @@ return True return False + def reload_upload_tools( app ): if hasattr( app, 'toolbox' ): app.toolbox.handle_datatypes_changed() https://bitbucket.org/galaxy/galaxy-central/commits/35d456f7edff/ Changeset: 35d456f7edff User: jmchilton Date: 2014-12-29 14:11:54+00:00 Summary: Method to abstract away ToolBox.shed_tool_confs from tool_util.py. Start a gradual process of referring to these as dynamic tool confs instead of shed tool confs. I would like like to make them more generally Galaxy-managed tool configurations and distinguish them from deployer-managed configurations - rather then having overtly ToolShed-only logic in ToolBox. Affected #: 2 files diff -r 739ca42b855530e8293cbb25298e1d4f6ef03d15 -r 35d456f7edffef9a8e4cd825503c3782f2be3617 lib/galaxy/tools/__init__.py --- a/lib/galaxy/tools/__init__.py +++ b/lib/galaxy/tools/__init__.py @@ -632,6 +632,16 @@ break return tool_version_select_field, tools, tool + def dynamic_conf_filenames( self, include_migrated_tool_conf=False ): + """ Return list of dynamic tool configuration filenames (shed_tools). + These must be used with various dynamic tool configuration update + operations (e.g. with update_shed_config). + """ + for dynamic_tool_conf_dict in self.shed_tool_confs: + dynamic_tool_conf_filename = dynamic_tool_conf_dict[ 'config_filename' ] + if include_migrated_tool_conf or (dynamic_tool_conf_filename != self.app.config.migrated_tools_config): + yield dynamic_tool_conf_filename + def build_tool_version_select_field( self, tools, tool_id, set_selected ): """Build a SelectField whose options are the ids for the received list of tools.""" options = [] diff -r 739ca42b855530e8293cbb25298e1d4f6ef03d15 -r 35d456f7edffef9a8e4cd825503c3782f2be3617 lib/tool_shed/util/tool_util.py --- a/lib/tool_shed/util/tool_util.py +++ b/lib/tool_shed/util/tool_util.py @@ -16,14 +16,12 @@ def build_shed_tool_conf_select_field( app ): """Build a SelectField whose options are the keys in app.toolbox.shed_tool_confs.""" options = [] - for shed_tool_conf_dict in app.toolbox.shed_tool_confs: - shed_tool_conf_filename = shed_tool_conf_dict[ 'config_filename' ] - if shed_tool_conf_filename != app.config.migrated_tools_config: - if shed_tool_conf_filename.startswith( './' ): - option_label = shed_tool_conf_filename.replace( './', '', 1 ) - else: - option_label = shed_tool_conf_filename - options.append( ( option_label, shed_tool_conf_filename ) ) + for dynamic_tool_conf_filename in app.toolbox.dynamic_conf_filenames(): + if dynamic_tool_conf_filename.startswith( './' ): + option_label = dynamic_tool_conf_filename.replace( './', '', 1 ) + else: + option_label = dynamic_tool_conf_filename + options.append( ( option_label, dynamic_tool_conf_filename ) ) select_field = SelectField( name='shed_tool_conf' ) for option_tup in options: select_field.add_option( option_tup[ 0 ], option_tup[ 1 ] ) https://bitbucket.org/galaxy/galaxy-central/commits/83dd43611918/ Changeset: 83dd43611918 User: jmchilton Date: 2014-12-29 14:11:54+00:00 Summary: Method to abstract away ToolBox.tool_panel from tool_util.py. Now only thing accessing ToolBox.tool_panel outside ToolBox itself is ToolPanelManager. Affected #: 2 files diff -r 35d456f7edffef9a8e4cd825503c3782f2be3617 -r 83dd436119182a688824218b0efda74e55e045c9 lib/galaxy/tools/__init__.py --- a/lib/galaxy/tools/__init__.py +++ b/lib/galaxy/tools/__init__.py @@ -1059,6 +1059,11 @@ status = 'done' return message, status + def get_sections( self ): + for k, v in self.tool_panel.items(): + if isinstance( v, ToolSection ): + yield (v.id, v.name) + def find_section_id( self, tool_panel_section_id ): """ Find the section ID referenced by the key or return '' indicating diff -r 35d456f7edffef9a8e4cd825503c3782f2be3617 -r 83dd436119182a688824218b0efda74e55e045c9 lib/tool_shed/util/tool_util.py --- a/lib/tool_shed/util/tool_util.py +++ b/lib/tool_shed/util/tool_util.py @@ -31,9 +31,8 @@ def build_tool_panel_section_select_field( app ): """Build a SelectField whose options are the sections of the current in-memory toolbox.""" options = [] - for k, v in app.toolbox.tool_panel.items(): - if isinstance( v, galaxy.tools.ToolSection ): - options.append( ( v.name, v.id ) ) + for section_id, section_name in app.toolbox.get_sections(): + options.append( ( section_name, section_id ) ) select_field = SelectField( name='tool_panel_section_id', display='radio' ) for option_tup in options: select_field.add_option( option_tup[ 0 ], option_tup[ 1 ] ) https://bitbucket.org/galaxy/galaxy-central/commits/6021c963ec92/ Changeset: 6021c963ec92 User: jmchilton Date: 2014-12-29 14:11:54+00:00 Summary: Reuse new ToolBox.get_section method in ToolPanelManager. This is a higher-level method and eliminates a couple more places where Tool Shed code is directly acceessing ToolBox.tool_panel. Affected #: 1 file diff -r 83dd436119182a688824218b0efda74e55e045c9 -r 6021c963ec925698353b134d070b403d2309609f lib/tool_shed/galaxy_install/tools/tool_panel_manager.py --- a/lib/tool_shed/galaxy_install/tools/tool_panel_manager.py +++ b/lib/tool_shed/galaxy_install/tools/tool_panel_manager.py @@ -325,8 +325,7 @@ tool_panel_section_id=section_id, new_tool_panel_section_label=new_tool_panel_section_label ) elif tool_panel_section_id: - tool_panel_section_key = str( tool_panel_section_id ) - tool_section = toolbox.tool_panel[ tool_panel_section_key ] + tool_panel_section_key, tool_section = toolbox.get_section( tool_panel_section_id) else: return None, None return tool_panel_section_key, tool_section @@ -495,8 +494,8 @@ # Remove the tool sub-element from the section element. config_elem.remove( tool_elem ) # Remove the tool from the section in the in-memory tool panel. - if section_key in toolbox.tool_panel: - tool_section = toolbox.tool_panel[ section_key ] + _, tool_section = toolbox.get_section( section_key ) + if tool_section: remove_from_panel( tool_elem, tool_section.elems, toolbox.integrated_tool_panel.get( section_key, {} ), section_key=section_key ) if len( config_elem ) < 1: # Keep a list of all empty section elements so they can be removed. https://bitbucket.org/galaxy/galaxy-central/commits/482b027099f5/ Changeset: 482b027099f5 User: jmchilton Date: 2014-12-29 14:11:54+00:00 Summary: Refactor logic for removing tools from panels into ToolBox. May seem sort of like Tool Shed specific stuff - but I think it can be reused elsewhere down the road. Regardless of this however as of this commit no external components directly access ToolBox.tool_panel or ToolBox.integrated_tool_panel - which I think is important - these should be internal details of the ToolBox (or newer modules it leverages). Affected #: 2 files diff -r 6021c963ec925698353b134d070b403d2309609f -r 482b027099f5aa0e15de928d093c6347a5297e18 lib/galaxy/tools/__init__.py --- a/lib/galaxy/tools/__init__.py +++ b/lib/galaxy/tools/__init__.py @@ -658,6 +658,55 @@ select_field.add_option( 'version %s' % option_tup[ 0 ], option_tup[ 1 ] ) return select_field + def remove_from_panel( self, tool_id, section_key='', remove_from_config=True ): + + def remove_from_dict( has_elems, integrated_has_elems ): + tool_key = 'tool_%s' % str( tool_id ) + available_tool_versions = self.get_loaded_tools_by_lineage( tool_id ) + if tool_key in has_elems: + if available_tool_versions: + available_tool_versions.reverse() + replacement_tool_key = None + replacement_tool_version = None + # Since we are going to remove the tool from the section, replace it with + # the newest loaded version of the tool. + for available_tool_version in available_tool_versions: + available_tool_section_id, available_tool_section_name = available_tool_version.get_panel_section() + # I suspect "available_tool_version.id in has_elems.keys()" doesn't + # belong in the following line or at least I don't understand what + # purpose it might serve. -John + if available_tool_version.id in has_elems.keys() or (available_tool_section_id == section_key): + replacement_tool_key = 'tool_%s' % str( available_tool_version.id ) + replacement_tool_version = available_tool_version + break + if replacement_tool_key and replacement_tool_version: + # Get the index of the tool_key in the tool_section. + for tool_panel_index, key in enumerate( has_elems.keys() ): + if key == tool_key: + break + # Remove the tool from the tool panel. + del has_elems[ tool_key ] + # Add the replacement tool at the same location in the tool panel. + has_elems.insert( tool_panel_index, + replacement_tool_key, + replacement_tool_version ) + else: + del has_elems[ tool_key ] + else: + del has_elems[ tool_key ] + if remove_from_config: + if hasattr( integrated_has_elems, "elems" ): + integrated_has_elems = integrated_has_elems.elems + if tool_key in integrated_has_elems: + del integrated_has_elems[ tool_key ] + + if section_key: + _, tool_section = self.get_section( section_key ) + if tool_section: + remove_from_dict( tool_section.elems, self.integrated_tool_panel.get( section_key, {} ) ) + else: + remove_from_dict( self.tool_panel, self.integrated_tool_panel ) + def load_tool_tag_set( self, elem, panel_dict, integrated_panel_dict, tool_path, load_panel_dict, guid=None, index=None ): try: path = elem.get( "file" ) diff -r 6021c963ec925698353b134d070b403d2309609f -r 482b027099f5aa0e15de928d093c6347a5297e18 lib/tool_shed/galaxy_install/tools/tool_panel_manager.py --- a/lib/tool_shed/galaxy_install/tools/tool_panel_manager.py +++ b/lib/tool_shed/galaxy_install/tools/tool_panel_manager.py @@ -426,50 +426,6 @@ def remove_guids( self, guids_to_remove, shed_tool_conf, uninstall ): toolbox = self.app.toolbox - - def remove_from_panel( tool_elem, has_elems, integrated_has_elems, section_key='' ): - # Hide tool from panel and promote next oldest version if - # available. - guid = tool_elem.get( 'guid' ) - tool_key = 'tool_%s' % str( guid ) - available_tool_versions = toolbox.get_loaded_tools_by_lineage( guid ) - if tool_key in has_elems: - if available_tool_versions: - available_tool_versions.reverse() - replacement_tool_key = None - replacement_tool_version = None - # Since we are going to remove the tool from the section, replace it with - # the newest loaded version of the tool. - for available_tool_version in available_tool_versions: - available_tool_section_id, available_tool_section_name = available_tool_version.get_panel_section() - # I suspect "available_tool_version.id in has_elems.keys()" doesn't - # belong in the following line or at least I don't understand what - # purpose it might serve. -John - if available_tool_version.id in has_elems.keys() or (available_tool_section_id == section_key): - replacement_tool_key = 'tool_%s' % str( available_tool_version.id ) - replacement_tool_version = available_tool_version - break - if replacement_tool_key and replacement_tool_version: - # Get the index of the tool_key in the tool_section. - for tool_panel_index, key in enumerate( has_elems.keys() ): - if key == tool_key: - break - # Remove the tool from the tool panel. - del has_elems[ tool_key ] - # Add the replacement tool at the same location in the tool panel. - has_elems.insert( tool_panel_index, - replacement_tool_key, - replacement_tool_version ) - else: - del has_elems[ tool_key ] - else: - del has_elems[ tool_key ] - if uninstall: - if hasattr( integrated_has_elems, "elems" ): - integrated_has_elems = integrated_has_elems.elems - if tool_key in integrated_has_elems: - del integrated_has_elems[ tool_key ] - # Remove the tools from the toolbox's tools_by_id dictionary. for guid_to_remove in guids_to_remove: # remove_from_tool_panel to false, will handling that logic below. @@ -494,17 +450,14 @@ # Remove the tool sub-element from the section element. config_elem.remove( tool_elem ) # Remove the tool from the section in the in-memory tool panel. - _, tool_section = toolbox.get_section( section_key ) - if tool_section: - remove_from_panel( tool_elem, tool_section.elems, toolbox.integrated_tool_panel.get( section_key, {} ), section_key=section_key ) + toolbox.remove_from_panel( tool_elem.get( "guid" ), section_key=section_key, remove_from_config=uninstall ) if len( config_elem ) < 1: # Keep a list of all empty section elements so they can be removed. config_elems_to_remove.append( config_elem ) elif config_elem.tag == 'tool': guid = config_elem.get( 'guid' ) if guid in guids_to_remove: - # get_panel_section return '' for un-sectioned tools. - remove_from_panel( config_elem, toolbox.tool_panel, toolbox.integrated_tool_panel, section_key='' ) + toolbox.remove_from_panel( guid, section_key='', remove_from_config=uninstall ) config_elems_to_remove.append( config_elem ) for config_elem in config_elems_to_remove: # Remove the element from the in-memory list of elements. https://bitbucket.org/galaxy/galaxy-central/commits/600d39d9bbac/ Changeset: 600d39d9bbac User: jmchilton Date: 2014-12-29 14:11:54+00:00 Summary: Update calls to ToolPanelManager.get_shed_tool_conf_dict for clarity... ... in other words to note where index in the resulting tuple is not actually used. Doing this to enable hiding this implementation detail (index) from components in subsequent commits. Affected #: 3 files diff -r 482b027099f5aa0e15de928d093c6347a5297e18 -r 600d39d9bbac52eb4321e65d16d1740a3f7c0871 lib/tool_shed/galaxy_install/install_manager.py --- a/lib/tool_shed/galaxy_install/install_manager.py +++ b/lib/tool_shed/galaxy_install/install_manager.py @@ -730,7 +730,7 @@ shed_tool_conf = install_options.get( 'shed_tool_conf', None ) if shed_tool_conf: # Get the tool_path setting. - index, shed_conf_dict = self.tpm.get_shed_tool_conf_dict( shed_tool_conf ) + _, shed_conf_dict = self.tpm.get_shed_tool_conf_dict( shed_tool_conf ) tool_path = shed_conf_dict[ 'tool_path' ] else: # Pick a semi-random shed-related tool panel configuration file and get the tool_path setting. diff -r 482b027099f5aa0e15de928d093c6347a5297e18 -r 600d39d9bbac52eb4321e65d16d1740a3f7c0871 lib/tool_shed/galaxy_install/metadata/installed_repository_metadata_manager.py --- a/lib/tool_shed/galaxy_install/metadata/installed_repository_metadata_manager.py +++ b/lib/tool_shed/galaxy_install/metadata/installed_repository_metadata_manager.py @@ -66,7 +66,7 @@ in the received tool shed repository metadata. """ repository_tools_tups = [] - index, shed_conf_dict = self.tpm.get_shed_tool_conf_dict( self.metadata_dict.get( 'shed_config_filename' ) ) + _, shed_conf_dict = self.tpm.get_shed_tool_conf_dict( self.metadata_dict.get( 'shed_config_filename' ) ) if 'tools' in self.metadata_dict: for tool_dict in self.metadata_dict[ 'tools' ]: load_relative_path = relative_path = tool_dict.get( 'tool_config', None ) diff -r 482b027099f5aa0e15de928d093c6347a5297e18 -r 600d39d9bbac52eb4321e65d16d1740a3f7c0871 lib/tool_shed/galaxy_install/tool_migration_manager.py --- a/lib/tool_shed/galaxy_install/tool_migration_manager.py +++ b/lib/tool_shed/galaxy_install/tool_migration_manager.py @@ -78,7 +78,7 @@ self.tool_shed_url = common_util.get_tool_shed_url_from_tool_shed_registry( self.app, defined_tool_shed_url ) self.tool_shed = common_util.remove_protocol_and_port_from_tool_shed_url( self.tool_shed_url ) self.repository_owner = common_util.REPOSITORY_OWNER - index, self.shed_config_dict = self.tpm.get_shed_tool_conf_dict( self.migrated_tools_config ) + _, self.shed_config_dict = self.tpm.get_shed_tool_conf_dict( self.migrated_tools_config ) # Since tool migration scripts can be executed any number of times, we need to # make sure the appropriate tools are defined in tool_conf.xml. If no tools # associated with the migration stage are defined, no repositories will be installed https://bitbucket.org/galaxy/galaxy-central/commits/7cee92f360c0/ Changeset: 7cee92f360c0 User: jmchilton Date: 2014-12-29 14:11:54+00:00 Summary: Update ToolBox.update_shed_config to use config_filename instead of index. Seems a more robust index, simplifies ToolPanelManager, etc.... Affected #: 2 files diff -r 600d39d9bbac52eb4321e65d16d1740a3f7c0871 -r 7cee92f360c0ca15d0f3f443c2e80a629d5eab1b lib/galaxy/tools/__init__.py --- a/lib/galaxy/tools/__init__.py +++ b/lib/galaxy/tools/__init__.py @@ -265,13 +265,15 @@ return shed_config_dict return default - def update_shed_config( self, shed_conf_index, shed_conf, integrated_panel_changes=True ): + def update_shed_config( self, shed_conf, integrated_panel_changes=True ): """ Update the in-memory descriptions of tools and write out the changes to integrated tool panel unless we are just deactivating a tool (since that doesn't affect that file). """ app = self.app - self.shed_tool_confs[ shed_conf_index ] = shed_conf + for index, my_shed_tool_conf in enumerate( self.shed_tool_confs ): + if shed_conf['config_filename'] == my_shed_tool_conf['config_filename']: + self.shed_tool_confs[ index ] = shed_conf if integrated_panel_changes and app.config.update_integrated_tool_panel: # Write the current in-memory version of the integrated_tool_panel.xml file to disk. self.write_integrated_tool_panel_config_file() diff -r 600d39d9bbac52eb4321e65d16d1740a3f7c0871 -r 7cee92f360c0ca15d0f3f443c2e80a629d5eab1b lib/tool_shed/galaxy_install/tools/tool_panel_manager.py --- a/lib/tool_shed/galaxy_install/tools/tool_panel_manager.py +++ b/lib/tool_shed/galaxy_install/tools/tool_panel_manager.py @@ -40,7 +40,7 @@ shed_tool_conf, tool_panel_dict, new_install=True ): """A tool shed repository is being installed or updated so handle tool panel alterations accordingly.""" # We need to change the in-memory version and the file system version of the shed_tool_conf file. - index, shed_tool_conf_dict = self.get_shed_tool_conf_dict( shed_tool_conf ) + _, shed_tool_conf_dict = self.get_shed_tool_conf_dict( shed_tool_conf ) tool_path = shed_tool_conf_dict[ 'tool_path' ] # Generate the list of ElementTree Element objects for each section or tool. elem_list = self.generate_tool_panel_elem_list( repository_name, @@ -66,7 +66,7 @@ ) # Replace the old list of in-memory config_elems with the new list for this shed_tool_conf_dict. shed_tool_conf_dict[ 'config_elems' ] = config_elems - self.app.toolbox.update_shed_config( index, shed_tool_conf_dict ) + self.app.toolbox.update_shed_config( shed_tool_conf_dict ) def config_elems_to_xml_file( self, config_elems, config_filename, tool_path ): """ @@ -430,7 +430,7 @@ for guid_to_remove in guids_to_remove: # remove_from_tool_panel to false, will handling that logic below. toolbox.remove_tool_by_id( guid_to_remove, remove_from_panel=False ) - index, shed_tool_conf_dict = self.get_shed_tool_conf_dict( shed_tool_conf ) + _, shed_tool_conf_dict = self.get_shed_tool_conf_dict( shed_tool_conf ) if uninstall: # Remove from the shed_tool_conf file on disk. self.remove_from_shed_tool_config( shed_tool_conf_dict, guids_to_remove ) @@ -464,4 +464,4 @@ config_elems.remove( config_elem ) # Update the config_elems of the in-memory shed_tool_conf_dict. shed_tool_conf_dict[ 'config_elems' ] = config_elems - toolbox.update_shed_config( index, shed_tool_conf_dict, integrated_panel_changes=uninstall ) + toolbox.update_shed_config( shed_tool_conf_dict, integrated_panel_changes=uninstall ) https://bitbucket.org/galaxy/galaxy-central/commits/22db1ba32b58/ Changeset: 22db1ba32b58 User: jmchilton Date: 2014-12-29 14:11:54+00:00 Summary: Simplifications for ToolPanelManager.get_shed_tool_conf_dict. None of the components using this method are using the index any more - so eliminate it. Also introduce a new ToolBox method to hide the details of ToolBox.shed_tool_confs from this method. Affected #: 6 files diff -r 7cee92f360c0ca15d0f3f443c2e80a629d5eab1b -r 22db1ba32b58093b68662db3f9f7f513e20c2bdf lib/galaxy/tools/__init__.py --- a/lib/galaxy/tools/__init__.py +++ b/lib/galaxy/tools/__init__.py @@ -634,15 +634,21 @@ break return tool_version_select_field, tools, tool + def dynamic_confs( self, include_migrated_tool_conf=False ): + confs = [] + for dynamic_tool_conf_dict in self.shed_tool_confs: + dynamic_tool_conf_filename = dynamic_tool_conf_dict[ 'config_filename' ] + if include_migrated_tool_conf or (dynamic_tool_conf_filename != self.app.config.migrated_tools_config): + confs.append( dynamic_tool_conf_dict ) + return confs + def dynamic_conf_filenames( self, include_migrated_tool_conf=False ): """ Return list of dynamic tool configuration filenames (shed_tools). These must be used with various dynamic tool configuration update operations (e.g. with update_shed_config). """ - for dynamic_tool_conf_dict in self.shed_tool_confs: - dynamic_tool_conf_filename = dynamic_tool_conf_dict[ 'config_filename' ] - if include_migrated_tool_conf or (dynamic_tool_conf_filename != self.app.config.migrated_tools_config): - yield dynamic_tool_conf_filename + for dynamic_tool_conf_dict in self.dynamic_confs( include_migrated_tool_conf=include_migrated_tool_conf ): + yield dynamic_tool_conf_dict[ 'config_filename' ] def build_tool_version_select_field( self, tools, tool_id, set_selected ): """Build a SelectField whose options are the ids for the received list of tools.""" diff -r 7cee92f360c0ca15d0f3f443c2e80a629d5eab1b -r 22db1ba32b58093b68662db3f9f7f513e20c2bdf lib/tool_shed/galaxy_install/install_manager.py --- a/lib/tool_shed/galaxy_install/install_manager.py +++ b/lib/tool_shed/galaxy_install/install_manager.py @@ -730,7 +730,7 @@ shed_tool_conf = install_options.get( 'shed_tool_conf', None ) if shed_tool_conf: # Get the tool_path setting. - _, shed_conf_dict = self.tpm.get_shed_tool_conf_dict( shed_tool_conf ) + shed_conf_dict = self.tpm.get_shed_tool_conf_dict( shed_tool_conf ) tool_path = shed_conf_dict[ 'tool_path' ] else: # Pick a semi-random shed-related tool panel configuration file and get the tool_path setting. diff -r 7cee92f360c0ca15d0f3f443c2e80a629d5eab1b -r 22db1ba32b58093b68662db3f9f7f513e20c2bdf lib/tool_shed/galaxy_install/metadata/installed_repository_metadata_manager.py --- a/lib/tool_shed/galaxy_install/metadata/installed_repository_metadata_manager.py +++ b/lib/tool_shed/galaxy_install/metadata/installed_repository_metadata_manager.py @@ -66,7 +66,7 @@ in the received tool shed repository metadata. """ repository_tools_tups = [] - _, shed_conf_dict = self.tpm.get_shed_tool_conf_dict( self.metadata_dict.get( 'shed_config_filename' ) ) + shed_conf_dict = self.tpm.get_shed_tool_conf_dict( self.metadata_dict.get( 'shed_config_filename' ) ) if 'tools' in self.metadata_dict: for tool_dict in self.metadata_dict[ 'tools' ]: load_relative_path = relative_path = tool_dict.get( 'tool_config', None ) diff -r 7cee92f360c0ca15d0f3f443c2e80a629d5eab1b -r 22db1ba32b58093b68662db3f9f7f513e20c2bdf lib/tool_shed/galaxy_install/tool_migration_manager.py --- a/lib/tool_shed/galaxy_install/tool_migration_manager.py +++ b/lib/tool_shed/galaxy_install/tool_migration_manager.py @@ -78,7 +78,7 @@ self.tool_shed_url = common_util.get_tool_shed_url_from_tool_shed_registry( self.app, defined_tool_shed_url ) self.tool_shed = common_util.remove_protocol_and_port_from_tool_shed_url( self.tool_shed_url ) self.repository_owner = common_util.REPOSITORY_OWNER - _, self.shed_config_dict = self.tpm.get_shed_tool_conf_dict( self.migrated_tools_config ) + self.shed_config_dict = self.tpm.get_shed_tool_conf_dict( self.migrated_tools_config ) # Since tool migration scripts can be executed any number of times, we need to # make sure the appropriate tools are defined in tool_conf.xml. If no tools # associated with the migration stage are defined, no repositories will be installed diff -r 7cee92f360c0ca15d0f3f443c2e80a629d5eab1b -r 22db1ba32b58093b68662db3f9f7f513e20c2bdf lib/tool_shed/galaxy_install/tools/tool_panel_manager.py --- a/lib/tool_shed/galaxy_install/tools/tool_panel_manager.py +++ b/lib/tool_shed/galaxy_install/tools/tool_panel_manager.py @@ -40,7 +40,7 @@ shed_tool_conf, tool_panel_dict, new_install=True ): """A tool shed repository is being installed or updated so handle tool panel alterations accordingly.""" # We need to change the in-memory version and the file system version of the shed_tool_conf file. - _, shed_tool_conf_dict = self.get_shed_tool_conf_dict( shed_tool_conf ) + shed_tool_conf_dict = self.get_shed_tool_conf_dict( shed_tool_conf ) tool_path = shed_tool_conf_dict[ 'tool_path' ] # Generate the list of ElementTree Element objects for each section or tool. elem_list = self.generate_tool_panel_elem_list( repository_name, @@ -304,13 +304,13 @@ Return the in-memory version of the shed_tool_conf file, which is stored in the config_elems entry in the shed_tool_conf_dict associated with the file. """ - for index, shed_tool_conf_dict in enumerate( self.app.toolbox.shed_tool_confs ): + for shed_tool_conf_dict in self.app.toolbox.dynamic_confs( include_migrated_tool_conf=True ): if shed_tool_conf == shed_tool_conf_dict[ 'config_filename' ]: - return index, shed_tool_conf_dict + return shed_tool_conf_dict else: file_name = basic_util.strip_path( shed_tool_conf_dict[ 'config_filename' ] ) if shed_tool_conf == file_name: - return index, shed_tool_conf_dict + return shed_tool_conf_dict def handle_tool_panel_section( self, toolbox, tool_panel_section_id=None, new_tool_panel_section_label=None ): """Return a ToolSection object retrieved from the current in-memory tool_panel.""" @@ -430,7 +430,7 @@ for guid_to_remove in guids_to_remove: # remove_from_tool_panel to false, will handling that logic below. toolbox.remove_tool_by_id( guid_to_remove, remove_from_panel=False ) - _, shed_tool_conf_dict = self.get_shed_tool_conf_dict( shed_tool_conf ) + shed_tool_conf_dict = self.get_shed_tool_conf_dict( shed_tool_conf ) if uninstall: # Remove from the shed_tool_conf file on disk. self.remove_from_shed_tool_config( shed_tool_conf_dict, guids_to_remove ) diff -r 7cee92f360c0ca15d0f3f443c2e80a629d5eab1b -r 22db1ba32b58093b68662db3f9f7f513e20c2bdf test/unit/tools_support.py --- a/test/unit/tools_support.py +++ b/test/unit/tools_support.py @@ -100,6 +100,7 @@ admin_users="mary@example.com", len_file_path=os.path.join( 'tool-data', 'shared', 'ucsc', 'chrom' ), builds_file_path=os.path.join( 'tool-data', 'shared', 'ucsc', 'builds.txt.sample' ), + migrated_tools_config=os.path.join(test_directory, "migrated_tools_conf.xml"), ) # Setup some attributes for downstream extension by specific tests. https://bitbucket.org/galaxy/galaxy-central/commits/68255ebe3f54/ Changeset: 68255ebe3f54 User: jmchilton Date: 2014-12-29 14:11:54+00:00 Summary: Reuse new ToolBox.dynamic_confs method other places... ... to shield ToolBox.shed_tool_confs from other components including Tool Shed install models, shed_util_common.py, and install_manager.py. After this commit - shed_tool_confs is only accessed directly by the ToolBox itself. Affected #: 3 files diff -r 22db1ba32b58093b68662db3f9f7f513e20c2bdf -r 68255ebe3f54500aa26db814c41f67f382731462 lib/galaxy/model/tool_shed_install/__init__.py --- a/lib/galaxy/model/tool_shed_install/__init__.py +++ b/lib/galaxy/model/tool_shed_install/__init__.py @@ -99,7 +99,7 @@ if not self.shed_config_filename: self.guess_shed_config( app, default=default ) if self.shed_config_filename: - for shed_tool_conf_dict in app.toolbox.shed_tool_confs: + for shed_tool_conf_dict in app.toolbox.dynamic_confs( include_migrated_tool_conf=True ): if self.shed_config_filename == shed_tool_conf_dict[ 'config_filename' ]: return shed_tool_conf_dict return default @@ -118,7 +118,7 @@ metadata = self.metadata or {} for tool in metadata.get( 'tools', [] ): tool_ids.append( tool.get( 'guid' ) ) - for shed_tool_conf_dict in app.toolbox.shed_tool_confs: + for shed_tool_conf_dict in app.toolbox.dynamic_confs( include_migrated_tool_conf=True ): name = shed_tool_conf_dict[ 'config_filename' ] for elem in shed_tool_conf_dict[ 'config_elems' ]: if elem.tag == 'tool': @@ -137,7 +137,7 @@ if self.includes_datatypes: # We need to search by file paths here, which is less desirable. tool_shed = common_util.remove_protocol_and_port_from_tool_shed_url( self.tool_shed ) - for shed_tool_conf_dict in app.toolbox.shed_tool_confs: + for shed_tool_conf_dict in app.toolbox.dynamic_confs( include_migrated_tool_conf=True ): tool_path = shed_tool_conf_dict[ 'tool_path' ] relative_path = os.path.join( tool_path, tool_shed, 'repos', self.owner, self.name, self.installed_changeset_revision ) if os.path.exists( relative_path ): @@ -284,7 +284,7 @@ def repo_path( self, app ): tool_shed = common_util.remove_protocol_and_port_from_tool_shed_url( self.tool_shed ) - for index, shed_tool_conf_dict in enumerate( app.toolbox.shed_tool_confs ): + for shed_tool_conf_dict in app.toolbox.dynamic_confs( include_migrated_tool_conf=True ): tool_path = shed_tool_conf_dict[ 'tool_path' ] relative_path = os.path.join( tool_path, tool_shed, 'repos', self.owner, self.name, self.installed_changeset_revision ) if os.path.exists( relative_path ): diff -r 22db1ba32b58093b68662db3f9f7f513e20c2bdf -r 68255ebe3f54500aa26db814c41f67f382731462 lib/tool_shed/galaxy_install/install_manager.py --- a/lib/tool_shed/galaxy_install/install_manager.py +++ b/lib/tool_shed/galaxy_install/install_manager.py @@ -733,15 +733,13 @@ shed_conf_dict = self.tpm.get_shed_tool_conf_dict( shed_tool_conf ) tool_path = shed_conf_dict[ 'tool_path' ] else: - # Pick a semi-random shed-related tool panel configuration file and get the tool_path setting. - for shed_config_dict in self.app.toolbox.shed_tool_confs: - # Don't use migrated_tools_conf.xml. - if shed_config_dict[ 'config_filename' ] != self.app.config.migrated_tools_config: - break + # Don't use migrated_tools_conf.xml. + try: + shed_config_dict = self.app.toolbox.dynamic_confs( include_migrated_tool_conf=False )[ 0 ] + except IndexError: + raise exceptions.RequestParameterMissingException( "Missing required parameter 'shed_tool_conf'." ) shed_tool_conf = shed_config_dict[ 'config_filename' ] tool_path = shed_config_dict[ 'tool_path' ] - if not shed_tool_conf: - raise exceptions.RequestParameterMissingException( "Missing required parameter 'shed_tool_conf'." ) tool_panel_section_id = self.app.toolbox.find_section_id( install_options.get( 'tool_panel_section_id', '' ) ) # Build the dictionary of information necessary for creating tool_shed_repository database records # for each repository being installed. diff -r 22db1ba32b58093b68662db3f9f7f513e20c2bdf -r 68255ebe3f54500aa26db814c41f67f382731462 lib/tool_shed/util/shed_util_common.py --- a/lib/tool_shed/util/shed_util_common.py +++ b/lib/tool_shed/util/shed_util_common.py @@ -722,7 +722,7 @@ shed_config_dict = repository.get_shed_config_dict( app ) if not shed_config_dict: # Just pick a semi-random shed config. - for shed_config_dict in app.toolbox.shed_tool_confs: + for shed_config_dict in app.toolbox.dynamic_confs( include_migrated_tool_conf=True ): if ( repository.dist_to_shed and shed_config_dict[ 'config_filename' ] == app.config.migrated_tools_config ) \ or ( not repository.dist_to_shed and shed_config_dict[ 'config_filename' ] != app.config.migrated_tools_config ): break @@ -736,7 +736,7 @@ Return the tool_path config setting for the received shed_tool_conf file by searching the tool box's in-memory list of shed_tool_confs for the dictionary whose config_filename key has a value matching the received shed_tool_conf. """ - for shed_tool_conf_dict in app.toolbox.shed_tool_confs: + for shed_tool_conf_dict in app.toolbox.dynamic_confs( include_migrated_tool_conf=True ): config_filename = shed_tool_conf_dict[ 'config_filename' ] if config_filename == shed_tool_conf: return shed_tool_conf_dict[ 'tool_path' ] @@ -1036,15 +1036,7 @@ def have_shed_tool_conf_for_install( app ): - if not app.toolbox.shed_tool_confs: - return False - migrated_tools_conf_path, migrated_tools_conf_name = os.path.split( app.config.migrated_tools_config ) - for shed_tool_conf_dict in app.toolbox.shed_tool_confs: - shed_tool_conf = shed_tool_conf_dict[ 'config_filename' ] - shed_tool_conf_path, shed_tool_conf_name = os.path.split( shed_tool_conf ) - if shed_tool_conf_name != migrated_tools_conf_name: - return True - return False + return bool( app.toolbox.dynamic_confs( include_migrated_tool_conf=False ) ) def is_tool_shed_client( app ): Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.