December 2014 - galaxy-commits - lists.galaxyproject.org

commit/galaxy-central: 12 new changesets
by commits-noreply＠bitbucket.org 29 Dec '14

29 Dec '14

12 new commits in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/8896bf67ff7a/ Changeset: 8896bf67ff7a User: jmchilton Date: 2014-12-29 14:11:54+00:00 Summary: Correct sample value for migrated tool config. Affected #: 1 file diff -r bfd679408c84a4c6f1d2b1a09eec1dab9c2d212b -r 8896bf67ff7ae6f4379ff0e18c30559a20cf6584 config/galaxy.ini.sample --- a/config/galaxy.ini.sample +++ b/config/galaxy.ini.sample @@ -148,7 +148,7 @@ # Tool config maintained by tool migration scripts. If you use the migration # scripts to install tools that have been migrated to the tool shed upon a new # release, they will be added to this tool config file. -#migrated_tools_config = migrated_tools_conf.xml +#migrated_tools_config = config/migrated_tools_conf.xml # File that contains the XML section and tool tags from all tool panel config # files integrated into a single file that defines the tool panel layout. This https://bitbucket.org/galaxy/galaxy-central/commits/d87f3d474c9a/ Changeset: d87f3d474c9a User: jmchilton Date: 2014-12-29 14:11:54+00:00 Summary: Small simplification to tool_panel_manager.py. Affected #: 1 file diff -r 8896bf67ff7ae6f4379ff0e18c30559a20cf6584 -r d87f3d474c9ab798e88286f80c09cef2bc41451a lib/tool_shed/galaxy_install/tools/tool_panel_manager.py --- a/lib/tool_shed/galaxy_install/tools/tool_panel_manager.py +++ b/lib/tool_shed/galaxy_install/tools/tool_panel_manager.py @@ -243,22 +243,13 @@ tool_file_path, tup_guid, tool = repository_tool_tup if tup_guid == guid: break - if inside_section: - tool_elem = self.generate_tool_elem( tool_shed, - repository_name, - changeset_revision, - owner, - tool_file_path, - tool, - tool_section ) - else: - tool_elem = self.generate_tool_elem( tool_shed, - repository_name, - changeset_revision, - owner, - tool_file_path, - tool, - None ) + tool_elem = self.generate_tool_elem( tool_shed, + repository_name, + changeset_revision, + owner, + tool_file_path, + tool, + tool_section if inside_section else None ) if inside_section: if section_in_elem_list: elem_list[ index ] = tool_section https://bitbucket.org/galaxy/galaxy-central/commits/000cd07f16f9/ Changeset: 000cd07f16f9 User: jmchilton Date: 2014-12-29 14:11:54+00:00 Summary: Reused get_section method to hide ToolBox.tool_panel from install_manager.py. Affected #: 4 files diff -r d87f3d474c9ab798e88286f80c09cef2bc41451a -r 000cd07f16f991cb78025e23f0872bafd58b9daf lib/galaxy/tools/__init__.py --- a/lib/galaxy/tools/__init__.py +++ b/lib/galaxy/tools/__init__.py @@ -277,13 +277,13 @@ self.write_integrated_tool_panel_config_file() app.reindex_tool_search() - def get_or_create_section( self, section_id, new_label=None ): + def get_section( self, section_id, new_label=None, create_if_needed=False ): tool_panel_section_key = str( section_id ) if tool_panel_section_key in self.tool_panel: # Appending a tool to an existing section in toolbox.tool_panel tool_section = self.tool_panel[ tool_panel_section_key ] log.debug( "Appending to tool panel section: %s" % str( tool_section.name ) ) - else: + elif create_if_needed: # Appending a new section to toolbox.tool_panel if new_label is None: # This might add an ugly section label to the tool panel, but, oh well... @@ -296,6 +296,8 @@ tool_section = ToolSection( section_dict ) self.tool_panel[ tool_panel_section_key ] = tool_section log.debug( "Loading new tool panel section: %s" % str( tool_section.name ) ) + else: + tool_section = None return tool_panel_section_key, tool_section def __resolve_tool_path(self, tool_path, config_filename): diff -r d87f3d474c9ab798e88286f80c09cef2bc41451a -r 000cd07f16f991cb78025e23f0872bafd58b9daf lib/tool_shed/galaxy_install/install_manager.py --- a/lib/tool_shed/galaxy_install/install_manager.py +++ b/lib/tool_shed/galaxy_install/install_manager.py @@ -809,12 +809,10 @@ def install_tool_shed_repository( self, tool_shed_repository, repo_info_dict, tool_panel_section_key, shed_tool_conf, tool_path, install_tool_dependencies, reinstalling=False ): if tool_panel_section_key: - try: - tool_section = self.app.toolbox.tool_panel[ tool_panel_section_key ] - except KeyError: + _, tool_section = self.app.toolbox.get_section( tool_panel_section_key ) + if tool_section is None: log.debug( 'Invalid tool_panel_section_key "%s" specified. Tools will be loaded outside of sections in the tool panel.', str( tool_panel_section_key ) ) - tool_section = None else: tool_section = None if isinstance( repo_info_dict, basestring ): diff -r d87f3d474c9ab798e88286f80c09cef2bc41451a -r 000cd07f16f991cb78025e23f0872bafd58b9daf lib/tool_shed/galaxy_install/tools/tool_panel_manager.py --- a/lib/tool_shed/galaxy_install/tools/tool_panel_manager.py +++ b/lib/tool_shed/galaxy_install/tools/tool_panel_manager.py @@ -297,7 +297,7 @@ return tool_section def get_or_create_tool_section( self, toolbox, tool_panel_section_id, new_tool_panel_section_label=None ): - return toolbox.get_or_create_section( section_id=tool_panel_section_id, new_label=new_tool_panel_section_label ) + return toolbox.get_section( section_id=tool_panel_section_id, new_label=new_tool_panel_section_label, create_if_needed=True ) def get_shed_tool_conf_dict( self, shed_tool_conf ): """ diff -r d87f3d474c9ab798e88286f80c09cef2bc41451a -r 000cd07f16f991cb78025e23f0872bafd58b9daf test/unit/tool_shed_unit_tests/test_tool_panel_manager.py --- a/test/unit/tool_shed_unit_tests/test_tool_panel_manager.py +++ b/test/unit/tool_shed_unit_tests/test_tool_panel_manager.py @@ -46,7 +46,7 @@ self.tool, ) ] - _, section = self.toolbox.get_or_create_section("tid1") + _, section = self.toolbox.get_section("tid1", create_if_needed=True) tpm = self.tpm tool_panel_dict = tpm.generate_tool_panel_dict_for_new_install( tool_dicts=new_tools, https://bitbucket.org/galaxy/galaxy-central/commits/739ca42b8555/ Changeset: 739ca42b8555 User: jmchilton Date: 2014-12-29 14:11:54+00:00 Summary: PEP-8 fixes for various Tool Shed components. - lib/tool_shed/util/tool_util.py - lib/tool_shed/galaxy_install/tool_migration_manager.py. - lib/tool_shed/util/shed_util_common.py. - lib/galaxy/model/tool_shed_install/__init__.py. Affected #: 4 files diff -r 000cd07f16f991cb78025e23f0872bafd58b9daf -r 739ca42b855530e8293cbb25298e1d4f6ef03d15 lib/galaxy/model/tool_shed_install/__init__.py --- a/lib/galaxy/model/tool_shed_install/__init__.py +++ b/lib/galaxy/model/tool_shed_install/__init__.py @@ -11,9 +11,9 @@ class ToolShedRepository( object ): dict_collection_visible_keys = ( 'id', 'tool_shed', 'name', 'owner', 'installed_changeset_revision', 'changeset_revision', 'ctx_rev', 'includes_datatypes', - 'tool_shed_status', 'deleted', 'uninstalled', 'dist_to_shed', 'status', 'error_message' ) + 'tool_shed_status', 'deleted', 'uninstalled', 'dist_to_shed', 'status', 'error_message' ) dict_element_visible_keys = ( 'id', 'tool_shed', 'name', 'owner', 'installed_changeset_revision', 'changeset_revision', 'ctx_rev', 'includes_datatypes', - 'tool_shed_status', 'deleted', 'uninstalled', 'dist_to_shed', 'status', 'error_message' ) + 'tool_shed_status', 'deleted', 'uninstalled', 'dist_to_shed', 'status', 'error_message' ) installation_status = Bunch( NEW='New', CLONING='Cloning', SETTING_TOOL_VERSIONS='Setting tool versions', @@ -356,12 +356,12 @@ for rd_tup in rd_tups: if len( rd_tup ) == 5: tool_shed, name, owner, changeset_revision, prior_installation_required, only_if_compiling_contained_td = \ - common_util.parse_repository_dependency_tuple( rd_tup, contains_error=False ) + common_util.parse_repository_dependency_tuple( rd_tup, contains_error=False ) if asbool( prior_installation_required ): required_rd_tups_that_must_be_installed.append( ( tool_shed, name, owner, changeset_revision, 'True', 'False' ) ) elif len( rd_tup ) == 6: tool_shed, name, owner, changeset_revision, prior_installation_required, only_if_compiling_contained_td = \ - common_util.parse_repository_dependency_tuple( rd_tup, contains_error=False ) + common_util.parse_repository_dependency_tuple( rd_tup, contains_error=False ) # The repository dependency will only be required to be previously installed if it does not fall into the category of # a repository that must be installed only so that its contained tool dependency can be used for compiling the tool # dependency of the dependent repository. @@ -493,12 +493,14 @@ class RepositoryRepositoryDependencyAssociation( object ): + def __init__( self, tool_shed_repository_id=None, repository_dependency_id=None ): self.tool_shed_repository_id = tool_shed_repository_id self.repository_dependency_id = repository_dependency_id class RepositoryDependency( object ): + def __init__( self, tool_shed_repository_id=None ): self.tool_shed_repository_id = tool_shed_repository_id @@ -647,6 +649,7 @@ class ToolVersionAssociation( object ): + def __init__( self, id=None, tool_id=None, parent_id=None ): self.id = id self.tool_id = tool_id @@ -654,6 +657,7 @@ class MigrateTools( object ): + def __init__( self, repository_id=None, repository_path=None, version=None ): self.repository_id = repository_id self.repository_path = repository_path diff -r 000cd07f16f991cb78025e23f0872bafd58b9daf -r 739ca42b855530e8293cbb25298e1d4f6ef03d15 lib/tool_shed/galaxy_install/tool_migration_manager.py --- a/lib/tool_shed/galaxy_install/tool_migration_manager.py +++ b/lib/tool_shed/galaxy_install/tool_migration_manager.py @@ -171,8 +171,6 @@ clone_dir = os.path.join( self.tool_path, relative_clone_dir ) if not self.__iscloned( clone_dir ): repository_clone_url = os.path.join( self.tool_shed_url, 'repos', owner, name ) - relative_install_dir = os.path.join( relative_clone_dir, name ) - install_dir = os.path.join( clone_dir, name ) ctx_rev = suc.get_ctx_rev( self.app, self.tool_shed_url, name, owner, changeset_revision ) tool_shed_repository = suc.create_or_update_tool_shed_repository( app=self.app, name=name, @@ -406,7 +404,7 @@ tool_configs_to_filter.append( tool_config ) else: print 'The tool "%s" (%s) has not been enabled because it is not defined in a proprietary tool config (%s).' \ - % ( guid, tool_config, ", ".join( self.proprietary_tool_confs or [] ) ) + % ( guid, tool_config, ", ".join( self.proprietary_tool_confs or [] ) ) if tool_configs_to_filter: lock = threading.Lock() lock.acquire( True ) @@ -421,7 +419,7 @@ repository=tool_shed_repository, changeset_revision=tool_shed_repository.changeset_revision, repository_clone_url=repository_clone_url, - shed_config_dict = self.shed_config_dict, + shed_config_dict=self.shed_config_dict, relative_install_dir=relative_install_dir, repository_files_dir=None, resetting_all_metadata_on_repository=False, @@ -504,11 +502,11 @@ datatypes_config = hg_util.get_config_from_disk( suc.DATATYPES_CONFIG_FILENAME, repo_install_dir ) # Load proprietary data types required by tools. The value of override is not # important here since the Galaxy server will be started after this installation - #completes. + # completes. converter_path, display_path = \ cdl.alter_config_and_load_prorietary_datatypes( datatypes_config, repo_install_dir, - override=False ) + override=False ) if converter_path or display_path: # Create a dictionary of tool shed repository related information. repository_dict = \ @@ -583,23 +581,23 @@ tool_version_dicts = json.loads( text ) tvm.handle_tool_versions( tool_version_dicts, tool_shed_repository ) else: - # Set the tool versions since they seem to be missing for this repository in the tool shed. - # CRITICAL NOTE: These default settings may not properly handle all parent/child associations. + # Set the tool versions since they seem to be missing + # for this repository in the tool shed. CRITICAL NOTE: + # These default settings may not properly handle all + # parent/child associations. for tool_dict in metadata_dict[ 'tools' ]: - flush_needed = False tool_id = tool_dict[ 'guid' ] old_tool_id = tool_dict[ 'id' ] - tool_version = tool_dict[ 'version' ] tool_version_using_old_id = tvm.get_tool_version( old_tool_id ) tool_version_using_guid = tvm.get_tool_version( tool_id ) if not tool_version_using_old_id: tool_version_using_old_id = self.app.install_model.ToolVersion( tool_id=old_tool_id, - tool_shed_repository=tool_shed_repository ) + tool_shed_repository=tool_shed_repository ) self.app.install_model.context.add( tool_version_using_old_id ) self.app.install_model.context.flush() if not tool_version_using_guid: tool_version_using_guid = self.app.install_model.ToolVersion( tool_id=tool_id, - tool_shed_repository=tool_shed_repository ) + tool_shed_repository=tool_shed_repository ) self.app.install_model.context.add( tool_version_using_guid ) self.app.install_model.context.flush() # Associate the two versions as parent / child. @@ -640,7 +638,6 @@ ordered_tsr_ids = [] processed_tsr_ids = [] prior_install_required_dict = self.get_prior_install_required_dict( tool_shed_repositories, repository_dependencies_dict ) - tsr_ids = [ tool_shed_repository.id for tool_shed_repository in tool_shed_repositories ] while len( processed_tsr_ids ) != len( prior_install_required_dict.keys() ): tsr_id = suc.get_next_prior_import_or_install_required_dict_entry( prior_install_required_dict, processed_tsr_ids ) processed_tsr_ids.append( tsr_id ) diff -r 000cd07f16f991cb78025e23f0872bafd58b9daf -r 739ca42b855530e8293cbb25298e1d4f6ef03d15 lib/tool_shed/util/shed_util_common.py --- a/lib/tool_shed/util/shed_util_common.py +++ b/lib/tool_shed/util/shed_util_common.py @@ -3,6 +3,7 @@ import os import re import shutil +import socket import string import sqlalchemy.orm.exc @@ -81,6 +82,7 @@ '${host}' """ + def create_or_update_tool_shed_repository( app, name, description, installed_changeset_revision, ctx_rev, repository_clone_url, metadata_dict, status, current_changeset_revision=None, owner='', dist_to_shed=False ): """ @@ -118,8 +120,8 @@ owner, installed_changeset_revision ) if tool_shed_repository: - log.debug( "Updating an existing row for repository '%s' in the tool_shed_repository table, status set to '%s'." % \ - ( str( name ), str( status ) ) ) + log.debug( "Updating an existing row for repository '%s' in the tool_shed_repository table, status set to '%s'." % + ( str( name ), str( status ) ) ) tool_shed_repository.description = description tool_shed_repository.changeset_revision = current_changeset_revision tool_shed_repository.ctx_rev = ctx_rev @@ -129,8 +131,8 @@ tool_shed_repository.uninstalled = uninstalled tool_shed_repository.status = status else: - log.debug( "Adding new row for repository '%s' in the tool_shed_repository table, status set to '%s'." % \ - ( str( name ), str( status ) ) ) + log.debug( "Adding new row for repository '%s' in the tool_shed_repository table, status set to '%s'." % + ( str( name ), str( status ) ) ) tool_shed_repository = \ app.install_model.ToolShedRepository( tool_shed=tool_shed, name=name, @@ -149,6 +151,7 @@ context.flush() return tool_shed_repository + def extract_components_from_tuple( repository_components_tuple ): '''Extract the repository components from the provided tuple in a backward-compatible manner.''' toolshed = repository_components_tuple[ 0 ] @@ -164,6 +167,7 @@ components_list = [ toolshed, name, owner, changeset_revision, prior_installation_required, only_if_compiling_contained_td ] return components_list + def generate_sharable_link_for_repository_in_tool_shed( repository, changeset_revision=None ): """Generate the URL for sharing a repository that is in the tool shed.""" base_url = url_for( '/', qualified=True ).rstrip( '/' ) @@ -173,6 +177,7 @@ sharable_url += '/%s' % changeset_revision return sharable_url + def generate_tool_guid( repository_clone_url, tool ): """ Generate a guid for the installed tool. It is critical that this guid matches the guid for @@ -182,6 +187,7 @@ tmp_url = common_util.remove_protocol_and_user_from_clone_url( repository_clone_url ) return '%s/%s/%s' % ( tmp_url, tool.id, tool.version ) + def generate_tool_shed_repository_install_dir( repository_clone_url, changeset_revision ): """ Generate a repository installation directory that guarantees repositories with the same @@ -196,6 +202,7 @@ tool_shed_url = common_util.remove_port_from_tool_shed_url( tool_shed_url ) return common_util.url_join( tool_shed_url, 'repos', repo_path, changeset_revision ) + def get_absolute_path_to_file_in_repository( repo_files_dir, file_name ): """Return the absolute path to a specified disk file contained in a repository.""" stripped_file_name = basic_util.strip_path( file_name ) @@ -207,19 +214,22 @@ return os.path.abspath( os.path.join( root, name ) ) return file_path + def get_categories( app ): """Get all categories from the database.""" sa_session = app.model.context.current return sa_session.query( app.model.Category ) \ - .filter( app.model.Category.table.c.deleted==False ) \ + .filter( app.model.Category.table.c.deleted == False ) \ .order_by( app.model.Category.table.c.name ) \ .all() + def get_category( app, id ): """Get a category from the database.""" sa_session = app.model.context.current return sa_session.query( app.model.Category ).get( app.security.decode_id( id ) ) + def get_category_by_name( app, name ): """Get a category from the database via name.""" sa_session = app.model.context.current @@ -228,6 +238,7 @@ except sqlalchemy.orm.exc.NoResultFound: return None + def get_ctx_rev( app, tool_shed_url, name, owner, changeset_revision ): """ Send a request to the tool shed to retrieve the ctx_rev for a repository defined by the @@ -240,6 +251,7 @@ ctx_rev = common_util.tool_shed_get( app, tool_shed_url, url ) return ctx_rev + def get_current_repository_metadata_for_changeset_revision( app, repository, changeset_revision ): encoded_repository_id = app.security.encode_id( repository.id ) repository_metadata = get_repository_metadata_by_changeset_revision( app, @@ -261,6 +273,7 @@ return repository_metadata return None + def get_ids_of_tool_shed_repositories_being_installed( app, as_string=False ): installing_repository_ids = [] new_status = app.install_model.ToolShedRepository.installation_status.NEW @@ -280,6 +293,7 @@ return ','.join( installing_repository_ids ) return installing_repository_ids + def get_latest_downloadable_changeset_revision( app, repository, repo ): repository_tip = repository.tip( app ) repository_metadata = get_repository_metadata_by_changeset_revision( app, app.security.encode_id( repository.id ), repository_tip ) @@ -290,6 +304,7 @@ return changeset_revisions[ -1 ] return hg_util.INITIAL_CHANGELOG_HASH + def get_next_downloadable_changeset_revision( repository, repo, after_changeset_revision ): """ Return the installable changeset_revision in the repository changelog after the changeset to which @@ -311,6 +326,7 @@ found_after_changeset_revision = True return None + def get_next_prior_import_or_install_required_dict_entry( prior_required_dict, processed_tsr_ids ): """ This method is used in the Tool Shed when exporting a repository and its dependencies, and in Galaxy @@ -344,6 +360,7 @@ continue return key + def get_ordered_metadata_changeset_revisions( repository, repo, downloadable=True ): """ Return an ordered list of changeset_revisions that are associated with metadata @@ -366,6 +383,7 @@ sorted_changeset_revisions = [ str( changeset_tup[ 1 ] ) for changeset_tup in sorted_changeset_tups ] return sorted_changeset_revisions + def get_prior_import_or_install_required_dict( app, tsr_ids, repo_info_dicts ): """ This method is used in the Tool Shed when exporting a repository and its dependencies, @@ -390,6 +408,7 @@ prior_import_or_install_required_dict[ encoded_repository_id ] = prior_import_or_install_ids return prior_import_or_install_required_dict + def get_repo_info_tuple_contents( repo_info_tuple ): """Take care in handling the repo_info_tuple as it evolves over time as new tool shed features are introduced.""" if len( repo_info_tuple ) == 6: @@ -399,6 +418,7 @@ description, repository_clone_url, changeset_revision, ctx_rev, repository_owner, repository_dependencies, tool_dependencies = repo_info_tuple return description, repository_clone_url, changeset_revision, ctx_rev, repository_owner, repository_dependencies, tool_dependencies + def get_repository_and_repository_dependencies_from_repo_info_dict( app, repo_info_dict ): """Return a tool_shed_repository or repository record defined by the information in the received repo_info_dict.""" repository_name = repo_info_dict.keys()[ 0 ] @@ -414,6 +434,7 @@ repository = get_repository_by_name_and_owner( app, repository_name, repository_owner ) return repository, repository_dependencies + def get_repository_by_id( app, id ): """Get a repository from the database via id.""" if is_tool_shed_client( app ): @@ -422,28 +443,31 @@ sa_session = app.model.context.current return sa_session.query( app.model.Repository ).get( app.security.decode_id( id ) ) + def get_repository_by_name( app, name ): """Get a repository from the database via name.""" repository_query = get_repository_query( app ) return repository_query.filter_by( name=name ).first() + def get_repository_by_name_and_owner( app, name, owner ): """Get a repository from the database via name and owner""" repository_query = get_repository_query( app ) if is_tool_shed_client( app ): return repository_query \ - .filter( and_( app.install_model.ToolShedRepository.table.c.name == name, - app.install_model.ToolShedRepository.table.c.owner == owner ) ) \ - .first() + .filter( and_( app.install_model.ToolShedRepository.table.c.name == name, + app.install_model.ToolShedRepository.table.c.owner == owner ) ) \ + .first() # We're in the tool shed. user = get_user_by_username( app, owner ) if user: return repository_query \ - .filter( and_( app.model.Repository.table.c.name == name, - app.model.Repository.table.c.user_id == user.id ) ) \ - .first() + .filter( and_( app.model.Repository.table.c.name == name, + app.model.Repository.table.c.user_id == user.id ) ) \ + .first() return None + def get_repository_dependency_types( repository_dependencies ): """ Inspect the received list of repository_dependencies tuples and return boolean values @@ -470,6 +494,7 @@ break return has_repository_dependencies, has_repository_dependencies_only_if_compiling_contained_td + def get_repository_for_dependency_relationship( app, tool_shed, name, owner, changeset_revision ): """ Return an installed tool_shed_repository database record that is defined by either the current changeset @@ -509,6 +534,7 @@ changeset_revision=text ) return repository + def get_repository_file_contents( file_path ): """Return the display-safe contents of a repository file for display in a browser.""" if checkers.is_gzip( file_path ): @@ -543,6 +569,7 @@ beginning_on_size_error=True ) return safe_str + def get_repository_files( folder_path ): """Return the file hierarchy of a tool shed repository.""" contents = [] @@ -558,6 +585,7 @@ contents.sort() return contents + def get_repository_from_refresh_on_change( app, **kwd ): # The changeset_revision_select_field in several grids performs a refresh_on_change which sends in request parameters like # changeset_revison_1, changeset_revision_2, etc. One of the many select fields on the grid performed the refresh_on_change, @@ -575,6 +603,7 @@ # This should never be reached - raise an exception? return v, None + def get_repository_ids_requiring_prior_import_or_install( app, tsr_ids, repository_dependencies ): """ This method is used in the Tool Shed when exporting a repository and its dependencies, @@ -593,11 +622,11 @@ continue for rd_tup in rd_tups: tool_shed, \ - name, \ - owner, \ - changeset_revision, \ - prior_installation_required, \ - only_if_compiling_contained_td = \ + name, \ + owner, \ + changeset_revision, \ + prior_installation_required, \ + only_if_compiling_contained_td = \ common_util.parse_repository_dependency_tuple( rd_tup ) # If only_if_compiling_contained_td is False, then the repository dependency # is not required to be installed prior to the dependent repository even if @@ -626,11 +655,13 @@ prior_tsr_ids.append( encoded_repository_id ) return prior_tsr_ids + def get_repository_in_tool_shed( app, id ): """Get a repository on the tool shed side from the database via id.""" sa_session = app.model.context.current return sa_session.query( app.model.Repository ).get( app.security.decode_id( id ) ) + def get_repository_metadata_by_changeset_revision( app, id, changeset_revision ): """Get metadata for a specified repository change set from the database.""" # Make sure there are no duplicate records, and return the single unique record for the changeset_revision. @@ -652,6 +683,7 @@ return all_metadata_records[ 0 ] return None + def get_repository_owner( cleaned_repository_url ): """Gvien a "cleaned" repository clone URL, return the owner of the repository.""" items = cleaned_repository_url.split( '/repos/' ) @@ -660,12 +692,13 @@ repo_path = repo_path.replace( '/', '', 1 ) return repo_path.lstrip( '/' ).split( '/' )[ 0 ] + def get_repository_owner_from_clone_url( repository_clone_url ): """Given a repository clone URL, return the owner of the repository.""" tmp_url = common_util.remove_protocol_and_user_from_clone_url( repository_clone_url ) - tool_shed = tmp_url.split( '/repos/' )[ 0 ].rstrip( '/' ) return get_repository_owner( tmp_url ) + def get_repository_query( app ): if is_tool_shed_client( app ): query = app.install_model.context.query( app.install_model.ToolShedRepository ) @@ -673,6 +706,7 @@ query = app.model.context.query( app.model.Repository ) return query + def get_tool_panel_config_tool_path_install_dir( app, repository ): """ Return shed-related tool panel config, the tool_path configured in it, and the relative path to @@ -690,12 +724,13 @@ # Just pick a semi-random shed config. for shed_config_dict in app.toolbox.shed_tool_confs: if ( repository.dist_to_shed and shed_config_dict[ 'config_filename' ] == app.config.migrated_tools_config ) \ - or ( not repository.dist_to_shed and shed_config_dict[ 'config_filename' ] != app.config.migrated_tools_config ): + or ( not repository.dist_to_shed and shed_config_dict[ 'config_filename' ] != app.config.migrated_tools_config ): break shed_tool_conf = shed_config_dict[ 'config_filename' ] tool_path = shed_config_dict[ 'tool_path' ] return shed_tool_conf, tool_path, relative_install_dir + def get_tool_path_by_shed_tool_conf_filename( app, shed_tool_conf ): """ Return the tool_path config setting for the received shed_tool_conf file by searching the tool box's in-memory list of shed_tool_confs for the @@ -711,10 +746,12 @@ return shed_tool_conf_dict[ 'tool_path' ] return None + def get_tool_shed_from_clone_url( repository_clone_url ): tmp_url = common_util.remove_protocol_and_user_from_clone_url( repository_clone_url ) return tmp_url.split( '/repos/' )[ 0 ].rstrip( '/' ) + def get_tool_shed_repository_by_id( app, repository_id ): """Return a tool shed repository database record defined by the id.""" # This method is used only in Galaxy, not the tool shed. @@ -722,6 +759,7 @@ .filter( app.install_model.ToolShedRepository.table.c.id == app.security.decode_id( repository_id ) ) \ .first() + def get_tool_shed_repository_by_shed_name_owner_changeset_revision( app, tool_shed, name, owner, changeset_revision ): """ Return a tool shed repository database record defined by the combination of a tool_shed, repository name, @@ -733,11 +771,12 @@ tool_shed = common_util.remove_protocol_from_tool_shed_url( tool_shed ) return repository_query \ .filter( and_( app.install_model.ToolShedRepository.table.c.tool_shed == tool_shed, - app.install_model.ToolShedRepository.table.c.name == name, - app.install_model.ToolShedRepository.table.c.owner == owner, - app.install_model.ToolShedRepository.table.c.changeset_revision == changeset_revision ) ) \ + app.install_model.ToolShedRepository.table.c.name == name, + app.install_model.ToolShedRepository.table.c.owner == owner, + app.install_model.ToolShedRepository.table.c.changeset_revision == changeset_revision ) ) \ .first() + def get_tool_shed_repository_by_shed_name_owner_installed_changeset_revision( app, tool_shed, name, owner, installed_changeset_revision ): """ @@ -750,11 +789,12 @@ tool_shed = common_util.remove_protocol_from_tool_shed_url( tool_shed ) return repository_query \ .filter( and_( app.install_model.ToolShedRepository.table.c.tool_shed == tool_shed, - app.install_model.ToolShedRepository.table.c.name == name, - app.install_model.ToolShedRepository.table.c.owner == owner, - app.install_model.ToolShedRepository.table.c.installed_changeset_revision == installed_changeset_revision ) ) \ + app.install_model.ToolShedRepository.table.c.name == name, + app.install_model.ToolShedRepository.table.c.owner == owner, + app.install_model.ToolShedRepository.table.c.installed_changeset_revision == installed_changeset_revision ) ) \ .first() + def get_tool_shed_status_for_installed_repository( app, repository ): """ Send a request to the tool shed to retrieve information about newer installable repository revisions, @@ -775,8 +815,8 @@ except HTTPError, e: # This should handle backward compatility to the Galaxy 12/20/12 release. We used to only handle updates for an installed revision # using a boolean value. - log.debug( "Error attempting to get tool shed status for installed repository %s: %s\nAttempting older 'check_for_updates' method.\n" % \ - ( str( repository.name ), str( e ) ) ) + log.debug( "Error attempting to get tool shed status for installed repository %s: %s\nAttempting older 'check_for_updates' method.\n" % + ( str( repository.name ), str( e ) ) ) params = '?name=%s&owner=%s&changeset_revision=%s&from_update_manager=True' % ( str( repository.name ), str( repository.owner ), str( repository.changeset_revision ) ) @@ -793,6 +833,7 @@ log.exception( "Error attempting to get tool shed status for installed repository %s: %s" % ( str( repository.name ), str( e ) ) ) return {} + def get_tool_shed_repository_status_label( app, tool_shed_repository=None, name=None, owner=None, changeset_revision=None, repository_clone_url=None ): """Return a color-coded label for the status of the received tool-shed_repository installed into Galaxy.""" if tool_shed_repository is None: @@ -842,6 +883,7 @@ status_label = 'unknown status' return '<div class="count-box state-color-%s">%s</div>' % ( bgcolor, status_label ) + def get_updated_changeset_revisions( app, name, owner, changeset_revision ): """ Return a string of comma-separated changeset revision hashes for all available updates to the received changeset @@ -863,6 +905,7 @@ return changeset_hashes_str return '' + def get_updated_changeset_revisions_from_tool_shed( app, tool_shed_url, name, owner, changeset_revision ): """ Get all appropriate newer changeset revisions for the repository defined by @@ -880,11 +923,13 @@ text = common_util.tool_shed_get( app, tool_shed_url, url ) return text + def get_user( app, id ): """Get a user from the database by id.""" sa_session = app.model.context.current return sa_session.query( app.model.User ).get( app.security.decode_id( id ) ) + def get_user_by_username( app, username ): """Get a user from the database by username.""" sa_session = app.model.context.current @@ -893,9 +938,10 @@ .filter( app.model.User.table.c.username == username ) \ .one() return user - except Exception, e: + except Exception: return None + def handle_email_alerts( app, host, repository, content_alert_str='', new_repo_alert=False, admin_only=False ): """ There are 2 complementary features that enable a tool shed user to receive email notification: @@ -947,7 +993,7 @@ admin_body = string.Template( template ).safe_substitute( host=host, sharable_link=sharable_link, repository_name=repository.name, - revision='%s:%s' %( str( ctx.rev() ), ctx ), + revision='%s:%s' % ( str( ctx.rev() ), ctx ), display_date=display_date, description=ctx.description(), username=username, @@ -955,7 +1001,7 @@ body = string.Template( template ).safe_substitute( host=host, sharable_link=sharable_link, repository_name=repository.name, - revision='%s:%s' %( str( ctx.rev() ), ctx ), + revision='%s:%s' % ( str( ctx.rev() ), ctx ), display_date=display_date, description=ctx.description(), username=username, @@ -985,9 +1031,10 @@ util.send_mail( frm, to, subject, admin_body, app.config ) else: util.send_mail( frm, to, subject, body, app.config ) - except Exception, e: + except Exception: log.exception( "An error occurred sending a tool shed repository update alert by email." ) + def have_shed_tool_conf_for_install( app ): if not app.toolbox.shed_tool_confs: return False @@ -999,6 +1046,7 @@ return True return False + def is_tool_shed_client( app ): """ The tool shed and clients to the tool (i.e. Galaxy) require a lot @@ -1008,6 +1056,7 @@ """ return hasattr( app, "install_model" ) + def open_repository_files_folder( folder_path ): """ Return a list of dictionaries, each of which contains information for a file or directory contained @@ -1026,14 +1075,15 @@ is_folder = True if filename: full_path = os.path.join( folder_path, filename ) - node = { "title" : filename, - "isFolder" : is_folder, - "isLazy" : is_folder, - "tooltip" : full_path, - "key" : full_path } + node = { "title": filename, + "isFolder": is_folder, + "isLazy": is_folder, + "tooltip": full_path, + "key": full_path } folder_contents.append( node ) return folder_contents + def repository_was_previously_installed( app, tool_shed_url, repository_name, repo_info_tuple, from_tip=False ): """ Find out if a repository is already installed into Galaxy - there are several scenarios where this @@ -1077,6 +1127,7 @@ return tool_shed_repository, previous_changeset_revision return None, None + def set_image_paths( app, encoded_repository_id, text ): """ Handle tool help image display for tools that are contained in repositories in @@ -1106,6 +1157,7 @@ text = re.sub( r'\.\. image:: (?!https?://)/?(.+)', r'.. image:: %s/\1' % route_to_images, text ) return text + def set_repository_attributes( app, repository, status, error_message, deleted, uninstalled, remove_from_disk=False ): if remove_from_disk: relative_install_dir = repository.repo_path( app ) @@ -1123,6 +1175,7 @@ app.install_model.context.add( repository ) app.install_model.context.flush() + def tool_shed_is_this_tool_shed( toolshed_base_url ): """Determine if a tool shed is the current tool shed.""" cleaned_toolshed_base_url = common_util.remove_protocol_from_tool_shed_url( toolshed_base_url ) diff -r 000cd07f16f991cb78025e23f0872bafd58b9daf -r 739ca42b855530e8293cbb25298e1d4f6ef03d15 lib/tool_shed/util/tool_util.py --- a/lib/tool_shed/util/tool_util.py +++ b/lib/tool_shed/util/tool_util.py @@ -5,15 +5,14 @@ import galaxy.tools from galaxy import util from galaxy.datatypes import checkers -from galaxy.tools import parameters from galaxy.util.expressions import ExpressionContext from galaxy.web.form_builder import SelectField -from galaxy.tools.actions.upload import UploadToolAction from tool_shed.util import basic_util log = logging.getLogger( __name__ ) + def build_shed_tool_conf_select_field( app ): """Build a SelectField whose options are the keys in app.toolbox.shed_tool_confs.""" options = [] @@ -30,6 +29,7 @@ select_field.add_option( option_tup[ 0 ], option_tup[ 1 ] ) return select_field + def build_tool_panel_section_select_field( app ): """Build a SelectField whose options are the sections of the current in-memory toolbox.""" options = [] @@ -41,6 +41,7 @@ select_field.add_option( option_tup[ 0 ], option_tup[ 1 ] ) return select_field + def copy_sample_file( app, filename, dest_path=None ): """ Copy xxx.sample to dest_path/xxx.sample and dest_path/xxx. The default value for dest_path @@ -61,6 +62,7 @@ if not os.path.exists( os.path.join( dest_path, copied_file ) ): shutil.copy( full_source_path, os.path.join( dest_path, copied_file ) ) + def copy_sample_files( app, sample_files, tool_path=None, sample_files_copied=None, dest_path=None ): """ Copy all appropriate files to dest_path in the local Galaxy environment that have not @@ -75,11 +77,12 @@ filename_sans_path = os.path.split( filename )[ 1 ] if filename_sans_path not in filenames_not_to_copy and filename not in sample_files_copied: if tool_path: - filename=os.path.join( tool_path, filename ) + filename = os.path.join( tool_path, filename ) # Attempt to ensure we're copying an appropriate file. if is_data_index_sample_file( filename ): copy_sample_file( app, filename, dest_path=dest_path ) + def generate_message_for_invalid_tools( app, invalid_file_tups, repository, metadata_dict, as_html=True, displaying_invalid_tool=False ): if as_html: @@ -124,6 +127,7 @@ message += "%s%s%s - %s%s" % ( bold_start, tool_file, bold_end, correction_msg, new_line ) return message + def get_headers( fname, sep, count=60, is_multi_byte=False ): """Returns a list with the first 'count' lines split by 'sep'.""" headers = [] @@ -137,6 +141,7 @@ break return headers + def get_tool_path_install_dir( partial_install_dir, shed_tool_conf_dict, tool_dict, config_elems ): for elem in config_elems: if elem.tag == 'tool': @@ -153,6 +158,7 @@ return tool_path, relative_install_dir return None, None + def handle_missing_index_file( app, tool_path, sample_files, repository_tools_tups, sample_files_copied ): """ Inspect each tool to see if it has any input parameters that are dynamically @@ -180,6 +186,7 @@ repository_tools_tups[ index ] = ( tup_path, guid, repository_tool ) return repository_tools_tups, sample_files_copied + def is_column_based( fname, sep='\t', skip=0, is_multi_byte=False ): """See if the file is column based with respect to a separator.""" headers = get_headers( fname, sep, is_multi_byte=is_multi_byte ) @@ -199,6 +206,7 @@ return False return True + def is_data_index_sample_file( file_path ): """ Attempt to determine if a .sample file is appropriate for copying to ~/tool-data when @@ -224,6 +232,7 @@ # Default to copying the file if none of the above are true. return True + def new_state( trans, tool, invalid=False ): """Create a new `DefaultToolState` for the received tool. Only inputs on the first page will be initialized.""" state = galaxy.tools.DefaultToolState() @@ -240,9 +249,10 @@ state.inputs[ input.name ] = [] return state + def panel_entry_per_tool( tool_section_dict ): # Return True if tool_section_dict looks like this. - # {<Tool guid> : + # {<Tool guid> : # [{ tool_config : <tool_config_file>, # id: <ToolSection id>, # version : <ToolSection version>, @@ -258,6 +268,7 @@ return True return False + def reload_upload_tools( app ): if hasattr( app, 'toolbox' ): app.toolbox.handle_datatypes_changed() https://bitbucket.org/galaxy/galaxy-central/commits/35d456f7edff/ Changeset: 35d456f7edff User: jmchilton Date: 2014-12-29 14:11:54+00:00 Summary: Method to abstract away ToolBox.shed_tool_confs from tool_util.py. Start a gradual process of referring to these as dynamic tool confs instead of shed tool confs. I would like like to make them more generally Galaxy-managed tool configurations and distinguish them from deployer-managed configurations - rather then having overtly ToolShed-only logic in ToolBox. Affected #: 2 files diff -r 739ca42b855530e8293cbb25298e1d4f6ef03d15 -r 35d456f7edffef9a8e4cd825503c3782f2be3617 lib/galaxy/tools/__init__.py --- a/lib/galaxy/tools/__init__.py +++ b/lib/galaxy/tools/__init__.py @@ -632,6 +632,16 @@ break return tool_version_select_field, tools, tool + def dynamic_conf_filenames( self, include_migrated_tool_conf=False ): + """ Return list of dynamic tool configuration filenames (shed_tools). + These must be used with various dynamic tool configuration update + operations (e.g. with update_shed_config). + """ + for dynamic_tool_conf_dict in self.shed_tool_confs: + dynamic_tool_conf_filename = dynamic_tool_conf_dict[ 'config_filename' ] + if include_migrated_tool_conf or (dynamic_tool_conf_filename != self.app.config.migrated_tools_config): + yield dynamic_tool_conf_filename + def build_tool_version_select_field( self, tools, tool_id, set_selected ): """Build a SelectField whose options are the ids for the received list of tools.""" options = [] diff -r 739ca42b855530e8293cbb25298e1d4f6ef03d15 -r 35d456f7edffef9a8e4cd825503c3782f2be3617 lib/tool_shed/util/tool_util.py --- a/lib/tool_shed/util/tool_util.py +++ b/lib/tool_shed/util/tool_util.py @@ -16,14 +16,12 @@ def build_shed_tool_conf_select_field( app ): """Build a SelectField whose options are the keys in app.toolbox.shed_tool_confs.""" options = [] - for shed_tool_conf_dict in app.toolbox.shed_tool_confs: - shed_tool_conf_filename = shed_tool_conf_dict[ 'config_filename' ] - if shed_tool_conf_filename != app.config.migrated_tools_config: - if shed_tool_conf_filename.startswith( './' ): - option_label = shed_tool_conf_filename.replace( './', '', 1 ) - else: - option_label = shed_tool_conf_filename - options.append( ( option_label, shed_tool_conf_filename ) ) + for dynamic_tool_conf_filename in app.toolbox.dynamic_conf_filenames(): + if dynamic_tool_conf_filename.startswith( './' ): + option_label = dynamic_tool_conf_filename.replace( './', '', 1 ) + else: + option_label = dynamic_tool_conf_filename + options.append( ( option_label, dynamic_tool_conf_filename ) ) select_field = SelectField( name='shed_tool_conf' ) for option_tup in options: select_field.add_option( option_tup[ 0 ], option_tup[ 1 ] ) https://bitbucket.org/galaxy/galaxy-central/commits/83dd43611918/ Changeset: 83dd43611918 User: jmchilton Date: 2014-12-29 14:11:54+00:00 Summary: Method to abstract away ToolBox.tool_panel from tool_util.py. Now only thing accessing ToolBox.tool_panel outside ToolBox itself is ToolPanelManager. Affected #: 2 files diff -r 35d456f7edffef9a8e4cd825503c3782f2be3617 -r 83dd436119182a688824218b0efda74e55e045c9 lib/galaxy/tools/__init__.py --- a/lib/galaxy/tools/__init__.py +++ b/lib/galaxy/tools/__init__.py @@ -1059,6 +1059,11 @@ status = 'done' return message, status + def get_sections( self ): + for k, v in self.tool_panel.items(): + if isinstance( v, ToolSection ): + yield (v.id, v.name) + def find_section_id( self, tool_panel_section_id ): """ Find the section ID referenced by the key or return '' indicating diff -r 35d456f7edffef9a8e4cd825503c3782f2be3617 -r 83dd436119182a688824218b0efda74e55e045c9 lib/tool_shed/util/tool_util.py --- a/lib/tool_shed/util/tool_util.py +++ b/lib/tool_shed/util/tool_util.py @@ -31,9 +31,8 @@ def build_tool_panel_section_select_field( app ): """Build a SelectField whose options are the sections of the current in-memory toolbox.""" options = [] - for k, v in app.toolbox.tool_panel.items(): - if isinstance( v, galaxy.tools.ToolSection ): - options.append( ( v.name, v.id ) ) + for section_id, section_name in app.toolbox.get_sections(): + options.append( ( section_name, section_id ) ) select_field = SelectField( name='tool_panel_section_id', display='radio' ) for option_tup in options: select_field.add_option( option_tup[ 0 ], option_tup[ 1 ] ) https://bitbucket.org/galaxy/galaxy-central/commits/6021c963ec92/ Changeset: 6021c963ec92 User: jmchilton Date: 2014-12-29 14:11:54+00:00 Summary: Reuse new ToolBox.get_section method in ToolPanelManager. This is a higher-level method and eliminates a couple more places where Tool Shed code is directly acceessing ToolBox.tool_panel. Affected #: 1 file diff -r 83dd436119182a688824218b0efda74e55e045c9 -r 6021c963ec925698353b134d070b403d2309609f lib/tool_shed/galaxy_install/tools/tool_panel_manager.py --- a/lib/tool_shed/galaxy_install/tools/tool_panel_manager.py +++ b/lib/tool_shed/galaxy_install/tools/tool_panel_manager.py @@ -325,8 +325,7 @@ tool_panel_section_id=section_id, new_tool_panel_section_label=new_tool_panel_section_label ) elif tool_panel_section_id: - tool_panel_section_key = str( tool_panel_section_id ) - tool_section = toolbox.tool_panel[ tool_panel_section_key ] + tool_panel_section_key, tool_section = toolbox.get_section( tool_panel_section_id) else: return None, None return tool_panel_section_key, tool_section @@ -495,8 +494,8 @@ # Remove the tool sub-element from the section element. config_elem.remove( tool_elem ) # Remove the tool from the section in the in-memory tool panel. - if section_key in toolbox.tool_panel: - tool_section = toolbox.tool_panel[ section_key ] + _, tool_section = toolbox.get_section( section_key ) + if tool_section: remove_from_panel( tool_elem, tool_section.elems, toolbox.integrated_tool_panel.get( section_key, {} ), section_key=section_key ) if len( config_elem ) < 1: # Keep a list of all empty section elements so they can be removed. https://bitbucket.org/galaxy/galaxy-central/commits/482b027099f5/ Changeset: 482b027099f5 User: jmchilton Date: 2014-12-29 14:11:54+00:00 Summary: Refactor logic for removing tools from panels into ToolBox. May seem sort of like Tool Shed specific stuff - but I think it can be reused elsewhere down the road. Regardless of this however as of this commit no external components directly access ToolBox.tool_panel or ToolBox.integrated_tool_panel - which I think is important - these should be internal details of the ToolBox (or newer modules it leverages). Affected #: 2 files diff -r 6021c963ec925698353b134d070b403d2309609f -r 482b027099f5aa0e15de928d093c6347a5297e18 lib/galaxy/tools/__init__.py --- a/lib/galaxy/tools/__init__.py +++ b/lib/galaxy/tools/__init__.py @@ -658,6 +658,55 @@ select_field.add_option( 'version %s' % option_tup[ 0 ], option_tup[ 1 ] ) return select_field + def remove_from_panel( self, tool_id, section_key='', remove_from_config=True ): + + def remove_from_dict( has_elems, integrated_has_elems ): + tool_key = 'tool_%s' % str( tool_id ) + available_tool_versions = self.get_loaded_tools_by_lineage( tool_id ) + if tool_key in has_elems: + if available_tool_versions: + available_tool_versions.reverse() + replacement_tool_key = None + replacement_tool_version = None + # Since we are going to remove the tool from the section, replace it with + # the newest loaded version of the tool. + for available_tool_version in available_tool_versions: + available_tool_section_id, available_tool_section_name = available_tool_version.get_panel_section() + # I suspect "available_tool_version.id in has_elems.keys()" doesn't + # belong in the following line or at least I don't understand what + # purpose it might serve. -John + if available_tool_version.id in has_elems.keys() or (available_tool_section_id == section_key): + replacement_tool_key = 'tool_%s' % str( available_tool_version.id ) + replacement_tool_version = available_tool_version + break + if replacement_tool_key and replacement_tool_version: + # Get the index of the tool_key in the tool_section. + for tool_panel_index, key in enumerate( has_elems.keys() ): + if key == tool_key: + break + # Remove the tool from the tool panel. + del has_elems[ tool_key ] + # Add the replacement tool at the same location in the tool panel. + has_elems.insert( tool_panel_index, + replacement_tool_key, + replacement_tool_version ) + else: + del has_elems[ tool_key ] + else: + del has_elems[ tool_key ] + if remove_from_config: + if hasattr( integrated_has_elems, "elems" ): + integrated_has_elems = integrated_has_elems.elems + if tool_key in integrated_has_elems: + del integrated_has_elems[ tool_key ] + + if section_key: + _, tool_section = self.get_section( section_key ) + if tool_section: + remove_from_dict( tool_section.elems, self.integrated_tool_panel.get( section_key, {} ) ) + else: + remove_from_dict( self.tool_panel, self.integrated_tool_panel ) + def load_tool_tag_set( self, elem, panel_dict, integrated_panel_dict, tool_path, load_panel_dict, guid=None, index=None ): try: path = elem.get( "file" ) diff -r 6021c963ec925698353b134d070b403d2309609f -r 482b027099f5aa0e15de928d093c6347a5297e18 lib/tool_shed/galaxy_install/tools/tool_panel_manager.py --- a/lib/tool_shed/galaxy_install/tools/tool_panel_manager.py +++ b/lib/tool_shed/galaxy_install/tools/tool_panel_manager.py @@ -426,50 +426,6 @@ def remove_guids( self, guids_to_remove, shed_tool_conf, uninstall ): toolbox = self.app.toolbox - - def remove_from_panel( tool_elem, has_elems, integrated_has_elems, section_key='' ): - # Hide tool from panel and promote next oldest version if - # available. - guid = tool_elem.get( 'guid' ) - tool_key = 'tool_%s' % str( guid ) - available_tool_versions = toolbox.get_loaded_tools_by_lineage( guid ) - if tool_key in has_elems: - if available_tool_versions: - available_tool_versions.reverse() - replacement_tool_key = None - replacement_tool_version = None - # Since we are going to remove the tool from the section, replace it with - # the newest loaded version of the tool. - for available_tool_version in available_tool_versions: - available_tool_section_id, available_tool_section_name = available_tool_version.get_panel_section() - # I suspect "available_tool_version.id in has_elems.keys()" doesn't - # belong in the following line or at least I don't understand what - # purpose it might serve. -John - if available_tool_version.id in has_elems.keys() or (available_tool_section_id == section_key): - replacement_tool_key = 'tool_%s' % str( available_tool_version.id ) - replacement_tool_version = available_tool_version - break - if replacement_tool_key and replacement_tool_version: - # Get the index of the tool_key in the tool_section. - for tool_panel_index, key in enumerate( has_elems.keys() ): - if key == tool_key: - break - # Remove the tool from the tool panel. - del has_elems[ tool_key ] - # Add the replacement tool at the same location in the tool panel. - has_elems.insert( tool_panel_index, - replacement_tool_key, - replacement_tool_version ) - else: - del has_elems[ tool_key ] - else: - del has_elems[ tool_key ] - if uninstall: - if hasattr( integrated_has_elems, "elems" ): - integrated_has_elems = integrated_has_elems.elems - if tool_key in integrated_has_elems: - del integrated_has_elems[ tool_key ] - # Remove the tools from the toolbox's tools_by_id dictionary. for guid_to_remove in guids_to_remove: # remove_from_tool_panel to false, will handling that logic below. @@ -494,17 +450,14 @@ # Remove the tool sub-element from the section element. config_elem.remove( tool_elem ) # Remove the tool from the section in the in-memory tool panel. - _, tool_section = toolbox.get_section( section_key ) - if tool_section: - remove_from_panel( tool_elem, tool_section.elems, toolbox.integrated_tool_panel.get( section_key, {} ), section_key=section_key ) + toolbox.remove_from_panel( tool_elem.get( "guid" ), section_key=section_key, remove_from_config=uninstall ) if len( config_elem ) < 1: # Keep a list of all empty section elements so they can be removed. config_elems_to_remove.append( config_elem ) elif config_elem.tag == 'tool': guid = config_elem.get( 'guid' ) if guid in guids_to_remove: - # get_panel_section return '' for un-sectioned tools. - remove_from_panel( config_elem, toolbox.tool_panel, toolbox.integrated_tool_panel, section_key='' ) + toolbox.remove_from_panel( guid, section_key='', remove_from_config=uninstall ) config_elems_to_remove.append( config_elem ) for config_elem in config_elems_to_remove: # Remove the element from the in-memory list of elements. https://bitbucket.org/galaxy/galaxy-central/commits/600d39d9bbac/ Changeset: 600d39d9bbac User: jmchilton Date: 2014-12-29 14:11:54+00:00 Summary: Update calls to ToolPanelManager.get_shed_tool_conf_dict for clarity... ... in other words to note where index in the resulting tuple is not actually used. Doing this to enable hiding this implementation detail (index) from components in subsequent commits. Affected #: 3 files diff -r 482b027099f5aa0e15de928d093c6347a5297e18 -r 600d39d9bbac52eb4321e65d16d1740a3f7c0871 lib/tool_shed/galaxy_install/install_manager.py --- a/lib/tool_shed/galaxy_install/install_manager.py +++ b/lib/tool_shed/galaxy_install/install_manager.py @@ -730,7 +730,7 @@ shed_tool_conf = install_options.get( 'shed_tool_conf', None ) if shed_tool_conf: # Get the tool_path setting. - index, shed_conf_dict = self.tpm.get_shed_tool_conf_dict( shed_tool_conf ) + _, shed_conf_dict = self.tpm.get_shed_tool_conf_dict( shed_tool_conf ) tool_path = shed_conf_dict[ 'tool_path' ] else: # Pick a semi-random shed-related tool panel configuration file and get the tool_path setting. diff -r 482b027099f5aa0e15de928d093c6347a5297e18 -r 600d39d9bbac52eb4321e65d16d1740a3f7c0871 lib/tool_shed/galaxy_install/metadata/installed_repository_metadata_manager.py --- a/lib/tool_shed/galaxy_install/metadata/installed_repository_metadata_manager.py +++ b/lib/tool_shed/galaxy_install/metadata/installed_repository_metadata_manager.py @@ -66,7 +66,7 @@ in the received tool shed repository metadata. """ repository_tools_tups = [] - index, shed_conf_dict = self.tpm.get_shed_tool_conf_dict( self.metadata_dict.get( 'shed_config_filename' ) ) + _, shed_conf_dict = self.tpm.get_shed_tool_conf_dict( self.metadata_dict.get( 'shed_config_filename' ) ) if 'tools' in self.metadata_dict: for tool_dict in self.metadata_dict[ 'tools' ]: load_relative_path = relative_path = tool_dict.get( 'tool_config', None ) diff -r 482b027099f5aa0e15de928d093c6347a5297e18 -r 600d39d9bbac52eb4321e65d16d1740a3f7c0871 lib/tool_shed/galaxy_install/tool_migration_manager.py --- a/lib/tool_shed/galaxy_install/tool_migration_manager.py +++ b/lib/tool_shed/galaxy_install/tool_migration_manager.py @@ -78,7 +78,7 @@ self.tool_shed_url = common_util.get_tool_shed_url_from_tool_shed_registry( self.app, defined_tool_shed_url ) self.tool_shed = common_util.remove_protocol_and_port_from_tool_shed_url( self.tool_shed_url ) self.repository_owner = common_util.REPOSITORY_OWNER - index, self.shed_config_dict = self.tpm.get_shed_tool_conf_dict( self.migrated_tools_config ) + _, self.shed_config_dict = self.tpm.get_shed_tool_conf_dict( self.migrated_tools_config ) # Since tool migration scripts can be executed any number of times, we need to # make sure the appropriate tools are defined in tool_conf.xml. If no tools # associated with the migration stage are defined, no repositories will be installed https://bitbucket.org/galaxy/galaxy-central/commits/7cee92f360c0/ Changeset: 7cee92f360c0 User: jmchilton Date: 2014-12-29 14:11:54+00:00 Summary: Update ToolBox.update_shed_config to use config_filename instead of index. Seems a more robust index, simplifies ToolPanelManager, etc.... Affected #: 2 files diff -r 600d39d9bbac52eb4321e65d16d1740a3f7c0871 -r 7cee92f360c0ca15d0f3f443c2e80a629d5eab1b lib/galaxy/tools/__init__.py --- a/lib/galaxy/tools/__init__.py +++ b/lib/galaxy/tools/__init__.py @@ -265,13 +265,15 @@ return shed_config_dict return default - def update_shed_config( self, shed_conf_index, shed_conf, integrated_panel_changes=True ): + def update_shed_config( self, shed_conf, integrated_panel_changes=True ): """ Update the in-memory descriptions of tools and write out the changes to integrated tool panel unless we are just deactivating a tool (since that doesn't affect that file). """ app = self.app - self.shed_tool_confs[ shed_conf_index ] = shed_conf + for index, my_shed_tool_conf in enumerate( self.shed_tool_confs ): + if shed_conf['config_filename'] == my_shed_tool_conf['config_filename']: + self.shed_tool_confs[ index ] = shed_conf if integrated_panel_changes and app.config.update_integrated_tool_panel: # Write the current in-memory version of the integrated_tool_panel.xml file to disk. self.write_integrated_tool_panel_config_file() diff -r 600d39d9bbac52eb4321e65d16d1740a3f7c0871 -r 7cee92f360c0ca15d0f3f443c2e80a629d5eab1b lib/tool_shed/galaxy_install/tools/tool_panel_manager.py --- a/lib/tool_shed/galaxy_install/tools/tool_panel_manager.py +++ b/lib/tool_shed/galaxy_install/tools/tool_panel_manager.py @@ -40,7 +40,7 @@ shed_tool_conf, tool_panel_dict, new_install=True ): """A tool shed repository is being installed or updated so handle tool panel alterations accordingly.""" # We need to change the in-memory version and the file system version of the shed_tool_conf file. - index, shed_tool_conf_dict = self.get_shed_tool_conf_dict( shed_tool_conf ) + _, shed_tool_conf_dict = self.get_shed_tool_conf_dict( shed_tool_conf ) tool_path = shed_tool_conf_dict[ 'tool_path' ] # Generate the list of ElementTree Element objects for each section or tool. elem_list = self.generate_tool_panel_elem_list( repository_name, @@ -66,7 +66,7 @@ ) # Replace the old list of in-memory config_elems with the new list for this shed_tool_conf_dict. shed_tool_conf_dict[ 'config_elems' ] = config_elems - self.app.toolbox.update_shed_config( index, shed_tool_conf_dict ) + self.app.toolbox.update_shed_config( shed_tool_conf_dict ) def config_elems_to_xml_file( self, config_elems, config_filename, tool_path ): """ @@ -430,7 +430,7 @@ for guid_to_remove in guids_to_remove: # remove_from_tool_panel to false, will handling that logic below. toolbox.remove_tool_by_id( guid_to_remove, remove_from_panel=False ) - index, shed_tool_conf_dict = self.get_shed_tool_conf_dict( shed_tool_conf ) + _, shed_tool_conf_dict = self.get_shed_tool_conf_dict( shed_tool_conf ) if uninstall: # Remove from the shed_tool_conf file on disk. self.remove_from_shed_tool_config( shed_tool_conf_dict, guids_to_remove ) @@ -464,4 +464,4 @@ config_elems.remove( config_elem ) # Update the config_elems of the in-memory shed_tool_conf_dict. shed_tool_conf_dict[ 'config_elems' ] = config_elems - toolbox.update_shed_config( index, shed_tool_conf_dict, integrated_panel_changes=uninstall ) + toolbox.update_shed_config( shed_tool_conf_dict, integrated_panel_changes=uninstall ) https://bitbucket.org/galaxy/galaxy-central/commits/22db1ba32b58/ Changeset: 22db1ba32b58 User: jmchilton Date: 2014-12-29 14:11:54+00:00 Summary: Simplifications for ToolPanelManager.get_shed_tool_conf_dict. None of the components using this method are using the index any more - so eliminate it. Also introduce a new ToolBox method to hide the details of ToolBox.shed_tool_confs from this method. Affected #: 6 files diff -r 7cee92f360c0ca15d0f3f443c2e80a629d5eab1b -r 22db1ba32b58093b68662db3f9f7f513e20c2bdf lib/galaxy/tools/__init__.py --- a/lib/galaxy/tools/__init__.py +++ b/lib/galaxy/tools/__init__.py @@ -634,15 +634,21 @@ break return tool_version_select_field, tools, tool + def dynamic_confs( self, include_migrated_tool_conf=False ): + confs = [] + for dynamic_tool_conf_dict in self.shed_tool_confs: + dynamic_tool_conf_filename = dynamic_tool_conf_dict[ 'config_filename' ] + if include_migrated_tool_conf or (dynamic_tool_conf_filename != self.app.config.migrated_tools_config): + confs.append( dynamic_tool_conf_dict ) + return confs + def dynamic_conf_filenames( self, include_migrated_tool_conf=False ): """ Return list of dynamic tool configuration filenames (shed_tools). These must be used with various dynamic tool configuration update operations (e.g. with update_shed_config). """ - for dynamic_tool_conf_dict in self.shed_tool_confs: - dynamic_tool_conf_filename = dynamic_tool_conf_dict[ 'config_filename' ] - if include_migrated_tool_conf or (dynamic_tool_conf_filename != self.app.config.migrated_tools_config): - yield dynamic_tool_conf_filename + for dynamic_tool_conf_dict in self.dynamic_confs( include_migrated_tool_conf=include_migrated_tool_conf ): + yield dynamic_tool_conf_dict[ 'config_filename' ] def build_tool_version_select_field( self, tools, tool_id, set_selected ): """Build a SelectField whose options are the ids for the received list of tools.""" diff -r 7cee92f360c0ca15d0f3f443c2e80a629d5eab1b -r 22db1ba32b58093b68662db3f9f7f513e20c2bdf lib/tool_shed/galaxy_install/install_manager.py --- a/lib/tool_shed/galaxy_install/install_manager.py +++ b/lib/tool_shed/galaxy_install/install_manager.py @@ -730,7 +730,7 @@ shed_tool_conf = install_options.get( 'shed_tool_conf', None ) if shed_tool_conf: # Get the tool_path setting. - _, shed_conf_dict = self.tpm.get_shed_tool_conf_dict( shed_tool_conf ) + shed_conf_dict = self.tpm.get_shed_tool_conf_dict( shed_tool_conf ) tool_path = shed_conf_dict[ 'tool_path' ] else: # Pick a semi-random shed-related tool panel configuration file and get the tool_path setting. diff -r 7cee92f360c0ca15d0f3f443c2e80a629d5eab1b -r 22db1ba32b58093b68662db3f9f7f513e20c2bdf lib/tool_shed/galaxy_install/metadata/installed_repository_metadata_manager.py --- a/lib/tool_shed/galaxy_install/metadata/installed_repository_metadata_manager.py +++ b/lib/tool_shed/galaxy_install/metadata/installed_repository_metadata_manager.py @@ -66,7 +66,7 @@ in the received tool shed repository metadata. """ repository_tools_tups = [] - _, shed_conf_dict = self.tpm.get_shed_tool_conf_dict( self.metadata_dict.get( 'shed_config_filename' ) ) + shed_conf_dict = self.tpm.get_shed_tool_conf_dict( self.metadata_dict.get( 'shed_config_filename' ) ) if 'tools' in self.metadata_dict: for tool_dict in self.metadata_dict[ 'tools' ]: load_relative_path = relative_path = tool_dict.get( 'tool_config', None ) diff -r 7cee92f360c0ca15d0f3f443c2e80a629d5eab1b -r 22db1ba32b58093b68662db3f9f7f513e20c2bdf lib/tool_shed/galaxy_install/tool_migration_manager.py --- a/lib/tool_shed/galaxy_install/tool_migration_manager.py +++ b/lib/tool_shed/galaxy_install/tool_migration_manager.py @@ -78,7 +78,7 @@ self.tool_shed_url = common_util.get_tool_shed_url_from_tool_shed_registry( self.app, defined_tool_shed_url ) self.tool_shed = common_util.remove_protocol_and_port_from_tool_shed_url( self.tool_shed_url ) self.repository_owner = common_util.REPOSITORY_OWNER - _, self.shed_config_dict = self.tpm.get_shed_tool_conf_dict( self.migrated_tools_config ) + self.shed_config_dict = self.tpm.get_shed_tool_conf_dict( self.migrated_tools_config ) # Since tool migration scripts can be executed any number of times, we need to # make sure the appropriate tools are defined in tool_conf.xml. If no tools # associated with the migration stage are defined, no repositories will be installed diff -r 7cee92f360c0ca15d0f3f443c2e80a629d5eab1b -r 22db1ba32b58093b68662db3f9f7f513e20c2bdf lib/tool_shed/galaxy_install/tools/tool_panel_manager.py --- a/lib/tool_shed/galaxy_install/tools/tool_panel_manager.py +++ b/lib/tool_shed/galaxy_install/tools/tool_panel_manager.py @@ -40,7 +40,7 @@ shed_tool_conf, tool_panel_dict, new_install=True ): """A tool shed repository is being installed or updated so handle tool panel alterations accordingly.""" # We need to change the in-memory version and the file system version of the shed_tool_conf file. - _, shed_tool_conf_dict = self.get_shed_tool_conf_dict( shed_tool_conf ) + shed_tool_conf_dict = self.get_shed_tool_conf_dict( shed_tool_conf ) tool_path = shed_tool_conf_dict[ 'tool_path' ] # Generate the list of ElementTree Element objects for each section or tool. elem_list = self.generate_tool_panel_elem_list( repository_name, @@ -304,13 +304,13 @@ Return the in-memory version of the shed_tool_conf file, which is stored in the config_elems entry in the shed_tool_conf_dict associated with the file. """ - for index, shed_tool_conf_dict in enumerate( self.app.toolbox.shed_tool_confs ): + for shed_tool_conf_dict in self.app.toolbox.dynamic_confs( include_migrated_tool_conf=True ): if shed_tool_conf == shed_tool_conf_dict[ 'config_filename' ]: - return index, shed_tool_conf_dict + return shed_tool_conf_dict else: file_name = basic_util.strip_path( shed_tool_conf_dict[ 'config_filename' ] ) if shed_tool_conf == file_name: - return index, shed_tool_conf_dict + return shed_tool_conf_dict def handle_tool_panel_section( self, toolbox, tool_panel_section_id=None, new_tool_panel_section_label=None ): """Return a ToolSection object retrieved from the current in-memory tool_panel.""" @@ -430,7 +430,7 @@ for guid_to_remove in guids_to_remove: # remove_from_tool_panel to false, will handling that logic below. toolbox.remove_tool_by_id( guid_to_remove, remove_from_panel=False ) - _, shed_tool_conf_dict = self.get_shed_tool_conf_dict( shed_tool_conf ) + shed_tool_conf_dict = self.get_shed_tool_conf_dict( shed_tool_conf ) if uninstall: # Remove from the shed_tool_conf file on disk. self.remove_from_shed_tool_config( shed_tool_conf_dict, guids_to_remove ) diff -r 7cee92f360c0ca15d0f3f443c2e80a629d5eab1b -r 22db1ba32b58093b68662db3f9f7f513e20c2bdf test/unit/tools_support.py --- a/test/unit/tools_support.py +++ b/test/unit/tools_support.py @@ -100,6 +100,7 @@ admin_users="mary(a)example.com", len_file_path=os.path.join( 'tool-data', 'shared', 'ucsc', 'chrom' ), builds_file_path=os.path.join( 'tool-data', 'shared', 'ucsc', 'builds.txt.sample' ), + migrated_tools_config=os.path.join(test_directory, "migrated_tools_conf.xml"), ) # Setup some attributes for downstream extension by specific tests. https://bitbucket.org/galaxy/galaxy-central/commits/68255ebe3f54/ Changeset: 68255ebe3f54 User: jmchilton Date: 2014-12-29 14:11:54+00:00 Summary: Reuse new ToolBox.dynamic_confs method other places... ... to shield ToolBox.shed_tool_confs from other components including Tool Shed install models, shed_util_common.py, and install_manager.py. After this commit - shed_tool_confs is only accessed directly by the ToolBox itself. Affected #: 3 files diff -r 22db1ba32b58093b68662db3f9f7f513e20c2bdf -r 68255ebe3f54500aa26db814c41f67f382731462 lib/galaxy/model/tool_shed_install/__init__.py --- a/lib/galaxy/model/tool_shed_install/__init__.py +++ b/lib/galaxy/model/tool_shed_install/__init__.py @@ -99,7 +99,7 @@ if not self.shed_config_filename: self.guess_shed_config( app, default=default ) if self.shed_config_filename: - for shed_tool_conf_dict in app.toolbox.shed_tool_confs: + for shed_tool_conf_dict in app.toolbox.dynamic_confs( include_migrated_tool_conf=True ): if self.shed_config_filename == shed_tool_conf_dict[ 'config_filename' ]: return shed_tool_conf_dict return default @@ -118,7 +118,7 @@ metadata = self.metadata or {} for tool in metadata.get( 'tools', [] ): tool_ids.append( tool.get( 'guid' ) ) - for shed_tool_conf_dict in app.toolbox.shed_tool_confs: + for shed_tool_conf_dict in app.toolbox.dynamic_confs( include_migrated_tool_conf=True ): name = shed_tool_conf_dict[ 'config_filename' ] for elem in shed_tool_conf_dict[ 'config_elems' ]: if elem.tag == 'tool': @@ -137,7 +137,7 @@ if self.includes_datatypes: # We need to search by file paths here, which is less desirable. tool_shed = common_util.remove_protocol_and_port_from_tool_shed_url( self.tool_shed ) - for shed_tool_conf_dict in app.toolbox.shed_tool_confs: + for shed_tool_conf_dict in app.toolbox.dynamic_confs( include_migrated_tool_conf=True ): tool_path = shed_tool_conf_dict[ 'tool_path' ] relative_path = os.path.join( tool_path, tool_shed, 'repos', self.owner, self.name, self.installed_changeset_revision ) if os.path.exists( relative_path ): @@ -284,7 +284,7 @@ def repo_path( self, app ): tool_shed = common_util.remove_protocol_and_port_from_tool_shed_url( self.tool_shed ) - for index, shed_tool_conf_dict in enumerate( app.toolbox.shed_tool_confs ): + for shed_tool_conf_dict in app.toolbox.dynamic_confs( include_migrated_tool_conf=True ): tool_path = shed_tool_conf_dict[ 'tool_path' ] relative_path = os.path.join( tool_path, tool_shed, 'repos', self.owner, self.name, self.installed_changeset_revision ) if os.path.exists( relative_path ): diff -r 22db1ba32b58093b68662db3f9f7f513e20c2bdf -r 68255ebe3f54500aa26db814c41f67f382731462 lib/tool_shed/galaxy_install/install_manager.py --- a/lib/tool_shed/galaxy_install/install_manager.py +++ b/lib/tool_shed/galaxy_install/install_manager.py @@ -733,15 +733,13 @@ shed_conf_dict = self.tpm.get_shed_tool_conf_dict( shed_tool_conf ) tool_path = shed_conf_dict[ 'tool_path' ] else: - # Pick a semi-random shed-related tool panel configuration file and get the tool_path setting. - for shed_config_dict in self.app.toolbox.shed_tool_confs: - # Don't use migrated_tools_conf.xml. - if shed_config_dict[ 'config_filename' ] != self.app.config.migrated_tools_config: - break + # Don't use migrated_tools_conf.xml. + try: + shed_config_dict = self.app.toolbox.dynamic_confs( include_migrated_tool_conf=False )[ 0 ] + except IndexError: + raise exceptions.RequestParameterMissingException( "Missing required parameter 'shed_tool_conf'." ) shed_tool_conf = shed_config_dict[ 'config_filename' ] tool_path = shed_config_dict[ 'tool_path' ] - if not shed_tool_conf: - raise exceptions.RequestParameterMissingException( "Missing required parameter 'shed_tool_conf'." ) tool_panel_section_id = self.app.toolbox.find_section_id( install_options.get( 'tool_panel_section_id', '' ) ) # Build the dictionary of information necessary for creating tool_shed_repository database records # for each repository being installed. diff -r 22db1ba32b58093b68662db3f9f7f513e20c2bdf -r 68255ebe3f54500aa26db814c41f67f382731462 lib/tool_shed/util/shed_util_common.py --- a/lib/tool_shed/util/shed_util_common.py +++ b/lib/tool_shed/util/shed_util_common.py @@ -722,7 +722,7 @@ shed_config_dict = repository.get_shed_config_dict( app ) if not shed_config_dict: # Just pick a semi-random shed config. - for shed_config_dict in app.toolbox.shed_tool_confs: + for shed_config_dict in app.toolbox.dynamic_confs( include_migrated_tool_conf=True ): if ( repository.dist_to_shed and shed_config_dict[ 'config_filename' ] == app.config.migrated_tools_config ) \ or ( not repository.dist_to_shed and shed_config_dict[ 'config_filename' ] != app.config.migrated_tools_config ): break @@ -736,7 +736,7 @@ Return the tool_path config setting for the received shed_tool_conf file by searching the tool box's in-memory list of shed_tool_confs for the dictionary whose config_filename key has a value matching the received shed_tool_conf. """ - for shed_tool_conf_dict in app.toolbox.shed_tool_confs: + for shed_tool_conf_dict in app.toolbox.dynamic_confs( include_migrated_tool_conf=True ): config_filename = shed_tool_conf_dict[ 'config_filename' ] if config_filename == shed_tool_conf: return shed_tool_conf_dict[ 'tool_path' ] @@ -1036,15 +1036,7 @@ def have_shed_tool_conf_for_install( app ): - if not app.toolbox.shed_tool_confs: - return False - migrated_tools_conf_path, migrated_tools_conf_name = os.path.split( app.config.migrated_tools_config ) - for shed_tool_conf_dict in app.toolbox.shed_tool_confs: - shed_tool_conf = shed_tool_conf_dict[ 'config_filename' ] - shed_tool_conf_path, shed_tool_conf_name = os.path.split( shed_tool_conf ) - if shed_tool_conf_name != migrated_tools_conf_name: - return True - return False + return bool( app.toolbox.dynamic_confs( include_migrated_tool_conf=False ) ) def is_tool_shed_client( app ): Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: jmchilton: Bugfix for 44dfa6f and typos in unit tests that caused it to go undetected.
by commits-noreply＠bitbucket.org 28 Dec '14

28 Dec '14

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/bfd679408c84/ Changeset: bfd679408c84 User: jmchilton Date: 2014-12-29 04:25:56+00:00 Summary: Bugfix for 44dfa6f and typos in unit tests that caused it to go undetected. Affected #: 3 files diff -r 1912dec92b728b974094456be8eb6d84d34a7ca6 -r bfd679408c84a4c6f1d2b1a09eec1dab9c2d212b lib/tool_shed/galaxy_install/tools/tool_panel_manager.py --- a/lib/tool_shed/galaxy_install/tools/tool_panel_manager.py +++ b/lib/tool_shed/galaxy_install/tools/tool_panel_manager.py @@ -475,8 +475,10 @@ else: del has_elems[ tool_key ] if uninstall: - if tool_key in toolbox.integrated_tool_panel: - del toolbox.integrated_tool_panel[ tool_key ] + if hasattr( integrated_has_elems, "elems" ): + integrated_has_elems = integrated_has_elems.elems + if tool_key in integrated_has_elems: + del integrated_has_elems[ tool_key ] # Remove the tools from the toolbox's tools_by_id dictionary. for guid_to_remove in guids_to_remove: diff -r 1912dec92b728b974094456be8eb6d84d34a7ca6 -r bfd679408c84a4c6f1d2b1a09eec1dab9c2d212b test/unit/tool_shed_unit_tests/test_tool_panel_manager.py --- a/test/unit/tool_shed_unit_tests/test_tool_panel_manager.py +++ b/test/unit/tool_shed_unit_tests/test_tool_panel_manager.py @@ -67,10 +67,10 @@ self._setup_two_versions_remove_one( section=True, uninstall=False ) self._verify_version_2_removed_from_panel( ) # Still in tool conf since not uninstalled only deactivated... - assert "github.com/galaxyproect/example/test_tool/0.2" in open(os.path.join(self.test_directory, "tool_conf.xml"), "r").read() + assert "github.com/galaxyproject/example/test_tool/0.2" in open(os.path.join(self.test_directory, "tool_conf.xml"), "r").read() self._verify_tool_confs() - self._remove_guids( ["github.com/galaxyproect/example/test_tool/0.1"], uninstall=False ) + self._remove_guids( ["github.com/galaxyproject/example/test_tool/0.1"], uninstall=False ) # Now no versions of this tool are returned by toolbox. all_versions = self.toolbox.get_tool( "test_tool", get_all_versions=True ) @@ -84,24 +84,25 @@ self._setup_two_versions_remove_one( section=True, uninstall=True ) self._verify_version_2_removed_from_panel( ) # Not in tool conf because it was uninstalled. - assert "github.com/galaxyproect/example/test_tool/0.2" not in open(os.path.join(self.test_directory, "tool_conf.xml"), "r").read() + assert "github.com/galaxyproject/example/test_tool/0.2" not in open(os.path.join(self.test_directory, "tool_conf.xml"), "r").read() + assert "tool_github.com/galaxyproject/example/test_tool/0.2" not in self.toolbox.integrated_tool_panel["tid"].elems self._verify_tool_confs() def test_deactivate_outside_section( self ): self._setup_two_versions_remove_one( section=False, uninstall=False ) self._verify_version_2_removed_from_panel( section=False ) # Still in tool conf since not uninstalled only deactivated... - assert "github.com/galaxyproect/example/test_tool/0.2" in open(os.path.join(self.test_directory, "tool_conf.xml"), "r").read() + assert "github.com/galaxyproject/example/test_tool/0.2" in open(os.path.join(self.test_directory, "tool_conf.xml"), "r").read() self._verify_tool_confs() def test_uninstall_outside_section( self ): self._setup_two_versions_remove_one( section=False, uninstall=True ) self._verify_version_2_removed_from_panel( section=False ) # Still in tool conf since not uninstalled only deactivated... - assert "github.com/galaxyproect/example/test_tool/0.2" not in open(os.path.join(self.test_directory, "tool_conf.xml"), "r").read() + assert "github.com/galaxyproject/example/test_tool/0.2" not in open(os.path.join(self.test_directory, "tool_conf.xml"), "r").read() self._verify_tool_confs() - self._remove_guids( ["github.com/galaxyproect/example/test_tool/0.1"], uninstall=True ) + self._remove_guids( ["github.com/galaxyproject/example/test_tool/0.1"], uninstall=True ) # Now no versions of this tool are returned by toolbox. all_versions = self.toolbox.get_tool( "test_tool", get_all_versions=True ) @@ -116,7 +117,7 @@ self._setup_two_versions_in_config( section=True ) self._setup_two_versions() self.toolbox - self._remove_guids( ["github.com/galaxyproect/example/test_tool/0.2"], uninstall=uninstall ) + self._remove_guids( ["github.com/galaxyproject/example/test_tool/0.2"], uninstall=uninstall ) def _verify_version_2_removed_from_panel( self, section=True ): # Check that test_tool now only has one version... @@ -127,12 +128,12 @@ if section: section = self.toolbox.tool_panel["tid"] assert len(section.elems) == 1 - assert section.elems.values()[0].id == "github.com/galaxyproect/example/test_tool/0.1" + assert section.elems.values()[0].id == "github.com/galaxyproject/example/test_tool/0.1" - assert "github.com/galaxyproect/example/test_tool/0.2" not in self.toolbox.integrated_tool_panel["tid"].elems + assert "github.com/galaxyproject/example/test_tool/0.2" not in self.toolbox.integrated_tool_panel["tid"].elems else: - self.toolbox.tool_panel.values()[0].id == "github.com/galaxyproect/example/test_tool/0.1" - assert "github.com/galaxyproect/example/test_tool/0.2" not in self.toolbox.integrated_tool_panel + self.toolbox.tool_panel.values()[0].id == "github.com/galaxyproject/example/test_tool/0.1" + assert "github.com/galaxyproject/example/test_tool/0.2" not in self.toolbox.integrated_tool_panel def _remove_guids( self, guids, uninstall, shed_tool_conf="tool_conf.xml" ): self.tpm.remove_guids( diff -r 1912dec92b728b974094456be8eb6d84d34a7ca6 -r bfd679408c84a4c6f1d2b1a09eec1dab9c2d212b test/unit/tools/test_toolbox.py --- a/test/unit/tools/test_toolbox.py +++ b/test/unit/tools/test_toolbox.py @@ -10,12 +10,12 @@ CONFIG_TEST_TOOL_VERSION_TEMPLATE = string.Template( - """ <tool file="tool.xml" guid="github.com/galaxyproect/example/test_tool/0.${version}"> + """ <tool file="tool.xml" guid="github.com/galaxyproject/example/test_tool/0.${version}"><tool_shed>github.com</tool_shed><repository_name>example</repository_name><repository_owner>galaxyproject</repository_owner><installed_changeset_revision>${version}</installed_changeset_revision> - <id>github.com/galaxyproect/example/test_tool/0.${version}</id> + <id>github.com/galaxyproject/example/test_tool/0.${version}</id><version>0.${version}</version></tool> """ @@ -72,14 +72,14 @@ def _setup_two_versions( self ): repository1 = self._repo_install( changeset="1" ) version1 = tool_shed_install.ToolVersion() - version1.tool_id = "github.com/galaxyproect/example/test_tool/0.1" + version1.tool_id = "github.com/galaxyproject/example/test_tool/0.1" version1.repository = repository1 self.app.install_model.context.add( version1 ) self.app.install_model.context.flush( ) repository2 = self._repo_install( changeset="2" ) version2 = tool_shed_install.ToolVersion() - version2.tool_id = "github.com/galaxyproect/example/test_tool/0.2" + version2.tool_id = "github.com/galaxyproject/example/test_tool/0.2" version2.repository = repository2 self.app.install_model.context.add( version2 ) @@ -184,7 +184,7 @@ # Assert only newer version of the tool loaded into the panel. section = self.toolbox.tool_panel["tid"] assert len(section.elems) == 1 - assert section.elems.values()[0].id == "github.com/galaxyproect/example/test_tool/0.2" + assert section.elems.values()[0].id == "github.com/galaxyproject/example/test_tool/0.2" def test_group_tools_out_of_section( self ): self._init_tool() @@ -212,10 +212,10 @@ self._init_tool() self._setup_two_versions_in_config( ) self._setup_two_versions() - assert self.toolbox.get_tool_id( "test_tool" ) == "github.com/galaxyproect/example/test_tool/0.1" - assert self.toolbox.get_tool_id( "github.com/galaxyproect/example/test_tool/0.1" ) == "github.com/galaxyproect/example/test_tool/0.1" - assert self.toolbox.get_tool_id( "github.com/galaxyproect/example/test_tool/0.2" ) == "github.com/galaxyproect/example/test_tool/0.2" - assert self.toolbox.get_tool_id( "github.com/galaxyproect/example/test_tool/0.3" ) is None + assert self.toolbox.get_tool_id( "test_tool" ) == "github.com/galaxyproject/example/test_tool/0.1" + assert self.toolbox.get_tool_id( "github.com/galaxyproject/example/test_tool/0.1" ) == "github.com/galaxyproject/example/test_tool/0.1" + assert self.toolbox.get_tool_id( "github.com/galaxyproject/example/test_tool/0.2" ) == "github.com/galaxyproject/example/test_tool/0.2" + assert self.toolbox.get_tool_id( "github.com/galaxyproject/example/test_tool/0.3" ) is None def test_tool_dir( self ): self._init_tool() @@ -285,14 +285,14 @@ # Verify lineage_ids on both tools is correctly ordered. for version in ["0.1", "0.2"]: - guid = "github.com/galaxyproect/example/test_tool/" + version + guid = "github.com/galaxyproject/example/test_tool/" + version lineage_ids = self.toolbox.get_tool( guid ).lineage_ids - assert lineage_ids[ 0 ] == "github.com/galaxyproect/example/test_tool/0.1" - assert lineage_ids[ 1 ] == "github.com/galaxyproect/example/test_tool/0.2" + assert lineage_ids[ 0 ] == "github.com/galaxyproject/example/test_tool/0.1" + assert lineage_ids[ 1 ] == "github.com/galaxyproject/example/test_tool/0.2" # Test tool_version attribute. - assert self.toolbox.get_tool( "test_tool", tool_version="0.1" ).guid == "github.com/galaxyproect/example/test_tool/0.1" - assert self.toolbox.get_tool( "test_tool", tool_version="0.2" ).guid == "github.com/galaxyproect/example/test_tool/0.2" + assert self.toolbox.get_tool( "test_tool", tool_version="0.1" ).guid == "github.com/galaxyproject/example/test_tool/0.1" + assert self.toolbox.get_tool( "test_tool", tool_version="0.2" ).guid == "github.com/galaxyproject/example/test_tool/0.2" def __remove_itp( self ): os.remove( os.path) Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: 6 new changesets
by commits-noreply＠bitbucket.org 28 Dec '14

28 Dec '14

6 new commits in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/50062f6e9b47/ Changeset: 50062f6e9b47 User: jmchilton Date: 2014-12-28 16:23:44+00:00 Summary: PEP-8 fixes for a few Tool Shed install components. - lib/tool_shed/galaxy_install/install_manager.py - lib/tool_shed/galaxy_install/repository_dependencies/repository_dependency_manager.py - lib/tool_shed/galaxy_install/repair_repository_manager.py Affected #: 3 files diff -r 97f46da40bb3767c6cf0d440b1d2d58e06edf5e6 -r 50062f6e9b47a36dad57393292611632908fe79a lib/tool_shed/galaxy_install/install_manager.py --- a/lib/tool_shed/galaxy_install/install_manager.py +++ b/lib/tool_shed/galaxy_install/install_manager.py @@ -42,11 +42,11 @@ class InstallToolDependencyManager( object ): - + def __init__( self, app ): self.app = app self.install_model = self.app.install_model - self.INSTALL_ACTIONS = [ 'download_binary', 'download_by_url', 'download_file', + self.INSTALL_ACTIONS = [ 'download_binary', 'download_by_url', 'download_file', 'setup_perl_environment', 'setup_python_environment', 'setup_r_environment', 'setup_ruby_environment', 'shell_command' ] @@ -142,9 +142,9 @@ # Since there was an installation error, update the tool dependency status to Error. The remove_installation_path option must # be left False here. error_message = '%s\n%s' % ( self.format_traceback(), str( e ) ) - tool_dependency = tool_dependency_util.handle_tool_dependency_installation_error( self.app, - tool_dependency, - error_message, + tool_dependency = tool_dependency_util.handle_tool_dependency_installation_error( self.app, + tool_dependency, + error_message, remove_installation_path=False ) tool_dependency = self.mark_tool_dependency_installed( tool_dependency ) return tool_dependency @@ -208,9 +208,9 @@ tool_dependency_db_records=tool_dependencies ) if ( tool_dependency.type == 'package' and proceed_with_install ): try: - tool_dependency = self.install_package( elem, - tool_shed_repository, - tool_dependencies=tool_dependencies, + tool_dependency = self.install_package( elem, + tool_shed_repository, + tool_dependencies=tool_dependencies, from_tool_migration_manager=from_tool_migration_manager ) except Exception, e: error_message = "Error installing tool dependency %s version %s: %s" % \ @@ -220,9 +220,9 @@ # Since there was an installation error, update the tool dependency status to Error. The # remove_installation_path option must be left False here. tool_dependency = \ - tool_dependency_util.handle_tool_dependency_installation_error( self.app, - tool_dependency, - error_message, + tool_dependency_util.handle_tool_dependency_installation_error( self.app, + tool_dependency, + error_message, remove_installation_path=False ) if tool_dependency and tool_dependency.status in [ self.install_model.ToolDependency.installation_status.INSTALLED, self.install_model.ToolDependency.installation_status.ERROR ]: @@ -236,7 +236,7 @@ def install_via_fabric( self, tool_shed_repository, tool_dependency, install_dir, package_name=None, custom_fabfile_path=None, actions_elem=None, action_elem=None, **kwd ): """ - Parse a tool_dependency.xml file's <actions> tag set to gather information for installation using + Parse a tool_dependency.xml file's <actions> tag set to gather information for installation using self.install_and_build_package(). The use of fabric is being eliminated, so some of these functions may need to be renamed at some point. """ @@ -363,8 +363,8 @@ else: # Process the next matching <actions> tag, or any defined <actions> tags that do not # contain platform dependent recipes. - log.debug( 'Error downloading binary for tool dependency %s version %s: %s' % \ - ( str( package_name ), str( package_version ), str( tool_dependency.error_message ) ) ) + log.debug( 'Error downloading binary for tool dependency %s version %s: %s' % + ( str( package_name ), str( package_version ), str( tool_dependency.error_message ) ) ) else: if actions_elem.tag == 'actions': # We've reached an <actions> tag that defines the recipe for installing and compiling from @@ -380,7 +380,7 @@ if removed: can_install_from_source = True else: - log.debug( 'Error removing old files from installation directory %s: %s' % \ + log.debug( 'Error removing old files from installation directory %s: %s' % ( str( installation_directory, str( error_message ) ) ) ) else: can_install_from_source = True @@ -389,7 +389,7 @@ if can_install_from_source: # We now know that binary installation was not successful, so proceed with the <actions> # tag set that defines the recipe to install and compile from source. - log.debug( 'Proceeding with install and compile recipe for tool dependency %s.' % \ + log.debug( 'Proceeding with install and compile recipe for tool dependency %s.' % str( tool_dependency.name ) ) tool_dependency = self.install_via_fabric( tool_shed_repository, tool_dependency, @@ -398,7 +398,7 @@ actions_elem=actions_elem, action_elem=None ) if actions_elem.tag == 'action' and \ - tool_dependency.status != self.install_model.ToolDependency.installation_status.ERROR: + tool_dependency.status != self.install_model.ToolDependency.installation_status.ERROR: # If the tool dependency is not in an error state, perform any final actions that have been # defined within the actions_group tag set, but outside of an <actions> tag, which defines # the recipe for installing and compiling from source. @@ -419,17 +419,17 @@ actions_elem=actions_elems, action_elem=None ) if tool_dependency.status != self.install_model.ToolDependency.installation_status.ERROR: - log.debug( 'Tool dependency %s version %s has been installed in %s.' % \ - ( str( package_name ), str( package_version ), str( install_dir ) ) ) + log.debug( 'Tool dependency %s version %s has been installed in %s.' % + ( str( package_name ), str( package_version ), str( install_dir ) ) ) return tool_dependency def mark_tool_dependency_installed( self, tool_dependency ): if tool_dependency.status not in [ self.install_model.ToolDependency.installation_status.ERROR, self.install_model.ToolDependency.installation_status.INSTALLED ]: - log.debug( 'Changing status for tool dependency %s from %s to %s.' % \ - ( str( tool_dependency.name ), - str( tool_dependency.status ), - str( self.install_model.ToolDependency.installation_status.INSTALLED ) ) ) + log.debug( 'Changing status for tool dependency %s from %s to %s.' % + ( str( tool_dependency.name ), + str( tool_dependency.status ), + str( self.install_model.ToolDependency.installation_status.INSTALLED ) ) ) status = self.install_model.ToolDependency.installation_status.INSTALLED tool_dependency = tool_dependency_util.set_tool_dependency_attributes( self.app, tool_dependency=tool_dependency, @@ -531,10 +531,10 @@ self.install_model.context.add( tool_shed_repository ) self.install_model.context.flush() if 'tool_dependencies' in irmm_metadata_dict and not reinstalling: - tool_dependencies = tool_dependency_util.create_tool_dependency_objects( self.app, - tool_shed_repository, - relative_install_dir, - set_status=True ) + tool_dependency_util.create_tool_dependency_objects( self.app, + tool_shed_repository, + relative_install_dir, + set_status=True ) if 'sample_files' in irmm_metadata_dict: sample_files = irmm_metadata_dict.get( 'sample_files', [] ) tool_index_sample_files = tdtm.get_tool_index_sample_files( sample_files ) @@ -579,12 +579,12 @@ new_install=True ) if 'data_manager' in irmm_metadata_dict: dmh = data_manager.DataManagerHandler( self.app ) - new_data_managers = dmh.install_data_managers( self.app.config.shed_data_manager_config_file, - irmm_metadata_dict, - shed_config_dict, - relative_install_dir, - tool_shed_repository, - repository_tools_tups ) + dmh.install_data_managers( self.app.config.shed_data_manager_config_file, + irmm_metadata_dict, + shed_config_dict, + relative_install_dir, + tool_shed_repository, + repository_tools_tups ) if 'datatypes' in irmm_metadata_dict: tool_shed_repository.status = self.install_model.ToolShedRepository.installation_status.LOADING_PROPRIETARY_DATATYPES if not tool_shed_repository.includes_datatypes: @@ -656,13 +656,9 @@ tool_shed_url = installation_dict[ 'tool_shed_url' ] # Handle contained tools. if includes_tools_for_display_in_tool_panel and ( new_tool_panel_section_label or tool_panel_section_id ): - tool_panel_section_key, tool_section = \ - self.tpm.handle_tool_panel_section( self.app.toolbox, - tool_panel_section_id=tool_panel_section_id, - new_tool_panel_section_label=new_tool_panel_section_label ) - else: - tool_panel_section_key = None - tool_section = None + self.tpm.handle_tool_panel_section( self.app.toolbox, + tool_panel_section_id=tool_panel_section_id, + new_tool_panel_section_label=new_tool_panel_section_label ) encoded_repository_ids = [ self.app.security.encode_id( tsr.id ) for tsr in created_or_updated_tool_shed_repositories ] new_kwd = dict( includes_tools=includes_tools, includes_tools_for_display_in_tool_panel=includes_tools_for_display_in_tool_panel, @@ -679,7 +675,7 @@ tool_shed_repository_ids=encoded_repository_ids, tool_shed_url=tool_shed_url ) encoded_kwd = encoding_util.tool_shed_encode( new_kwd ) - tsr_ids = [ r.id for r in created_or_updated_tool_shed_repositories ] + tsr_ids = [ r.id for r in created_or_updated_tool_shed_repositories ] tool_shed_repositories = [] for tsr_id in tsr_ids: tsr = self.install_model.context.query( self.install_model.ToolShedRepository ).get( tsr_id ) @@ -895,10 +891,10 @@ # Get the tool_dependencies.xml file from the repository. tool_dependencies_config = hg_util.get_config_from_disk( 'tool_dependencies.xml', install_dir ) itdm = InstallToolDependencyManager( self.app ) - installed_tool_dependencies = itdm.install_specified_tool_dependencies( tool_shed_repository=tool_shed_repository, - tool_dependencies_config=tool_dependencies_config, - tool_dependencies=tool_shed_repository.tool_dependencies, - from_tool_migration_manager=False ) + itdm.install_specified_tool_dependencies( tool_shed_repository=tool_shed_repository, + tool_dependencies_config=tool_dependencies_config, + tool_dependencies=tool_shed_repository.tool_dependencies, + from_tool_migration_manager=False ) basic_util.remove_dir( work_dir ) self.update_tool_shed_repository_status( tool_shed_repository, self.install_model.ToolShedRepository.installation_status.INSTALLED ) diff -r 97f46da40bb3767c6cf0d440b1d2d58e06edf5e6 -r 50062f6e9b47a36dad57393292611632908fe79a lib/tool_shed/galaxy_install/repair_repository_manager.py --- a/lib/tool_shed/galaxy_install/repair_repository_manager.py +++ b/lib/tool_shed/galaxy_install/repair_repository_manager.py @@ -65,7 +65,7 @@ repair_dict = {} irm = install_manager.InstallRepositoryManager( self.app ) # Get a dictionary of all repositories upon which the contents of the current repository_metadata - #record depend. + # record depend. repository_dependencies_dict = rdim.get_repository_dependencies_for_installed_tool_shed_repository( self.app, repository ) if repository_dependencies_dict: @@ -142,14 +142,14 @@ return repo_info_dict, tool_panel_section_key def repair_tool_shed_repository( self, repository, repo_info_dict ): - + def add_repair_dict_entry( repository_name, error_message ): if repository_name in repair_dict: repair_dict[ repository_name ].append( error_message ) else: repair_dict[ repository_name ] = [ error_message ] return repair_dict - + metadata = repository.metadata repair_dict = {} tpm = tool_panel_manager.ToolPanelManager( self.app ) diff -r 97f46da40bb3767c6cf0d440b1d2d58e06edf5e6 -r 50062f6e9b47a36dad57393292611632908fe79a lib/tool_shed/galaxy_install/repository_dependencies/repository_dependency_manager.py --- a/lib/tool_shed/galaxy_install/repository_dependencies/repository_dependency_manager.py +++ b/lib/tool_shed/galaxy_install/repository_dependencies/repository_dependency_manager.py @@ -38,13 +38,13 @@ for repo_info_dict in repo_info_dicts: for name, repo_info_tuple in repo_info_dict.items(): description, \ - repository_clone_url, \ - changeset_revision, \ - ctx_rev, \ - repository_owner, \ - repository_dependencies, \ - tool_dependencies = \ - suc.get_repo_info_tuple_contents( repo_info_tuple ) + repository_clone_url, \ + changeset_revision, \ + ctx_rev, \ + repository_owner, \ + repository_dependencies, \ + tool_dependencies = \ + suc.get_repo_info_tuple_contents( repo_info_tuple ) if repository_dependencies: for key, val in repository_dependencies.items(): if key in [ 'root_key', 'description' ]: @@ -71,12 +71,12 @@ for repository_dependency_components_list in val: required_repository = None rd_toolshed, \ - rd_name, \ - rd_owner, \ - rd_changeset_revision, \ - rd_prior_installation_required, \ - rd_only_if_compiling_contained_td = \ - common_util.parse_repository_dependency_tuple( repository_dependency_components_list ) + rd_name, \ + rd_owner, \ + rd_changeset_revision, \ + rd_prior_installation_required, \ + rd_only_if_compiling_contained_td = \ + common_util.parse_repository_dependency_tuple( repository_dependency_components_list ) # Get the the tool_shed_repository defined by rd_name, rd_owner and rd_changeset_revision. This # is the repository that will be required by the current d_repository. # TODO: Check tool_shed_repository.tool_shed as well when repository dependencies across tool sheds is supported. @@ -101,10 +101,10 @@ repository_dependency = self.get_repository_dependency_by_repository_id( install_model, required_repository.id ) if not repository_dependency: - log.debug( 'Creating new repository_dependency record for installed revision %s of repository: %s owned by %s.' % \ - ( str( required_repository.installed_changeset_revision ), - str( required_repository.name ), - str( required_repository.owner ) ) ) + log.debug( 'Creating new repository_dependency record for installed revision %s of repository: %s owned by %s.' % + ( str( required_repository.installed_changeset_revision ), + str( required_repository.name ), + str( required_repository.owner ) ) ) repository_dependency = install_model.RepositoryDependency( tool_shed_repository_id=required_repository.id ) install_model.context.add( repository_dependency ) install_model.context.flush() @@ -159,13 +159,13 @@ for name, repo_info_tuple in repo_info_dict.items(): can_update_db_record = False description, \ - repository_clone_url, \ - changeset_revision, \ - ctx_rev, \ - repository_owner, \ - repository_dependencies, \ - tool_dependencies = \ - suc.get_repo_info_tuple_contents( repo_info_tuple ) + repository_clone_url, \ + changeset_revision, \ + ctx_rev, \ + repository_owner, \ + repository_dependencies, \ + tool_dependencies = \ + suc.get_repo_info_tuple_contents( repo_info_tuple ) # See if the repository has an existing record in the database. repository_db_record, installed_changeset_revision = \ suc.repository_was_previously_installed( self.app, tool_shed_url, name, repo_info_tuple, from_tip=False ) @@ -190,8 +190,6 @@ # record in the database. name = repository_db_record.name installed_changeset_revision = repository_db_record.installed_changeset_revision - metadata_dict = repository_db_record.metadata - dist_to_shed = repository_db_record.dist_to_shed can_update_db_record = True elif repository_db_record.status in [ install_model.ToolShedRepository.installation_status.DEACTIVATED ]: # The current tool shed repository is deactivated, so updating its database record @@ -209,8 +207,6 @@ else: # No record exists in the database for the repository currently being processed. installed_changeset_revision = changeset_revision - metadata_dict = {} - dist_to_shed = False can_update_db_record = True if can_update_db_record: # The database record for the tool shed repository currently being processed can be updated. @@ -348,13 +344,10 @@ # a dependency of the dependent repository's contained tool dependency, and only if that # tool dependency requires compilation. # For backward compatibility to the 12/20/12 Galaxy release. - prior_installation_required = 'False' only_if_compiling_contained_td = 'False' if len( components_list ) == 4: - prior_installation_required = 'False' only_if_compiling_contained_td = 'False' elif len( components_list ) == 5: - prior_installation_required = components_list[ 4 ] only_if_compiling_contained_td = 'False' if not asbool( only_if_compiling_contained_td ): if components_list not in required_repository_tups: https://bitbucket.org/galaxy/galaxy-central/commits/032eefd299cb/ Changeset: 032eefd299cb User: jmchilton Date: 2014-12-28 16:23:44+00:00 Summary: Tweak two ToolShed modules to clarify they don't use ToolSection... ... objects (directly anyway). I would like to refactor all the tool shed components to not deal directly with ToolSection objects but instead just pass around IDs. Affected #: 2 files diff -r 50062f6e9b47a36dad57393292611632908fe79a -r 032eefd299cbfd103ea85103e64147d436068fb5 lib/tool_shed/galaxy_install/repair_repository_manager.py --- a/lib/tool_shed/galaxy_install/repair_repository_manager.py +++ b/lib/tool_shed/galaxy_install/repair_repository_manager.py @@ -123,7 +123,7 @@ tool_panel_section_name = section_dict[ 'name' ] if tool_panel_section_id: tpm = tool_panel_manager.ToolPanelManager( self.app ) - tool_panel_section_key, tool_panel_section = \ + tool_panel_section_key, _ = \ tpm.get_or_create_tool_section( self.app.toolbox, tool_panel_section_id=tool_panel_section_id, new_tool_panel_section_label=tool_panel_section_name ) @@ -165,7 +165,7 @@ suc.get_tool_panel_config_tool_path_install_dir( self.app, repository ) # Reset the repository attributes to the New state for installation. if metadata: - tool_section, tool_panel_section_key = \ + _, tool_panel_section_key = \ tpm.handle_tool_panel_selection( self.app.toolbox, metadata, no_changes_checked=True, diff -r 50062f6e9b47a36dad57393292611632908fe79a -r 032eefd299cbfd103ea85103e64147d436068fb5 lib/tool_shed/galaxy_install/repository_dependencies/repository_dependency_manager.py --- a/lib/tool_shed/galaxy_install/repository_dependencies/repository_dependency_manager.py +++ b/lib/tool_shed/galaxy_install/repository_dependencies/repository_dependency_manager.py @@ -213,7 +213,7 @@ # Get the repository metadata to see where it was previously located in the tool panel. tpm = tool_panel_manager.ToolPanelManager( self.app ) if repository_db_record and repository_db_record.metadata: - tool_section, tool_panel_section_key = \ + _, tool_panel_section_key = \ tpm.handle_tool_panel_selection( toolbox=self.app.toolbox, metadata=repository_db_record.metadata, no_changes_checked=no_changes_checked, @@ -221,7 +221,7 @@ new_tool_panel_section_label=new_tool_panel_section_label ) else: # We're installing a new tool shed repository that does not yet have a database record. - tool_panel_section_key, tool_section = \ + tool_panel_section_key, _ = \ tpm.handle_tool_panel_section( self.app.toolbox, tool_panel_section_id=tool_panel_section_id, new_tool_panel_section_label=new_tool_panel_section_label ) https://bitbucket.org/galaxy/galaxy-central/commits/6d4909f34123/ Changeset: 6d4909f34123 User: jmchilton Date: 2014-12-28 16:23:44+00:00 Summary: More centralization of code for registering tools in ToolBox. Affected #: 1 file diff -r 032eefd299cbfd103ea85103e64147d436068fb5 -r 6d4909f34123fe53f1497dd2ea27a611039ae5c0 lib/galaxy/tools/__init__.py --- a/lib/galaxy/tools/__init__.py +++ b/lib/galaxy/tools/__init__.py @@ -679,7 +679,7 @@ # changed, so the tool should always be reloaded here. We used to # only load the tool if it was not found in self.tools_by_id, but # performing that check did not enable this scenario. - self.tools_by_id[ tool.id ] = tool + self.register_tool( tool ) if load_panel_dict: self.__add_tool_to_tool_panel( tool, panel_dict, section=isinstance( panel_dict, ToolSection ) ) https://bitbucket.org/galaxy/galaxy-central/commits/b499a560826b/ Changeset: b499a560826b User: jmchilton Date: 2014-12-28 16:23:44+00:00 Summary: Eliminate one reference to ToolBox.tool_panel in install_manager.py. Affected #: 2 files diff -r 6d4909f34123fe53f1497dd2ea27a611039ae5c0 -r b499a560826b2d3a3cfc16dc44b99dafa7c3802e lib/galaxy/tools/__init__.py --- a/lib/galaxy/tools/__init__.py +++ b/lib/galaxy/tools/__init__.py @@ -1021,6 +1021,24 @@ status = 'done' return message, status + def find_section_id( self, tool_panel_section_id ): + """ + Find the section ID referenced by the key or return '' indicating + no such section id. + """ + if not tool_panel_section_id: + tool_panel_section_id = '' + else: + if tool_panel_section_id not in self.tool_panel: + # Hack introduced without comment in a29d54619813d5da992b897557162a360b8d610c- + # not sure why it is needed. + fixed_tool_panel_section_id = 'section_%s' % tool_panel_section_id + if fixed_tool_panel_section_id in self.tool_panel: + tool_panel_section_id = fixed_tool_panel_section_id + else: + tool_panel_section_id = '' + return tool_panel_section_id + def load_workflow( self, workflow_id ): """ Return an instance of 'Workflow' identified by `id`, diff -r 6d4909f34123fe53f1497dd2ea27a611039ae5c0 -r b499a560826b2d3a3cfc16dc44b99dafa7c3802e lib/tool_shed/galaxy_install/install_manager.py --- a/lib/tool_shed/galaxy_install/install_manager.py +++ b/lib/tool_shed/galaxy_install/install_manager.py @@ -742,16 +742,7 @@ tool_path = shed_config_dict[ 'tool_path' ] if not shed_tool_conf: raise exceptions.RequestParameterMissingException( "Missing required parameter 'shed_tool_conf'." ) - tool_panel_section_id = install_options.get( 'tool_panel_section_id', '' ) - if tool_panel_section_id not in [ None, '' ]: - if tool_panel_section_id not in self.app.toolbox.tool_panel: - fixed_tool_panel_section_id = 'section_%s' % tool_panel_section_id - if fixed_tool_panel_section_id in self.app.toolbox.tool_panel: - tool_panel_section_id = fixed_tool_panel_section_id - else: - tool_panel_section_id = '' - else: - tool_panel_section_id = '' + tool_panel_section_id = self.app.toolbox.find_section_id( install_options.get( 'tool_panel_section_id', '' ) ) # Build the dictionary of information necessary for creating tool_shed_repository database records # for each repository being installed. installation_dict = dict( install_repository_dependencies=install_repository_dependencies, https://bitbucket.org/galaxy/galaxy-central/commits/54b778976b7b/ Changeset: 54b778976b7b User: jmchilton Date: 2014-12-28 16:23:44+00:00 Summary: Simplify shed-y logic in load_tool_tag_set. Drops cycolomatic complexity from 14 to 10 for that method. Affected #: 2 files diff -r b499a560826b2d3a3cfc16dc44b99dafa7c3802e -r 54b778976b7bb297160955170fda6ff1efd2d62a lib/galaxy/tools/__init__.py --- a/lib/galaxy/tools/__init__.py +++ b/lib/galaxy/tools/__init__.py @@ -559,6 +559,23 @@ def tools( self ): return self.tools_by_id.iteritems() + def __elem_to_tool_shed_repository( self, elem ): + # The tool is contained in an installed tool shed repository, so load + # the tool only if the repository has not been marked deleted. + tool_shed = elem.find( "tool_shed" ).text + repository_name = elem.find( "repository_name" ).text + repository_owner = elem.find( "repository_owner" ).text + installed_changeset_revision_elem = elem.find( "installed_changeset_revision" ) + if installed_changeset_revision_elem is None: + # Backward compatibility issue - the tag used to be named 'changeset_revision'. + installed_changeset_revision_elem = elem.find( "changeset_revision" ) + installed_changeset_revision = installed_changeset_revision_elem.text + tool_shed_repository = self.__get_tool_shed_repository( tool_shed, + repository_name, + repository_owner, + installed_changeset_revision ) + return tool_shed_repository + def __get_tool_shed_repository( self, tool_shed, name, owner, installed_changeset_revision ): # We store only the port, if one exists, in the database. tool_shed = common_util.remove_protocol_from_tool_shed_url( tool_shed ) @@ -613,42 +630,29 @@ try: path = elem.get( "file" ) repository_id = None - if guid is None: - tool_shed_repository = None - can_load_into_panel_dict = True - else: + + tool_shed_repository = None + can_load_into_panel_dict = True + if guid is not None: # The tool is contained in an installed tool shed repository, so load # the tool only if the repository has not been marked deleted. - tool_shed = elem.find( "tool_shed" ).text - repository_name = elem.find( "repository_name" ).text - repository_owner = elem.find( "repository_owner" ).text - installed_changeset_revision_elem = elem.find( "installed_changeset_revision" ) - if installed_changeset_revision_elem is None: - # Backward compatibility issue - the tag used to be named 'changeset_revision'. - installed_changeset_revision_elem = elem.find( "changeset_revision" ) - installed_changeset_revision = installed_changeset_revision_elem.text - tool_shed_repository = self.__get_tool_shed_repository( tool_shed, - repository_name, - repository_owner, - installed_changeset_revision ) + tool_shed_repository = self.__elem_to_tool_shed_repository( elem ) if tool_shed_repository: # Only load tools if the repository is not deactivated or uninstalled. can_load_into_panel_dict = not tool_shed_repository.deleted repository_id = self.app.security.encode_id( tool_shed_repository.id ) - else: - # If there is not yet a tool_shed_repository record, we're in the process of installing - # a new repository, so any included tools can be loaded into the tool panel. - can_load_into_panel_dict = True + # Else there is not yet a tool_shed_repository record, we're in the process of installing + # a new repository, so any included tools can be loaded into the tool panel. tool = self.load_tool( os.path.join( tool_path, path ), guid=guid, repository_id=repository_id ) if string_as_bool(elem.get( 'hidden', False )): tool.hidden = True key = 'tool_%s' % str( tool.id ) if can_load_into_panel_dict: if guid is not None: - tool.tool_shed = tool_shed - tool.repository_name = repository_name - tool.repository_owner = repository_owner - tool.installed_changeset_revision = installed_changeset_revision + tool.tool_shed = tool_shed_repository.tool_shed + tool.repository_name = tool_shed_repository.name + tool.repository_owner = tool_shed_repository.owner + tool.installed_changeset_revision = tool_shed_repository.installed_changeset_revision tool.guid = guid tool.version = elem.find( "version" ).text # Make sure the tool has a tool_version. diff -r b499a560826b2d3a3cfc16dc44b99dafa7c3802e -r 54b778976b7bb297160955170fda6ff1efd2d62a test/unit/tools/test_toolbox.py --- a/test/unit/tools/test_toolbox.py +++ b/test/unit/tools/test_toolbox.py @@ -136,6 +136,27 @@ assert toolbox.get_tool( "test_tool" ) is not None assert toolbox.get_tool( "not_a_test_tool" ) is None + def test_tool_shed_properties( self ): + self._init_tool() + self._setup_two_versions_in_config( section=False ) + self._setup_two_versions() + + test_tool = self.toolbox.get_tool( "test_tool" ) + assert test_tool.tool_shed == "github.com" + assert test_tool.repository_owner == "galaxyproject" + assert test_tool.repository_name == "example" + assert test_tool.installed_changeset_revision == "1" + + def test_tool_shed_properties_only_on_installed_tools( self ): + self._init_tool() + self._add_config( """<toolbox><tool file="tool.xml" /></toolbox>""" ) + toolbox = self.toolbox + test_tool = toolbox.get_tool( "test_tool" ) + assert test_tool.tool_shed is None + assert test_tool.repository_name is None + assert test_tool.repository_owner is None + assert test_tool.installed_changeset_revision is None + def test_load_file_in_section( self ): self._init_tool() self._add_config( """<toolbox><section id="t" name="test"><tool file="tool.xml" /></section></toolbox>""" ) https://bitbucket.org/galaxy/galaxy-central/commits/1912dec92b72/ Changeset: 1912dec92b72 User: jmchilton Date: 2014-12-28 16:23:44+00:00 Summary: De-duplication for loading elements into the tool panel. Create one public method load_item on ToolBox that should be used externally instead of load_tool_tag_set, load_section_tag_set, etc.... This reduces the duplication within the toolbox and between the toolbox and the Tool Shed's ToolPanelManager. Probably more importantly it also is another step down the road toward hiding implementation details such as toolbox.tool_panel and toolbox.intergrated_tool_panel from the ToolPanelManager. Already had unit test coverage for ToolPanelManager stuff here - but added test coverage for loading labels, workflows, and tool directories. Affected #: 3 files diff -r 54b778976b7bb297160955170fda6ff1efd2d62a -r 1912dec92b728b974094456be8eb6d84d34a7ca6 lib/galaxy/tools/__init__.py --- a/lib/galaxy/tools/__init__.py +++ b/lib/galaxy/tools/__init__.py @@ -222,16 +222,14 @@ self.index += 1 if parsing_shed_tool_conf: config_elems.append( elem ) - if elem.tag == 'tool': - self.load_tool_tag_set( elem, self.tool_panel, self.integrated_tool_panel, tool_path, load_panel_dict, guid=elem.get( 'guid' ), index=index ) - elif elem.tag == 'workflow': - self.load_workflow_tag_set( elem, self.tool_panel, self.integrated_tool_panel, load_panel_dict, index=index ) - elif elem.tag == 'section': - self.load_section_tag_set( elem, tool_path, load_panel_dict, index=index ) - elif elem.tag == 'label': - self.load_label_tag_set( elem, self.tool_panel, self.integrated_tool_panel, load_panel_dict, index=index ) - elif elem.tag == 'tool_dir': - self.load_tooldir_tag_set( elem, self.tool_panel, self.integrated_tool_panel, tool_path, load_panel_dict ) + self.load_item( + elem, + tool_path=tool_path, + load_panel_dict=load_panel_dict, + guid=elem.get( 'guid' ), + index=index, + internal=True + ) if parsing_shed_tool_conf: shed_tool_conf_dict = dict( config_filename=config_filename, @@ -239,6 +237,28 @@ config_elems=config_elems ) self.shed_tool_confs.append( shed_tool_conf_dict ) + def load_item( self, elem, tool_path, panel_dict=None, integrated_panel_dict=None, load_panel_dict=True, guid=None, index=None, internal=False ): + item_type = elem.tag + if item_type not in ['tool', 'section'] and not internal: + # External calls from tool shed code cannot load labels or tool + # directories. + return + + if panel_dict is None: + panel_dict = self.tool_panel + if integrated_panel_dict is None: + integrated_panel_dict = self.integrated_tool_panel + if item_type == 'tool': + self.load_tool_tag_set( elem, panel_dict=panel_dict, integrated_panel_dict=integrated_panel_dict, tool_path=tool_path, load_panel_dict=load_panel_dict, guid=guid, index=index ) + elif item_type == 'workflow': + self.load_workflow_tag_set( elem, panel_dict=panel_dict, integrated_panel_dict=integrated_panel_dict, load_panel_dict=load_panel_dict, index=index ) + elif item_type == 'section': + self.load_section_tag_set( elem, tool_path=tool_path, load_panel_dict=load_panel_dict, index=index ) + elif item_type == 'label': + self.load_label_tag_set( elem, panel_dict=panel_dict, integrated_panel_dict=integrated_panel_dict, load_panel_dict=load_panel_dict, index=index ) + elif item_type == 'tool_dir': + self.load_tooldir_tag_set( elem, panel_dict, tool_path, integrated_panel_dict, load_panel_dict=load_panel_dict ) + def get_shed_config_dict_by_filename( self, filename, default=None ): for shed_config_dict in self.shed_tool_confs: if shed_config_dict[ 'config_filename' ] == filename: @@ -729,14 +749,16 @@ integrated_section = ToolSection( elem ) integrated_elems = integrated_section.elems for sub_index, sub_elem in enumerate( elem ): - if sub_elem.tag == 'tool': - self.load_tool_tag_set( sub_elem, elems, integrated_elems, tool_path, load_panel_dict, guid=sub_elem.get( 'guid' ), index=sub_index ) - elif sub_elem.tag == 'workflow': - self.load_workflow_tag_set( sub_elem, elems, integrated_elems, load_panel_dict, index=sub_index ) - elif sub_elem.tag == 'label': - self.load_label_tag_set( sub_elem, elems, integrated_elems, load_panel_dict, index=sub_index ) - elif sub_elem.tag == 'tool_dir': - self.load_tooldir_tag_set( sub_elem, elems, tool_path, integrated_elems, load_panel_dict ) + self.load_item( + sub_elem, + tool_path=tool_path, + panel_dict=elems, + integrated_panel_dict=integrated_elems, + load_panel_dict=load_panel_dict, + guid=sub_elem.get( 'guid' ), + index=sub_index, + internal=True, + ) if load_panel_dict: self.tool_panel[ key ] = section # Always load sections into the integrated_tool_panel. diff -r 54b778976b7bb297160955170fda6ff1efd2d62a -r 1912dec92b728b974094456be8eb6d84d34a7ca6 lib/tool_shed/galaxy_install/tools/tool_panel_manager.py --- a/lib/tool_shed/galaxy_install/tools/tool_panel_manager.py +++ b/lib/tool_shed/galaxy_install/tools/tool_panel_manager.py @@ -58,22 +58,12 @@ # Add the new elements to the in-memory list of config_elems. config_elems.append( config_elem ) # Load the tools into the in-memory tool panel. - if config_elem.tag == 'section': - self.app.toolbox.load_section_tag_set( config_elem, tool_path, load_panel_dict=True ) - elif config_elem.tag == 'workflow': - self.app.toolbox.load_workflow_tag_set( config_elem, - self.app.toolbox.tool_panel, - self.app.toolbox.integrated_tool_panel, - load_panel_dict=True ) - elif config_elem.tag == 'tool': - self.app.toolbox.load_tool_tag_set( - config_elem, - self.app.toolbox.tool_panel, - self.app.toolbox.integrated_tool_panel, - tool_path, - load_panel_dict=True, - guid=config_elem.get( 'guid' ) - ) + self.app.toolbox.load_item( + config_elem, + tool_path=tool_path, + load_panel_dict=True, + guid=config_elem.get( 'guid' ), + ) # Replace the old list of in-memory config_elems with the new list for this shed_tool_conf_dict. shed_tool_conf_dict[ 'config_elems' ] = config_elems self.app.toolbox.update_shed_config( index, shed_tool_conf_dict ) diff -r 54b778976b7bb297160955170fda6ff1efd2d62a -r 1912dec92b728b974094456be8eb6d84d34a7ca6 test/unit/tools/test_toolbox.py --- a/test/unit/tools/test_toolbox.py +++ b/test/unit/tools/test_toolbox.py @@ -3,6 +3,7 @@ import unittest from galaxy.tools import ToolBox +from galaxy import model from galaxy.model import tool_shed_install from galaxy.model.tool_shed_install import mapping import tools_support @@ -216,6 +217,67 @@ assert self.toolbox.get_tool_id( "github.com/galaxyproect/example/test_tool/0.2" ) == "github.com/galaxyproect/example/test_tool/0.2" assert self.toolbox.get_tool_id( "github.com/galaxyproect/example/test_tool/0.3" ) is None + def test_tool_dir( self ): + self._init_tool() + self._add_config( """<toolbox><tool_dir dir="%s" /></toolbox>""" % self.test_directory ) + + toolbox = self.toolbox + assert toolbox.get_tool( "test_tool" ) is not None + + def test_workflow_in_panel( self ): + stored_workflow = self.__test_workflow() + encoded_id = self.app.security.encode_id( stored_workflow.id ) + self._add_config( """<toolbox><workflow id="%s" /></toolbox>""" % encoded_id ) + assert len( self.toolbox.tool_panel ) == 1 + panel_workflow = self.toolbox.tool_panel.values()[ 0 ] + assert panel_workflow == stored_workflow.latest_workflow + # TODO: test to_dict with workflows + + def test_workflow_in_section( self ): + stored_workflow = self.__test_workflow() + encoded_id = self.app.security.encode_id( stored_workflow.id ) + self._add_config( """<toolbox><section id="tid" name="TID"><workflow id="%s" /></section></toolbox>""" % encoded_id ) + assert len( self.toolbox.tool_panel ) == 1 + section = self.toolbox.tool_panel[ 'tid' ] + assert len( section.elems ) == 1 + panel_workflow = section.elems.values()[ 0 ] + assert panel_workflow == stored_workflow.latest_workflow + + def test_label_in_panel( self ): + self._add_config( """<toolbox><label id="lab1" text="Label 1" /><label id="lab2" text="Label 2" /></toolbox>""" ) + assert len( self.toolbox.tool_panel ) == 2 + self.__check_test_labels( self.toolbox.tool_panel ) + + def test_label_in_section( self ): + self._add_config( """<toolbox><section id="tid" name="TID"><label id="lab1" text="Label 1" /><label id="lab2" text="Label 2" /></section></toolbox>""" ) + assert len( self.toolbox.tool_panel ) == 1 + section = self.toolbox.tool_panel[ 'tid' ] + self.__check_test_labels( section.elems ) + + def __check_test_labels( self, panel_dict ): + assert panel_dict.keys() == ["label_lab1", "label_lab2"] + label1 = panel_dict.values()[ 0 ] + assert label1.id == "lab1" + assert label1.text == "Label 1" + + label2 = panel_dict[ "label_lab2" ] + assert label2.id == "lab2" + assert label2.text == "Label 2" + + def __test_workflow( self ): + stored_workflow = model.StoredWorkflow() + workflow = model.Workflow() + workflow.stored_workflow = stored_workflow + stored_workflow.latest_workflow = workflow + user = model.User() + user.email = "test(a)example.com" + user.password = "passw0rD1" + stored_workflow.user = user + self.app.model.context.add( workflow ) + self.app.model.context.add( stored_workflow ) + self.app.model.context.flush() + return stored_workflow + def __verify_two_test_tools( self ): # Assert tool versions of the tool with simple id 'test_tool' all_versions = self.toolbox.get_tool( "test_tool", get_all_versions=True ) Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: jmchilton: Fix remaining tool shed test cases for new escaped values.
by commits-noreply＠bitbucket.org 27 Dec '14

27 Dec '14

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/97f46da40bb3/ Changeset: 97f46da40bb3 User: jmchilton Date: 2014-12-28 04:46:22+00:00 Summary: Fix remaining tool shed test cases for new escaped values. I think in this case the values on the page are correct now - but the test framework needs to be updated to handle the escaped value. Affected #: 1 file diff -r 461a6a9379001959b5b3259f90a2b9471ce391ed -r 97f46da40bb3767c6cf0d440b1d2d58e06edf5e6 test/tool_shed/base/twilltestcase.py --- a/test/tool_shed/base/twilltestcase.py +++ b/test/tool_shed/base/twilltestcase.py @@ -430,6 +430,8 @@ url = '/admin_toolshed/manage_repository?id=%s' % self.security.encode_id( installed_repository.id ) self.visit_galaxy_url( url ) strings_displayed.append( str( installed_repository.installed_changeset_revision ) ) + # Every place Galaxy's XXXX tool appears in attribute - need to quote. + strings_displayed = map( lambda x: x.replace("'", "'"), strings_displayed ) self.check_for_strings( strings_displayed, strings_not_displayed ) def display_installed_workflow_image( self, repository, workflow_name, strings_displayed=[], strings_not_displayed=[] ): Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: 2 new changesets
by commits-noreply＠bitbucket.org 27 Dec '14

27 Dec '14

2 new commits in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/86241a27f4d8/ Changeset: 86241a27f4d8 User: jmchilton Date: 2014-12-27 22:49:16+00:00 Summary: Add --no_cleanup option to tests. Affected #: 1 file diff -r 1ed5a63aae0a5c8091483658c579d9d762a4c3fb -r 86241a27f4d8d7065fb7908a0e687d6b11a062eb run_tests.sh --- a/run_tests.sh +++ b/run_tests.sh @@ -21,6 +21,12 @@ '${0##*/} -unit testscriptath' running particular tests scripts '${0##*/} -qunit' for running qunit JavaScript tests '${0##*/} -qunit testname' for running single JavaScript test with given name + +Extra options: + + --no_cleanup Do not delete temp files for Python functional tests (-toolshed, -framework, etc...) + --report_file Path of HTML report to produce (for Python Galaxy functional tests). + --xunit_report_file Path of XUnit report to produce (for Python Galaxy functional tests). EOF } @@ -196,6 +202,16 @@ shift 1 fi ;; + --no_cleanup) + GALAXY_TEST_NO_CLEANUP=1 + export GALAXY_TEST_NO_CLEANUP + TOOL_SHED_TEST_NO_CLEANUP=1 + export TOOL_SHED_TEST_NO_CLEANUP + GALAXY_INSTALL_TEST_NO_CLEANUP=1 + export GALAXY_INSTALL_TEST_NO_CLEANUP + echo "Skipping Python test clean up." + shift + ;; -watch|--watch) # Have grunt watch test or directory for changes, only # valid for javascript testing. https://bitbucket.org/galaxy/galaxy-central/commits/461a6a937900/ Changeset: 461a6a937900 User: jmchilton Date: 2014-12-27 22:49:16+00:00 Summary: Skip checking tool migrations for Galaxy in tool shed tests. Affected #: 1 file diff -r 86241a27f4d8d7065fb7908a0e687d6b11a062eb -r 461a6a9379001959b5b3259f90a2b9471ce391ed test/tool_shed/functional_tests.py --- a/test/tool_shed/functional_tests.py +++ b/test/tool_shed/functional_tests.py @@ -359,6 +359,7 @@ database_connection = galaxy_database_connection, database_auto_migrate = galaxy_database_auto_migrate, datatype_converters_config_file = "datatype_converters_conf.xml.sample", + check_migrate_tools = False, enable_tool_shed_check = True, file_path = galaxy_file_path, global_conf = galaxy_global_conf, Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: jmchilton: Merge next-stable.
by commits-noreply＠bitbucket.org 27 Dec '14

27 Dec '14

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/1ed5a63aae0a/ Changeset: 1ed5a63aae0a User: jmchilton Date: 2014-12-27 22:35:51+00:00 Summary: Merge next-stable. Affected #: 9 files diff -r 63ec80be64bb00414be0f7122a51907114ffb714 -r 1ed5a63aae0a5c8091483658c579d9d762a4c3fb lib/galaxy/web/base/controllers/admin.py --- a/lib/galaxy/web/base/controllers/admin.py +++ b/lib/galaxy/web/base/controllers/admin.py @@ -7,7 +7,7 @@ from galaxy.web.form_builder import CheckboxField from string import punctuation as PUNCTUATION import galaxy.queue_worker -from markupsafe import escape +from tool_shed.util.web_util import escape from tool_shed.util import shed_util_common as suc diff -r 63ec80be64bb00414be0f7122a51907114ffb714 -r 1ed5a63aae0a5c8091483658c579d9d762a4c3fb lib/galaxy/webapps/galaxy/controllers/admin.py --- a/lib/galaxy/webapps/galaxy/controllers/admin.py +++ b/lib/galaxy/webapps/galaxy/controllers/admin.py @@ -17,7 +17,7 @@ from galaxy.web.params import QuotaParamParser from tool_shed.util import common_util from tool_shed.util import encoding_util -from markupsafe import escape +from tool_shed.util.web_util import escape log = logging.getLogger( __name__ ) diff -r 63ec80be64bb00414be0f7122a51907114ffb714 -r 1ed5a63aae0a5c8091483658c579d9d762a4c3fb lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py --- a/lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py +++ b/lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py @@ -8,9 +8,9 @@ from galaxy.web.form_builder import CheckboxField from galaxy.util import json from galaxy.model.orm import or_ -from markupsafe import escape import tool_shed.repository_types.util as rt_util +from tool_shed.util.web_util import escape from tool_shed.util import common_util from tool_shed.util import encoding_util diff -r 63ec80be64bb00414be0f7122a51907114ffb714 -r 1ed5a63aae0a5c8091483658c579d9d762a4c3fb lib/galaxy/webapps/tool_shed/controllers/admin.py --- a/lib/galaxy/webapps/tool_shed/controllers/admin.py +++ b/lib/galaxy/webapps/tool_shed/controllers/admin.py @@ -3,7 +3,7 @@ from galaxy import util from galaxy.util import inflector from galaxy import web -from markupsafe import escape +from tool_shed.util.web_util import escape from galaxy.web.base.controller import BaseUIController from galaxy.web.base.controllers.admin import Admin diff -r 63ec80be64bb00414be0f7122a51907114ffb714 -r 1ed5a63aae0a5c8091483658c579d9d762a4c3fb lib/galaxy/webapps/tool_shed/controllers/repository.py --- a/lib/galaxy/webapps/tool_shed/controllers/repository.py +++ b/lib/galaxy/webapps/tool_shed/controllers/repository.py @@ -6,7 +6,6 @@ from time import strftime from datetime import date from datetime import datetime -from markupsafe import escape from galaxy import util from galaxy import web @@ -19,6 +18,7 @@ from tool_shed.capsule import capsule_manager from tool_shed.dependencies.repository import relation_builder +from tool_shed.util.web_util import escape from tool_shed.galaxy_install import dependency_display from tool_shed.metadata import repository_metadata_manager from tool_shed.utility_containers import ToolShedUtilityContainerManager diff -r 63ec80be64bb00414be0f7122a51907114ffb714 -r 1ed5a63aae0a5c8091483658c579d9d762a4c3fb lib/galaxy/webapps/tool_shed/controllers/repository_review.py --- a/lib/galaxy/webapps/tool_shed/controllers/repository_review.py +++ b/lib/galaxy/webapps/tool_shed/controllers/repository_review.py @@ -2,7 +2,7 @@ import os from sqlalchemy.sql.expression import func -from markupsafe import escape +from tool_shed.util.web_util import escape from galaxy import util from galaxy import web diff -r 63ec80be64bb00414be0f7122a51907114ffb714 -r 1ed5a63aae0a5c8091483658c579d9d762a4c3fb lib/galaxy/webapps/tool_shed/controllers/upload.py --- a/lib/galaxy/webapps/tool_shed/controllers/upload.py +++ b/lib/galaxy/webapps/tool_shed/controllers/upload.py @@ -9,7 +9,7 @@ from galaxy import web from galaxy.datatypes import checkers from galaxy.web.base.controller import BaseUIController -from markupsafe import escape +from tool_shed.util.web_util import escape from tool_shed.dependencies import attribute_handlers from tool_shed.galaxy_install import dependency_display diff -r 63ec80be64bb00414be0f7122a51907114ffb714 -r 1ed5a63aae0a5c8091483658c579d9d762a4c3fb lib/tool_shed/util/repository_util.py --- a/lib/tool_shed/util/repository_util.py +++ b/lib/tool_shed/util/repository_util.py @@ -7,7 +7,7 @@ from galaxy import web from galaxy.web.form_builder import build_select_field from galaxy.webapps.tool_shed.model import directory_hash_id -from markupsafe import escape +from tool_shed.util.web_util import escape from tool_shed.dependencies.repository import relation_builder diff -r 63ec80be64bb00414be0f7122a51907114ffb714 -r 1ed5a63aae0a5c8091483658c579d9d762a4c3fb lib/tool_shed/util/web_util.py --- /dev/null +++ b/lib/tool_shed/util/web_util.py @@ -0,0 +1,21 @@ +from markupsafe import escape as raw_escape + +ALLOWED_ELEMENTS = ["<b>", "</b>", "<br/>"] +ALLOWED_MAP = dict(map(lambda x: (x, raw_escape(x)), ALLOWED_ELEMENTS)) + + +def escape( value ): + """ A tool shed variant of markupsafe.escape that allows a select few + HTML elements that are repeatedly used in messages created deep + in the toolshed components. Ideally abstract things would be produced + in these components and messages in the views or client side - this is + what should be worked toward - but for now - we have this hack. + + >>> escape("A <b>repo</b>") + u'A <b>repo</b>' + """ + escaped = str( raw_escape( value ) ) + # Unescape few selected tags. + for key, value in ALLOWED_MAP.iteritems(): + escaped = escaped.replace(value, key) + return escaped Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: jmchilton: Fixes for over escaping in c2bed0a.
by commits-noreply＠bitbucket.org 27 Dec '14

27 Dec '14

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/93a868eb314d/ Changeset: 93a868eb314d Branch: next-stable User: jmchilton Date: 2014-12-27 22:30:59+00:00 Summary: Fixes for over escaping in c2bed0a. Fixes dozens of tool functional tests. Affected #: 9 files diff -r 501f510ff101849c45d9b0925bb55fe59319088b -r 93a868eb314d257c21bb19380b360bcf0d8dd82c lib/galaxy/web/base/controllers/admin.py --- a/lib/galaxy/web/base/controllers/admin.py +++ b/lib/galaxy/web/base/controllers/admin.py @@ -7,7 +7,7 @@ from galaxy.web.form_builder import CheckboxField from string import punctuation as PUNCTUATION import galaxy.queue_worker -from markupsafe import escape +from tool_shed.util.web_util import escape from tool_shed.util import shed_util_common as suc diff -r 501f510ff101849c45d9b0925bb55fe59319088b -r 93a868eb314d257c21bb19380b360bcf0d8dd82c lib/galaxy/webapps/galaxy/controllers/admin.py --- a/lib/galaxy/webapps/galaxy/controllers/admin.py +++ b/lib/galaxy/webapps/galaxy/controllers/admin.py @@ -17,7 +17,7 @@ from galaxy.web.params import QuotaParamParser from tool_shed.util import common_util from tool_shed.util import encoding_util -from markupsafe import escape +from tool_shed.util.web_util import escape log = logging.getLogger( __name__ ) diff -r 501f510ff101849c45d9b0925bb55fe59319088b -r 93a868eb314d257c21bb19380b360bcf0d8dd82c lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py --- a/lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py +++ b/lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py @@ -8,9 +8,9 @@ from galaxy.web.form_builder import CheckboxField from galaxy.util import json from galaxy.model.orm import or_ -from markupsafe import escape import tool_shed.repository_types.util as rt_util +from tool_shed.util.web_util import escape from tool_shed.util import common_util from tool_shed.util import encoding_util diff -r 501f510ff101849c45d9b0925bb55fe59319088b -r 93a868eb314d257c21bb19380b360bcf0d8dd82c lib/galaxy/webapps/tool_shed/controllers/admin.py --- a/lib/galaxy/webapps/tool_shed/controllers/admin.py +++ b/lib/galaxy/webapps/tool_shed/controllers/admin.py @@ -3,7 +3,7 @@ from galaxy import util from galaxy.util import inflector from galaxy import web -from markupsafe import escape +from tool_shed.util.web_util import escape from galaxy.web.base.controller import BaseUIController from galaxy.web.base.controllers.admin import Admin diff -r 501f510ff101849c45d9b0925bb55fe59319088b -r 93a868eb314d257c21bb19380b360bcf0d8dd82c lib/galaxy/webapps/tool_shed/controllers/repository.py --- a/lib/galaxy/webapps/tool_shed/controllers/repository.py +++ b/lib/galaxy/webapps/tool_shed/controllers/repository.py @@ -6,7 +6,6 @@ from time import strftime from datetime import date from datetime import datetime -from markupsafe import escape from galaxy import util from galaxy import web @@ -19,6 +18,7 @@ from tool_shed.capsule import capsule_manager from tool_shed.dependencies.repository import relation_builder +from tool_shed.util.web_util import escape from tool_shed.galaxy_install import dependency_display from tool_shed.metadata import repository_metadata_manager from tool_shed.utility_containers import ToolShedUtilityContainerManager diff -r 501f510ff101849c45d9b0925bb55fe59319088b -r 93a868eb314d257c21bb19380b360bcf0d8dd82c lib/galaxy/webapps/tool_shed/controllers/repository_review.py --- a/lib/galaxy/webapps/tool_shed/controllers/repository_review.py +++ b/lib/galaxy/webapps/tool_shed/controllers/repository_review.py @@ -2,7 +2,7 @@ import os from sqlalchemy.sql.expression import func -from markupsafe import escape +from tool_shed.util.web_util import escape from galaxy import util from galaxy import web diff -r 501f510ff101849c45d9b0925bb55fe59319088b -r 93a868eb314d257c21bb19380b360bcf0d8dd82c lib/galaxy/webapps/tool_shed/controllers/upload.py --- a/lib/galaxy/webapps/tool_shed/controllers/upload.py +++ b/lib/galaxy/webapps/tool_shed/controllers/upload.py @@ -9,7 +9,7 @@ from galaxy import web from galaxy.datatypes import checkers from galaxy.web.base.controller import BaseUIController -from markupsafe import escape +from tool_shed.util.web_util import escape from tool_shed.dependencies import attribute_handlers from tool_shed.galaxy_install import dependency_display diff -r 501f510ff101849c45d9b0925bb55fe59319088b -r 93a868eb314d257c21bb19380b360bcf0d8dd82c lib/tool_shed/util/repository_util.py --- a/lib/tool_shed/util/repository_util.py +++ b/lib/tool_shed/util/repository_util.py @@ -7,7 +7,7 @@ from galaxy import web from galaxy.web.form_builder import build_select_field from galaxy.webapps.tool_shed.model import directory_hash_id -from markupsafe import escape +from tool_shed.util.web_util import escape from tool_shed.dependencies.repository import relation_builder diff -r 501f510ff101849c45d9b0925bb55fe59319088b -r 93a868eb314d257c21bb19380b360bcf0d8dd82c lib/tool_shed/util/web_util.py --- /dev/null +++ b/lib/tool_shed/util/web_util.py @@ -0,0 +1,21 @@ +from markupsafe import escape as raw_escape + +ALLOWED_ELEMENTS = ["<b>", "</b>", "<br/>"] +ALLOWED_MAP = dict(map(lambda x: (x, raw_escape(x)), ALLOWED_ELEMENTS)) + + +def escape( value ): + """ A tool shed variant of markupsafe.escape that allows a select few + HTML elements that are repeatedly used in messages created deep + in the toolshed components. Ideally abstract things would be produced + in these components and messages in the views or client side - this is + what should be worked toward - but for now - we have this hack. + + >>> escape("A <b>repo</b>") + u'A <b>repo</b>' + """ + escaped = str( raw_escape( value ) ) + # Unescape few selected tags. + for key, value in ALLOWED_MAP.iteritems(): + escaped = escaped.replace(value, key) + return escaped Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

[galaxyproject/usegalaxy-playbook] b8c80f: Disable Stampede for now, it's being unstable.
by GitHub 26 Dec '14

26 Dec '14

Branch: refs/heads/master Home: https://github.com/galaxyproject/usegalaxy-playbook Commit: b8c80fd82c9cd8c602d09bcb97e7a4d567ba84e3 https://github.com/galaxyproject/usegalaxy-playbook/commit/b8c80fd82c9cd8c6… Author: Nate Coraor <nate(a)bx.psu.edu> Date: 2014-12-26 (Fri, 26 Dec 2014) Changed paths: M templates/galaxy/usegalaxy.org/config/job_conf.xml.j2 Log Message: ----------- Disable Stampede for now, it's being unstable.

1 0

commit/galaxy-central: 6 new changesets
by commits-noreply＠bitbucket.org 26 Dec '14

26 Dec '14

6 new commits in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/919dd242fd62/ Changeset: 919dd242fd62 Branch: arff_datatype User: BjoernGruening Date: 2014-12-16 11:40:29+00:00 Summary: Created new branch arff_datatype Affected #: 0 files https://bitbucket.org/galaxy/galaxy-central/commits/807b1e8a7815/ Changeset: 807b1e8a7815 Branch: arff_datatype User: BjoernGruening Date: 2014-12-16 12:08:50+00:00 Summary: Add Arff datatype class with sniffer and metadata. Affected #: 1 file diff -r 919dd242fd625a71d26b3935cabfbdb9b48c97b4 -r 807b1e8a7815f1514712146e909a508acc31a93d lib/galaxy/datatypes/text.py --- a/lib/galaxy/datatypes/text.py +++ b/lib/galaxy/datatypes/text.py @@ -6,6 +6,7 @@ from galaxy.datatypes.data import Text from galaxy.datatypes.data import get_file_peek from galaxy.datatypes.data import nice_size +from galaxy.datatypes.metadata import MetadataElement from galaxy import util import tempfile @@ -155,3 +156,106 @@ return True return False + +class Arff( Text ): + """ + An ARFF (Attribute-Relation File Format) file is an ASCII text file that describes a list of instances sharing a set of attributes. + http://weka.wikispaces.com/ARFF + """ + file_ext = "arff" + + + """Add metadata elements""" + MetadataElement( name="comment_lines", default=0, desc="Number of comment lines", readonly=True, optional=True, no_value=0 ) + MetadataElement( name="columns", default=0, desc="Number of columns", readonly=True, visible=True, no_value=0 ) + + def set_peek( self, dataset, is_multi_byte=False ): + if not dataset.dataset.purged: + dataset.peek = get_file_peek( dataset.file_name, is_multi_byte=is_multi_byte ) + dataset.blurb = "Attribute-Relation File Format (ARFF)" + dataset.blurb += ", %s comments, %s attributes" % ( dataset.metadata.comment_lines, dataset.metadata.columns ) + else: + dataset.peek = 'file does not exist' + dataset.blurb = 'file purged from disc' + + def sniff( self, filename ): + """ + Try to guess the Arff filetype. + It usually starts with a "format-version:" string and has several stanzas which starts with "id:". + """ + with open( filename ) as handle: + relation_found = False + attribute_found = False + prefix = "" + for line_count, line in enumerate( handle ): + if line_count > 1000: + # only investigate the first 1000 lines + return False + line = line.strip() + if not line: + continue + + start_string = line[:20].upper() + if start_string.startswith("@RELATION"): + relation_found = True + elif start_string.startswith("@ATTRIBUTE"): + attribute_found = True + elif start_string.startswith("@DATA"): + # @DATA should be the last data block + if relation_found and attribute_found: + return True + return False + + def set_meta( self, dataset, **kwd ): + """ + Trying to count the comment lines and the number of columns included. + A typical ARFF data block looks like this: + @DATA + 5.1,3.5,1.4,0.2,Iris-setosa + 4.9,3.0,1.4,0.2,Iris-setosa + """ + if dataset.has_data(): + comment_lines = 0 + first_real_line = False + data_block = False + with open( dataset.file_name ) as handle: + for line in handle: + line = line.strip() + if not line: + continue + if line.startswith('%') and not first_real_line: + comment_lines += 1 + else: + first_real_line = True + if data_block: + if line.startswith('{'): + # Sparse representation + """ + @data + 0, X, 0, Y, "class A", {5} + or + @data + {1 X, 3 Y, 4 "class A"}, {5} + """ + token = line.split('}',1) + first_part = token[0] + last_column = first_part.split(',')[-1].strip() + numeric_value = last_column.split()[0] + column_count = int(numeric_value) + if len(token) > 1: + # we have an additional weight + column_count -= 1 + else: + columns = line.strip().split(',') + column_count = len(columns) + if columns[-1].strip().startswith('{'): + # we have an additional weight at the end + column_count -= 1 + + # We have now the column_count and we know the initial comment lines. So we can terminate here. + break + if line[:5].upper() == "@DATA": + data_block = True + dataset.metadata.comment_lines = comment_lines + dataset.metadata.columns = column_count + https://bitbucket.org/galaxy/galaxy-central/commits/a876fb01dba3/ Changeset: a876fb01dba3 Branch: arff_datatype User: BjoernGruening Date: 2014-12-16 12:16:00+00:00 Summary: Add Arff datatype and sniffer to datatypes_conf.xml. Affected #: 1 file diff -r 807b1e8a7815f1514712146e909a508acc31a93d -r a876fb01dba3595d68ae99fea8e2070fa3ffad55 config/datatypes_conf.xml.sample --- a/config/datatypes_conf.xml.sample +++ b/config/datatypes_conf.xml.sample @@ -3,6 +3,7 @@ <registration converters_path="lib/galaxy/datatypes/converters" display_path="display_applications"><datatype extension="ab1" type="galaxy.datatypes.binary:Ab1" mimetype="application/octet-stream" display_in_upload="true" description="A binary sequence file in 'ab1' format with a '.ab1' file extension. You must manually select this 'File Format' when uploading the file." description_url="https://wiki.galaxyproject.org/Learn/Datatypes#Ab1"/><datatype extension="afg" type="galaxy.datatypes.assembly:Amos" display_in_upload="false" /> + <datatype extension="asn1" type="galaxy.datatypes.data:GenericAsn1" mimetype="text/plain" display_in_upload="true" /><datatype extension="asn1-binary" type="galaxy.datatypes.binary:GenericAsn1Binary" mimetype="application/octet-stream" display_in_upload="true" /><datatype extension="axt" type="galaxy.datatypes.sequence:Axt" display_in_upload="true" description="blastz pairwise alignment format. Each alignment block in an axt file contains three lines: a summary line and 2 sequence lines. Blocks are separated from one another by blank lines. The summary line contains chromosomal position and size information about the alignment. It consists of 9 required fields." description_url="https://wiki.galaxyproject.org/Learn/Datatypes#Axt"/> @@ -298,6 +299,7 @@ <sniffer type="galaxy.datatypes.data:Newick"/><sniffer type="galaxy.datatypes.data:Nexus"/><sniffer type="galaxy.datatypes.text:Obo"/> + <sniffer type="galaxy.datatypes.text:Ipynb"/><sniffer type="galaxy.datatypes.text:Json"/><sniffer type="galaxy.datatypes.images:Jpg"/> https://bitbucket.org/galaxy/galaxy-central/commits/d91d8e8beada/ Changeset: d91d8e8beada Branch: arff_datatype User: BjoernGruening Date: 2014-12-23 00:27:36+00:00 Summary: Activate arff datatype by default Affected #: 1 file diff -r a876fb01dba3595d68ae99fea8e2070fa3ffad55 -r d91d8e8beadac2a187645f86c199119064a5ccb0 config/datatypes_conf.xml.sample --- a/config/datatypes_conf.xml.sample +++ b/config/datatypes_conf.xml.sample @@ -3,7 +3,7 @@ <registration converters_path="lib/galaxy/datatypes/converters" display_path="display_applications"><datatype extension="ab1" type="galaxy.datatypes.binary:Ab1" mimetype="application/octet-stream" display_in_upload="true" description="A binary sequence file in 'ab1' format with a '.ab1' file extension. You must manually select this 'File Format' when uploading the file." description_url="https://wiki.galaxyproject.org/Learn/Datatypes#Ab1"/><datatype extension="afg" type="galaxy.datatypes.assembly:Amos" display_in_upload="false" /> -  + <datatype extension="arff" type="galaxy.datatypes.text:Arff" mimetype="text/plain" display_in_upload="True" /><datatype extension="asn1" type="galaxy.datatypes.data:GenericAsn1" mimetype="text/plain" display_in_upload="true" /><datatype extension="asn1-binary" type="galaxy.datatypes.binary:GenericAsn1Binary" mimetype="application/octet-stream" display_in_upload="true" /><datatype extension="axt" type="galaxy.datatypes.sequence:Axt" display_in_upload="true" description="blastz pairwise alignment format. Each alignment block in an axt file contains three lines: a summary line and 2 sequence lines. Blocks are separated from one another by blank lines. The summary line contains chromosomal position and size information about the alignment. It consists of 9 required fields." description_url="https://wiki.galaxyproject.org/Learn/Datatypes#Axt"/> @@ -299,7 +299,7 @@ <sniffer type="galaxy.datatypes.data:Newick"/><sniffer type="galaxy.datatypes.data:Nexus"/><sniffer type="galaxy.datatypes.text:Obo"/> -  + <sniffer type="galaxy.datatypes.text.Arff"/><sniffer type="galaxy.datatypes.text:Ipynb"/><sniffer type="galaxy.datatypes.text:Json"/><sniffer type="galaxy.datatypes.images:Jpg"/> https://bitbucket.org/galaxy/galaxy-central/commits/63ec80be64bb/ Changeset: 63ec80be64bb User: jmchilton Date: 2014-12-26 22:33:02+00:00 Summary: Merge pull request #614. Affected #: 2 files diff -r 338a32cc6067d92055324e4b827697051ec99254 -r 63ec80be64bb00414be0f7122a51907114ffb714 config/datatypes_conf.xml.sample --- a/config/datatypes_conf.xml.sample +++ b/config/datatypes_conf.xml.sample @@ -3,6 +3,7 @@ <registration converters_path="lib/galaxy/datatypes/converters" display_path="display_applications"><datatype extension="ab1" type="galaxy.datatypes.binary:Ab1" mimetype="application/octet-stream" display_in_upload="true" description="A binary sequence file in 'ab1' format with a '.ab1' file extension. You must manually select this 'File Format' when uploading the file." description_url="https://wiki.galaxyproject.org/Learn/Datatypes#Ab1"/><datatype extension="afg" type="galaxy.datatypes.assembly:Amos" display_in_upload="false" /> + <datatype extension="arff" type="galaxy.datatypes.text:Arff" mimetype="text/plain" display_in_upload="True" /><datatype extension="asn1" type="galaxy.datatypes.data:GenericAsn1" mimetype="text/plain" display_in_upload="true" /><datatype extension="asn1-binary" type="galaxy.datatypes.binary:GenericAsn1Binary" mimetype="application/octet-stream" display_in_upload="true" /><datatype extension="axt" type="galaxy.datatypes.sequence:Axt" display_in_upload="true" description="blastz pairwise alignment format. Each alignment block in an axt file contains three lines: a summary line and 2 sequence lines. Blocks are separated from one another by blank lines. The summary line contains chromosomal position and size information about the alignment. It consists of 9 required fields." description_url="https://wiki.galaxyproject.org/Learn/Datatypes#Axt"/> @@ -298,6 +299,7 @@ <sniffer type="galaxy.datatypes.data:Newick"/><sniffer type="galaxy.datatypes.data:Nexus"/><sniffer type="galaxy.datatypes.text:Obo"/> + <sniffer type="galaxy.datatypes.text.Arff"/><sniffer type="galaxy.datatypes.text:Ipynb"/><sniffer type="galaxy.datatypes.text:Json"/><sniffer type="galaxy.datatypes.images:Jpg"/> diff -r 338a32cc6067d92055324e4b827697051ec99254 -r 63ec80be64bb00414be0f7122a51907114ffb714 lib/galaxy/datatypes/text.py --- a/lib/galaxy/datatypes/text.py +++ b/lib/galaxy/datatypes/text.py @@ -6,6 +6,7 @@ from galaxy.datatypes.data import Text from galaxy.datatypes.data import get_file_peek from galaxy.datatypes.data import nice_size +from galaxy.datatypes.metadata import MetadataElement from galaxy import util import tempfile @@ -155,3 +156,106 @@ return True return False + +class Arff( Text ): + """ + An ARFF (Attribute-Relation File Format) file is an ASCII text file that describes a list of instances sharing a set of attributes. + http://weka.wikispaces.com/ARFF + """ + file_ext = "arff" + + + """Add metadata elements""" + MetadataElement( name="comment_lines", default=0, desc="Number of comment lines", readonly=True, optional=True, no_value=0 ) + MetadataElement( name="columns", default=0, desc="Number of columns", readonly=True, visible=True, no_value=0 ) + + def set_peek( self, dataset, is_multi_byte=False ): + if not dataset.dataset.purged: + dataset.peek = get_file_peek( dataset.file_name, is_multi_byte=is_multi_byte ) + dataset.blurb = "Attribute-Relation File Format (ARFF)" + dataset.blurb += ", %s comments, %s attributes" % ( dataset.metadata.comment_lines, dataset.metadata.columns ) + else: + dataset.peek = 'file does not exist' + dataset.blurb = 'file purged from disc' + + def sniff( self, filename ): + """ + Try to guess the Arff filetype. + It usually starts with a "format-version:" string and has several stanzas which starts with "id:". + """ + with open( filename ) as handle: + relation_found = False + attribute_found = False + prefix = "" + for line_count, line in enumerate( handle ): + if line_count > 1000: + # only investigate the first 1000 lines + return False + line = line.strip() + if not line: + continue + + start_string = line[:20].upper() + if start_string.startswith("@RELATION"): + relation_found = True + elif start_string.startswith("@ATTRIBUTE"): + attribute_found = True + elif start_string.startswith("@DATA"): + # @DATA should be the last data block + if relation_found and attribute_found: + return True + return False + + def set_meta( self, dataset, **kwd ): + """ + Trying to count the comment lines and the number of columns included. + A typical ARFF data block looks like this: + @DATA + 5.1,3.5,1.4,0.2,Iris-setosa + 4.9,3.0,1.4,0.2,Iris-setosa + """ + if dataset.has_data(): + comment_lines = 0 + first_real_line = False + data_block = False + with open( dataset.file_name ) as handle: + for line in handle: + line = line.strip() + if not line: + continue + if line.startswith('%') and not first_real_line: + comment_lines += 1 + else: + first_real_line = True + if data_block: + if line.startswith('{'): + # Sparse representation + """ + @data + 0, X, 0, Y, "class A", {5} + or + @data + {1 X, 3 Y, 4 "class A"}, {5} + """ + token = line.split('}',1) + first_part = token[0] + last_column = first_part.split(',')[-1].strip() + numeric_value = last_column.split()[0] + column_count = int(numeric_value) + if len(token) > 1: + # we have an additional weight + column_count -= 1 + else: + columns = line.strip().split(',') + column_count = len(columns) + if columns[-1].strip().startswith('{'): + # we have an additional weight at the end + column_count -= 1 + + # We have now the column_count and we know the initial comment lines. So we can terminate here. + break + if line[:5].upper() == "@DATA": + data_block = True + dataset.metadata.comment_lines = comment_lines + dataset.metadata.columns = column_count + https://bitbucket.org/galaxy/galaxy-central/commits/3ce3ebfdd734/ Changeset: 3ce3ebfdd734 Branch: arff_datatype User: jmchilton Date: 2014-12-26 22:34:57+00:00 Summary: Close arff_datatype. Affected #: 0 files Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: 3 new changesets
by commits-noreply＠bitbucket.org 26 Dec '14

26 Dec '14

3 new commits in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/f7b777f46f9b/ Changeset: f7b777f46f9b User: jmchilton Date: 2014-12-26 14:55:27+00:00 Summary: Remove unused method lib/galaxy/workflow/modules.py. Affected #: 1 file diff -r 4844fa1099447fb8397159be710c99a82726821a -r f7b777f46f9b69b3cd2ca2d90a30988d9f4fa0ab lib/galaxy/workflow/modules.py --- a/lib/galaxy/workflow/modules.py +++ b/lib/galaxy/workflow/modules.py @@ -557,13 +557,6 @@ def normalize_runtime_state( self, runtime_state ): return runtime_state.encode( self.tool, self.trans.app, secure=False ) - @classmethod - def __get_tool_version( cls, trans, tool_id ): - # Return a ToolVersion if one exists for tool_id. - return trans.install_model.context.query( trans.install_model.ToolVersion ) \ - .filter( trans.install_model.ToolVersion.table.c.tool_id == tool_id ) \ - .first() - def save_to_step( self, step ): step.type = self.type step.tool_id = self.tool_id https://bitbucket.org/galaxy/galaxy-central/commits/8a2b4ebdf492/ Changeset: 8a2b4ebdf492 User: jmchilton Date: 2014-12-26 14:55:27+00:00 Summary: Eliminate direct in/get checks against toolbox.tools_by_id - use get_tool or has_tool. Affected #: 6 files diff -r f7b777f46f9b69b3cd2ca2d90a30988d9f4fa0ab -r 8a2b4ebdf4927fccbe44eda80eaba17b8033802c lib/galaxy/jobs/__init__.py --- a/lib/galaxy/jobs/__init__.py +++ b/lib/galaxy/jobs/__init__.py @@ -711,7 +711,7 @@ self.job_id = job.id self.session_id = job.session_id self.user_id = job.user_id - self.tool = queue.app.toolbox.tools_by_id.get( job.tool_id, None ) + self.tool = queue.app.toolbox.get_tool( job.tool_id, exact=True ) self.queue = queue self.app = queue.app self.sa_session = self.app.model.context diff -r f7b777f46f9b69b3cd2ca2d90a30988d9f4fa0ab -r 8a2b4ebdf4927fccbe44eda80eaba17b8033802c lib/galaxy/jobs/handler.py --- a/lib/galaxy/jobs/handler.py +++ b/lib/galaxy/jobs/handler.py @@ -119,7 +119,7 @@ & ( model.Job.handler == self.app.config.server_name ) ).all() for job in jobs_at_startup: - if job.tool_id not in self.app.toolbox.tools_by_id: + if not self.app.toolbox.has_tool( job.tool_id, exact=True ): log.warning( "(%s) Tool '%s' removed from tool config, unable to recover job" % ( job.id, job.tool_id ) ) self.job_wrapper( job ).fail( 'This tool was disabled before the job completed. Please contact your Galaxy administrator.' ) elif job.job_runner_name is not None and job.job_runner_external_id is None: diff -r f7b777f46f9b69b3cd2ca2d90a30988d9f4fa0ab -r 8a2b4ebdf4927fccbe44eda80eaba17b8033802c lib/galaxy/managers/workflows.py --- a/lib/galaxy/managers/workflows.py +++ b/lib/galaxy/managers/workflows.py @@ -224,7 +224,7 @@ errors = [] for key, step_dict in data['steps'].iteritems(): is_tool = is_tool_module_type( step_dict[ 'type' ] ) - if is_tool and step_dict['tool_id'] not in trans.app.toolbox.tools_by_id: + if is_tool and not trans.app.toolbox.has_tool( step_dict['tool_id'], exact=True ): errors.append("Step %s requires tool '%s'." % (step_dict['id'], step_dict['tool_id'])) if errors: raise MissingToolsException(workflow, errors) diff -r f7b777f46f9b69b3cd2ca2d90a30988d9f4fa0ab -r 8a2b4ebdf4927fccbe44eda80eaba17b8033802c lib/galaxy/tools/__init__.py --- a/lib/galaxy/tools/__init__.py +++ b/lib/galaxy/tools/__init__.py @@ -478,8 +478,11 @@ shutil.move( filename, os.path.abspath( self.integrated_tool_panel_config ) ) os.chmod( self.integrated_tool_panel_config, 0644 ) - def get_tool( self, tool_id, tool_version=None, get_all_versions=False ): + def get_tool( self, tool_id, tool_version=None, get_all_versions=False, exact=False ): """Attempt to locate a tool in the tool box.""" + if get_all_versions and exact: + raise AssertionError("Cannot specify get_tool with both get_all_versions and exact as True") + if tool_id in self.tools_by_id and not get_all_versions: #tool_id exactly matches an available tool by id (which is 'old' tool_id or guid) return self.tools_by_id[ tool_id ] @@ -513,6 +516,9 @@ return[ self.tools_by_id[ tool_id ] ] return None + def has_tool( self, tool_id, exact=False ): + return self.get_tool( tool_id, exact=exact ) is not None + def get_loaded_tools_by_lineage( self, tool_id ): """Get all loaded tools associated by lineage to the tool whose id is tool_id.""" tv = self.__get_tool_version( tool_id ) diff -r f7b777f46f9b69b3cd2ca2d90a30988d9f4fa0ab -r 8a2b4ebdf4927fccbe44eda80eaba17b8033802c lib/galaxy/webapps/galaxy/controllers/admin.py --- a/lib/galaxy/webapps/galaxy/controllers/admin.py +++ b/lib/galaxy/webapps/galaxy/controllers/admin.py @@ -486,7 +486,8 @@ class ToolIdColumn( grids.TextColumn ): def get_value( self, trans, grid, tool_version ): - if tool_version.tool_id in trans.app.toolbox.tools_by_id: + toolbox = trans.app.toolbox + if toolbox.has_tool( tool_version.tool_id, exact=True ): link = url_for( controller='tool_runner', tool_id=tool_version.tool_id ) link_str = '<a href="%s">' % link return '<div class="count-box state-color-ok">%s%s</a></div>' % ( link_str, tool_version.tool_id ) @@ -497,8 +498,9 @@ def get_value( self, trans, grid, tool_version ): tool_ids_str = '' + toolbox = trans.app.toolbox for tool_id in tool_version.get_version_ids( trans.app ): - if tool_id in trans.app.toolbox.tools_by_id: + if toolbox.has_tool( tool_id, exact=True ): link = url_for( controller='tool_runner', tool_id=tool_version.tool_id ) link_str = '<a href="%s">' % link tool_ids_str += '<div class="count-box state-color-ok">%s%s</a></div><br/>' % ( link_str, tool_id ) diff -r f7b777f46f9b69b3cd2ca2d90a30988d9f4fa0ab -r 8a2b4ebdf4927fccbe44eda80eaba17b8033802c test/unit/jobs/test_job_wrapper.py --- a/test/unit/jobs/test_job_wrapper.py +++ b/test/unit/jobs/test_job_wrapper.py @@ -190,14 +190,14 @@ def __init__(self, test_tool): self.test_tool = test_tool - @property - def tools_by_id(self): - return self - def get(self, tool_id, default=None): assert tool_id == TEST_TOOL_ID return self.test_tool + def get_tool( self, tool_id, exact=False ): + tool = self.get(tool_id) + return tool + class MockObjectStore(object): https://bitbucket.org/galaxy/galaxy-central/commits/338a32cc6067/ Changeset: 338a32cc6067 User: jmchilton Date: 2014-12-26 14:55:27+00:00 Summary: Introduce abstraction shielding toolbox.tools_by_id from galaxy.workflow.modules. That was the last module outside of ToolBox itself that was accessing tools_by_id directly. Affected #: 5 files diff -r 8a2b4ebdf4927fccbe44eda80eaba17b8033802c -r 338a32cc6067d92055324e4b827697051ec99254 lib/galaxy/tools/__init__.py --- a/lib/galaxy/tools/__init__.py +++ b/lib/galaxy/tools/__init__.py @@ -519,6 +519,21 @@ def has_tool( self, tool_id, exact=False ): return self.get_tool( tool_id, exact=exact ) is not None + def get_tool_id( self, tool_id ): + """ Take a tool id (potentially from a different Galaxy instance or that + is no longer loaded - and find the closest match to the currently loaded + tools (using get_tool for inexact matches which currently returns the oldest + tool shed installed tool with the same short id). + """ + if tool_id not in self.tools_by_id: + tool = self.get_tool( tool_id ) + if tool: + tool_id = tool.id + else: + tool_id = None + # else exact match - leave unmodified. + return tool_id + def get_loaded_tools_by_lineage( self, tool_id ): """Get all loaded tools associated by lineage to the tool whose id is tool_id.""" tv = self.__get_tool_version( tool_id ) diff -r 8a2b4ebdf4927fccbe44eda80eaba17b8033802c -r 338a32cc6067d92055324e4b827697051ec99254 lib/galaxy/workflow/modules.py --- a/lib/galaxy/workflow/modules.py +++ b/lib/galaxy/workflow/modules.py @@ -508,15 +508,13 @@ def from_workflow_step( Class, trans, step ): toolbox = trans.app.toolbox tool_id = step.tool_id - if toolbox and tool_id not in toolbox.tools_by_id: + if toolbox: # See if we have access to a different version of the tool. # TODO: If workflows are ever enhanced to use tool version # in addition to tool id, enhance the selection process here # to retrieve the correct version of the tool. - tool = toolbox.get_tool( tool_id ) - if tool: - tool_id = tool.id - if ( toolbox and tool_id in toolbox.tools_by_id ): + tool_id = toolbox.get_tool_id( tool_id ) + if ( toolbox and tool_id ): if step.config: # This step has its state saved in the config field due to the # tool being previously unavailable. diff -r 8a2b4ebdf4927fccbe44eda80eaba17b8033802c -r 338a32cc6067d92055324e4b827697051ec99254 test/unit/tools/test_toolbox.py --- a/test/unit/tools/test_toolbox.py +++ b/test/unit/tools/test_toolbox.py @@ -186,6 +186,15 @@ # No changes, should be regenerated self.assert_integerated_tool_panel(exists=False) + def test_get_tool_id( self ): + self._init_tool() + self._setup_two_versions_in_config( ) + self._setup_two_versions() + assert self.toolbox.get_tool_id( "test_tool" ) == "github.com/galaxyproect/example/test_tool/0.1" + assert self.toolbox.get_tool_id( "github.com/galaxyproect/example/test_tool/0.1" ) == "github.com/galaxyproect/example/test_tool/0.1" + assert self.toolbox.get_tool_id( "github.com/galaxyproect/example/test_tool/0.2" ) == "github.com/galaxyproect/example/test_tool/0.2" + assert self.toolbox.get_tool_id( "github.com/galaxyproect/example/test_tool/0.3" ) is None + def __verify_two_test_tools( self ): # Assert tool versions of the tool with simple id 'test_tool' all_versions = self.toolbox.get_tool( "test_tool", get_all_versions=True ) diff -r 8a2b4ebdf4927fccbe44eda80eaba17b8033802c -r 338a32cc6067d92055324e4b827697051ec99254 test/unit/workflows/test_modules.py --- a/test/unit/workflows/test_modules.py +++ b/test/unit/workflows/test_modules.py @@ -5,6 +5,7 @@ import mock from galaxy import model +from galaxy.util import bunch from galaxy.workflow import modules from .workflow_support import MockTrans @@ -201,6 +202,37 @@ assert exception +def test_updated_tool_version(): + trans = MockTrans() + mock_tool = __mock_tool(id="cat1", version="0.9") + trans.app.toolbox.tools[ "cat1" ] = mock_tool + module = __from_step( + trans=trans, + type="tool", + tool_id="cat1", + tool_version="0.7", + config=None, + ) + # Make sure there is a warnin with tool id, old version, + # and new version. + for val in "cat1", "0.7", "0.9": + assert val in module.version_changes[0] + + +def test_tool_version_same(): + trans = MockTrans() + mock_tool = __mock_tool(id="cat1", version="1.0") + trans.app.toolbox.tools[ "cat1" ] = mock_tool + module = __from_step( + trans=trans, + type="tool", + tool_id="cat1", + tool_version="1.0", + config=None, + ) + assert not module.version_changes + + def __assert_has_runtime_input( module, label=None, collection_type=None ): inputs = module.get_runtime_inputs() assert len( inputs ) == 1 @@ -221,7 +253,11 @@ def __from_step( **kwds ): - trans = MockTrans() + if "trans" in kwds: + trans = kwds["trans"] + del kwds["trans"] + else: + trans = MockTrans() step = __step( **kwds ) @@ -238,3 +274,20 @@ setattr( step, key, value ) return step + + +def __mock_tool( + id="cat1", + version="1.0", +): + # For now ignoring inputs, params_from_strings, and + # check_and_update_param_values since only have unit tests for version + # handling - but need to write tests for all of this longer term. + tool = bunch.Bunch( + id=id, + version=version, + inputs={}, + params_from_strings=mock.Mock(), + check_and_update_param_values=mock.Mock(), + ) + return tool diff -r 8a2b4ebdf4927fccbe44eda80eaba17b8033802c -r 338a32cc6067d92055324e4b827697051ec99254 test/unit/workflows/workflow_support.py --- a/test/unit/workflows/workflow_support.py +++ b/test/unit/workflows/workflow_support.py @@ -40,3 +40,7 @@ def get_tool( self, tool_id ): # Real tool box returns None of missing tool also return self.tools.get( tool_id, None ) + + def get_tool_id( self, tool_id ): + tool = self.get_tool( tool_id ) + return tool and tool.id Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0