- galaxy-commits - lists.galaxyproject.org

commit/galaxy-central: 6 new changesets
by commits-noreply＠bitbucket.org 28 Dec '14

28 Dec '14

6 new commits in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/50062f6e9b47/ Changeset: 50062f6e9b47 User: jmchilton Date: 2014-12-28 16:23:44+00:00 Summary: PEP-8 fixes for a few Tool Shed install components. - lib/tool_shed/galaxy_install/install_manager.py - lib/tool_shed/galaxy_install/repository_dependencies/repository_dependency_manager.py - lib/tool_shed/galaxy_install/repair_repository_manager.py Affected #: 3 files diff -r 97f46da40bb3767c6cf0d440b1d2d58e06edf5e6 -r 50062f6e9b47a36dad57393292611632908fe79a lib/tool_shed/galaxy_install/install_manager.py --- a/lib/tool_shed/galaxy_install/install_manager.py +++ b/lib/tool_shed/galaxy_install/install_manager.py @@ -42,11 +42,11 @@ class InstallToolDependencyManager( object ): - + def __init__( self, app ): self.app = app self.install_model = self.app.install_model - self.INSTALL_ACTIONS = [ 'download_binary', 'download_by_url', 'download_file', + self.INSTALL_ACTIONS = [ 'download_binary', 'download_by_url', 'download_file', 'setup_perl_environment', 'setup_python_environment', 'setup_r_environment', 'setup_ruby_environment', 'shell_command' ] @@ -142,9 +142,9 @@ # Since there was an installation error, update the tool dependency status to Error. The remove_installation_path option must # be left False here. error_message = '%s\n%s' % ( self.format_traceback(), str( e ) ) - tool_dependency = tool_dependency_util.handle_tool_dependency_installation_error( self.app, - tool_dependency, - error_message, + tool_dependency = tool_dependency_util.handle_tool_dependency_installation_error( self.app, + tool_dependency, + error_message, remove_installation_path=False ) tool_dependency = self.mark_tool_dependency_installed( tool_dependency ) return tool_dependency @@ -208,9 +208,9 @@ tool_dependency_db_records=tool_dependencies ) if ( tool_dependency.type == 'package' and proceed_with_install ): try: - tool_dependency = self.install_package( elem, - tool_shed_repository, - tool_dependencies=tool_dependencies, + tool_dependency = self.install_package( elem, + tool_shed_repository, + tool_dependencies=tool_dependencies, from_tool_migration_manager=from_tool_migration_manager ) except Exception, e: error_message = "Error installing tool dependency %s version %s: %s" % \ @@ -220,9 +220,9 @@ # Since there was an installation error, update the tool dependency status to Error. The # remove_installation_path option must be left False here. tool_dependency = \ - tool_dependency_util.handle_tool_dependency_installation_error( self.app, - tool_dependency, - error_message, + tool_dependency_util.handle_tool_dependency_installation_error( self.app, + tool_dependency, + error_message, remove_installation_path=False ) if tool_dependency and tool_dependency.status in [ self.install_model.ToolDependency.installation_status.INSTALLED, self.install_model.ToolDependency.installation_status.ERROR ]: @@ -236,7 +236,7 @@ def install_via_fabric( self, tool_shed_repository, tool_dependency, install_dir, package_name=None, custom_fabfile_path=None, actions_elem=None, action_elem=None, **kwd ): """ - Parse a tool_dependency.xml file's <actions> tag set to gather information for installation using + Parse a tool_dependency.xml file's <actions> tag set to gather information for installation using self.install_and_build_package(). The use of fabric is being eliminated, so some of these functions may need to be renamed at some point. """ @@ -363,8 +363,8 @@ else: # Process the next matching <actions> tag, or any defined <actions> tags that do not # contain platform dependent recipes. - log.debug( 'Error downloading binary for tool dependency %s version %s: %s' % \ - ( str( package_name ), str( package_version ), str( tool_dependency.error_message ) ) ) + log.debug( 'Error downloading binary for tool dependency %s version %s: %s' % + ( str( package_name ), str( package_version ), str( tool_dependency.error_message ) ) ) else: if actions_elem.tag == 'actions': # We've reached an <actions> tag that defines the recipe for installing and compiling from @@ -380,7 +380,7 @@ if removed: can_install_from_source = True else: - log.debug( 'Error removing old files from installation directory %s: %s' % \ + log.debug( 'Error removing old files from installation directory %s: %s' % ( str( installation_directory, str( error_message ) ) ) ) else: can_install_from_source = True @@ -389,7 +389,7 @@ if can_install_from_source: # We now know that binary installation was not successful, so proceed with the <actions> # tag set that defines the recipe to install and compile from source. - log.debug( 'Proceeding with install and compile recipe for tool dependency %s.' % \ + log.debug( 'Proceeding with install and compile recipe for tool dependency %s.' % str( tool_dependency.name ) ) tool_dependency = self.install_via_fabric( tool_shed_repository, tool_dependency, @@ -398,7 +398,7 @@ actions_elem=actions_elem, action_elem=None ) if actions_elem.tag == 'action' and \ - tool_dependency.status != self.install_model.ToolDependency.installation_status.ERROR: + tool_dependency.status != self.install_model.ToolDependency.installation_status.ERROR: # If the tool dependency is not in an error state, perform any final actions that have been # defined within the actions_group tag set, but outside of an <actions> tag, which defines # the recipe for installing and compiling from source. @@ -419,17 +419,17 @@ actions_elem=actions_elems, action_elem=None ) if tool_dependency.status != self.install_model.ToolDependency.installation_status.ERROR: - log.debug( 'Tool dependency %s version %s has been installed in %s.' % \ - ( str( package_name ), str( package_version ), str( install_dir ) ) ) + log.debug( 'Tool dependency %s version %s has been installed in %s.' % + ( str( package_name ), str( package_version ), str( install_dir ) ) ) return tool_dependency def mark_tool_dependency_installed( self, tool_dependency ): if tool_dependency.status not in [ self.install_model.ToolDependency.installation_status.ERROR, self.install_model.ToolDependency.installation_status.INSTALLED ]: - log.debug( 'Changing status for tool dependency %s from %s to %s.' % \ - ( str( tool_dependency.name ), - str( tool_dependency.status ), - str( self.install_model.ToolDependency.installation_status.INSTALLED ) ) ) + log.debug( 'Changing status for tool dependency %s from %s to %s.' % + ( str( tool_dependency.name ), + str( tool_dependency.status ), + str( self.install_model.ToolDependency.installation_status.INSTALLED ) ) ) status = self.install_model.ToolDependency.installation_status.INSTALLED tool_dependency = tool_dependency_util.set_tool_dependency_attributes( self.app, tool_dependency=tool_dependency, @@ -531,10 +531,10 @@ self.install_model.context.add( tool_shed_repository ) self.install_model.context.flush() if 'tool_dependencies' in irmm_metadata_dict and not reinstalling: - tool_dependencies = tool_dependency_util.create_tool_dependency_objects( self.app, - tool_shed_repository, - relative_install_dir, - set_status=True ) + tool_dependency_util.create_tool_dependency_objects( self.app, + tool_shed_repository, + relative_install_dir, + set_status=True ) if 'sample_files' in irmm_metadata_dict: sample_files = irmm_metadata_dict.get( 'sample_files', [] ) tool_index_sample_files = tdtm.get_tool_index_sample_files( sample_files ) @@ -579,12 +579,12 @@ new_install=True ) if 'data_manager' in irmm_metadata_dict: dmh = data_manager.DataManagerHandler( self.app ) - new_data_managers = dmh.install_data_managers( self.app.config.shed_data_manager_config_file, - irmm_metadata_dict, - shed_config_dict, - relative_install_dir, - tool_shed_repository, - repository_tools_tups ) + dmh.install_data_managers( self.app.config.shed_data_manager_config_file, + irmm_metadata_dict, + shed_config_dict, + relative_install_dir, + tool_shed_repository, + repository_tools_tups ) if 'datatypes' in irmm_metadata_dict: tool_shed_repository.status = self.install_model.ToolShedRepository.installation_status.LOADING_PROPRIETARY_DATATYPES if not tool_shed_repository.includes_datatypes: @@ -656,13 +656,9 @@ tool_shed_url = installation_dict[ 'tool_shed_url' ] # Handle contained tools. if includes_tools_for_display_in_tool_panel and ( new_tool_panel_section_label or tool_panel_section_id ): - tool_panel_section_key, tool_section = \ - self.tpm.handle_tool_panel_section( self.app.toolbox, - tool_panel_section_id=tool_panel_section_id, - new_tool_panel_section_label=new_tool_panel_section_label ) - else: - tool_panel_section_key = None - tool_section = None + self.tpm.handle_tool_panel_section( self.app.toolbox, + tool_panel_section_id=tool_panel_section_id, + new_tool_panel_section_label=new_tool_panel_section_label ) encoded_repository_ids = [ self.app.security.encode_id( tsr.id ) for tsr in created_or_updated_tool_shed_repositories ] new_kwd = dict( includes_tools=includes_tools, includes_tools_for_display_in_tool_panel=includes_tools_for_display_in_tool_panel, @@ -679,7 +675,7 @@ tool_shed_repository_ids=encoded_repository_ids, tool_shed_url=tool_shed_url ) encoded_kwd = encoding_util.tool_shed_encode( new_kwd ) - tsr_ids = [ r.id for r in created_or_updated_tool_shed_repositories ] + tsr_ids = [ r.id for r in created_or_updated_tool_shed_repositories ] tool_shed_repositories = [] for tsr_id in tsr_ids: tsr = self.install_model.context.query( self.install_model.ToolShedRepository ).get( tsr_id ) @@ -895,10 +891,10 @@ # Get the tool_dependencies.xml file from the repository. tool_dependencies_config = hg_util.get_config_from_disk( 'tool_dependencies.xml', install_dir ) itdm = InstallToolDependencyManager( self.app ) - installed_tool_dependencies = itdm.install_specified_tool_dependencies( tool_shed_repository=tool_shed_repository, - tool_dependencies_config=tool_dependencies_config, - tool_dependencies=tool_shed_repository.tool_dependencies, - from_tool_migration_manager=False ) + itdm.install_specified_tool_dependencies( tool_shed_repository=tool_shed_repository, + tool_dependencies_config=tool_dependencies_config, + tool_dependencies=tool_shed_repository.tool_dependencies, + from_tool_migration_manager=False ) basic_util.remove_dir( work_dir ) self.update_tool_shed_repository_status( tool_shed_repository, self.install_model.ToolShedRepository.installation_status.INSTALLED ) diff -r 97f46da40bb3767c6cf0d440b1d2d58e06edf5e6 -r 50062f6e9b47a36dad57393292611632908fe79a lib/tool_shed/galaxy_install/repair_repository_manager.py --- a/lib/tool_shed/galaxy_install/repair_repository_manager.py +++ b/lib/tool_shed/galaxy_install/repair_repository_manager.py @@ -65,7 +65,7 @@ repair_dict = {} irm = install_manager.InstallRepositoryManager( self.app ) # Get a dictionary of all repositories upon which the contents of the current repository_metadata - #record depend. + # record depend. repository_dependencies_dict = rdim.get_repository_dependencies_for_installed_tool_shed_repository( self.app, repository ) if repository_dependencies_dict: @@ -142,14 +142,14 @@ return repo_info_dict, tool_panel_section_key def repair_tool_shed_repository( self, repository, repo_info_dict ): - + def add_repair_dict_entry( repository_name, error_message ): if repository_name in repair_dict: repair_dict[ repository_name ].append( error_message ) else: repair_dict[ repository_name ] = [ error_message ] return repair_dict - + metadata = repository.metadata repair_dict = {} tpm = tool_panel_manager.ToolPanelManager( self.app ) diff -r 97f46da40bb3767c6cf0d440b1d2d58e06edf5e6 -r 50062f6e9b47a36dad57393292611632908fe79a lib/tool_shed/galaxy_install/repository_dependencies/repository_dependency_manager.py --- a/lib/tool_shed/galaxy_install/repository_dependencies/repository_dependency_manager.py +++ b/lib/tool_shed/galaxy_install/repository_dependencies/repository_dependency_manager.py @@ -38,13 +38,13 @@ for repo_info_dict in repo_info_dicts: for name, repo_info_tuple in repo_info_dict.items(): description, \ - repository_clone_url, \ - changeset_revision, \ - ctx_rev, \ - repository_owner, \ - repository_dependencies, \ - tool_dependencies = \ - suc.get_repo_info_tuple_contents( repo_info_tuple ) + repository_clone_url, \ + changeset_revision, \ + ctx_rev, \ + repository_owner, \ + repository_dependencies, \ + tool_dependencies = \ + suc.get_repo_info_tuple_contents( repo_info_tuple ) if repository_dependencies: for key, val in repository_dependencies.items(): if key in [ 'root_key', 'description' ]: @@ -71,12 +71,12 @@ for repository_dependency_components_list in val: required_repository = None rd_toolshed, \ - rd_name, \ - rd_owner, \ - rd_changeset_revision, \ - rd_prior_installation_required, \ - rd_only_if_compiling_contained_td = \ - common_util.parse_repository_dependency_tuple( repository_dependency_components_list ) + rd_name, \ + rd_owner, \ + rd_changeset_revision, \ + rd_prior_installation_required, \ + rd_only_if_compiling_contained_td = \ + common_util.parse_repository_dependency_tuple( repository_dependency_components_list ) # Get the the tool_shed_repository defined by rd_name, rd_owner and rd_changeset_revision. This # is the repository that will be required by the current d_repository. # TODO: Check tool_shed_repository.tool_shed as well when repository dependencies across tool sheds is supported. @@ -101,10 +101,10 @@ repository_dependency = self.get_repository_dependency_by_repository_id( install_model, required_repository.id ) if not repository_dependency: - log.debug( 'Creating new repository_dependency record for installed revision %s of repository: %s owned by %s.' % \ - ( str( required_repository.installed_changeset_revision ), - str( required_repository.name ), - str( required_repository.owner ) ) ) + log.debug( 'Creating new repository_dependency record for installed revision %s of repository: %s owned by %s.' % + ( str( required_repository.installed_changeset_revision ), + str( required_repository.name ), + str( required_repository.owner ) ) ) repository_dependency = install_model.RepositoryDependency( tool_shed_repository_id=required_repository.id ) install_model.context.add( repository_dependency ) install_model.context.flush() @@ -159,13 +159,13 @@ for name, repo_info_tuple in repo_info_dict.items(): can_update_db_record = False description, \ - repository_clone_url, \ - changeset_revision, \ - ctx_rev, \ - repository_owner, \ - repository_dependencies, \ - tool_dependencies = \ - suc.get_repo_info_tuple_contents( repo_info_tuple ) + repository_clone_url, \ + changeset_revision, \ + ctx_rev, \ + repository_owner, \ + repository_dependencies, \ + tool_dependencies = \ + suc.get_repo_info_tuple_contents( repo_info_tuple ) # See if the repository has an existing record in the database. repository_db_record, installed_changeset_revision = \ suc.repository_was_previously_installed( self.app, tool_shed_url, name, repo_info_tuple, from_tip=False ) @@ -190,8 +190,6 @@ # record in the database. name = repository_db_record.name installed_changeset_revision = repository_db_record.installed_changeset_revision - metadata_dict = repository_db_record.metadata - dist_to_shed = repository_db_record.dist_to_shed can_update_db_record = True elif repository_db_record.status in [ install_model.ToolShedRepository.installation_status.DEACTIVATED ]: # The current tool shed repository is deactivated, so updating its database record @@ -209,8 +207,6 @@ else: # No record exists in the database for the repository currently being processed. installed_changeset_revision = changeset_revision - metadata_dict = {} - dist_to_shed = False can_update_db_record = True if can_update_db_record: # The database record for the tool shed repository currently being processed can be updated. @@ -348,13 +344,10 @@ # a dependency of the dependent repository's contained tool dependency, and only if that # tool dependency requires compilation. # For backward compatibility to the 12/20/12 Galaxy release. - prior_installation_required = 'False' only_if_compiling_contained_td = 'False' if len( components_list ) == 4: - prior_installation_required = 'False' only_if_compiling_contained_td = 'False' elif len( components_list ) == 5: - prior_installation_required = components_list[ 4 ] only_if_compiling_contained_td = 'False' if not asbool( only_if_compiling_contained_td ): if components_list not in required_repository_tups: https://bitbucket.org/galaxy/galaxy-central/commits/032eefd299cb/ Changeset: 032eefd299cb User: jmchilton Date: 2014-12-28 16:23:44+00:00 Summary: Tweak two ToolShed modules to clarify they don't use ToolSection... ... objects (directly anyway). I would like to refactor all the tool shed components to not deal directly with ToolSection objects but instead just pass around IDs. Affected #: 2 files diff -r 50062f6e9b47a36dad57393292611632908fe79a -r 032eefd299cbfd103ea85103e64147d436068fb5 lib/tool_shed/galaxy_install/repair_repository_manager.py --- a/lib/tool_shed/galaxy_install/repair_repository_manager.py +++ b/lib/tool_shed/galaxy_install/repair_repository_manager.py @@ -123,7 +123,7 @@ tool_panel_section_name = section_dict[ 'name' ] if tool_panel_section_id: tpm = tool_panel_manager.ToolPanelManager( self.app ) - tool_panel_section_key, tool_panel_section = \ + tool_panel_section_key, _ = \ tpm.get_or_create_tool_section( self.app.toolbox, tool_panel_section_id=tool_panel_section_id, new_tool_panel_section_label=tool_panel_section_name ) @@ -165,7 +165,7 @@ suc.get_tool_panel_config_tool_path_install_dir( self.app, repository ) # Reset the repository attributes to the New state for installation. if metadata: - tool_section, tool_panel_section_key = \ + _, tool_panel_section_key = \ tpm.handle_tool_panel_selection( self.app.toolbox, metadata, no_changes_checked=True, diff -r 50062f6e9b47a36dad57393292611632908fe79a -r 032eefd299cbfd103ea85103e64147d436068fb5 lib/tool_shed/galaxy_install/repository_dependencies/repository_dependency_manager.py --- a/lib/tool_shed/galaxy_install/repository_dependencies/repository_dependency_manager.py +++ b/lib/tool_shed/galaxy_install/repository_dependencies/repository_dependency_manager.py @@ -213,7 +213,7 @@ # Get the repository metadata to see where it was previously located in the tool panel. tpm = tool_panel_manager.ToolPanelManager( self.app ) if repository_db_record and repository_db_record.metadata: - tool_section, tool_panel_section_key = \ + _, tool_panel_section_key = \ tpm.handle_tool_panel_selection( toolbox=self.app.toolbox, metadata=repository_db_record.metadata, no_changes_checked=no_changes_checked, @@ -221,7 +221,7 @@ new_tool_panel_section_label=new_tool_panel_section_label ) else: # We're installing a new tool shed repository that does not yet have a database record. - tool_panel_section_key, tool_section = \ + tool_panel_section_key, _ = \ tpm.handle_tool_panel_section( self.app.toolbox, tool_panel_section_id=tool_panel_section_id, new_tool_panel_section_label=new_tool_panel_section_label ) https://bitbucket.org/galaxy/galaxy-central/commits/6d4909f34123/ Changeset: 6d4909f34123 User: jmchilton Date: 2014-12-28 16:23:44+00:00 Summary: More centralization of code for registering tools in ToolBox. Affected #: 1 file diff -r 032eefd299cbfd103ea85103e64147d436068fb5 -r 6d4909f34123fe53f1497dd2ea27a611039ae5c0 lib/galaxy/tools/__init__.py --- a/lib/galaxy/tools/__init__.py +++ b/lib/galaxy/tools/__init__.py @@ -679,7 +679,7 @@ # changed, so the tool should always be reloaded here. We used to # only load the tool if it was not found in self.tools_by_id, but # performing that check did not enable this scenario. - self.tools_by_id[ tool.id ] = tool + self.register_tool( tool ) if load_panel_dict: self.__add_tool_to_tool_panel( tool, panel_dict, section=isinstance( panel_dict, ToolSection ) ) https://bitbucket.org/galaxy/galaxy-central/commits/b499a560826b/ Changeset: b499a560826b User: jmchilton Date: 2014-12-28 16:23:44+00:00 Summary: Eliminate one reference to ToolBox.tool_panel in install_manager.py. Affected #: 2 files diff -r 6d4909f34123fe53f1497dd2ea27a611039ae5c0 -r b499a560826b2d3a3cfc16dc44b99dafa7c3802e lib/galaxy/tools/__init__.py --- a/lib/galaxy/tools/__init__.py +++ b/lib/galaxy/tools/__init__.py @@ -1021,6 +1021,24 @@ status = 'done' return message, status + def find_section_id( self, tool_panel_section_id ): + """ + Find the section ID referenced by the key or return '' indicating + no such section id. + """ + if not tool_panel_section_id: + tool_panel_section_id = '' + else: + if tool_panel_section_id not in self.tool_panel: + # Hack introduced without comment in a29d54619813d5da992b897557162a360b8d610c- + # not sure why it is needed. + fixed_tool_panel_section_id = 'section_%s' % tool_panel_section_id + if fixed_tool_panel_section_id in self.tool_panel: + tool_panel_section_id = fixed_tool_panel_section_id + else: + tool_panel_section_id = '' + return tool_panel_section_id + def load_workflow( self, workflow_id ): """ Return an instance of 'Workflow' identified by `id`, diff -r 6d4909f34123fe53f1497dd2ea27a611039ae5c0 -r b499a560826b2d3a3cfc16dc44b99dafa7c3802e lib/tool_shed/galaxy_install/install_manager.py --- a/lib/tool_shed/galaxy_install/install_manager.py +++ b/lib/tool_shed/galaxy_install/install_manager.py @@ -742,16 +742,7 @@ tool_path = shed_config_dict[ 'tool_path' ] if not shed_tool_conf: raise exceptions.RequestParameterMissingException( "Missing required parameter 'shed_tool_conf'." ) - tool_panel_section_id = install_options.get( 'tool_panel_section_id', '' ) - if tool_panel_section_id not in [ None, '' ]: - if tool_panel_section_id not in self.app.toolbox.tool_panel: - fixed_tool_panel_section_id = 'section_%s' % tool_panel_section_id - if fixed_tool_panel_section_id in self.app.toolbox.tool_panel: - tool_panel_section_id = fixed_tool_panel_section_id - else: - tool_panel_section_id = '' - else: - tool_panel_section_id = '' + tool_panel_section_id = self.app.toolbox.find_section_id( install_options.get( 'tool_panel_section_id', '' ) ) # Build the dictionary of information necessary for creating tool_shed_repository database records # for each repository being installed. installation_dict = dict( install_repository_dependencies=install_repository_dependencies, https://bitbucket.org/galaxy/galaxy-central/commits/54b778976b7b/ Changeset: 54b778976b7b User: jmchilton Date: 2014-12-28 16:23:44+00:00 Summary: Simplify shed-y logic in load_tool_tag_set. Drops cycolomatic complexity from 14 to 10 for that method. Affected #: 2 files diff -r b499a560826b2d3a3cfc16dc44b99dafa7c3802e -r 54b778976b7bb297160955170fda6ff1efd2d62a lib/galaxy/tools/__init__.py --- a/lib/galaxy/tools/__init__.py +++ b/lib/galaxy/tools/__init__.py @@ -559,6 +559,23 @@ def tools( self ): return self.tools_by_id.iteritems() + def __elem_to_tool_shed_repository( self, elem ): + # The tool is contained in an installed tool shed repository, so load + # the tool only if the repository has not been marked deleted. + tool_shed = elem.find( "tool_shed" ).text + repository_name = elem.find( "repository_name" ).text + repository_owner = elem.find( "repository_owner" ).text + installed_changeset_revision_elem = elem.find( "installed_changeset_revision" ) + if installed_changeset_revision_elem is None: + # Backward compatibility issue - the tag used to be named 'changeset_revision'. + installed_changeset_revision_elem = elem.find( "changeset_revision" ) + installed_changeset_revision = installed_changeset_revision_elem.text + tool_shed_repository = self.__get_tool_shed_repository( tool_shed, + repository_name, + repository_owner, + installed_changeset_revision ) + return tool_shed_repository + def __get_tool_shed_repository( self, tool_shed, name, owner, installed_changeset_revision ): # We store only the port, if one exists, in the database. tool_shed = common_util.remove_protocol_from_tool_shed_url( tool_shed ) @@ -613,42 +630,29 @@ try: path = elem.get( "file" ) repository_id = None - if guid is None: - tool_shed_repository = None - can_load_into_panel_dict = True - else: + + tool_shed_repository = None + can_load_into_panel_dict = True + if guid is not None: # The tool is contained in an installed tool shed repository, so load # the tool only if the repository has not been marked deleted. - tool_shed = elem.find( "tool_shed" ).text - repository_name = elem.find( "repository_name" ).text - repository_owner = elem.find( "repository_owner" ).text - installed_changeset_revision_elem = elem.find( "installed_changeset_revision" ) - if installed_changeset_revision_elem is None: - # Backward compatibility issue - the tag used to be named 'changeset_revision'. - installed_changeset_revision_elem = elem.find( "changeset_revision" ) - installed_changeset_revision = installed_changeset_revision_elem.text - tool_shed_repository = self.__get_tool_shed_repository( tool_shed, - repository_name, - repository_owner, - installed_changeset_revision ) + tool_shed_repository = self.__elem_to_tool_shed_repository( elem ) if tool_shed_repository: # Only load tools if the repository is not deactivated or uninstalled. can_load_into_panel_dict = not tool_shed_repository.deleted repository_id = self.app.security.encode_id( tool_shed_repository.id ) - else: - # If there is not yet a tool_shed_repository record, we're in the process of installing - # a new repository, so any included tools can be loaded into the tool panel. - can_load_into_panel_dict = True + # Else there is not yet a tool_shed_repository record, we're in the process of installing + # a new repository, so any included tools can be loaded into the tool panel. tool = self.load_tool( os.path.join( tool_path, path ), guid=guid, repository_id=repository_id ) if string_as_bool(elem.get( 'hidden', False )): tool.hidden = True key = 'tool_%s' % str( tool.id ) if can_load_into_panel_dict: if guid is not None: - tool.tool_shed = tool_shed - tool.repository_name = repository_name - tool.repository_owner = repository_owner - tool.installed_changeset_revision = installed_changeset_revision + tool.tool_shed = tool_shed_repository.tool_shed + tool.repository_name = tool_shed_repository.name + tool.repository_owner = tool_shed_repository.owner + tool.installed_changeset_revision = tool_shed_repository.installed_changeset_revision tool.guid = guid tool.version = elem.find( "version" ).text # Make sure the tool has a tool_version. diff -r b499a560826b2d3a3cfc16dc44b99dafa7c3802e -r 54b778976b7bb297160955170fda6ff1efd2d62a test/unit/tools/test_toolbox.py --- a/test/unit/tools/test_toolbox.py +++ b/test/unit/tools/test_toolbox.py @@ -136,6 +136,27 @@ assert toolbox.get_tool( "test_tool" ) is not None assert toolbox.get_tool( "not_a_test_tool" ) is None + def test_tool_shed_properties( self ): + self._init_tool() + self._setup_two_versions_in_config( section=False ) + self._setup_two_versions() + + test_tool = self.toolbox.get_tool( "test_tool" ) + assert test_tool.tool_shed == "github.com" + assert test_tool.repository_owner == "galaxyproject" + assert test_tool.repository_name == "example" + assert test_tool.installed_changeset_revision == "1" + + def test_tool_shed_properties_only_on_installed_tools( self ): + self._init_tool() + self._add_config( """<toolbox><tool file="tool.xml" /></toolbox>""" ) + toolbox = self.toolbox + test_tool = toolbox.get_tool( "test_tool" ) + assert test_tool.tool_shed is None + assert test_tool.repository_name is None + assert test_tool.repository_owner is None + assert test_tool.installed_changeset_revision is None + def test_load_file_in_section( self ): self._init_tool() self._add_config( """<toolbox><section id="t" name="test"><tool file="tool.xml" /></section></toolbox>""" ) https://bitbucket.org/galaxy/galaxy-central/commits/1912dec92b72/ Changeset: 1912dec92b72 User: jmchilton Date: 2014-12-28 16:23:44+00:00 Summary: De-duplication for loading elements into the tool panel. Create one public method load_item on ToolBox that should be used externally instead of load_tool_tag_set, load_section_tag_set, etc.... This reduces the duplication within the toolbox and between the toolbox and the Tool Shed's ToolPanelManager. Probably more importantly it also is another step down the road toward hiding implementation details such as toolbox.tool_panel and toolbox.intergrated_tool_panel from the ToolPanelManager. Already had unit test coverage for ToolPanelManager stuff here - but added test coverage for loading labels, workflows, and tool directories. Affected #: 3 files diff -r 54b778976b7bb297160955170fda6ff1efd2d62a -r 1912dec92b728b974094456be8eb6d84d34a7ca6 lib/galaxy/tools/__init__.py --- a/lib/galaxy/tools/__init__.py +++ b/lib/galaxy/tools/__init__.py @@ -222,16 +222,14 @@ self.index += 1 if parsing_shed_tool_conf: config_elems.append( elem ) - if elem.tag == 'tool': - self.load_tool_tag_set( elem, self.tool_panel, self.integrated_tool_panel, tool_path, load_panel_dict, guid=elem.get( 'guid' ), index=index ) - elif elem.tag == 'workflow': - self.load_workflow_tag_set( elem, self.tool_panel, self.integrated_tool_panel, load_panel_dict, index=index ) - elif elem.tag == 'section': - self.load_section_tag_set( elem, tool_path, load_panel_dict, index=index ) - elif elem.tag == 'label': - self.load_label_tag_set( elem, self.tool_panel, self.integrated_tool_panel, load_panel_dict, index=index ) - elif elem.tag == 'tool_dir': - self.load_tooldir_tag_set( elem, self.tool_panel, self.integrated_tool_panel, tool_path, load_panel_dict ) + self.load_item( + elem, + tool_path=tool_path, + load_panel_dict=load_panel_dict, + guid=elem.get( 'guid' ), + index=index, + internal=True + ) if parsing_shed_tool_conf: shed_tool_conf_dict = dict( config_filename=config_filename, @@ -239,6 +237,28 @@ config_elems=config_elems ) self.shed_tool_confs.append( shed_tool_conf_dict ) + def load_item( self, elem, tool_path, panel_dict=None, integrated_panel_dict=None, load_panel_dict=True, guid=None, index=None, internal=False ): + item_type = elem.tag + if item_type not in ['tool', 'section'] and not internal: + # External calls from tool shed code cannot load labels or tool + # directories. + return + + if panel_dict is None: + panel_dict = self.tool_panel + if integrated_panel_dict is None: + integrated_panel_dict = self.integrated_tool_panel + if item_type == 'tool': + self.load_tool_tag_set( elem, panel_dict=panel_dict, integrated_panel_dict=integrated_panel_dict, tool_path=tool_path, load_panel_dict=load_panel_dict, guid=guid, index=index ) + elif item_type == 'workflow': + self.load_workflow_tag_set( elem, panel_dict=panel_dict, integrated_panel_dict=integrated_panel_dict, load_panel_dict=load_panel_dict, index=index ) + elif item_type == 'section': + self.load_section_tag_set( elem, tool_path=tool_path, load_panel_dict=load_panel_dict, index=index ) + elif item_type == 'label': + self.load_label_tag_set( elem, panel_dict=panel_dict, integrated_panel_dict=integrated_panel_dict, load_panel_dict=load_panel_dict, index=index ) + elif item_type == 'tool_dir': + self.load_tooldir_tag_set( elem, panel_dict, tool_path, integrated_panel_dict, load_panel_dict=load_panel_dict ) + def get_shed_config_dict_by_filename( self, filename, default=None ): for shed_config_dict in self.shed_tool_confs: if shed_config_dict[ 'config_filename' ] == filename: @@ -729,14 +749,16 @@ integrated_section = ToolSection( elem ) integrated_elems = integrated_section.elems for sub_index, sub_elem in enumerate( elem ): - if sub_elem.tag == 'tool': - self.load_tool_tag_set( sub_elem, elems, integrated_elems, tool_path, load_panel_dict, guid=sub_elem.get( 'guid' ), index=sub_index ) - elif sub_elem.tag == 'workflow': - self.load_workflow_tag_set( sub_elem, elems, integrated_elems, load_panel_dict, index=sub_index ) - elif sub_elem.tag == 'label': - self.load_label_tag_set( sub_elem, elems, integrated_elems, load_panel_dict, index=sub_index ) - elif sub_elem.tag == 'tool_dir': - self.load_tooldir_tag_set( sub_elem, elems, tool_path, integrated_elems, load_panel_dict ) + self.load_item( + sub_elem, + tool_path=tool_path, + panel_dict=elems, + integrated_panel_dict=integrated_elems, + load_panel_dict=load_panel_dict, + guid=sub_elem.get( 'guid' ), + index=sub_index, + internal=True, + ) if load_panel_dict: self.tool_panel[ key ] = section # Always load sections into the integrated_tool_panel. diff -r 54b778976b7bb297160955170fda6ff1efd2d62a -r 1912dec92b728b974094456be8eb6d84d34a7ca6 lib/tool_shed/galaxy_install/tools/tool_panel_manager.py --- a/lib/tool_shed/galaxy_install/tools/tool_panel_manager.py +++ b/lib/tool_shed/galaxy_install/tools/tool_panel_manager.py @@ -58,22 +58,12 @@ # Add the new elements to the in-memory list of config_elems. config_elems.append( config_elem ) # Load the tools into the in-memory tool panel. - if config_elem.tag == 'section': - self.app.toolbox.load_section_tag_set( config_elem, tool_path, load_panel_dict=True ) - elif config_elem.tag == 'workflow': - self.app.toolbox.load_workflow_tag_set( config_elem, - self.app.toolbox.tool_panel, - self.app.toolbox.integrated_tool_panel, - load_panel_dict=True ) - elif config_elem.tag == 'tool': - self.app.toolbox.load_tool_tag_set( - config_elem, - self.app.toolbox.tool_panel, - self.app.toolbox.integrated_tool_panel, - tool_path, - load_panel_dict=True, - guid=config_elem.get( 'guid' ) - ) + self.app.toolbox.load_item( + config_elem, + tool_path=tool_path, + load_panel_dict=True, + guid=config_elem.get( 'guid' ), + ) # Replace the old list of in-memory config_elems with the new list for this shed_tool_conf_dict. shed_tool_conf_dict[ 'config_elems' ] = config_elems self.app.toolbox.update_shed_config( index, shed_tool_conf_dict ) diff -r 54b778976b7bb297160955170fda6ff1efd2d62a -r 1912dec92b728b974094456be8eb6d84d34a7ca6 test/unit/tools/test_toolbox.py --- a/test/unit/tools/test_toolbox.py +++ b/test/unit/tools/test_toolbox.py @@ -3,6 +3,7 @@ import unittest from galaxy.tools import ToolBox +from galaxy import model from galaxy.model import tool_shed_install from galaxy.model.tool_shed_install import mapping import tools_support @@ -216,6 +217,67 @@ assert self.toolbox.get_tool_id( "github.com/galaxyproect/example/test_tool/0.2" ) == "github.com/galaxyproect/example/test_tool/0.2" assert self.toolbox.get_tool_id( "github.com/galaxyproect/example/test_tool/0.3" ) is None + def test_tool_dir( self ): + self._init_tool() + self._add_config( """<toolbox><tool_dir dir="%s" /></toolbox>""" % self.test_directory ) + + toolbox = self.toolbox + assert toolbox.get_tool( "test_tool" ) is not None + + def test_workflow_in_panel( self ): + stored_workflow = self.__test_workflow() + encoded_id = self.app.security.encode_id( stored_workflow.id ) + self._add_config( """<toolbox><workflow id="%s" /></toolbox>""" % encoded_id ) + assert len( self.toolbox.tool_panel ) == 1 + panel_workflow = self.toolbox.tool_panel.values()[ 0 ] + assert panel_workflow == stored_workflow.latest_workflow + # TODO: test to_dict with workflows + + def test_workflow_in_section( self ): + stored_workflow = self.__test_workflow() + encoded_id = self.app.security.encode_id( stored_workflow.id ) + self._add_config( """<toolbox><section id="tid" name="TID"><workflow id="%s" /></section></toolbox>""" % encoded_id ) + assert len( self.toolbox.tool_panel ) == 1 + section = self.toolbox.tool_panel[ 'tid' ] + assert len( section.elems ) == 1 + panel_workflow = section.elems.values()[ 0 ] + assert panel_workflow == stored_workflow.latest_workflow + + def test_label_in_panel( self ): + self._add_config( """<toolbox><label id="lab1" text="Label 1" /><label id="lab2" text="Label 2" /></toolbox>""" ) + assert len( self.toolbox.tool_panel ) == 2 + self.__check_test_labels( self.toolbox.tool_panel ) + + def test_label_in_section( self ): + self._add_config( """<toolbox><section id="tid" name="TID"><label id="lab1" text="Label 1" /><label id="lab2" text="Label 2" /></section></toolbox>""" ) + assert len( self.toolbox.tool_panel ) == 1 + section = self.toolbox.tool_panel[ 'tid' ] + self.__check_test_labels( section.elems ) + + def __check_test_labels( self, panel_dict ): + assert panel_dict.keys() == ["label_lab1", "label_lab2"] + label1 = panel_dict.values()[ 0 ] + assert label1.id == "lab1" + assert label1.text == "Label 1" + + label2 = panel_dict[ "label_lab2" ] + assert label2.id == "lab2" + assert label2.text == "Label 2" + + def __test_workflow( self ): + stored_workflow = model.StoredWorkflow() + workflow = model.Workflow() + workflow.stored_workflow = stored_workflow + stored_workflow.latest_workflow = workflow + user = model.User() + user.email = "test(a)example.com" + user.password = "passw0rD1" + stored_workflow.user = user + self.app.model.context.add( workflow ) + self.app.model.context.add( stored_workflow ) + self.app.model.context.flush() + return stored_workflow + def __verify_two_test_tools( self ): # Assert tool versions of the tool with simple id 'test_tool' all_versions = self.toolbox.get_tool( "test_tool", get_all_versions=True ) Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: jmchilton: Fix remaining tool shed test cases for new escaped values.
by commits-noreply＠bitbucket.org 28 Dec '14

28 Dec '14

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/97f46da40bb3/ Changeset: 97f46da40bb3 User: jmchilton Date: 2014-12-28 04:46:22+00:00 Summary: Fix remaining tool shed test cases for new escaped values. I think in this case the values on the page are correct now - but the test framework needs to be updated to handle the escaped value. Affected #: 1 file diff -r 461a6a9379001959b5b3259f90a2b9471ce391ed -r 97f46da40bb3767c6cf0d440b1d2d58e06edf5e6 test/tool_shed/base/twilltestcase.py --- a/test/tool_shed/base/twilltestcase.py +++ b/test/tool_shed/base/twilltestcase.py @@ -430,6 +430,8 @@ url = '/admin_toolshed/manage_repository?id=%s' % self.security.encode_id( installed_repository.id ) self.visit_galaxy_url( url ) strings_displayed.append( str( installed_repository.installed_changeset_revision ) ) + # Every place Galaxy's XXXX tool appears in attribute - need to quote. + strings_displayed = map( lambda x: x.replace("'", "'"), strings_displayed ) self.check_for_strings( strings_displayed, strings_not_displayed ) def display_installed_workflow_image( self, repository, workflow_name, strings_displayed=[], strings_not_displayed=[] ): Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: 2 new changesets
by commits-noreply＠bitbucket.org 27 Dec '14

27 Dec '14

2 new commits in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/86241a27f4d8/ Changeset: 86241a27f4d8 User: jmchilton Date: 2014-12-27 22:49:16+00:00 Summary: Add --no_cleanup option to tests. Affected #: 1 file diff -r 1ed5a63aae0a5c8091483658c579d9d762a4c3fb -r 86241a27f4d8d7065fb7908a0e687d6b11a062eb run_tests.sh --- a/run_tests.sh +++ b/run_tests.sh @@ -21,6 +21,12 @@ '${0##*/} -unit testscriptath' running particular tests scripts '${0##*/} -qunit' for running qunit JavaScript tests '${0##*/} -qunit testname' for running single JavaScript test with given name + +Extra options: + + --no_cleanup Do not delete temp files for Python functional tests (-toolshed, -framework, etc...) + --report_file Path of HTML report to produce (for Python Galaxy functional tests). + --xunit_report_file Path of XUnit report to produce (for Python Galaxy functional tests). EOF } @@ -196,6 +202,16 @@ shift 1 fi ;; + --no_cleanup) + GALAXY_TEST_NO_CLEANUP=1 + export GALAXY_TEST_NO_CLEANUP + TOOL_SHED_TEST_NO_CLEANUP=1 + export TOOL_SHED_TEST_NO_CLEANUP + GALAXY_INSTALL_TEST_NO_CLEANUP=1 + export GALAXY_INSTALL_TEST_NO_CLEANUP + echo "Skipping Python test clean up." + shift + ;; -watch|--watch) # Have grunt watch test or directory for changes, only # valid for javascript testing. https://bitbucket.org/galaxy/galaxy-central/commits/461a6a937900/ Changeset: 461a6a937900 User: jmchilton Date: 2014-12-27 22:49:16+00:00 Summary: Skip checking tool migrations for Galaxy in tool shed tests. Affected #: 1 file diff -r 86241a27f4d8d7065fb7908a0e687d6b11a062eb -r 461a6a9379001959b5b3259f90a2b9471ce391ed test/tool_shed/functional_tests.py --- a/test/tool_shed/functional_tests.py +++ b/test/tool_shed/functional_tests.py @@ -359,6 +359,7 @@ database_connection = galaxy_database_connection, database_auto_migrate = galaxy_database_auto_migrate, datatype_converters_config_file = "datatype_converters_conf.xml.sample", + check_migrate_tools = False, enable_tool_shed_check = True, file_path = galaxy_file_path, global_conf = galaxy_global_conf, Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: jmchilton: Merge next-stable.
by commits-noreply＠bitbucket.org 27 Dec '14

27 Dec '14

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/1ed5a63aae0a/ Changeset: 1ed5a63aae0a User: jmchilton Date: 2014-12-27 22:35:51+00:00 Summary: Merge next-stable. Affected #: 9 files diff -r 63ec80be64bb00414be0f7122a51907114ffb714 -r 1ed5a63aae0a5c8091483658c579d9d762a4c3fb lib/galaxy/web/base/controllers/admin.py --- a/lib/galaxy/web/base/controllers/admin.py +++ b/lib/galaxy/web/base/controllers/admin.py @@ -7,7 +7,7 @@ from galaxy.web.form_builder import CheckboxField from string import punctuation as PUNCTUATION import galaxy.queue_worker -from markupsafe import escape +from tool_shed.util.web_util import escape from tool_shed.util import shed_util_common as suc diff -r 63ec80be64bb00414be0f7122a51907114ffb714 -r 1ed5a63aae0a5c8091483658c579d9d762a4c3fb lib/galaxy/webapps/galaxy/controllers/admin.py --- a/lib/galaxy/webapps/galaxy/controllers/admin.py +++ b/lib/galaxy/webapps/galaxy/controllers/admin.py @@ -17,7 +17,7 @@ from galaxy.web.params import QuotaParamParser from tool_shed.util import common_util from tool_shed.util import encoding_util -from markupsafe import escape +from tool_shed.util.web_util import escape log = logging.getLogger( __name__ ) diff -r 63ec80be64bb00414be0f7122a51907114ffb714 -r 1ed5a63aae0a5c8091483658c579d9d762a4c3fb lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py --- a/lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py +++ b/lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py @@ -8,9 +8,9 @@ from galaxy.web.form_builder import CheckboxField from galaxy.util import json from galaxy.model.orm import or_ -from markupsafe import escape import tool_shed.repository_types.util as rt_util +from tool_shed.util.web_util import escape from tool_shed.util import common_util from tool_shed.util import encoding_util diff -r 63ec80be64bb00414be0f7122a51907114ffb714 -r 1ed5a63aae0a5c8091483658c579d9d762a4c3fb lib/galaxy/webapps/tool_shed/controllers/admin.py --- a/lib/galaxy/webapps/tool_shed/controllers/admin.py +++ b/lib/galaxy/webapps/tool_shed/controllers/admin.py @@ -3,7 +3,7 @@ from galaxy import util from galaxy.util import inflector from galaxy import web -from markupsafe import escape +from tool_shed.util.web_util import escape from galaxy.web.base.controller import BaseUIController from galaxy.web.base.controllers.admin import Admin diff -r 63ec80be64bb00414be0f7122a51907114ffb714 -r 1ed5a63aae0a5c8091483658c579d9d762a4c3fb lib/galaxy/webapps/tool_shed/controllers/repository.py --- a/lib/galaxy/webapps/tool_shed/controllers/repository.py +++ b/lib/galaxy/webapps/tool_shed/controllers/repository.py @@ -6,7 +6,6 @@ from time import strftime from datetime import date from datetime import datetime -from markupsafe import escape from galaxy import util from galaxy import web @@ -19,6 +18,7 @@ from tool_shed.capsule import capsule_manager from tool_shed.dependencies.repository import relation_builder +from tool_shed.util.web_util import escape from tool_shed.galaxy_install import dependency_display from tool_shed.metadata import repository_metadata_manager from tool_shed.utility_containers import ToolShedUtilityContainerManager diff -r 63ec80be64bb00414be0f7122a51907114ffb714 -r 1ed5a63aae0a5c8091483658c579d9d762a4c3fb lib/galaxy/webapps/tool_shed/controllers/repository_review.py --- a/lib/galaxy/webapps/tool_shed/controllers/repository_review.py +++ b/lib/galaxy/webapps/tool_shed/controllers/repository_review.py @@ -2,7 +2,7 @@ import os from sqlalchemy.sql.expression import func -from markupsafe import escape +from tool_shed.util.web_util import escape from galaxy import util from galaxy import web diff -r 63ec80be64bb00414be0f7122a51907114ffb714 -r 1ed5a63aae0a5c8091483658c579d9d762a4c3fb lib/galaxy/webapps/tool_shed/controllers/upload.py --- a/lib/galaxy/webapps/tool_shed/controllers/upload.py +++ b/lib/galaxy/webapps/tool_shed/controllers/upload.py @@ -9,7 +9,7 @@ from galaxy import web from galaxy.datatypes import checkers from galaxy.web.base.controller import BaseUIController -from markupsafe import escape +from tool_shed.util.web_util import escape from tool_shed.dependencies import attribute_handlers from tool_shed.galaxy_install import dependency_display diff -r 63ec80be64bb00414be0f7122a51907114ffb714 -r 1ed5a63aae0a5c8091483658c579d9d762a4c3fb lib/tool_shed/util/repository_util.py --- a/lib/tool_shed/util/repository_util.py +++ b/lib/tool_shed/util/repository_util.py @@ -7,7 +7,7 @@ from galaxy import web from galaxy.web.form_builder import build_select_field from galaxy.webapps.tool_shed.model import directory_hash_id -from markupsafe import escape +from tool_shed.util.web_util import escape from tool_shed.dependencies.repository import relation_builder diff -r 63ec80be64bb00414be0f7122a51907114ffb714 -r 1ed5a63aae0a5c8091483658c579d9d762a4c3fb lib/tool_shed/util/web_util.py --- /dev/null +++ b/lib/tool_shed/util/web_util.py @@ -0,0 +1,21 @@ +from markupsafe import escape as raw_escape + +ALLOWED_ELEMENTS = ["<b>", "</b>", "<br/>"] +ALLOWED_MAP = dict(map(lambda x: (x, raw_escape(x)), ALLOWED_ELEMENTS)) + + +def escape( value ): + """ A tool shed variant of markupsafe.escape that allows a select few + HTML elements that are repeatedly used in messages created deep + in the toolshed components. Ideally abstract things would be produced + in these components and messages in the views or client side - this is + what should be worked toward - but for now - we have this hack. + + >>> escape("A <b>repo</b>") + u'A <b>repo</b>' + """ + escaped = str( raw_escape( value ) ) + # Unescape few selected tags. + for key, value in ALLOWED_MAP.iteritems(): + escaped = escaped.replace(value, key) + return escaped Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: jmchilton: Fixes for over escaping in c2bed0a.
by commits-noreply＠bitbucket.org 27 Dec '14

27 Dec '14

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/93a868eb314d/ Changeset: 93a868eb314d Branch: next-stable User: jmchilton Date: 2014-12-27 22:30:59+00:00 Summary: Fixes for over escaping in c2bed0a. Fixes dozens of tool functional tests. Affected #: 9 files diff -r 501f510ff101849c45d9b0925bb55fe59319088b -r 93a868eb314d257c21bb19380b360bcf0d8dd82c lib/galaxy/web/base/controllers/admin.py --- a/lib/galaxy/web/base/controllers/admin.py +++ b/lib/galaxy/web/base/controllers/admin.py @@ -7,7 +7,7 @@ from galaxy.web.form_builder import CheckboxField from string import punctuation as PUNCTUATION import galaxy.queue_worker -from markupsafe import escape +from tool_shed.util.web_util import escape from tool_shed.util import shed_util_common as suc diff -r 501f510ff101849c45d9b0925bb55fe59319088b -r 93a868eb314d257c21bb19380b360bcf0d8dd82c lib/galaxy/webapps/galaxy/controllers/admin.py --- a/lib/galaxy/webapps/galaxy/controllers/admin.py +++ b/lib/galaxy/webapps/galaxy/controllers/admin.py @@ -17,7 +17,7 @@ from galaxy.web.params import QuotaParamParser from tool_shed.util import common_util from tool_shed.util import encoding_util -from markupsafe import escape +from tool_shed.util.web_util import escape log = logging.getLogger( __name__ ) diff -r 501f510ff101849c45d9b0925bb55fe59319088b -r 93a868eb314d257c21bb19380b360bcf0d8dd82c lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py --- a/lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py +++ b/lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py @@ -8,9 +8,9 @@ from galaxy.web.form_builder import CheckboxField from galaxy.util import json from galaxy.model.orm import or_ -from markupsafe import escape import tool_shed.repository_types.util as rt_util +from tool_shed.util.web_util import escape from tool_shed.util import common_util from tool_shed.util import encoding_util diff -r 501f510ff101849c45d9b0925bb55fe59319088b -r 93a868eb314d257c21bb19380b360bcf0d8dd82c lib/galaxy/webapps/tool_shed/controllers/admin.py --- a/lib/galaxy/webapps/tool_shed/controllers/admin.py +++ b/lib/galaxy/webapps/tool_shed/controllers/admin.py @@ -3,7 +3,7 @@ from galaxy import util from galaxy.util import inflector from galaxy import web -from markupsafe import escape +from tool_shed.util.web_util import escape from galaxy.web.base.controller import BaseUIController from galaxy.web.base.controllers.admin import Admin diff -r 501f510ff101849c45d9b0925bb55fe59319088b -r 93a868eb314d257c21bb19380b360bcf0d8dd82c lib/galaxy/webapps/tool_shed/controllers/repository.py --- a/lib/galaxy/webapps/tool_shed/controllers/repository.py +++ b/lib/galaxy/webapps/tool_shed/controllers/repository.py @@ -6,7 +6,6 @@ from time import strftime from datetime import date from datetime import datetime -from markupsafe import escape from galaxy import util from galaxy import web @@ -19,6 +18,7 @@ from tool_shed.capsule import capsule_manager from tool_shed.dependencies.repository import relation_builder +from tool_shed.util.web_util import escape from tool_shed.galaxy_install import dependency_display from tool_shed.metadata import repository_metadata_manager from tool_shed.utility_containers import ToolShedUtilityContainerManager diff -r 501f510ff101849c45d9b0925bb55fe59319088b -r 93a868eb314d257c21bb19380b360bcf0d8dd82c lib/galaxy/webapps/tool_shed/controllers/repository_review.py --- a/lib/galaxy/webapps/tool_shed/controllers/repository_review.py +++ b/lib/galaxy/webapps/tool_shed/controllers/repository_review.py @@ -2,7 +2,7 @@ import os from sqlalchemy.sql.expression import func -from markupsafe import escape +from tool_shed.util.web_util import escape from galaxy import util from galaxy import web diff -r 501f510ff101849c45d9b0925bb55fe59319088b -r 93a868eb314d257c21bb19380b360bcf0d8dd82c lib/galaxy/webapps/tool_shed/controllers/upload.py --- a/lib/galaxy/webapps/tool_shed/controllers/upload.py +++ b/lib/galaxy/webapps/tool_shed/controllers/upload.py @@ -9,7 +9,7 @@ from galaxy import web from galaxy.datatypes import checkers from galaxy.web.base.controller import BaseUIController -from markupsafe import escape +from tool_shed.util.web_util import escape from tool_shed.dependencies import attribute_handlers from tool_shed.galaxy_install import dependency_display diff -r 501f510ff101849c45d9b0925bb55fe59319088b -r 93a868eb314d257c21bb19380b360bcf0d8dd82c lib/tool_shed/util/repository_util.py --- a/lib/tool_shed/util/repository_util.py +++ b/lib/tool_shed/util/repository_util.py @@ -7,7 +7,7 @@ from galaxy import web from galaxy.web.form_builder import build_select_field from galaxy.webapps.tool_shed.model import directory_hash_id -from markupsafe import escape +from tool_shed.util.web_util import escape from tool_shed.dependencies.repository import relation_builder diff -r 501f510ff101849c45d9b0925bb55fe59319088b -r 93a868eb314d257c21bb19380b360bcf0d8dd82c lib/tool_shed/util/web_util.py --- /dev/null +++ b/lib/tool_shed/util/web_util.py @@ -0,0 +1,21 @@ +from markupsafe import escape as raw_escape + +ALLOWED_ELEMENTS = ["<b>", "</b>", "<br/>"] +ALLOWED_MAP = dict(map(lambda x: (x, raw_escape(x)), ALLOWED_ELEMENTS)) + + +def escape( value ): + """ A tool shed variant of markupsafe.escape that allows a select few + HTML elements that are repeatedly used in messages created deep + in the toolshed components. Ideally abstract things would be produced + in these components and messages in the views or client side - this is + what should be worked toward - but for now - we have this hack. + + >>> escape("A <b>repo</b>") + u'A <b>repo</b>' + """ + escaped = str( raw_escape( value ) ) + # Unescape few selected tags. + for key, value in ALLOWED_MAP.iteritems(): + escaped = escaped.replace(value, key) + return escaped Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

[galaxyproject/usegalaxy-playbook] b8c80f: Disable Stampede for now, it's being unstable.
by GitHub 26 Dec '14

26 Dec '14

Branch: refs/heads/master Home: https://github.com/galaxyproject/usegalaxy-playbook Commit: b8c80fd82c9cd8c602d09bcb97e7a4d567ba84e3 https://github.com/galaxyproject/usegalaxy-playbook/commit/b8c80fd82c9cd8c6… Author: Nate Coraor <nate(a)bx.psu.edu> Date: 2014-12-26 (Fri, 26 Dec 2014) Changed paths: M templates/galaxy/usegalaxy.org/config/job_conf.xml.j2 Log Message: ----------- Disable Stampede for now, it's being unstable.

1 0

commit/galaxy-central: 6 new changesets
by commits-noreply＠bitbucket.org 26 Dec '14

26 Dec '14

6 new commits in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/919dd242fd62/ Changeset: 919dd242fd62 Branch: arff_datatype User: BjoernGruening Date: 2014-12-16 11:40:29+00:00 Summary: Created new branch arff_datatype Affected #: 0 files https://bitbucket.org/galaxy/galaxy-central/commits/807b1e8a7815/ Changeset: 807b1e8a7815 Branch: arff_datatype User: BjoernGruening Date: 2014-12-16 12:08:50+00:00 Summary: Add Arff datatype class with sniffer and metadata. Affected #: 1 file diff -r 919dd242fd625a71d26b3935cabfbdb9b48c97b4 -r 807b1e8a7815f1514712146e909a508acc31a93d lib/galaxy/datatypes/text.py --- a/lib/galaxy/datatypes/text.py +++ b/lib/galaxy/datatypes/text.py @@ -6,6 +6,7 @@ from galaxy.datatypes.data import Text from galaxy.datatypes.data import get_file_peek from galaxy.datatypes.data import nice_size +from galaxy.datatypes.metadata import MetadataElement from galaxy import util import tempfile @@ -155,3 +156,106 @@ return True return False + +class Arff( Text ): + """ + An ARFF (Attribute-Relation File Format) file is an ASCII text file that describes a list of instances sharing a set of attributes. + http://weka.wikispaces.com/ARFF + """ + file_ext = "arff" + + + """Add metadata elements""" + MetadataElement( name="comment_lines", default=0, desc="Number of comment lines", readonly=True, optional=True, no_value=0 ) + MetadataElement( name="columns", default=0, desc="Number of columns", readonly=True, visible=True, no_value=0 ) + + def set_peek( self, dataset, is_multi_byte=False ): + if not dataset.dataset.purged: + dataset.peek = get_file_peek( dataset.file_name, is_multi_byte=is_multi_byte ) + dataset.blurb = "Attribute-Relation File Format (ARFF)" + dataset.blurb += ", %s comments, %s attributes" % ( dataset.metadata.comment_lines, dataset.metadata.columns ) + else: + dataset.peek = 'file does not exist' + dataset.blurb = 'file purged from disc' + + def sniff( self, filename ): + """ + Try to guess the Arff filetype. + It usually starts with a "format-version:" string and has several stanzas which starts with "id:". + """ + with open( filename ) as handle: + relation_found = False + attribute_found = False + prefix = "" + for line_count, line in enumerate( handle ): + if line_count > 1000: + # only investigate the first 1000 lines + return False + line = line.strip() + if not line: + continue + + start_string = line[:20].upper() + if start_string.startswith("@RELATION"): + relation_found = True + elif start_string.startswith("@ATTRIBUTE"): + attribute_found = True + elif start_string.startswith("@DATA"): + # @DATA should be the last data block + if relation_found and attribute_found: + return True + return False + + def set_meta( self, dataset, **kwd ): + """ + Trying to count the comment lines and the number of columns included. + A typical ARFF data block looks like this: + @DATA + 5.1,3.5,1.4,0.2,Iris-setosa + 4.9,3.0,1.4,0.2,Iris-setosa + """ + if dataset.has_data(): + comment_lines = 0 + first_real_line = False + data_block = False + with open( dataset.file_name ) as handle: + for line in handle: + line = line.strip() + if not line: + continue + if line.startswith('%') and not first_real_line: + comment_lines += 1 + else: + first_real_line = True + if data_block: + if line.startswith('{'): + # Sparse representation + """ + @data + 0, X, 0, Y, "class A", {5} + or + @data + {1 X, 3 Y, 4 "class A"}, {5} + """ + token = line.split('}',1) + first_part = token[0] + last_column = first_part.split(',')[-1].strip() + numeric_value = last_column.split()[0] + column_count = int(numeric_value) + if len(token) > 1: + # we have an additional weight + column_count -= 1 + else: + columns = line.strip().split(',') + column_count = len(columns) + if columns[-1].strip().startswith('{'): + # we have an additional weight at the end + column_count -= 1 + + # We have now the column_count and we know the initial comment lines. So we can terminate here. + break + if line[:5].upper() == "@DATA": + data_block = True + dataset.metadata.comment_lines = comment_lines + dataset.metadata.columns = column_count + https://bitbucket.org/galaxy/galaxy-central/commits/a876fb01dba3/ Changeset: a876fb01dba3 Branch: arff_datatype User: BjoernGruening Date: 2014-12-16 12:16:00+00:00 Summary: Add Arff datatype and sniffer to datatypes_conf.xml. Affected #: 1 file diff -r 807b1e8a7815f1514712146e909a508acc31a93d -r a876fb01dba3595d68ae99fea8e2070fa3ffad55 config/datatypes_conf.xml.sample --- a/config/datatypes_conf.xml.sample +++ b/config/datatypes_conf.xml.sample @@ -3,6 +3,7 @@ <registration converters_path="lib/galaxy/datatypes/converters" display_path="display_applications"><datatype extension="ab1" type="galaxy.datatypes.binary:Ab1" mimetype="application/octet-stream" display_in_upload="true" description="A binary sequence file in 'ab1' format with a '.ab1' file extension. You must manually select this 'File Format' when uploading the file." description_url="https://wiki.galaxyproject.org/Learn/Datatypes#Ab1"/><datatype extension="afg" type="galaxy.datatypes.assembly:Amos" display_in_upload="false" /> + <datatype extension="asn1" type="galaxy.datatypes.data:GenericAsn1" mimetype="text/plain" display_in_upload="true" /><datatype extension="asn1-binary" type="galaxy.datatypes.binary:GenericAsn1Binary" mimetype="application/octet-stream" display_in_upload="true" /><datatype extension="axt" type="galaxy.datatypes.sequence:Axt" display_in_upload="true" description="blastz pairwise alignment format. Each alignment block in an axt file contains three lines: a summary line and 2 sequence lines. Blocks are separated from one another by blank lines. The summary line contains chromosomal position and size information about the alignment. It consists of 9 required fields." description_url="https://wiki.galaxyproject.org/Learn/Datatypes#Axt"/> @@ -298,6 +299,7 @@ <sniffer type="galaxy.datatypes.data:Newick"/><sniffer type="galaxy.datatypes.data:Nexus"/><sniffer type="galaxy.datatypes.text:Obo"/> + <sniffer type="galaxy.datatypes.text:Ipynb"/><sniffer type="galaxy.datatypes.text:Json"/><sniffer type="galaxy.datatypes.images:Jpg"/> https://bitbucket.org/galaxy/galaxy-central/commits/d91d8e8beada/ Changeset: d91d8e8beada Branch: arff_datatype User: BjoernGruening Date: 2014-12-23 00:27:36+00:00 Summary: Activate arff datatype by default Affected #: 1 file diff -r a876fb01dba3595d68ae99fea8e2070fa3ffad55 -r d91d8e8beadac2a187645f86c199119064a5ccb0 config/datatypes_conf.xml.sample --- a/config/datatypes_conf.xml.sample +++ b/config/datatypes_conf.xml.sample @@ -3,7 +3,7 @@ <registration converters_path="lib/galaxy/datatypes/converters" display_path="display_applications"><datatype extension="ab1" type="galaxy.datatypes.binary:Ab1" mimetype="application/octet-stream" display_in_upload="true" description="A binary sequence file in 'ab1' format with a '.ab1' file extension. You must manually select this 'File Format' when uploading the file." description_url="https://wiki.galaxyproject.org/Learn/Datatypes#Ab1"/><datatype extension="afg" type="galaxy.datatypes.assembly:Amos" display_in_upload="false" /> -  + <datatype extension="arff" type="galaxy.datatypes.text:Arff" mimetype="text/plain" display_in_upload="True" /><datatype extension="asn1" type="galaxy.datatypes.data:GenericAsn1" mimetype="text/plain" display_in_upload="true" /><datatype extension="asn1-binary" type="galaxy.datatypes.binary:GenericAsn1Binary" mimetype="application/octet-stream" display_in_upload="true" /><datatype extension="axt" type="galaxy.datatypes.sequence:Axt" display_in_upload="true" description="blastz pairwise alignment format. Each alignment block in an axt file contains three lines: a summary line and 2 sequence lines. Blocks are separated from one another by blank lines. The summary line contains chromosomal position and size information about the alignment. It consists of 9 required fields." description_url="https://wiki.galaxyproject.org/Learn/Datatypes#Axt"/> @@ -299,7 +299,7 @@ <sniffer type="galaxy.datatypes.data:Newick"/><sniffer type="galaxy.datatypes.data:Nexus"/><sniffer type="galaxy.datatypes.text:Obo"/> -  + <sniffer type="galaxy.datatypes.text.Arff"/><sniffer type="galaxy.datatypes.text:Ipynb"/><sniffer type="galaxy.datatypes.text:Json"/><sniffer type="galaxy.datatypes.images:Jpg"/> https://bitbucket.org/galaxy/galaxy-central/commits/63ec80be64bb/ Changeset: 63ec80be64bb User: jmchilton Date: 2014-12-26 22:33:02+00:00 Summary: Merge pull request #614. Affected #: 2 files diff -r 338a32cc6067d92055324e4b827697051ec99254 -r 63ec80be64bb00414be0f7122a51907114ffb714 config/datatypes_conf.xml.sample --- a/config/datatypes_conf.xml.sample +++ b/config/datatypes_conf.xml.sample @@ -3,6 +3,7 @@ <registration converters_path="lib/galaxy/datatypes/converters" display_path="display_applications"><datatype extension="ab1" type="galaxy.datatypes.binary:Ab1" mimetype="application/octet-stream" display_in_upload="true" description="A binary sequence file in 'ab1' format with a '.ab1' file extension. You must manually select this 'File Format' when uploading the file." description_url="https://wiki.galaxyproject.org/Learn/Datatypes#Ab1"/><datatype extension="afg" type="galaxy.datatypes.assembly:Amos" display_in_upload="false" /> + <datatype extension="arff" type="galaxy.datatypes.text:Arff" mimetype="text/plain" display_in_upload="True" /><datatype extension="asn1" type="galaxy.datatypes.data:GenericAsn1" mimetype="text/plain" display_in_upload="true" /><datatype extension="asn1-binary" type="galaxy.datatypes.binary:GenericAsn1Binary" mimetype="application/octet-stream" display_in_upload="true" /><datatype extension="axt" type="galaxy.datatypes.sequence:Axt" display_in_upload="true" description="blastz pairwise alignment format. Each alignment block in an axt file contains three lines: a summary line and 2 sequence lines. Blocks are separated from one another by blank lines. The summary line contains chromosomal position and size information about the alignment. It consists of 9 required fields." description_url="https://wiki.galaxyproject.org/Learn/Datatypes#Axt"/> @@ -298,6 +299,7 @@ <sniffer type="galaxy.datatypes.data:Newick"/><sniffer type="galaxy.datatypes.data:Nexus"/><sniffer type="galaxy.datatypes.text:Obo"/> + <sniffer type="galaxy.datatypes.text.Arff"/><sniffer type="galaxy.datatypes.text:Ipynb"/><sniffer type="galaxy.datatypes.text:Json"/><sniffer type="galaxy.datatypes.images:Jpg"/> diff -r 338a32cc6067d92055324e4b827697051ec99254 -r 63ec80be64bb00414be0f7122a51907114ffb714 lib/galaxy/datatypes/text.py --- a/lib/galaxy/datatypes/text.py +++ b/lib/galaxy/datatypes/text.py @@ -6,6 +6,7 @@ from galaxy.datatypes.data import Text from galaxy.datatypes.data import get_file_peek from galaxy.datatypes.data import nice_size +from galaxy.datatypes.metadata import MetadataElement from galaxy import util import tempfile @@ -155,3 +156,106 @@ return True return False + +class Arff( Text ): + """ + An ARFF (Attribute-Relation File Format) file is an ASCII text file that describes a list of instances sharing a set of attributes. + http://weka.wikispaces.com/ARFF + """ + file_ext = "arff" + + + """Add metadata elements""" + MetadataElement( name="comment_lines", default=0, desc="Number of comment lines", readonly=True, optional=True, no_value=0 ) + MetadataElement( name="columns", default=0, desc="Number of columns", readonly=True, visible=True, no_value=0 ) + + def set_peek( self, dataset, is_multi_byte=False ): + if not dataset.dataset.purged: + dataset.peek = get_file_peek( dataset.file_name, is_multi_byte=is_multi_byte ) + dataset.blurb = "Attribute-Relation File Format (ARFF)" + dataset.blurb += ", %s comments, %s attributes" % ( dataset.metadata.comment_lines, dataset.metadata.columns ) + else: + dataset.peek = 'file does not exist' + dataset.blurb = 'file purged from disc' + + def sniff( self, filename ): + """ + Try to guess the Arff filetype. + It usually starts with a "format-version:" string and has several stanzas which starts with "id:". + """ + with open( filename ) as handle: + relation_found = False + attribute_found = False + prefix = "" + for line_count, line in enumerate( handle ): + if line_count > 1000: + # only investigate the first 1000 lines + return False + line = line.strip() + if not line: + continue + + start_string = line[:20].upper() + if start_string.startswith("@RELATION"): + relation_found = True + elif start_string.startswith("@ATTRIBUTE"): + attribute_found = True + elif start_string.startswith("@DATA"): + # @DATA should be the last data block + if relation_found and attribute_found: + return True + return False + + def set_meta( self, dataset, **kwd ): + """ + Trying to count the comment lines and the number of columns included. + A typical ARFF data block looks like this: + @DATA + 5.1,3.5,1.4,0.2,Iris-setosa + 4.9,3.0,1.4,0.2,Iris-setosa + """ + if dataset.has_data(): + comment_lines = 0 + first_real_line = False + data_block = False + with open( dataset.file_name ) as handle: + for line in handle: + line = line.strip() + if not line: + continue + if line.startswith('%') and not first_real_line: + comment_lines += 1 + else: + first_real_line = True + if data_block: + if line.startswith('{'): + # Sparse representation + """ + @data + 0, X, 0, Y, "class A", {5} + or + @data + {1 X, 3 Y, 4 "class A"}, {5} + """ + token = line.split('}',1) + first_part = token[0] + last_column = first_part.split(',')[-1].strip() + numeric_value = last_column.split()[0] + column_count = int(numeric_value) + if len(token) > 1: + # we have an additional weight + column_count -= 1 + else: + columns = line.strip().split(',') + column_count = len(columns) + if columns[-1].strip().startswith('{'): + # we have an additional weight at the end + column_count -= 1 + + # We have now the column_count and we know the initial comment lines. So we can terminate here. + break + if line[:5].upper() == "@DATA": + data_block = True + dataset.metadata.comment_lines = comment_lines + dataset.metadata.columns = column_count + https://bitbucket.org/galaxy/galaxy-central/commits/3ce3ebfdd734/ Changeset: 3ce3ebfdd734 Branch: arff_datatype User: jmchilton Date: 2014-12-26 22:34:57+00:00 Summary: Close arff_datatype. Affected #: 0 files Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: 3 new changesets
by commits-noreply＠bitbucket.org 26 Dec '14

26 Dec '14

3 new commits in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/f7b777f46f9b/ Changeset: f7b777f46f9b User: jmchilton Date: 2014-12-26 14:55:27+00:00 Summary: Remove unused method lib/galaxy/workflow/modules.py. Affected #: 1 file diff -r 4844fa1099447fb8397159be710c99a82726821a -r f7b777f46f9b69b3cd2ca2d90a30988d9f4fa0ab lib/galaxy/workflow/modules.py --- a/lib/galaxy/workflow/modules.py +++ b/lib/galaxy/workflow/modules.py @@ -557,13 +557,6 @@ def normalize_runtime_state( self, runtime_state ): return runtime_state.encode( self.tool, self.trans.app, secure=False ) - @classmethod - def __get_tool_version( cls, trans, tool_id ): - # Return a ToolVersion if one exists for tool_id. - return trans.install_model.context.query( trans.install_model.ToolVersion ) \ - .filter( trans.install_model.ToolVersion.table.c.tool_id == tool_id ) \ - .first() - def save_to_step( self, step ): step.type = self.type step.tool_id = self.tool_id https://bitbucket.org/galaxy/galaxy-central/commits/8a2b4ebdf492/ Changeset: 8a2b4ebdf492 User: jmchilton Date: 2014-12-26 14:55:27+00:00 Summary: Eliminate direct in/get checks against toolbox.tools_by_id - use get_tool or has_tool. Affected #: 6 files diff -r f7b777f46f9b69b3cd2ca2d90a30988d9f4fa0ab -r 8a2b4ebdf4927fccbe44eda80eaba17b8033802c lib/galaxy/jobs/__init__.py --- a/lib/galaxy/jobs/__init__.py +++ b/lib/galaxy/jobs/__init__.py @@ -711,7 +711,7 @@ self.job_id = job.id self.session_id = job.session_id self.user_id = job.user_id - self.tool = queue.app.toolbox.tools_by_id.get( job.tool_id, None ) + self.tool = queue.app.toolbox.get_tool( job.tool_id, exact=True ) self.queue = queue self.app = queue.app self.sa_session = self.app.model.context diff -r f7b777f46f9b69b3cd2ca2d90a30988d9f4fa0ab -r 8a2b4ebdf4927fccbe44eda80eaba17b8033802c lib/galaxy/jobs/handler.py --- a/lib/galaxy/jobs/handler.py +++ b/lib/galaxy/jobs/handler.py @@ -119,7 +119,7 @@ & ( model.Job.handler == self.app.config.server_name ) ).all() for job in jobs_at_startup: - if job.tool_id not in self.app.toolbox.tools_by_id: + if not self.app.toolbox.has_tool( job.tool_id, exact=True ): log.warning( "(%s) Tool '%s' removed from tool config, unable to recover job" % ( job.id, job.tool_id ) ) self.job_wrapper( job ).fail( 'This tool was disabled before the job completed. Please contact your Galaxy administrator.' ) elif job.job_runner_name is not None and job.job_runner_external_id is None: diff -r f7b777f46f9b69b3cd2ca2d90a30988d9f4fa0ab -r 8a2b4ebdf4927fccbe44eda80eaba17b8033802c lib/galaxy/managers/workflows.py --- a/lib/galaxy/managers/workflows.py +++ b/lib/galaxy/managers/workflows.py @@ -224,7 +224,7 @@ errors = [] for key, step_dict in data['steps'].iteritems(): is_tool = is_tool_module_type( step_dict[ 'type' ] ) - if is_tool and step_dict['tool_id'] not in trans.app.toolbox.tools_by_id: + if is_tool and not trans.app.toolbox.has_tool( step_dict['tool_id'], exact=True ): errors.append("Step %s requires tool '%s'." % (step_dict['id'], step_dict['tool_id'])) if errors: raise MissingToolsException(workflow, errors) diff -r f7b777f46f9b69b3cd2ca2d90a30988d9f4fa0ab -r 8a2b4ebdf4927fccbe44eda80eaba17b8033802c lib/galaxy/tools/__init__.py --- a/lib/galaxy/tools/__init__.py +++ b/lib/galaxy/tools/__init__.py @@ -478,8 +478,11 @@ shutil.move( filename, os.path.abspath( self.integrated_tool_panel_config ) ) os.chmod( self.integrated_tool_panel_config, 0644 ) - def get_tool( self, tool_id, tool_version=None, get_all_versions=False ): + def get_tool( self, tool_id, tool_version=None, get_all_versions=False, exact=False ): """Attempt to locate a tool in the tool box.""" + if get_all_versions and exact: + raise AssertionError("Cannot specify get_tool with both get_all_versions and exact as True") + if tool_id in self.tools_by_id and not get_all_versions: #tool_id exactly matches an available tool by id (which is 'old' tool_id or guid) return self.tools_by_id[ tool_id ] @@ -513,6 +516,9 @@ return[ self.tools_by_id[ tool_id ] ] return None + def has_tool( self, tool_id, exact=False ): + return self.get_tool( tool_id, exact=exact ) is not None + def get_loaded_tools_by_lineage( self, tool_id ): """Get all loaded tools associated by lineage to the tool whose id is tool_id.""" tv = self.__get_tool_version( tool_id ) diff -r f7b777f46f9b69b3cd2ca2d90a30988d9f4fa0ab -r 8a2b4ebdf4927fccbe44eda80eaba17b8033802c lib/galaxy/webapps/galaxy/controllers/admin.py --- a/lib/galaxy/webapps/galaxy/controllers/admin.py +++ b/lib/galaxy/webapps/galaxy/controllers/admin.py @@ -486,7 +486,8 @@ class ToolIdColumn( grids.TextColumn ): def get_value( self, trans, grid, tool_version ): - if tool_version.tool_id in trans.app.toolbox.tools_by_id: + toolbox = trans.app.toolbox + if toolbox.has_tool( tool_version.tool_id, exact=True ): link = url_for( controller='tool_runner', tool_id=tool_version.tool_id ) link_str = '<a href="%s">' % link return '<div class="count-box state-color-ok">%s%s</a></div>' % ( link_str, tool_version.tool_id ) @@ -497,8 +498,9 @@ def get_value( self, trans, grid, tool_version ): tool_ids_str = '' + toolbox = trans.app.toolbox for tool_id in tool_version.get_version_ids( trans.app ): - if tool_id in trans.app.toolbox.tools_by_id: + if toolbox.has_tool( tool_id, exact=True ): link = url_for( controller='tool_runner', tool_id=tool_version.tool_id ) link_str = '<a href="%s">' % link tool_ids_str += '<div class="count-box state-color-ok">%s%s</a></div><br/>' % ( link_str, tool_id ) diff -r f7b777f46f9b69b3cd2ca2d90a30988d9f4fa0ab -r 8a2b4ebdf4927fccbe44eda80eaba17b8033802c test/unit/jobs/test_job_wrapper.py --- a/test/unit/jobs/test_job_wrapper.py +++ b/test/unit/jobs/test_job_wrapper.py @@ -190,14 +190,14 @@ def __init__(self, test_tool): self.test_tool = test_tool - @property - def tools_by_id(self): - return self - def get(self, tool_id, default=None): assert tool_id == TEST_TOOL_ID return self.test_tool + def get_tool( self, tool_id, exact=False ): + tool = self.get(tool_id) + return tool + class MockObjectStore(object): https://bitbucket.org/galaxy/galaxy-central/commits/338a32cc6067/ Changeset: 338a32cc6067 User: jmchilton Date: 2014-12-26 14:55:27+00:00 Summary: Introduce abstraction shielding toolbox.tools_by_id from galaxy.workflow.modules. That was the last module outside of ToolBox itself that was accessing tools_by_id directly. Affected #: 5 files diff -r 8a2b4ebdf4927fccbe44eda80eaba17b8033802c -r 338a32cc6067d92055324e4b827697051ec99254 lib/galaxy/tools/__init__.py --- a/lib/galaxy/tools/__init__.py +++ b/lib/galaxy/tools/__init__.py @@ -519,6 +519,21 @@ def has_tool( self, tool_id, exact=False ): return self.get_tool( tool_id, exact=exact ) is not None + def get_tool_id( self, tool_id ): + """ Take a tool id (potentially from a different Galaxy instance or that + is no longer loaded - and find the closest match to the currently loaded + tools (using get_tool for inexact matches which currently returns the oldest + tool shed installed tool with the same short id). + """ + if tool_id not in self.tools_by_id: + tool = self.get_tool( tool_id ) + if tool: + tool_id = tool.id + else: + tool_id = None + # else exact match - leave unmodified. + return tool_id + def get_loaded_tools_by_lineage( self, tool_id ): """Get all loaded tools associated by lineage to the tool whose id is tool_id.""" tv = self.__get_tool_version( tool_id ) diff -r 8a2b4ebdf4927fccbe44eda80eaba17b8033802c -r 338a32cc6067d92055324e4b827697051ec99254 lib/galaxy/workflow/modules.py --- a/lib/galaxy/workflow/modules.py +++ b/lib/galaxy/workflow/modules.py @@ -508,15 +508,13 @@ def from_workflow_step( Class, trans, step ): toolbox = trans.app.toolbox tool_id = step.tool_id - if toolbox and tool_id not in toolbox.tools_by_id: + if toolbox: # See if we have access to a different version of the tool. # TODO: If workflows are ever enhanced to use tool version # in addition to tool id, enhance the selection process here # to retrieve the correct version of the tool. - tool = toolbox.get_tool( tool_id ) - if tool: - tool_id = tool.id - if ( toolbox and tool_id in toolbox.tools_by_id ): + tool_id = toolbox.get_tool_id( tool_id ) + if ( toolbox and tool_id ): if step.config: # This step has its state saved in the config field due to the # tool being previously unavailable. diff -r 8a2b4ebdf4927fccbe44eda80eaba17b8033802c -r 338a32cc6067d92055324e4b827697051ec99254 test/unit/tools/test_toolbox.py --- a/test/unit/tools/test_toolbox.py +++ b/test/unit/tools/test_toolbox.py @@ -186,6 +186,15 @@ # No changes, should be regenerated self.assert_integerated_tool_panel(exists=False) + def test_get_tool_id( self ): + self._init_tool() + self._setup_two_versions_in_config( ) + self._setup_two_versions() + assert self.toolbox.get_tool_id( "test_tool" ) == "github.com/galaxyproect/example/test_tool/0.1" + assert self.toolbox.get_tool_id( "github.com/galaxyproect/example/test_tool/0.1" ) == "github.com/galaxyproect/example/test_tool/0.1" + assert self.toolbox.get_tool_id( "github.com/galaxyproect/example/test_tool/0.2" ) == "github.com/galaxyproect/example/test_tool/0.2" + assert self.toolbox.get_tool_id( "github.com/galaxyproect/example/test_tool/0.3" ) is None + def __verify_two_test_tools( self ): # Assert tool versions of the tool with simple id 'test_tool' all_versions = self.toolbox.get_tool( "test_tool", get_all_versions=True ) diff -r 8a2b4ebdf4927fccbe44eda80eaba17b8033802c -r 338a32cc6067d92055324e4b827697051ec99254 test/unit/workflows/test_modules.py --- a/test/unit/workflows/test_modules.py +++ b/test/unit/workflows/test_modules.py @@ -5,6 +5,7 @@ import mock from galaxy import model +from galaxy.util import bunch from galaxy.workflow import modules from .workflow_support import MockTrans @@ -201,6 +202,37 @@ assert exception +def test_updated_tool_version(): + trans = MockTrans() + mock_tool = __mock_tool(id="cat1", version="0.9") + trans.app.toolbox.tools[ "cat1" ] = mock_tool + module = __from_step( + trans=trans, + type="tool", + tool_id="cat1", + tool_version="0.7", + config=None, + ) + # Make sure there is a warnin with tool id, old version, + # and new version. + for val in "cat1", "0.7", "0.9": + assert val in module.version_changes[0] + + +def test_tool_version_same(): + trans = MockTrans() + mock_tool = __mock_tool(id="cat1", version="1.0") + trans.app.toolbox.tools[ "cat1" ] = mock_tool + module = __from_step( + trans=trans, + type="tool", + tool_id="cat1", + tool_version="1.0", + config=None, + ) + assert not module.version_changes + + def __assert_has_runtime_input( module, label=None, collection_type=None ): inputs = module.get_runtime_inputs() assert len( inputs ) == 1 @@ -221,7 +253,11 @@ def __from_step( **kwds ): - trans = MockTrans() + if "trans" in kwds: + trans = kwds["trans"] + del kwds["trans"] + else: + trans = MockTrans() step = __step( **kwds ) @@ -238,3 +274,20 @@ setattr( step, key, value ) return step + + +def __mock_tool( + id="cat1", + version="1.0", +): + # For now ignoring inputs, params_from_strings, and + # check_and_update_param_values since only have unit tests for version + # handling - but need to write tests for all of this longer term. + tool = bunch.Bunch( + id=id, + version=version, + inputs={}, + params_from_strings=mock.Mock(), + check_and_update_param_values=mock.Mock(), + ) + return tool diff -r 8a2b4ebdf4927fccbe44eda80eaba17b8033802c -r 338a32cc6067d92055324e4b827697051ec99254 test/unit/workflows/workflow_support.py --- a/test/unit/workflows/workflow_support.py +++ b/test/unit/workflows/workflow_support.py @@ -40,3 +40,7 @@ def get_tool( self, tool_id ): # Real tool box returns None of missing tool also return self.tools.get( tool_id, None ) + + def get_tool_id( self, tool_id ): + tool = self.get_tool( tool_id ) + return tool and tool.id Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: 7 new changesets
by commits-noreply＠bitbucket.org 25 Dec '14

25 Dec '14

7 new commits in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/2b6e21823ea7/ Changeset: 2b6e21823ea7 User: jmchilton Date: 2014-12-25 18:35:13+00:00 Summary: More ToolPanelManager unit testing. Affected #: 1 file diff -r 0587bc5413d66e85673d7b429bc04d34e47ad4c4 -r 2b6e21823ea70429b7913b913010f6501f10a601 test/unit/tool_shed_unit_tests/test_tool_panel_manager.py --- a/test/unit/tool_shed_unit_tests/test_tool_panel_manager.py +++ b/test/unit/tool_shed_unit_tests/test_tool_panel_manager.py @@ -101,6 +101,16 @@ assert "github.com/galaxyproect/example/test_tool/0.2" not in open(os.path.join(self.test_directory, "tool_conf.xml"), "r").read() self._verify_tool_confs() + self._remove_guids( ["github.com/galaxyproect/example/test_tool/0.1"], uninstall=True ) + + # Now no versions of this tool are returned by toolbox. + all_versions = self.toolbox.get_tool( "test_tool", get_all_versions=True ) + assert not all_versions + + # Check that tool panel has reverted to old value... + section = self.toolbox.tool_panel["tid"] + assert len(section.elems) == 0 + def _setup_two_versions_remove_one( self, section, uninstall ): self._init_tool() self._setup_two_versions_in_config( section=True ) https://bitbucket.org/galaxy/galaxy-central/commits/8ffb20fd4300/ Changeset: 8ffb20fd4300 User: jmchilton Date: 2014-12-25 18:35:13+00:00 Summary: Eliminate repeated nested object access in ToolPanelManager.remove_guids. Affected #: 1 file diff -r 2b6e21823ea70429b7913b913010f6501f10a601 -r 8ffb20fd4300fbf3b4cc5ac21022e55cd2c1da68 lib/tool_shed/galaxy_install/tools/tool_panel_manager.py --- a/lib/tool_shed/galaxy_install/tools/tool_panel_manager.py +++ b/lib/tool_shed/galaxy_install/tools/tool_panel_manager.py @@ -445,10 +445,11 @@ self.remove_guids( guids_to_remove, shed_tool_conf, uninstall ) def remove_guids( self, guids_to_remove, shed_tool_conf, uninstall ): + toolbox = self.app.toolbox # Remove the tools from the toolbox's tools_by_id dictionary. for guid_to_remove in guids_to_remove: - if guid_to_remove in self.app.toolbox.tools_by_id: - del self.app.toolbox.tools_by_id[ guid_to_remove ] + if guid_to_remove in toolbox.tools_by_id: + del toolbox.tools_by_id[ guid_to_remove ] index, shed_tool_conf_dict = self.get_shed_tool_conf_dict( shed_tool_conf ) if uninstall: # Remove from the shed_tool_conf file on disk. @@ -469,12 +470,12 @@ # Remove the tool sub-element from the section element. config_elem.remove( tool_elem ) # Remove the tool from the section in the in-memory tool panel. - if section_key in self.app.toolbox.tool_panel: - tool_section = self.app.toolbox.tool_panel[ section_key ] + if section_key in toolbox.tool_panel: + tool_section = toolbox.tool_panel[ section_key ] guid = tool_elem.get( 'guid' ) tool_key = 'tool_%s' % str( guid ) # Get the list of versions of this tool that are currently available in the toolbox. - available_tool_versions = self.app.toolbox.get_loaded_tools_by_lineage( guid ) + available_tool_versions = toolbox.get_loaded_tools_by_lineage( guid ) if tool_key in tool_section.elems: if available_tool_versions: available_tool_versions.reverse() @@ -505,8 +506,8 @@ del tool_section.elems[ tool_key ] if uninstall: # Remove the tool from the section in the in-memory integrated tool panel. - if section_key in self.app.toolbox.integrated_tool_panel: - tool_section = self.app.toolbox.integrated_tool_panel[ section_key ] + if section_key in toolbox.integrated_tool_panel: + tool_section = toolbox.integrated_tool_panel[ section_key ] tool_key = 'tool_%s' % str( tool_elem.get( 'guid' ) ) if tool_key in tool_section.elems: del tool_section.elems[ tool_key ] @@ -518,8 +519,8 @@ if guid in guids_to_remove: tool_key = 'tool_%s' % str( config_elem.get( 'guid' ) ) # Get the list of versions of this tool that are currently available in the toolbox. - available_tool_versions = self.app.toolbox.get_loaded_tools_by_lineage( guid ) - if tool_key in self.app.toolbox.tool_panel: + available_tool_versions = toolbox.get_loaded_tools_by_lineage( guid ) + if tool_key in toolbox.tool_panel: if available_tool_versions: available_tool_versions.reverse() replacement_tool_key = None @@ -528,32 +529,32 @@ # the newest loaded version of the tool. for available_tool_version in available_tool_versions: available_tool_section_id, available_tool_section_name = available_tool_version.get_panel_section() - if available_tool_version.id in self.app.toolbox.tool_panel.keys() or not available_tool_section_id: + if available_tool_version.id in toolbox.tool_panel.keys() or not available_tool_section_id: replacement_tool_key = 'tool_%s' % str( available_tool_version.id ) replacement_tool_version = available_tool_version break if replacement_tool_key and replacement_tool_version: # Get the index of the tool_key in the tool_section. - for tool_panel_index, key in enumerate( self.app.toolbox.tool_panel.keys() ): + for tool_panel_index, key in enumerate( toolbox.tool_panel.keys() ): if key == tool_key: break # Remove the tool from the tool panel. - del self.app.toolbox.tool_panel[ tool_key ] + del toolbox.tool_panel[ tool_key ] # Add the replacement tool at the same location in the tool panel. - self.app.toolbox.tool_panel.insert( tool_panel_index, - replacement_tool_key, - replacement_tool_version ) + toolbox.tool_panel.insert( tool_panel_index, + replacement_tool_key, + replacement_tool_version ) else: - del self.app.toolbox.tool_panel[ tool_key ] + del toolbox.tool_panel[ tool_key ] else: - del self.app.toolbox.tool_panel[ tool_key ] + del toolbox.tool_panel[ tool_key ] if uninstall: - if tool_key in self.app.toolbox.integrated_tool_panel: - del self.app.toolbox.integrated_tool_panel[ tool_key ] + if tool_key in toolbox.integrated_tool_panel: + del toolbox.integrated_tool_panel[ tool_key ] config_elems_to_remove.append( config_elem ) for config_elem in config_elems_to_remove: # Remove the element from the in-memory list of elements. config_elems.remove( config_elem ) # Update the config_elems of the in-memory shed_tool_conf_dict. shed_tool_conf_dict[ 'config_elems' ] = config_elems - self.app.toolbox.update_shed_config( index, shed_tool_conf_dict, integrated_panel_changes=uninstall ) + toolbox.update_shed_config( index, shed_tool_conf_dict, integrated_panel_changes=uninstall ) https://bitbucket.org/galaxy/galaxy-central/commits/44dfa6fc2701/ Changeset: 44dfa6fc2701 User: jmchilton Date: 2014-12-25 18:35:13+00:00 Summary: Reduce duplication/complexity in tool_panel_manager.py. Big conditional in remove_guids switched on sectioned versus un-sectioned with very little difference in what was actually being done. This reduces the cyclomatic complexity of that method from 35 down to 26 (though it is still the most 'complex' method in that file by a wide margin). Affected #: 1 file diff -r 8ffb20fd4300fbf3b4cc5ac21022e55cd2c1da68 -r 44dfa6fc270161fb12a3f152d5f24f53d757cd29 lib/tool_shed/galaxy_install/tools/tool_panel_manager.py --- a/lib/tool_shed/galaxy_install/tools/tool_panel_manager.py +++ b/lib/tool_shed/galaxy_install/tools/tool_panel_manager.py @@ -446,6 +446,48 @@ def remove_guids( self, guids_to_remove, shed_tool_conf, uninstall ): toolbox = self.app.toolbox + + def remove_from_panel( tool_elem, has_elems, integrated_has_elems, section_key='' ): + # Hide tool from panel and promote next oldest version if + # available. + guid = tool_elem.get( 'guid' ) + tool_key = 'tool_%s' % str( guid ) + available_tool_versions = toolbox.get_loaded_tools_by_lineage( guid ) + if tool_key in has_elems: + if available_tool_versions: + available_tool_versions.reverse() + replacement_tool_key = None + replacement_tool_version = None + # Since we are going to remove the tool from the section, replace it with + # the newest loaded version of the tool. + for available_tool_version in available_tool_versions: + available_tool_section_id, available_tool_section_name = available_tool_version.get_panel_section() + # I suspect "available_tool_version.id in has_elems.keys()" doesn't + # belong in the following line or at least I don't understand what + # purpose it might serve. -John + if available_tool_version.id in has_elems.keys() or (available_tool_section_id == section_key): + replacement_tool_key = 'tool_%s' % str( available_tool_version.id ) + replacement_tool_version = available_tool_version + break + if replacement_tool_key and replacement_tool_version: + # Get the index of the tool_key in the tool_section. + for tool_panel_index, key in enumerate( has_elems.keys() ): + if key == tool_key: + break + # Remove the tool from the tool panel. + del has_elems[ tool_key ] + # Add the replacement tool at the same location in the tool panel. + has_elems.insert( tool_panel_index, + replacement_tool_key, + replacement_tool_version ) + else: + del has_elems[ tool_key ] + else: + del has_elems[ tool_key ] + if uninstall: + if tool_key in toolbox.integrated_tool_panel: + del toolbox.integrated_tool_panel[ tool_key ] + # Remove the tools from the toolbox's tools_by_id dictionary. for guid_to_remove in guids_to_remove: if guid_to_remove in toolbox.tools_by_id: @@ -472,85 +514,15 @@ # Remove the tool from the section in the in-memory tool panel. if section_key in toolbox.tool_panel: tool_section = toolbox.tool_panel[ section_key ] - guid = tool_elem.get( 'guid' ) - tool_key = 'tool_%s' % str( guid ) - # Get the list of versions of this tool that are currently available in the toolbox. - available_tool_versions = toolbox.get_loaded_tools_by_lineage( guid ) - if tool_key in tool_section.elems: - if available_tool_versions: - available_tool_versions.reverse() - replacement_tool_key = None - replacement_tool_version = None - # Since we are going to remove the tool from the section, replace it with the - # newest loaded version of the tool. - for available_tool_version in available_tool_versions: - available_tool_section_id, available_tool_section_name = available_tool_version.get_panel_section() - if available_tool_version.id in tool_section.elems.keys() or section_key == available_tool_section_id: - replacement_tool_key = 'tool_%s' % str( available_tool_version.id ) - replacement_tool_version = available_tool_version - break - if replacement_tool_key and replacement_tool_version: - # Get the index of the tool_key in the tool_section. - for tool_section_elems_index, key in enumerate( tool_section.elems.keys() ): - if key == tool_key: - break - # Remove the tool from the tool section. - del tool_section.elems[ tool_key ] - # Add the replacement tool at the same location in the tool section. - tool_section.elems.insert( tool_section_elems_index, - replacement_tool_key, - replacement_tool_version ) - else: - del tool_section.elems[ tool_key ] - else: - del tool_section.elems[ tool_key ] - if uninstall: - # Remove the tool from the section in the in-memory integrated tool panel. - if section_key in toolbox.integrated_tool_panel: - tool_section = toolbox.integrated_tool_panel[ section_key ] - tool_key = 'tool_%s' % str( tool_elem.get( 'guid' ) ) - if tool_key in tool_section.elems: - del tool_section.elems[ tool_key ] + remove_from_panel( tool_elem, tool_section.elems, toolbox.integrated_tool_panel.get( section_key, {} ), section_key=section_key ) if len( config_elem ) < 1: # Keep a list of all empty section elements so they can be removed. config_elems_to_remove.append( config_elem ) elif config_elem.tag == 'tool': guid = config_elem.get( 'guid' ) if guid in guids_to_remove: - tool_key = 'tool_%s' % str( config_elem.get( 'guid' ) ) - # Get the list of versions of this tool that are currently available in the toolbox. - available_tool_versions = toolbox.get_loaded_tools_by_lineage( guid ) - if tool_key in toolbox.tool_panel: - if available_tool_versions: - available_tool_versions.reverse() - replacement_tool_key = None - replacement_tool_version = None - # Since we are going to remove the tool from the section, replace it with - # the newest loaded version of the tool. - for available_tool_version in available_tool_versions: - available_tool_section_id, available_tool_section_name = available_tool_version.get_panel_section() - if available_tool_version.id in toolbox.tool_panel.keys() or not available_tool_section_id: - replacement_tool_key = 'tool_%s' % str( available_tool_version.id ) - replacement_tool_version = available_tool_version - break - if replacement_tool_key and replacement_tool_version: - # Get the index of the tool_key in the tool_section. - for tool_panel_index, key in enumerate( toolbox.tool_panel.keys() ): - if key == tool_key: - break - # Remove the tool from the tool panel. - del toolbox.tool_panel[ tool_key ] - # Add the replacement tool at the same location in the tool panel. - toolbox.tool_panel.insert( tool_panel_index, - replacement_tool_key, - replacement_tool_version ) - else: - del toolbox.tool_panel[ tool_key ] - else: - del toolbox.tool_panel[ tool_key ] - if uninstall: - if tool_key in toolbox.integrated_tool_panel: - del toolbox.integrated_tool_panel[ tool_key ] + # get_panel_section return '' for un-sectioned tools. + remove_from_panel( config_elem, toolbox.tool_panel, toolbox.integrated_tool_panel, section_key='' ) config_elems_to_remove.append( config_elem ) for config_elem in config_elems_to_remove: # Remove the element from the in-memory list of elements. https://bitbucket.org/galaxy/galaxy-central/commits/01293fc1f167/ Changeset: 01293fc1f167 User: jmchilton Date: 2014-12-25 18:35:13+00:00 Summary: Abstract out ToolBox internals from tool_shed.util.tool_util. Instead of reloading upload actions at this level and exposing tools_by_id - move this logic into the ToolBox in a new method called handle_datatypes_changed(). Affected #: 2 files diff -r 44dfa6fc270161fb12a3f152d5f24f53d757cd29 -r 01293fc1f16715121524b987fcab031f3e149a02 lib/galaxy/tools/__init__.py --- a/lib/galaxy/tools/__init__.py +++ b/lib/galaxy/tools/__init__.py @@ -34,6 +34,7 @@ from galaxy import exceptions from galaxy.tools import watcher from galaxy.tools.actions import DefaultToolAction +from galaxy.tools.actions.upload import UploadToolAction from galaxy.tools.actions.data_source import DataSourceToolAction from galaxy.tools.actions.data_manager import DataManagerToolAction from galaxy.tools.deps import build_dependency_manager @@ -992,6 +993,13 @@ def init_dependency_manager( self ): self.dependency_manager = build_dependency_manager( self.app.config ) + def handle_datatypes_changed( self ): + """ Refresh upload tools when new datatypes are added. """ + for tool_id in self.tools_by_id: + tool = self.tools_by_id[ tool_id ] + if isinstance( tool.tool_action, UploadToolAction ): + self.reload_tool_by_id( tool_id ) + @property def sa_session( self ): """ diff -r 44dfa6fc270161fb12a3f152d5f24f53d757cd29 -r 01293fc1f16715121524b987fcab031f3e149a02 lib/tool_shed/util/tool_util.py --- a/lib/tool_shed/util/tool_util.py +++ b/lib/tool_shed/util/tool_util.py @@ -260,7 +260,4 @@ def reload_upload_tools( app ): if hasattr( app, 'toolbox' ): - for tool_id in app.toolbox.tools_by_id: - tool = app.toolbox.tools_by_id[ tool_id ] - if isinstance( tool.tool_action, UploadToolAction ): - app.toolbox.reload_tool_by_id( tool_id ) + app.toolbox.handle_datatypes_changed() https://bitbucket.org/galaxy/galaxy-central/commits/7716c8fbb891/ Changeset: 7716c8fbb891 User: jmchilton Date: 2014-12-25 18:35:13+00:00 Summary: Centralize abstraction for removing tools from toolbox. Shield's toolbox.tools_by_id from lib/tool_shed/galaxy_install/tools/tool_panel_manager.py. Affected #: 3 files diff -r 01293fc1f16715121524b987fcab031f3e149a02 -r 7716c8fbb891eb18f81bc80fc5f85f735d025357 lib/galaxy/datatypes/registry.py --- a/lib/galaxy/datatypes/registry.py +++ b/lib/galaxy/datatypes/registry.py @@ -529,8 +529,7 @@ converter.id = tool_dict[ 'guid' ] break if deactivate: - if converter.id in toolbox.tools_by_id: - del toolbox.tools_by_id[ converter.id ] + toolbox.remove_tool_by_id( converter.id, remove_from_panel=False ) if source_datatype in self.datatype_converters: if target_datatype in self.datatype_converters[ source_datatype ]: del self.datatype_converters[ source_datatype ][ target_datatype ] diff -r 01293fc1f16715121524b987fcab031f3e149a02 -r 7716c8fbb891eb18f81bc80fc5f85f735d025357 lib/galaxy/tools/__init__.py --- a/lib/galaxy/tools/__init__.py +++ b/lib/galaxy/tools/__init__.py @@ -952,9 +952,13 @@ status = 'done' return message, status - def remove_tool_by_id( self, tool_id ): + def remove_tool_by_id( self, tool_id, remove_from_panel=True ): """ - Attempt to remove the tool identified by 'tool_id'. + Attempt to remove the tool identified by 'tool_id'. Ignores + tool lineage - so to remove a tool with potentially multiple + versions send remove_from_panel=False and handle the logic of + promoting the next newest version of the tool into the panel + if needed. """ if tool_id not in self.tools_by_id: message = "No tool with id %s" % escape( tool_id ) @@ -962,17 +966,18 @@ else: tool = self.tools_by_id[ tool_id ] del self.tools_by_id[ tool_id ] - tool_key = 'tool_' + tool_id - for key, val in self.tool_panel.items(): - if key == tool_key: - del self.tool_panel[ key ] - break - elif key.startswith( 'section' ): - if tool_key in val.elems: - del self.tool_panel[ key ].elems[ tool_key ] + if remove_from_panel: + tool_key = 'tool_' + tool_id + for key, val in self.tool_panel.items(): + if key == tool_key: + del self.tool_panel[ key ] break - if tool_id in self.data_manager_tools: - del self.data_manager_tools[ tool_id ] + elif key.startswith( 'section' ): + if tool_key in val.elems: + del self.tool_panel[ key ].elems[ tool_key ] + break + if tool_id in self.data_manager_tools: + del self.data_manager_tools[ tool_id ] #TODO: do we need to manually remove from the integrated panel here? message = "Removed the tool:<br/>" message += "<b>name:</b> %s<br/>" % tool.name diff -r 01293fc1f16715121524b987fcab031f3e149a02 -r 7716c8fbb891eb18f81bc80fc5f85f735d025357 lib/tool_shed/galaxy_install/tools/tool_panel_manager.py --- a/lib/tool_shed/galaxy_install/tools/tool_panel_manager.py +++ b/lib/tool_shed/galaxy_install/tools/tool_panel_manager.py @@ -490,8 +490,8 @@ # Remove the tools from the toolbox's tools_by_id dictionary. for guid_to_remove in guids_to_remove: - if guid_to_remove in toolbox.tools_by_id: - del toolbox.tools_by_id[ guid_to_remove ] + # remove_from_tool_panel to false, will handling that logic below. + toolbox.remove_tool_by_id( guid_to_remove, remove_from_panel=False ) index, shed_tool_conf_dict = self.get_shed_tool_conf_dict( shed_tool_conf ) if uninstall: # Remove from the shed_tool_conf file on disk. https://bitbucket.org/galaxy/galaxy-central/commits/75e5db40bd2e/ Changeset: 75e5db40bd2e User: jmchilton Date: 2014-12-25 18:35:13+00:00 Summary: Eliminate walking over tools_by_id externally from toolbox. Affected #: 2 files diff -r 7716c8fbb891eb18f81bc80fc5f85f735d025357 -r 75e5db40bd2e6bfb481b500081a3d8b066063541 lib/galaxy/tools/__init__.py --- a/lib/galaxy/tools/__init__.py +++ b/lib/galaxy/tools/__init__.py @@ -535,6 +535,9 @@ .filter( self.app.install_model.ToolVersion.table.c.tool_id == tool_id ) \ .first() + def tools( self ): + return self.tools_by_id.iteritems() + def __get_tool_shed_repository( self, tool_shed, name, owner, installed_changeset_revision ): # We store only the port, if one exists, in the database. tool_shed = common_util.remove_protocol_from_tool_shed_url( tool_shed ) diff -r 7716c8fbb891eb18f81bc80fc5f85f735d025357 -r 75e5db40bd2e6bfb481b500081a3d8b066063541 lib/galaxy/tools/search/__init__.py --- a/lib/galaxy/tools/search/__init__.py +++ b/lib/galaxy/tools/search/__init__.py @@ -27,7 +27,7 @@ self.index = self.storage.create_index( schema ) writer = self.index.writer() ## TODO: would also be nice to search section headers. - for id, tool in self.toolbox.tools_by_id.iteritems(): + for id, tool in self.toolbox.tools(): add_doc_kwds = { "id": id, "title": to_unicode(tool.name), https://bitbucket.org/galaxy/galaxy-central/commits/4844fa109944/ Changeset: 4844fa109944 User: jmchilton Date: 2014-12-25 18:35:13+00:00 Summary: Abstractions for registering tools from outside toolbox. Part of effort to further limit exposing tools_by_id directly outside Toolbox. Affected #: 4 files diff -r 75e5db40bd2e6bfb481b500081a3d8b066063541 -r 4844fa1099447fb8397159be710c99a82726821a lib/galaxy/datatypes/registry.py --- a/lib/galaxy/datatypes/registry.py +++ b/lib/galaxy/datatypes/registry.py @@ -535,7 +535,7 @@ del self.datatype_converters[ source_datatype ][ target_datatype ] self.log.debug( "Deactivated converter: %s", converter.id ) else: - toolbox.tools_by_id[ converter.id ] = converter + toolbox.register_tool( converter ) if source_datatype not in self.datatype_converters: self.datatype_converters[ source_datatype ] = odict() self.datatype_converters[ source_datatype ][ target_datatype ] = converter @@ -644,8 +644,7 @@ tmp_name = tempfile.NamedTemporaryFile() tmp_name.write( tool_xml_text ) tmp_name.flush() - set_meta_tool = toolbox.load_tool( tmp_name.name ) - toolbox.tools_by_id[ set_meta_tool.id ] = set_meta_tool + set_meta_tool = toolbox.load_hidden_tool( tmp_name.name ) self.set_external_metadata_tool = set_meta_tool self.log.debug( "Loaded external metadata tool: %s", self.set_external_metadata_tool.id ) diff -r 75e5db40bd2e6bfb481b500081a3d8b066063541 -r 4844fa1099447fb8397159be710c99a82726821a lib/galaxy/tools/__init__.py --- a/lib/galaxy/tools/__init__.py +++ b/lib/galaxy/tools/__init__.py @@ -804,6 +804,17 @@ self.tool_watcher.watch_file( config_file, tool.id ) return tool + def load_hidden_tool( self, config_file, **kwds ): + """ Load a hidden tool (in this context meaning one that does not + appear in the tool panel) and register it in tools_by_id. + """ + tool = self.load_tool( config_file, **kwds ) + self.register_tool( tool ) + return tool + + def register_tool( self, tool ): + self.tools_by_id[ tool.id ] = tool + def package_tool( self, trans, tool_id ): """ Create a tarball with the tool's xml, help images, and test data. diff -r 75e5db40bd2e6bfb481b500081a3d8b066063541 -r 4844fa1099447fb8397159be710c99a82726821a lib/galaxy/tools/data_manager/manager.py --- a/lib/galaxy/tools/data_manager/manager.py +++ b/lib/galaxy/tools/data_manager/manager.py @@ -244,12 +244,12 @@ return self.guid or self.declared_id #if we have a guid, we will use that as the data_manager id def load_tool( self, tool_filename, guid=None, data_manager_id=None, tool_shed_repository_id=None ): - tool = self.data_managers.app.toolbox.load_tool( tool_filename, - guid=guid, - data_manager_id=data_manager_id, - repository_id=tool_shed_repository_id ) + toolbox = self.data_managers.app.toolbox + tool = toolbox.load_hidden_tool( tool_filename, + guid=guid, + data_manager_id=data_manager_id, + repository_id=tool_shed_repository_id ) self.data_managers.app.toolbox.data_manager_tools[ tool.id ] = tool - self.data_managers.app.toolbox.tools_by_id[ tool.id ] = tool self.tool = tool return tool diff -r 75e5db40bd2e6bfb481b500081a3d8b066063541 -r 4844fa1099447fb8397159be710c99a82726821a lib/galaxy/tools/imp_exp/__init__.py --- a/lib/galaxy/tools/imp_exp/__init__.py +++ b/lib/galaxy/tools/imp_exp/__init__.py @@ -40,14 +40,12 @@ tmp_name = tempfile.NamedTemporaryFile() tmp_name.write( tool_xml_text ) tmp_name.flush() - history_exp_tool = toolbox.load_tool( tmp_name.name ) - toolbox.tools_by_id[ history_exp_tool.id ] = history_exp_tool + history_exp_tool = toolbox.load_hidden_tool( tmp_name.name ) log.debug( "Loaded history export tool: %s", history_exp_tool.id ) # Load import tool. tool_xml = os.path.join( os.getcwd(), "lib/galaxy/tools/imp_exp/imp_history_from_archive.xml" ) - history_imp_tool = toolbox.load_tool( tool_xml ) - toolbox.tools_by_id[ history_imp_tool.id ] = history_imp_tool + history_imp_tool = toolbox.load_hidden_tool( tool_xml ) log.debug( "Loaded history import tool: %s", history_imp_tool.id ) Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: 5 new changesets
by commits-noreply＠bitbucket.org 25 Dec '14

25 Dec '14

5 new commits in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/38c8222d9e79/ Changeset: 38c8222d9e79 User: jmchilton Date: 2014-12-25 17:24:23+00:00 Summary: Bugfix: Allow passing key to GET /api/datatypes/mapping. All anonymous API endpoints should also allow authenticated access and just ignore the key. Affected #: 1 file diff -r 179199007ec93cdfa45b200e9a27486035ff2477 -r 38c8222d9e79a237dea9179a63b486447aeb9999 lib/galaxy/webapps/galaxy/api/datatypes.py --- a/lib/galaxy/webapps/galaxy/api/datatypes.py +++ b/lib/galaxy/webapps/galaxy/api/datatypes.py @@ -44,7 +44,7 @@ return { 'error': str( exception ) } @web.expose_api_anonymous - def mapping( self, trans ): + def mapping( self, trans, **kwd ): ''' GET /api/datatypes/mapping Return a dictionary of class to class mappings. https://bitbucket.org/galaxy/galaxy-central/commits/cd529da77278/ Changeset: cd529da77278 User: jmchilton Date: 2014-12-25 17:24:23+00:00 Summary: Suite of API tests for exist datatypes endpoint. Affected #: 1 file diff -r 38c8222d9e79a237dea9179a63b486447aeb9999 -r cd529da7727870e472d18a4ba9037aa554bb9e81 test/api/test_datatypes.py --- /dev/null +++ b/test/api/test_datatypes.py @@ -0,0 +1,51 @@ +from base import api + + +HIDDEN_DURING_UPLOAD_DATATYPE = "fli" + + +class DatatypesApiTestCase( api.ApiTestCase ): + + def test_index( self ): + datatypes = self._index_datatypes() + for common_type in ["tabular", "fasta"]: + assert common_type in datatypes, "%s not in %s" % (common_type, datatypes) + + def test_index_upload_only( self ): + # fli is not displayed in upload - so only show it if upload_only + # is explicitly false. + datatypes = self._index_datatypes( data={ "upload_only": False } ) + assert HIDDEN_DURING_UPLOAD_DATATYPE in datatypes + + datatypes = self._index_datatypes( data={ "upload_only": True } ) + assert HIDDEN_DURING_UPLOAD_DATATYPE not in datatypes + + datatypes = self._index_datatypes( ) + assert HIDDEN_DURING_UPLOAD_DATATYPE not in datatypes + + def test_full_index( self ): + datatypes = self._index_datatypes( data={ "extension_only": False } ) + for datatype in datatypes: + self._assert_has_keys( datatype, "extension", "description", "description_url" ) + assert datatype["extension"] != HIDDEN_DURING_UPLOAD_DATATYPE + + def test_mapping( self ): + response = self._get( "datatypes/mapping" ) + self._assert_status_code_is( response, 200 ) + mapping_dict = response.json() + self._assert_has_keys( mapping_dict, "ext_to_class_name", "class_to_classes" ) + + def test_sniffers( self ): + response = self._get( "datatypes/sniffers" ) + self._assert_status_code_is( response, 200 ) + sniffer_list = response.json() + owl_index = sniffer_list.index( "galaxy.datatypes.xml:Owl" ) + xml_index = sniffer_list.index( "galaxy.datatypes.xml:GenericXml" ) + assert owl_index < xml_index + + def _index_datatypes( self, data={} ): + response = self._get( "datatypes", data=data ) + self._assert_status_code_is( response, 200 ) + datatypes = response.json() + assert isinstance( datatypes, list ) + return datatypes https://bitbucket.org/galaxy/galaxy-central/commits/afc01429c885/ Changeset: afc01429c885 User: jmchilton Date: 2014-12-25 17:24:23+00:00 Summary: Update datatypes API to newer API decoarator/exception handling. Affected #: 1 file diff -r cd529da7727870e472d18a4ba9037aa554bb9e81 -r afc01429c88520099bbbb1715f8412d995d4b92c lib/galaxy/webapps/galaxy/api/datatypes.py --- a/lib/galaxy/webapps/galaxy/api/datatypes.py +++ b/lib/galaxy/webapps/galaxy/api/datatypes.py @@ -2,7 +2,8 @@ API operations allowing clients to determine datatype supported by Galaxy. """ -from galaxy import web +from galaxy.web import _future_expose_api_anonymous as expose_api_anonymous +from galaxy import exceptions from galaxy.web.base.controller import BaseAPIController from galaxy.util import asbool from galaxy.datatypes.data import Data @@ -13,7 +14,7 @@ class DatatypesController( BaseAPIController ): - @web.expose_api_anonymous + @expose_api_anonymous def index( self, trans, **kwd ): """ GET /api/datatypes @@ -38,12 +39,14 @@ dictionary[key] = elem.get(key) rval.append(dictionary) return rval - except Exception, exception: + except Exception as exception: log.error( 'could not get datatypes: %s', str( exception ), exc_info=True ) - trans.response.status = 500 - return { 'error': str( exception ) } + if not isinstance( exception, exceptions.MessageException ): + raise exceptions.InternalServerError( str( exception ) ) + else: + raise - @web.expose_api_anonymous + @expose_api_anonymous def mapping( self, trans, **kwd ): ''' GET /api/datatypes/mapping @@ -69,12 +72,15 @@ visit_bases( types, c ) class_to_classes[ n ] = dict( ( t, True ) for t in types ) return dict( ext_to_class_name=ext_to_class_name, class_to_classes=class_to_classes ) - except Exception, exception: + + except Exception as exception: log.error( 'could not get datatype mapping: %s', str( exception ), exc_info=True ) - trans.response.status = 500 - return { 'error': str( exception ) } + if not isinstance( exception, exceptions.MessageException ): + raise exceptions.InternalServerError( str( exception ) ) + else: + raise - @web.expose_api_anonymous + @expose_api_anonymous def sniffers( self, trans, **kwd ): ''' GET /api/datatypes/sniffers @@ -87,7 +93,9 @@ if datatype is not None: rval.append( datatype ) return rval - except Exception, exception: + except Exception as exception: log.error( 'could not get datatypes: %s', str( exception ), exc_info=True ) - trans.response.status = 500 - return { 'error': str( exception ) } + if not isinstance( exception, exceptions.MessageException ): + raise exceptions.InternalServerError( str( exception ) ) + else: + raise https://bitbucket.org/galaxy/galaxy-central/commits/750775b64cb7/ Changeset: 750775b64cb7 User: jmchilton Date: 2014-12-25 17:24:23+00:00 Summary: Reduce repeated nested object access in galaxy.api.datatypes. Affected #: 1 file diff -r afc01429c88520099bbbb1715f8412d995d4b92c -r 750775b64cb758876745af058550b97ae1c47105 lib/galaxy/webapps/galaxy/api/datatypes.py --- a/lib/galaxy/webapps/galaxy/api/datatypes.py +++ b/lib/galaxy/webapps/galaxy/api/datatypes.py @@ -20,17 +20,18 @@ GET /api/datatypes Return an object containing upload datatypes. """ + datatypes_registry = self._datatypes_registry extension_only = asbool( kwd.get( 'extension_only', True ) ) upload_only = asbool( kwd.get( 'upload_only', True ) ) try: if extension_only: if upload_only: - return trans.app.datatypes_registry.upload_file_formats + return datatypes_registry.upload_file_formats else: - return [ ext for ext in trans.app.datatypes_registry.datatypes_by_extension ] + return [ ext for ext in datatypes_registry.datatypes_by_extension ] else: rval = [] - for elem in trans.app.datatypes_registry.datatype_elems: + for elem in datatypes_registry.datatype_elems: if not asbool(elem.get('display_in_upload')) and upload_only: continue keys = ['extension', 'description', 'description_url'] @@ -55,7 +56,7 @@ try: ext_to_class_name = dict() classes = [] - for k, v in trans.app.datatypes_registry.datatypes_by_extension.iteritems(): + for k, v in self._datatypes_registry.datatypes_by_extension.iteritems(): c = v.__class__ ext_to_class_name[k] = c.__module__ + "." + c.__name__ classes.append( c ) @@ -88,7 +89,7 @@ ''' try: rval = [] - for sniffer_elem in trans.app.datatypes_registry.sniffer_elems: + for sniffer_elem in self._datatypes_registry.sniffer_elems: datatype = sniffer_elem.get( 'type' ) if datatype is not None: rval.append( datatype ) @@ -99,3 +100,7 @@ raise exceptions.InternalServerError( str( exception ) ) else: raise + + @property + def _datatypes_registry( self ): + return self.app.datatypes_registry https://bitbucket.org/galaxy/galaxy-central/commits/0587bc5413d6/ Changeset: 0587bc5413d6 User: jmchilton Date: 2014-12-25 17:24:23+00:00 Summary: Expose datatype converter information via datatype API. With test case. Affected #: 3 files diff -r 750775b64cb758876745af058550b97ae1c47105 -r 0587bc5413d66e85673d7b429bc04d34e47ad4c4 lib/galaxy/webapps/galaxy/api/datatypes.py --- a/lib/galaxy/webapps/galaxy/api/datatypes.py +++ b/lib/galaxy/webapps/galaxy/api/datatypes.py @@ -101,6 +101,19 @@ else: raise + @expose_api_anonymous + def converters( self, trans, **kwd ): + converters = [] + for (source_type, targets) in self._datatypes_registry.datatype_converters.iteritems(): + for target_type in targets: + converters.append( { + 'source': source_type, + 'target': target_type, + 'tool_id': targets[ target_type ].id, + } ) + + return converters + @property def _datatypes_registry( self ): return self.app.datatypes_registry diff -r 750775b64cb758876745af058550b97ae1c47105 -r 0587bc5413d66e85673d7b429bc04d34e47ad4c4 lib/galaxy/webapps/galaxy/buildapp.py --- a/lib/galaxy/webapps/galaxy/buildapp.py +++ b/lib/galaxy/webapps/galaxy/buildapp.py @@ -198,7 +198,7 @@ webapp.mapper.resource( 'datatype', 'datatypes', path_prefix='/api', - collection={ 'sniffers': 'GET', 'mapping' : 'GET' }, + collection={ 'sniffers': 'GET', 'mapping' : 'GET', 'converters': 'GET' }, parent_resources=dict( member_name='datatype', collection_name='datatypes' ) ) #webapp.mapper.connect( 'run_workflow', '/api/workflow/{workflow_id}/library/{library_id}', controller='workflows', action='run', workflow_id=None, library_id=None, conditions=dict(method=["GET"]) ) webapp.mapper.resource( 'search', 'search', path_prefix='/api' ) diff -r 750775b64cb758876745af058550b97ae1c47105 -r 0587bc5413d66e85673d7b429bc04d34e47ad4c4 test/api/test_datatypes.py --- a/test/api/test_datatypes.py +++ b/test/api/test_datatypes.py @@ -43,6 +43,19 @@ xml_index = sniffer_list.index( "galaxy.datatypes.xml:GenericXml" ) assert owl_index < xml_index + def test_converters( self ): + response = self._get( "datatypes/converters" ) + self._assert_status_code_is( response, 200 ) + converters_list = response.json() + found_fasta_to_tabular = False + + for converter in converters_list: + self._assert_has_key( converter, "source", "target", "tool_id" ) + if converter["source"] == "fasta" and converter["target"] == "tabular": + found_fasta_to_tabular = True + + assert found_fasta_to_tabular + def _index_datatypes( self, data={} ): response = self._get( "datatypes", data=data ) self._assert_status_code_is( response, 200 ) Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0