1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/changeset/90b28e58bb11/ changeset: 90b28e58bb11 user: greg date: 2012-12-13 22:00:34 summary: Tool shed util refactoring and import fixes and cleanup. affected #: 5 files diff -r 7fb4047a3d6be5847f38acba440e4113eb21e12c -r 90b28e58bb11eb65bd4f461e0b380c5921c626e2 lib/galaxy/tool_shed/__init__.py --- a/lib/galaxy/tool_shed/__init__.py +++ b/lib/galaxy/tool_shed/__init__.py @@ -3,6 +3,7 @@ """ import os import galaxy.util.shed_util +import galaxy.util.shed_util_common from galaxy.model.orm import and_ from galaxy import eggs @@ -27,7 +28,7 @@ ElementInclude.include( root ) tool_path = root.get( 'tool_path', None ) if tool_path: - tool_shed = galaxy.util.shed_util.clean_tool_shed_url( tool_shed_repository.tool_shed ) + tool_shed = galaxy.util.shed_util_common.clean_tool_shed_url( tool_shed_repository.tool_shed ) relative_path = os.path.join( tool_path, tool_shed, 'repos', diff -r 7fb4047a3d6be5847f38acba440e4113eb21e12c -r 90b28e58bb11eb65bd4f461e0b380c5921c626e2 lib/galaxy/tool_shed/install_manager.py --- a/lib/galaxy/tool_shed/install_manager.py +++ b/lib/galaxy/tool_shed/install_manager.py @@ -37,7 +37,7 @@ self.tool_shed_install_config = tool_shed_install_config tree = util.parse_xml( tool_shed_install_config ) root = tree.getroot() - self.tool_shed = shed_util.clean_tool_shed_url( root.get( 'name' ) ) + self.tool_shed = suc.clean_tool_shed_url( root.get( 'name' ) ) self.repository_owner = common_util.REPOSITORY_OWNER index, self.shed_config_dict = shed_util.get_shed_tool_conf_dict( app, self.migrated_tools_config ) # Since tool migration scripts can be executed any number of times, we need to make sure the appropriate tools are defined in @@ -198,6 +198,7 @@ tool_dependencies = None if 'tools' in metadata_dict: sample_files = metadata_dict.get( 'sample_files', [] ) + sample_files = [ str( s ) for s in sample_files ] tool_index_sample_files = shed_util.get_tool_index_sample_files( sample_files ) shed_util.copy_sample_files( self.app, tool_index_sample_files, tool_path=self.tool_path ) sample_files_copied = [ s for s in tool_index_sample_files ] diff -r 7fb4047a3d6be5847f38acba440e4113eb21e12c -r 90b28e58bb11eb65bd4f461e0b380c5921c626e2 lib/galaxy/util/shed_util.py --- a/lib/galaxy/util/shed_util.py +++ b/lib/galaxy/util/shed_util.py @@ -166,11 +166,6 @@ os.close( fd ) shutil.move( filename, os.path.abspath( config_filename ) ) os.chmod( config_filename, 0644 ) -def clean_tool_shed_url( tool_shed_url ): - if tool_shed_url.find( ':' ) > 0: - # Eliminate the port, if any, since it will result in an invalid directory name. - return tool_shed_url.split( ':' )[ 0 ] - return tool_shed_url.rstrip( '/' ) def copy_sample_files( app, sample_files, tool_path=None, sample_files_copied=None, dest_path=None ): """ Copy all appropriate files to dest_path in the local Galaxy environment that have not already been copied. Those that have been copied @@ -379,53 +374,6 @@ else: tool_panel_dict[ guid ] = [ tool_section_dict ] return tool_panel_dict -def generate_tool_panel_dict_from_shed_tool_conf_entries( trans, repository ): - """ - Keep track of the section in the tool panel in which this repository's tools will be contained by parsing the shed-tool_conf in - which the repository's tools are defined and storing the tool panel definition of each tool in the repository. This method is called - only when the repository is being deactivated or uninstalled and allows for activation or reinstallation using the original layout. - """ - tool_panel_dict = {} - shed_tool_conf, tool_path, relative_install_dir = get_tool_panel_config_tool_path_install_dir( trans.app, repository ) - metadata = repository.metadata - # Create a dictionary of tool guid and tool config file name for each tool in the repository. - guids_and_configs = {} - for tool_dict in metadata[ 'tools' ]: - guid = tool_dict[ 'guid' ] - tool_config = tool_dict[ 'tool_config' ] - file_name = suc.strip_path( tool_config ) - guids_and_configs[ guid ] = file_name - # Parse the shed_tool_conf file in which all of this repository's tools are defined and generate the tool_panel_dict. - tree = util.parse_xml( shed_tool_conf ) - root = tree.getroot() - for elem in root: - if elem.tag == 'tool': - guid = elem.get( 'guid' ) - if guid in guids_and_configs: - # The tool is displayed in the tool panel outside of any tool sections. - tool_section_dict = dict( tool_config=guids_and_configs[ guid ], id='', name='', version='' ) - if guid in tool_panel_dict: - tool_panel_dict[ guid ].append( tool_section_dict ) - else: - tool_panel_dict[ guid ] = [ tool_section_dict ] - elif elem.tag == 'section': - section_id = elem.get( 'id' ) or '' - section_name = elem.get( 'name' ) or '' - section_version = elem.get( 'version' ) or '' - for section_elem in elem: - if section_elem.tag == 'tool': - guid = section_elem.get( 'guid' ) - if guid in guids_and_configs: - # The tool is displayed in the tool panel inside the current tool section. - tool_section_dict = dict( tool_config=guids_and_configs[ guid ], - id=section_id, - name=section_name, - version=section_version ) - if guid in tool_panel_dict: - tool_panel_dict[ guid ].append( tool_section_dict ) - else: - tool_panel_dict[ guid ] = [ tool_section_dict ] - return tool_panel_dict def generate_tool_panel_dict_for_tool_config( guid, tool_config, tool_sections=None ): """ Create a dictionary of the following type for a single tool config file name. The intent is to call this method for every tool config @@ -583,7 +531,7 @@ for s in sample_files: # The problem with this is that Galaxy does not follow a standard naming convention for file names. if s.endswith( '.loc.sample' ) or s.endswith( '.xml.sample' ) or s.endswith( '.txt.sample' ): - tool_index_sample_files.append( s ) + tool_index_sample_files.append( str( s ) ) return tool_index_sample_files def get_tool_dependency( trans, id ): """Get a tool_dependency from the database via id""" @@ -598,23 +546,6 @@ if as_string: return ','.join( tool_dependency_ids ) return tool_dependency_ids -def get_tool_panel_config_tool_path_install_dir( app, repository ): - # Return shed-related tool panel config, the tool_path configured in it, and the relative path to the directory where the - # repository is installed. This method assumes all repository tools are defined in a single shed-related tool panel config. - tool_shed = clean_tool_shed_url( repository.tool_shed ) - partial_install_dir = '%s/repos/%s/%s/%s' % ( tool_shed, repository.owner, repository.name, repository.installed_changeset_revision ) - # Get the relative tool installation paths from each of the shed tool configs. - relative_install_dir = None - shed_config_dict = repository.get_shed_config_dict( app ) - if not shed_config_dict: - #just pick a semi-random shed config - for shed_config_dict in app.toolbox.shed_tool_confs: - if ( repository.dist_to_shed and shed_config_dict['config_filename'] == app.config.migrated_tools_config ) or ( not repository.dist_to_shed and shed_config_dict['config_filename'] != app.config.migrated_tools_config ): - break - shed_tool_conf = shed_config_dict[ 'config_filename' ] - tool_path = shed_config_dict[ 'tool_path' ] - relative_install_dir = partial_install_dir - return shed_tool_conf, tool_path, relative_install_dir def get_tool_path_install_dir( partial_install_dir, shed_tool_conf_dict, tool_dict, config_elems ): for elem in config_elems: if elem.tag == 'tool': @@ -913,7 +844,7 @@ """A tool shed repository is being deactivated or uninstalled so handle tool panel alterations accordingly.""" # Determine where the tools are currently defined in the tool panel and store this information so the tools can be displayed # in the same way when the repository is activated or reinstalled. - tool_panel_dict = generate_tool_panel_dict_from_shed_tool_conf_entries( trans, repository ) + tool_panel_dict = suc.generate_tool_panel_dict_from_shed_tool_conf_entries( trans, repository ) repository.metadata[ 'tool_panel_section' ] = tool_panel_dict trans.sa_session.add( repository ) trans.sa_session.flush() diff -r 7fb4047a3d6be5847f38acba440e4113eb21e12c -r 90b28e58bb11eb65bd4f461e0b380c5921c626e2 lib/galaxy/util/shed_util_common.py --- a/lib/galaxy/util/shed_util_common.py +++ b/lib/galaxy/util/shed_util_common.py @@ -328,6 +328,11 @@ if can_delete: trans.sa_session.delete( repository_metadata ) trans.sa_session.flush() +def clean_tool_shed_url( tool_shed_url ): + if tool_shed_url.find( ':' ) > 0: + # Eliminate the port, if any, since it will result in an invalid directory name. + return tool_shed_url.split( ':' )[ 0 ] + return tool_shed_url.rstrip( '/' ) def clone_repository( repository_clone_url, repository_file_dir, ctx_rev ): """Clone the repository up to the specified changeset_revision. No subsequent revisions will be present in the cloned repository.""" try: @@ -567,8 +572,8 @@ return repo_info_dict def generate_clone_url_for_installed_repository( trans, repository ): """Generate the URL for cloning a repository that has been installed into a Galaxy instance.""" - tool_shed_url = suc.get_url_from_repository_tool_shed( trans.app, repository ) - return suc.url_join( tool_shed_url, 'repos', repository.owner, repository.name ) + tool_shed_url = get_url_from_repository_tool_shed( trans.app, repository ) + return url_join( tool_shed_url, 'repos', repository.owner, repository.name ) def generate_clone_url_for_repository_in_tool_shed( trans, repository ): """Generate the URL for cloning a repository that is in the tool shed.""" base_url = url_for( '/', qualified=True ).rstrip( '/' ) @@ -689,6 +694,10 @@ original_repository_metadata = None readme_file_names = get_readme_file_names( repository.name ) metadata_dict = { 'shed_config_filename' : shed_config_dict.get( 'config_filename' ) } + # If we're regenerating metadata for a repository that contains tools, make sure we keep the tool panel section information. + # Fixme: do we need this? + #if original_repository_metadata and 'tool_panel_section' in original_repository_metadata: + # metadata_dict[ 'tool_panel_section' ] = original_repository_metadata[ 'tool_panel_section' ] readme_files = [] invalid_file_tups = [] invalid_tool_configs = [] @@ -994,6 +1003,53 @@ else: metadata_dict[ 'tools' ] = [ tool_dict ] return metadata_dict +def generate_tool_panel_dict_from_shed_tool_conf_entries( trans, repository ): + """ + Keep track of the section in the tool panel in which this repository's tools will be contained by parsing the shed-tool_conf in + which the repository's tools are defined and storing the tool panel definition of each tool in the repository. This method is called + only when the repository is being deactivated or uninstalled and allows for activation or reinstallation using the original layout. + """ + tool_panel_dict = {} + shed_tool_conf, tool_path, relative_install_dir = get_tool_panel_config_tool_path_install_dir( trans.app, repository ) + metadata = repository.metadata + # Create a dictionary of tool guid and tool config file name for each tool in the repository. + guids_and_configs = {} + for tool_dict in metadata[ 'tools' ]: + guid = tool_dict[ 'guid' ] + tool_config = tool_dict[ 'tool_config' ] + file_name = strip_path( tool_config ) + guids_and_configs[ guid ] = file_name + # Parse the shed_tool_conf file in which all of this repository's tools are defined and generate the tool_panel_dict. + tree = util.parse_xml( shed_tool_conf ) + root = tree.getroot() + for elem in root: + if elem.tag == 'tool': + guid = elem.get( 'guid' ) + if guid in guids_and_configs: + # The tool is displayed in the tool panel outside of any tool sections. + tool_section_dict = dict( tool_config=guids_and_configs[ guid ], id='', name='', version='' ) + if guid in tool_panel_dict: + tool_panel_dict[ guid ].append( tool_section_dict ) + else: + tool_panel_dict[ guid ] = [ tool_section_dict ] + elif elem.tag == 'section': + section_id = elem.get( 'id' ) or '' + section_name = elem.get( 'name' ) or '' + section_version = elem.get( 'version' ) or '' + for section_elem in elem: + if section_elem.tag == 'tool': + guid = section_elem.get( 'guid' ) + if guid in guids_and_configs: + # The tool is displayed in the tool panel inside the current tool section. + tool_section_dict = dict( tool_config=guids_and_configs[ guid ], + id=section_id, + name=section_name, + version=section_version ) + if guid in tool_panel_dict: + tool_panel_dict[ guid ].append( tool_section_dict ) + else: + tool_panel_dict[ guid ] = [ tool_section_dict ] + return tool_panel_dict def generate_workflow_metadata( relative_path, exported_workflow_dict, metadata_dict ): """Update the received metadata_dict with changes that have been applied to the received exported_workflow_dict.""" if 'workflows' in metadata_dict: @@ -1372,6 +1428,23 @@ relative_path_to_sample_file = relative_path_to_sample_file[ len( tool_path ) + 1 :] sample_file_metadata_paths.append( relative_path_to_sample_file ) return sample_file_metadata_paths, sample_file_copy_paths +def get_tool_panel_config_tool_path_install_dir( app, repository ): + # Return shed-related tool panel config, the tool_path configured in it, and the relative path to the directory where the + # repository is installed. This method assumes all repository tools are defined in a single shed-related tool panel config. + tool_shed = clean_tool_shed_url( repository.tool_shed ) + partial_install_dir = '%s/repos/%s/%s/%s' % ( tool_shed, repository.owner, repository.name, repository.installed_changeset_revision ) + # Get the relative tool installation paths from each of the shed tool configs. + relative_install_dir = None + shed_config_dict = repository.get_shed_config_dict( app ) + if not shed_config_dict: + #just pick a semi-random shed config + for shed_config_dict in app.toolbox.shed_tool_confs: + if ( repository.dist_to_shed and shed_config_dict['config_filename'] == app.config.migrated_tools_config ) or ( not repository.dist_to_shed and shed_config_dict['config_filename'] != app.config.migrated_tools_config ): + break + shed_tool_conf = shed_config_dict[ 'config_filename' ] + tool_path = shed_config_dict[ 'tool_path' ] + relative_install_dir = partial_install_dir + return shed_tool_conf, tool_path, relative_install_dir def get_tool_shed_from_clone_url( repository_clone_url ): tmp_url = clean_repository_clone_url( repository_clone_url ) return tmp_url.split( 'repos' )[ 0 ].rstrip( '/' ) @@ -2019,7 +2092,7 @@ str( markupsafe.escape( ''.join( translated ) ) ) return ''.join( translated ) def tool_shed_from_repository_clone_url( repository_clone_url ): - return suc.clean_repository_clone_url( repository_clone_url ).split( 'repos' )[ 0 ].rstrip( '/' ) + return clean_repository_clone_url( repository_clone_url ).split( 'repos' )[ 0 ].rstrip( '/' ) def tool_shed_is_this_tool_shed( toolshed_base_url ): return toolshed_base_url.rstrip( '/' ) == str( url_for( '/', qualified=True ) ).rstrip( '/' ) def translate_string( raw_text, to_html=True ): @@ -2106,13 +2179,14 @@ shed_tool_conf = shed_conf_dict[ 'config_filename' ] tool_path = shed_conf_dict[ 'tool_path' ] + # TODO Fix this - we should be able to pass only app - we should not need trans... #hack for 'trans.app' used in lots of places. These places should just directly use app trans = util.bunch.Bunch() trans.app = app tool_panel_dict = generate_tool_panel_dict_from_shed_tool_conf_entries( trans, repository ) repository_tools_tups = get_repository_tools_tups( app, repository.metadata ) - cleaned_repository_clone_url = suc.clean_repository_clone_url( suc.generate_clone_url_for_installed_repository( trans, repository ) ) + cleaned_repository_clone_url = clean_repository_clone_url( generate_clone_url_for_installed_repository( trans, repository ) ) tool_shed = tool_shed_from_repository_clone_url( cleaned_repository_clone_url ) owner = repository.owner if not owner: diff -r 7fb4047a3d6be5847f38acba440e4113eb21e12c -r 90b28e58bb11eb65bd4f461e0b380c5921c626e2 lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py --- a/lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py +++ b/lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py @@ -309,7 +309,7 @@ def activate_repository( self, trans, **kwd ): """Activate a repository that was deactivated but not uninstalled.""" repository = suc.get_installed_tool_shed_repository( trans, kwd[ 'id' ] ) - shed_tool_conf, tool_path, relative_install_dir = shed_util.get_tool_panel_config_tool_path_install_dir( trans.app, repository ) + shed_tool_conf, tool_path, relative_install_dir = suc.get_tool_panel_config_tool_path_install_dir( trans.app, repository ) repository_clone_url = suc.generate_clone_url_for_installed_repository( trans, repository ) repository.deleted = False repository.status = trans.model.ToolShedRepository.installation_status.INSTALLED @@ -460,7 +460,7 @@ remove_from_disk = params.get( 'remove_from_disk', '' ) remove_from_disk_checked = CheckboxField.is_checked( remove_from_disk ) tool_shed_repository = suc.get_installed_tool_shed_repository( trans, kwd[ 'id' ] ) - shed_tool_conf, tool_path, relative_install_dir = shed_util.get_tool_panel_config_tool_path_install_dir( trans.app, tool_shed_repository ) + shed_tool_conf, tool_path, relative_install_dir = suc.get_tool_panel_config_tool_path_install_dir( trans.app, tool_shed_repository ) if relative_install_dir: if tool_path: relative_install_dir = os.path.join( tool_path, relative_install_dir ) @@ -553,7 +553,7 @@ items = tmp_url.split( 'repos' ) tool_shed_url = items[ 0 ] repo_path = items[ 1 ] - tool_shed_url = shed_util.clean_tool_shed_url( tool_shed_url ) + tool_shed_url = suc.clean_tool_shed_url( tool_shed_url ) return suc.url_join( tool_shed_url, 'repos', repo_path, changeset_revision ) @web.json @web.require_admin @@ -794,7 +794,7 @@ sample_files = metadata_dict.get( 'sample_files', [] ) tool_index_sample_files = shed_util.get_tool_index_sample_files( sample_files ) shed_util.copy_sample_files( self.app, tool_index_sample_files, tool_path=tool_path ) - sample_files_copied = [ s for s in tool_index_sample_files ] + sample_files_copied = [ str( s ) for s in tool_index_sample_files ] repository_tools_tups = shed_util.get_repository_tools_tups( trans.app, metadata_dict ) if repository_tools_tups: # Handle missing data table entries for tool parameters that are dynamically generated select lists. @@ -866,7 +866,7 @@ ( repository.name, repository.owner, repository.installed_changeset_revision, ( url_for( '/', qualified=True ) ) ) ) return trans.response.send_redirect( url ) description = util.restore_text( params.get( 'description', repository.description ) ) - shed_tool_conf, tool_path, relative_install_dir = shed_util.get_tool_panel_config_tool_path_install_dir( trans.app, repository ) + shed_tool_conf, tool_path, relative_install_dir = suc.get_tool_panel_config_tool_path_install_dir( trans.app, repository ) if relative_install_dir: repo_files_dir = os.path.abspath( os.path.join( tool_path, relative_install_dir, repository.name ) ) else: @@ -1294,7 +1294,7 @@ install_tool_dependencies = CheckboxField.is_checked( kwd.get( 'install_tool_dependencies', '' ) ) new_tool_panel_section = kwd.get( 'new_tool_panel_section', '' ) tool_panel_section = kwd.get( 'tool_panel_section', '' ) - shed_tool_conf, tool_path, relative_install_dir = shed_util.get_tool_panel_config_tool_path_install_dir( trans.app, tool_shed_repository ) + shed_tool_conf, tool_path, relative_install_dir = suc.get_tool_panel_config_tool_path_install_dir( trans.app, tool_shed_repository ) repository_clone_url = suc.generate_clone_url_for_installed_repository( trans, tool_shed_repository ) clone_dir = os.path.join( tool_path, self.generate_tool_path( repository_clone_url, tool_shed_repository.installed_changeset_revision ) ) relative_install_dir = os.path.join( clone_dir, tool_shed_repository.name ) @@ -1660,7 +1660,7 @@ message += "Reset all of this reppository's metadata in the tool shed, then set the installed tool versions " message ++ "from the installed repository's <b>Repository Actions</b> menu. " status = 'error' - shed_tool_conf, tool_path, relative_install_dir = shed_util.get_tool_panel_config_tool_path_install_dir( trans.app, repository ) + shed_tool_conf, tool_path, relative_install_dir = suc.get_tool_panel_config_tool_path_install_dir( trans.app, repository ) repo_files_dir = os.path.abspath( os.path.join( relative_install_dir, repository.name ) ) return trans.fill_template( '/admin/tool_shed_repository/manage_repository.mako', repository=repository, @@ -1749,7 +1749,7 @@ if changeset_revision == latest_changeset_revision: message = "The installed repository named '%s' is current, there are no updates available. " % name else: - shed_tool_conf, tool_path, relative_install_dir = shed_util.get_tool_panel_config_tool_path_install_dir( trans.app, repository ) + shed_tool_conf, tool_path, relative_install_dir = suc.get_tool_panel_config_tool_path_install_dir( trans.app, repository ) if relative_install_dir: if tool_path: repo_files_dir = os.path.abspath( os.path.join( tool_path, relative_install_dir, name ) ) @@ -1759,7 +1759,7 @@ repository_clone_url = os.path.join( tool_shed_url, 'repos', owner, name ) shed_util.pull_repository( repo, repository_clone_url, latest_ctx_rev ) suc.update_repository( repo, latest_ctx_rev ) - tool_shed = shed_util.clean_tool_shed_url( tool_shed_url ) + tool_shed = suc.clean_tool_shed_url( tool_shed_url ) # Update the repository metadata. metadata_dict, invalid_file_tups = suc.generate_metadata_for_changeset_revision( app=trans.app, repository=repository, @@ -1777,12 +1777,13 @@ repository.update_available = False trans.sa_session.add( repository ) trans.sa_session.flush() + # Fixme: call shed_util.add_to_tool_panel here? # Create tool_dependency records if necessary. if 'tool_dependencies' in metadata_dict: tool_dependencies = shed_util.create_tool_dependency_objects( trans.app, repository, relative_install_dir, set_status=False ) message = "The installed repository named '%s' has been updated to change set revision '%s'. " % ( name, latest_changeset_revision ) # See if any tool dependencies can be installed. - shed_tool_conf, tool_path, relative_install_dir = shed_util.get_tool_panel_config_tool_path_install_dir( trans.app, repository ) + shed_tool_conf, tool_path, relative_install_dir = suc.get_tool_panel_config_tool_path_install_dir( trans.app, repository ) if repository.missing_tool_dependencies: message += "Click the name of one of the missing tool dependencies listed below to install tool dependencies." else: Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.