[galaxy-commits] commit/galaxy-central: greg: Move some functions out of the tool shed's shed_util_common module into appropriate classes.

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/55b4556e0b7e/ Changeset: 55b4556e0b7e User: greg Date: 2014-07-21 22:44:32 Summary: Move some functions out of the tool shed's shed_util_common module into appropriate classes. Affected #: 16 files diff -r e9de99ef77105d221df55771724ca56b86485a4b -r 55b4556e0b7eba1ae3bfeb6e850831e25b3238ac lib/galaxy/webapps/galaxy/api/tool_shed_repositories.py --- a/lib/galaxy/webapps/galaxy/api/tool_shed_repositories.py +++ b/lib/galaxy/webapps/galaxy/api/tool_shed_repositories.py @@ -13,6 +13,7 @@ from tool_shed.galaxy_install.install_manager import InstallRepositoryManager from tool_shed.galaxy_install.metadata.installed_repository_metadata_manager import InstalledRepositoryMetadataManager from tool_shed.galaxy_install.repair_repository_manager import RepairRepositoryManager + from tool_shed.util import common_util from tool_shed.util import encoding_util from tool_shed.util import hg_util diff -r e9de99ef77105d221df55771724ca56b86485a4b -r 55b4556e0b7eba1ae3bfeb6e850831e25b3238ac lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py --- a/lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py +++ b/lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py @@ -46,7 +46,7 @@ def activate_repository( self, trans, **kwd ): """Activate a repository that was deactivated but not uninstalled.""" repository_id = kwd[ 'id' ] - repository = suc.get_installed_tool_shed_repository( trans.app, repository_id ) + repository = repository_maintenance_util.get_installed_tool_shed_repository( trans.app, repository_id ) try: trans.app.installed_repository_manager.activate_repository( repository ) except Exception, e: @@ -72,7 +72,7 @@ def browse_repository( self, trans, **kwd ): message = kwd.get( 'message', '' ) status = kwd.get( 'status', 'done' ) - repository = suc.get_installed_tool_shed_repository( trans.app, kwd[ 'id' ] ) + repository = repository_maintenance_util.get_installed_tool_shed_repository( trans.app, kwd[ 'id' ] ) return trans.fill_template( '/admin/tool_shed_repository/browse_repository.mako', repository=repository, message=message, @@ -105,7 +105,7 @@ action='purge_repository', **kwd ) ) if operation == "activate or reinstall": - repository = suc.get_installed_tool_shed_repository( trans.app, kwd[ 'id' ] ) + repository = repository_maintenance_util.get_installed_tool_shed_repository( trans.app, kwd[ 'id' ] ) if repository.uninstalled: # Since we're reinstalling the repository we need to find the latest changeset revision to which it can # be updated so that we can reset the metadata if necessary. This will ensure that information about @@ -206,7 +206,7 @@ def check_for_updates( self, trans, **kwd ): """Send a request to the relevant tool shed to see if there are any updates.""" repository_id = kwd.get( 'id', None ) - repository = suc.get_installed_tool_shed_repository( trans.app, repository_id ) + repository = repository_maintenance_util.get_installed_tool_shed_repository( trans.app, repository_id ) tool_shed_url = common_util.get_tool_shed_url_from_tool_shed_registry( trans.app, str( repository.tool_shed ) ) params = '?galaxy_url=%s&name=%s&owner=%s&changeset_revision=%s' % \ ( web.url_for( '/', qualified=True ), @@ -232,7 +232,7 @@ status = kwd.get( 'status', 'done' ) remove_from_disk = kwd.get( 'remove_from_disk', '' ) remove_from_disk_checked = CheckboxField.is_checked( remove_from_disk ) - tool_shed_repository = suc.get_installed_tool_shed_repository( trans.app, kwd[ 'id' ] ) + tool_shed_repository = repository_maintenance_util.get_installed_tool_shed_repository( trans.app, kwd[ 'id' ] ) shed_tool_conf, tool_path, relative_install_dir = \ suc.get_tool_panel_config_tool_path_install_dir( trans.app, tool_shed_repository ) if relative_install_dir: @@ -394,7 +394,7 @@ of the installed tool shed repository in Galaxy. We need it so that we can derive the tool shed from which it was installed. """ - repository = suc.get_installed_tool_shed_repository( trans.app, repository_id ) + repository = repository_maintenance_util.get_installed_tool_shed_repository( trans.app, repository_id ) tool_shed_url = common_util.get_tool_shed_url_from_tool_shed_registry( trans.app, str( repository.tool_shed ) ) if tool_shed_url is None or repository_name is None or repository_owner is None or changeset_revision is None: message = "Unable to retrieve tool dependencies from the Tool Shed because one or more of the following required " @@ -419,7 +419,7 @@ Send a request to the appropriate tool shed to retrieve the dictionary of information required to reinstall an updated revision of an uninstalled tool shed repository. """ - repository = suc.get_installed_tool_shed_repository( trans.app, repository_id ) + repository = repository_maintenance_util.get_installed_tool_shed_repository( trans.app, repository_id ) tool_shed_url = common_util.get_tool_shed_url_from_tool_shed_registry( trans.app, str( repository.tool_shed ) ) if tool_shed_url is None or repository_name is None or repository_owner is None or changeset_revision is None: message = "Unable to retrieve updated repository information from the Tool Shed because one or more of the following " @@ -513,7 +513,7 @@ status = kwd.get( 'status', 'done' ) repository_id = kwd.get( 'id', None ) if repository_id is not None: - repository = suc.get_installed_tool_shed_repository( trans.app, repository_id ) + repository = repository_maintenance_util.get_installed_tool_shed_repository( trans.app, repository_id ) if repository is not None: tool_shed_url = common_util.get_tool_shed_url_from_tool_shed_registry( trans.app, str( repository.tool_shed ) ) name = str( repository.name ) @@ -571,7 +571,7 @@ dependencies are included in the updated revision. """ updating_repository_id = kwd.get( 'updating_repository_id', None ) - repository = suc.get_installed_tool_shed_repository( trans.app, updating_repository_id ) + repository = repository_maintenance_util.get_installed_tool_shed_repository( trans.app, updating_repository_id ) # All received dependencies need to be installed - confirmed by the caller. encoded_tool_dependencies_dict = kwd.get( 'encoded_tool_dependencies_dict', None ) if encoded_tool_dependencies_dict is not None: @@ -747,7 +747,7 @@ if repository_id is None: return trans.show_error_message( 'Missing required encoded repository id.' ) operation = kwd.get( 'operation', None ) - repository = suc.get_installed_tool_shed_repository( trans.app, repository_id ) + repository = repository_maintenance_util.get_installed_tool_shed_repository( trans.app, repository_id ) if repository is None: return trans.show_error_message( 'Invalid repository specified.' ) tool_shed_url = common_util.get_tool_shed_url_from_tool_shed_registry( trans.app, str( repository.tool_shed ) ) @@ -866,15 +866,15 @@ message += ' and restart your Galaxy server to install tool dependencies.' status = 'error' installed_tool_dependencies_select_field = \ - suc.build_tool_dependencies_select_field( trans.app, - tool_shed_repository=tool_shed_repository, - name='inst_td_ids', - uninstalled_only=False ) + tool_dependency_util.build_tool_dependencies_select_field( trans.app, + tool_shed_repository=tool_shed_repository, + name='inst_td_ids', + uninstalled_only=False ) uninstalled_tool_dependencies_select_field = \ - suc.build_tool_dependencies_select_field( trans.app, - tool_shed_repository=tool_shed_repository, - name='uninstalled_tool_dependency_ids', - uninstalled_only=True ) + tool_dependency_util.build_tool_dependencies_select_field( trans.app, + tool_shed_repository=tool_shed_repository, + name='uninstalled_tool_dependency_ids', + uninstalled_only=True ) return trans.fill_template( '/admin/tool_shed_repository/manage_repository_tool_dependencies.mako', repository=tool_shed_repository, installed_tool_dependencies_select_field=installed_tool_dependencies_select_field, @@ -1276,7 +1276,7 @@ repository_id = kwd.get( 'id', None ) new_kwd = {} if repository_id is not None: - repository = suc.get_installed_tool_shed_repository( trans.app, repository_id ) + repository = repository_maintenance_util.get_installed_tool_shed_repository( trans.app, repository_id ) if repository: if repository.is_new: if kwd.get( 'purge_repository_button', False ): @@ -1314,7 +1314,7 @@ message = kwd.get( 'message', '' ) status = kwd.get( 'status', 'done' ) repository_id = kwd[ 'id' ] - tool_shed_repository = suc.get_installed_tool_shed_repository( trans.app, repository_id ) + tool_shed_repository = repository_maintenance_util.get_installed_tool_shed_repository( trans.app, repository_id ) no_changes = kwd.get( 'no_changes', '' ) no_changes_checked = CheckboxField.is_checked( no_changes ) install_repository_dependencies = CheckboxField.is_checked( kwd.get( 'install_repository_dependencies', '' ) ) @@ -1457,7 +1457,7 @@ action='browse_repositories', message=message, status=status ) ) - tool_shed_repository = suc.get_installed_tool_shed_repository( trans.app, repository_id ) + tool_shed_repository = repository_maintenance_util.get_installed_tool_shed_repository( trans.app, repository_id ) rrm = RepairRepositoryManager( trans.app ) if kwd.get( 'repair_repository_button', False ): encoded_repair_dict = kwd.get( 'repair_dict', None ) @@ -1475,7 +1475,7 @@ repository = trans.install_model.context.query( trans.install_model.ToolShedRepository ).get( trans.security.decode_id( tsr_id ) ) repositories_for_repair.append( repository ) return self.repair_tool_shed_repositories( trans, rrm, repositories_for_repair, ordered_repo_info_dicts ) - tool_shed_repository = suc.get_installed_tool_shed_repository( trans.app, repository_id ) + tool_shed_repository = repository_maintenance_util.get_installed_tool_shed_repository( trans.app, repository_id ) repair_dict = rrm.get_repair_dict( tool_shed_repository ) encoded_repair_dict = encoding_util.tool_shed_encode( repair_dict ) ordered_tsr_ids = repair_dict.get( 'ordered_tsr_ids', [] ) @@ -1537,7 +1537,7 @@ repository_id = kwd.get( 'id', None ) latest_changeset_revision = kwd.get( 'latest_changeset_revision', None ) latest_ctx_rev = kwd.get( 'latest_ctx_rev', None ) - tool_shed_repository = suc.get_installed_tool_shed_repository( trans.app, repository_id ) + tool_shed_repository = repository_maintenance_util.get_installed_tool_shed_repository( trans.app, repository_id ) repository_clone_url = common_util.generate_clone_url_for_installed_repository( trans.app, tool_shed_repository ) metadata = tool_shed_repository.metadata tool_shed_url = common_util.get_tool_shed_url_from_tool_shed_registry( trans.app, str( tool_shed_repository.tool_shed ) ) @@ -1724,7 +1724,7 @@ @web.require_admin def reset_repository_metadata( self, trans, id ): """Reset all metadata on a single installed tool shed repository.""" - repository = suc.get_installed_tool_shed_repository( trans.app, id ) + repository = repository_maintenance_util.get_installed_tool_shed_repository( trans.app, id ) repository_clone_url = common_util.generate_clone_url_for_installed_repository( trans.app, repository ) tool_path, relative_install_dir = repository.get_tool_relative_path( trans.app ) if relative_install_dir: @@ -1742,7 +1742,7 @@ persist=False ) repository.metadata = metadata_dict if metadata_dict != original_metadata_dict: - suc.update_in_shed_tool_config( trans.app, repository ) + irmm.update_in_shed_tool_config( repository ) trans.install_model.context.add( repository ) trans.install_model.context.flush() message = 'Metadata has been reset on repository <b>%s</b>.' % repository.name @@ -1763,7 +1763,7 @@ @web.require_admin def reset_to_install( self, trans, **kwd ): """An error occurred while cloning the repository, so reset everything necessary to enable another attempt.""" - repository = suc.get_installed_tool_shed_repository( trans.app, kwd[ 'id' ] ) + repository = repository_maintenance_util.get_installed_tool_shed_repository( trans.app, kwd[ 'id' ] ) if kwd.get( 'reset_repository', False ): suc.set_repository_attributes( trans.app, repository, @@ -1789,7 +1789,7 @@ Get the tool_versions from the tool shed for each tool in the installed revision of a selected tool shed repository and update the metadata for the repository's revision in the Galaxy database. """ - repository = suc.get_installed_tool_shed_repository( trans.app, kwd[ 'id' ] ) + repository = repository_maintenance_util.get_installed_tool_shed_repository( trans.app, kwd[ 'id' ] ) tool_shed_url = common_util.get_tool_shed_url_from_tool_shed_registry( trans.app, str( repository.tool_shed ) ) params = '?name=%s&owner=%s&changeset_revision=%s' % ( str( repository.name ), str( repository.owner ), @@ -1947,8 +1947,8 @@ tpm = tool_panel_manager.ToolPanelManager( trans.app ) tool_panel_dict = metadata_dict.get( 'tool_panel_section', None ) if tool_panel_dict is None: - tool_panel_dict = suc.generate_tool_panel_dict_from_shed_tool_conf_entries( trans.app, repository ) - repository_tools_tups = suc.get_repository_tools_tups( trans.app, metadata_dict ) + tool_panel_dict = tpm.generate_tool_panel_dict_from_shed_tool_conf_entries( repository ) + repository_tools_tups = irmm.get_repository_tools_tups( metadata_dict ) tpm.add_to_tool_panel( repository_name=str( repository.name ), repository_clone_url=repository_clone_url, changeset_revision=str( repository.installed_changeset_revision ), @@ -2069,8 +2069,10 @@ repository_names_not_updated = [] updated_count = 0 for repository in trans.install_model.context.query( trans.install_model.ToolShedRepository ) \ - .filter( trans.install_model.ToolShedRepository.table.c.deleted == False ): - ok, updated = suc.check_or_update_tool_shed_status_for_installed_repository( trans.app, repository ) + .filter( trans.install_model.ToolShedRepository.table.c.deleted == False ): + ok, updated = \ + repository_maintenance_util.check_or_update_tool_shed_status_for_installed_repository( trans.app, + repository ) if ok: success_count += 1 else: @@ -2085,7 +2087,9 @@ else: repository_id = kwd.get( 'id', None ) repository = suc.get_tool_shed_repository_by_id( trans.app, repository_id ) - ok, updated = suc.check_or_update_tool_shed_status_for_installed_repository( trans.app, repository ) + ok, updated = \ + repository_maintenance_util.check_or_update_tool_shed_status_for_installed_repository( trans.app, + repository ) if ok: if updated: message = "The tool shed status for repository <b>%s</b> has been updated." % str( repository.name ) @@ -2104,7 +2108,7 @@ def view_tool_metadata( self, trans, repository_id, tool_id, **kwd ): message = kwd.get( 'message', '' ) status = kwd.get( 'status', 'done' ) - repository = suc.get_installed_tool_shed_repository( trans.app, repository_id ) + repository = repository_maintenance_util.get_installed_tool_shed_repository( trans.app, repository_id ) repository_metadata = repository.metadata shed_config_dict = repository.get_shed_config_dict( trans.app ) tool_metadata = {} diff -r e9de99ef77105d221df55771724ca56b86485a4b -r 55b4556e0b7eba1ae3bfeb6e850831e25b3238ac lib/galaxy/webapps/tool_shed/controllers/repository.py --- a/lib/galaxy/webapps/tool_shed/controllers/repository.py +++ b/lib/galaxy/webapps/tool_shed/controllers/repository.py @@ -2755,7 +2755,7 @@ # This method is called only from the ~/templates/webapps/tool_shed/repository/manage_repository.mako template. rmm = repository_metadata_manager.RepositoryMetadataManager( trans.app, trans.user ) invalid_file_tups, metadata_dict = \ - rmm.reset_all_metadata_on_repository_in_tool_shed( id, **kwd ) + rmm.reset_all_metadata_on_repository_in_tool_shed( id ) if invalid_file_tups: repository = suc.get_repository_in_tool_shed( trans.app, id ) message = tool_util.generate_message_for_invalid_tools( trans.app, diff -r e9de99ef77105d221df55771724ca56b86485a4b -r 55b4556e0b7eba1ae3bfeb6e850831e25b3238ac lib/tool_shed/galaxy_install/install_manager.py --- a/lib/tool_shed/galaxy_install/install_manager.py +++ b/lib/tool_shed/galaxy_install/install_manager.py @@ -42,6 +42,7 @@ def __init__( self, app ): self.app = app + self.install_model = self.app.install_model self.INSTALL_ACTIONS = [ 'download_binary', 'download_by_url', 'download_file', 'setup_perl_environment', 'setup_python_environment', 'setup_r_environment', 'setup_ruby_environment', 'shell_command' ] @@ -118,7 +119,7 @@ work_dir=work_dir, current_dir=current_dir, initial_download=False ) - if tool_dependency.status in [ self.app.install_model.ToolDependency.installation_status.ERROR ]: + if tool_dependency.status in [ self.install_model.ToolDependency.installation_status.ERROR ]: # If the tool_dependency status is in an error state, return it with no additional # processing. return tool_dependency @@ -130,7 +131,6 @@ return tool_dependency def install_and_build_package_via_fabric( self, tool_shed_repository, tool_dependency, actions_dict ): - sa_session = self.app.install_model.context try: # There is currently only one fabric method. tool_dependency = self.install_and_build_package( tool_shed_repository, tool_dependency, actions_dict ) @@ -221,8 +221,8 @@ tool_dependency, error_message, remove_installation_path=False ) - if tool_dependency and tool_dependency.status in [ self.app.install_model.ToolDependency.installation_status.INSTALLED, - self.app.install_model.ToolDependency.installation_status.ERROR ]: + if tool_dependency and tool_dependency.status in [ self.install_model.ToolDependency.installation_status.INSTALLED, + self.install_model.ToolDependency.installation_status.ERROR ]: installed_packages.append( tool_dependency ) if self.app.config.manage_dependency_relationships: # Add the tool_dependency to the in-memory dictionaries in the installed_repository_manager. @@ -237,7 +237,6 @@ self.install_and_build_package(). The use of fabric is being eliminated, so some of these functions may need to be renamed at some point. """ - sa_session = self.app.install_model.context if not os.path.exists( install_dir ): os.makedirs( install_dir ) actions_dict = dict( install_dir=install_dir ) @@ -353,7 +352,7 @@ package_name=package_name, actions_elem=actions_elem, action_elem=None ) - if tool_dependency.status == self.app.install_model.ToolDependency.installation_status.INSTALLED: + if tool_dependency.status == self.install_model.ToolDependency.installation_status.INSTALLED: # If an <actions> tag was found that matches the current platform, and # self.install_via_fabric() did not result in an error state, set binary_installed # to True in order to skip any remaining platform-specific <actions> tags. @@ -396,7 +395,7 @@ actions_elem=actions_elem, action_elem=None ) if actions_elem.tag == 'action' and \ - tool_dependency.status != self.app.install_model.ToolDependency.installation_status.ERROR: + tool_dependency.status != self.install_model.ToolDependency.installation_status.ERROR: # If the tool dependency is not in an error state, perform any final actions that have been # defined within the actions_group tag set, but outside of an <actions> tag, which defines # the recipe for installing and compiling from source. @@ -416,19 +415,19 @@ package_name=package_name, actions_elem=actions_elems, action_elem=None ) - if tool_dependency.status != self.app.install_model.ToolDependency.installation_status.ERROR: + if tool_dependency.status != self.install_model.ToolDependency.installation_status.ERROR: log.debug( 'Tool dependency %s version %s has been installed in %s.' % \ ( str( package_name ), str( package_version ), str( install_dir ) ) ) return tool_dependency def mark_tool_dependency_installed( self, tool_dependency ): - if tool_dependency.status not in [ self.app.install_model.ToolDependency.installation_status.ERROR, - self.app.install_model.ToolDependency.installation_status.INSTALLED ]: + if tool_dependency.status not in [ self.install_model.ToolDependency.installation_status.ERROR, + self.install_model.ToolDependency.installation_status.INSTALLED ]: log.debug( 'Changing status for tool dependency %s from %s to %s.' % \ ( str( tool_dependency.name ), str( tool_dependency.status ), - str( self.app.install_model.ToolDependency.installation_status.INSTALLED ) ) ) - status = self.app.install_model.ToolDependency.installation_status.INSTALLED + str( self.install_model.ToolDependency.installation_status.INSTALLED ) ) ) + status = self.install_model.ToolDependency.installation_status.INSTALLED tool_dependency = tool_dependency_util.set_tool_dependency_attributes( self.app, tool_dependency=tool_dependency, status=status, @@ -441,6 +440,8 @@ def __init__( self, app ): self.app = app + self.install_model = self.app.install_model + self.tpm = tool_panel_manager.ToolPanelManager( self.app ) def get_repository_components_for_installation( self, encoded_tsr_id, encoded_tsr_ids, repo_info_dicts, tool_panel_section_keys ): @@ -497,10 +498,10 @@ def handle_repository_contents( self, tool_shed_repository, tool_path, repository_clone_url, relative_install_dir, tool_shed=None, tool_section=None, shed_tool_conf=None, reinstalling=False ): """ - Generate the metadata for the installed tool shed repository, among other things. This method is called from Galaxy - (never the tool shed) when an administrator is installing a new repository or reinstalling an uninstalled repository. + Generate the metadata for the installed tool shed repository, among other things. + This method is called when an administrator is installing a new repository or + reinstalling an uninstalled repository. """ - install_model = self.app.install_model shed_config_dict = self.app.toolbox.get_shed_config_dict_by_filename( shed_tool_conf ) irmm = InstalledRepositoryMetadataManager( self.app ) metadata_dict, invalid_file_tups = \ @@ -518,8 +519,8 @@ tool_shed_status_dict = suc.get_tool_shed_status_for_installed_repository( self.app, tool_shed_repository ) if tool_shed_status_dict: tool_shed_repository.tool_shed_status = tool_shed_status_dict - install_model.context.add( tool_shed_repository ) - install_model.context.flush() + self.install_model.context.add( tool_shed_repository ) + self.install_model.context.flush() if 'tool_dependencies' in metadata_dict and not reinstalling: tool_dependencies = tool_dependency_util.create_tool_dependency_objects( self.app, tool_shed_repository, @@ -536,13 +537,12 @@ self.app.config.shed_tool_data_table_config, persist=True ) if 'tools' in metadata_dict: - tpm = tool_panel_manager.ToolPanelManager( self.app ) - tool_panel_dict = tpm.generate_tool_panel_dict_for_new_install( metadata_dict[ 'tools' ], tool_section ) + tool_panel_dict = self.tpm.generate_tool_panel_dict_for_new_install( metadata_dict[ 'tools' ], tool_section ) sample_files = metadata_dict.get( 'sample_files', [] ) tool_index_sample_files = tool_util.get_tool_index_sample_files( sample_files ) tool_util.copy_sample_files( self.app, tool_index_sample_files, tool_path=tool_path ) sample_files_copied = [ str( s ) for s in tool_index_sample_files ] - repository_tools_tups = suc.get_repository_tools_tups( self.app, metadata_dict ) + repository_tools_tups = irmm.get_repository_tools_tups( metadata_dict ) if repository_tools_tups: # Handle missing data table entries for tool parameters that are dynamically generated select lists. repository_tools_tups = tool_util.handle_missing_data_table_entry( self.app, @@ -561,14 +561,14 @@ sample_files, tool_path=tool_path, sample_files_copied=sample_files_copied ) - tpm.add_to_tool_panel( repository_name=tool_shed_repository.name, - repository_clone_url=repository_clone_url, - changeset_revision=tool_shed_repository.installed_changeset_revision, - repository_tools_tups=repository_tools_tups, - owner=tool_shed_repository.owner, - shed_tool_conf=shed_tool_conf, - tool_panel_dict=tool_panel_dict, - new_install=True ) + self.tpm.add_to_tool_panel( repository_name=tool_shed_repository.name, + repository_clone_url=repository_clone_url, + changeset_revision=tool_shed_repository.installed_changeset_revision, + repository_tools_tups=repository_tools_tups, + owner=tool_shed_repository.owner, + shed_tool_conf=shed_tool_conf, + tool_panel_dict=tool_panel_dict, + new_install=True ) if 'data_manager' in metadata_dict: new_data_managers = data_manager_util.install_data_managers( self.app, self.app.config.shed_data_manager_config_file, @@ -578,11 +578,11 @@ tool_shed_repository, repository_tools_tups ) if 'datatypes' in metadata_dict: - tool_shed_repository.status = install_model.ToolShedRepository.installation_status.LOADING_PROPRIETARY_DATATYPES + tool_shed_repository.status = self.install_model.ToolShedRepository.installation_status.LOADING_PROPRIETARY_DATATYPES if not tool_shed_repository.includes_datatypes: tool_shed_repository.includes_datatypes = True - install_model.context.add( tool_shed_repository ) - install_model.context.flush() + self.install_model.context.add( tool_shed_repository ) + self.install_model.context.flush() files_dir = relative_install_dir if shed_config_dict.get( 'tool_path' ): files_dir = os.path.join( shed_config_dict[ 'tool_path' ], files_dir ) @@ -628,7 +628,6 @@ return created_or_updated_tool_shed_repositories, tool_panel_section_keys, repo_info_dicts, filtered_repo_info_dicts def initiate_repository_installation( self, installation_dict ): - install_model = self.app.install_model # The following installation_dict entries are all required. created_or_updated_tool_shed_repositories = installation_dict[ 'created_or_updated_tool_shed_repositories' ] filtered_repo_info_dicts = installation_dict[ 'filtered_repo_info_dicts' ] @@ -648,11 +647,10 @@ tool_shed_url = installation_dict[ 'tool_shed_url' ] # Handle contained tools. if includes_tools_for_display_in_tool_panel and ( new_tool_panel_section_label or tool_panel_section_id ): - tpm = tool_panel_manager.ToolPanelManager( self.app ) tool_panel_section_key, tool_section = \ - tpm.handle_tool_panel_section( self.app.toolbox, - tool_panel_section_id=tool_panel_section_id, - new_tool_panel_section_label=new_tool_panel_section_label ) + self.tpm.handle_tool_panel_section( self.app.toolbox, + tool_panel_section_id=tool_panel_section_id, + new_tool_panel_section_label=new_tool_panel_section_label ) else: tool_panel_section_key = None tool_section = None @@ -675,12 +673,12 @@ tsr_ids = [ r.id for r in created_or_updated_tool_shed_repositories ] tool_shed_repositories = [] for tsr_id in tsr_ids: - tsr = install_model.context.query( install_model.ToolShedRepository ).get( tsr_id ) + tsr = self.install_model.context.query( self.install_model.ToolShedRepository ).get( tsr_id ) tool_shed_repositories.append( tsr ) clause_list = [] for tsr_id in tsr_ids: - clause_list.append( install_model.ToolShedRepository.table.c.id == tsr_id ) - query = install_model.context.query( install_model.ToolShedRepository ).filter( or_( *clause_list ) ) + clause_list.append( self.install_model.ToolShedRepository.table.c.id == tsr_id ) + query = self.install_model.context.query( self.install_model.ToolShedRepository ).filter( or_( *clause_list ) ) return encoded_kwd, query, tool_shed_repositories, encoded_repository_ids def install( self, tool_shed_url, name, owner, changeset_revision, install_options ): @@ -726,7 +724,7 @@ shed_tool_conf = install_options.get( 'shed_tool_conf', None ) if shed_tool_conf: # Get the tool_path setting. - index, shed_conf_dict = suc.get_shed_tool_conf_dict( self.app, shed_tool_conf ) + index, shed_conf_dict = self.tpm.get_shed_tool_conf_dict( shed_tool_conf ) tool_path = shed_conf_dict[ 'tool_path' ] else: # Pick a semi-random shed-related tool panel configuration file and get the tool_path setting. @@ -792,11 +790,10 @@ self.order_components_for_installation( tsr_ids, repo_info_dicts, tool_panel_section_keys=tool_panel_section_keys ) # Install the repositories, keeping track of each one for later display. for index, tsr_id in enumerate( ordered_tsr_ids ): - install_model = self.app.install_model - tool_shed_repository = install_model.context.query( install_model.ToolShedRepository ) \ - .get( self.app.security.decode_id( tsr_id ) ) - if tool_shed_repository.status in [ install_model.ToolShedRepository.installation_status.NEW, - install_model.ToolShedRepository.installation_status.UNINSTALLED ]: + tool_shed_repository = self.install_model.context.query( self.install_model.ToolShedRepository ) \ + .get( self.app.security.decode_id( tsr_id ) ) + if tool_shed_repository.status in [ self.install_model.ToolShedRepository.installation_status.NEW, + self.install_model.ToolShedRepository.installation_status.UNINSTALLED ]: repo_info_dict = ordered_repo_info_dicts[ index ] tool_panel_section_key = ordered_tool_panel_section_keys[ index ] self.install_tool_shed_repository( tool_shed_repository, @@ -814,7 +811,6 @@ def install_tool_shed_repository( self, tool_shed_repository, repo_info_dict, tool_panel_section_key, shed_tool_conf, tool_path, install_tool_dependencies, reinstalling=False ): - install_model = self.app.install_model if tool_panel_section_key: try: tool_section = self.app.toolbox.tool_panel[ tool_panel_section_key ] @@ -827,9 +823,8 @@ if isinstance( repo_info_dict, basestring ): repo_info_dict = encoding_util.tool_shed_decode( repo_info_dict ) # Clone each repository to the configured location. - suc.update_tool_shed_repository_status( self.app, - tool_shed_repository, - install_model.ToolShedRepository.installation_status.CLONING ) + self.update_tool_shed_repository_status( tool_shed_repository, + self.install_model.ToolShedRepository.installation_status.CLONING ) repo_info_tuple = repo_info_dict[ tool_shed_repository.name ] description, repository_clone_url, changeset_revision, ctx_rev, repository_owner, repository_dependencies, tool_dependencies = repo_info_tuple relative_clone_dir = suc.generate_tool_shed_repository_install_dir( repository_clone_url, @@ -859,13 +854,12 @@ tool_section=tool_section, shed_tool_conf=shed_tool_conf, reinstalling=reinstalling ) - install_model.context.refresh( tool_shed_repository ) + self.install_model.context.refresh( tool_shed_repository ) metadata = tool_shed_repository.metadata if 'tools' in metadata: # Get the tool_versions from the tool shed for each tool in the installed change set. - suc.update_tool_shed_repository_status( self.app, - tool_shed_repository, - install_model.ToolShedRepository.installation_status.SETTING_TOOL_VERSIONS ) + self.update_tool_shed_repository_status( tool_shed_repository, + self.install_model.ToolShedRepository.installation_status.SETTING_TOOL_VERSIONS ) tool_shed_url = common_util.get_tool_shed_url_from_tool_shed_registry( self.app, str( tool_shed_repository.tool_shed ) ) params = '?name=%s&owner=%s&changeset_revision=%s' % ( str( tool_shed_repository.name ), str( tool_shed_repository.owner ), @@ -885,9 +879,8 @@ if install_tool_dependencies and tool_shed_repository.tool_dependencies and 'tool_dependencies' in metadata: work_dir = tempfile.mkdtemp( prefix="tmp-toolshed-itsr" ) # Install tool dependencies. - suc.update_tool_shed_repository_status( self.app, - tool_shed_repository, - install_model.ToolShedRepository.installation_status.INSTALLING_TOOL_DEPENDENCIES ) + self.update_tool_shed_repository_status( tool_shed_repository, + self.install_model.ToolShedRepository.installation_status.INSTALLING_TOOL_DEPENDENCIES ) # Get the tool_dependencies.xml file from the repository. tool_dependencies_config = hg_util.get_config_from_disk( 'tool_dependencies.xml', install_dir ) itdm = InstallToolDependencyManager( self.app ) @@ -896,9 +889,8 @@ tool_dependencies=tool_shed_repository.tool_dependencies, from_tool_migration_manager=False ) basic_util.remove_dir( work_dir ) - suc.update_tool_shed_repository_status( self.app, - tool_shed_repository, - install_model.ToolShedRepository.installation_status.INSTALLED ) + self.update_tool_shed_repository_status( tool_shed_repository, + self.install_model.ToolShedRepository.installation_status.INSTALLED ) if self.app.config.manage_dependency_relationships: # Add the installed repository and any tool dependencies to the in-memory dictionaries # in the installed_repository_manager. @@ -907,7 +899,7 @@ # An error occurred while cloning the repository, so reset everything necessary to enable another attempt. suc.set_repository_attributes( self.app, tool_shed_repository, - status=install_model.ToolShedRepository.installation_status.ERROR, + status=self.install_model.ToolShedRepository.installation_status.ERROR, error_message=error_message, deleted=False, uninstalled=False, @@ -962,3 +954,13 @@ ordered_repo_info_dicts.append( repo_info_dict ) ordered_tool_panel_section_keys.append( tool_panel_section_key ) return ordered_tsr_ids, ordered_repo_info_dicts, ordered_tool_panel_section_keys + + def update_tool_shed_repository_status( self, tool_shed_repository, status, error_message=None ): + """ + Update the status of a tool shed repository in the process of being installed into Galaxy. + """ + tool_shed_repository.status = status + if error_message: + tool_shed_repository.error_message = str( error_message ) + self.install_model.context.add( tool_shed_repository ) + self.install_model.context.flush() diff -r e9de99ef77105d221df55771724ca56b86485a4b -r 55b4556e0b7eba1ae3bfeb6e850831e25b3238ac lib/tool_shed/galaxy_install/installed_repository_manager.py --- a/lib/tool_shed/galaxy_install/installed_repository_manager.py +++ b/lib/tool_shed/galaxy_install/installed_repository_manager.py @@ -14,6 +14,7 @@ from tool_shed.util import xml_util from galaxy.model.orm import and_ +from tool_shed.galaxy_install.metadata.installed_repository_metadata_manager import InstalledRepositoryMetadataManager from tool_shed.galaxy_install.repository_dependencies import repository_dependency_manager from tool_shed.galaxy_install.tools import tool_panel_manager @@ -80,9 +81,10 @@ repository.deleted = False repository.status = self.install_model.ToolShedRepository.installation_status.INSTALLED if repository.includes_tools_for_display_in_tool_panel: + irmm = InstalledRepositoryMetadataManager( self.app ) tpm = tool_panel_manager.ToolPanelManager( self.app ) metadata = repository.metadata - repository_tools_tups = suc.get_repository_tools_tups( self.app, metadata ) + repository_tools_tups = irmm.get_repository_tools_tups( metadata ) # Reload tools into the appropriate tool panel section. tool_panel_dict = repository.metadata[ 'tool_panel_section' ] tpm.add_to_tool_panel( repository.name, diff -r e9de99ef77105d221df55771724ca56b86485a4b -r 55b4556e0b7eba1ae3bfeb6e850831e25b3238ac lib/tool_shed/galaxy_install/metadata/installed_repository_metadata_manager.py --- a/lib/tool_shed/galaxy_install/metadata/installed_repository_metadata_manager.py +++ b/lib/tool_shed/galaxy_install/metadata/installed_repository_metadata_manager.py @@ -5,11 +5,14 @@ from galaxy.util import inflector from galaxy.web.form_builder import SelectField +from tool_shed.galaxy_install.tools import tool_panel_manager from tool_shed.metadata import metadata_generator from tool_shed.util import common_util +from tool_shed.util import repository_maintenance_util from tool_shed.util import shed_util_common as suc from tool_shed.util import tool_util +from tool_shed.util import xml_util log = logging.getLogger( __name__ ) @@ -19,6 +22,7 @@ def __init__( self, app ): super( InstalledRepositoryMetadataManager, self ).__init__( app ) self.app = app + self.tpm = tool_panel_manager.ToolPanelManager( self.app ) def build_repository_ids_select_field( self, name='repository_ids', multiple=True, display='checkboxes' ): """Generate the current list of repositories for resetting metadata.""" @@ -46,11 +50,32 @@ return self.app.install_model.context.query( self.app.install_model.ToolShedRepository ) \ .filter( self.app.install_model.ToolShedRepository.table.c.uninstalled == False ) + def get_repository_tools_tups( self, metadata_dict ): + """ + Return a list of tuples of the form (relative_path, guid, tool) for each tool defined + in the received tool shed repository metadata. + """ + repository_tools_tups = [] + index, shed_conf_dict = self.tpm.get_shed_tool_conf_dict( metadata_dict.get( 'shed_config_filename' ) ) + if 'tools' in metadata_dict: + for tool_dict in metadata_dict[ 'tools' ]: + load_relative_path = relative_path = tool_dict.get( 'tool_config', None ) + if shed_conf_dict.get( 'tool_path' ): + load_relative_path = os.path.join( shed_conf_dict.get( 'tool_path' ), relative_path ) + guid = tool_dict.get( 'guid', None ) + if relative_path and guid: + tool = self.app.toolbox.load_tool( os.path.abspath( load_relative_path ), guid=guid ) + else: + tool = None + if tool: + repository_tools_tups.append( ( relative_path, guid, tool ) ) + return repository_tools_tups + def reset_all_metadata_on_installed_repository( self, id ): """Reset all metadata on a single tool shed repository installed into a Galaxy instance.""" invalid_file_tups = [] metadata_dict = {} - repository = suc.get_installed_tool_shed_repository( self.app, id ) + repository = repository_maintenance_util.get_installed_tool_shed_repository( self.app, id ) repository_clone_url = common_util.generate_clone_url_for_installed_repository( self.app, repository ) tool_path, relative_install_dir = repository.get_tool_relative_path( self.app ) if relative_install_dir: @@ -67,7 +92,7 @@ persist=False ) repository.metadata = metadata_dict if metadata_dict != original_metadata_dict: - suc.update_in_shed_tool_config( self.app, repository ) + self.update_in_shed_tool_config( repository ) self.app.install_model.context.add( repository ) self.app.install_model.context.flush() log.debug( 'Metadata has been reset on repository %s.' % repository.name ) @@ -90,7 +115,7 @@ unsuccessful_count = 0 for repository_id in repository_ids: try: - repository = suc.get_installed_tool_shed_repository( self.app, repository_id ) + repository = repository_maintenance_util.get_installed_tool_shed_repository( self.app, repository_id ) owner = str( repository.owner ) invalid_file_tups, metadata_dict = \ self.reset_all_metadata_on_installed_repository( repository_id ) @@ -117,3 +142,49 @@ message = 'Select at least one repository to on which to reset all metadata.' status = 'error' return message, status + + def tool_shed_from_repository_clone_url( self, repository_clone_url ): + """Given a repository clone URL, return the tool shed that contains the repository.""" + return common_util.remove_protocol_and_user_from_clone_url( repository_clone_url ).split( '/repos/' )[ 0 ].rstrip( '/' ) + + def update_in_shed_tool_config( self, repository ): + """ + A tool shed repository is being updated so change the shed_tool_conf file. Parse the config + file to generate the entire list of config_elems instead of using the in-memory list. + """ + shed_conf_dict = repository.get_shed_config_dict( self.app ) + shed_tool_conf = shed_conf_dict[ 'config_filename' ] + tool_path = shed_conf_dict[ 'tool_path' ] + tool_panel_dict = self.tpm.generate_tool_panel_dict_from_shed_tool_conf_entries( repository ) + repository_tools_tups = self.get_repository_tools_tups( repository.metadata ) + clone_url = common_util.generate_clone_url_for_installed_repository( self.app, repository ) + cleaned_repository_clone_url = common_util.remove_protocol_and_user_from_clone_url( clone_url ) + tool_shed = self.tool_shed_from_repository_clone_url( cleaned_repository_clone_url ) + owner = repository.owner + if not owner: + owner = suc.get_repository_owner( cleaned_repository_clone_url ) + guid_to_tool_elem_dict = {} + for tool_config_filename, guid, tool in repository_tools_tups: + guid_to_tool_elem_dict[ guid ] = self.tpm.generate_tool_elem( tool_shed, + repository.name, + repository.changeset_revision, + repository.owner or '', + tool_config_filename, + tool, + None ) + config_elems = [] + tree, error_message = xml_util.parse_xml( shed_tool_conf ) + if tree: + root = tree.getroot() + for elem in root: + if elem.tag == 'section': + for i, tool_elem in enumerate( elem ): + guid = tool_elem.attrib.get( 'guid' ) + if guid in guid_to_tool_elem_dict: + elem[i] = guid_to_tool_elem_dict[ guid ] + elif elem.tag == 'tool': + guid = elem.attrib.get( 'guid' ) + if guid in guid_to_tool_elem_dict: + elem = guid_to_tool_elem_dict[ guid ] + config_elems.append( elem ) + self.tpm.config_elems_to_xml_file( config_elems, shed_tool_conf, tool_path ) diff -r e9de99ef77105d221df55771724ca56b86485a4b -r 55b4556e0b7eba1ae3bfeb6e850831e25b3238ac lib/tool_shed/galaxy_install/repair_repository_manager.py --- a/lib/tool_shed/galaxy_install/repair_repository_manager.py +++ b/lib/tool_shed/galaxy_install/repair_repository_manager.py @@ -190,6 +190,7 @@ if repository.status in [ self.app.install_model.ToolShedRepository.installation_status.ERROR ]: repair_dict = add_repair_dict_entry( repository.name, repository.error_message ) else: + irm = install_manager.InstallRepositoryManager( self.app ) # We have an installed tool shed repository, so handle tool dependencies if necessary. if repository.missing_tool_dependencies and metadata and 'tool_dependencies' in metadata: work_dir = tempfile.mkdtemp( prefix="tmp-toolshed-itdep" ) @@ -204,8 +205,7 @@ error_message=None, remove_from_disk=True ) # Install tool dependencies. - suc.update_tool_shed_repository_status( self.app, - repository, + irm.update_tool_shed_repository_status( repository, self.app.install_model.ToolShedRepository.installation_status.INSTALLING_TOOL_DEPENDENCIES ) # Get the tool_dependencies.xml file from the repository. tool_dependencies_config = hg_util.get_config_from_disk( 'tool_dependencies.xml', repository.repo_path( self.app ) ) @@ -218,5 +218,6 @@ if installed_tool_dependency.status in [ self.app.install_model.ToolDependency.installation_status.ERROR ]: repair_dict = add_repair_dict_entry( repository.name, installed_tool_dependency.error_message ) basic_util.remove_dir( work_dir ) - suc.update_tool_shed_repository_status( self.app, repository, self.app.install_model.ToolShedRepository.installation_status.INSTALLED ) + irm.update_tool_shed_repository_status( repository, + self.app.install_model.ToolShedRepository.installation_status.INSTALLED ) return repair_dict \ No newline at end of file diff -r e9de99ef77105d221df55771724ca56b86485a4b -r 55b4556e0b7eba1ae3bfeb6e850831e25b3238ac lib/tool_shed/galaxy_install/repository_dependencies/repository_dependency_manager.py --- a/lib/tool_shed/galaxy_install/repository_dependencies/repository_dependency_manager.py +++ b/lib/tool_shed/galaxy_install/repository_dependencies/repository_dependency_manager.py @@ -206,7 +206,7 @@ # Set changeset_revision here so suc.create_or_update_tool_shed_repository will find # the previously installed and uninstalled repository instead of creating a new record. changeset_revision = repository_db_record.installed_changeset_revision - suc.reset_previously_installed_repository( install_model, repository_db_record ) + self.reset_previously_installed_repository( repository_db_record ) can_update_db_record = True else: # No record exists in the database for the repository currently being processed. @@ -409,3 +409,26 @@ return False return True return False + + def reset_previously_installed_repository( self, repository ): + """ + Reset the attributes of a tool_shed_repository that was previously installed. + The repository will be in some state other than INSTALLED, so all attributes + will be set to the default NEW state. This will enable the repository to be + freshly installed. + """ + debug_msg = "Resetting tool_shed_repository '%s' for installation.\n" % str( repository.name ) + debug_msg += "The current state of the tool_shed_repository is:\n" + debug_msg += "deleted: %s\n" % str( repository.deleted ) + debug_msg += "tool_shed_status: %s\n" % str( repository.tool_shed_status ) + debug_msg += "uninstalled: %s\n" % str( repository.uninstalled ) + debug_msg += "status: %s\n" % str( repository.status ) + debug_msg += "error_message: %s\n" % str( repository.error_message ) + log.debug( debug_msg ) + repository.deleted = False + repository.tool_shed_status = None + repository.uninstalled = False + repository.status = self.app.install_model.ToolShedRepository.installation_status.NEW + repository.error_message = None + self.app.install_model.context.add( repository ) + self.app.install_model.context.flush() diff -r e9de99ef77105d221df55771724ca56b86485a4b -r 55b4556e0b7eba1ae3bfeb6e850831e25b3238ac lib/tool_shed/galaxy_install/tool_migration_manager.py --- a/lib/tool_shed/galaxy_install/tool_migration_manager.py +++ b/lib/tool_shed/galaxy_install/tool_migration_manager.py @@ -74,7 +74,7 @@ self.tool_shed_url = common_util.get_tool_shed_url_from_tool_shed_registry( self.app, defined_tool_shed_url ) self.tool_shed = common_util.remove_protocol_and_port_from_tool_shed_url( self.tool_shed_url ) self.repository_owner = common_util.REPOSITORY_OWNER - index, self.shed_config_dict = suc.get_shed_tool_conf_dict( app, self.migrated_tools_config ) + index, self.shed_config_dict = self.tpm.get_shed_tool_conf_dict( self.migrated_tools_config ) # Since tool migration scripts can be executed any number of times, we need to # make sure the appropriate tools are defined in tool_conf.xml. If no tools # associated with the migration stage are defined, no repositories will be installed @@ -442,7 +442,7 @@ tool_index_sample_files = tool_util.get_tool_index_sample_files( sample_files ) tool_util.copy_sample_files( self.app, tool_index_sample_files, tool_path=self.tool_path ) sample_files_copied = [ s for s in tool_index_sample_files ] - repository_tools_tups = suc.get_repository_tools_tups( self.app, metadata_dict ) + repository_tools_tups = irmm.get_repository_tools_tups( metadata_dict ) if repository_tools_tups: # Handle missing data table entries for tool parameters that are dynamically generated select lists. repository_tools_tups = tool_util.handle_missing_data_table_entry( self.app, @@ -471,12 +471,12 @@ new_install=True ) if install_dependencies and tool_dependencies and has_tool_dependencies: # Install tool dependencies. - suc.update_tool_shed_repository_status( self.app, - tool_shed_repository, + irm = install_manager.InstallRepositoryManager( self.app ) + itdm = install_manager.InstallToolDependencyManager( self.app ) + irm.update_tool_shed_repository_status( tool_shed_repository, self.app.install_model.ToolShedRepository.installation_status.INSTALLING_TOOL_DEPENDENCIES ) # Get the tool_dependencies.xml file from disk. tool_dependencies_config = hg_util.get_config_from_disk( 'tool_dependencies.xml', repo_install_dir ) - itdm = install_manager.InstallToolDependencyManager( self.app ) installed_tool_dependencies = itdm.install_specified_tool_dependencies( tool_shed_repository=tool_shed_repository, tool_dependencies_config=tool_dependencies_config, tool_dependencies=tool_dependencies, @@ -533,6 +533,7 @@ if cloned_ok and is_installed: print "Skipping automatic install of repository '", tool_shed_repository.name, "' because it has already been installed in location ", clone_dir else: + irm = install_manager.InstallRepositoryManager( self.app ) repository_clone_url = os.path.join( self.tool_shed_url, 'repos', tool_shed_repository.owner, tool_shed_repository.name ) relative_install_dir = os.path.join( relative_clone_dir, tool_shed_repository.name ) install_dir = os.path.join( clone_dir, tool_shed_repository.name ) @@ -542,8 +543,7 @@ tool_shed_repository.owner, tool_shed_repository.installed_changeset_revision ) if not cloned_ok: - suc.update_tool_shed_repository_status( self.app, - tool_shed_repository, + irm.update_tool_shed_repository_status( tool_shed_repository, self.app.install_model.ToolShedRepository.installation_status.CLONING ) cloned_ok, error_message = hg_util.clone_repository( repository_clone_url, os.path.abspath( install_dir ), ctx_rev ) if cloned_ok and not is_installed: @@ -556,8 +556,7 @@ self.app.install_model.context.refresh( tool_shed_repository ) metadata_dict = tool_shed_repository.metadata if 'tools' in metadata_dict: - suc.update_tool_shed_repository_status( self.app, - tool_shed_repository, + irm.update_tool_shed_repository_status( tool_shed_repository, self.app.install_model.ToolShedRepository.installation_status.SETTING_TOOL_VERSIONS ) # Get the tool_versions from the tool shed for each tool in the installed change set. url = '%s/repository/get_tool_versions?name=%s&owner=%s&changeset_revision=%s' % \ @@ -595,11 +594,11 @@ parent_id=tool_version_using_old_id.id ) self.app.install_model.context.add( tool_version_association ) self.app.install_model.context.flush() - suc.update_tool_shed_repository_status( self.app, tool_shed_repository, self.app.install_model.ToolShedRepository.installation_status.INSTALLED ) + irm.update_tool_shed_repository_status( tool_shed_repository, + self.app.install_model.ToolShedRepository.installation_status.INSTALLED ) else: print 'Error attempting to clone repository %s: %s' % ( str( tool_shed_repository.name ), str( error_message ) ) - suc.update_tool_shed_repository_status( self.app, - tool_shed_repository, + irm.update_tool_shed_repository_status( tool_shed_repository, self.app.install_model.ToolShedRepository.installation_status.ERROR, error_message=error_message ) diff -r e9de99ef77105d221df55771724ca56b86485a4b -r 55b4556e0b7eba1ae3bfeb6e850831e25b3238ac lib/tool_shed/galaxy_install/tools/tool_panel_manager.py --- a/lib/tool_shed/galaxy_install/tools/tool_panel_manager.py +++ b/lib/tool_shed/galaxy_install/tools/tool_panel_manager.py @@ -1,5 +1,7 @@ import logging import os +import shutil +import tempfile import galaxy.tools from galaxy.tools.search import ToolBoxSearch @@ -19,9 +21,11 @@ self.app = app def add_to_shed_tool_config( self, shed_tool_conf_dict, elem_list ): - # A tool shed repository is being installed so change the shed_tool_conf file. Parse the - # config file to generate the entire list of config_elems instead of using the in-memory list - # since it will be a subset of the entire list if one or more repositories have been deactivated. + """ + "A tool shed repository is being installed so change the shed_tool_conf file. Parse the + config file to generate the entire list of config_elems instead of using the in-memory list + since it will be a subset of the entire list if one or more repositories have been deactivated. + """ shed_tool_conf = shed_tool_conf_dict[ 'config_filename' ] tool_path = shed_tool_conf_dict[ 'tool_path' ] config_elems = [] @@ -34,13 +38,13 @@ for elem_entry in elem_list: config_elems.append( elem_entry ) # Persist the altered shed_tool_config file. - suc.config_elems_to_xml_file( self.app, config_elems, shed_tool_conf, tool_path ) + self.config_elems_to_xml_file( config_elems, shed_tool_conf, tool_path ) def add_to_tool_panel( self, repository_name, repository_clone_url, changeset_revision, repository_tools_tups, owner, shed_tool_conf, tool_panel_dict, new_install=True ): """A tool shed repository is being installed or updated so handle tool panel alterations accordingly.""" # We need to change the in-memory version and the file system version of the shed_tool_conf file. - index, shed_tool_conf_dict = suc.get_shed_tool_conf_dict( self.app, shed_tool_conf ) + index, shed_tool_conf_dict = self.get_shed_tool_conf_dict( shed_tool_conf ) tool_path = shed_tool_conf_dict[ 'tool_path' ] # Generate the list of ElementTree Element objects for each section or tool. elem_list = self.generate_tool_panel_elem_list( repository_name, @@ -81,6 +85,44 @@ self.app.toolbox.write_integrated_tool_panel_config_file() self.app.toolbox_search = ToolBoxSearch( self.app.toolbox ) + def config_elems_to_xml_file( self, config_elems, config_filename, tool_path ): + """ + Persist the current in-memory list of config_elems to a file named by the + value of config_filename. + """ + fd, filename = tempfile.mkstemp( prefix="tmp-toolshed-cetxf" ) + os.write( fd, '<?xml version="1.0"?>\n' ) + os.write( fd, '<toolbox tool_path="%s">\n' % str( tool_path ) ) + for elem in config_elems: + os.write( fd, '%s' % xml_util.xml_to_string( elem, use_indent=True ) ) + os.write( fd, '</toolbox>\n' ) + os.close( fd ) + shutil.move( filename, os.path.abspath( config_filename ) ) + os.chmod( config_filename, 0644 ) + + def generate_tool_elem( self, tool_shed, repository_name, changeset_revision, owner, tool_file_path, + tool, tool_section ): + """Create and return an ElementTree tool Element.""" + if tool_section is not None: + tool_elem = XmlET.SubElement( tool_section, 'tool' ) + else: + tool_elem = XmlET.Element( 'tool' ) + tool_elem.attrib[ 'file' ] = tool_file_path + tool_elem.attrib[ 'guid' ] = tool.guid + tool_shed_elem = XmlET.SubElement( tool_elem, 'tool_shed' ) + tool_shed_elem.text = tool_shed + repository_name_elem = XmlET.SubElement( tool_elem, 'repository_name' ) + repository_name_elem.text = repository_name + repository_owner_elem = XmlET.SubElement( tool_elem, 'repository_owner' ) + repository_owner_elem.text = owner + changeset_revision_elem = XmlET.SubElement( tool_elem, 'installed_changeset_revision' ) + changeset_revision_elem.text = changeset_revision + id_elem = XmlET.SubElement( tool_elem, 'id' ) + id_elem.text = tool.id + version_elem = XmlET.SubElement( tool_elem, 'version' ) + version_elem.text = tool.version + return tool_elem + def generate_tool_panel_dict_for_new_install( self, tool_dicts, tool_section=None ): """ When installing a repository that contains tools, all tools must currently be defined @@ -125,6 +167,60 @@ tool_panel_dict[ guid ] = tool_section_dicts return tool_panel_dict + def generate_tool_panel_dict_from_shed_tool_conf_entries( self, repository ): + """ + Keep track of the section in the tool panel in which this repository's tools + will be contained by parsing the shed_tool_conf in which the repository's tools + are defined and storing the tool panel definition of each tool in the repository. + This method is called only when the repository is being deactivated or un-installed + and allows for activation or re-installation using the original layout. + """ + tool_panel_dict = {} + shed_tool_conf, tool_path, relative_install_dir = \ + suc.get_tool_panel_config_tool_path_install_dir( self.app, repository ) + metadata = repository.metadata + # Create a dictionary of tool guid and tool config file name for each tool in the repository. + guids_and_configs = {} + if 'tools' in metadata: + for tool_dict in metadata[ 'tools' ]: + guid = tool_dict[ 'guid' ] + tool_config = tool_dict[ 'tool_config' ] + file_name = basic_util.strip_path( tool_config ) + guids_and_configs[ guid ] = file_name + # Parse the shed_tool_conf file in which all of this repository's tools are defined and generate the tool_panel_dict. + tree, error_message = xml_util.parse_xml( shed_tool_conf ) + if tree is None: + return tool_panel_dict + root = tree.getroot() + for elem in root: + if elem.tag == 'tool': + guid = elem.get( 'guid' ) + if guid in guids_and_configs: + # The tool is displayed in the tool panel outside of any tool sections. + tool_section_dict = dict( tool_config=guids_and_configs[ guid ], id='', name='', version='' ) + if guid in tool_panel_dict: + tool_panel_dict[ guid ].append( tool_section_dict ) + else: + tool_panel_dict[ guid ] = [ tool_section_dict ] + elif elem.tag == 'section': + section_id = elem.get( 'id' ) or '' + section_name = elem.get( 'name' ) or '' + section_version = elem.get( 'version' ) or '' + for section_elem in elem: + if section_elem.tag == 'tool': + guid = section_elem.get( 'guid' ) + if guid in guids_and_configs: + # The tool is displayed in the tool panel inside the current tool section. + tool_section_dict = dict( tool_config=guids_and_configs[ guid ], + id=section_id, + name=section_name, + version=section_version ) + if guid in tool_panel_dict: + tool_panel_dict[ guid ].append( tool_section_dict ) + else: + tool_panel_dict[ guid ] = [ tool_section_dict ] + return tool_panel_dict + def generate_tool_panel_elem_list( self, repository_name, repository_clone_url, changeset_revision, tool_panel_dict, repository_tools_tups, owner='' ): """Generate a list of ElementTree Element objects for each section or tool.""" @@ -157,21 +253,21 @@ if tup_guid == guid: break if inside_section: - tool_elem = suc.generate_tool_elem( tool_shed, - repository_name, - changeset_revision, - owner, - tool_file_path, - tool, - tool_section ) + tool_elem = self.generate_tool_elem( tool_shed, + repository_name, + changeset_revision, + owner, + tool_file_path, + tool, + tool_section ) else: - tool_elem = suc.generate_tool_elem( tool_shed, - repository_name, - changeset_revision, - owner, - tool_file_path, - tool, - None ) + tool_elem = self.generate_tool_elem( tool_shed, + repository_name, + changeset_revision, + owner, + tool_file_path, + tool, + None ) if inside_section: if section_in_elem_list: elem_list[ index ] = tool_section @@ -238,6 +334,19 @@ log.debug( "Loading new tool panel section: %s" % str( tool_section.name ) ) return tool_panel_section_key, tool_section + def get_shed_tool_conf_dict( self, shed_tool_conf ): + """ + Return the in-memory version of the shed_tool_conf file, which is stored in + the config_elems entry in the shed_tool_conf_dict associated with the file. + """ + for index, shed_tool_conf_dict in enumerate( self.app.toolbox.shed_tool_confs ): + if shed_tool_conf == shed_tool_conf_dict[ 'config_filename' ]: + return index, shed_tool_conf_dict + else: + file_name = basic_util.strip_path( shed_tool_conf_dict[ 'config_filename' ] ) + if shed_tool_conf == file_name: + return index, shed_tool_conf_dict + def handle_tool_panel_section( self, toolbox, tool_panel_section_id=None, new_tool_panel_section_label=None ): """Return a ToolSection object retrieved from the current in-memory tool_panel.""" # If tool_panel_section_id is received, the section exists in the tool panel. In this @@ -299,10 +408,12 @@ return tool_section, tool_panel_section_key def remove_from_shed_tool_config( self, shed_tool_conf_dict, guids_to_remove ): - # A tool shed repository is being uninstalled so change the shed_tool_conf file. - # Parse the config file to generate the entire list of config_elems instead of - # using the in-memory list since it will be a subset of the entire list if one - # or more repositories have been deactivated. + """ + A tool shed repository is being uninstalled so change the shed_tool_conf file. + Parse the config file to generate the entire list of config_elems instead of + using the in-memory list since it will be a subset of the entire list if one + or more repositories have been deactivated. + """ shed_tool_conf = shed_tool_conf_dict[ 'config_filename' ] tool_path = shed_tool_conf_dict[ 'tool_path' ] config_elems = [] @@ -330,7 +441,7 @@ for config_elem in config_elems_to_remove: config_elems.remove( config_elem ) # Persist the altered in-memory version of the tool config. - suc.config_elems_to_xml_file( self.app, config_elems, shed_tool_conf, tool_path ) + self.config_elems_to_xml_file( config_elems, shed_tool_conf, tool_path ) def remove_from_tool_panel( self, repository, shed_tool_conf, uninstall ): """ @@ -340,7 +451,7 @@ # Determine where the tools are currently defined in the tool panel and store this # information so the tools can be displayed in the same way when the repository is # activated or reinstalled. - tool_panel_dict = suc.generate_tool_panel_dict_from_shed_tool_conf_entries( self.app, repository ) + tool_panel_dict = self.generate_tool_panel_dict_from_shed_tool_conf_entries( repository ) repository.metadata[ 'tool_panel_section' ] = tool_panel_dict self.app.install_model.context.add( repository ) self.app.install_model.context.flush() @@ -351,7 +462,7 @@ for guid_to_remove in guids_to_remove: if guid_to_remove in self.app.toolbox.tools_by_id: del self.app.toolbox.tools_by_id[ guid_to_remove ] - index, shed_tool_conf_dict = suc.get_shed_tool_conf_dict( self.app, shed_tool_conf ) + index, shed_tool_conf_dict = self.get_shed_tool_conf_dict( shed_tool_conf ) if uninstall: # Remove from the shed_tool_conf file on disk. self.remove_from_shed_tool_config( shed_tool_conf_dict, guids_to_remove ) diff -r e9de99ef77105d221df55771724ca56b86485a4b -r 55b4556e0b7eba1ae3bfeb6e850831e25b3238ac lib/tool_shed/metadata/repository_metadata_manager.py --- a/lib/tool_shed/metadata/repository_metadata_manager.py +++ b/lib/tool_shed/metadata/repository_metadata_manager.py @@ -951,11 +951,9 @@ unsuccessful_count = 0 for repository_id in repository_ids: try: - # We're in the tool shed. repository = suc.get_repository_in_tool_shed( self.app, repository_id ) - owner = str( repository.user.username ) invalid_file_tups, metadata_dict = \ - self.reset_all_metadata_on_repository_in_tool_shed( self.user, repository_id ) + self.reset_all_metadata_on_repository_in_tool_shed( repository_id ) if invalid_file_tups: message = tool_util.generate_message_for_invalid_tools( self.app, invalid_file_tups, @@ -966,7 +964,7 @@ unsuccessful_count += 1 else: log.debug( "Successfully reset metadata on repository %s owned by %s" % \ - ( str( repository.name ), owner ) ) + ( str( repository.name ), str( repository.user.username ) ) ) successful_count += 1 except: log.exception( "Error attempting to reset metadata on repository %s" % str( repository.name ) ) diff -r e9de99ef77105d221df55771724ca56b86485a4b -r 55b4556e0b7eba1ae3bfeb6e850831e25b3238ac lib/tool_shed/util/data_manager_util.py --- a/lib/tool_shed/util/data_manager_util.py +++ b/lib/tool_shed/util/data_manager_util.py @@ -1,7 +1,10 @@ import logging import os + +from tool_shed.galaxy_install.tools import tool_panel_manager + +from tool_shed.util import shed_util_common as suc from tool_shed.util import xml_util -import tool_shed.util.shed_util_common as suc log = logging.getLogger( __name__ ) @@ -17,6 +20,7 @@ def install_data_managers( app, shed_data_manager_conf_filename, metadata_dict, shed_config_dict, relative_install_dir, repository, repository_tools_tups ): rval = [] if 'data_manager' in metadata_dict: + tpm = tool_panel_manager.ToolPanelManager( app ) repository_tools_by_guid = {} for tool_tup in repository_tools_tups: repository_tools_by_guid[ tool_tup[ 1 ] ] = dict( tool_config_filename=tool_tup[ 0 ], tool=tool_tup[ 2 ] ) @@ -70,13 +74,13 @@ elem.set( 'shed_conf_file', shed_config_dict['config_filename'] ) if elem.get( 'tool_file', None ) is not None: del elem.attrib[ 'tool_file' ] #remove old tool_file info - tool_elem = suc.generate_tool_elem( repository.tool_shed, - repository.name, - repository.installed_changeset_revision, - repository.owner, - tool_config_filename, - tool, - None ) + tool_elem = tpm.generate_tool_elem( repository.tool_shed, + repository.name, + repository.installed_changeset_revision, + repository.owner, + tool_config_filename, + tool, + None ) elem.insert( 0, tool_elem ) data_manager = app.data_managers.load_manager_from_elem( elem, tool_path=shed_config_dict.get( 'tool_path', '' ), replace_existing=True ) if data_manager: diff -r e9de99ef77105d221df55771724ca56b86485a4b -r 55b4556e0b7eba1ae3bfeb6e850831e25b3238ac lib/tool_shed/util/repository_maintenance_util.py --- a/lib/tool_shed/util/repository_maintenance_util.py +++ b/lib/tool_shed/util/repository_maintenance_util.py @@ -40,6 +40,20 @@ config.write( new_file ) new_file.close() +def check_or_update_tool_shed_status_for_installed_repository( app, repository ): + updated = False + tool_shed_status_dict = suc.get_tool_shed_status_for_installed_repository( app, repository ) + if tool_shed_status_dict: + ok = True + if tool_shed_status_dict != repository.tool_shed_status: + repository.tool_shed_status = tool_shed_status_dict + app.install_model.context.add( repository ) + app.install_model.context.flush() + updated = True + else: + ok = False + return ok, updated + def create_repo_info_dict( app, repository_clone_url, changeset_revision, ctx_rev, repository_owner, repository_name=None, repository=None, repository_metadata=None, tool_dependencies=None, repository_dependencies=None ): """ @@ -171,6 +185,11 @@ sa_session.flush() return role +def get_installed_tool_shed_repository( app, id ): + """Get a tool shed repository record from the Galaxy database defined by the id.""" + return app.install_model.context.query( app.install_model.ToolShedRepository ) \ + .get( app.security.decode_id( id ) ) + def get_repo_info_dict( app, user, repository_id, changeset_revision ): repository = suc.get_repository_in_tool_shed( app, repository_id ) repo = hg_util.get_repo_for_repository( app, repository=repository, repo_path=None, create=False ) diff -r e9de99ef77105d221df55771724ca56b86485a4b -r 55b4556e0b7eba1ae3bfeb6e850831e25b3238ac lib/tool_shed/util/shed_util_common.py --- a/lib/tool_shed/util/shed_util_common.py +++ b/lib/tool_shed/util/shed_util_common.py @@ -1,24 +1,22 @@ +import json import logging import os import re import shutil import string -import tempfile +import sqlalchemy.orm.exc + from galaxy import util -from galaxy.util import asbool -from galaxy.util import json from galaxy.web import url_for -from galaxy.web.form_builder import SelectField from galaxy.datatypes import checkers from galaxy.model.orm import and_ from galaxy.model.orm import or_ -import sqlalchemy.orm.exc + from tool_shed.util import basic_util from tool_shed.util import common_util from tool_shed.util import encoding_util from tool_shed.util import hg_util -from tool_shed.util import xml_util -import tool_shed.repository_types.util as rt_util + from xml.etree import ElementTree as XmlET from urllib2 import HTTPError @@ -84,49 +82,6 @@ '${host}' """ -def build_tool_dependencies_select_field( app, tool_shed_repository, name, multiple=True, display='checkboxes', uninstalled_only=False ): - """Method called from Galaxy to generate the current list of tool dependency ids for an installed tool shed repository.""" - tool_dependencies_select_field = SelectField( name=name, multiple=multiple, display=display ) - for tool_dependency in tool_shed_repository.tool_dependencies: - if uninstalled_only: - if tool_dependency.status not in [ app.install_model.ToolDependency.installation_status.NEVER_INSTALLED, - app.install_model.ToolDependency.installation_status.UNINSTALLED ]: - continue - else: - if tool_dependency.status in [ app.install_model.ToolDependency.installation_status.NEVER_INSTALLED, - app.install_model.ToolDependency.installation_status.UNINSTALLED ]: - continue - option_label = '%s version %s' % ( str( tool_dependency.name ), str( tool_dependency.version ) ) - option_value = app.security.encode_id( tool_dependency.id ) - tool_dependencies_select_field.add_option( option_label, option_value ) - return tool_dependencies_select_field - -def check_or_update_tool_shed_status_for_installed_repository( app, repository ): - updated = False - tool_shed_status_dict = get_tool_shed_status_for_installed_repository( app, repository ) - if tool_shed_status_dict: - ok = True - if tool_shed_status_dict != repository.tool_shed_status: - repository.tool_shed_status = tool_shed_status_dict - app.install_model.context.add( repository ) - app.install_model.context.flush() - updated = True - else: - ok = False - return ok, updated - -def config_elems_to_xml_file( app, config_elems, config_filename, tool_path ): - """Persist the current in-memory list of config_elems to a file named by the value of config_filename.""" - fd, filename = tempfile.mkstemp( prefix="tmp-toolshed-cetxf" ) - os.write( fd, '<?xml version="1.0"?>\n' ) - os.write( fd, '<toolbox tool_path="%s">\n' % str( tool_path ) ) - for elem in config_elems: - os.write( fd, '%s' % xml_util.xml_to_string( elem, use_indent=True ) ) - os.write( fd, '</toolbox>\n' ) - os.close( fd ) - shutil.move( filename, os.path.abspath( config_filename ) ) - os.chmod( config_filename, 0644 ) - def create_or_update_tool_shed_repository( app, name, description, installed_changeset_revision, ctx_rev, repository_clone_url, metadata_dict, status, current_changeset_revision=None, owner='', dist_to_shed=False ): """ @@ -248,28 +203,6 @@ sharable_url += '/%s' % changeset_revision return sharable_url -def generate_tool_elem( tool_shed, repository_name, changeset_revision, owner, tool_file_path, tool, tool_section ): - """Create and return an ElementTree tool Element.""" - if tool_section is not None: - tool_elem = XmlET.SubElement( tool_section, 'tool' ) - else: - tool_elem = XmlET.Element( 'tool' ) - tool_elem.attrib[ 'file' ] = tool_file_path - tool_elem.attrib[ 'guid' ] = tool.guid - tool_shed_elem = XmlET.SubElement( tool_elem, 'tool_shed' ) - tool_shed_elem.text = tool_shed - repository_name_elem = XmlET.SubElement( tool_elem, 'repository_name' ) - repository_name_elem.text = repository_name - repository_owner_elem = XmlET.SubElement( tool_elem, 'repository_owner' ) - repository_owner_elem.text = owner - changeset_revision_elem = XmlET.SubElement( tool_elem, 'installed_changeset_revision' ) - changeset_revision_elem.text = changeset_revision - id_elem = XmlET.SubElement( tool_elem, 'id' ) - id_elem.text = tool.id - version_elem = XmlET.SubElement( tool_elem, 'version' ) - version_elem.text = tool.version - return tool_elem - def generate_tool_guid( repository_clone_url, tool ): """ Generate a guid for the installed tool. It is critical that this guid matches the guid for @@ -279,57 +212,6 @@ tmp_url = common_util.remove_protocol_and_user_from_clone_url( repository_clone_url ) return '%s/%s/%s' % ( tmp_url, tool.id, tool.version ) -def generate_tool_panel_dict_from_shed_tool_conf_entries( app, repository ): - """ - Keep track of the section in the tool panel in which this repository's tools will be contained by parsing the shed_tool_conf in - which the repository's tools are defined and storing the tool panel definition of each tool in the repository. This method is called - only when the repository is being deactivated or uninstalled and allows for activation or reinstallation using the original layout. - """ - tool_panel_dict = {} - shed_tool_conf, tool_path, relative_install_dir = get_tool_panel_config_tool_path_install_dir( app, repository ) - metadata = repository.metadata - # Create a dictionary of tool guid and tool config file name for each tool in the repository. - guids_and_configs = {} - if 'tools' in metadata: - for tool_dict in metadata[ 'tools' ]: - guid = tool_dict[ 'guid' ] - tool_config = tool_dict[ 'tool_config' ] - file_name = basic_util.strip_path( tool_config ) - guids_and_configs[ guid ] = file_name - # Parse the shed_tool_conf file in which all of this repository's tools are defined and generate the tool_panel_dict. - tree, error_message = xml_util.parse_xml( shed_tool_conf ) - if tree is None: - return tool_panel_dict - root = tree.getroot() - for elem in root: - if elem.tag == 'tool': - guid = elem.get( 'guid' ) - if guid in guids_and_configs: - # The tool is displayed in the tool panel outside of any tool sections. - tool_section_dict = dict( tool_config=guids_and_configs[ guid ], id='', name='', version='' ) - if guid in tool_panel_dict: - tool_panel_dict[ guid ].append( tool_section_dict ) - else: - tool_panel_dict[ guid ] = [ tool_section_dict ] - elif elem.tag == 'section': - section_id = elem.get( 'id' ) or '' - section_name = elem.get( 'name' ) or '' - section_version = elem.get( 'version' ) or '' - for section_elem in elem: - if section_elem.tag == 'tool': - guid = section_elem.get( 'guid' ) - if guid in guids_and_configs: - # The tool is displayed in the tool panel inside the current tool section. - tool_section_dict = dict( tool_config=guids_and_configs[ guid ], - id=section_id, - name=section_name, - version=section_version ) - if guid in tool_panel_dict: - tool_panel_dict[ guid ].append( tool_section_dict ) - else: - tool_panel_dict[ guid ] = [ tool_section_dict ] - return tool_panel_dict - def generate_tool_shed_repository_install_dir( repository_clone_url, changeset_revision ): """ Generate a repository installation directory that guarantees repositories with the same @@ -483,10 +365,6 @@ return ','.join( installing_repository_ids ) return installing_repository_ids -def get_installed_tool_shed_repository( app, id ): - """Get a tool shed repository record from the Galaxy database defined by the id.""" - return app.install_model.context.query( app.install_model.ToolShedRepository ).get( app.security.decode_id( id ) ) - def get_latest_changeset_revision( app, repository, repo ): repository_tip = repository.tip( app ) repository_metadata = get_repository_metadata_by_changeset_revision( app, @@ -565,8 +443,10 @@ def get_or_create_tool_shed_repository( app, tool_shed, name, owner, changeset_revision ): """ - Return a tool shed repository database record defined by the combination of tool shed, repository name, repository owner and changeset_revision - or installed_changeset_revision. A new tool shed repository record will be created if one is not located. + Return a tool shed repository database record defined by the combination of + tool shed, repository name, repository owner and changeset_revision or + installed_changeset_revision. A new tool shed repository record will be + created if one is not located. """ install_model = app.install_model # We store the port in the database. @@ -697,7 +577,7 @@ for rd_tup in repository_dependencies: tool_shed, name, owner, changeset_revision, prior_installation_required, only_if_compiling_contained_td = \ common_util.parse_repository_dependency_tuple( rd_tup ) - if not asbool( only_if_compiling_contained_td ): + if not util.asbool( only_if_compiling_contained_td ): has_repository_dependencies = True break # Set has_repository_dependencies_only_if_compiling_contained_td, which will be True only if at @@ -706,7 +586,7 @@ for rd_tup in repository_dependencies: tool_shed, name, owner, changeset_revision, prior_installation_required, only_if_compiling_contained_td = \ common_util.parse_repository_dependency_tuple( rd_tup ) - if asbool( only_if_compiling_contained_td ): + if util.asbool( only_if_compiling_contained_td ): has_repository_dependencies_only_if_compiling_contained_td = True break return has_repository_dependencies, has_repository_dependencies_only_if_compiling_contained_td @@ -838,8 +718,8 @@ # where the repository dependency is not installed prior to the dependent repository's tool dependency compilation process, the tool # dependency compilation framework will install the repository dependency prior to compilation of the dependent repository's tool # dependency. - if not asbool( only_if_compiling_contained_td ): - if asbool( prior_installation_required ): + if not util.asbool( only_if_compiling_contained_td ): + if util.asbool( prior_installation_required ): if is_tool_shed_client( app ): # We store the port, if one exists, in the database. tool_shed = common_util.remove_protocol_from_tool_shed_url( tool_shed ) @@ -899,34 +779,6 @@ query = app.model.context.query( app.model.Repository ) return query -def get_repository_tools_tups( app, metadata_dict ): - """Return a list of tuples of the form (relative_path, guid, tool) for each tool defined in the received tool shed repository metadata.""" - repository_tools_tups = [] - index, shed_conf_dict = get_shed_tool_conf_dict( app, metadata_dict.get( 'shed_config_filename' ) ) - if 'tools' in metadata_dict: - for tool_dict in metadata_dict[ 'tools' ]: - load_relative_path = relative_path = tool_dict.get( 'tool_config', None ) - if shed_conf_dict.get( 'tool_path' ): - load_relative_path = os.path.join( shed_conf_dict.get( 'tool_path' ), relative_path ) - guid = tool_dict.get( 'guid', None ) - if relative_path and guid: - tool = app.toolbox.load_tool( os.path.abspath( load_relative_path ), guid=guid ) - else: - tool = None - if tool: - repository_tools_tups.append( ( relative_path, guid, tool ) ) - return repository_tools_tups - -def get_shed_tool_conf_dict( app, shed_tool_conf ): - """Return the in-memory version of the shed_tool_conf file, which is stored in the config_elems entry in the shed_tool_conf_dict associated with the file.""" - for index, shed_tool_conf_dict in enumerate( app.toolbox.shed_tool_confs ): - if shed_tool_conf == shed_tool_conf_dict[ 'config_filename' ]: - return index, shed_tool_conf_dict - else: - file_name = basic_util.strip_path( shed_tool_conf_dict[ 'config_filename' ] ) - if shed_tool_conf == file_name: - return index, shed_tool_conf_dict - def get_skip_tool_test_by_changeset_revision( app, changeset_revision ): """Return a skip_tool_test record whose initial_changeset_revision is the received changeset_revision.""" # There should only be one, but we'll use first() so callers won't have to handle exceptions. @@ -1238,7 +1090,7 @@ email_alerts.append( user.email ) else: subject = "Galaxy tool shed update alert for repository named %s" % str( repository.name ) - email_alerts = json.from_json_string( repository.email_alerts ) + email_alerts = json.loads( repository.email_alerts ) for email in email_alerts: to = email.strip() # Send it @@ -1262,7 +1114,8 @@ return False def is_tool_shed_client( app ): - """ The tool shed and clients to the tool (i.e. Galaxy) require a lot + """ + The tool shed and clients to the tool (i.e. Galaxy) require a lot of similar functionality in this file but with small differences. This method should determine if the app performing the action is the tool shed or a client of the tool shed. @@ -1338,32 +1191,12 @@ return tool_shed_repository, previous_changeset_revision return None, None -def reset_previously_installed_repository( install_model, repository ): - """ - Reset the atrributes of a tool_shed_repository that was previsouly installed. The repository will be in some state other than INSTALLED, - so all atributes will be set to the default NEW state. This will enable the repository to be freshly installed. - """ - debug_msg = "Resetting tool_shed_repository '%s' for installation.\n" % str( repository.name ) - debug_msg += "The current state of the tool_shed_repository is:\n" - debug_msg += "deleted: %s\n" % str( repository.deleted ) - debug_msg += "tool_shed_status: %s\n" % str( repository.tool_shed_status ) - debug_msg += "uninstalled: %s\n" % str( repository.uninstalled ) - debug_msg += "status: %s\n" % str( repository.status ) - debug_msg += "error_message: %s\n" % str( repository.error_message ) - log.debug( debug_msg ) - repository.deleted = False - repository.tool_shed_status = None - repository.uninstalled = False - repository.status = install_model.ToolShedRepository.installation_status.NEW - repository.error_message = None - install_model.context.add( repository ) - install_model.context.flush() - def set_image_paths( app, encoded_repository_id, text ): """ - Handle tool help image display for tools that are contained in repositories in the tool shed or installed into Galaxy as well as image - display in repository README files. This method will determine the location of the image file and return the path to it that will enable - the caller to open the file. + Handle tool help image display for tools that are contained in repositories in + the tool shed or installed into Galaxy as well as image display in repository + README files. This method will determine the location of the image file and + return the path to it that will enable the caller to open the file. """ if text: if is_tool_shed_client( app ): @@ -1373,22 +1206,32 @@ route_to_images = '/repository/static/images/%s' % encoded_repository_id # We used to require $PATH_TO_IMAGES, but we now eliminate it if it's used. text = text.replace( '$PATH_TO_IMAGES', '' ) - # Eliminate the invalid setting of ./static/images since the routes will properly display images contained in that directory. + # Eliminate the invalid setting of ./static/images since the routes will + # properly display images contained in that directory. text = text.replace( './static/images', '' ) - # Eliminate the default setting of /static/images since the routes will properly display images contained in that directory. + # Eliminate the default setting of /static/images since the routes will + # properly display images contained in that directory. text = text.replace( '/static/images', '' ) - # Use regex to instantiate routes into the defined image paths, but replace paths that start with neither http:// nor https://, - # which will allow for settings like .. images:: http_files/images/help.png + # Use regex to instantiate routes into the defined image paths, but replace + # paths that start with neither http:// nor https://, which will allow for + # settings like .. images:: http_files/images/help.png for match in re.findall( '.. image:: (?!http)/?(.+)', text ): text = text.replace( match, match.replace( '/', '%2F' ) ) text = re.sub( r'\.\. image:: (?!https?://)/?(.+)', r'.. image:: %s/\1' % route_to_images, text ) return text def set_only_if_compiling_contained_td( repository, required_repository ): - """Return True if the received required_repository is only needed to compile a tool dependency defined for the received repository.""" - # This method is called only from Galaxy when rendering repository dependencies for an installed tool shed repository. + """ + Return True if the received required_repository is only needed to compile a tool + dependency defined for the received repository. + """ + # This method is called only from Galaxy when rendering repository dependencies + # for an installed tool shed repository. # TODO: Do we need to check more than changeset_revision here? - required_repository_tup = [ required_repository.tool_shed, required_repository.name, required_repository.owner, required_repository.changeset_revision ] + required_repository_tup = [ required_repository.tool_shed, \ + required_repository.name, \ + required_repository.owner, \ + required_repository.changeset_revision ] for tup in repository.tuples_of_repository_dependencies_needed_for_compiling_td: partial_tup = tup[ 0:4 ] if partial_tup == required_repository_tup: @@ -1430,63 +1273,8 @@ app.install_model.context.add( repository ) app.install_model.context.flush() -def tool_shed_from_repository_clone_url( repository_clone_url ): - """Given a repository clone URL, return the tool shed that contains the repository.""" - return common_util.remove_protocol_and_user_from_clone_url( repository_clone_url ).split( '/repos/' )[ 0 ].rstrip( '/' ) - def tool_shed_is_this_tool_shed( toolshed_base_url ): """Determine if a tool shed is the current tool shed.""" cleaned_toolshed_base_url = common_util.remove_protocol_from_tool_shed_url( toolshed_base_url ) cleaned_tool_shed = common_util.remove_protocol_from_tool_shed_url( str( url_for( '/', qualified=True ) ) ) return cleaned_toolshed_base_url == cleaned_tool_shed - -def update_in_shed_tool_config( app, repository ): - """ - A tool shed repository is being updated so change the shed_tool_conf file. Parse the config - file to generate the entire list of config_elems instead of using the in-memory list. - """ - shed_conf_dict = repository.get_shed_config_dict( app ) - shed_tool_conf = shed_conf_dict[ 'config_filename' ] - tool_path = shed_conf_dict[ 'tool_path' ] - tool_panel_dict = generate_tool_panel_dict_from_shed_tool_conf_entries( app, repository ) - repository_tools_tups = get_repository_tools_tups( app, repository.metadata ) - clone_url = common_util.generate_clone_url_for_installed_repository( app, repository ) - cleaned_repository_clone_url = common_util.remove_protocol_and_user_from_clone_url( clone_url ) - tool_shed = tool_shed_from_repository_clone_url( cleaned_repository_clone_url ) - owner = repository.owner - if not owner: - owner = get_repository_owner( cleaned_repository_clone_url ) - guid_to_tool_elem_dict = {} - for tool_config_filename, guid, tool in repository_tools_tups: - guid_to_tool_elem_dict[ guid ] = generate_tool_elem( tool_shed, - repository.name, - repository.changeset_revision, - repository.owner or '', - tool_config_filename, - tool, - None ) - config_elems = [] - tree, error_message = xml_util.parse_xml( shed_tool_conf ) - if tree: - root = tree.getroot() - for elem in root: - if elem.tag == 'section': - for i, tool_elem in enumerate( elem ): - guid = tool_elem.attrib.get( 'guid' ) - if guid in guid_to_tool_elem_dict: - elem[i] = guid_to_tool_elem_dict[ guid ] - elif elem.tag == 'tool': - guid = elem.attrib.get( 'guid' ) - if guid in guid_to_tool_elem_dict: - elem = guid_to_tool_elem_dict[ guid ] - config_elems.append( elem ) - config_elems_to_xml_file( app, config_elems, shed_tool_conf, tool_path ) - -def update_tool_shed_repository_status( app, tool_shed_repository, status, error_message=None ): - """Update the status of a tool shed repository in the process of being installed into Galaxy.""" - context = app.install_model.context - tool_shed_repository.status = status - if error_message: - tool_shed_repository.error_message = str( error_message ) - context.add( tool_shed_repository ) - context.flush() diff -r e9de99ef77105d221df55771724ca56b86485a4b -r 55b4556e0b7eba1ae3bfeb6e850831e25b3238ac lib/tool_shed/util/tool_dependency_util.py --- a/lib/tool_shed/util/tool_dependency_util.py +++ b/lib/tool_shed/util/tool_dependency_util.py @@ -4,11 +4,34 @@ from galaxy import util from galaxy.model.orm import and_ +from galaxy.web.form_builder import SelectField + from tool_shed.util import hg_util from tool_shed.util import xml_util log = logging.getLogger( __name__ ) +def build_tool_dependencies_select_field( app, tool_shed_repository, name, multiple=True, display='checkboxes', + uninstalled_only=False ): + """ + Generate a SelectField consisting of the current list of tool dependency ids + for an installed tool shed repository. + """ + tool_dependencies_select_field = SelectField( name=name, multiple=multiple, display=display ) + for tool_dependency in tool_shed_repository.tool_dependencies: + if uninstalled_only: + if tool_dependency.status not in [ app.install_model.ToolDependency.installation_status.NEVER_INSTALLED, + app.install_model.ToolDependency.installation_status.UNINSTALLED ]: + continue + else: + if tool_dependency.status in [ app.install_model.ToolDependency.installation_status.NEVER_INSTALLED, + app.install_model.ToolDependency.installation_status.UNINSTALLED ]: + continue + option_label = '%s version %s' % ( str( tool_dependency.name ), str( tool_dependency.version ) ) + option_value = app.security.encode_id( tool_dependency.id ) + tool_dependencies_select_field.add_option( option_label, option_value ) + return tool_dependencies_select_field + def create_or_update_tool_dependency( app, tool_shed_repository, name, version, type, status, set_status=True ): """Create or update a tool_dependency record in the Galaxy database.""" # Called from Galaxy (never the tool shed) when a new repository is being installed or when an uninstalled diff -r e9de99ef77105d221df55771724ca56b86485a4b -r 55b4556e0b7eba1ae3bfeb6e850831e25b3238ac lib/tool_shed/util/xml_util.py --- a/lib/tool_shed/util/xml_util.py +++ b/lib/tool_shed/util/xml_util.py @@ -137,8 +137,8 @@ def xml_to_string( elem, encoding='utf-8', use_indent=False, level=0 ): if elem is not None: if use_indent: - # We were called from suc.config_elems_to_xml_file(), so set the level to 1 since level 0 - # is the <toolbox> tag set. + # We were called from ToolPanelManager.config_elems_to_xml_file(), so + # set the level to 1 since level 0 is the <toolbox> tag set. indent( elem, level=level ) if using_python_27: xml_str = '%s\n' % xml.etree.ElementTree.tostring( elem, encoding=encoding, method="xml" ) Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.