[galaxy-commits] commit/galaxy-central: greg: Handle combinations of newly defined tool dependencies and repository dependencies that are included in updates being pulled to tool shed repositories installed into Galaxy.

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/90b4baa5c2b1/ Changeset: 90b4baa5c2b1 User: greg Date: 2014-03-26 19:58:08 Summary: Handle combinations of newly defined tool dependencies and repository dependencies that are included in updates being pulled to tool shed repositories installed into Galaxy. Affected #: 15 files diff -r 09985439d17f9bef026554938b05d0d6eedd06cb -r 90b4baa5c2b18d49bcea5a0807f0df8924abebb1 lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py --- a/lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py +++ b/lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py @@ -201,9 +201,13 @@ repository_id = kwd.get( 'id', None ) repository = suc.get_installed_tool_shed_repository( trans, repository_id ) tool_shed_url = suc.get_url_from_tool_shed( trans.app, repository.tool_shed ) + params = '?galaxy_url=%s&name=%s&owner=%s&changeset_revision=%s' % \ + ( web.url_for( '/', qualified=True ), + str( repository.name ), + str( repository.owner ), + str( repository.changeset_revision ) ) url = suc.url_join( tool_shed_url, - 'repository/check_for_updates?galaxy_url=%s&name=%s&owner=%s&changeset_revision=%s' % \ - ( web.url_for( '/', qualified=True ), repository.name, repository.owner, repository.changeset_revision ) ) + 'repository/check_for_updates%s' % params ) return trans.response.send_redirect( url ) @web.expose @@ -367,9 +371,10 @@ @web.require_admin def get_tool_dependencies( self, trans, repository_id, repository_name, repository_owner, changeset_revision ): """ - Send a request to the appropriate tool shed to retrieve the dictionary of tool dependencies defined for the received repository name, - owner and changeset revision. The received repository_id is the encoded id of the installed tool shed repository in Galaxy. We need - it so that we can derive the tool shed from which it was installed. + Send a request to the appropriate tool shed to retrieve the dictionary of tool dependencies defined for + the received repository name, owner and changeset revision. The received repository_id is the encoded id + of the installed tool shed repository in Galaxy. We need it so that we can derive the tool shed from which + it was installed. """ repository = suc.get_installed_tool_shed_repository( trans, repository_id ) tool_shed_url = suc.get_url_from_tool_shed( trans.app, repository.tool_shed ) @@ -388,8 +393,8 @@ @web.require_admin def get_updated_repository_information( self, trans, repository_id, repository_name, repository_owner, changeset_revision ): """ - Send a request to the appropriate tool shed to retrieve the dictionary of information required to reinstall an updated revision of an - uninstalled tool shed repository. + Send a request to the appropriate tool shed to retrieve the dictionary of information required to reinstall + an updated revision of an uninstalled tool shed repository. """ repository = suc.get_installed_tool_shed_repository( trans, repository_id ) tool_shed_url = suc.get_url_from_tool_shed( trans.app, repository.tool_shed ) @@ -436,13 +441,16 @@ @web.expose @web.require_admin - def initiate_tool_dependency_installation( self, trans, tool_dependencies ): + def initiate_tool_dependency_installation( self, trans, tool_dependencies, **kwd ): """Install specified dependencies for repository tools.""" # Get the tool_shed_repository from one of the tool_dependencies. - message = '' + message = kwd.get( 'message', '' ) + status = kwd.get( 'status', 'done' ) + err_msg = '' tool_shed_repository = tool_dependencies[ 0 ].tool_shed_repository # Get the tool_dependencies.xml file from the repository. - tool_dependencies_config = suc.get_config_from_disk( suc.TOOL_DEPENDENCY_DEFINITION_FILENAME, tool_shed_repository.repo_path( trans.app ) ) + tool_dependencies_config = suc.get_config_from_disk( suc.TOOL_DEPENDENCY_DEFINITION_FILENAME, + tool_shed_repository.repo_path( trans.app ) ) installed_tool_dependencies = common_install_util.handle_tool_dependencies( app=trans.app, tool_shed_repository=tool_shed_repository, tool_dependencies_config=tool_dependencies_config, @@ -452,13 +460,12 @@ if installed_tool_dependency.status == trans.app.install_model.ToolDependency.installation_status.ERROR: text = util.unicodify( installed_tool_dependency.error_message ) if text is not None: - message += ' %s' % text + err_msg += ' %s' % text tool_dependency_ids = [ trans.security.encode_id( td.id ) for td in tool_dependencies ] - if message: + if err_msg: + message += err_msg status = 'error' - else: - message = "Installed tool dependencies: %s" % ','.join( td.name for td in installed_tool_dependencies ) - status = 'done' + message += "Installed tool dependencies: %s" % ','.join( td.name for td in installed_tool_dependencies ) td_ids = [ trans.security.encode_id( td.id ) for td in tool_shed_repository.tool_dependencies ] return trans.response.send_redirect( web.url_for( controller='admin_toolshed', action='manage_tool_dependencies', @@ -506,9 +513,10 @@ status = 'error' else: # Install the latest downloadable revision of the repository. + params = '?name=%s&owner=%s&changeset_revisions=%s&galaxy_url=%s' % \ + ( name, owner, str( latest_downloadable_revision ), web.url_for( '/', qualified=True ) ) url = suc.url_join( tool_shed_url, - 'repository/install_repositories_by_revision?name=%s&owner=%s&changeset_revisions=%s&galaxy_url=%s' % \ - ( name, owner, latest_downloadable_revision, web.url_for( '/', qualified=True ) ) ) + 'repository/install_repositories_by_revision%s' % params ) return trans.response.send_redirect( url ) else: message = 'Cannot locate installed tool shed repository with encoded id <b>%s</b>.' % str( repository_id ) @@ -521,43 +529,74 @@ message=message, status=status ) ) - @web.expose @web.require_admin - def install_tool_dependencies( self, trans, **kwd ): + def install_tool_dependencies_with_update( self, trans, **kwd ): + """ + Updating an installed tool shed repository where new tool dependencies but no new repository + dependencies are included in the updated revision. + """ + updating_repository_id = kwd.get( 'updating_repository_id', None ) + repository = suc.get_installed_tool_shed_repository( trans, updating_repository_id ) + # All received dependencies need to be installed - confirmed by the caller. + encoded_tool_dependencies_dict = kwd.get( 'encoded_tool_dependencies_dict', None ) + if encoded_tool_dependencies_dict is not None: + tool_dependencies_dict = encoding_util.tool_shed_decode( encoded_tool_dependencies_dict ) + else: + tool_dependencies_dict = {} + encoded_relative_install_dir = kwd.get( 'encoded_relative_install_dir', None ) + if encoded_relative_install_dir is not None: + relative_install_dir = encoding_util.tool_shed_decode( encoded_relative_install_dir ) + else: + relative_install_dir = '' + updating_to_changeset_revision = kwd.get( 'updating_to_changeset_revision', None ) + updating_to_ctx_rev = kwd.get( 'updating_to_ctx_rev', None ) + encoded_updated_metadata = kwd.get( 'encoded_updated_metadata', None ) message = kwd.get( 'message', '' ) status = kwd.get( 'status', 'done' ) - tool_dependency_ids = tool_dependency_util.get_tool_dependency_ids( as_string=False, **kwd ) - tool_dependencies = [] - for tool_dependency_id in tool_dependency_ids: - tool_dependency = tool_dependency_util.get_tool_dependency( trans, tool_dependency_id ) - tool_dependencies.append( tool_dependency ) - if kwd.get( 'install_tool_dependencies_button', False ): - # Filter tool dependencies to only those that are installed. - tool_dependencies_for_installation = [] - for tool_dependency in tool_dependencies: - if tool_dependency.status in [ trans.install_model.ToolDependency.installation_status.UNINSTALLED, - trans.install_model.ToolDependency.installation_status.ERROR ]: - tool_dependencies_for_installation.append( tool_dependency ) - if tool_dependencies_for_installation: - # Redirect back to the ToolDependencyGrid before initiating installation. - encoded_tool_dependency_for_installation_ids = [ trans.security.encode_id( td.id ) for td in tool_dependencies_for_installation ] - new_kwd = dict( action='manage_tool_dependencies', - operation='initiate_tool_dependency_installation', - tool_dependency_ids=encoded_tool_dependency_for_installation_ids, - message=message, - status=status ) - return self.tool_dependency_grid( trans, **new_kwd ) + install_tool_dependencies = CheckboxField.is_checked( kwd.get( 'install_tool_dependencies', '' ) ) + if 'install_tool_dependencies_with_update_button' in kwd: + # Now that the user has chosen whether to install tool dependencies or not, we can + # update the repository record with the changes in the updated revision. + if encoded_updated_metadata: + updated_metadata = encoding_util.tool_shed_decode( encoded_updated_metadata ) else: - message = 'All of the selected tool dependencies are already installed.' - status = 'error' - return trans.response.send_redirect( web.url_for( controller='admin_toolshed', - action='manage_repository_tool_dependencies', - tool_dependency_ids=tool_dependency_ids, - status=status, - message=message ) ) - return trans.fill_template( '/admin/tool_shed_repository/install_tool_dependencies.mako', - tool_dependencies=tool_dependencies, + updated_metadata = None + repository = repository_util.update_repository_record( trans, + repository=repository, + updated_metadata_dict=updated_metadata, + updated_changeset_revision=updating_to_changeset_revision, + updated_ctx_rev=updating_to_ctx_rev ) + if install_tool_dependencies: + tool_dependencies = tool_dependency_util.create_tool_dependency_objects( trans.app, + repository, + relative_install_dir, + set_status=False ) + message = "The installed repository named '%s' has been updated to change set revision '%s'. " % \ + ( str( repository.name ), updating_to_changeset_revision ) + self.initiate_tool_dependency_installation( trans, tool_dependencies, message=message, status=status ) + # Handle tool dependencies check box. + if trans.app.config.tool_dependency_dir is None: + if includes_tool_dependencies: + message = "Tool dependencies defined in this repository can be automatically installed if you set " + message += "the value of your <b>tool_dependency_dir</b> setting in your Galaxy config file " + message += "(universe_wsgi.ini) and restart your Galaxy server." + status = "warning" + install_tool_dependencies_check_box_checked = False + else: + install_tool_dependencies_check_box_checked = True + install_tool_dependencies_check_box = CheckboxField( 'install_tool_dependencies', + checked=install_tool_dependencies_check_box_checked ) + return trans.fill_template( '/admin/tool_shed_repository/install_tool_dependencies_with_update.mako', + repository=repository, + updating_repository_id=updating_repository_id, + updating_to_ctx_rev=updating_to_ctx_rev, + updating_to_changeset_revision=updating_to_changeset_revision, + encoded_updated_metadata=encoded_updated_metadata, + encoded_relative_install_dir=encoded_relative_install_dir, + encoded_tool_dependencies_dict=encoded_tool_dependencies_dict, + install_tool_dependencies_check_box=install_tool_dependencies_check_box, + tool_dependencies_dict=tool_dependencies_dict, message=message, status=status ) @@ -611,7 +650,8 @@ # TODO: I believe this block should be removed, but make sure.. repositories_for_uninstallation = [] for repository_id in tsridslist: - repository = trans.install_model.context.query( trans.install_model.ToolShedRepository ).get( trans.security.decode_id( repository_id ) ) + repository = trans.install_model.context.query( trans.install_model.ToolShedRepository ) \ + .get( trans.security.decode_id( repository_id ) ) if repository.status in [ trans.install_model.ToolShedRepository.installation_status.INSTALLED, trans.install_model.ToolShedRepository.installation_status.ERROR ]: repositories_for_uninstallation.append( repository ) @@ -632,28 +672,34 @@ filtered_repo_info_dicts = [] filtered_tool_panel_section_keys = [] repositories_for_installation = [] - # Some repositories may have repository dependencies that are required to be installed before the dependent repository, so we'll - # order the list of tsr_ids to ensure all repositories install in the required order. + # Some repositories may have repository dependencies that are required to be installed before the + # dependent repository, so we'll order the list of tsr_ids to ensure all repositories install in the + # required order. ordered_tsr_ids, ordered_repo_info_dicts, ordered_tool_panel_section_keys = \ repository_util.order_components_for_installation( trans, tsr_ids, repo_info_dicts, tool_panel_section_keys=tool_panel_section_keys ) for tsr_id in ordered_tsr_ids: - repository = trans.install_model.context.query( trans.install_model.ToolShedRepository ).get( trans.security.decode_id( tsr_id ) ) + repository = trans.install_model.context.query( trans.install_model.ToolShedRepository ) \ + .get( trans.security.decode_id( tsr_id ) ) if repository.status in [ trans.install_model.ToolShedRepository.installation_status.NEW, trans.install_model.ToolShedRepository.installation_status.UNINSTALLED ]: repositories_for_installation.append( repository ) - repo_info_dict, tool_panel_section_key = repository_util.get_repository_components_for_installation( tsr_id, - ordered_tsr_ids, - ordered_repo_info_dicts, - ordered_tool_panel_section_keys ) + repo_info_dict, tool_panel_section_key = \ + repository_util.get_repository_components_for_installation( tsr_id, + ordered_tsr_ids, + ordered_repo_info_dicts, + ordered_tool_panel_section_keys ) filtered_repo_info_dicts.append( repo_info_dict ) filtered_tool_panel_section_keys.append( tool_panel_section_key ) if repositories_for_installation: decoded_kwd[ 'repo_info_dicts' ] = filtered_repo_info_dicts decoded_kwd[ 'tool_panel_section_keys' ] = filtered_tool_panel_section_keys - self.install_tool_shed_repositories( trans, repositories_for_installation, reinstalling=reinstalling, **decoded_kwd ) + self.install_tool_shed_repositories( trans, + repositories_for_installation, + reinstalling=reinstalling, + **decoded_kwd ) else: kwd[ 'message' ] = 'All selected tool shed repositories are already installed.' kwd[ 'status' ] = 'error' @@ -772,7 +818,10 @@ trans.install_model.ToolDependency.installation_status.UNINSTALLED ]: tool_dependencies_for_installation.append( tool_dependency ) if tool_dependencies_for_installation: - self.initiate_tool_dependency_installation( trans, tool_dependencies_for_installation ) + self.initiate_tool_dependency_installation( trans, + tool_dependencies_for_installation, + message=message, + status=status ) else: message = 'All selected tool dependencies are already installed.' status = 'error' @@ -780,14 +829,16 @@ message = 'Set the value of your <b>tool_dependency_dir</b> setting in your Galaxy config file (universe_wsgi.ini) ' message += ' and restart your Galaxy server to install tool dependencies.' status = 'error' - installed_tool_dependencies_select_field = suc.build_tool_dependencies_select_field( trans, - tool_shed_repository=tool_shed_repository, - name='inst_td_ids', - uninstalled_only=False ) - uninstalled_tool_dependencies_select_field = suc.build_tool_dependencies_select_field( trans, - tool_shed_repository=tool_shed_repository, - name='uninstalled_tool_dependency_ids', - uninstalled_only=True ) + installed_tool_dependencies_select_field = \ + suc.build_tool_dependencies_select_field( trans, + tool_shed_repository=tool_shed_repository, + name='inst_td_ids', + uninstalled_only=False ) + uninstalled_tool_dependencies_select_field = \ + suc.build_tool_dependencies_select_field( trans, + tool_shed_repository=tool_shed_repository, + name='uninstalled_tool_dependency_ids', + uninstalled_only=True ) return trans.fill_template( '/admin/tool_shed_repository/manage_repository_tool_dependencies.mako', repository=tool_shed_repository, installed_tool_dependencies_select_field=installed_tool_dependencies_select_field, @@ -836,7 +887,10 @@ trans.install_model.ToolDependency.installation_status.UNINSTALLED ]: tool_dependencies_for_installation.append( tool_dependency ) if tool_dependencies_for_installation: - self.initiate_tool_dependency_installation( trans, tool_dependencies_for_installation ) + self.initiate_tool_dependency_installation( trans, + tool_dependencies_for_installation, + message=message, + status=status ) else: kwd[ 'message' ] = 'All selected tool dependencies are already installed.' kwd[ 'status' ] = 'error' @@ -876,18 +930,21 @@ @web.require_admin def prepare_for_install( self, trans, **kwd ): if not suc.have_shed_tool_conf_for_install( trans ): - message = 'The <b>tool_config_file</b> setting in <b>universe_wsgi.ini</b> must include at least one shed tool configuration file name with a ' - message += '<b><toolbox></b> tag that includes a <b>tool_path</b> attribute value which is a directory relative to the Galaxy installation ' - message += 'directory in order to automatically install tools from a Galaxy tool shed (e.g., the file name <b>shed_tool_conf.xml</b> whose ' - message += '<b><toolbox></b> tag is <b><toolbox tool_path="../shed_tools"></b>).<p/>See the ' - message += '<a href="http://wiki.g2.bx.psu.edu/InstallingRepositoriesToGalaxy" target="_blank">Installation of Galaxy tool shed repository tools ' - message += 'into a local Galaxy instance</a> section of the Galaxy tool shed wiki for all of the details.' + message = 'The <b>tool_config_file</b> setting in <b>universe_wsgi.ini</b> must include at least one ' + message += 'shed tool configuration file name with a <b><toolbox></b> tag that includes a <b>tool_path</b> ' + message += 'attribute value which is a directory relative to the Galaxy installation directory in order ' + message += 'to automatically install tools from a Galaxy Tool Shed (e.g., the file name <b>shed_tool_conf.xml</b> ' + message += 'whose <b><toolbox></b> tag is <b><toolbox tool_path="../shed_tools"></b>).<p/>See the ' + message += '<a href="http://wiki.g2.bx.psu.edu/InstallingRepositoriesToGalaxy" target="_blank">Installation ' + message += 'of Galaxy Tool Shed repository tools into a local Galaxy instance</a> section of the Galaxy Tool ' + message += 'Shed wiki for all of the details.' return trans.show_error_message( message ) message = kwd.get( 'message', '' ) status = kwd.get( 'status', 'done' ) shed_tool_conf = kwd.get( 'shed_tool_conf', None ) tool_shed_url = kwd.get( 'tool_shed_url', None ) - # Handle repository dependencies, which do not include those that are required only for compiling a dependent repository's tool dependencies. + # Handle repository dependencies, which do not include those that are required only for compiling a dependent + # repository's tool dependencies. has_repository_dependencies = util.string_as_bool( kwd.get( 'has_repository_dependencies', False ) ) install_repository_dependencies = kwd.get( 'install_repository_dependencies', '' ) # Every repository will be installed into the same tool panel section or all will be installed outside of any sections. @@ -900,13 +957,36 @@ includes_tools_for_display_in_tool_panel = util.string_as_bool( kwd.get( 'includes_tools_for_display_in_tool_panel', False ) ) includes_tool_dependencies = util.string_as_bool( kwd.get( 'includes_tool_dependencies', False ) ) install_tool_dependencies = kwd.get( 'install_tool_dependencies', '' ) + # In addition to installing new repositories, this method is called when updating an installed repository + # to a new changeset_revision where the update includes newly defined repository dependencies. + updating = util.asbool( kwd.get( 'updating', False ) ) + updating_repository_id = kwd.get( 'updating_repository_id', None ) + updating_to_changeset_revision = kwd.get( 'updating_to_changeset_revision', None ) + updating_to_ctx_rev = kwd.get( 'updating_to_ctx_rev', None ) + encoded_updated_metadata = kwd.get( 'encoded_updated_metadata', None ) encoded_repo_info_dicts = kwd.get( 'encoded_repo_info_dicts', '' ) if encoded_repo_info_dicts: encoded_repo_info_dicts = encoded_repo_info_dicts.split( encoding_util.encoding_sep ) if not encoded_repo_info_dicts: - # The request originated in the tool shed via a tool search. + # The request originated in the tool shed via a tool search or from this controller's + # update_to_changeset_revision() method. repository_ids = kwd.get( 'repository_ids', None ) - changeset_revisions = kwd.get( 'changeset_revisions', None ) + if updating: + # We have updated an installed repository where the updates included newly defined repository + # and possibly tool dependencies. We will have arrived here only if the updates include newly + # defined repository dependencies. We're preparing to allow the user to elect to install these + # dependencies. At this point, the repository has been updated to the latest changeset revision, + # but the received repository id is from the Galaxy side (the caller is this controller's + # update_to_changeset_revision() method. We need to get the id of the same repository from the + # Tool Shed side. + repository = suc.get_tool_shed_repository_by_id( trans, updating_repository_id ) + url = suc.url_join( tool_shed_url, + 'repository/get_repository_id?name=%s&owner=%s' % \ + ( str( repository.name ), str( repository.owner ) ) ) + repository_ids = common_util.tool_shed_get( trans.app, tool_shed_url, url ) + changeset_revisions = updating_to_changeset_revision + else: + changeset_revisions = kwd.get( 'changeset_revisions', None ) # Get the information necessary to install each repository. url = suc.url_join( tool_shed_url, 'repository/get_repository_information?repository_ids=%s&changeset_revisions=%s' % \ @@ -918,7 +998,8 @@ if not includes_tools: includes_tools = util.string_as_bool( decoded_repo_info_dict.get( 'includes_tools', False ) ) if not includes_tools_for_display_in_tool_panel: - includes_tools_for_display_in_tool_panel = util.string_as_bool( decoded_repo_info_dict.get( 'includes_tools_for_display_in_tool_panel', False ) ) + includes_tools_for_display_in_tool_panel = \ + util.string_as_bool( decoded_repo_info_dict.get( 'includes_tools_for_display_in_tool_panel', False ) ) if not has_repository_dependencies: has_repository_dependencies = util.string_as_bool( repo_information_dict.get( 'has_repository_dependencies', False ) ) if not includes_tool_dependencies: @@ -927,6 +1008,19 @@ repo_info_dicts = [ encoding_util.tool_shed_decode( encoded_repo_info_dict ) for encoded_repo_info_dict in encoded_repo_info_dicts ] if ( not includes_tools_for_display_in_tool_panel and kwd.get( 'select_shed_tool_panel_config_button', False ) ) or \ ( includes_tools_for_display_in_tool_panel and kwd.get( 'select_tool_panel_section_button', False ) ): + if updating: + encoded_updated_metadata_dict = kwd.get( 'encoded_updated_metadata_dict', None ) + updated_changeset_revision = kwd.get( 'updated_changeset_revision', None ) + updated_ctx_rev = kwd.get( 'updated_ctx_rev', None ) + repository = suc.get_tool_shed_repository_by_id( trans, updating_repository_id ) + decoded_updated_metadata = encoding_util.tool_shed_decode( encoded_updated_metadata ) + # Now that the user has decided whether they will handle dependencies, we can update + # the repository to the latest revision. + repository = repository_util.update_repository_record( trans, + repository=repository, + updated_metadata_dict=decoded_updated_metadata, + updated_changeset_revision=updating_to_changeset_revision, + updated_ctx_rev=updating_to_ctx_rev ) install_repository_dependencies = CheckboxField.is_checked( install_repository_dependencies ) if includes_tool_dependencies: install_tool_dependencies = CheckboxField.is_checked( install_tool_dependencies ) @@ -959,7 +1053,8 @@ tool_panel_section_keys=tool_panel_section_keys, tool_path=tool_path, tool_shed_url=tool_shed_url ) - encoded_kwd, query, tool_shed_repositories, encoded_repository_ids = repository_util.initiate_repository_installation( trans, installation_dict ) + encoded_kwd, query, tool_shed_repositories, encoded_repository_ids = \ + repository_util.initiate_repository_installation( trans, installation_dict ) return trans.fill_template( 'admin/tool_shed_repository/initiate_repository_installation.mako', encoded_kwd=encoded_kwd, query=query, @@ -976,12 +1071,14 @@ tool_path = suc.get_tool_path_by_shed_tool_conf_filename( trans, shed_tool_conf ) tool_panel_section_select_field = tool_util.build_tool_panel_section_select_field( trans ) if len( repo_info_dicts ) == 1: - # If we're installing a single repository, see if it contains a readme or dependencies that we can display. + # If we're installing or updating a single repository, see if it contains a readme or + # dependencies that we can display. repo_info_dict = repo_info_dicts[ 0 ] dependencies_for_repository_dict = common_install_util.get_dependencies_for_repository( trans, tool_shed_url, repo_info_dict, - includes_tool_dependencies ) + includes_tool_dependencies, + updating=updating ) changeset_revision = dependencies_for_repository_dict.get( 'changeset_revision', None ) if not has_repository_dependencies: has_repository_dependencies = dependencies_for_repository_dict.get( 'has_repository_dependencies', False ) @@ -990,7 +1087,8 @@ if not includes_tools: includes_tools = dependencies_for_repository_dict.get( 'includes_tools', False ) if not includes_tools_for_display_in_tool_panel: - includes_tools_for_display_in_tool_panel = dependencies_for_repository_dict.get( 'includes_tools_for_display_in_tool_panel', False ) + includes_tools_for_display_in_tool_panel = \ + dependencies_for_repository_dict.get( 'includes_tools_for_display_in_tool_panel', False ) installed_repository_dependencies = dependencies_for_repository_dict.get( 'installed_repository_dependencies', None ) installed_tool_dependencies = dependencies_for_repository_dict.get( 'installed_tool_dependencies', None ) missing_repository_dependencies = dependencies_for_repository_dict.get( 'missing_repository_dependencies', None ) @@ -998,19 +1096,25 @@ name = dependencies_for_repository_dict.get( 'name', None ) repository_owner = dependencies_for_repository_dict.get( 'repository_owner', None ) readme_files_dict = readme_util.get_readme_files_dict_for_display( trans, tool_shed_url, repo_info_dict ) - # We're handling 1 of 2 scenarios here: (1) we're installing a tool shed repository for the first time, so we've retrieved the list of installed - # and missing repository dependencies from the database (2) we're handling the scenario where an error occurred during the installation process, - # so we have a tool_shed_repository record in the database with associated repository dependency records. Since we have the repository - # dependencies in either case, we'll merge the list of missing repository dependencies into the list of installed repository dependencies since - # each displayed repository dependency will display a status, whether installed or missing. - containers_dict = repository_util.populate_containers_dict_for_new_install( trans=trans, - tool_shed_url=tool_shed_url, - tool_path=tool_path, - readme_files_dict=readme_files_dict, - installed_repository_dependencies=installed_repository_dependencies, - missing_repository_dependencies=missing_repository_dependencies, - installed_tool_dependencies=installed_tool_dependencies, - missing_tool_dependencies=missing_tool_dependencies ) + # We're handling 1 of 3 scenarios here: (1) we're installing a tool shed repository for the first time, so we've + # retrieved the list of installed and missing repository dependencies from the database (2) we're handling the + # scenario where an error occurred during the installation process, so we have a tool_shed_repository record in + # the database with associated repository dependency records. Since we have the repository dependencies in both + # of the above 2 cases, we'll merge the list of missing repository dependencies into the list of installed + # repository dependencies since each displayed repository dependency will display a status, whether installed or + # missing. The 3rd scenario is where we're updating an installed repository and the updates include newly + # defined repository (and possibly tool) dependencies. In this case, merging will result in newly defined + # dependencies to be lost. We pass the updating parameter to make sure merging occurs only when appropriate. + containers_dict = \ + repository_util.populate_containers_dict_for_new_install( trans=trans, + tool_shed_url=tool_shed_url, + tool_path=tool_path, + readme_files_dict=readme_files_dict, + installed_repository_dependencies=installed_repository_dependencies, + missing_repository_dependencies=missing_repository_dependencies, + installed_tool_dependencies=installed_tool_dependencies, + missing_tool_dependencies=missing_tool_dependencies, + updating=updating ) else: # We're installing a list of repositories, each of which may have tool dependencies or repository dependencies. containers_dicts = [] @@ -1018,7 +1122,8 @@ dependencies_for_repository_dict = common_install_util.get_dependencies_for_repository( trans, tool_shed_url, repo_info_dict, - includes_tool_dependencies ) + includes_tool_dependencies, + updating=updating ) changeset_revision = dependencies_for_repository_dict.get( 'changeset_revision', None ) if not has_repository_dependencies: has_repository_dependencies = dependencies_for_repository_dict.get( 'has_repository_dependencies', False ) @@ -1027,34 +1132,39 @@ if not includes_tools: includes_tools = dependencies_for_repository_dict.get( 'includes_tools', False ) if not includes_tools_for_display_in_tool_panel: - includes_tools_for_display_in_tool_panel = dependencies_for_repository_dict.get( 'includes_tools_for_display_in_tool_panel', False ) + includes_tools_for_display_in_tool_panel = \ + dependencies_for_repository_dict.get( 'includes_tools_for_display_in_tool_panel', False ) installed_repository_dependencies = dependencies_for_repository_dict.get( 'installed_repository_dependencies', None ) installed_tool_dependencies = dependencies_for_repository_dict.get( 'installed_tool_dependencies', None ) missing_repository_dependencies = dependencies_for_repository_dict.get( 'missing_repository_dependencies', None ) missing_tool_dependencies = dependencies_for_repository_dict.get( 'missing_tool_dependencies', None ) name = dependencies_for_repository_dict.get( 'name', None ) repository_owner = dependencies_for_repository_dict.get( 'repository_owner', None ) - containers_dict = repository_util.populate_containers_dict_for_new_install( trans=trans, - tool_shed_url=tool_shed_url, - tool_path=tool_path, - readme_files_dict=None, - installed_repository_dependencies=installed_repository_dependencies, - missing_repository_dependencies=missing_repository_dependencies, - installed_tool_dependencies=installed_tool_dependencies, - missing_tool_dependencies=missing_tool_dependencies ) + containers_dict = \ + repository_util.populate_containers_dict_for_new_install( trans=trans, + tool_shed_url=tool_shed_url, + tool_path=tool_path, + readme_files_dict=None, + installed_repository_dependencies=installed_repository_dependencies, + missing_repository_dependencies=missing_repository_dependencies, + installed_tool_dependencies=installed_tool_dependencies, + missing_tool_dependencies=missing_tool_dependencies, + updating=updating ) containers_dicts.append( containers_dict ) # Merge all containers into a single container. containers_dict = repository_util.merge_containers_dicts_for_new_install( containers_dicts ) # Handle tool dependencies check box. if trans.app.config.tool_dependency_dir is None: if includes_tool_dependencies: - message = "Tool dependencies defined in this repository can be automatically installed if you set the value of your <b>tool_dependency_dir</b> " - message += "setting in your Galaxy config file (universe_wsgi.ini) and restart your Galaxy server before installing the repository." + message = "Tool dependencies defined in this repository can be automatically installed if you set " + message += "the value of your <b>tool_dependency_dir</b> setting in your Galaxy config file " + message += "(universe_wsgi.ini) and restart your Galaxy server before installing the repository." status = "warning" install_tool_dependencies_check_box_checked = False else: install_tool_dependencies_check_box_checked = True - install_tool_dependencies_check_box = CheckboxField( 'install_tool_dependencies', checked=install_tool_dependencies_check_box_checked ) + install_tool_dependencies_check_box = CheckboxField( 'install_tool_dependencies', + checked=install_tool_dependencies_check_box_checked ) # Handle repository dependencies check box. install_repository_dependencies_check_box = CheckboxField( 'install_repository_dependencies', checked=True ) encoded_repo_info_dicts = encoding_util.encoding_sep.join( encoded_repo_info_dicts ) @@ -1062,6 +1172,11 @@ if includes_tools_for_display_in_tool_panel: return trans.fill_template( '/admin/tool_shed_repository/select_tool_panel_section.mako', encoded_repo_info_dicts=encoded_repo_info_dicts, + updating=updating, + updating_repository_id=updating_repository_id, + updating_to_ctx_rev=updating_to_ctx_rev, + updating_to_changeset_revision=updating_to_changeset_revision, + encoded_updated_metadata=encoded_updated_metadata, includes_tools=includes_tools, includes_tools_for_display_in_tool_panel=includes_tools_for_display_in_tool_panel, includes_tool_dependencies=includes_tool_dependencies, @@ -1077,10 +1192,16 @@ message=message, status=status ) else: - # If installing repositories that includes no tools and has no repository dependencies, display a page allowing the Galaxy administrator to - # select a shed-related tool panel configuration file whose tool_path setting will be the location the repositories will be installed. + # If installing repositories that includes no tools and has no repository dependencies, display a page + # allowing the Galaxy administrator to select a shed-related tool panel configuration file whose tool_path + # setting will be the location the repositories will be installed. return trans.fill_template( '/admin/tool_shed_repository/select_shed_tool_panel_config.mako', encoded_repo_info_dicts=encoded_repo_info_dicts, + updating=updating, + updating_repository_id=updating_repository_id, + updating_to_ctx_rev=updating_to_ctx_rev, + updating_to_changeset_revision=updating_to_changeset_revision, + encoded_updated_metadata=encoded_updated_metadata, includes_tools=includes_tools, includes_tools_for_display_in_tool_panel=includes_tools_for_display_in_tool_panel, includes_tool_dependencies=includes_tool_dependencies, @@ -1400,7 +1521,8 @@ dependencies_for_repository_dict = common_install_util.get_dependencies_for_repository( trans, tool_shed_url, repo_info_dict, - includes_tool_dependencies ) + includes_tool_dependencies, + updating=False ) changeset_revision = dependencies_for_repository_dict.get( 'changeset_revision', None ) has_repository_dependencies = dependencies_for_repository_dict.get( 'has_repository_dependencies', False ) includes_tool_dependencies = dependencies_for_repository_dict.get( 'includes_tool_dependencies', False ) @@ -1449,14 +1571,16 @@ original_section_name = '' tool_panel_section_select_field = None shed_tool_conf_select_field = tool_util.build_shed_tool_conf_select_field( trans ) - containers_dict = repository_util.populate_containers_dict_for_new_install( trans=trans, - tool_shed_url=tool_shed_url, - tool_path=tool_path, - readme_files_dict=readme_files_dict, - installed_repository_dependencies=installed_repository_dependencies, - missing_repository_dependencies=missing_repository_dependencies, - installed_tool_dependencies=installed_tool_dependencies, - missing_tool_dependencies=missing_tool_dependencies ) + containers_dict = \ + repository_util.populate_containers_dict_for_new_install( trans=trans, + tool_shed_url=tool_shed_url, + tool_path=tool_path, + readme_files_dict=readme_files_dict, + installed_repository_dependencies=installed_repository_dependencies, + missing_repository_dependencies=missing_repository_dependencies, + installed_tool_dependencies=installed_tool_dependencies, + missing_tool_dependencies=missing_tool_dependencies, + updating=False ) # Since we're reinstalling we'll merge the list of missing repository dependencies into the list of installed repository dependencies since each displayed # repository dependency will display a status, whether installed or missing. containers_dict = repository_dependency_util.merge_missing_repository_dependencies_to_installed_container( containers_dict ) @@ -1574,8 +1698,8 @@ @web.require_admin def set_tool_versions( self, trans, **kwd ): """ - Get the tool_versions from the tool shed for each tool in the installed revision of a selected tool shed repository and update the - metadata for the repository's revision in the Galaxy database. + Get the tool_versions from the tool shed for each tool in the installed revision of a selected tool shed + repository and update the metadata for the repository's revision in the Galaxy database. """ repository = suc.get_installed_tool_shed_repository( trans, kwd[ 'id' ] ) tool_shed_url = suc.get_url_from_tool_shed( trans.app, repository.tool_shed ) @@ -1720,29 +1844,17 @@ resetting_all_metadata_on_repository=False, updating_installed_repository=True, persist=True ) - repository.metadata = metadata_dict - # Update the repository.changeset_revision column in the database. - repository.changeset_revision = latest_changeset_revision - repository.ctx_rev = latest_ctx_rev - # Update the repository.tool_shed_status column in the database. - tool_shed_status_dict = suc.get_tool_shed_status_for_installed_repository( trans.app, repository ) - if tool_shed_status_dict: - repository.tool_shed_status = tool_shed_status_dict - else: - repository.tool_shed_status = None - trans.install_model.context.add( repository ) - trans.install_model.context.flush() if 'tools' in metadata_dict: tool_panel_dict = metadata_dict.get( 'tool_panel_section', None ) if tool_panel_dict is None: tool_panel_dict = suc.generate_tool_panel_dict_from_shed_tool_conf_entries( trans.app, repository ) repository_tools_tups = suc.get_repository_tools_tups( trans.app, metadata_dict ) tool_util.add_to_tool_panel( app=trans.app, - repository_name=repository.name, + repository_name=str( repository.name ), repository_clone_url=repository_clone_url, - changeset_revision=repository.installed_changeset_revision, + changeset_revision=str( repository.installed_changeset_revision ), repository_tools_tups=repository_tools_tups, - owner=repository.owner, + owner=str( repository.owner ), shed_tool_conf=shed_tool_conf, tool_panel_dict=tool_panel_dict, new_install=False ) @@ -1755,19 +1867,40 @@ os.path.join( relative_install_dir, name ), repository, repository_tools_tups ) - # Create tool_dependency records if necessary. - if 'tool_dependencies' in metadata_dict: - tool_dependencies = tool_dependency_util.create_tool_dependency_objects( trans.app, - repository, - relative_install_dir, - set_status=False ) + if 'repository_dependencies' in metadata_dict or 'tool_dependencies' in metadata_dict: + if 'repository_dependencies' in metadata_dict: + # Updates received include newly defined repository dependencies, so allow the user + # the option of installting them. We cannot update the repository with the changes + # until that happens, so we have to send them along. + new_kwd = dict( tool_shed_url=tool_shed_url, + updating_repository_id=trans.security.encode_id( repository.id ), + updating_to_ctx_rev=latest_ctx_rev, + updating_to_changeset_revision=latest_changeset_revision, + encoded_updated_metadata=encoding_util.tool_shed_encode( metadata_dict ), + updating=True ) + return self.prepare_for_install( trans, **new_kwd ) + # Updates received did not include any newly defined repository dependencies but did include + # newly defined tool dependencies. + encoded_tool_dependencies_dict = encoding_util.tool_shed_encode( metadata_dict.get( 'tool_dependencies', {} ) ) + encoded_relative_install_dir = encoding_util.tool_shed_encode( relative_install_dir ) + new_kwd = dict( updating_repository_id=trans.security.encode_id( repository.id ), + updating_to_ctx_rev=latest_ctx_rev, + updating_to_changeset_revision=latest_changeset_revision, + encoded_updated_metadata=encoding_util.tool_shed_encode( metadata_dict ), + encoded_relative_install_dir=encoded_relative_install_dir, + encoded_tool_dependencies_dict=encoded_tool_dependencies_dict, + message=message, + status = status ) + return self.install_tool_dependencies_with_update( trans, **new_kwd ) + # Updates received did not include any newly defined repository dependencies or newly defined + # tool dependencies. + repository = repository_util.update_repository_record( trans, + repository=repository, + updated_metadata_dict=metadata_dict, + updated_changeset_revision=latest_changeset_revision, + updated_ctx_rev=latest_ctx_rev ) message = "The installed repository named '%s' has been updated to change set revision '%s'. " % \ ( name, latest_changeset_revision ) - # See if any tool dependencies can be installed. - shed_tool_conf, tool_path, relative_install_dir = \ - suc.get_tool_panel_config_tool_path_install_dir( trans.app, repository ) - if repository.missing_tool_dependencies: - message += "Click the name of one of the missing tool dependencies listed below to install tool dependencies." else: message = "The directory containing the installed repository named '%s' cannot be found. " % name status = 'error' diff -r 09985439d17f9bef026554938b05d0d6eedd06cb -r 90b4baa5c2b18d49bcea5a0807f0df8924abebb1 lib/galaxy/webapps/tool_shed/controllers/repository.py --- a/lib/galaxy/webapps/tool_shed/controllers/repository.py +++ b/lib/galaxy/webapps/tool_shed/controllers/repository.py @@ -866,11 +866,13 @@ no_update = 'false' elif galaxy_url: # Start building up the url to redirect back to the calling Galaxy instance. + params = '?tool_shed_url=%s&name=%s&owner=%s&changeset_revision=%s&latest_changeset_revision=' % \ + ( web.url_for( '/', qualified=True ), repository.name, repository.user.username, changeset_revision ) url = suc.url_join( galaxy_url, - 'admin_toolshed/update_to_changeset_revision?tool_shed_url=%s&name=%s&owner=%s&changeset_revision=%s&latest_changeset_revision=' % \ - ( web.url_for( '/', qualified=True ), repository.name, repository.user.username, changeset_revision ) ) + 'admin_toolshed/update_to_changeset_revision%s' % params ) else: - message = 'Unable to check for updates due to an invalid Galaxy URL: <b>%s</b>. You may need to enable third-party cookies in your browser. ' % galaxy_url + message = 'Unable to check for updates due to an invalid Galaxy URL: <b>%s</b>. ' % galaxy_url + message += 'You may need to enable third-party cookies in your browser. ' return trans.show_error_message( message ) if changeset_revision == repository.tip( trans.app ): # If changeset_revision is the repository tip, there are no additional updates. @@ -883,15 +885,17 @@ trans.security.encode_id( repository.id ), changeset_revision ) if repository_metadata: - # If changeset_revision is in the repository_metadata table for this repository, there are no additional updates. + # If changeset_revision is in the repository_metadata table for this repository, there are no + # additional updates. if from_update_manager: return no_update else: # Return the same value for changeset_revision and latest_changeset_revision. url += latest_changeset_revision else: - # The changeset_revision column in the repository_metadata table has been updated with a new changeset_revision value since the - # repository was installed. We need to find the changeset_revision to which we need to update. + # The changeset_revision column in the repository_metadata table has been updated with a new + # changeset_revision value since the repository was installed. We need to find the changeset_revision + # to which we need to update. update_to_changeset_hash = None for changeset in repo.changelog: changeset_hash = str( repo.changectx( changeset ) ) @@ -1680,6 +1684,16 @@ return encoding_util.tool_shed_encode( repository_dependencies ) return '' + @web.expose + def get_repository_id( self, trans, **kwd ): + """Given a repository name and owner, return the encoded repository id.""" + repository_name = kwd[ 'name' ] + repository_owner = kwd[ 'owner' ] + repository = suc.get_repository_by_name_and_owner( trans.app, repository_name, repository_owner ) + if repository: + return trans.security.encode_id( repository.id ) + return '' + @web.json def get_repository_information( self, trans, repository_ids, changeset_revisions, **kwd ): """ @@ -1824,7 +1838,9 @@ if not repository_metadata: # The received changeset_revision is no longer associated with metadata, so get the next changeset_revision in the repository # changelog that is associated with metadata. - changeset_revision = suc.get_next_downloadable_changeset_revision( repository, repo, after_changeset_revision=changeset_revision ) + changeset_revision = suc.get_next_downloadable_changeset_revision( repository, + repo, + after_changeset_revision=changeset_revision ) repository_metadata = suc.get_repository_metadata_by_changeset_revision( trans, repository_id, changeset_revision ) ctx = suc.get_changectx_for_changeset( repo, changeset_revision ) repo_info_dict = repository_util.create_repo_info_dict( trans=trans, @@ -2039,11 +2055,12 @@ galaxy_url = suc.handle_galaxy_url( trans, **kwd ) if galaxy_url: # Redirect back to local Galaxy to perform install. + params = '?tool_shed_url=%s&repository_ids=%s&changeset_revisions=%s' % \ + ( web.url_for( '/', qualified=True ), + ','.join( util.listify( repository_ids ) ), + ','.join( util.listify( changeset_revisions ) ) ) url = suc.url_join( galaxy_url, - 'admin_toolshed/prepare_for_install?tool_shed_url=%s&repository_ids=%s&changeset_revisions=%s' % \ - ( web.url_for( '/', qualified=True ), - ','.join( util.listify( repository_ids ) ), - ','.join( util.listify( changeset_revisions ) ) ) ) + 'admin_toolshed/prepare_for_install%s' % params ) return trans.response.send_redirect( url ) else: message = 'Repository installation is not possible due to an invalid Galaxy URL: <b>%s</b>. ' % galaxy_url @@ -2477,9 +2494,10 @@ @web.expose def next_installable_changeset_revision( self, trans, **kwd ): """ - Handle a request from a Galaxy instance where the changeset_revision defined for a repository in a dependency definition file is older - than the changeset_revision associated with the installed repository. This will occur with repository's of type tool_dependency_definition, - and this scenario will occur while repository dependency hierarchies are bing installed. + Handle a request from a Galaxy instance where the changeset_revision defined for a repository + in a dependency definition file is older than the changeset_revision associated with the installed + repository. This will occur with repository's of type tool_dependency_definition, and this scenario + will occur while repository dependency hierarchies are bing installed. """ name = kwd.get( 'name', None ) owner = kwd.get( 'owner', None ) @@ -2981,12 +2999,13 @@ @web.expose def updated_changeset_revisions( self, trans, **kwd ): """ - Handle a request from a local Galaxy instance to retrieve the list of changeset revisions to which an installed repository can be updated. This - method will return a string of comma-separated changeset revision hashes for all available updates to the received changeset revision. Among - other things , this method handles the scenario where an installed tool shed repository's tool_dependency definition file defines a changeset - revision for a complex repository dependency that is outdated. In other words, a defined changeset revision is older than the current changeset - revision for the required repository, making it impossible to discover the repository without knowledge of revisions to which it could have been - updated. + Handle a request from a local Galaxy instance to retrieve the list of changeset revisions to which an + installed repository can be updated. This method will return a string of comma-separated changeset revision + hashes for all available updates to the received changeset revision. Among other things , this method + handles the scenario where an installed tool shed repository's tool_dependency definition file defines a + changeset revision for a complex repository dependency that is outdated. In other words, a defined changeset + revision is older than the current changeset revision for the required repository, making it impossible to + discover the repository without knowledge of revisions to which it could have been updated. """ name = kwd.get( 'name', None ) owner = kwd.get( 'owner', None ) diff -r 09985439d17f9bef026554938b05d0d6eedd06cb -r 90b4baa5c2b18d49bcea5a0807f0df8924abebb1 lib/tool_shed/galaxy_install/repository_util.py --- a/lib/tool_shed/galaxy_install/repository_util.py +++ b/lib/tool_shed/galaxy_install/repository_util.py @@ -37,15 +37,19 @@ Galaxy instance. The dictionary will also contain the recursive list of repository dependencies defined for the repository, as well as the defined tool dependencies. - This method is called from Galaxy under three scenarios: + This method is called from Galaxy under four scenarios: 1. During the tool shed repository installation process via the tool shed's get_repository_information() - method. In this case both the received repository and repository_metadata will be objects., but + method. In this case both the received repository and repository_metadata will be objects, but tool_dependencies and repository_dependencies will be None. - 2. When a tool shed repository that was uninstalled from a Galaxy instance is being reinstalled with no + 2. When getting updates for an install repository where the updates include newly defined repository + dependency definitions. This scenario is similar to 1. above. The tool shed's get_repository_information() + method is the caller, and both the received repository and repository_metadata will be objects, but + tool_dependencies and repository_dependencies will be None. + 3. When a tool shed repository that was uninstalled from a Galaxy instance is being reinstalled with no updates available. In this case, both repository and repository_metadata will be None, but tool_dependencies and repository_dependencies will be objects previously retrieved from the tool shed if the repository includes definitions for them. - 3. When a tool shed repository that was uninstalled from a Galaxy instance is being reinstalled with updates + 4. When a tool shed repository that was uninstalled from a Galaxy instance is being reinstalled with updates available. In this case, this method is reached via the tool shed's get_updated_repository_information() method, and both repository and repository_metadata will be objects but tool_dependencies and repository_dependencies will be None. @@ -184,16 +188,22 @@ repo_dir = repository.repo_path( trans.app ) repo = hg.repository( suc.get_configured_ui(), repo_dir ) repository_clone_url = suc.generate_clone_url_for_repository_in_tool_shed( trans, repository ) - repository_metadata = suc.get_repository_metadata_by_changeset_revision( trans, repository_id, changeset_revision ) + repository_metadata = suc.get_repository_metadata_by_changeset_revision( trans, + repository_id, + changeset_revision ) if not repository_metadata: - # The received changeset_revision is no longer installable, so get the next changeset_revision in the repository's changelog. - # This generally occurs only with repositories of type tool_dependency_definition. - next_downloadable_changeset_revision = suc.get_next_downloadable_changeset_revision( repository, repo, changeset_revision ) + # The received changeset_revision is no longer installable, so get the next changeset_revision + # in the repository's changelog. This generally occurs only with repositories of type + # tool_dependency_definition. + next_downloadable_changeset_revision = \ + suc.get_next_downloadable_changeset_revision( repository,repo, changeset_revision ) if next_downloadable_changeset_revision: - repository_metadata = suc.get_repository_metadata_by_changeset_revision( trans, repository_id, next_downloadable_changeset_revision ) + repository_metadata = \ + suc.get_repository_metadata_by_changeset_revision( trans, repository_id, next_downloadable_changeset_revision ) if repository_metadata: - # For now, we'll always assume that we'll get repository_metadata, but if we discover our assumption is not valid we'll have to - # enhance the callers to handle repository_metadata values of None in the returned repo_info_dict. + # For now, we'll always assume that we'll get repository_metadata, but if we discover our assumption + # is not valid we'll have to enhance the callers to handle repository_metadata values of None in the + # returned repo_info_dict. metadata = repository_metadata.metadata if 'tools' in metadata: includes_tools = True @@ -743,9 +753,13 @@ ordered_tool_panel_section_keys.append( tool_panel_section_key ) return ordered_tsr_ids, ordered_repo_info_dicts, ordered_tool_panel_section_keys -def populate_containers_dict_for_new_install( trans, tool_shed_url, tool_path, readme_files_dict, installed_repository_dependencies, missing_repository_dependencies, - installed_tool_dependencies, missing_tool_dependencies ): - """Return the populated containers for a repository being installed for the first time.""" +def populate_containers_dict_for_new_install( trans, tool_shed_url, tool_path, readme_files_dict, installed_repository_dependencies, + missing_repository_dependencies, installed_tool_dependencies, missing_tool_dependencies, + updating=False ): + """ + Return the populated containers for a repository being installed for the first time or for an installed repository + that is being updated and the updates include newly defined repository (and possibly tool) dependencies. + """ installed_tool_dependencies, missing_tool_dependencies = \ tool_dependency_util.populate_tool_dependencies_dicts( trans=trans, tool_shed_url=tool_shed_url, @@ -753,27 +767,32 @@ repository_installed_tool_dependencies=installed_tool_dependencies, repository_missing_tool_dependencies=missing_tool_dependencies, required_repo_info_dicts=None ) - # Since we are installing a new repository, most of the repository contents are set to None since we don't yet know what they are. - containers_dict = container_util.build_repository_containers_for_galaxy( trans=trans, - repository=None, - datatypes=None, - invalid_tools=None, - missing_repository_dependencies=missing_repository_dependencies, - missing_tool_dependencies=missing_tool_dependencies, - readme_files_dict=readme_files_dict, - repository_dependencies=installed_repository_dependencies, - tool_dependencies=installed_tool_dependencies, - valid_tools=None, - workflows=None, - valid_data_managers=None, - invalid_data_managers=None, - data_managers_errors=None, - new_install=True, - reinstalling=False ) - # Merge the missing_repository_dependencies container contents to the installed_repository_dependencies container. - containers_dict = repository_dependency_util.merge_missing_repository_dependencies_to_installed_container( containers_dict ) - # Merge the missing_tool_dependencies container contents to the installed_tool_dependencies container. - containers_dict = tool_dependency_util.merge_missing_tool_dependencies_to_installed_container( containers_dict ) + # Most of the repository contents are set to None since we don't yet know what they are. + containers_dict = \ + container_util.build_repository_containers_for_galaxy( trans=trans, + repository=None, + datatypes=None, + invalid_tools=None, + missing_repository_dependencies=missing_repository_dependencies, + missing_tool_dependencies=missing_tool_dependencies, + readme_files_dict=readme_files_dict, + repository_dependencies=installed_repository_dependencies, + tool_dependencies=installed_tool_dependencies, + valid_tools=None, + workflows=None, + valid_data_managers=None, + invalid_data_managers=None, + data_managers_errors=None, + new_install=True, + reinstalling=False ) + if not updating: + # If we installing a new repository and not updaing an installed repository, we can merge + # the missing_repository_dependencies container contents to the installed_repository_dependencies + # container. When updating an installed repository, merging will result in losing newly defined + # dependencies included in the updates. + containers_dict = repository_dependency_util.merge_missing_repository_dependencies_to_installed_container( containers_dict ) + # Merge the missing_tool_dependencies container contents to the installed_tool_dependencies container. + containers_dict = tool_dependency_util.merge_missing_tool_dependencies_to_installed_container( containers_dict ) return containers_dict def pull_repository( repo, repository_clone_url, ctx_rev ): @@ -875,3 +894,23 @@ repository.uninstalled = uninstalled trans.install_model.context.add( repository ) trans.install_model.context.flush() + +def update_repository_record( trans, repository, updated_metadata_dict, updated_changeset_revision, updated_ctx_rev ): + """ + Update a tool_shed_repository database record with new information retrieved from the + Tool Shed. This happens when updating an installed repository to a new changeset revision. + """ + repository.metadata = updated_metadata_dict + # Update the repository.changeset_revision column in the database. + repository.changeset_revision = updated_changeset_revision + repository.ctx_rev = updated_ctx_rev + # Update the repository.tool_shed_status column in the database. + tool_shed_status_dict = suc.get_tool_shed_status_for_installed_repository( trans.app, repository ) + if tool_shed_status_dict: + repository.tool_shed_status = tool_shed_status_dict + else: + repository.tool_shed_status = None + trans.install_model.context.add( repository ) + trans.install_model.context.flush() + trans.install_model.context.refresh( repository ) + return repository diff -r 09985439d17f9bef026554938b05d0d6eedd06cb -r 90b4baa5c2b18d49bcea5a0807f0df8924abebb1 lib/tool_shed/galaxy_install/tool_dependencies/install_util.py --- a/lib/tool_shed/galaxy_install/tool_dependencies/install_util.py +++ b/lib/tool_shed/galaxy_install/tool_dependencies/install_util.py @@ -359,13 +359,14 @@ can_install_tool_dependency = True if can_install_tool_dependency: package_install_version = package_elem.get( 'version', '1.0' ) - tool_dependency = tool_dependency_util.create_or_update_tool_dependency( app=app, - tool_shed_repository=tool_shed_repository, - name=package_name, - version=package_version, - type='package', - status=app.install_model.ToolDependency.installation_status.INSTALLING, - set_status=True ) + tool_dependency = \ + tool_dependency_util.create_or_update_tool_dependency( app=app, + tool_shed_repository=tool_shed_repository, + name=package_name, + version=package_version, + type='package', + status=app.install_model.ToolDependency.installation_status.INSTALLING, + set_status=True ) # Get the information about the current platform in case the tool dependency definition includes tag sets # for installing compiled binaries. platform_info_dict = tool_dependency_util.get_platform_info_dict() @@ -879,25 +880,28 @@ # Tool dependencies of type "set_environmnet" always have the version attribute set to None. attr_tup = ( env_var_name, None, 'set_environment' ) if attr_tup in attr_tups_of_dependencies_for_install: - install_dir = tool_dependency_util.get_tool_dependency_install_dir( app=app, - repository_name=tool_shed_repository.name, - repository_owner=tool_shed_repository.owner, - repository_changeset_revision=tool_shed_repository.installed_changeset_revision, - tool_dependency_type='set_environment', - tool_dependency_name=env_var_name, - tool_dependency_version=None ) + install_dir = \ + tool_dependency_util.get_tool_dependency_install_dir( app=app, + repository_name=tool_shed_repository.name, + repository_owner=tool_shed_repository.owner, + repository_changeset_revision=tool_shed_repository.installed_changeset_revision, + tool_dependency_type='set_environment', + tool_dependency_name=env_var_name, + tool_dependency_version=None ) tool_shed_repository_install_dir = get_tool_shed_repository_install_dir( app, tool_shed_repository ) env_var_dict = td_common_util.create_env_var_dict( env_var_elem, tool_shed_repository_install_dir=tool_shed_repository_install_dir ) if env_var_dict: if not os.path.exists( install_dir ): os.makedirs( install_dir ) - tool_dependency = tool_dependency_util.create_or_update_tool_dependency( app=app, - tool_shed_repository=tool_shed_repository, - name=env_var_name, - version=None, - type='set_environment', - status=app.install_model.ToolDependency.installation_status.INSTALLING, - set_status=True ) + status = app.install_model.ToolDependency.installation_status.INSTALLING + tool_dependency = \ + tool_dependency_util.create_or_update_tool_dependency( app=app, + tool_shed_repository=tool_shed_repository, + name=env_var_name, + version=None, + type='set_environment', + status=status, + set_status=True ) if env_var_version == '1.0': # Create this tool dependency's env.sh file. env_file_builder = fabric_util.EnvFileBuilder( install_dir ) @@ -906,28 +910,34 @@ error_message = 'Error creating env.sh file for tool dependency %s, return_code: %s' % \ ( str( tool_dependency.name ), str( return_code ) ) log.debug( error_message ) - tool_dependency = tool_dependency_util.set_tool_dependency_attributes( app, - tool_dependency=tool_dependency, - status=app.install_model.ToolDependency.installation_status.ERROR, - error_message=error_message, - remove_from_disk=False ) + status = app.install_model.ToolDependency.installation_status.ERROR + tool_dependency = \ + tool_dependency_util.set_tool_dependency_attributes( app, + tool_dependency=tool_dependency, + status=status, + error_message=error_message, + remove_from_disk=False ) else: if tool_dependency.status not in [ app.install_model.ToolDependency.installation_status.ERROR, app.install_model.ToolDependency.installation_status.INSTALLED ]: - tool_dependency = tool_dependency_util.set_tool_dependency_attributes( app, - tool_dependency=tool_dependency, - status=app.install_model.ToolDependency.installation_status.INSTALLED, - error_message=None, - remove_from_disk=False ) + status = app.install_model.ToolDependency.installation_status.INSTALLED + tool_dependency = \ + tool_dependency_util.set_tool_dependency_attributes( app, + tool_dependency=tool_dependency, + status=status, + error_message=None, + remove_from_disk=False ) log.debug( 'Environment variable %s set in %s for tool dependency %s.' % \ ( str( env_var_name ), str( install_dir ), str( tool_dependency.name ) ) ) else: error_message = 'Only set_environment version 1.0 is currently supported (i.e., change your tag to be <set_environment version="1.0">).' - tool_dependency = tool_dependency_util.set_tool_dependency_attributes( app, - tool_dependency=tool_dependency, - status=app.install_model.ToolDependency.installation_status.ERROR, - error_message=error_message, - remove_from_disk=False ) + status = app.install_model.ToolDependency.installation_status.ERROR + tool_dependency = \ + tool_dependency_util.set_tool_dependency_attributes( app, + tool_dependency=tool_dependency, + status=status, + error_message=error_message, + remove_from_disk=False ) return tool_dependency def strip_path( fpath ): diff -r 09985439d17f9bef026554938b05d0d6eedd06cb -r 90b4baa5c2b18d49bcea5a0807f0df8924abebb1 lib/tool_shed/util/common_install_util.py --- a/lib/tool_shed/util/common_install_util.py +++ b/lib/tool_shed/util/common_install_util.py @@ -70,7 +70,7 @@ if display_path is not None: datatype_util.load_installed_display_applications( trans.app, installed_repository_dict, deactivate=False ) -def get_dependencies_for_repository( trans, tool_shed_url, repo_info_dict, includes_tool_dependencies ): +def get_dependencies_for_repository( trans, tool_shed_url, repo_info_dict, includes_tool_dependencies, updating=False ): """ Return dictionaries containing the sets of installed and missing tool dependencies and repository dependencies associated with the repository defined by the received repo_info_dict. @@ -90,7 +90,7 @@ # Inspect the tool_dependencies dictionary to separate the installed and missing tool dependencies. # We don't add to installed_td and missing_td here because at this point they are empty. installed_td, missing_td = \ - get_installed_and_missing_tool_dependencies_for_installing_repository( trans, tool_shed_url, tool_dependencies ) + get_installed_and_missing_tool_dependencies_for_repository( trans, tool_shed_url, tool_dependencies ) # In cases where a repository dependency is required only for compiling a dependent repository's # tool dependency, the value of repository_dependencies will be an empty dictionary here. if repository_dependencies: @@ -101,10 +101,11 @@ name, repository_owner, changeset_revision ) - if repository and repository.metadata: + if not updating and repository and repository.metadata: installed_rd, missing_rd = get_installed_and_missing_repository_dependencies( trans, repository ) else: - installed_rd, missing_rd = get_installed_and_missing_repository_dependencies_for_new_install( trans, repo_info_tuple ) + installed_rd, missing_rd = \ + get_installed_and_missing_repository_dependencies_for_new_or_updated_install( trans, repo_info_tuple ) # Discover all repository dependencies and retrieve information for installing them. all_repo_info_dict = get_required_repo_info_dicts( trans, tool_shed_url, util.listify( repo_info_dict ) ) has_repository_dependencies = all_repo_info_dict.get( 'has_repository_dependencies', False ) @@ -119,7 +120,8 @@ required_tool_dependencies = {} for rid in required_repo_info_dicts: for name, repo_info_tuple in rid.items(): - description, repository_clone_url, changeset_revision, ctx_rev, repository_owner, rid_repository_dependencies, rid_tool_dependencies = \ + description, repository_clone_url, changeset_revision, ctx_rev, \ + repository_owner, rid_repository_dependencies, rid_tool_dependencies = \ suc.get_repo_info_tuple_contents( repo_info_tuple ) if rid_tool_dependencies: for td_key, td_dict in rid_tool_dependencies.items(): @@ -128,7 +130,9 @@ if required_tool_dependencies: # Discover and categorize all tool dependencies defined for this repository's repository dependencies. required_installed_td, required_missing_td = \ - get_installed_and_missing_tool_dependencies_for_installing_repository( trans, tool_shed_url, required_tool_dependencies ) + get_installed_and_missing_tool_dependencies_for_repository( trans, + tool_shed_url, + required_tool_dependencies ) if required_installed_td: if not includes_tool_dependencies: includes_tool_dependencies = True @@ -168,11 +172,12 @@ def get_installed_and_missing_repository_dependencies( trans, repository ): """ - Return the installed and missing repository dependencies for a tool shed repository that has a record in the Galaxy database, but - may or may not be installed. In this case, the repository dependencies are associated with the repository in the database. Do not - include a repository dependency if it is required only to compile a tool dependency defined for the dependent repository since these - special kinds of repository dependencies are really a dependency of the dependent repository's contained tool dependency, and only if - that tool dependency requires compilation. + Return the installed and missing repository dependencies for a tool shed repository that has a record + in the Galaxy database, but may or may not be installed. In this case, the repository dependencies are + associated with the repository in the database. Do not include a repository dependency if it is required + only to compile a tool dependency defined for the dependent repository since these special kinds of repository + dependencies are really a dependency of the dependent repository's contained tool dependency, and only + if that tool dependency requires compilation. """ missing_repository_dependencies = {} installed_repository_dependencies = {} @@ -228,7 +233,7 @@ missing_repository_dependencies[ 'description' ] = description return installed_repository_dependencies, missing_repository_dependencies -def get_installed_and_missing_repository_dependencies_for_new_install( trans, repo_info_tuple ): +def get_installed_and_missing_repository_dependencies_for_new_or_updated_install( trans, repo_info_tuple ): """ Parse the received repository_dependencies dictionary that is associated with a repository being installed into Galaxy for the first time and attempt to determine repository dependencies that are @@ -264,13 +269,13 @@ tmp_repo_info_tuple ) if repository: new_rd_tup = [ tool_shed, - name, - owner, - changeset_revision, - prior_installation_required, - only_if_compiling_contained_td, - repository.id, - repository.status ] + name, + owner, + changeset_revision, + prior_installation_required, + only_if_compiling_contained_td, + repository.id, + repository.status ] if repository.status == trans.install_model.ToolShedRepository.installation_status.INSTALLED: if new_rd_tup not in installed_rd_tups: installed_rd_tups.append( new_rd_tup ) @@ -285,12 +290,12 @@ missing_rd_tups.append( new_rd_tup ) else: new_rd_tup = [ tool_shed, - name, - owner, - changeset_revision, - prior_installation_required, - only_if_compiling_contained_td, - None, + name, + owner, + changeset_revision, + prior_installation_required, + only_if_compiling_contained_td, + None, 'Never installed' ] if not util.asbool( only_if_compiling_contained_td ): # A repository dependency that is not installed will not be considered missing if it's value for @@ -307,11 +312,17 @@ missing_repository_dependencies[ 'description' ] = description return installed_repository_dependencies, missing_repository_dependencies -def get_installed_and_missing_tool_dependencies_for_installing_repository( trans, tool_shed_url, tool_dependencies_dict ): +def get_installed_and_missing_tool_dependencies_for_repository( trans, tool_shed_url, tool_dependencies_dict ): """ Return the lists of installed tool dependencies and missing tool dependencies for a set of repositories being installed into Galaxy. """ + # FIXME: This implementation breaks when updates to a repository contain dependencies that result in + # multiple entries for a specific tool dependency. A scenario where this can happen is where 2 repositories + # define the same dependency internally (not using the complex repository dependency definition to a separate + # package repository approach). If 2 repositories contain the same tool_dependencies.xml file, one dependency + # will be lost since the values in these returned dictionaries are not lists. All tool dependency dictionaries + # should have lists as values. These scenarios are probably extreme corner cases, but still should be handled. installed_tool_dependencies = {} missing_tool_dependencies = {} if tool_dependencies_dict: diff -r 09985439d17f9bef026554938b05d0d6eedd06cb -r 90b4baa5c2b18d49bcea5a0807f0df8924abebb1 lib/tool_shed/util/container_util.py --- a/lib/tool_shed/util/container_util.py +++ b/lib/tool_shed/util/container_util.py @@ -821,8 +821,8 @@ tool_dependencies = metadata[ 'tool_dependencies' ] if trans.webapp.name == 'tool_shed': if 'orphan_tool_dependencies' in metadata: - # The use of the orphan_tool_dependencies category in metadata has been deprecated, but we still need to check in case - # the metadata is out of date. + # The use of the orphan_tool_dependencies category in metadata has been deprecated, + # but we still need to check in case the metadata is out of date. orphan_tool_dependencies = metadata[ 'orphan_tool_dependencies' ] tool_dependencies.update( orphan_tool_dependencies ) # Tool dependencies can be categorized as orphans only if the repository contains tools. diff -r 09985439d17f9bef026554938b05d0d6eedd06cb -r 90b4baa5c2b18d49bcea5a0807f0df8924abebb1 lib/tool_shed/util/encoding_util.py --- a/lib/tool_shed/util/encoding_util.py +++ b/lib/tool_shed/util/encoding_util.py @@ -21,7 +21,6 @@ try: values = json.loads( value ) except Exception, e: - #log.debug( "Decoding json value from tool shed for value '%s' threw exception: %s" % ( str( value ), str( e ) ) ) pass if values is not None: try: @@ -34,7 +33,7 @@ return values def tool_shed_encode( val ): - if isinstance( val, dict ): + if isinstance( val, dict ) or isinstance( val, list ): value = json.dumps( val ) else: value = val diff -r 09985439d17f9bef026554938b05d0d6eedd06cb -r 90b4baa5c2b18d49bcea5a0807f0df8924abebb1 lib/tool_shed/util/metadata_util.py --- a/lib/tool_shed/util/metadata_util.py +++ b/lib/tool_shed/util/metadata_util.py @@ -662,9 +662,11 @@ # See if we have a repository dependencies defined. if name == suc.REPOSITORY_DEPENDENCY_DEFINITION_FILENAME: path_to_repository_dependencies_config = os.path.join( root, name ) - metadata_dict, error_message = generate_repository_dependency_metadata( app, - path_to_repository_dependencies_config, - metadata_dict ) + metadata_dict, error_message = \ + generate_repository_dependency_metadata( app, + path_to_repository_dependencies_config, + metadata_dict, + updating_installed_repository=updating_installed_repository ) if error_message: invalid_file_tups.append( ( name, error_message ) ) # See if we have one or more READ_ME files. @@ -795,7 +797,10 @@ # is True, the tool dependency definition will be set as invalid. This is currently the only case # where a tool dependency definition is considered invalid. repository_dependency_tup, repository_dependency_is_valid, error_message = \ - handle_repository_elem( app=app, repository_elem=sub_elem, only_if_compiling_contained_td=False ) + handle_repository_elem( app=app, + repository_elem=sub_elem, + only_if_compiling_contained_td=False, + updating_installed_repository=False ) elif sub_elem.tag == 'install': package_install_version = sub_elem.get( 'version', '1.0' ) if package_install_version == '1.0': @@ -829,7 +834,8 @@ repository_dependency_tup, repository_dependency_is_valid, error_message = \ handle_repository_elem( app=app, repository_elem=sub_action_elem, - only_if_compiling_contained_td=True ) + only_if_compiling_contained_td=True, + updating_installed_repository=False ) if requirements_dict: dependency_key = '%s/%s' % ( package_name, package_version ) if repository_dependency_is_valid: @@ -841,10 +847,10 @@ return valid_tool_dependencies_dict, invalid_tool_dependencies_dict, repository_dependency_tup, \ repository_dependency_is_valid, error_message -def generate_repository_dependency_metadata( app, repository_dependencies_config, metadata_dict ): +def generate_repository_dependency_metadata( app, repository_dependencies_config, metadata_dict, updating_installed_repository=False ): """ - Generate a repository dependencies dictionary based on valid information defined in the received repository_dependencies_config. This method - is called from the tool shed as well as from Galaxy. + Generate a repository dependencies dictionary based on valid information defined in the received + repository_dependencies_config. This method is called from the tool shed as well as from Galaxy. """ error_message = '' # Make sure we're looking at a valid repository_dependencies.xml file. @@ -861,7 +867,10 @@ valid_repository_dependency_tups = [] for repository_elem in root.findall( 'repository' ): repository_dependency_tup, repository_dependency_is_valid, err_msg = \ - handle_repository_elem( app, repository_elem, only_if_compiling_contained_td=False ) + handle_repository_elem( app, + repository_elem, + only_if_compiling_contained_td=False, + updating_installed_repository=updating_installed_repository ) if repository_dependency_is_valid: valid_repository_dependency_tups.append( repository_dependency_tup ) else: @@ -885,11 +894,12 @@ metadata_dict[ 'repository_dependencies' ] = valid_repository_dependencies_dict return metadata_dict, error_message -def generate_tool_dependency_metadata( app, repository, changeset_revision, repository_clone_url, tool_dependencies_config, metadata_dict, - original_repository_metadata=None ): +def generate_tool_dependency_metadata( app, repository, changeset_revision, repository_clone_url, tool_dependencies_config, + metadata_dict, original_repository_metadata=None ): """ - If the combination of name, version and type of each element is defined in the <requirement> tag for at least one tool in the repository, - then update the received metadata_dict with information from the parsed tool_dependencies_config. + If the combination of name, version and type of each element is defined in the <requirement> tag for + at least one tool in the repository, then update the received metadata_dict with information from the + parsed tool_dependencies_config. """ error_message = '' if original_repository_metadata: @@ -1171,7 +1181,7 @@ deleted_tool_dependency_names.append( original_dependency_val_dict[ 'name' ] ) return updated_tool_dependency_names, deleted_tool_dependency_names -def handle_repository_elem( app, repository_elem, only_if_compiling_contained_td=False ): +def handle_repository_elem( app, repository_elem, only_if_compiling_contained_td=False, updating_installed_repository=False ): """ Process the received repository_elem which is a <repository> tag either from a repository_dependencies.xml file or a tool_dependencies.xml file. If the former, we're generating repository dependencies metadata for @@ -1187,14 +1197,17 @@ changeset_revision = repository_elem.get( 'changeset_revision', None ) prior_installation_required = str( repository_elem.get( 'prior_installation_required', False ) ) if app.name == 'galaxy': - # We're installing a repository into Galaxy, so make sure its contained repository dependency definition - # is valid. - if toolshed is None or name is None or owner is None or changeset_revision is None: - # Raise an exception here instead of returning an error_message to keep the installation from - # proceeding. Reaching here implies a bug in the Tool Shed framework. - error_message = 'Installation halted because the following repository dependency definition is invalid:\n' - error_message += xml_util.xml_to_string( repository_elem, use_indent=True ) - raise Exception( error_message ) + if updating_installed_repository: + pass + else: + # We're installing a repository into Galaxy, so make sure its contained repository dependency definition + # is valid. + if toolshed is None or name is None or owner is None or changeset_revision is None: + # Raise an exception here instead of returning an error_message to keep the installation from + # proceeding. Reaching here implies a bug in the Tool Shed framework. + error_message = 'Installation halted because the following repository dependency definition is invalid:\n' + error_message += xml_util.xml_to_string( repository_elem, use_indent=True ) + raise Exception( error_message ) if not toolshed: # Default to the current tool shed. toolshed = str( url_for( '/', qualified=True ) ).rstrip( '/' ) @@ -1230,6 +1243,14 @@ updated_changeset_revision ) if repository: return repository_dependency_tup, is_valid, error_message + if updating_installed_repository: + # The repository dependency was included in an update to the installed repository, so it will + # not yet be installed. Return the tuple for later installation. + return repository_dependency_tup, is_valid, error_message + if updating_installed_repository: + # The repository dependency was included in an update to the installed repository, so it will not yet + # be installed. Return the tuple for later installation. + return repository_dependency_tup, is_valid, error_message # Don't generate an error message for missing repository dependencies that are required only if compiling the # dependent repository's tool dependency. if not only_if_compiling_contained_td: diff -r 09985439d17f9bef026554938b05d0d6eedd06cb -r 90b4baa5c2b18d49bcea5a0807f0df8924abebb1 lib/tool_shed/util/tool_dependency_util.py --- a/lib/tool_shed/util/tool_dependency_util.py +++ b/lib/tool_shed/util/tool_dependency_util.py @@ -90,7 +90,6 @@ if tree is None: return tool_dependency_objects root = tree.getroot() - fabric_version_checked = False for elem in root: tool_dependency_type = elem.tag if tool_dependency_type == 'package': @@ -574,8 +573,8 @@ sa_session.flush() return tool_dependency -def sync_database_with_file_system( app, tool_shed_repository, tool_dependency_name, tool_dependency_version, tool_dependency_install_dir, - tool_dependency_type='package' ): +def sync_database_with_file_system( app, tool_shed_repository, tool_dependency_name, tool_dependency_version, + tool_dependency_install_dir, tool_dependency_type='package' ): """ The installation directory defined by the received tool_dependency_install_dir exists, so check for the presence of fabric_util.INSTALLATION_LOG. If the files exists, we'll assume the tool dependency is installed, but not diff -r 09985439d17f9bef026554938b05d0d6eedd06cb -r 90b4baa5c2b18d49bcea5a0807f0df8924abebb1 templates/admin/tool_shed_repository/common.mako --- a/templates/admin/tool_shed_repository/common.mako +++ b/templates/admin/tool_shed_repository/common.mako @@ -90,6 +90,7 @@ return str( self.count ) repository_dependencies_root_folder = containers_dict.get( 'repository_dependencies', None ) + missing_repository_dependencies_root_folder = containers_dict.get( 'missing_repository_dependencies', None ) tool_dependencies_root_folder = containers_dict.get( 'tool_dependencies', None ) missing_tool_dependencies_root_folder = containers_dict.get( 'missing_tool_dependencies', None ) env_settings_heaader_row_displayed = False @@ -112,7 +113,7 @@ </p></div></div> - %if repository_dependencies_root_folder: + %if repository_dependencies_root_folder or missing_repository_dependencies_root_folder: %if repository_dependencies_check_box is not None: <div class="form-row"> %if export: @@ -131,14 +132,26 @@ </div><div style="clear: both"></div> %endif - <div class="form-row"> - <p/> - <% row_counter = RowCounter() %> - <table cellspacing="2" cellpadding="2" border="0" width="100%" class="tables container-table"> - ${render_folder( repository_dependencies_root_folder, 0, parent=None, row_counter=row_counter, is_root_folder=True )} - </table> - <div style="clear: both"></div> - </div> + %if repository_dependencies_root_folder: + <div class="form-row"> + <p/> + <% row_counter = RowCounter() %> + <table cellspacing="2" cellpadding="2" border="0" width="100%" class="tables container-table"> + ${render_folder( repository_dependencies_root_folder, 0, parent=None, row_counter=row_counter, is_root_folder=True )} + </table> + <div style="clear: both"></div> + </div> + %endif + %if missing_repository_dependencies_root_folder: + <div class="form-row"> + <p/> + <% row_counter = RowCounter() %> + <table cellspacing="2" cellpadding="2" border="0" width="100%" class="tables container-table"> + ${render_folder( missing_repository_dependencies_root_folder, 0, parent=None, row_counter=row_counter, is_root_folder=True )} + </table> + <div style="clear: both"></div> + </div> + %endif %endif %if tool_dependencies_root_folder or missing_tool_dependencies_root_folder: %if install_tool_dependencies_check_box is not None: diff -r 09985439d17f9bef026554938b05d0d6eedd06cb -r 90b4baa5c2b18d49bcea5a0807f0df8924abebb1 templates/admin/tool_shed_repository/install_tool_dependencies.mako --- a/templates/admin/tool_shed_repository/install_tool_dependencies.mako +++ /dev/null @@ -1,81 +0,0 @@ -<%inherit file="/base.mako"/> -<%namespace file="/message.mako" import="render_msg" /> - -<% import os %> - -%if message: - ${render_msg( message, status )} -%endif - -<div class="warningmessage"> - <p> - The tool dependencies listed below can be automatically installed with the repository. Installing them provides significant - benefits and Galaxy includes various features to manage them. - </p> - <p> - Each of these dependencies may require their own build requirements (e.g., CMake, g++, etc). Galaxy will not attempt to install - these build requirements, so if any are missing from your environment tool dependency installation may partially fail. The - repository and all of it's contents will be installed in any case. - </p> - <p> - If tool dependency installation fails in any way, you can install the missing build requirements and have Galaxy attempt to install - the tool dependencies again using the <b>Install tool dependencies</b> pop-up menu option on the <b>Manage repository</b> page. - </p> -</div> - -<div class="toolForm"> - <div class="toolFormBody"> - <form name="install_tool_dependenceies" id="install_tool_dependenceies" action="${h.url_for( controller='admin_toolshed', action='install_tool_dependencies' )}" method="post" > - <div class="form-row"> - <table class="grid"> - <tr><td colspan="4" bgcolor="#D8D8D8"><b>Tool dependencies</b></td></tr> - <tr> - <th>Name</th> - <th>Version</th> - <th>Type</th> - <th>Install directory</th> - </tr> - <% tool_shed_repository = None %> - %for tool_dependency in tool_dependencies: - <input type="hidden" name="tool_dependency_ids" value="${trans.security.encode_id( tool_dependency.id )}"/> - <% - readme_text = None - if tool_shed_repository is None: - tool_shed_repository = tool_dependency.tool_shed_repository - metadata = tool_shed_repository.metadata - tool_dependencies_dict = metadata[ 'tool_dependencies' ] - for key, requirements_dict in tool_dependencies_dict.items(): - key_items = key.split( '/' ) - key_name = key_items[ 0 ] - key_version = key_items[ 1 ] - if key_name == tool_dependency.name and key_version == tool_dependency.version: - readme_text = requirements_dict.get( 'readme', None ) - install_dir = os.path.join( trans.app.config.tool_dependency_dir, - tool_dependency.name, - tool_dependency.version, - tool_shed_repository.owner, - tool_shed_repository.name, - tool_shed_repository.installed_changeset_revision ) - %> - %if not os.path.exists( install_dir ): - <tr> - <td>${tool_dependency.name}</td> - <td>${tool_dependency.version}</td> - <td>${tool_dependency.type}</td> - <td>${install_dir}</td> - </tr> - %if readme_text: - <tr><td colspan="4" bgcolor="#FFFFCC">${tool_dependency.name} ${tool_dependency.version} requirements and installation information</td></tr> - <tr><td colspan="4"><pre>${readme_text}</pre></td></tr> - %endif - %endif - %endfor - </table> - <div style="clear: both"></div> - </div> - <div class="form-row"> - <input type="submit" name="install_tool_dependencies_button" value="Install"/> - </div> - </form> - </div> -</div> diff -r 09985439d17f9bef026554938b05d0d6eedd06cb -r 90b4baa5c2b18d49bcea5a0807f0df8924abebb1 templates/admin/tool_shed_repository/install_tool_dependencies_with_update.mako --- /dev/null +++ b/templates/admin/tool_shed_repository/install_tool_dependencies_with_update.mako @@ -0,0 +1,91 @@ +<%inherit file="/base.mako"/> +<%namespace file="/admin/tool_shed_repository/repository_actions_menu.mako" import="*" /> +<%namespace file="/message.mako" import="render_msg" /> + +<% + import os + from tool_shed.util.common_util import parse_repository_dependency_tuple +%> + +${render_galaxy_repository_actions( repository )} + +%if message: + ${render_msg( message, status )} +%endif + +<div class="warningmessage"> + <p> + The updates to the <b>${repository.name}</b> repository require the following packages. Click the <b>Install</b> button to install them. + Installing some packages may take a while, but you can continue to use Galaxy during installation. + </p> +</div> + +<div class="toolForm"> + <div class="toolFormBody"> + <form name="install_tool_dependencies_with_update" id="install_tool_dependencies_with_update" action="${h.url_for( controller='admin_toolshed', action='install_tool_dependencies_with_update' )}" method="post" > + <input type="hidden" name="updating_repository_id" value="${updating_repository_id}"/> + <input type="hidden" name="updating_to_ctx_rev" value="${updating_to_ctx_rev}"/> + <input type="hidden" name="updating_to_changeset_revision" value="${updating_to_changeset_revision}"/> + <input type="hidden" name="encoded_updated_metadata" value="${encoded_updated_metadata}"/> + <input type="hidden" name="encoded_relative_install_dir" value="${encoded_relative_install_dir}"/> + <input type="hidden" name="encoded_tool_dependencies_dict" value="${encoded_tool_dependencies_dict}"/> + %if tool_dependencies_dict: + %if install_tool_dependencies_check_box is not None: + <div class="form-row"> + <label>Handle tool dependencies?</label> + <% disabled = trans.app.config.tool_dependency_dir is None %> + ${install_tool_dependencies_check_box.get_html( disabled=disabled )} + <div class="toolParamHelp" style="clear: both;"> + %if disabled: + Set the tool_dependency_dir configuration value in your Galaxy config to automatically handle tool dependencies. + %else: + Un-check to skip automatic handling of these tool dependencies. + %endif + </div> + </div> + <div style="clear: both"></div> + %endif + <div class="form-row"> + <table class="grid"> + <tr><td colspan="4" bgcolor="#D8D8D8"><b>New tool dependencies included in update</b></td></tr> + <tr> + <th>Name</th> + <th>Version</th> + <th>Install directory</th> + </tr> + %for key, requirements_dict in tool_dependencies_dict.items(): + <% + readme_text = None + key_items = key.split( '/' ) + key_name = key_items[ 0 ] + key_version = key_items[ 1 ] + readme_text = requirements_dict.get( 'readme', None ) + install_dir = os.path.join( trans.app.config.tool_dependency_dir, + key_name, + key_version, + repository.owner, + repository.name, + repository.installed_changeset_revision ) + %> + %if not os.path.exists( install_dir ): + <tr> + <td>${key_name}</td> + <td>${key_version}</td> + <td>${install_dir}</td> + </tr> + %if readme_text: + <tr><td colspan="4" bgcolor="#FFFFCC">${key_name} ${key_version} requirements and installation information</td></tr> + <tr><td colspan="4"><pre>${readme_text}</pre></td></tr> + %endif + %endif + %endfor + </table> + <div style="clear: both"></div> + </div> + %endif + <div class="form-row"> + <input type="submit" name="install_tool_dependencies_with_update_button" value="Install"/> + </div> + </form> + </div> +</div> diff -r 09985439d17f9bef026554938b05d0d6eedd06cb -r 90b4baa5c2b18d49bcea5a0807f0df8924abebb1 templates/admin/tool_shed_repository/manage_repository_tool_dependencies.mako --- a/templates/admin/tool_shed_repository/manage_repository_tool_dependencies.mako +++ b/templates/admin/tool_shed_repository/manage_repository_tool_dependencies.mako @@ -30,8 +30,6 @@ <tr><th bgcolor="#D8D8D8">Name</th><th bgcolor="#D8D8D8">Version</th><th bgcolor="#D8D8D8">Type</th><th bgcolor="#D8D8D8">Status</th><th bgcolor="#D8D8D8">Error</th></tr> %for tool_dependency in repository.tool_dependencies: <% - # Tool dependencies cannot be uninstalled if they have a status of 'Installed'. Only the containing repository - # can be uninstalled (only if it has no dependent repositories) if a tool dependency has been successfully installed. if tool_dependency.error_message: error_message = tool_dependency.error_message else: @@ -41,8 +39,7 @@ trans.install_model.ToolDependency.installation_status.UNINSTALLED ]: can_install = True if not can_uninstall: - if tool_dependency.status not in [ trans.install_model.ToolDependency.installation_status.INSTALLED, - trans.install_model.ToolDependency.installation_status.NEVER_INSTALLED, + if tool_dependency.status not in [ trans.install_model.ToolDependency.installation_status.NEVER_INSTALLED, trans.install_model.ToolDependency.installation_status.UNINSTALLED ]: can_uninstall = True %> diff -r 09985439d17f9bef026554938b05d0d6eedd06cb -r 90b4baa5c2b18d49bcea5a0807f0df8924abebb1 templates/admin/tool_shed_repository/select_shed_tool_panel_config.mako --- a/templates/admin/tool_shed_repository/select_shed_tool_panel_config.mako +++ b/templates/admin/tool_shed_repository/select_shed_tool_panel_config.mako @@ -46,8 +46,8 @@ <div class="warningmessage"><p> - The core Galaxy development team does not maintain the contents of many Galaxy tool shed repositories. Some repository tools - may include code that produces malicious behavior, so be aware of what you are installing. + The Galaxy development team does not maintain the contents of many Galaxy Tool Shed repositories. Some + repository tools may include code that produces malicious behavior, so be aware of what you are installing. </p><p> If you discover a repository that causes problems after installation, contact <a href="http://wiki.g2.bx.psu.edu/Support" target="_blank">Galaxy support</a>, @@ -63,6 +63,11 @@ <form name="select_shed_tool_panel_config" id="select_shed_tool_panel_config" action="${h.url_for( controller='admin_toolshed', action='prepare_for_install' )}" method="post" ><div class="form-row"><input type="hidden" name="encoded_repo_info_dicts" value="${encoded_repo_info_dicts}" /> + <input type="hidden" name="updating" value="${updating}" /> + <input type="hidden" name="updating_repository_id" value="${updating_repository_id}" /> + <input type="hidden" name="updating_to_ctx_rev" value="${updating_to_ctx_rev}" /> + <input type="hidden" name="updating_to_changeset_revision" value="${updating_to_changeset_revision}" /> + <input type="hidden" name="encoded_updated_metadata" value="${encoded_updated_metadata}" /><input type="hidden" name="includes_tools" value="${includes_tools}" /><input type="hidden" name="includes_tool_dependencies" value="${includes_tool_dependencies}" /><input type="hidden" name="includes_tools_for_display_in_tool_panel" value="${includes_tools_for_display_in_tool_panel}" /> diff -r 09985439d17f9bef026554938b05d0d6eedd06cb -r 90b4baa5c2b18d49bcea5a0807f0df8924abebb1 templates/admin/tool_shed_repository/select_tool_panel_section.mako --- a/templates/admin/tool_shed_repository/select_tool_panel_section.mako +++ b/templates/admin/tool_shed_repository/select_tool_panel_section.mako @@ -46,8 +46,8 @@ <div class="warningmessage"><p> - The core Galaxy development team does not maintain the contents of many Galaxy tool shed repositories. Some repository tools - may include code that produces malicious behavior, so be aware of what you are installing. + The Galaxy development team does not maintain the contents of many Galaxy Tool Shed repositories. Some + repository tools may include code that produces malicious behavior, so be aware of what you are installing. </p><p> If you discover a repository that causes problems after installation, contact <a href="http://wiki.g2.bx.psu.edu/Support" target="_blank">Galaxy support</a>, @@ -67,6 +67,11 @@ <input type="hidden" name="includes_tools_for_display_in_tool_panel" value="${includes_tools_for_display_in_tool_panel}" /><input type="hidden" name="tool_shed_url" value="${tool_shed_url}" /><input type="hidden" name="encoded_repo_info_dicts" value="${encoded_repo_info_dicts}" /> + <input type="hidden" name="updating" value="${updating}" /> + <input type="hidden" name="updating_repository_id" value="${updating_repository_id}" /> + <input type="hidden" name="updating_to_ctx_rev" value="${updating_to_ctx_rev}" /> + <input type="hidden" name="updating_to_changeset_revision" value="${updating_to_changeset_revision}" /> + <input type="hidden" name="encoded_updated_metadata" value="${encoded_updated_metadata}" /></div><div style="clear: both"></div><% readme_files_dict = containers_dict.get( 'readme_files', None ) %> Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.