12 new commits in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/a3be81bd096a/ Changeset: a3be81bd096a User: jmchilton Date: 2015-02-27 20:58:02+00:00 Summary: Reduce use of bare excepts in install_manager.py. Affected #: 1 file diff -r 179c7c3fbb0db10765a97eb632bfc34c570ef0ff -r a3be81bd096a90d8a2528921d84e66d0c97479fb lib/tool_shed/galaxy_install/install_manager.py --- a/lib/tool_shed/galaxy_install/install_manager.py +++ b/lib/tool_shed/galaxy_install/install_manager.py @@ -710,19 +710,19 @@ installed_tool_shed_repositories = [] try: has_repository_dependencies = repository_revision_dict[ 'has_repository_dependencies' ] - except: + except KeyError: raise exceptions.InternalServerError( "Tool shed response missing required parameter 'has_repository_dependencies'." ) try: includes_tools = repository_revision_dict[ 'includes_tools' ] - except: + except KeyError: raise exceptions.InternalServerError( "Tool shed response missing required parameter 'includes_tools'." ) try: includes_tool_dependencies = repository_revision_dict[ 'includes_tool_dependencies' ] - except: + except KeyError: raise exceptions.InternalServerError( "Tool shed response missing required parameter 'includes_tool_dependencies'." ) try: includes_tools_for_display_in_tool_panel = repository_revision_dict[ 'includes_tools_for_display_in_tool_panel' ] - except: + except KeyError: raise exceptions.InternalServerError( "Tool shed response missing required parameter 'includes_tools_for_display_in_tool_panel'." ) # Get the information about the Galaxy components (e.g., tool pane section, tool config file, etc) that will contain the repository information. install_repository_dependencies = install_options.get( 'install_repository_dependencies', False ) https://bitbucket.org/galaxy/galaxy-central/commits/f1ea6093d943/ Changeset: f1ea6093d943 User: jmchilton Date: 2015-02-27 20:58:02+00:00 Summary: Eliminate unused using_api parameter in install_manager. Affected #: 2 files diff -r a3be81bd096a90d8a2528921d84e66d0c97479fb -r f1ea6093d9435b2c6015c50f8a1defecbc26f8ef lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py --- a/lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py +++ b/lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py @@ -1092,7 +1092,7 @@ tool_path=tool_path, tool_shed_url=tool_shed_url ) created_or_updated_tool_shed_repositories, tool_panel_section_keys, repo_info_dicts, filtered_repo_info_dicts = \ - install_repository_manager.handle_tool_shed_repositories( installation_dict, using_api=False ) + install_repository_manager.handle_tool_shed_repositories( installation_dict ) if created_or_updated_tool_shed_repositories: installation_dict = dict( created_or_updated_tool_shed_repositories=created_or_updated_tool_shed_repositories, filtered_repo_info_dicts=filtered_repo_info_dicts, diff -r a3be81bd096a90d8a2528921d84e66d0c97479fb -r f1ea6093d9435b2c6015c50f8a1defecbc26f8ef lib/tool_shed/galaxy_install/install_manager.py --- a/lib/tool_shed/galaxy_install/install_manager.py +++ b/lib/tool_shed/galaxy_install/install_manager.py @@ -623,7 +623,7 @@ # Load proprietary datatype display applications self.app.datatypes_registry.load_display_applications( installed_repository_dict=repository_dict ) - def handle_tool_shed_repositories( self, installation_dict, using_api=False ): + def handle_tool_shed_repositories( self, installation_dict ): # The following installation_dict entries are all required. install_repository_dependencies = installation_dict[ 'install_repository_dependencies' ] new_tool_panel_section_label = installation_dict[ 'new_tool_panel_section_label' ] @@ -759,7 +759,7 @@ tool_shed_url=tool_shed_url ) # Create the tool_shed_repository database records and gather additional information for repository installation. created_or_updated_tool_shed_repositories, tool_panel_section_keys, repo_info_dicts, filtered_repo_info_dicts = \ - self.handle_tool_shed_repositories( installation_dict, using_api=True ) + self.handle_tool_shed_repositories( installation_dict ) if created_or_updated_tool_shed_repositories: # Build the dictionary of information necessary for installing the repositories. installation_dict = dict( created_or_updated_tool_shed_repositories=created_or_updated_tool_shed_repositories, https://bitbucket.org/galaxy/galaxy-central/commits/f95ba7774644/ Changeset: f95ba7774644 User: jmchilton Date: 2015-02-27 20:58:02+00:00 Summary: Small refactoring of install_manager.py. Break tool dependency dir checking out into its own function to reduce complexity of __install_repositories. Affected #: 1 file diff -r f1ea6093d9435b2c6015c50f8a1defecbc26f8ef -r f95ba77746449eeff2593a06a457b57ebe803eca lib/tool_shed/galaxy_install/install_manager.py --- a/lib/tool_shed/galaxy_install/install_manager.py +++ b/lib/tool_shed/galaxy_install/install_manager.py @@ -728,11 +728,7 @@ install_repository_dependencies = install_options.get( 'install_repository_dependencies', False ) install_tool_dependencies = install_options.get( 'install_tool_dependencies', False ) if install_tool_dependencies: - if self.app.config.tool_dependency_dir is None: - no_tool_dependency_dir_message = "Tool dependencies can be automatically installed only if you set " - no_tool_dependency_dir_message += "the value of your 'tool_dependency_dir' setting in your Galaxy " - no_tool_dependency_dir_message += "configuration file (galaxy.ini) and restart your Galaxy server. " - raise exceptions.ConfigDoesNotAllowException( no_tool_dependency_dir_message ) + self.__assert_can_install_dependencies() new_tool_panel_section_label = install_options.get( 'new_tool_panel_section_label', '' ) shed_tool_conf = install_options.get( 'shed_tool_conf', None ) if shed_tool_conf: @@ -957,6 +953,13 @@ self.install_model.context.add( tool_shed_repository ) self.install_model.context.flush() + def __assert_can_install_dependencies(self): + if self.app.config.tool_dependency_dir is None: + no_tool_dependency_dir_message = "Tool dependencies can be automatically installed only if you set " + no_tool_dependency_dir_message += "the value of your 'tool_dependency_dir' setting in your Galaxy " + no_tool_dependency_dir_message += "configuration file (galaxy.ini) and restart your Galaxy server. " + raise exceptions.ConfigDoesNotAllowException( no_tool_dependency_dir_message ) + def fetch_tool_versions( app, tool_shed_repository ): """ Fetch a data structure describing tool shed versions from the tool shed https://bitbucket.org/galaxy/galaxy-central/commits/0688265255e4/ Changeset: 0688265255e4 User: jmchilton Date: 2015-02-27 20:58:02+00:00 Summary: Make more clear function in admin_toolshed is not entry point. Affected #: 1 file diff -r f95ba77746449eeff2593a06a457b57ebe803eca -r 0688265255e47eea3e90ce0947cadc520ab1145f lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py --- a/lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py +++ b/lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py @@ -639,32 +639,6 @@ @web.expose @web.require_admin - def install_tool_shed_repositories( self, trans, tool_shed_repositories, reinstalling=False, **kwd ): - """Install specified tool shed repositories.""" - shed_tool_conf = kwd.get( 'shed_tool_conf', '' ) - tool_path = kwd[ 'tool_path' ] - install_tool_dependencies = CheckboxField.is_checked( kwd.get( 'install_tool_dependencies', '' ) ) - # There must be a one-to-one mapping between items in the 3 lists: tool_shed_repositories, tool_panel_section_keys, repo_info_dicts. - tool_panel_section_keys = util.listify( kwd[ 'tool_panel_section_keys' ] ) - repo_info_dicts = util.listify( kwd[ 'repo_info_dicts' ] ) - irm = install_manager.InstallRepositoryManager( trans.app ) - for index, tool_shed_repository in enumerate( tool_shed_repositories ): - repo_info_dict = repo_info_dicts[ index ] - tool_panel_section_key = tool_panel_section_keys[ index ] - irm.install_tool_shed_repository( tool_shed_repository, - repo_info_dict, - tool_panel_section_key, - shed_tool_conf, - tool_path, - install_tool_dependencies, - reinstalling=reinstalling ) - tsr_ids_for_monitoring = [ trans.security.encode_id( tsr.id ) for tsr in tool_shed_repositories ] - return trans.response.send_redirect( web.url_for( controller='admin_toolshed', - action='monitor_repository_installation', - tool_shed_repository_ids=tsr_ids_for_monitoring ) ) - - @web.expose - @web.require_admin def manage_repositories( self, trans, **kwd ): message = escape( kwd.get( 'message', '' ) ) status = kwd.get( 'status', 'done' ) @@ -733,10 +707,10 @@ if repositories_for_installation: decoded_kwd[ 'repo_info_dicts' ] = filtered_repo_info_dicts decoded_kwd[ 'tool_panel_section_keys' ] = filtered_tool_panel_section_keys - self.install_tool_shed_repositories( trans, - repositories_for_installation, - reinstalling=reinstalling, - **decoded_kwd ) + self.__install_tool_shed_repositories( trans, + repositories_for_installation, + reinstalling=reinstalling, + **decoded_kwd ) else: kwd[ 'message' ] = 'All selected tool shed repositories are already installed.' kwd[ 'status' ] = 'error' @@ -2162,3 +2136,27 @@ metadata=metadata, message=message, status=status ) + + def __install_tool_shed_repositories( self, trans, tool_shed_repositories, reinstalling=False, **kwd ): + """Install specified tool shed repositories.""" + shed_tool_conf = kwd.get( 'shed_tool_conf', '' ) + tool_path = kwd[ 'tool_path' ] + install_tool_dependencies = CheckboxField.is_checked( kwd.get( 'install_tool_dependencies', '' ) ) + # There must be a one-to-one mapping between items in the 3 lists: tool_shed_repositories, tool_panel_section_keys, repo_info_dicts. + tool_panel_section_keys = util.listify( kwd[ 'tool_panel_section_keys' ] ) + repo_info_dicts = util.listify( kwd[ 'repo_info_dicts' ] ) + irm = install_manager.InstallRepositoryManager( trans.app ) + for index, tool_shed_repository in enumerate( tool_shed_repositories ): + repo_info_dict = repo_info_dicts[ index ] + tool_panel_section_key = tool_panel_section_keys[ index ] + irm.install_tool_shed_repository( tool_shed_repository, + repo_info_dict, + tool_panel_section_key, + shed_tool_conf, + tool_path, + install_tool_dependencies, + reinstalling=reinstalling ) + tsr_ids_for_monitoring = [ trans.security.encode_id( tsr.id ) for tsr in tool_shed_repositories ] + return trans.response.send_redirect( web.url_for( controller='admin_toolshed', + action='monitor_repository_installation', + tool_shed_repository_ids=tsr_ids_for_monitoring ) ) https://bitbucket.org/galaxy/galaxy-central/commits/2998ac978d89/ Changeset: 2998ac978d89 User: jmchilton Date: 2015-02-27 20:58:02+00:00 Summary: Extract an exception out in install_manager.py. Will make it easier to catch this specific problem in admin_toolshed in subsequent refactoring. Affected #: 1 file diff -r 0688265255e47eea3e90ce0947cadc520ab1145f -r 2998ac978d8935c7d9ba655ada297e2f2d205ebd lib/tool_shed/galaxy_install/install_manager.py --- a/lib/tool_shed/galaxy_install/install_manager.py +++ b/lib/tool_shed/galaxy_install/install_manager.py @@ -804,7 +804,7 @@ installed_tool_shed_repositories.append( tool_shed_repository ) else: # We're attempting to install more than 1 repository, and all of them have already been installed. - raise exceptions.RequestParameterInvalidException( 'All repositories that you are attempting to install have been previously installed.' ) + raise RepositoriesInstalledException() return installed_tool_shed_repositories def install_tool_shed_repository( self, tool_shed_repository, repo_info_dict, tool_panel_section_key, shed_tool_conf, tool_path, @@ -961,6 +961,12 @@ raise exceptions.ConfigDoesNotAllowException( no_tool_dependency_dir_message ) +class RepositoriesInstalledException(exceptions.RequestParameterInvalidException): + + def __init__(self): + super(RepositoriesInstalledException, self).__init__('All repositories that you are attempting to install have been previously installed.') + + def fetch_tool_versions( app, tool_shed_repository ): """ Fetch a data structure describing tool shed versions from the tool shed corresponding to a tool_shed_repository object. https://bitbucket.org/galaxy/galaxy-central/commits/d7849c8a5de0/ Changeset: d7849c8a5de0 User: jmchilton Date: 2015-02-27 20:58:02+00:00 Summary: Refactor a bunch of repository installation out of controller... ... and into install_manager.py. There is a bunch of duplication between this and the code already in install manager - which is a good indication this is where it belongs - hopefully we can de-duplicate it with subsequent changes. This will also allow more of install_manager's methods to be marked as internal. Affected #: 2 files diff -r 2998ac978d8935c7d9ba655ada297e2f2d205ebd -r d7849c8a5de0878a9da97ce4ab52a0272a904a13 lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py --- a/lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py +++ b/lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py @@ -678,41 +678,20 @@ reinstalling = util.string_as_bool( kwd.get( 'reinstalling', False ) ) encoded_kwd = kwd[ 'encoded_kwd' ] decoded_kwd = encoding_util.tool_shed_decode( encoded_kwd ) - tsr_ids = decoded_kwd[ 'tool_shed_repository_ids' ] - tool_panel_section_keys = decoded_kwd[ 'tool_panel_section_keys' ] - repo_info_dicts = decoded_kwd[ 'repo_info_dicts' ] - filtered_repo_info_dicts = [] - filtered_tool_panel_section_keys = [] - repositories_for_installation = [] - # Some repositories may have repository dependencies that are required to be installed before the - # dependent repository, so we'll order the list of tsr_ids to ensure all repositories install in the - # required order. - ordered_tsr_ids, ordered_repo_info_dicts, ordered_tool_panel_section_keys = \ - irm.order_components_for_installation( tsr_ids, - repo_info_dicts, - tool_panel_section_keys=tool_panel_section_keys ) - for tsr_id in ordered_tsr_ids: - repository = trans.install_model.context.query( trans.install_model.ToolShedRepository ) \ - .get( trans.security.decode_id( tsr_id ) ) - if repository.status in [ trans.install_model.ToolShedRepository.installation_status.NEW, - trans.install_model.ToolShedRepository.installation_status.UNINSTALLED ]: - repositories_for_installation.append( repository ) - repo_info_dict, tool_panel_section_key = \ - irm.get_repository_components_for_installation( tsr_id, - ordered_tsr_ids, - ordered_repo_info_dicts, - ordered_tool_panel_section_keys ) - filtered_repo_info_dicts.append( repo_info_dict ) - filtered_tool_panel_section_keys.append( tool_panel_section_key ) - if repositories_for_installation: - decoded_kwd[ 'repo_info_dicts' ] = filtered_repo_info_dicts - decoded_kwd[ 'tool_panel_section_keys' ] = filtered_tool_panel_section_keys - self.__install_tool_shed_repositories( trans, - repositories_for_installation, - reinstalling=reinstalling, - **decoded_kwd ) - else: - kwd[ 'message' ] = 'All selected tool shed repositories are already installed.' + install_tool_dependencies = CheckboxField.is_checked( decoded_kwd.get( 'install_tool_dependencies', '' ) ) + decoded_kwd['install_tool_dependencies'] = install_tool_dependencies + try: + tool_shed_repositories = irm.install_web( + trans=trans, + decoded_kwd=decoded_kwd, + reinstalling=reinstalling, + ) + tsr_ids_for_monitoring = [ trans.security.encode_id( tsr.id ) for tsr in tool_shed_repositories ] + trans.response.send_redirect( web.url_for( controller='admin_toolshed', + action='monitor_repository_installation', + tool_shed_repository_ids=tsr_ids_for_monitoring ) ) + except install_manager.RepositoriesInstalledException as e: + kwd[ 'message' ] = e.message kwd[ 'status' ] = 'error' return self.repository_installation_grid( trans, **kwd ) @@ -2136,27 +2115,3 @@ metadata=metadata, message=message, status=status ) - - def __install_tool_shed_repositories( self, trans, tool_shed_repositories, reinstalling=False, **kwd ): - """Install specified tool shed repositories.""" - shed_tool_conf = kwd.get( 'shed_tool_conf', '' ) - tool_path = kwd[ 'tool_path' ] - install_tool_dependencies = CheckboxField.is_checked( kwd.get( 'install_tool_dependencies', '' ) ) - # There must be a one-to-one mapping between items in the 3 lists: tool_shed_repositories, tool_panel_section_keys, repo_info_dicts. - tool_panel_section_keys = util.listify( kwd[ 'tool_panel_section_keys' ] ) - repo_info_dicts = util.listify( kwd[ 'repo_info_dicts' ] ) - irm = install_manager.InstallRepositoryManager( trans.app ) - for index, tool_shed_repository in enumerate( tool_shed_repositories ): - repo_info_dict = repo_info_dicts[ index ] - tool_panel_section_key = tool_panel_section_keys[ index ] - irm.install_tool_shed_repository( tool_shed_repository, - repo_info_dict, - tool_panel_section_key, - shed_tool_conf, - tool_path, - install_tool_dependencies, - reinstalling=reinstalling ) - tsr_ids_for_monitoring = [ trans.security.encode_id( tsr.id ) for tsr in tool_shed_repositories ] - return trans.response.send_redirect( web.url_for( controller='admin_toolshed', - action='monitor_repository_installation', - tool_shed_repository_ids=tsr_ids_for_monitoring ) ) diff -r 2998ac978d8935c7d9ba655ada297e2f2d205ebd -r d7849c8a5de0878a9da97ce4ab52a0272a904a13 lib/tool_shed/galaxy_install/install_manager.py --- a/lib/tool_shed/galaxy_install/install_manager.py +++ b/lib/tool_shed/galaxy_install/install_manager.py @@ -7,6 +7,7 @@ from galaxy import exceptions from galaxy import eggs +from galaxy import util eggs.require( 'paramiko' ) eggs.require( 'ssh' ) @@ -807,6 +808,54 @@ raise RepositoriesInstalledException() return installed_tool_shed_repositories + def install_web( self, trans, decoded_kwd, reinstalling ): + shed_tool_conf = decoded_kwd.get( 'shed_tool_conf', '' ) + tool_path = decoded_kwd[ 'tool_path' ] + tsr_ids = decoded_kwd[ 'tool_shed_repository_ids' ] + tool_panel_section_keys = util.listify( decoded_kwd[ 'tool_panel_section_keys' ] ) + repo_info_dicts = util.listify( decoded_kwd[ 'repo_info_dicts' ] ) + install_tool_dependencies = decoded_kwd['install_tool_dependencies'] + filtered_repo_info_dicts = [] + filtered_tool_panel_section_keys = [] + repositories_for_installation = [] + # Some repositories may have repository dependencies that are required to be installed before the + # dependent repository, so we'll order the list of tsr_ids to ensure all repositories install in the + # required order. + ordered_tsr_ids, ordered_repo_info_dicts, ordered_tool_panel_section_keys = \ + self.order_components_for_installation( tsr_ids, + repo_info_dicts, + tool_panel_section_keys=tool_panel_section_keys ) + for tsr_id in ordered_tsr_ids: + repository = trans.install_model.context.query( trans.install_model.ToolShedRepository ) \ + .get( trans.security.decode_id( tsr_id ) ) + if repository.status in [ trans.install_model.ToolShedRepository.installation_status.NEW, + trans.install_model.ToolShedRepository.installation_status.UNINSTALLED ]: + repositories_for_installation.append( repository ) + repo_info_dict, tool_panel_section_key = \ + self.get_repository_components_for_installation( tsr_id, + ordered_tsr_ids, + ordered_repo_info_dicts, + ordered_tool_panel_section_keys ) + filtered_repo_info_dicts.append( repo_info_dict ) + filtered_tool_panel_section_keys.append( tool_panel_section_key ) + + installed_tool_shed_repositories = [] + if repositories_for_installation: + for index, tool_shed_repository in enumerate( repositories_for_installation ): + repo_info_dict = filtered_repo_info_dicts[ index ] + tool_panel_section_key = filtered_tool_panel_section_keys[ index ] + self.install_tool_shed_repository( tool_shed_repository, + repo_info_dict=repo_info_dict, + tool_panel_section_key=tool_panel_section_key, + shed_tool_conf=shed_tool_conf, + tool_path=tool_path, + install_tool_dependencies=install_tool_dependencies, + reinstalling=reinstalling ) + installed_tool_shed_repositories.append( tool_shed_repository ) + else: + raise RepositoriesInstalledException() + return installed_tool_shed_repositories + def install_tool_shed_repository( self, tool_shed_repository, repo_info_dict, tool_panel_section_key, shed_tool_conf, tool_path, install_tool_dependencies, reinstalling=False ): if tool_panel_section_key: https://bitbucket.org/galaxy/galaxy-central/commits/4a04ee11725e/ Changeset: 4a04ee11725e User: jmchilton Date: 2015-02-27 20:58:02+00:00 Summary: PEP-8 comments in install_manager.py. Affected #: 1 file diff -r d7849c8a5de0878a9da97ce4ab52a0272a904a13 -r 4a04ee11725e11127848775ae46e68818843e768 lib/tool_shed/galaxy_install/install_manager.py --- a/lib/tool_shed/galaxy_install/install_manager.py +++ b/lib/tool_shed/galaxy_install/install_manager.py @@ -455,10 +455,12 @@ def get_repository_components_for_installation( self, encoded_tsr_id, encoded_tsr_ids, repo_info_dicts, tool_panel_section_keys ): """ - The received encoded_tsr_ids, repo_info_dicts, and tool_panel_section_keys are 3 lists that - contain associated elements at each location in the list. This method will return the elements - from repo_info_dicts and tool_panel_section_keys associated with the received encoded_tsr_id - by determining its location in the received encoded_tsr_ids list. + The received encoded_tsr_ids, repo_info_dicts, and + tool_panel_section_keys are 3 lists that contain associated elements + at each location in the list. This method will return the elements + from repo_info_dicts and tool_panel_section_keys associated with the + received encoded_tsr_id by determining its location in the received + encoded_tsr_ids list. """ for index, tsr_id in enumerate( encoded_tsr_ids ): if tsr_id == encoded_tsr_id: @@ -775,15 +777,18 @@ tool_panel_section_keys=tool_panel_section_keys, tool_path=tool_path, tool_shed_url=tool_shed_url ) - # Prepare the repositories for installation. Even though this method receives a single combination - # of tool_shed_url, name, owner and changeset_revision, there may be multiple repositories for installation - # at this point because repository dependencies may have added additional repositories for installation - # along with the single specified repository. + # Prepare the repositories for installation. Even though this + # method receives a single combination of tool_shed_url, name, + # owner and changeset_revision, there may be multiple repositories + # for installation at this point because repository dependencies + # may have added additional repositories for installation along + # with the single specified repository. encoded_kwd, query, tool_shed_repositories, encoded_repository_ids = \ self.initiate_repository_installation( installation_dict ) - # Some repositories may have repository dependencies that are required to be installed before the - # dependent repository, so we'll order the list of tsr_ids to ensure all repositories install in - # the required order. + # Some repositories may have repository dependencies that are + # required to be installed before the dependent repository, so + # we'll order the list of tsr_ids to ensure all repositories + # install in the required order. tsr_ids = [ self.app.security.encode_id( tool_shed_repository.id ) for tool_shed_repository in tool_shed_repositories ] ordered_tsr_ids, ordered_repo_info_dicts, ordered_tool_panel_section_keys = \ self.order_components_for_installation( tsr_ids, repo_info_dicts, tool_panel_section_keys=tool_panel_section_keys ) https://bitbucket.org/galaxy/galaxy-central/commits/77c9941c3d8b/ Changeset: 77c9941c3d8b User: jmchilton Date: 2015-02-27 20:58:02+00:00 Summary: Refactor __install_repositories so it looks more like newer install_web. Affected #: 1 file diff -r 4a04ee11725e11127848775ae46e68818843e768 -r 77c9941c3d8befb3d2678862633f05216cca1c18 lib/tool_shed/galaxy_install/install_manager.py --- a/lib/tool_shed/galaxy_install/install_manager.py +++ b/lib/tool_shed/galaxy_install/install_manager.py @@ -709,8 +709,6 @@ return installed_tool_shed_repositories def __install_repositories( self, tool_shed_url, repository_revision_dict, repo_info_dicts, install_options ): - # Keep track of all repositories that are installed - there may be more than one if repository dependencies are installed. - installed_tool_shed_repositories = [] try: has_repository_dependencies = repository_revision_dict[ 'has_repository_dependencies' ] except KeyError: @@ -790,6 +788,10 @@ # we'll order the list of tsr_ids to ensure all repositories # install in the required order. tsr_ids = [ self.app.security.encode_id( tool_shed_repository.id ) for tool_shed_repository in tool_shed_repositories ] + + # Keep track of all repositories that are installed - there may be more than one if repository dependencies are installed. + installed_tool_shed_repositories = [] + ordered_tsr_ids, ordered_repo_info_dicts, ordered_tool_panel_section_keys = \ self.order_components_for_installation( tsr_ids, repo_info_dicts, tool_panel_section_keys=tool_panel_section_keys ) # Install the repositories, keeping track of each one for later display. @@ -808,10 +810,10 @@ install_tool_dependencies, reinstalling=False ) installed_tool_shed_repositories.append( tool_shed_repository ) + return installed_tool_shed_repositories else: # We're attempting to install more than 1 repository, and all of them have already been installed. raise RepositoriesInstalledException() - return installed_tool_shed_repositories def install_web( self, trans, decoded_kwd, reinstalling ): shed_tool_conf = decoded_kwd.get( 'shed_tool_conf', '' ) https://bitbucket.org/galaxy/galaxy-central/commits/1b96a2122ea6/ Changeset: 1b96a2122ea6 User: jmchilton Date: 2015-02-27 20:58:02+00:00 Summary: Eliminate duplication between ts installing from API and web controller. Favoring the web controller variant slightly - it is known to work better and has test coverage. Affected #: 2 files diff -r 77c9941c3d8befb3d2678862633f05216cca1c18 -r 1b96a2122ea6607e84e3fb89e5e9209fbd119fd4 lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py --- a/lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py +++ b/lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py @@ -679,10 +679,11 @@ encoded_kwd = kwd[ 'encoded_kwd' ] decoded_kwd = encoding_util.tool_shed_decode( encoded_kwd ) install_tool_dependencies = CheckboxField.is_checked( decoded_kwd.get( 'install_tool_dependencies', '' ) ) + tsr_ids = decoded_kwd[ 'tool_shed_repository_ids' ] decoded_kwd['install_tool_dependencies'] = install_tool_dependencies try: - tool_shed_repositories = irm.install_web( - trans=trans, + tool_shed_repositories = irm.install_repositories( + tsr_ids=tsr_ids, decoded_kwd=decoded_kwd, reinstalling=reinstalling, ) diff -r 77c9941c3d8befb3d2678862633f05216cca1c18 -r 1b96a2122ea6607e84e3fb89e5e9209fbd119fd4 lib/tool_shed/galaxy_install/install_manager.py --- a/lib/tool_shed/galaxy_install/install_manager.py +++ b/lib/tool_shed/galaxy_install/install_manager.py @@ -702,13 +702,15 @@ name, owner, changeset_revision ) - installed_tool_shed_repositories = self.__install_repositories( tool_shed_url, - repository_revision_dict, - repo_info_dicts, - install_options ) + installed_tool_shed_repositories = self.__initiate_and_install_repositories( + tool_shed_url, + repository_revision_dict, + repo_info_dicts, + install_options + ) return installed_tool_shed_repositories - def __install_repositories( self, tool_shed_url, repository_revision_dict, repo_info_dicts, install_options ): + def __initiate_and_install_repositories( self, tool_shed_url, repository_revision_dict, repo_info_dicts, install_options ): try: has_repository_dependencies = repository_revision_dict[ 'has_repository_dependencies' ] except KeyError: @@ -789,36 +791,18 @@ # install in the required order. tsr_ids = [ self.app.security.encode_id( tool_shed_repository.id ) for tool_shed_repository in tool_shed_repositories ] - # Keep track of all repositories that are installed - there may be more than one if repository dependencies are installed. - installed_tool_shed_repositories = [] + decoded_kwd = dict( + shed_tool_conf=shed_tool_conf, + tool_path=tool_path, + tool_panel_section_keys=tool_panel_section_keys, + repo_info_dicts=repo_info_dicts, + install_tool_dependencies=install_tool_dependencies, + ) + return self.install_repositories(tsr_ids, decoded_kwd, reinstalling=False) - ordered_tsr_ids, ordered_repo_info_dicts, ordered_tool_panel_section_keys = \ - self.order_components_for_installation( tsr_ids, repo_info_dicts, tool_panel_section_keys=tool_panel_section_keys ) - # Install the repositories, keeping track of each one for later display. - for index, tsr_id in enumerate( ordered_tsr_ids ): - tool_shed_repository = self.install_model.context.query( self.install_model.ToolShedRepository ) \ - .get( self.app.security.decode_id( tsr_id ) ) - if tool_shed_repository.status in [ self.install_model.ToolShedRepository.installation_status.NEW, - self.install_model.ToolShedRepository.installation_status.UNINSTALLED ]: - repo_info_dict = ordered_repo_info_dicts[ index ] - tool_panel_section_key = ordered_tool_panel_section_keys[ index ] - self.install_tool_shed_repository( tool_shed_repository, - repo_info_dict, - tool_panel_section_key, - shed_tool_conf, - tool_path, - install_tool_dependencies, - reinstalling=False ) - installed_tool_shed_repositories.append( tool_shed_repository ) - return installed_tool_shed_repositories - else: - # We're attempting to install more than 1 repository, and all of them have already been installed. - raise RepositoriesInstalledException() - - def install_web( self, trans, decoded_kwd, reinstalling ): + def install_repositories( self, tsr_ids, decoded_kwd, reinstalling ): shed_tool_conf = decoded_kwd.get( 'shed_tool_conf', '' ) tool_path = decoded_kwd[ 'tool_path' ] - tsr_ids = decoded_kwd[ 'tool_shed_repository_ids' ] tool_panel_section_keys = util.listify( decoded_kwd[ 'tool_panel_section_keys' ] ) repo_info_dicts = util.listify( decoded_kwd[ 'repo_info_dicts' ] ) install_tool_dependencies = decoded_kwd['install_tool_dependencies'] @@ -833,10 +817,10 @@ repo_info_dicts, tool_panel_section_keys=tool_panel_section_keys ) for tsr_id in ordered_tsr_ids: - repository = trans.install_model.context.query( trans.install_model.ToolShedRepository ) \ - .get( trans.security.decode_id( tsr_id ) ) - if repository.status in [ trans.install_model.ToolShedRepository.installation_status.NEW, - trans.install_model.ToolShedRepository.installation_status.UNINSTALLED ]: + repository = self.install_model.context.query( self.install_model.ToolShedRepository ) \ + .get( self.app.security.decode_id( tsr_id ) ) + if repository.status in [ self.install_model.ToolShedRepository.installation_status.NEW, + self.install_model.ToolShedRepository.installation_status.UNINSTALLED ]: repositories_for_installation.append( repository ) repo_info_dict, tool_panel_section_key = \ self.get_repository_components_for_installation( tsr_id, https://bitbucket.org/galaxy/galaxy-central/commits/ebe2008f5e49/ Changeset: ebe2008f5e49 User: jmchilton Date: 2015-02-27 20:58:02+00:00 Summary: Use append_section abstraction. Affected #: 1 file diff -r 1b96a2122ea6607e84e3fb89e5e9209fbd119fd4 -r ebe2008f5e49998cc74264ee085e7e7a4fab57bc lib/galaxy/tools/toolbox/base.py --- a/lib/galaxy/tools/toolbox/base.py +++ b/lib/galaxy/tools/toolbox/base.py @@ -237,7 +237,7 @@ 'version': '', } tool_section = ToolSection( section_dict ) - self._tool_panel[ tool_panel_section_key ] = tool_section + self._tool_panel.append_section( tool_panel_section_key, tool_section ) log.debug( "Loading new tool panel section: %s" % str( tool_section.name ) ) else: tool_section = None https://bitbucket.org/galaxy/galaxy-central/commits/4f0678e797a1/ Changeset: 4f0678e797a1 User: jmchilton Date: 2015-02-27 20:58:02+00:00 Summary: PEP-8 in toolbox/base.py. Affected #: 1 file diff -r ebe2008f5e49998cc74264ee085e7e7a4fab57bc -r 4f0678e797a1247103459fba186c4dc8765b8aa2 lib/galaxy/tools/toolbox/base.py --- a/lib/galaxy/tools/toolbox/base.py +++ b/lib/galaxy/tools/toolbox/base.py @@ -457,9 +457,9 @@ if tool_id in self._tools_by_id and not get_all_versions: if tool_version and tool_version in self._tool_versions_by_id[ tool_id ]: return self._tool_versions_by_id[ tool_id ][ tool_version ] - #tool_id exactly matches an available tool by id (which is 'old' tool_id or guid) + # tool_id exactly matches an available tool by id (which is 'old' tool_id or guid) return self._tools_by_id[ tool_id ] - #exact tool id match not found, or all versions requested, search for other options, e.g. migrated tools or different versions + # exact tool id match not found, or all versions requested, search for other options, e.g. migrated tools or different versions rval = [] tool_lineage = self._lineage_map.get( tool_id ) if tool_lineage: @@ -469,7 +469,7 @@ if lineage_tool: rval.append( lineage_tool ) if not rval: - #still no tool, do a deeper search and try to match by old ids + # still no tool, do a deeper search and try to match by old ids for tool in self._tools_by_id.itervalues(): if tool.old_id == tool_id: rval.append( tool ) @@ -478,14 +478,14 @@ return rval else: if tool_version: - #return first tool with matching version + # return first tool with matching version for tool in rval: if tool.version == tool_version: return tool - #No tool matches by version, simply return the first available tool found + # No tool matches by version, simply return the first available tool found return rval[0] - #We now likely have a Toolshed guid passed in, but no supporting database entries - #If the tool exists by exact id and is loaded then provide exact match within a list + # We now likely have a Toolshed guid passed in, but no supporting database entries + # If the tool exists by exact id and is loaded then provide exact match within a list if tool_id in self._tools_by_id: return[ self._tools_by_id[ tool_id ] ] return None @@ -1012,7 +1012,7 @@ break if tool_id in self.data_manager_tools: del self.data_manager_tools[ tool_id ] - #TODO: do we need to manually remove from the integrated panel here? + # TODO: do we need to manually remove from the integrated panel here? message = "Removed the tool:<br/>" message += "<b>name:</b> %s<br/>" % tool.name message += "<b>id:</b> %s<br/>" % tool.id https://bitbucket.org/galaxy/galaxy-central/commits/0468d285f89c/ Changeset: 0468d285f89c User: jmchilton Date: 2015-02-27 20:58:02+00:00 Summary: Move integrated tool panel logic into its own module/mixin. Affected #: 2 files diff -r 4f0678e797a1247103459fba186c4dc8765b8aa2 -r 0468d285f89c799559926c94f300c42d05e8c47a lib/galaxy/tools/toolbox/base.py --- a/lib/galaxy/tools/toolbox/base.py +++ b/lib/galaxy/tools/toolbox/base.py @@ -1,6 +1,5 @@ import os import re -import shutil import string import tarfile import tempfile @@ -23,6 +22,7 @@ from .panel import ToolSectionLabel from .panel import ToolSection from .panel import panel_item_types +from .integrated_panel import ManagesIntegratedToolPanelMixin from .lineages import LineageMap from .tags import tool_tag_manager @@ -37,22 +37,7 @@ log = logging.getLogger( __name__ ) -INTEGRATED_TOOL_PANEL_DESCRIPTION = """ -This is Galaxy's integrated tool panel and should be modified directly only for -reordering tools inside a section. Each time Galaxy starts up, this file is -synchronized with the various tool config files: tools, sections and labels -added to one of these files, will be added also here in the appropriate place, -while elements removed from the tool config files will be correspondingly -deleted from this file. -To modify locally managed tools (e.g. from tool_conf.xml) modify that file -directly and restart Galaxy. Whenever possible Tool Shed managed tools (e.g. -from shed_tool_conf.xml) should be managed from within the Galaxy interface or -via its API - but if changes are necessary (such as to hide a tool or re-assign -its section) modify that file and restart Galaxy. -""" - - -class AbstractToolBox( object, Dictifiable ): +class AbstractToolBox( object, Dictifiable, ManagesIntegratedToolPanelMixin ): """ Abstract container for managing a ToolPanel - containing tools and workflows optionally in labelled sections. @@ -78,15 +63,8 @@ self._index = 0 self.data_manager_tools = odict() self._lineage_map = LineageMap( app ) - # File that contains the XML section and tool tags from all tool panel config files integrated into a - # single file that defines the tool panel layout. This file can be changed by the Galaxy administrator - # (in a way similar to the single tool_conf.xml file in the past) to alter the layout of the tool panel. - self._integrated_tool_panel_config = app.config.integrated_tool_panel_config - # In-memory dictionary that defines the layout of the tool_panel.xml file on disk. - self._integrated_tool_panel = ToolPanelElements() - self._integrated_tool_panel_config_has_contents = os.path.exists( self._integrated_tool_panel_config ) and os.stat( self._integrated_tool_panel_config ).st_size > 0 - if self._integrated_tool_panel_config_has_contents: - self._load_integrated_tool_panel_keys() + # Sets self._integrated_tool_panel and self._integrated_tool_panel_config_has_contents + self._init_integrated_tool_panel( app.config ) # The following refers to the tool_path config setting for backward compatibility. The shed-related # (e.g., shed_tool_conf.xml) files include the tool_path attribute within the <toolbox> tag. self._tool_root_dir = tool_root_dir @@ -98,12 +76,7 @@ if self.app.name == 'galaxy' and self._integrated_tool_panel_config_has_contents: # Load self._tool_panel based on the order in self._integrated_tool_panel. self._load_tool_panel() - if app.config.update_integrated_tool_panel: - # Write the current in-memory integrated_tool_panel to the integrated_tool_panel.xml file. - # This will cover cases where the Galaxy administrator manually edited one or more of the tool panel - # config files, adding or removing locally developed tools or workflows. The value of integrated_tool_panel - # will be False when things like functional tests are the caller. - self._write_integrated_tool_panel_config_file() + self._save_integrated_tool_panel() def create_tool( self, config_file, repository_id=None, guid=None, **kwds ): raise NotImplementedError() @@ -215,9 +188,8 @@ for index, my_shed_tool_conf in enumerate( self._dynamic_tool_confs ): if shed_conf['config_filename'] == my_shed_tool_conf['config_filename']: self._dynamic_tool_confs[ index ] = shed_conf - if integrated_panel_changes and app.config.update_integrated_tool_panel: - # Write the current in-memory version of the integrated_tool_panel.xml file to disk. - self._write_integrated_tool_panel_config_file() + if integrated_panel_changes: + self._save_integrated_tool_panel() app.reindex_tool_search() def get_section( self, section_id, new_label=None, create_if_needed=False ): @@ -400,52 +372,6 @@ elif elem.tag == 'label': self._integrated_tool_panel.stub_label( key ) - def _write_integrated_tool_panel_config_file( self ): - """ - Write the current in-memory version of the integrated_tool_panel.xml file to disk. Since Galaxy administrators - use this file to manage the tool panel, we'll not use xml_to_string() since it doesn't write XML quite right. - """ - fd, filename = tempfile.mkstemp() - os.write( fd, '<?xml version="1.0"?>\n' ) - os.write( fd, '<toolbox>\n' ) - os.write( fd, ' <!--\n ') - os.write( fd, '\n '.join( [ l for l in INTEGRATED_TOOL_PANEL_DESCRIPTION.split("\n") if l ] ) ) - os.write( fd, '\n -->\n') - for key, item_type, item in self._integrated_tool_panel.panel_items_iter(): - if item: - if item_type == panel_item_types.TOOL: - os.write( fd, ' <tool id="%s" />\n' % item.id ) - elif item_type == panel_item_types.WORKFLOW: - os.write( fd, ' <workflow id="%s" />\n' % item.id ) - elif item_type == panel_item_types.LABEL: - label_id = item.id or '' - label_text = item.text or '' - label_version = item.version or '' - os.write( fd, ' <label id="%s" text="%s" version="%s" />\n' % ( label_id, label_text, label_version ) ) - elif item_type == panel_item_types.SECTION: - section_id = item.id or '' - section_name = item.name or '' - section_version = item.version or '' - os.write( fd, ' <section id="%s" name="%s" version="%s">\n' % ( section_id, section_name, section_version ) ) - for section_key, section_item_type, section_item in item.panel_items_iter(): - if section_item_type == panel_item_types.TOOL: - if section_item: - os.write( fd, ' <tool id="%s" />\n' % section_item.id ) - elif section_item_type == panel_item_types.WORKFLOW: - if section_item: - os.write( fd, ' <workflow id="%s" />\n' % section_item.id ) - elif section_item_type == panel_item_types.LABEL: - if section_item: - label_id = section_item.id or '' - label_text = section_item.text or '' - label_version = section_item.version or '' - os.write( fd, ' <label id="%s" text="%s" version="%s" />\n' % ( label_id, label_text, label_version ) ) - os.write( fd, ' </section>\n' ) - os.write( fd, '</toolbox>\n' ) - os.close( fd ) - shutil.move( filename, os.path.abspath( self._integrated_tool_panel_config ) ) - os.chmod( self._integrated_tool_panel_config, 0644 ) - def get_tool( self, tool_id, tool_version=None, get_all_versions=False, exact=False ): """Attempt to locate a tool in the tool box.""" if tool_version: @@ -777,13 +703,7 @@ if async: self._load_tool_panel() - - if self.app.config.update_integrated_tool_panel: - # Write the current in-memory integrated_tool_panel to the integrated_tool_panel.xml file. - # This will cover cases where the Galaxy administrator manually edited one or more of the tool panel - # config files, adding or removing locally developed tools or workflows. The value of integrated_tool_panel - # will be False when things like functional tests are the caller. - self._write_integrated_tool_panel_config_file() + self._save_integrated_tool_panel() return tool.id except Exception: log.exception("Failed to load potential tool %s." % tool_file) diff -r 4f0678e797a1247103459fba186c4dc8765b8aa2 -r 0468d285f89c799559926c94f300c42d05e8c47a lib/galaxy/tools/toolbox/integrated_panel.py --- /dev/null +++ b/lib/galaxy/tools/toolbox/integrated_panel.py @@ -0,0 +1,87 @@ +import os +import shutil +import tempfile + +from .panel import ToolPanelElements +from .panel import panel_item_types + + +INTEGRATED_TOOL_PANEL_DESCRIPTION = """ +This is Galaxy's integrated tool panel and should be modified directly only for +reordering tools inside a section. Each time Galaxy starts up, this file is +synchronized with the various tool config files: tools, sections and labels +added to one of these files, will be added also here in the appropriate place, +while elements removed from the tool config files will be correspondingly +deleted from this file. +To modify locally managed tools (e.g. from tool_conf.xml) modify that file +directly and restart Galaxy. Whenever possible Tool Shed managed tools (e.g. +from shed_tool_conf.xml) should be managed from within the Galaxy interface or +via its API - but if changes are necessary (such as to hide a tool or re-assign +its section) modify that file and restart Galaxy. +""" + + +class ManagesIntegratedToolPanelMixin: + + def _init_integrated_tool_panel(self, config): + self.update_integrated_tool_panel = config.update_integrated_tool_panel + self._integrated_tool_panel_config = config.integrated_tool_panel_config + # In-memory dictionary that defines the layout of the tool_panel.xml file on disk. + self._integrated_tool_panel = ToolPanelElements() + self._integrated_tool_panel_config_has_contents = os.path.exists( self._integrated_tool_panel_config ) and os.stat( self._integrated_tool_panel_config ).st_size > 0 + if self._integrated_tool_panel_config_has_contents: + self._load_integrated_tool_panel_keys() + + def _save_integrated_tool_panel(self): + if self.update_integrated_tool_panel: + # Write the current in-memory integrated_tool_panel to the integrated_tool_panel.xml file. + # This will cover cases where the Galaxy administrator manually edited one or more of the tool panel + # config files, adding or removing locally developed tools or workflows. The value of integrated_tool_panel + # will be False when things like functional tests are the caller. + self._write_integrated_tool_panel_config_file() + + def _write_integrated_tool_panel_config_file( self ): + """ + Write the current in-memory version of the integrated_tool_panel.xml file to disk. Since Galaxy administrators + use this file to manage the tool panel, we'll not use xml_to_string() since it doesn't write XML quite right. + """ + fd, filename = tempfile.mkstemp() + os.write( fd, '<?xml version="1.0"?>\n' ) + os.write( fd, '<toolbox>\n' ) + os.write( fd, ' <!--\n ') + os.write( fd, '\n '.join( [ l for l in INTEGRATED_TOOL_PANEL_DESCRIPTION.split("\n") if l ] ) ) + os.write( fd, '\n -->\n') + for key, item_type, item in self._integrated_tool_panel.panel_items_iter(): + if item: + if item_type == panel_item_types.TOOL: + os.write( fd, ' <tool id="%s" />\n' % item.id ) + elif item_type == panel_item_types.WORKFLOW: + os.write( fd, ' <workflow id="%s" />\n' % item.id ) + elif item_type == panel_item_types.LABEL: + label_id = item.id or '' + label_text = item.text or '' + label_version = item.version or '' + os.write( fd, ' <label id="%s" text="%s" version="%s" />\n' % ( label_id, label_text, label_version ) ) + elif item_type == panel_item_types.SECTION: + section_id = item.id or '' + section_name = item.name or '' + section_version = item.version or '' + os.write( fd, ' <section id="%s" name="%s" version="%s">\n' % ( section_id, section_name, section_version ) ) + for section_key, section_item_type, section_item in item.panel_items_iter(): + if section_item_type == panel_item_types.TOOL: + if section_item: + os.write( fd, ' <tool id="%s" />\n' % section_item.id ) + elif section_item_type == panel_item_types.WORKFLOW: + if section_item: + os.write( fd, ' <workflow id="%s" />\n' % section_item.id ) + elif section_item_type == panel_item_types.LABEL: + if section_item: + label_id = section_item.id or '' + label_text = section_item.text or '' + label_version = section_item.version or '' + os.write( fd, ' <label id="%s" text="%s" version="%s" />\n' % ( label_id, label_text, label_version ) ) + os.write( fd, ' </section>\n' ) + os.write( fd, '</toolbox>\n' ) + os.close( fd ) + shutil.move( filename, os.path.abspath( self._integrated_tool_panel_config ) ) + os.chmod( self._integrated_tool_panel_config, 0644 ) Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.