1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/20a8a71805a6/ Changeset: 20a8a71805a6 User: greg Date: 2014-03-27 21:03:53 Summary: Add the new repository type "Repository suite definition". This new type is associated with repositories that will contain a single file named repository_dependencies.xml that defines simple repository dependency relationships to a collection of repositories that contain related Galaxy utilities. The chemicaltoolbox repository is an example. Affected #: 14 files diff -r f4d81f13120a8995b3486bd05c00d0430f2d44c7 -r 20a8a71805a64f9718d03b986ae767c61f2cf035 lib/galaxy/webapps/tool_shed/controllers/repository.py --- a/lib/galaxy/webapps/tool_shed/controllers/repository.py +++ b/lib/galaxy/webapps/tool_shed/controllers/repository.py @@ -2346,9 +2346,15 @@ key_rd_dicts_to_be_processed=None, all_repository_dependencies=None, handled_key_rd_dicts=None ) - if str( repository.type ) != rt_util.TOOL_DEPENDENCY_DEFINITION: + if str( repository.type ) != rt_util.REPOSITORY_SUITE_DEFINITION: # Handle messaging for resetting repository type to the optimal value. - change_repository_type_message = tool_dependency_util.generate_message_for_repository_type_change( trans, repository ) + change_repository_type_message = suc.generate_message_for_repository_type_change( trans, repository ) + if change_repository_type_message: + message += change_repository_type_message + status = 'warning' + elif str( repository.type ) != rt_util.TOOL_DEPENDENCY_DEFINITION: + # Handle messaging for resetting repository type to the optimal value. + change_repository_type_message = suc.generate_message_for_repository_type_change( trans, repository ) if change_repository_type_message: message += change_repository_type_message status = 'warning' @@ -2496,8 +2502,7 @@ """ Handle a request from a Galaxy instance where the changeset_revision defined for a repository in a dependency definition file is older than the changeset_revision associated with the installed - repository. This will occur with repository's of type tool_dependency_definition, and this scenario - will occur while repository dependency hierarchies are bing installed. + repository. """ name = kwd.get( 'name', None ) owner = kwd.get( 'owner', None ) diff -r f4d81f13120a8995b3486bd05c00d0430f2d44c7 -r 20a8a71805a64f9718d03b986ae767c61f2cf035 lib/galaxy/webapps/tool_shed/controllers/upload.py --- a/lib/galaxy/webapps/tool_shed/controllers/upload.py +++ b/lib/galaxy/webapps/tool_shed/controllers/upload.py @@ -123,16 +123,40 @@ istar = False if istar: ok, message, files_to_remove, content_alert_str, undesirable_dirs_removed, undesirable_files_removed = \ - self.upload_tar( trans, repository, tar, uploaded_file, upload_point, remove_repo_files_not_in_tar, commit_message, new_repo_alert ) + self.upload_tar( trans, + repository, + tar, + uploaded_file, + upload_point, + remove_repo_files_not_in_tar, + commit_message, + new_repo_alert ) elif uploaded_directory: ok, message, files_to_remove, content_alert_str, undesirable_dirs_removed, undesirable_files_removed = \ - self.upload_directory( trans, repository, uploaded_directory, upload_point, remove_repo_files_not_in_tar, commit_message, new_repo_alert ) + self.upload_directory( trans, + repository, + uploaded_directory, + upload_point, + remove_repo_files_not_in_tar, + commit_message, + new_repo_alert ) else: if ( isgzip or isbz2 ) and uncompress_file: - uploaded_file_filename = commit_util.uncompress( repository, uploaded_file_name, uploaded_file_filename, isgzip=isgzip, isbz2=isbz2 ) - if repository.type == rt_util.TOOL_DEPENDENCY_DEFINITION and uploaded_file_filename != suc.TOOL_DEPENDENCY_DEFINITION_FILENAME: + uploaded_file_filename = commit_util.uncompress( repository, + uploaded_file_name, + uploaded_file_filename, + isgzip=isgzip, + isbz2=isbz2 ) + if repository.type == rt_util.REPOSITORY_SUITE_DEFINITION and \ + uploaded_file_filename != suc.REPOSITORY_DEPENDENCY_DEFINITION_FILENAME: ok = False - message = 'Repositories of type <b>Tool dependency definition</b> can only contain a single file named <b>tool_dependencies.xml</b>.' + message = 'Repositories of type <b>Repository suite definition</b> can only contain a single file named ' + message += '<b>repository_dependencies.xml</b>.' + elif repository.type == rt_util.TOOL_DEPENDENCY_DEFINITION and \ + uploaded_file_filename != suc.TOOL_DEPENDENCY_DEFINITION_FILENAME: + ok = False + message = 'Repositories of type <b>Tool dependency definition</b> can only contain a single file named ' + message += '<b>tool_dependencies.xml</b>.' if ok: if upload_point is not None: full_path = os.path.abspath( os.path.join( repo_dir, upload_point, uploaded_file_filename ) ) @@ -140,10 +164,12 @@ full_path = os.path.abspath( os.path.join( repo_dir, uploaded_file_filename ) ) # Move some version of the uploaded file to the load_point within the repository hierarchy. if uploaded_file_filename in [ suc.REPOSITORY_DEPENDENCY_DEFINITION_FILENAME ]: - # Inspect the contents of the file to see if changeset_revision values are missing and if so, set them appropriately. - altered, root_elem, error_message = commit_util.handle_repository_dependencies_definition( trans, - uploaded_file_name, - unpopulate=False ) + # Inspect the contents of the file to see if changeset_revision values are missing and if so, + # set them appropriately. + altered, root_elem, error_message = \ + commit_util.handle_repository_dependencies_definition( trans, + uploaded_file_name, + unpopulate=False ) if error_message: ok = False message = error_message @@ -153,10 +179,12 @@ shutil.move( tmp_filename, full_path ) else: shutil.move( uploaded_file_name, full_path ) - elif uploaded_file_filename in [ suc.TOOL_DEPENDENCY_DEFINITION_FILENAME ]: - # Inspect the contents of the file to see if it defines a complex repository dependency definition whose changeset_revision values - # are missing and if so, set them appropriately. - altered, root_elem, error_message = commit_util.handle_tool_dependencies_definition( trans, uploaded_file_name ) + elif uploaded_file_filename in [ suc.REPOSITORY_DEPENDENCY_DEFINITION_FILENAME, + suc.TOOL_DEPENDENCY_DEFINITION_FILENAME ]: + # Inspect the contents of the file to see if changeset_revision values are + # missing and if so, set them appropriately. + altered, root_elem, error_message = \ + commit_util.handle_tool_dependencies_definition( trans, uploaded_file_name ) if error_message: ok = False message = error_message @@ -170,8 +198,8 @@ else: shutil.move( uploaded_file_name, full_path ) if ok: - # See if any admin users have chosen to receive email alerts when a repository is updated. If so, check every uploaded file to ensure - # content is appropriate. + # See if any admin users have chosen to receive email alerts when a repository is updated. + # If so, check every uploaded file to ensure content is appropriate. check_contents = commit_util.check_file_contents_for_email_alerts( trans ) if check_contents and os.path.isfile( full_path ): content_alert_str = commit_util.check_file_content_for_html_and_images( full_path ) @@ -182,8 +210,9 @@ full_path = full_path.encode( 'ascii', 'replace' ) commands.commit( repo.ui, repo, full_path, user=trans.user.username, message=commit_message ) if full_path.endswith( 'tool_data_table_conf.xml.sample' ): - # Handle the special case where a tool_data_table_conf.xml.sample file is being uploaded by parsing the file and adding new entries - # to the in-memory trans.app.tool_data_tables dictionary. + # Handle the special case where a tool_data_table_conf.xml.sample file is being uploaded + # by parsing the file and adding new entries to the in-memory trans.app.tool_data_tables + # dictionary. error, error_message = tool_util.handle_sample_tool_data_table_conf_file( trans.app, full_path ) if error: message = '%s<br/>%s' % ( message, error_message ) @@ -212,43 +241,54 @@ else: source_type = "file" source = uploaded_file_filename - message = "The %s <b>%s</b> has been successfully%suploaded to the repository. " % ( source_type, source, uncompress_str ) + message = "The %s <b>%s</b> has been successfully%suploaded to the repository. " % \ + ( source_type, source, uncompress_str ) if istar and ( undesirable_dirs_removed or undesirable_files_removed ): items_removed = undesirable_dirs_removed + undesirable_files_removed - message += " %d undesirable items (.hg .svn .git directories, .DS_Store, hgrc files, etc) were removed from the archive. " % items_removed + message += " %d undesirable items (.hg .svn .git directories, .DS_Store, hgrc files, etc) " % items_removed + message += "were removed from the archive. " if istar and remove_repo_files_not_in_tar and files_to_remove: if upload_point is not None: - message += " %d files were removed from the repository relative to the selected upload point '%s'. " % ( len( files_to_remove ), upload_point ) + message += " %d files were removed from the repository relative to the selected upload point '%s'. " % \ + ( len( files_to_remove ), upload_point ) else: message += " %d files were removed from the repository root. " % len( files_to_remove ) kwd[ 'message' ] = message metadata_util.set_repository_metadata_due_to_new_tip( trans, repository, content_alert_str=content_alert_str, **kwd ) if repository.metadata_revisions: - # A repository's metadata revisions are order descending by update_time, so the zeroth revision will be the tip just after an upload. - metadata_dict = repository.metadata_revisions[0].metadata + # A repository's metadata revisions are order descending by update_time, so the zeroth revision + # will be the tip just after an upload. + metadata_dict = repository.metadata_revisions[ 0 ].metadata else: metadata_dict = {} - if str( repository.type ) != rt_util.TOOL_DEPENDENCY_DEFINITION: - change_repository_type_message = tool_dependency_util.generate_message_for_repository_type_change( trans, repository ) + if str( repository.type ) not in [ rt_util.REPOSITORY_SUITE_DEFINITION, + rt_util.TOOL_DEPENDENCY_DEFINITION ]: + change_repository_type_message = suc.generate_message_for_repository_type_change( trans, repository ) if change_repository_type_message: message += change_repository_type_message status = 'warning' else: - # Provide a warning message if a tool_dependencies.xml file is provided, but tool dependencies weren't loaded due to a requirement tag mismatch - # or some other problem. Tool dependency definitions can define orphan tool dependencies (no relationship to any tools contained in the repository), - # so warning messages are important because orphans are always valid. The repository owner must be warned in case they did not intend to define an - # orphan dependency, but simply provided incorrect information (tool shed, name owner, changeset_revision) for the definition. - orphan_message = tool_dependency_util.generate_message_for_orphan_tool_dependencies( trans, repository, metadata_dict ) + # Provide a warning message if a tool_dependencies.xml file is provided, but tool dependencies + # weren't loaded due to a requirement tag mismatch or some other problem. Tool dependency + # definitions can define orphan tool dependencies (no relationship to any tools contained in the + # repository), so warning messages are important because orphans are always valid. The repository + # owner must be warned in case they did not intend to define an orphan dependency, but simply + # provided incorrect information (tool shed, name owner, changeset_revision) for the definition. + orphan_message = tool_dependency_util.generate_message_for_orphan_tool_dependencies( trans, + repository, + metadata_dict ) if orphan_message: message += orphan_message status = 'warning' # Handle messaging for invalid tool dependencies. - invalid_tool_dependencies_message = tool_dependency_util.generate_message_for_invalid_tool_dependencies( metadata_dict ) + invalid_tool_dependencies_message = \ + tool_dependency_util.generate_message_for_invalid_tool_dependencies( metadata_dict ) if invalid_tool_dependencies_message: message += invalid_tool_dependencies_message status = 'error' # Handle messaging for invalid repository dependencies. - invalid_repository_dependencies_message = repository_dependency_util.generate_message_for_invalid_repository_dependencies( metadata_dict ) + invalid_repository_dependencies_message = \ + repository_dependency_util.generate_message_for_invalid_repository_dependencies( metadata_dict ) if invalid_repository_dependencies_message: message += invalid_repository_dependencies_message status = 'error' @@ -279,7 +319,8 @@ message=message, status=status ) - def upload_directory( self, trans, repository, uploaded_directory, upload_point, remove_repo_files_not_in_tar, commit_message, new_repo_alert ): + def upload_directory( self, trans, repository, uploaded_directory, upload_point, remove_repo_files_not_in_tar, + commit_message, new_repo_alert ): repo_dir = repository.repo_path( trans.app ) repo = hg.repository( suc.get_configured_ui(), repo_dir ) undesirable_dirs_removed = 0 @@ -292,7 +333,9 @@ for root, dirs, files in os.walk( uploaded_directory ): for uploaded_file in files: relative_path = os.path.normpath( os.path.join( os.path.relpath( root, uploaded_directory ), uploaded_file ) ) - if repository.type == rt_util.TOOL_DEPENDENCY_DEFINITION: + if repository.type == rt_util.REPOSITORY_SUITE_DEFINITION: + ok = os.path.basename( uploaded_file ) == suc.REPOSITORY_DEPENDENCY_DEFINITION_FILENAME + elif repository.type == rt_util.TOOL_DEPENDENCY_DEFINITION: ok = os.path.basename( uploaded_file ) == suc.TOOL_DEPENDENCY_DEFINITION_FILENAME else: ok = os.path.basename( uploaded_file ) not in commit_util.UNDESIRABLE_FILES @@ -307,10 +350,12 @@ if ok: uploaded_file_name = os.path.abspath( os.path.join( root, uploaded_file ) ) if os.path.split( uploaded_file_name )[ -1 ] == suc.REPOSITORY_DEPENDENCY_DEFINITION_FILENAME: - # Inspect the contents of the file to see if changeset_revision values are missing and if so, set them appropriately. - altered, root_elem, error_message = commit_util.handle_repository_dependencies_definition( trans, - uploaded_file_name, - unpopulate=False ) + # Inspect the contents of the file to see if changeset_revision values are missing and + # if so, set them appropriately. + altered, root_elem, error_message = \ + commit_util.handle_repository_dependencies_definition( trans, + uploaded_file_name, + unpopulate=False ) if error_message: return False, error_message, [], '', [], [] elif altered: diff -r f4d81f13120a8995b3486bd05c00d0430f2d44c7 -r 20a8a71805a64f9718d03b986ae767c61f2cf035 lib/galaxy/webapps/tool_shed/framework/middleware/hg.py --- a/lib/galaxy/webapps/tool_shed/framework/middleware/hg.py +++ b/lib/galaxy/webapps/tool_shed/framework/middleware/hg.py @@ -120,8 +120,28 @@ for row in result_set: # Should only be 1 row... repository_type = str( row[ 'type' ] ) - if repository_type == rt_util.TOOL_DEPENDENCY_DEFINITION: - # Handle repositories of type tool_dependency_definition, which can only contain a single file named tool_dependencies.xml. + if repository_type == rt_util.REPOSITORY_SUITE_DEFINITION: + # Handle repositories of type repository_suite_definition, which can only contain a single + # file named repository_dependencies.xml. + for entry in changeset_groups: + if len( entry ) == 2: + # We possibly found an altered file entry. + filename, change_list = entry + if filename and isinstance( filename, str ): + if filename == suc.REPOSITORY_DEPENDENCY_DEFINITION_FILENAME: + # Make sure the any complex repository dependency definitions contain valid <repository> tags. + is_valid, error_msg = commit_util.repository_tags_are_valid( filename, change_list ) + if not is_valid: + log.debug( error_msg ) + return self.__display_exception_remotely( start_response, error_msg ) + else: + msg = "Only a single file named repository_dependencies.xml can be pushed to a repository " + msg += "of type 'Repository suite definition'." + log.debug( msg ) + return self.__display_exception_remotely( start_response, msg ) + elif repository_type == rt_util.TOOL_DEPENDENCY_DEFINITION: + # Handle repositories of type tool_dependency_definition, which can only contain a single + # file named tool_dependencies.xml. for entry in changeset_groups: if len( entry ) == 2: # We possibly found an altered file entry. @@ -134,20 +154,24 @@ log.debug( error_msg ) return self.__display_exception_remotely( start_response, error_msg ) else: - msg = "Only a single file named tool_dependencies.xml can be pushed to a repository of type 'Tool dependency definition'." + msg = "Only a single file named tool_dependencies.xml can be pushed to a repository " + msg += "of type 'Tool dependency definition'." log.debug( msg ) return self.__display_exception_remotely( start_response, msg ) else: - # If the changeset includes changes to dependency definition files, make sure tag sets are not missing "toolshed" or - # "changeset_revision" attributes since automatically populating them is not supported when pushing from the command line. - # These attributes are automatically populated only when using the tool shed upload utility. + # If the changeset includes changes to dependency definition files, make sure tag sets + # are not missing "toolshed" or "changeset_revision" attributes since automatically populating + # them is not supported when pushing from the command line. These attributes are automatically + # populated only when using the tool shed upload utility. for entry in changeset_groups: if len( entry ) == 2: # We possibly found an altered file entry. filename, change_list = entry if filename and isinstance( filename, str ): - if filename in [ suc.REPOSITORY_DEPENDENCY_DEFINITION_FILENAME, suc.TOOL_DEPENDENCY_DEFINITION_FILENAME ]: - # We check both files since tool dependency definitions files can contain complex repository dependency definitions. + if filename in [ suc.REPOSITORY_DEPENDENCY_DEFINITION_FILENAME, + suc.TOOL_DEPENDENCY_DEFINITION_FILENAME ]: + # We check both files since tool dependency definitions files can contain complex + # repository dependency definitions. is_valid, error_msg = commit_util.repository_tags_are_valid( filename, change_list ) if not is_valid: log.debug( error_msg ) diff -r f4d81f13120a8995b3486bd05c00d0430f2d44c7 -r 20a8a71805a64f9718d03b986ae767c61f2cf035 lib/galaxy/webapps/tool_shed/model/__init__.py --- a/lib/galaxy/webapps/tool_shed/model/__init__.py +++ b/lib/galaxy/webapps/tool_shed/model/__init__.py @@ -194,6 +194,8 @@ return False def can_change_type_to( self, app, new_type_label ): + if self.type == new_type_label: + return False if self.can_change_type( app ): new_type = app.repository_types_registry.get_class_by_label( new_type_label ) if new_type.is_valid_for_type( app, self ): diff -r f4d81f13120a8995b3486bd05c00d0430f2d44c7 -r 20a8a71805a64f9718d03b986ae767c61f2cf035 lib/tool_shed/galaxy_install/repository_util.py --- a/lib/tool_shed/galaxy_install/repository_util.py +++ b/lib/tool_shed/galaxy_install/repository_util.py @@ -194,7 +194,7 @@ if not repository_metadata: # The received changeset_revision is no longer installable, so get the next changeset_revision # in the repository's changelog. This generally occurs only with repositories of type - # tool_dependency_definition. + # repository_suite_definition or tool_dependency_definition. next_downloadable_changeset_revision = \ suc.get_next_downloadable_changeset_revision( repository,repo, changeset_revision ) if next_downloadable_changeset_revision: diff -r f4d81f13120a8995b3486bd05c00d0430f2d44c7 -r 20a8a71805a64f9718d03b986ae767c61f2cf035 lib/tool_shed/galaxy_install/tool_dependencies/install_util.py --- a/lib/tool_shed/galaxy_install/tool_dependencies/install_util.py +++ b/lib/tool_shed/galaxy_install/tool_dependencies/install_util.py @@ -211,12 +211,13 @@ tool_dependency_name=package_name, tool_dependency_version=package_version ) if os.path.exists( dependent_install_dir ): - # The install manager handles tool migration stages and the sync_database_with_file_system() method handles two - # scenarios: (1) where a Galaxy file system environment related to installed tool shed repositories and tool dependencies - # has somehow (over time )gotten out of sync with the Galaxy database tables associated with these installed items, and - # (2) the Tool Shed's install and test framework which installs repositories in 2 stages, those of type - # tool_dependency_definition followed by those containing valid tools and tool functional test components. Neither of - # these scenarios apply when the install manager is running. + # The install manager handles tool migration stages and the sync_database_with_file_system() + # method handles two scenarios: (1) where a Galaxy file system environment related to installed + # Tool Shed repositories and tool dependencies has somehow gotten out of sync with the Galaxy + # database tables associated with these installed items, and (2) the Tool Shed's install and test + # framework which installs repositories in 2 stages, those of type tool_dependency_definition + # followed by those containing valid tools and tool functional test components. Neither of these + # scenarios apply when the install manager is running. if from_install_manager: can_install_tool_dependency = True else: @@ -334,12 +335,13 @@ tool_dependency_name=package_name, tool_dependency_version=package_version ) if os.path.exists( install_dir ): - # The install manager handles tool migration stages and the sync_database_with_file_system() method handles two - # scenarios: (1) where a Galaxy file system environment related to installed tool shed repositories and tool dependencies - # has somehow (over time )gotten out of sync with the Galaxy database tables associated with these installed items, and - # (2) the Tool Shed's install and test framework which installs repositories in 2 stages, those of type - # tool_dependency_definition followed by those containing valid tools and tool functional test components. Neither of - # these scenarios apply when the install manager is running. + # The install manager handles tool migration stages and the sync_database_with_file_system() + # method handles two scenarios: (1) where a Galaxy file system environment related to installed + # Tool Shed repositories and tool dependencies has somehow gotten out of sync with the Galaxy + # database tables associated with these installed items, and (2) the Tool Shed's install and test + # framework which installs repositories in 2 stages, those of type tool_dependency_definition + # followed by those containing valid tools and tool functional test components. Neither of these + # scenarios apply when the install manager is running. if from_install_manager: can_install_tool_dependency = True else: diff -r f4d81f13120a8995b3486bd05c00d0430f2d44c7 -r 20a8a71805a64f9718d03b986ae767c61f2cf035 lib/tool_shed/grids/repository_grids.py --- a/lib/tool_shed/grids/repository_grids.py +++ b/lib/tool_shed/grids/repository_grids.py @@ -175,7 +175,8 @@ log.exception( str( e ) ) return 'unknown' else: - # Here repository.type must be rt_util.TOOL_DEPENDENCY_DEFINITION. + # Here repository.type must be either rt_util.REPOSITORY_SUITE_DEFINITION or + # rt_util.TOOL_DEPENDENCY_DEFINITION. try: if len( repository.metadata_revisions ) > 0: displayed_metadata_revision = repository.metadata_revisions[ -1 ] @@ -1771,7 +1772,7 @@ """ Inspect the latest downloadable changeset revision for the received repository to see if it includes at least 1 tool that has at least 1 failing test. This will filter out repositories - of type tool_dependency_definition. + of type repository_suite_definition and tool_dependency_definition. """ repository_metadata = get_latest_downloadable_repository_metadata_if_it_includes_tools( trans, repository ) if repository_metadata \ @@ -1787,7 +1788,8 @@ Inspect the latest downloadable changeset revision for the received repository to see if it includes tools that are either missing functional tests or functional test data. If the changset revision includes tools but is missing tool test components, return the changeset - revision hash. This will filter out repositories of type tool_dependency_definition. + revision hash. This will filter out repositories of type repository_suite_definition and + tool_dependency_definition. """ repository_metadata = get_latest_downloadable_repository_metadata_if_it_includes_tools( trans, repository ) if repository_metadata and repository_metadata.missing_test_components: @@ -1797,7 +1799,8 @@ def filter_by_latest_downloadable_changeset_revision_that_has_no_failing_tool_tests( trans, repository ): """ Inspect the latest downloadable changeset revision for the received repository to see if it - includes tools with no failing tests. This will filter out repositories of type tool_dependency_definition. + includes tools with no failing tests. This will filter out repositories of type repository_suite_definition + and tool_dependency_definition. """ repository_metadata = get_latest_downloadable_repository_metadata_if_it_includes_tools( trans, repository ) if repository_metadata is not None and \ @@ -1809,7 +1812,8 @@ def filter_by_latest_metadata_changeset_revision_that_has_invalid_tools( trans, repository ): """ Inspect the latest changeset revision with associated metadata for the received repository - to see if it has invalid tools. This will filter out repositories of type tool_dependency_definition. + to see if it has invalid tools. This will filter out repositories of type repository_suite_definition + and tool_dependency_definition. """ repository_metadata = get_latest_repository_metadata_if_it_includes_invalid_tools( trans, repository ) if repository_metadata is not None: @@ -1820,7 +1824,7 @@ """ Inspect the latest downloadable changeset revision for the received repository to see if it has tool test installation errors. This will return repositories of type unrestricted - as well as type tool_dependency_definition. + as well as types repository_suite_definition and tool_dependency_definition. """ repository_metadata = get_latest_downloadable_repository_metadata_if_it_has_test_install_errors( trans, repository ) # Filter further by eliminating repositories that are missing test components. @@ -1831,7 +1835,8 @@ def filter_by_latest_downloadable_changeset_revision_with_skip_tests_checked( trans, repository ): """ Inspect the latest downloadable changeset revision for the received repository to see if skip tests - is checked. This will return repositories of type unrestricted as well as type tool_dependency_definition. + is checked. This will return repositories of type unrestricted as well as types repository_suite_definition + and tool_dependency_definition. """ repository_metadata = get_latest_downloadable_repository_metadata( trans, repository ) # The skip_tool_tests attribute is a SkipToolTest table mapping backref to the RepositoryMetadata table. @@ -1842,7 +1847,8 @@ def get_latest_downloadable_repository_metadata( trans, repository ): """ Return the latest downloadable repository_metadata record for the received repository. This will - return repositories of type unrestricted as well as type tool_dependency_definition. + return repositories of type unrestricted as well as types repository_suite_definition and + tool_dependency_definition. """ encoded_repository_id = trans.security.encode_id( repository.id ) repo = hg.repository( suc.get_configured_ui(), repository.repo_path( trans.app ) ) @@ -1867,7 +1873,8 @@ def get_latest_downloadable_repository_metadata_if_it_includes_tools( trans, repository ): """ Return the latest downloadable repository_metadata record for the received repository if its - includes_tools attribute is True. This will filter out repositories of type tool_dependency_definition. + includes_tools attribute is True. This will filter out repositories of type repository_suite_definition + and tool_dependency_definition. """ repository_metadata = get_latest_downloadable_repository_metadata( trans, repository ) if repository_metadata is not None and repository_metadata.includes_tools: @@ -1878,7 +1885,7 @@ """ Return the latest downloadable repository_metadata record for the received repository if its test_install_error attribute is True. This will return repositories of type unrestricted as - well as type tool_dependency_definition. + well as types repository_suite_definition and tool_dependency_definition. """ repository_metadata = get_latest_downloadable_repository_metadata( trans, repository ) if repository_metadata is not None and repository_metadata.test_install_error: @@ -1888,7 +1895,8 @@ def get_latest_repository_metadata( trans, repository ): """ Return the latest repository_metadata record for the received repository if it exists. This will - return repositories of type unrestricted as well as type tool_dependency_definition. + return repositories of type unrestricted as well as types repository_suite_definition and + tool_dependency_definition. """ encoded_repository_id = trans.security.encode_id( repository.id ) repo = hg.repository( suc.get_configured_ui(), repository.repo_path( trans.app ) ) @@ -1908,7 +1916,8 @@ def get_latest_repository_metadata_if_it_includes_invalid_tools( trans, repository ): """ Return the latest repository_metadata record for the received repository that contains invalid - tools if one exists. This will filter out repositories of type tool_dependency_definition. + tools if one exists. This will filter out repositories of type repository_suite_definition and + tool_dependency_definition. """ repository_metadata = get_latest_repository_metadata( trans, repository ) if repository_metadata is not None: diff -r f4d81f13120a8995b3486bd05c00d0430f2d44c7 -r 20a8a71805a64f9718d03b986ae767c61f2cf035 lib/tool_shed/repository_types/registry.py --- a/lib/tool_shed/repository_types/registry.py +++ b/lib/tool_shed/repository_types/registry.py @@ -1,5 +1,6 @@ import logging import unrestricted +import repository_suite_definition import tool_dependency_definition from galaxy.util.odict import odict @@ -11,6 +12,7 @@ def __init__( self ): self.repository_types_by_label = odict() self.repository_types_by_label[ 'unrestricted' ] = unrestricted.Unrestricted() + self.repository_types_by_label[ 'repository_suite_definition' ] = repository_suite_definition.RepositorySuiteDefinition() self.repository_types_by_label[ 'tool_dependency_definition' ] = tool_dependency_definition.ToolDependencyDefinition() def get_class_by_label( self, label ): diff -r f4d81f13120a8995b3486bd05c00d0430f2d44c7 -r 20a8a71805a64f9718d03b986ae767c61f2cf035 lib/tool_shed/repository_types/repository_suite_definition.py --- /dev/null +++ b/lib/tool_shed/repository_types/repository_suite_definition.py @@ -0,0 +1,41 @@ +import logging +from tool_shed.repository_types.metadata import TipOnly +import tool_shed.repository_types.util as rt_util +import tool_shed.util.shed_util_common as suc + +from galaxy import eggs +eggs.require( 'mercurial' ) +from mercurial import hg +from mercurial import ui + +log = logging.getLogger( __name__ ) + +class RepositorySuiteDefinition( TipOnly ): + + def __init__( self ): + self.type = rt_util.REPOSITORY_SUITE_DEFINITION + self.label = 'Repository suite definition' + self.valid_file_names = [ 'repository_dependencies.xml' ] + + def is_valid_for_type( self, app, repository, revisions_to_check=None ): + """ + Inspect the received repository's contents to determine if they abide by the rules defined for + the contents of this type. If the received revisions_to_check is a list of changeset revisions, + then inspection will be restricted to the revisions in the list. + """ + repo = hg.repository( ui.ui(), repository.repo_path( app ) ) + if revisions_to_check: + changeset_revisions = revisions_to_check + else: + changeset_revisions = repo.changelog + for changeset in changeset_revisions: + changeset_revision = str( repo.changectx( changeset ) ) + ctx = repo.changectx( changeset ) + # Inspect all files in the changeset (in sorted order) to make sure there is only one and it + # is named repository_dependencies.xml. + files_changed_in_changeset = ctx.files() + for file_path in files_changed_in_changeset: + file_name = suc.strip_path( file_path ) + if file_name not in self.valid_file_names: + return False + return True diff -r f4d81f13120a8995b3486bd05c00d0430f2d44c7 -r 20a8a71805a64f9718d03b986ae767c61f2cf035 lib/tool_shed/repository_types/util.py --- a/lib/tool_shed/repository_types/util.py +++ b/lib/tool_shed/repository_types/util.py @@ -4,6 +4,7 @@ log = logging.getLogger( __name__ ) UNRESTRICTED = 'unrestricted' +REPOSITORY_SUITE_DEFINITION = 'repository_suite_definition' TOOL_DEPENDENCY_DEFINITION = 'tool_dependency_definition' types = [ UNRESTRICTED, TOOL_DEPENDENCY_DEFINITION ] diff -r f4d81f13120a8995b3486bd05c00d0430f2d44c7 -r 20a8a71805a64f9718d03b986ae767c61f2cf035 lib/tool_shed/util/commit_util.py --- a/lib/tool_shed/util/commit_util.py +++ b/lib/tool_shed/util/commit_util.py @@ -50,6 +50,9 @@ if member.name in [ 'hgrc' ]: message = "Uploaded archives cannot contain hgrc files." return False, message + if repository.type == rt_util.REPOSITORY_SUITE_DEFINITION and member.name != suc.REPOSITORY_DEPENDENCY_DEFINITION_FILENAME: + message = 'Repositories of type <b>Repsoitory suite definition</b> can contain only a single file named <b>repository_dependencies.xml</b>.' + return False, message if repository.type == rt_util.TOOL_DEPENDENCY_DEFINITION and member.name != suc.TOOL_DEPENDENCY_DEFINITION_FILENAME: message = 'Repositories of type <b>Tool dependency definition</b> can contain only a single file named <b>tool_dependencies.xml</b>.' return False, message @@ -478,8 +481,8 @@ def repository_tag_is_valid( filename, line ): """ - Checks changes made to <repository> tags in a dependency definition file being pushed to the tool shed from the command line to ensure that - all required attributes exist. + Checks changes made to <repository> tags in a dependency definition file being pushed to the + Tool Shed from the command line to ensure that all required attributes exist. """ required_attributes = [ 'toolshed', 'name', 'owner', 'changeset_revision' ] defined_attributes = line.split() @@ -490,15 +493,17 @@ defined = True break if not defined: - error_msg = 'The %s file contains a <repository> tag that is missing the required attribute %s. ' % ( filename, required_attribute ) - error_msg += 'Automatically populating dependency definition attributes occurs only when using the tool shed upload utility. ' + error_msg = 'The %s file contains a <repository> tag that is missing the required attribute %s. ' % \ + ( filename, required_attribute ) + error_msg += 'Automatically populating dependency definition attributes occurs only when using ' + error_msg += 'the Tool Shed upload utility. ' return False, error_msg return True, '' def repository_tags_are_valid( filename, change_list ): """ - Make sure the any complex repository dependency definitions contain valid <repository> tags when pushing changes to the tool shed on the command - line. + Make sure the any complex repository dependency definitions contain valid <repository> tags when pushing + changes to the tool shed on the command line. """ tag = '<repository' for change_dict in change_list: @@ -519,8 +524,9 @@ def unpack_chunks( hg_unbundle10_obj ): """ - This method provides a generator of parsed chunks of a "group" in a mercurial unbundle10 object which is created when a changeset that is pushed - to a tool shed repository using hg push from the command line is read using readbundle. + This method provides a generator of parsed chunks of a "group" in a mercurial unbundle10 object which + is created when a changeset that is pushed to a Tool Shed repository using hg push from the command line + is read using readbundle. """ while True: length, = struct.unpack( '>l', readexactly( hg_unbundle10_obj, 4 ) ) @@ -538,8 +544,9 @@ def unpack_groups( hg_unbundle10_obj ): """ - This method provides a generator of parsed groups from a mercurial unbundle10 object which is created when a changeset that is pushed - to a tool shed repository using hg push from the command line is read using readbundle. + This method provides a generator of parsed groups from a mercurial unbundle10 object which is + created when a changeset that is pushed to a Tool Shed repository using hg push from the command + line is read using readbundle. """ # Process the changelog group. yield [ chunk for chunk in unpack_chunks( hg_unbundle10_obj ) ] @@ -556,8 +563,8 @@ def unpack_patches( hg_unbundle10_obj, remaining ): """ - This method provides a generator of patches from the data field in a chunk. As there is no delimiter for this data field, a length argument is - required. + This method provides a generator of patches from the data field in a chunk. As there is no delimiter + for this data field, a length argument is required. """ while remaining >= 12: start, end, blocklen = struct.unpack( '>lll', readexactly( hg_unbundle10_obj, 12 ) ) diff -r f4d81f13120a8995b3486bd05c00d0430f2d44c7 -r 20a8a71805a64f9718d03b986ae767c61f2cf035 lib/tool_shed/util/shed_util_common.py --- a/lib/tool_shed/util/shed_util_common.py +++ b/lib/tool_shed/util/shed_util_common.py @@ -333,6 +333,25 @@ # Don't include the changeset_revision in clone urls. return url_join( toolshed, 'repos', owner, name ) +def generate_message_for_repository_type_change( trans, repository ): + message = '' + if repository.can_change_type_to( trans.app, rt_util.REPOSITORY_SUITE_DEFINITION ): + repository_suite_definition_type_class = \ + trans.app.repository_types_registry.get_class_by_label( rt_util.REPOSITORY_SUITE_DEFINITION ) + message += "This repository currently contains a single file named <b>%s</b>. If the intent of this repository is " % \ + REPOSITORY_DEPENDENCY_DEFINITION_FILENAME + message += "to define relationships to a collection of repositories that contain related Galaxy utilities with " + message += "no plans to add additional files, then consider setting its type to <b>%s</b>.<br/>" % \ + repository_suite_definition_type_class.label + elif repository.can_change_type_to( trans.app, rt_util.TOOL_DEPENDENCY_DEFINITION ): + tool_dependency_definition_type_class = \ + trans.app.repository_types_registry.get_class_by_label( rt_util.TOOL_DEPENDENCY_DEFINITION ) + message += "This repository currently contains a single file named <b>%s</b>. If additional files will " % \ + TOOL_DEPENDENCY_DEFINITION_FILENAME + message += "not be added to this repository, consider setting its type to <b>%s</b>.<br/>" % \ + tool_dependency_definition_type_class.label + return message + def generate_repository_info_elem( tool_shed, repository_name, changeset_revision, owner, parent_elem=None, **kwd ): """Create and return an ElementTree repository info Element.""" if parent_elem is None: @@ -880,15 +899,17 @@ a page. When called from either the Tool Shed or Galaxy API, order is False. """ if trans.webapp.name == 'tool_shed': - # When called from the Tool Shed API, the metadata is reset on all repositories of type tool_dependency_definition in addition - # to other selected repositories. + # When called from the Tool Shed API, the metadata is reset on all repositories of types + # repository_suite_definition and tool_dependency_definition in addition to other selected + # repositories. if my_writable: username = trans.user.username clause_list = [] for repository in trans.sa_session.query( trans.model.Repository ) \ .filter( trans.model.Repository.table.c.deleted == False ): - # Always reset metadata on all repositories of type tool_dependency_definition. - if repository.type == rt_util.TOOL_DEPENDENCY_DEFINITION: + # Always reset metadata on all repositories of types repository_suite_definition and + # tool_dependency_definition. + if repository.type in [ rt_util.REPOSITORY_SUITE_DEFINITION, rt_util.TOOL_DEPENDENCY_DEFINITION ]: clause_list.append( trans.model.Repository.table.c.id == repository.id ) else: allow_push = repository.allow_push( trans.app ) diff -r f4d81f13120a8995b3486bd05c00d0430f2d44c7 -r 20a8a71805a64f9718d03b986ae767c61f2cf035 lib/tool_shed/util/tool_dependency_util.py --- a/lib/tool_shed/util/tool_dependency_util.py +++ b/lib/tool_shed/util/tool_dependency_util.py @@ -198,17 +198,6 @@ message += set_environment_orphans_str return message -def generate_message_for_repository_type_change( trans, repository ): - message = '' - if repository.can_change_type_to( trans.app, rt_util.TOOL_DEPENDENCY_DEFINITION ): - tool_dependency_definition_type_class = \ - trans.app.repository_types_registry.get_class_by_label( rt_util.TOOL_DEPENDENCY_DEFINITION ) - message += "This repository currently contains a single file named <b>%s</b>. If additional files will " % \ - suc.TOOL_DEPENDENCY_DEFINITION_FILENAME - message += "not be added to this repository, then it's type should be set to <b>%s</b>.<br/>" % \ - tool_dependency_definition_type_class.label - return message - def get_download_url_for_platform( url_templates, platform_info_dict ): ''' Compare the dict returned by get_platform_info() with the values specified in the url_template element. Return @@ -583,7 +572,7 @@ """ # This method should be reached very rarely. It implies that either the Galaxy environment became corrupted (i.e., # the database records for installed tool dependencies is not synchronized with tool dependencies on disk) or the Tool - # Shed's install and test framework is running. The Tool Shed's install and test framework which installs repositories + # Shed's install and test framework is running. The Tool Shed's install and test framework installs repositories # in 2 stages, those of type tool_dependency_definition followed by those containing valid tools and tool functional # test components. log.debug( "Synchronizing the database with the file system..." ) diff -r f4d81f13120a8995b3486bd05c00d0430f2d44c7 -r 20a8a71805a64f9718d03b986ae767c61f2cf035 templates/webapps/tool_shed/repository/common.mako --- a/templates/webapps/tool_shed/repository/common.mako +++ b/templates/webapps/tool_shed/repository/common.mako @@ -191,6 +191,9 @@ %if util.UNRESTRICTED in repository_types: <li><b>Unrestricted</b> - contents can be any set of valid Galaxy utilities or files %endif + %if util.REPOSITORY_SUITE_DEFINITION in repository_types: + <li><b>Repository suite definition</b> - contents will always be restricted to one file named repository_dependencies.xml + %endif %if util.TOOL_DEPENDENCY_DEFINITION in repository_types: <li><b>Tool dependency definition</b> - contents will always be restricted to one file named tool_dependencies.xml %endif Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.