1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/cd8ca90e93a1/ Changeset: cd8ca90e93a1 User: greg Date: 2014-01-05 01:25:43 Summary: Set a repository and its dependent repositories created from a capsule to be defined as installable only if its creation resulted in no errors. Affected #: 7 files diff -r f11a729d42cb66753b8b1616ad625c1b2a306b3d -r cd8ca90e93a1f896ee6ff3ac56665e64373ace00 lib/galaxy/webapps/tool_shed/api/repositories.py --- a/lib/galaxy/webapps/tool_shed/api/repositories.py +++ b/lib/galaxy/webapps/tool_shed/api/repositories.py @@ -219,11 +219,12 @@ import_results_tups = repository_maintenance_util.create_repository_and_import_archive( trans, repository_status_info_dict, import_results_tups ) + import_util.check_status_and_reset_downloadable( trans, import_results_tups ) suc.remove_dir( file_path ) # NOTE: the order of installation is defined in import_results_tups, but order will be lost when transferred to return_dict. return_dict = {} for import_results_tup in import_results_tups: - name_owner, message = import_results_tup + ok, name_owner, message = import_results_tup name, owner = name_owner key = 'Archive of repository "%s" owned by "%s"' % ( str( name ), str( owner ) ) val = message.replace( '<b>', '"' ).replace( '</b>', '"' ) diff -r f11a729d42cb66753b8b1616ad625c1b2a306b3d -r cd8ca90e93a1f896ee6ff3ac56665e64373ace00 lib/galaxy/webapps/tool_shed/controllers/repository.py --- a/lib/galaxy/webapps/tool_shed/controllers/repository.py +++ b/lib/galaxy/webapps/tool_shed/controllers/repository.py @@ -1883,9 +1883,11 @@ # Add the capsule_file_name and encoded_file_path to the repository_status_info_dict. repository_status_info_dict[ 'capsule_file_name' ] = capsule_file_name repository_status_info_dict[ 'encoded_file_path' ] = encoded_file_path - import_results_tups = repository_maintenance_util.create_repository_and_import_archive( trans, - repository_status_info_dict, - import_results_tups ) + import_results_tups = \ + repository_maintenance_util.create_repository_and_import_archive( trans, + repository_status_info_dict, + import_results_tups ) + import_util.check_status_and_reset_downloadable( trans, import_results_tups ) suc.remove_dir( file_path ) return trans.fill_template( '/webapps/tool_shed/repository/import_capsule_results.mako', export_info_dict=export_info_dict, diff -r f11a729d42cb66753b8b1616ad625c1b2a306b3d -r cd8ca90e93a1f896ee6ff3ac56665e64373ace00 lib/tool_shed/util/import_util.py --- a/lib/tool_shed/util/import_util.py +++ b/lib/tool_shed/util/import_util.py @@ -21,6 +21,37 @@ log = logging.getLogger( __name__ ) +def check_status_and_reset_downloadable( trans, import_results_tups ): + """Check the status of each imported repository and set downloadable to False if errors.""" + flush = False + for import_results_tup in import_results_tups: + ok, name_owner, message = import_results_tup + name, owner = name_owner + if not ok: + repository = suc.get_repository_by_name_and_owner( trans.app, name, owner ) + # Do not allow the repository to be automatically installed if population resulted in errors. + tip_changeset_revision = repository.tip( trans.app ) + repository_metadata = suc.get_repository_metadata_by_changeset_revision( trans, + trans.security.encode_id( repository.id ), + tip_changeset_revision ) + if repository_metadata: + if repository_metadata.downloadable: + repository_metadata.downloadable = False + trans.sa_session.add( repository_metadata ) + if not flush: + flush = True + # Do not allow dependent repository revisions to be automatically installed if population + # resulted in errors. + dependent_downloadable_revisions = suc.get_dependent_downloadable_revisions( trans, repository_metadata ) + for dependent_downloadable_revision in dependent_downloadable_revisions: + if dependent_downloadable_revision.downloadable: + dependent_downloadable_revision.downloadable = False + trans.sa_session.add( dependent_downloadable_revision ) + if not flush: + flush = True + if flush: + trans.sa_session.flush() + def extract_capsule_files( trans, **kwd ): """Extract the uploaded capsule archive into a temporary location for inspection, validation and potential import.""" return_dict = {} @@ -243,13 +274,16 @@ commit_message, undesirable_dirs_removed, undesirable_files_removed ) + if error_message: + results_dict[ 'ok' ] = False + results_dict[ 'error_message' ] += error_message try: - metadata_util.set_repository_metadata_due_to_new_tip( trans, repository, content_alert_str=content_alert_str ) + metadata_util.set_repository_metadata_due_to_new_tip( trans, + repository, + content_alert_str=content_alert_str ) except Exception, e: log.debug( "Error setting metadata on repository %s created from imported archive %s: %s" % \ ( str( repository.name ), str( archive_file_name ), str( e ) ) ) - results_dict[ 'ok' ] = ok - results_dict[ 'error_message' ] += error_message else: archive.close() results_dict[ 'ok' ] = False diff -r f11a729d42cb66753b8b1616ad625c1b2a306b3d -r cd8ca90e93a1f896ee6ff3ac56665e64373ace00 lib/tool_shed/util/metadata_util.py --- a/lib/tool_shed/util/metadata_util.py +++ b/lib/tool_shed/util/metadata_util.py @@ -296,14 +296,19 @@ includes_tool_dependencies = True if 'workflows' in metadata_dict: includes_workflows = True - if has_repository_dependencies or has_repository_dependencies_only_if_compiling_contained_td or includes_datatypes or \ - includes_tools or includes_tool_dependencies or includes_workflows: + if has_repository_dependencies or \ + has_repository_dependencies_only_if_compiling_contained_td or \ + includes_datatypes or \ + includes_tools or \ + includes_tool_dependencies or \ + includes_workflows: downloadable = True else: downloadable = False repository_metadata = suc.get_repository_metadata_by_changeset_revision( trans, id, changeset_revision ) if repository_metadata: - # A repository metadata record already exists with the received changeset_revision, so we don't need to check the skip_tool_test table. + # A repository metadata record already exists with the received changeset_revision, so we don't need to + # check the skip_tool_test table. check_skip_tool_test = False repository_metadata.metadata = metadata_dict repository_metadata.downloadable = downloadable @@ -313,7 +318,8 @@ repository_metadata.includes_tool_dependencies = includes_tool_dependencies repository_metadata.includes_workflows = includes_workflows else: - # No repository_metadata record exists for the received changeset_revision, so we may need to update the skip_tool_test table. + # No repository_metadata record exists for the received changeset_revision, so we may need to update the + # skip_tool_test table. check_skip_tool_test = True repository_metadata = trans.model.RepositoryMetadata( repository_id=repository.id, changeset_revision=changeset_revision, @@ -324,7 +330,8 @@ includes_tools=includes_tools, includes_tool_dependencies=includes_tool_dependencies, includes_workflows=includes_workflows ) - # Always set the default values for the following columns. When resetting all metadata on a repository, this will reset the values. + # Always set the default values for the following columns. When resetting all metadata on a repository + # this will reset the values. repository_metadata.tools_functionally_correct = False repository_metadata.missing_test_components = False repository_metadata.test_install_error = False @@ -1738,7 +1745,8 @@ # NO_METADATA - no metadata for either ancestor or current, so continue from current # EQUAL - ancestor metadata is equivalent to current metadata, so continue from current # SUBSET - ancestor metadata is a subset of current metadata, so continue from current - # NOT_EQUAL_AND_NOT_SUBSET - ancestor metadata is neither equal to nor a subset of current metadata, so persist ancestor metadata. + # NOT_EQUAL_AND_NOT_SUBSET - ancestor metadata is neither equal to nor a subset of current + # metadata, so persist ancestor metadata. comparison = compare_changeset_revisions( trans, ancestor_changeset_revision, ancestor_metadata_dict, @@ -1750,7 +1758,11 @@ elif comparison == NOT_EQUAL_AND_NOT_SUBSET: metadata_changeset_revision = ancestor_changeset_revision metadata_dict = ancestor_metadata_dict - repository_metadata = create_or_update_repository_metadata( trans, id, repository, metadata_changeset_revision, metadata_dict ) + repository_metadata = create_or_update_repository_metadata( trans, + id, + repository, + metadata_changeset_revision, + metadata_dict ) changeset_revisions.append( metadata_changeset_revision ) ancestor_changeset_revision = current_changeset_revision ancestor_metadata_dict = current_metadata_dict @@ -1762,7 +1774,11 @@ metadata_changeset_revision = current_changeset_revision metadata_dict = current_metadata_dict # We're at the end of the change log. - repository_metadata = create_or_update_repository_metadata( trans, id, repository, metadata_changeset_revision, metadata_dict ) + repository_metadata = create_or_update_repository_metadata( trans, + id, + repository, + metadata_changeset_revision, + metadata_dict ) changeset_revisions.append( metadata_changeset_revision ) ancestor_changeset_revision = None ancestor_metadata_dict = None @@ -1770,14 +1786,20 @@ # We reach here only if current_metadata_dict is empty and ancestor_metadata_dict is not. if not ctx.children(): # We're at the end of the change log. - repository_metadata = create_or_update_repository_metadata( trans, id, repository, metadata_changeset_revision, metadata_dict ) + repository_metadata = create_or_update_repository_metadata( trans, + id, + repository, + metadata_changeset_revision, + metadata_dict ) changeset_revisions.append( metadata_changeset_revision ) ancestor_changeset_revision = None ancestor_metadata_dict = None suc.remove_dir( work_dir ) - # Delete all repository_metadata records for this repository that do not have a changeset_revision value in changeset_revisions. + # Delete all repository_metadata records for this repository that do not have a changeset_revision + # value in changeset_revisions. clean_repository_metadata( trans, id, changeset_revisions ) - # Set tool version information for all downloadable changeset revisions. Get the list of changeset revisions from the changelog. + # Set tool version information for all downloadable changeset revisions. Get the list of changeset + # revisions from the changelog. reset_all_tool_versions( trans, id, repo ) # Reset the tool_data_tables by loading the empty tool_data_table_conf.xml file. tool_util.reset_tool_data_tables( trans.app ) @@ -1785,8 +1807,8 @@ def reset_metadata_on_selected_repositories( trans, **kwd ): """ - Inspect the repository changelog to reset metadata for all appropriate changeset revisions. This method is called from both Galaxy and the - Tool Shed. + Inspect the repository changelog to reset metadata for all appropriate changeset revisions. + This method is called from both Galaxy and the Tool Shed. """ repository_ids = util.listify( kwd.get( 'repository_ids', None ) ) message = '' @@ -1879,7 +1901,11 @@ tip_only = isinstance( repository_type_class, TipOnly ) if not tip_only and new_metadata_required_for_utilities( trans, repository, metadata_dict ): # Create a new repository_metadata table row. - repository_metadata = create_or_update_repository_metadata( trans, encoded_id, repository, repository.tip( trans.app ), metadata_dict ) + repository_metadata = create_or_update_repository_metadata( trans, + encoded_id, + repository, + repository.tip( trans.app ), + metadata_dict ) # If this is the first record stored for this repository, see if we need to send any email alerts. if len( repository.downloadable_revisions ) == 1: suc.handle_email_alerts( trans, repository, content_alert_str='', new_repo_alert=True, admin_only=False ) @@ -1896,8 +1922,8 @@ repository_metadata.includes_datatypes = True else: repository_metadata.includes_datatypes = False - # We don't store information about the special type of repository dependency that is needed only for compiling a tool dependency - # defined for the dependent repository. + # We don't store information about the special type of repository dependency that is needed only for + # compiling a tool dependency defined for the dependent repository. repository_dependencies_dict = metadata_dict.get( 'repository_dependencies', {} ) repository_dependencies = repository_dependencies_dict.get( 'repository_dependencies', [] ) has_repository_dependencies, has_repository_dependencies_only_if_compiling_contained_td = \ @@ -1924,9 +1950,14 @@ trans.sa_session.flush() else: # There are no metadata records associated with the repository. - repository_metadata = create_or_update_repository_metadata( trans, encoded_id, repository, repository.tip( trans.app ), metadata_dict ) + repository_metadata = create_or_update_repository_metadata( trans, + encoded_id, + repository, + repository.tip( trans.app ), + metadata_dict ) if 'tools' in metadata_dict and repository_metadata and status != 'error': - # Set tool versions on the new downloadable change set. The order of the list of changesets is critical, so we use the repo's changelog. + # Set tool versions on the new downloadable change set. The order of the list of changesets is + # critical, so we use the repo's changelog. changeset_revisions = [] for changeset in repo.changelog: changeset_revision = str( repo.changectx( changeset ) ) @@ -1945,8 +1976,12 @@ return message, status def set_repository_metadata_due_to_new_tip( trans, repository, content_alert_str=None, **kwd ): - """Set metadata on the repository tip in the tool shed - this method is not called from Galaxy.""" - error_message, status = set_repository_metadata( trans, repository, content_alert_str=content_alert_str, **kwd ) + """Set metadata on the repository tip in the tool shed.""" + # This method is not called from Galaxy. + error_message, status = set_repository_metadata( trans, + repository, + content_alert_str=content_alert_str, + **kwd ) if error_message: # FIXME: This probably should not redirect since this method is called from the upload controller as well as the repository controller. # If there is an error, display it. @@ -1994,8 +2029,12 @@ break if new_dependency_name and new_dependency_type and new_dependency_version: # Update all attributes of the tool_dependency record in the database. - log.debug( "Updating tool dependency '%s' with type '%s' and version '%s' to have new type '%s' and version '%s'." % \ - ( str( tool_dependency.name ), str( tool_dependency.type ), str( tool_dependency.version ), str( new_dependency_type ), str( new_dependency_version ) ) ) + log.debug( "Updating version %s of tool dependency %s %s to have new version %s and type %s." % \ + ( str( tool_dependency.version ), + str( tool_dependency.type ), + str( tool_dependency.name ), + str( new_dependency_version ), + str( new_dependency_type ) ) ) tool_dependency.type = new_dependency_type tool_dependency.version = new_dependency_version tool_dependency.status = app.install_model.ToolDependency.installation_status.UNINSTALLED @@ -2004,9 +2043,10 @@ context.flush() new_tool_dependency = tool_dependency else: - # We have no new tool dependency definition based on a matching dependency name, so remove the existing tool dependency record from the database. - log.debug( "Deleting tool dependency with name '%s', type '%s' and version '%s' from the database since it is no longer defined." % \ - ( str( tool_dependency.name ), str( tool_dependency.type ), str( tool_dependency.version ) ) ) + # We have no new tool dependency definition based on a matching dependency name, so remove + # the existing tool dependency record from the database. + log.debug( "Deleting version %s of tool dependency %s %s from the database since it is no longer defined." % \ + ( str( tool_dependency.version ), str( tool_dependency.type ), str( tool_dependency.name ) ) ) context.delete( tool_dependency ) context.flush() return new_tool_dependency diff -r f11a729d42cb66753b8b1616ad625c1b2a306b3d -r cd8ca90e93a1f896ee6ff3ac56665e64373ace00 lib/tool_shed/util/repository_maintenance_util.py --- a/lib/tool_shed/util/repository_maintenance_util.py +++ b/lib/tool_shed/util/repository_maintenance_util.py @@ -102,15 +102,17 @@ def create_repository_and_import_archive( trans, repository_archive_dict, import_results_tups ): """ - Create a new repository in the tool shed and populate it with the contents of a gzip compressed tar archive that was exported - as part or all of the contents of a capsule. + Create a new repository in the tool shed and populate it with the contents of a gzip compressed tar archive + that was exported as part or all of the contents of a capsule. """ results_message = '' name = repository_archive_dict.get( 'name', None ) username = repository_archive_dict.get( 'owner', None ) if name is None or username is None: - results_message += 'Import failed: required repository name <b>%s</b> or owner <b>%s</b> is missing.' % ( str( name ), str( username )) - import_results_tups.append( ( ( str( name ), str( username ) ), results_message ) ) + ok = False + results_message += 'Import failed: required repository name <b>%s</b> or owner <b>%s</b> is missing.' % \ + ( str( name ), str( username )) + import_results_tups.append( ( ok, ( str( name ), str( username ) ), results_message ) ) else: if repository_archive_dict[ 'status' ] is None: # The repository does not yet exist in this Tool Shed and the current user is authorized to import @@ -122,8 +124,9 @@ # the exported repository archive. user = suc.get_user_by_username( trans.app, username ) if user is None: + ok = False results_message += 'Import failed: repository owner <b>%s</b> does not have an account in this Tool Shed.' % str( username ) - import_results_tups.append( ( ( str( name ), str( username ) ), results_message ) ) + import_results_tups.append( ( ok, ( str( name ), str( username ) ), results_message ) ) else: user_id = user.id # The categories entry in the repository_archive_dict is a list of category names. If a name does not @@ -134,8 +137,8 @@ for category_name in category_names: category = suc.get_category_by_name( trans, category_name ) if category is None: - results_message += 'This Tool Shed does not have the category <b>%s</b> so it will not be associated with this repository.' % \ - str( category_name ) + results_message += 'This Tool Shed does not have the category <b>%s</b> so it ' % str( category_name ) + results_message += 'will not be associated with this repository.' else: category_ids.append( trans.security.encode_id( category.id ) ) # Create the repository record in the database. @@ -150,11 +153,14 @@ results_message += create_message # Populate the new repository with the contents of exported repository archive. results_dict = import_util.import_repository_archive( trans, repository, repository_archive_dict ) - import_results_tups.append( ( ( str( name ), str( username ) ), results_message ) ) + ok = results_dict.get( 'ok', False ) + import_results_tups.append( ( ok, ( str( name ), str( username ) ), results_message ) ) else: # The repository either already exists in this Tool Shed or the current user is not authorized to create it. - results_message += 'Import not necessary: repository status for this Tool Shed is: %s.' % str( repository_archive_dict[ 'status' ] ) - import_results_tups.append( ( ( str( name ), str( username ) ), results_message ) ) + ok = True + results_message += 'Import not necessary: repository status for this Tool Shed is: %s.' % \ + str( repository_archive_dict[ 'status' ] ) + import_results_tups.append( ( ok, ( str( name ), str( username ) ), results_message ) ) return import_results_tups def validate_repository_name( app, name, user ): diff -r f11a729d42cb66753b8b1616ad625c1b2a306b3d -r cd8ca90e93a1f896ee6ff3ac56665e64373ace00 lib/tool_shed/util/shed_util_common.py --- a/lib/tool_shed/util/shed_util_common.py +++ b/lib/tool_shed/util/shed_util_common.py @@ -559,12 +559,65 @@ return repository_metadata return None +def get_dependent_downloadable_revisions( trans, repository_metadata ): + """ + Return all repository_metadata records that are downloadable and that depend upon the received + repository_metadata record. + """ + # This method is called only from the tool shed. + rm_changeset_revision = repository_metadata.changeset_revision + rm_repository = repository_metadata.repository + rm_repository_name = str( rm_repository.name ) + rm_repository_owner = str( rm_repository.user.username ) + dependent_downloadable_revisions = [] + for repository in trans.sa_session.query( trans.model.Repository ) \ + .filter( and_( trans.model.Repository.table.c.id != rm_repository.id, + trans.model.Repository.table.c.deleted == False, + trans.model.Repository.table.c.deprecated == False ) ): + downloadable_revisions = repository.downloadable_revisions + if downloadable_revisions: + for downloadable_revision in downloadable_revisions: + if downloadable_revision.has_repository_dependencies: + metadata = downloadable_revision.metadata + if metadata: + repository_dependencies_dict = metadata.get( 'repository_dependencies', {} ) + repository_dependencies_tups = repository_dependencies_dict.get( 'repository_dependencies', [] ) + for repository_dependencies_tup in repository_dependencies_tups: + tool_shed, name, owner, changeset_revision, prior_installation_required, only_if_compiling_contained_td = \ + common_util.parse_repository_dependency_tuple( repository_dependencies_tup ) + if name == rm_repository_name and owner == rm_repository_owner: + # We've discovered a repository revision that depends upon the repository associated + # with the received repository_metadata record, but we need to make sure it depends + # upon the revision. + if changeset_revision == rm_changeset_revision: + dependent_downloadable_revisions.append( downloadable_revision ) + else: + # Make sure the defined changeset_revision is current. + defined_repository_metadata = \ + trans.sa_session.query( trans.model.RepositoryMetadata ) \ + .filter( trans.model.RepositoryMetadata.table.c.changeset_revision == changeset_revision ) \ + .first() + if defined_repository_metadata is None: + # The defined changeset_revision is not associated with a repository_metadata + # record, so updates must be necessary. + defined_repository = get_repository_by_name_and_owner( trans.app, name, owner ) + defined_repo_dir = defined_repository.repo_path( trans.app ) + defined_repo = hg.repository( get_configured_ui(), defined_repo_dir ) + updated_changeset_revision = \ + get_next_downloadable_changeset_revision( defined_repository, + defined_repo, + changeset_revision ) + if updated_changeset_revision == rm_changeset_revision: + dependent_downloadable_revisions.append( downloadable_revision ) + return dependent_downloadable_revisions + def get_file_context_from_ctx( ctx, filename ): """Return the mercurial file context for a specified file.""" - # We have to be careful in determining if we found the correct file because multiple files with the same name may be in different directories - # within ctx if the files were moved within the change set. For example, in the following ctx.files() list, the former may have been moved to - # the latter: ['tmap_wrapper_0.0.19/tool_data_table_conf.xml.sample', 'tmap_wrapper_0.3.3/tool_data_table_conf.xml.sample']. Another scenario - # is that the file has been deleted. + # We have to be careful in determining if we found the correct file because multiple files with + # the same name may be in different directories within ctx if the files were moved within the change + # set. For example, in the following ctx.files() list, the former may have been moved to the latter: + # ['tmap_wrapper_0.0.19/tool_data_table_conf.xml.sample', 'tmap_wrapper_0.3.3/tool_data_table_conf.xml.sample']. + # Another scenario is that the file has been deleted. deleted = False filename = strip_path( filename ) for ctx_file in ctx.files(): diff -r f11a729d42cb66753b8b1616ad625c1b2a306b3d -r cd8ca90e93a1f896ee6ff3ac56665e64373ace00 templates/webapps/tool_shed/repository/import_capsule_results.mako --- a/templates/webapps/tool_shed/repository/import_capsule_results.mako +++ b/templates/webapps/tool_shed/repository/import_capsule_results.mako @@ -70,7 +70,7 @@ <table class="grid"> %for import_results_tup in import_results_tups: <% - name_owner_tup, results_message = import_results_tup + ok, name_owner_tup, results_message = import_results_tup name, owner = name_owner_tup %><tr><td>Archive of repository <b>${name}</b> owned by <b>${owner}</b><br/>${results_message}</td></tr> Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.