May 2014 - galaxy-commits - lists.galaxyproject.org

commit/galaxy-central: greg: Use the new hg_util.get_repo_for_repository() function to retrieve a mercurial repo for a repository.
by commits-noreply＠bitbucket.org 21 May '14

21 May '14

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/7eaa7e058c28/ Changeset: 7eaa7e058c28 User: greg Date: 2014-05-21 18:22:16 Summary: Use the new hg_util.get_repo_for_repository() function to retrieve a mercurial repo for a repository. Affected #: 23 files diff -r df18a38465c2cd7d7e8014febc3d955758f84671 -r 7eaa7e058c2898b10674b4ca43a0a76e566517c3 lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py --- a/lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py +++ b/lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py @@ -33,11 +33,6 @@ import pkg_resources -eggs.require( 'mercurial' ) -from mercurial import commands -from mercurial import hg -from mercurial import ui - pkg_resources.require( 'elementtree' ) from elementtree import ElementTree from elementtree.ElementTree import Element @@ -1865,7 +1860,10 @@ repo_files_dir = os.path.abspath( os.path.join( tool_path, relative_install_dir, name ) ) else: repo_files_dir = os.path.abspath( os.path.join( relative_install_dir, name ) ) - repo = hg.repository( hg_util.get_configured_ui(), path=repo_files_dir ) + repo = hg_util.get_repo_for_repository( trans.app, + repository=None, + repo_path=repo_files_dir, + create=False ) repository_clone_url = os.path.join( tool_shed_url, 'repos', owner, name ) repository_util.pull_repository( repo, repository_clone_url, latest_ctx_rev ) hg_util.update_repository( repo, latest_ctx_rev ) diff -r df18a38465c2cd7d7e8014febc3d955758f84671 -r 7eaa7e058c2898b10674b4ca43a0a76e566517c3 lib/galaxy/webapps/tool_shed/api/repositories.py --- a/lib/galaxy/webapps/tool_shed/api/repositories.py +++ b/lib/galaxy/webapps/tool_shed/api/repositories.py @@ -19,10 +19,6 @@ from tool_shed.util import repository_maintenance_util from tool_shed.util import tool_util -eggs.require( 'mercurial' ) - -from mercurial import hg - log = logging.getLogger( __name__ ) @@ -49,8 +45,7 @@ error_message += "cannot locate repository %s owned by %s." % ( str( name ), str( owner ) ) log.debug( error_message ) return [] - repo_dir = repository.repo_path( trans.app ) - repo = hg.repository( hg_util.get_configured_ui(), repo_dir ) + repo = hg_util.get_repo_for_repository( trans.app, repository=repository, repo_path=None, create=False ) ordered_installable_revisions = suc.get_ordered_metadata_changeset_revisions( repository, repo, downloadable=True ) return ordered_installable_revisions else: @@ -133,8 +128,7 @@ if repository_metadata is None: # The changeset_revision column in the repository_metadata table has been updated with a new # value value, so find the changeset_revision to which we need to update. - repo_dir = repository.repo_path( trans.app ) - repo = hg.repository( hg_util.get_configured_ui(), repo_dir ) + repo = hg_util.get_repo_for_repository( trans.app, repository=repository, repo_path=None, create=False ) new_changeset_revision = suc.get_next_downloadable_changeset_revision( repository, repo, changeset_revision ) repository_metadata = suc.get_repository_metadata_by_changeset_revision( trans.app, encoded_repository_id, diff -r df18a38465c2cd7d7e8014febc3d955758f84671 -r 7eaa7e058c2898b10674b4ca43a0a76e566517c3 lib/galaxy/webapps/tool_shed/api/repository_revisions.py --- a/lib/galaxy/webapps/tool_shed/api/repository_revisions.py +++ b/lib/galaxy/webapps/tool_shed/api/repository_revisions.py @@ -9,11 +9,6 @@ from tool_shed.util import hg_util import tool_shed.util.shed_util_common as suc -from galaxy import eggs -eggs.require( 'mercurial' ) - -from mercurial import hg - log = logging.getLogger( __name__ ) @@ -168,8 +163,10 @@ if repository_dependency_repository_metadata is None: # The changeset_revision column in the repository_metadata table has been updated with a new # value value, so find the changeset_revision to which we need to update. - repo_dir = repository_dependency.repo_path( trans.app ) - repo = hg.repository( hg_util.get_configured_ui(), repo_dir ) + repo = hg_util.get_repo_for_repository( trans.app, + repository=repository_dependency, + repo_path=None, + create=False ) new_changeset_revision = suc.get_next_downloadable_changeset_revision( repository_dependency, repo, changeset_revision ) diff -r df18a38465c2cd7d7e8014febc3d955758f84671 -r 7eaa7e058c2898b10674b4ca43a0a76e566517c3 lib/galaxy/webapps/tool_shed/controllers/repository.py --- a/lib/galaxy/webapps/tool_shed/controllers/repository.py +++ b/lib/galaxy/webapps/tool_shed/controllers/repository.py @@ -41,10 +41,8 @@ eggs.require( 'mercurial' ) from mercurial import commands -from mercurial import hg from mercurial import mdiff from mercurial import patch -from mercurial import ui log = logging.getLogger( __name__ ) @@ -720,7 +718,7 @@ status = kwd.get( 'status', 'done' ) commit_message = kwd.get( 'commit_message', 'Deleted selected files' ) repository = suc.get_repository_in_tool_shed( trans, id ) - repo = hg.repository( hg_util.get_configured_ui(), repository.repo_path( trans.app ) ) + repo = hg_util.get_repo_for_repository( trans.app, repository=repository, repo_path=None, create=False ) # Update repository files for browsing. hg_util.update_repository( repo ) changeset_revision = repository.tip( trans.app ) @@ -891,8 +889,7 @@ owner = kwd.get( 'owner', None ) changeset_revision = kwd.get( 'changeset_revision', None ) repository = suc.get_repository_by_name_and_owner( trans.app, name, owner ) - repo_dir = repository.repo_path( trans.app ) - repo = hg.repository( hg_util.get_configured_ui(), repo_dir ) + repo = hg_util.get_repo_for_repository( trans.app, repository=repository, repo_path=None, create=False ) # Default to the current changeset revision. update_to_ctx = hg_util.get_changectx_for_changeset( repo, changeset_revision ) latest_changeset_revision = changeset_revision @@ -1464,10 +1461,10 @@ includes_tools_for_display_in_tool_panel = has_galaxy_utilities_dict[ 'includes_tools_for_display_in_tool_panel' ] includes_tool_dependencies = has_galaxy_utilities_dict[ 'includes_tool_dependencies' ] has_repository_dependencies = has_galaxy_utilities_dict[ 'has_repository_dependencies' ] - has_repository_dependencies_only_if_compiling_contained_td = has_galaxy_utilities_dict[ 'has_repository_dependencies_only_if_compiling_contained_td' ] + has_repository_dependencies_only_if_compiling_contained_td = \ + has_galaxy_utilities_dict[ 'has_repository_dependencies_only_if_compiling_contained_td' ] includes_workflows = has_galaxy_utilities_dict[ 'includes_workflows' ] - repo_dir = repository.repo_path( trans.app ) - repo = hg.repository( hg_util.get_configured_ui(), repo_dir ) + repo = hg_util.get_repo_for_repository( trans.app, repository=repository, repo_path=None, create=False ) # Default to the received changeset revision and ctx_rev. update_to_ctx = hg_util.get_changectx_for_changeset( repo, changeset_revision ) ctx_rev = str( update_to_ctx.rev() ) @@ -1544,8 +1541,7 @@ repository_owner = kwd[ 'owner' ] changeset_revision = kwd[ 'changeset_revision' ] repository = suc.get_repository_by_name_and_owner( trans.app, repository_name, repository_owner ) - repo_dir = repository.repo_path( trans.app ) - repo = hg.repository( hg_util.get_configured_ui(), repo_dir ) + repo = hg_util.get_repo_for_repository( trans.app, repository=repository, repo_path=None, create=False ) ctx = hg_util.get_changectx_for_changeset( repo, changeset_revision ) if ctx: return str( ctx.rev() ) @@ -1558,14 +1554,6 @@ trans.response.headers['Expires'] = '0' return suc.get_repository_file_contents( file_path ) - def get_file_from_changeset_revision( self, repo_files_dir, changeset_revision, file_name, dir ): - """Return file_name from the received changeset_revision of the repository manifest.""" - stripped_file_name = suc.strip_path( file_name ) - repo = hg.repository( hg_util.get_configured_ui(), repo_files_dir ) - ctx = hg_util.get_changectx_for_changeset( repo, changeset_revision ) - named_tmp_file = suc.get_named_tmpfile_from_ctx( ctx, file_name, dir ) - return named_tmp_file - @web.expose def get_functional_test_rss( self, trans, **kwd ): '''Return an RSS feed of the functional test results for the provided user, optionally filtered by the 'status' parameter.''' @@ -1604,8 +1592,7 @@ trans.model.Repository.table.c.deprecated == False, trans.model.Repository.table.c.user_id == user.id ) ): repository = repository_metadata.repository - repo_dir = repository.repo_path( trans.app ) - repo = hg.repository( hg_util.get_configured_ui(), repo_dir ) + repo = hg_util.get_repo_for_repository( trans.app, repository=repository, repo_path=None, create=False ) latest_downloadable_changeset_revsion = suc.get_latest_downloadable_changeset_revision( trans.app, repository, repo ) if repository_metadata.changeset_revision == latest_downloadable_changeset_revsion: # We'll display only the test run for the latest installable revision in the rss feed. @@ -1673,7 +1660,7 @@ if repository_name is not None and repository_owner is not None: repository = suc.get_repository_by_name_and_owner( trans.app, repository_name, repository_owner ) if repository: - repo = hg.repository( hg_util.get_configured_ui(), repository.repo_path( trans.app ) ) + repo = hg_util.get_repo_for_repository( trans.app, repository=repository, repo_path=None, create=False ) return suc.get_latest_downloadable_changeset_revision( trans.app, repository, repo ) return suc.INITIAL_CHANGELOG_HASH @@ -1860,8 +1847,7 @@ owner = kwd[ 'owner' ] changeset_revision = kwd[ 'changeset_revision' ] repository = suc.get_repository_by_name_and_owner( trans.app, name, owner ) - repo_dir = repository.repo_path( trans.app ) - repo = hg.repository( hg_util.get_configured_ui(), repo_dir ) + repo = hg_util.get_repo_for_repository( trans.app, repository=repository, repo_path=None, create=False ) tool_version_dicts = [] for changeset in repo.changelog: current_changeset_revision = str( repo.changectx( changeset ) ) @@ -1885,8 +1871,7 @@ repository = suc.get_repository_by_name_and_owner( trans.app, name, owner ) repository_id = trans.security.encode_id( repository.id ) repository_clone_url = common_util.generate_clone_url_for_repository_in_tool_shed( trans, repository ) - repo_dir = repository.repo_path( trans.app ) - repo = hg.repository( hg_util.get_configured_ui(), repo_dir ) + repo = hg_util.get_repo_for_repository( trans.app, repository=repository, repo_path=None, create=False ) repository_metadata = suc.get_repository_metadata_by_changeset_revision( trans.app, repository_id, changeset_revision ) if not repository_metadata: # The received changeset_revision is no longer associated with metadata, so get the next changeset_revision in the repository @@ -2171,7 +2156,7 @@ repository = suc.get_repository_in_tool_shed( trans, id ) repository_type = kwd.get( 'repository_type', str( repository.type ) ) repo_dir = repository.repo_path( trans.app ) - repo = hg.repository( hg_util.get_configured_ui(), repo_dir ) + repo = hg_util.get_repo_for_repository( trans.app, repository=None, repo_path=repo_dir, create=False ) repo_name = kwd.get( 'repo_name', repository.name ) changeset_revision = kwd.get( 'changeset_revision', repository.tip( trans.app ) ) description = kwd.get( 'description', repository.description ) @@ -2450,8 +2435,7 @@ else: # There is no repository_metadata defined for the changeset_revision, so see if it was defined # in a previous changeset in the changelog. - repo_dir = repository.repo_path( trans.app ) - repo = hg.repository( hg_util.get_configured_ui(), repo_dir ) + repo = hg_util.get_repo_for_repository( trans.app, repository=repository, repo_path=None, create=False ) previous_changeset_revision = \ suc.get_previous_metadata_changeset_revision( repository, repo, @@ -2527,8 +2511,7 @@ owner = kwd.get( 'owner', None ) changeset_revision = kwd.get( 'changeset_revision', None ) repository = suc.get_repository_by_name_and_owner( trans.app, name, owner ) - repo_dir = repository.repo_path( trans.app ) - repo = hg.repository( hg_util.get_configured_ui(), repo_dir ) + repo = hg_util.get_repo_for_repository( trans.app, repository=repository, repo_path=None, create=False ) # Get the next installable changeset_revision beyond the received changeset_revision. changeset_revision = suc.get_next_downloadable_changeset_revision( repository, repo, changeset_revision ) if changeset_revision: @@ -2547,8 +2530,7 @@ message = kwd.get( 'message', '' ) status = kwd.get( 'status', 'done' ) repository = suc.get_repository_in_tool_shed( trans, repository_id ) - repo_dir = repository.repo_path( trans.app ) - repo = hg.repository( hg_util.get_configured_ui(), repo_dir ) + repo = hg_util.get_repo_for_repository( trans.app, repository=repository, repo_path=None, create=False ) changeset_revision = kwd.get( 'changeset_revision', repository.tip( trans.app ) ) repository_metadata = suc.get_repository_metadata_by_changeset_revision( trans.app, repository_id, changeset_revision ) if repository_metadata: @@ -2614,8 +2596,7 @@ owner = kwd.get( 'owner', None ) changeset_revision = kwd.get( 'changeset_revision', None ) repository = suc.get_repository_by_name_and_owner( trans.app, name, owner ) - repo_dir = repository.repo_path( trans.app ) - repo = hg.repository( hg_util.get_configured_ui(), repo_dir ) + repo = hg_util.get_repo_for_repository( trans.app, repository=repository, repo_path=None, create=False ) # Get the lower bound changeset revision. lower_bound_changeset_revision = suc.get_previous_metadata_changeset_revision( repository, repo, changeset_revision, downloadable=True ) # Build the list of changeset revision hashes. @@ -2641,7 +2622,7 @@ status='error' ) ) repository = suc.get_repository_in_tool_shed( trans, id ) changeset_revision = repository.tip( trans.app ) - repo = hg.repository( hg_util.get_configured_ui(), repository.repo_path( trans.app ) ) + repo = hg_util.get_repo_for_repository( trans.app, repository=repository, repo_path=None, create=False ) if repository.user == trans.user: return trans.response.send_redirect( web.url_for( controller='repository', action='browse_repositories', @@ -2710,7 +2691,7 @@ commit_message = kwd.get( 'commit_message', 'Deleted selected files' ) repository = suc.get_repository_in_tool_shed( trans, id ) repo_dir = repository.repo_path( trans.app ) - repo = hg.repository( hg_util.get_configured_ui(), repo_dir ) + repo = hg_util.get_repo_for_repository( trans.app, repository=None, repo_path=repo_dir, create=False ) selected_files_to_delete = kwd.get( 'selected_files_to_delete', '' ) if kwd.get( 'select_files_to_delete_button', False ): if selected_files_to_delete: @@ -2750,7 +2731,6 @@ # Update the repository files for browsing. hg_util.update_repository( repo ) # Get the new repository tip. - repo = hg.repository( hg_util.get_configured_ui(), repo_dir ) if tip == repository.tip( trans.app ): message += 'No changes to repository. ' kwd[ 'message' ] = message @@ -2935,15 +2915,18 @@ repository = None if repository: repository_id = trans.security.encode_id( repository.id ) - repository_metadata = metadata_util.get_repository_metadata_by_repository_id_changeset_revision( trans, repository_id, changeset_revision ) + repository_metadata = metadata_util.get_repository_metadata_by_repository_id_changeset_revision( trans, + repository_id, + changeset_revision ) if not repository_metadata: # Get updates to the received changeset_revision if any exist. - repo_dir = repository.repo_path( trans.app ) - repo = hg.repository( hg_util.get_configured_ui(), repo_dir ) + repo = hg_util.get_repo_for_repository( trans.app, repository=repository, repo_path=None, create=False ) upper_bound_changeset_revision = suc.get_next_downloadable_changeset_revision( repository, repo, changeset_revision ) if upper_bound_changeset_revision: changeset_revision = upper_bound_changeset_revision - repository_metadata = metadata_util.get_repository_metadata_by_repository_id_changeset_revision( trans, repository_id, changeset_revision ) + repository_metadata = metadata_util.get_repository_metadata_by_repository_id_changeset_revision( trans, + repository_id, + changeset_revision ) if repository_metadata: return trans.response.send_redirect( web.url_for( controller='repository', action='index', @@ -2978,8 +2961,7 @@ repository_metadata = suc.get_repository_metadata_by_changeset_revision( trans.app, trans.security.encode_id( repository.id ), changeset_revision ) - repo_dir = repository.repo_path( trans.app ) - repo = hg.repository( hg_util.get_configured_ui(), repo_dir ) + repo = hg_util.get_repo_for_repository( trans.app, repository=repository, repo_path=None, create=False ) tool_shed_status_dict = {} # Handle repository deprecation. tool_shed_status_dict[ 'repository_deprecated' ] = str( repository.deprecated ) @@ -3071,7 +3053,7 @@ message = kwd.get( 'message', '' ) status = kwd.get( 'status', 'done' ) repository = suc.get_repository_in_tool_shed( trans, id ) - repo = hg.repository( hg_util.get_configured_ui(), repository.repo_path( trans.app ) ) + repo = hg_util.get_repo_for_repository( trans.app, repository=repository, repo_path=None, create=False ) changesets = [] for changeset in repo.changelog: ctx = repo.changectx( changeset ) @@ -3106,7 +3088,7 @@ message = kwd.get( 'message', '' ) status = kwd.get( 'status', 'done' ) repository = suc.get_repository_in_tool_shed( trans, id ) - repo = hg.repository( hg_util.get_configured_ui(), repository.repo_path( trans.app ) ) + repo = hg_util.get_repo_for_repository( trans.app, repository=repository, repo_path=None, create=False ) ctx = hg_util.get_changectx_for_changeset( repo, ctx_str ) if ctx is None: message = "Repository does not include changeset revision '%s'." % str( ctx_str ) @@ -3195,7 +3177,7 @@ message = kwd.get( 'message', '' ) status = kwd.get( 'status', 'done' ) repository = suc.get_repository_in_tool_shed( trans, id ) - repo = hg.repository( hg_util.get_configured_ui(), repository.repo_path( trans.app ) ) + repo = hg_util.get_repo_for_repository( trans.app, repository=repository, repo_path=None, create=False ) avg_rating, num_ratings = self.get_ave_item_rating_data( trans.sa_session, repository, webapp_model=trans.model ) changeset_revision = kwd.get( 'changeset_revision', repository.tip( trans.app ) ) display_reviews = kwd.get( 'display_reviews', False ) @@ -3289,7 +3271,7 @@ render_repository_actions_for = kwd.get( 'render_repository_actions_for', 'tool_shed' ) repository = suc.get_repository_in_tool_shed( trans, repository_id ) repo_files_dir = repository.repo_path( trans.app ) - repo = hg.repository( hg_util.get_configured_ui(), repo_files_dir ) + repo = hg_util.get_repo_for_repository( trans.app, repository=None, repo_path=repo_files_dir, create=False ) tool_metadata_dict = {} tool_lineage = [] tool = None diff -r df18a38465c2cd7d7e8014febc3d955758f84671 -r 7eaa7e058c2898b10674b4ca43a0a76e566517c3 lib/galaxy/webapps/tool_shed/controllers/repository_review.py --- a/lib/galaxy/webapps/tool_shed/controllers/repository_review.py +++ b/lib/galaxy/webapps/tool_shed/controllers/repository_review.py @@ -13,10 +13,6 @@ import tool_shed.grids.repository_review_grids as repository_review_grids import tool_shed.grids.util as grids_util -from galaxy import eggs -eggs.require('mercurial') -from mercurial import hg, ui, patch, commands - log = logging.getLogger( __name__ ) @@ -73,7 +69,7 @@ status = kwd.get( 'status', 'done' ) review = review_util.get_review( trans, kwd[ 'id' ] ) repository = review.repository - repo = hg.repository( hg_util.get_configured_ui(), repository.repo_path( trans.app ) ) + repo = hg_util.get_repo_for_repository( trans.app, repository=repository, repo_path=None, create=False ) rev, changeset_revision_label = hg_util.get_rev_label_from_changeset_revision( repo, review.changeset_revision ) return trans.fill_template( '/webapps/tool_shed/repository_review/browse_review.mako', repository=repository, @@ -233,7 +229,7 @@ for component in review_util.get_components( trans ): components_dict[ component.name ] = dict( component=component, component_review=None ) repository = review.repository - repo = hg.repository( hg_util.get_configured_ui(), repository.repo_path( trans.app ) ) + repo = hg_util.get_repo_for_repository( trans.app, repository=repository, repo_path=None, create=False ) for component_review in review.component_reviews: if component_review and component_review.component: component_name = component_review.component.name @@ -470,8 +466,7 @@ repository_id = kwd.get( 'id', None ) if repository_id: repository = suc.get_repository_in_tool_shed( trans, repository_id ) - repo_dir = repository.repo_path( trans.app ) - repo = hg.repository( hg_util.get_configured_ui(), repo_dir ) + repo = hg_util.get_repo_for_repository( trans.app, repository=repository, repo_path=None, create=False ) metadata_revision_hashes = [ metadata_revision.changeset_revision for metadata_revision in repository.metadata_revisions ] reviewed_revision_hashes = [ review.changeset_revision for review in repository.reviews ] reviews_dict = odict() @@ -517,8 +512,7 @@ repository_id = kwd.get( 'id', None ) changeset_revision = kwd.get( 'changeset_revision', None ) repository = suc.get_repository_in_tool_shed( trans, repository_id ) - repo_dir = repository.repo_path( trans.app ) - repo = hg.repository( hg_util.get_configured_ui(), repo_dir ) + repo = hg_util.get_repo_for_repository( trans.app, repository=repository, repo_path=None, create=False ) installable = changeset_revision in [ metadata_revision.changeset_revision for metadata_revision in repository.metadata_revisions ] rev, changeset_revision_label = hg_util.get_rev_label_from_changeset_revision( repo, changeset_revision ) reviews = review_util.get_reviews_by_repository_id_changeset_revision( trans, repository_id, changeset_revision ) @@ -583,7 +577,7 @@ status = kwd.get( 'status', 'done' ) repository = suc.get_repository_in_tool_shed( trans, kwd[ 'id' ] ) changeset_revision = kwd.get( 'changeset_revision', None ) - repo = hg.repository( hg_util.get_configured_ui(), repository.repo_path( trans.app ) ) + repo = hg_util.get_repo_for_repository( trans.app, repository=repository, repo_path=None, create=False ) previous_reviews_dict = review_util.get_previous_repository_reviews( trans, repository, changeset_revision ) rev, changeset_revision_label = hg_util.get_rev_label_from_changeset_revision( repo, changeset_revision ) return trans.fill_template( '/webapps/tool_shed/repository_review/select_previous_review.mako', diff -r df18a38465c2cd7d7e8014febc3d955758f84671 -r 7eaa7e058c2898b10674b4ca43a0a76e566517c3 lib/galaxy/webapps/tool_shed/controllers/upload.py --- a/lib/galaxy/webapps/tool_shed/controllers/upload.py +++ b/lib/galaxy/webapps/tool_shed/controllers/upload.py @@ -21,8 +21,6 @@ from galaxy import eggs eggs.require( 'mercurial' ) from mercurial import commands -from mercurial import hg -from mercurial import ui log = logging.getLogger( __name__ ) @@ -40,7 +38,7 @@ repository_id = kwd.get( 'repository_id', '' ) repository = suc.get_repository_in_tool_shed( trans, repository_id ) repo_dir = repository.repo_path( trans.app ) - repo = hg.repository( hg_util.get_configured_ui(), repo_dir ) + repo = hg_util.get_repo_for_repository( trans.app, repository=None, repo_path=repo_dir, create=False ) uncompress_file = util.string_as_bool( kwd.get( 'uncompress_file', 'true' ) ) remove_repo_files_not_in_tar = util.string_as_bool( kwd.get( 'remove_repo_files_not_in_tar', 'true' ) ) uploaded_file = None @@ -325,7 +323,7 @@ def upload_directory( self, trans, repository, uploaded_directory, upload_point, remove_repo_files_not_in_tar, commit_message, new_repo_alert ): repo_dir = repository.repo_path( trans.app ) - repo = hg.repository( hg_util.get_configured_ui(), repo_dir ) + repo = hg_util.get_repo_for_repository( trans.app, repository=None, repo_path=repo_dir, create=False ) undesirable_dirs_removed = 0 undesirable_files_removed = 0 if upload_point is not None: @@ -389,7 +387,7 @@ def upload_tar( self, trans, repository, tar, uploaded_file, upload_point, remove_repo_files_not_in_tar, commit_message, new_repo_alert ): # Upload a tar archive of files. repo_dir = repository.repo_path( trans.app ) - repo = hg.repository( hg_util.get_configured_ui(), repo_dir ) + repo = hg_util.get_repo_for_repository( trans.app, repository=None, repo_path=repo_dir, create=False ) undesirable_dirs_removed = 0 undesirable_files_removed = 0 ok, message = commit_util.check_archive( repository, tar ) diff -r df18a38465c2cd7d7e8014febc3d955758f84671 -r 7eaa7e058c2898b10674b4ca43a0a76e566517c3 lib/tool_shed/galaxy_install/repository_util.py --- a/lib/tool_shed/galaxy_install/repository_util.py +++ b/lib/tool_shed/galaxy_install/repository_util.py @@ -26,8 +26,6 @@ eggs.require( 'mercurial' ) from mercurial import commands -from mercurial import hg -from mercurial import ui log = logging.getLogger( __name__ ) @@ -186,8 +184,7 @@ def get_repo_info_dict( trans, repository_id, changeset_revision ): repository = suc.get_repository_in_tool_shed( trans, repository_id ) - repo_dir = repository.repo_path( trans.app ) - repo = hg.repository( hg_util.get_configured_ui(), repo_dir ) + repo = hg_util.get_repo_for_repository( trans.app, repository=repository, repo_path=None, create=False ) repository_clone_url = common_util.generate_clone_url_for_repository_in_tool_shed( trans, repository ) repository_metadata = suc.get_repository_metadata_by_changeset_revision( trans.app, repository_id, @@ -578,7 +575,10 @@ current_changeset_revision = changeset_revision_dict.get( 'changeset_revision', None ) current_ctx_rev = changeset_revision_dict.get( 'ctx_rev', None ) if current_ctx_rev != ctx_rev: - repo = hg.repository( hg_util.get_configured_ui(), path=os.path.abspath( install_dir ) ) + repo = hg_util.get_repo_for_repository( trans.app, + repository=None, + repo_path=os.path.abspath( install_dir ), + create=False ) pull_repository( repo, repository_clone_url, current_changeset_revision ) hg_util.update_repository( repo, ctx_rev=current_ctx_rev ) handle_repository_contents( trans, diff -r df18a38465c2cd7d7e8014febc3d955758f84671 -r 7eaa7e058c2898b10674b4ca43a0a76e566517c3 lib/tool_shed/grids/repository_grids.py --- a/lib/tool_shed/grids/repository_grids.py +++ b/lib/tool_shed/grids/repository_grids.py @@ -17,12 +17,6 @@ eggs.require('markupsafe') from markupsafe import escape as escape_html -eggs.require('mercurial') -from mercurial import commands -from mercurial import hg -from mercurial import patch -from mercurial import ui - log = logging.getLogger( __name__ ) class CategoryGrid( grids.Grid ): @@ -103,8 +97,7 @@ def get_value( self, trans, grid, repository ): """Display the current repository heads.""" - repo_dir = repository.repo_path( trans.app ) - repo = hg.repository( hg_util.get_configured_ui(), repo_dir ) + repo = hg_util.get_repo_for_repository( trans.app, repository=repository, repo_path=None, create=False ) heads = suc.get_repository_heads( repo ) multiple_heads = len( heads ) > 1 if multiple_heads: @@ -1477,8 +1470,10 @@ required_repository_id, changeset_revision ) if not required_repository_metadata: - repo_dir = required_repository.repo_path( trans.app ) - repo = hg.repository( hg_util.get_configured_ui(), repo_dir ) + repo = hg_util.get_repo_for_repository( trans.app, + repository=required_repository, + repo_path=None, + create=False ) updated_changeset_revision = \ suc.get_next_downloadable_changeset_revision( required_repository, repo, diff -r df18a38465c2cd7d7e8014febc3d955758f84671 -r 7eaa7e058c2898b10674b4ca43a0a76e566517c3 lib/tool_shed/grids/repository_review_grids.py --- a/lib/tool_shed/grids/repository_review_grids.py +++ b/lib/tool_shed/grids/repository_review_grids.py @@ -9,13 +9,6 @@ from tool_shed.util import hg_util from tool_shed.util import metadata_util -from galaxy import eggs -eggs.require('mercurial') -from mercurial import commands -from mercurial import hg -from mercurial import patch -from mercurial import ui - log = logging.getLogger( __name__ ) @@ -67,7 +60,7 @@ # Restrict to revisions that have been reviewed. if repository.reviews: rval = '' - repo = hg.repository( hg_util.get_configured_ui(), repository.repo_path( trans.app ) ) + repo = hg_util.get_repo_for_repository( trans.app, repository=repository, repo_path=None, create=False ) for review in repository.reviews: changeset_revision = review.changeset_revision rev, label = hg_util.get_rev_label_from_changeset_revision( repo, changeset_revision ) diff -r df18a38465c2cd7d7e8014febc3d955758f84671 -r 7eaa7e058c2898b10674b4ca43a0a76e566517c3 lib/tool_shed/grids/util.py --- a/lib/tool_shed/grids/util.py +++ b/lib/tool_shed/grids/util.py @@ -168,7 +168,7 @@ tool_dependency_definition. """ encoded_repository_id = trans.security.encode_id( repository.id ) - repo = hg.repository( hg_util.get_configured_ui(), repository.repo_path( trans.app ) ) + repo = hg_util.get_repo_for_repository( trans.app, repository=repository, repo_path=None, create=False ) tip_ctx = str( repo.changectx( repo.changelog.tip() ) ) repository_metadata = None try: @@ -218,7 +218,7 @@ tool_dependency_definition. """ encoded_repository_id = trans.security.encode_id( repository.id ) - repo = hg.repository( hg_util.get_configured_ui(), repository.repo_path( trans.app ) ) + repo = hg_util.get_repo_for_repository( trans.app, repository=repository, repo_path=None, create=False ) tip_ctx = str( repo.changectx( repo.changelog.tip() ) ) try: repository_metadata = suc.get_repository_metadata_by_changeset_revision( trans.app, encoded_repository_id, tip_ctx ) diff -r df18a38465c2cd7d7e8014febc3d955758f84671 -r 7eaa7e058c2898b10674b4ca43a0a76e566517c3 lib/tool_shed/repository_registry.py --- a/lib/tool_shed/repository_registry.py +++ b/lib/tool_shed/repository_registry.py @@ -62,7 +62,7 @@ """ if repository.deleted: return ( None, False ) - repo = hg_util.get_repo_for_repository( self.app, repository ) + repo = hg_util.get_repo_for_repository( self.app, repository=repository, repo_path=None, create=False ) # Get the latest installable changeset revision since that is all that is currently configured for testing. latest_installable_changeset_revision = suc.get_latest_downloadable_changeset_revision( self.app, repository, repo ) if latest_installable_changeset_revision not in [ None, suc.INITIAL_CHANGELOG_HASH ]: diff -r df18a38465c2cd7d7e8014febc3d955758f84671 -r 7eaa7e058c2898b10674b4ca43a0a76e566517c3 lib/tool_shed/scripts/check_repositories_for_functional_tests.py --- a/lib/tool_shed/scripts/check_repositories_for_functional_tests.py +++ b/lib/tool_shed/scripts/check_repositories_for_functional_tests.py @@ -28,8 +28,6 @@ from galaxy.model.orm import not_ from galaxy.model.orm import select from galaxy.util import listify -from mercurial import hg -from mercurial import ui from mercurial import __version__ from optparse import OptionParser from time import strftime @@ -110,7 +108,7 @@ if tool_dicts is not None: # Clone the repository up to the changeset revision we're checking. repo_dir = repository.repo_path( app ) - repo = hg.repository( hg_util.get_configured_ui(), repo_dir ) + repo = hg_util.get_repo_for_repository( app, repository=None, repo_path=repo_dir, create=False ) work_dir = tempfile.mkdtemp( prefix="tmp-toolshed-cafr" ) cloned_ok, error_message = hg_util.clone_repository( repo_dir, work_dir, changeset_revision ) if cloned_ok: @@ -316,14 +314,6 @@ missing_test_files.append( required_file ) return missing_test_files -def get_repo_changelog_tuples( repo_path ): - repo = hg.repository( ui.ui(), repo_path ) - changelog_tuples = [] - for changeset in repo.changelog: - ctx = repo.changectx( changeset ) - changelog_tuples.append( ( ctx.rev(), str( ctx ) ) ) - return changelog_tuples - def main(): '''Script that checks repositories to see if the tools contained within them have functional tests defined.''' parser = OptionParser() @@ -368,8 +358,7 @@ would be redundant. """ if not testable_revision: - repo_dir = repository.repo_path( app ) - repo = hg.repository( hg_util.get_configured_ui(), repo_dir ) + repo = hg_util.get_repo_for_repository( app, repository=repository, repo_path=None, create=False ) changeset_revisions = suc.get_ordered_metadata_changeset_revisions( repository, repo, downloadable=True ) if len( changeset_revisions ) > 1: latest_downloadable_revision = changeset_revisions[ -1 ] diff -r df18a38465c2cd7d7e8014febc3d955758f84671 -r 7eaa7e058c2898b10674b4ca43a0a76e566517c3 lib/tool_shed/util/commit_util.py --- a/lib/tool_shed/util/commit_util.py +++ b/lib/tool_shed/util/commit_util.py @@ -20,11 +20,8 @@ from galaxy import eggs eggs.require( 'mercurial' ) from mercurial import commands -from mercurial import hg -from mercurial import ui from mercurial.changegroup import readbundle from mercurial.changegroup import readexactly -from mercurial.changegroup import writebundle log = logging.getLogger( __name__ ) @@ -156,8 +153,7 @@ def handle_directory_changes( trans, repository, full_path, filenames_in_archive, remove_repo_files_not_in_tar, new_repo_alert, commit_message, undesirable_dirs_removed, undesirable_files_removed ): - repo_dir = repository.repo_path( trans.app ) - repo = hg.repository( hg_util.get_configured_ui(), repo_dir ) + repo = hg_util.get_repo_for_repository( trans.app, repository=repository, repo_path=None, create=False ) content_alert_str = '' files_to_remove = [] filenames_in_archive = [ os.path.join( full_path, name ) for name in filenames_in_archive ] @@ -342,8 +338,7 @@ # revision are valid. repository = suc.get_repository_by_name_and_owner( trans.app, name, owner ) if repository: - repo_dir = repository.repo_path( trans.app ) - repo = hg.repository( hg_util.get_configured_ui(), repo_dir ) + repo = hg_util.get_repo_for_repository( trans.app, repository=repository, repo_path=None, create=False ) lastest_installable_changeset_revision = suc.get_latest_downloadable_changeset_revision( trans.app, repository, repo ) if lastest_installable_changeset_revision != suc.INITIAL_CHANGELOG_HASH: elem.attrib[ 'changeset_revision' ] = lastest_installable_changeset_revision diff -r df18a38465c2cd7d7e8014febc3d955758f84671 -r 7eaa7e058c2898b10674b4ca43a0a76e566517c3 lib/tool_shed/util/export_util.py --- a/lib/tool_shed/util/export_util.py +++ b/lib/tool_shed/util/export_util.py @@ -21,10 +21,7 @@ eggs.require( 'mercurial' ) -import mercurial.util from mercurial import commands -from mercurial import hg -from mercurial import patch from mercurial import ui log = logging.getLogger( __name__ ) @@ -39,7 +36,7 @@ def archive_repository_revision( trans, ui, repository, archive_dir, changeset_revision ): '''Create an un-versioned archive of a repository.''' - repo = hg.repository( hg_util.get_configured_ui(), repository.repo_path( trans.app ) ) + repo = hg_util.get_repo_for_repository( trans.app, repository=repository, repo_path=None, create=False ) options_dict = hg_util.get_mercurial_default_options_dict( 'archive' ) options_dict[ 'rev' ] = changeset_revision error_message = '' @@ -242,7 +239,7 @@ key_rd_dicts_to_be_processed=None, all_repository_dependencies=None, handled_key_rd_dicts=None ) - repo = hg.repository( hg_util.get_configured_ui(), repository.repo_path( trans.app ) ) + repo = hg_util.get_repo_for_repository( trans.app, repository=repository, repo_path=None, create=False ) ctx = hg_util.get_changectx_for_changeset( repo, changeset_revision ) repo_info_dict = {} # Cast unicode to string. diff -r df18a38465c2cd7d7e8014febc3d955758f84671 -r 7eaa7e058c2898b10674b4ca43a0a76e566517c3 lib/tool_shed/util/hg_util.py --- a/lib/tool_shed/util/hg_util.py +++ b/lib/tool_shed/util/hg_util.py @@ -69,15 +69,18 @@ ctx_date = date.strftime( "%Y-%m-%d" ) return ctx_date -def get_repo_for_repository( app, repository ): - return hg.repository( get_configured_ui(), repository.repo_path( app ) ) +def get_repo_for_repository( app, repository=None, repo_path=None, create=False ): + if repository is not None: + return hg.repository( get_configured_ui(), repository.repo_path( app ), create=create ) + if repo_path is not None: + return hg.repository( get_configured_ui(), repo_path, create=create ) def get_revision_label( trans, repository, changeset_revision, include_date=True, include_hash=True ): """ Return a string consisting of the human read-able changeset rev and the changeset revision string which includes the revision date if the receive include_date is True. """ - repo = get_repo_for_repository( trans.app, repository ) + repo = get_repo_for_repository( trans.app, repository=repository, repo_path=None ) ctx = get_changectx_for_changeset( repo, changeset_revision ) if ctx: return get_revision_label_from_ctx( ctx, include_date=include_date, include_hash=include_hash ) diff -r df18a38465c2cd7d7e8014febc3d955758f84671 -r 7eaa7e058c2898b10674b4ca43a0a76e566517c3 lib/tool_shed/util/import_util.py --- a/lib/tool_shed/util/import_util.py +++ b/lib/tool_shed/util/import_util.py @@ -14,13 +14,6 @@ import tool_shed.util.shed_util_common as suc import tool_shed.repository_types.util as rt_util -from galaxy import eggs -eggs.require( 'mercurial' ) - -from mercurial import commands -from mercurial import hg -from mercurial import ui - log = logging.getLogger( __name__ ) def check_status_and_reset_downloadable( trans, import_results_tups ): @@ -214,7 +207,7 @@ archive_file_path = os.path.join( file_path, archive_file_name ) archive = tarfile.open( archive_file_path, 'r:*' ) repo_dir = repository.repo_path( trans.app ) - repo = hg.repository( hg_util.get_configured_ui(), repo_dir ) + repo = hg_util.get_repo_for_repository( trans.app, repository=None, repo_path=repo_dir, create=False ) undesirable_dirs_removed = 0 undesirable_files_removed = 0 ok, error_message = commit_util.check_archive( repository, archive ) diff -r df18a38465c2cd7d7e8014febc3d955758f84671 -r 7eaa7e058c2898b10674b4ca43a0a76e566517c3 lib/tool_shed/util/metadata_util.py --- a/lib/tool_shed/util/metadata_util.py +++ b/lib/tool_shed/util/metadata_util.py @@ -1,7 +1,7 @@ import logging import os import tempfile -from galaxy import eggs + from galaxy import util from galaxy import web from galaxy.datatypes import checkers @@ -10,6 +10,7 @@ from galaxy.util import inflector from galaxy.util import json from galaxy.web import url_for + import tool_shed.util.shed_util_common as suc from tool_shed.repository_types.metadata import TipOnly from tool_shed.util import common_util @@ -23,12 +24,6 @@ from tool_shed.galaxy_install.tool_dependencies import td_common_util import tool_shed.repository_types.util as rt_util -eggs.require( 'mercurial' ) - -from mercurial import commands -from mercurial import hg -from mercurial import ui - log = logging.getLogger( __name__ ) # Repository metadata comparisons for changeset revisions. @@ -352,7 +347,7 @@ # changeset revision in the received repository's changelog (up to the received changeset revision) to see if it is contained in the # skip_tool_test table. If it is, but is not associated with a repository_metadata record, reset that skip_tool_test record to the # newly created repository_metadata record. - repo = hg.repository( hg_util.get_configured_ui(), repository.repo_path( trans.app ) ) + repo = hg_util.get_repo_for_repository( trans.app, repository=repository, repo_path=None, create=False ) for changeset in repo.changelog: changeset_hash = str( repo.changectx( changeset ) ) skip_tool_test = suc.get_skip_tool_test_by_changeset_revision( trans, changeset_hash ) @@ -1051,7 +1046,7 @@ def get_latest_repository_metadata( trans, decoded_repository_id, downloadable=False ): """Get last metadata defined for a specified repository from the database.""" repository = trans.sa_session.query( trans.model.Repository ).get( decoded_repository_id ) - repo = hg.repository( hg_util.get_configured_ui(), repository.repo_path( trans.app ) ) + repo = hg_util.get_repo_for_repository( trans.app, repository=repository, repo_path=None, create=False ) if downloadable: changeset_revision = suc.get_latest_downloadable_changeset_revision( trans.app, repository, @@ -1288,7 +1283,8 @@ log.debug( error_message ) is_valid = False return repository_dependency_tup, is_valid, error_message - repo = hg.repository( hg_util.get_configured_ui(), repository.repo_path( app ) ) + repo = hg_util.get_repo_for_repository( app, repository=repository, repo_path=None, create=False ) + # The received changeset_revision may be None since defining it in the dependency definition is optional. # If this is the case, the default will be to set its value to the repository dependency tip revision. # This probably occurs only when handling circular dependency definitions. @@ -1734,7 +1730,7 @@ repository = suc.get_repository_in_tool_shed( trans, id ) log.debug( "Resetting all metadata on repository: %s" % repository.name ) repo_dir = repository.repo_path( trans.app ) - repo = hg.repository( hg_util.get_configured_ui(), repo_dir ) + repo = hg_util.get_repo_for_repository( trans.app, repository=None, repo_path=repo_dir, create=False ) repository_clone_url = common_util.generate_clone_url_for_repository_in_tool_shed( trans, repository ) # The list of changeset_revisions refers to repository_metadata records that have been created or updated. When the following loop # completes, we'll delete all repository_metadata records for this repository that do not have a changeset_revision value in this list. @@ -1916,7 +1912,7 @@ encoded_id = trans.security.encode_id( repository.id ) repository_clone_url = common_util.generate_clone_url_for_repository_in_tool_shed( trans, repository ) repo_dir = repository.repo_path( trans.app ) - repo = hg.repository( hg_util.get_configured_ui(), repo_dir ) + repo = hg_util.get_repo_for_repository( trans.app, repository=None, repo_path=repo_dir, create=False ) metadata_dict, invalid_file_tups = generate_metadata_for_changeset_revision( app=trans.app, repository=repository, changeset_revision=repository.tip( trans.app ), diff -r df18a38465c2cd7d7e8014febc3d955758f84671 -r 7eaa7e058c2898b10674b4ca43a0a76e566517c3 lib/tool_shed/util/readme_util.py --- a/lib/tool_shed/util/readme_util.py +++ b/lib/tool_shed/util/readme_util.py @@ -1,20 +1,18 @@ import logging import os import threading + from mako.template import Template -from galaxy import eggs + from galaxy import web from galaxy.util import json from galaxy.util import rst_to_html from galaxy.util import unicodify + import tool_shed.util.shed_util_common as suc from tool_shed.util import common_util from tool_shed.util import hg_util -eggs.require( 'mercurial' ) - -from mercurial import hg - log = logging.getLogger( __name__ ) def build_readme_files_dict( trans, repository, changeset_revision, metadata, tool_path=None ): @@ -26,7 +24,7 @@ if trans.webapp.name == 'galaxy': can_use_disk_files = True else: - repo = hg.repository( hg_util.get_configured_ui(), repository.repo_path( trans.app ) ) + repo = hg_util.get_repo_for_repository( trans.app, repository=repository, repo_path=None, create=False ) latest_downloadable_changeset_revision = suc.get_latest_downloadable_changeset_revision( trans.app, repository, repo ) can_use_disk_files = changeset_revision == latest_downloadable_changeset_revision readme_files_dict = {} diff -r df18a38465c2cd7d7e8014febc3d955758f84671 -r 7eaa7e058c2898b10674b4ca43a0a76e566517c3 lib/tool_shed/util/repository_dependency_util.py --- a/lib/tool_shed/util/repository_dependency_util.py +++ b/lib/tool_shed/util/repository_dependency_util.py @@ -1,9 +1,10 @@ import logging import os -from galaxy import eggs + from galaxy.util import asbool from galaxy.util import json from galaxy.util import listify + import tool_shed.util.shed_util_common as suc from tool_shed.util import common_util from tool_shed.util import common_install_util @@ -13,12 +14,6 @@ from tool_shed.util import metadata_util from tool_shed.util import tool_util -eggs.require( 'mercurial' ) - -from mercurial import commands -from mercurial import hg -from mercurial import ui - log = logging.getLogger( __name__ ) def build_repository_dependency_relationships( trans, repo_info_dicts, tool_shed_repositories ): @@ -536,12 +531,12 @@ updated_key_rd_dicts.append( key_rd_dict ) else: # The repository changeset_revision is no longer installable, so see if there's been an update. - repo_dir = repository.repo_path( trans.app ) - repo = hg.repository( hg_util.get_configured_ui(), repo_dir ) + repo = hg_util.get_repo_for_repository( trans.app, repository=repository, repo_path=None, create=False ) changeset_revision = suc.get_next_downloadable_changeset_revision( repository, repo, rd_changeset_revision ) - repository_metadata = metadata_util.get_repository_metadata_by_repository_id_changeset_revision( trans, - trans.security.encode_id( repository.id ), - changeset_revision ) + repository_metadata = \ + metadata_util.get_repository_metadata_by_repository_id_changeset_revision( trans, + trans.security.encode_id( repository.id ), + changeset_revision ) if repository_metadata: new_key_rd_dict = {} new_key_rd_dict[ key ] = \ diff -r df18a38465c2cd7d7e8014febc3d955758f84671 -r 7eaa7e058c2898b10674b4ca43a0a76e566517c3 lib/tool_shed/util/repository_maintenance_util.py --- a/lib/tool_shed/util/repository_maintenance_util.py +++ b/lib/tool_shed/util/repository_maintenance_util.py @@ -9,14 +9,6 @@ from galaxy.web.form_builder import build_select_field from galaxy.webapps.tool_shed.model import directory_hash_id -from galaxy import eggs -eggs.require( 'mercurial' ) - -from mercurial import commands -from mercurial import hg -from mercurial import patch -from mercurial import ui - log = logging.getLogger( __name__ ) VALID_REPOSITORYNAME_RE = re.compile( "^[a-z0-9\_]+$" ) @@ -48,7 +40,7 @@ # Since we support both http and https, we set push_ssl to False to override the default (which is True) in the mercurial api. The hg # purge extension purges all files and directories not being tracked by mercurial in the current repository. It'll remove unknown files # and empty directories. This is not currently used because it is not supported in the mercurial API. - repo = hg.repository( hg_util.get_configured_ui(), path=repository.repo_path( trans.app ) ) + repo = hg_util.get_repo_for_repository( trans.app, repository=repository, repo_path=None, create=False ) fp = repo.opener( 'hgrc', 'wb' ) fp.write( '[paths]\n' ) fp.write( 'default = .\n' ) @@ -84,7 +76,7 @@ if not os.path.exists( repository_path ): os.makedirs( repository_path ) # Create the local repository. - repo = hg.repository( hg_util.get_configured_ui(), repository_path, create=True ) + repo = hg_util.get_repo_for_repository( trans.app, repository=None, repo_path=repository_path, create=True ) # Add an entry in the hgweb.config file for the local repository. lhs = "repos/%s/%s" % ( repository.user.username, repository.name ) trans.app.hgweb_config_manager.add_entry( lhs, repository_path ) diff -r df18a38465c2cd7d7e8014febc3d955758f84671 -r 7eaa7e058c2898b10674b4ca43a0a76e566517c3 lib/tool_shed/util/review_util.py --- a/lib/tool_shed/util/review_util.py +++ b/lib/tool_shed/util/review_util.py @@ -1,17 +1,10 @@ import logging import os -from galaxy import eggs from galaxy.model.orm import and_ from galaxy.util.odict import odict from tool_shed.util import hg_util import tool_shed.util.shed_util_common as suc -eggs.require( 'mercurial' ) - -from mercurial import commands -from mercurial import hg -from mercurial import ui - log = logging.getLogger( __name__ ) def can_browse_repository_reviews( trans, repository ): @@ -59,7 +52,7 @@ def get_previous_repository_reviews( trans, repository, changeset_revision ): """Return an ordered dictionary of repository reviews up to and including the received changeset revision.""" - repo = hg.repository( hg_util.get_configured_ui(), repository.repo_path( trans.app ) ) + repo = hg_util.get_repo_for_repository( trans.app, repository=repository, repo_path=None, create=False ) reviewed_revision_hashes = [ review.changeset_revision for review in repository.reviews ] previous_reviews_dict = odict() for changeset in suc.reversed_upper_bounded_changelog( repo, changeset_revision ): @@ -94,7 +87,7 @@ def has_previous_repository_reviews( trans, repository, changeset_revision ): """Determine if a repository has a changeset revision review prior to the received changeset revision.""" - repo = hg.repository( hg_util.get_configured_ui(), repository.repo_path( trans.app ) ) + repo = hg_util.get_repo_for_repository( trans.app, repository=repository, repo_path=None, create=False ) reviewed_revision_hashes = [ review.changeset_revision for review in repository.reviews ] for changeset in suc.reversed_upper_bounded_changelog( repo, changeset_revision ): previous_changeset_revision = str( repo.changectx( changeset ) ) diff -r df18a38465c2cd7d7e8014febc3d955758f84671 -r 7eaa7e058c2898b10674b4ca43a0a76e566517c3 lib/tool_shed/util/shed_util_common.py --- a/lib/tool_shed/util/shed_util_common.py +++ b/lib/tool_shed/util/shed_util_common.py @@ -23,9 +23,6 @@ from urllib2 import HTTPError from galaxy import eggs -eggs.require( 'mercurial' ) - -from mercurial import hg eggs.require( 'markupsafe' ) import markupsafe @@ -474,7 +471,7 @@ return repository_metadata # The installable changeset_revision may have been changed because it was "moved ahead" # in the repository changelog. - repo = hg.repository( hg_util.get_configured_ui(), repository.repo_path( app ) ) + repo = hg_util.get_repo_for_repository( app, repository=repository, repo_path=None, create=False ) updated_changeset_revision = get_next_downloadable_changeset_revision( repository, repo, after_changeset_revision=changeset_revision ) @@ -528,8 +525,10 @@ # The defined changeset_revision is not associated with a repository_metadata # record, so updates must be necessary. defined_repository = get_repository_by_name_and_owner( trans.app, name, owner ) - defined_repo_dir = defined_repository.repo_path( trans.app ) - defined_repo = hg.repository( hg_util.get_configured_ui(), defined_repo_dir ) + defined_repo = hg_util.get_repo_for_repository( trans.app, + repository=defined_repository, + repo_path=None, + create=False ) updated_changeset_revision = \ get_next_downloadable_changeset_revision( defined_repository, defined_repo, @@ -1324,8 +1323,7 @@ revision for the repository defined by the received name and owner. """ repository = get_repository_by_name_and_owner( trans.app, name, owner ) - repo_dir = repository.repo_path( trans.app ) - repo = hg.repository( hg_util.get_configured_ui(), repo_dir ) + repo = hg_util.get_repo_for_repository( trans.app, repository=repository, repo_path=None, create=False ) # Get the upper bound changeset revision. upper_bound_changeset_revision = get_next_downloadable_changeset_revision( repository, repo, changeset_revision ) # Build the list of changeset revision hashes defining each available update up to, but excluding, upper_bound_changeset_revision. @@ -1389,8 +1387,7 @@ user is not an admin user, the email will not include any information about both HTML and image content that was included in the change set. """ - repo_dir = repository.repo_path( trans.app ) - repo = hg.repository( hg_util.get_configured_ui(), repo_dir ) + repo = hg_util.get_repo_for_repository( trans.app, repository=repository, repo_path=None, create=False ) sharable_link = generate_sharable_link_for_repository_in_tool_shed( repository, changeset_revision=None ) smtp_server = trans.app.config.smtp_server if smtp_server and ( new_repo_alert or repository.email_alerts ): diff -r df18a38465c2cd7d7e8014febc3d955758f84671 -r 7eaa7e058c2898b10674b4ca43a0a76e566517c3 lib/tool_shed/util/tool_util.py --- a/lib/tool_shed/util/tool_util.py +++ b/lib/tool_shed/util/tool_util.py @@ -4,7 +4,6 @@ import shutil import tempfile import galaxy.tools -from galaxy import eggs from galaxy import util from galaxy.datatypes import checkers from galaxy.model.orm import and_ @@ -20,12 +19,6 @@ import tool_shed.util.shed_util_common as suc from xml.etree import ElementTree as XmlET -eggs.require( 'mercurial' ) - -from mercurial import commands -from mercurial import hg -from mercurial import ui - log = logging.getLogger( __name__ ) def add_to_shed_tool_config( app, shed_tool_conf_dict, elem_list ): @@ -538,8 +531,7 @@ repsitory_metadata.tool_versions. """ repository = suc.get_repository_by_id( trans, repository_id ) - repo_dir = repository.repo_path( trans.app ) - repo = hg.repository( hg_util.get_configured_ui(), repo_dir ) + repo = hg_util.get_repo_for_repository( trans.app, repository=repository, repo_path=None, create=False ) # Initialize the tool lineage version_lineage = [ guid ] # Get all ancestor guids of the received guid. @@ -867,7 +859,7 @@ original_tool_data_path = trans.app.config.tool_data_path repository = suc.get_repository_in_tool_shed( trans, repository_id ) repo_files_dir = repository.repo_path( trans.app ) - repo = hg.repository( hg_util.get_configured_ui(), repo_files_dir ) + repo = hg_util.get_repo_for_repository( trans.app, repository=None, repo_path=repo_files_dir, create=False ) message = '' tool = None can_use_disk_file = False Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: jmchilton: More stdio handling tests.
by commits-noreply＠bitbucket.org 21 May '14

21 May '14

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/df18a38465c2/ Changeset: df18a38465c2 User: jmchilton Date: 2014-05-21 18:11:39 Summary: More stdio handling tests. Affected #: 1 file diff -r d0979f737213cbc8e000a46f90a7cd5501c709d1 -r df18a38465c2cd7d7e8014febc3d955758f84671 test/unit/jobs/test_job_output_checker.py --- a/test/unit/jobs/test_job_output_checker.py +++ b/test/unit/jobs/test_job_output_checker.py @@ -2,6 +2,7 @@ from galaxy.util.bunch import Bunch from galaxy.jobs.output_checker import check_output from galaxy.jobs.error_level import StdioErrorLevel +from galaxy.tools import ToolStdioRegex class OutputCheckerTestCase( TestCase ): @@ -39,19 +40,53 @@ self.tool_exit_code = 0 self.__assertSuccessful() - def test_problematic_strings( self ): + def test_problematic_strings_matching( self ): problematic_str = '\x80abc' - regex_rule = Bunch( match=r'.abc', stdout_match=False, stderr_match=True, error_level=StdioErrorLevel.FATAL, desc=None ) - self.tool.stdio_regexes = [ regex_rule ] + self.__add_regex( Bunch( match=r'.abc', stdout_match=False, stderr_match=True, error_level=StdioErrorLevel.FATAL, desc=None ) ) self.stderr = problematic_str self.__assertNotSuccessful() + def test_problematic_strings_not_matching( self ): problematic_str = '\x80abc' - regex_rule = Bunch( match=r'.abcd', stdout_match=False, stderr_match=True, error_level=StdioErrorLevel.FATAL, desc=None ) - self.tool.stdio_regexes = [ regex_rule ] + self.__add_regex( Bunch( match=r'.abcd', stdout_match=False, stderr_match=True, error_level=StdioErrorLevel.FATAL, desc=None ) ) self.stderr = problematic_str self.__assertSuccessful() + def test_stderr_regex_negative_match( self ): + regex = ToolStdioRegex() + regex.stderr_match = True + regex.match = "foobar" + self.__add_regex( regex ) + self.stderr = "foo" + self.__assertSuccessful() + + def test_stderr_regex_positive_match( self ): + regex = ToolStdioRegex() + regex.stderr_match = True + regex.match = "foo" + self.__add_regex( regex ) + self.stderr = "foobar" + self.__assertNotSuccessful() + + def test_stdout_ignored_for_stderr_regexes( self ): + regex = ToolStdioRegex() + regex.stderr_match = True + regex.match = "foo" + self.__add_regex( regex ) + self.stdout = "foobar" + self.__assertSuccessful() + + def test_stderr_ignored_for_stdout_regexes( self ): + regex = ToolStdioRegex() + regex.stdout_match = True + regex.match = "foo" + self.__add_regex( regex ) + self.stderr = "foobar" + self.__assertSuccessful() + + def __add_regex( self, regex ): + self.tool.stdio_regexes.append( regex ) + def __assertSuccessful( self ): self.assertTrue( self.__check_output() ) Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: greg: Change function signatures for several utility functions that used to require trans but now only need app.
by commits-noreply＠bitbucket.org 21 May '14

21 May '14

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/d0979f737213/ Changeset: d0979f737213 User: greg Date: 2014-05-21 15:55:12 Summary: Change function signatures for several utility functions that used to require trans but now only need app. Affected #: 13 files diff -r e3c0070b64bae25215e9a30822f6e7488e938969 -r d0979f737213cbc8e000a46f90a7cd5501c709d1 lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py --- a/lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py +++ b/lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py @@ -1,6 +1,7 @@ import logging import os import shutil + from admin import AdminGalaxy from galaxy import eggs from galaxy import web @@ -10,7 +11,10 @@ from galaxy.web.framework.helpers import iff from galaxy.util import json from galaxy.model.orm import or_ + import tool_shed.util.shed_util_common as suc +import tool_shed.repository_types.util as rt_util + from tool_shed.util import common_util from tool_shed.util import common_install_util from tool_shed.util import data_manager_util @@ -26,6 +30,7 @@ from tool_shed.util import xml_util from tool_shed.galaxy_install import repository_util import tool_shed.galaxy_install.grids.admin_toolshed_grids as admin_toolshed_grids + import pkg_resources eggs.require( 'mercurial' ) @@ -458,7 +463,7 @@ err_msg = '' tool_shed_repository = tool_dependencies[ 0 ].tool_shed_repository # Get the tool_dependencies.xml file from the repository. - tool_dependencies_config = suc.get_config_from_disk( suc.TOOL_DEPENDENCY_DEFINITION_FILENAME, + tool_dependencies_config = suc.get_config_from_disk( rt_util.TOOL_DEPENDENCY_DEFINITION_FILENAME, tool_shed_repository.repo_path( trans.app ) ) installed_tool_dependencies = \ common_install_util.install_specified_packages( app=trans.app, diff -r e3c0070b64bae25215e9a30822f6e7488e938969 -r d0979f737213cbc8e000a46f90a7cd5501c709d1 lib/galaxy/webapps/tool_shed/controllers/repository.py --- a/lib/galaxy/webapps/tool_shed/controllers/repository.py +++ b/lib/galaxy/webapps/tool_shed/controllers/repository.py @@ -1713,11 +1713,14 @@ changeset_revision = kwd.get( 'changeset_revision', None ) repository = suc.get_repository_by_name_and_owner( trans.app, name, owner ) repository_id = trans.security.encode_id( repository.id ) - # We aren't concerned with repositories of type tool_dependency_definition here if a repository_metadata record is not returned - # because repositories of this type will never have repository dependencies. However, if a readme file is uploaded, or some other - # change is made that does not create a new downloadable changeset revision but updates the existing one, we still want to be able - # to get repository dependencies. - repository_metadata = suc.get_current_repository_metadata_for_changeset_revision( trans, repository, changeset_revision ) + # We aren't concerned with repositories of type tool_dependency_definition here if a + # repository_metadata record is not returned because repositories of this type will never + # have repository dependencies. However, if a readme file is uploaded, or some other change + # is made that does not create a new downloadable changeset revision but updates the existing + # one, we still want to be able to get repository dependencies. + repository_metadata = suc.get_current_repository_metadata_for_changeset_revision( trans.app, + repository, + changeset_revision ) if repository_metadata: metadata = repository_metadata.metadata if metadata: @@ -1838,7 +1841,7 @@ #manafest. repo_dir = repository.repo_path( trans.app ) # Get the tool_dependencies.xml file from disk. - tool_dependencies_config = suc.get_config_from_disk( suc.TOOL_DEPENDENCY_DEFINITION_FILENAME, repo_dir ) + tool_dependencies_config = suc.get_config_from_disk( rt_util.TOOL_DEPENDENCY_DEFINITION_FILENAME, repo_dir ) # Return the encoded contents of the tool_dependencies.xml file. if tool_dependencies_config: tool_dependencies_config_file = open( tool_dependencies_config, 'rb' ) @@ -2362,13 +2365,15 @@ handled_key_rd_dicts=None ) if str( repository.type ) != rt_util.REPOSITORY_SUITE_DEFINITION: # Handle messaging for resetting repository type to the optimal value. - change_repository_type_message = suc.generate_message_for_repository_type_change( trans, repository ) + change_repository_type_message = rt_util.generate_message_for_repository_type_change( trans.app, + repository ) if change_repository_type_message: message += change_repository_type_message status = 'warning' elif str( repository.type ) != rt_util.TOOL_DEPENDENCY_DEFINITION: # Handle messaging for resetting repository type to the optimal value. - change_repository_type_message = suc.generate_message_for_repository_type_change( trans, repository ) + change_repository_type_message = rt_util.generate_message_for_repository_type_change( trans.app, + repository ) if change_repository_type_message: message += change_repository_type_message status = 'warning' @@ -3245,7 +3250,7 @@ status = 'warning' else: metadata = None - is_malicious = suc.changeset_is_malicious( trans, id, repository.tip( trans.app ) ) + is_malicious = suc.changeset_is_malicious( trans.app, id, repository.tip( trans.app ) ) if is_malicious: if trans.app.security_agent.can_push( trans.app, trans.user, repository ): message += malicious_error_can_push diff -r e3c0070b64bae25215e9a30822f6e7488e938969 -r d0979f737213cbc8e000a46f90a7cd5501c709d1 lib/galaxy/webapps/tool_shed/controllers/upload.py --- a/lib/galaxy/webapps/tool_shed/controllers/upload.py +++ b/lib/galaxy/webapps/tool_shed/controllers/upload.py @@ -149,12 +149,12 @@ isgzip=isgzip, isbz2=isbz2 ) if repository.type == rt_util.REPOSITORY_SUITE_DEFINITION and \ - uploaded_file_filename != suc.REPOSITORY_DEPENDENCY_DEFINITION_FILENAME: + uploaded_file_filename != rt_util.REPOSITORY_DEPENDENCY_DEFINITION_FILENAME: ok = False message = 'Repositories of type <b>Repository suite definition</b> can only contain a single file named ' message += '<b>repository_dependencies.xml</b>.' elif repository.type == rt_util.TOOL_DEPENDENCY_DEFINITION and \ - uploaded_file_filename != suc.TOOL_DEPENDENCY_DEFINITION_FILENAME: + uploaded_file_filename != rt_util.TOOL_DEPENDENCY_DEFINITION_FILENAME: ok = False message = 'Repositories of type <b>Tool dependency definition</b> can only contain a single file named ' message += '<b>tool_dependencies.xml</b>.' @@ -164,7 +164,7 @@ else: full_path = os.path.abspath( os.path.join( repo_dir, uploaded_file_filename ) ) # Move some version of the uploaded file to the load_point within the repository hierarchy. - if uploaded_file_filename in [ suc.REPOSITORY_DEPENDENCY_DEFINITION_FILENAME ]: + if uploaded_file_filename in [ rt_util.REPOSITORY_DEPENDENCY_DEFINITION_FILENAME ]: # Inspect the contents of the file to see if changeset_revision values are missing and if so, # set them appropriately. altered, root_elem, error_message = \ @@ -180,8 +180,8 @@ shutil.move( tmp_filename, full_path ) else: shutil.move( uploaded_file_name, full_path ) - elif uploaded_file_filename in [ suc.REPOSITORY_DEPENDENCY_DEFINITION_FILENAME, - suc.TOOL_DEPENDENCY_DEFINITION_FILENAME ]: + elif uploaded_file_filename in [ rt_util.REPOSITORY_DEPENDENCY_DEFINITION_FILENAME, + rt_util.TOOL_DEPENDENCY_DEFINITION_FILENAME ]: # Inspect the contents of the file to see if changeset_revision values are # missing and if so, set them appropriately. altered, root_elem, error_message = \ @@ -264,7 +264,8 @@ metadata_dict = {} if str( repository.type ) not in [ rt_util.REPOSITORY_SUITE_DEFINITION, rt_util.TOOL_DEPENDENCY_DEFINITION ]: - change_repository_type_message = suc.generate_message_for_repository_type_change( trans, repository ) + change_repository_type_message = rt_util.generate_message_for_repository_type_change( trans.app, + repository ) if change_repository_type_message: message += change_repository_type_message status = 'warning' @@ -336,9 +337,9 @@ for uploaded_file in files: relative_path = os.path.normpath( os.path.join( os.path.relpath( root, uploaded_directory ), uploaded_file ) ) if repository.type == rt_util.REPOSITORY_SUITE_DEFINITION: - ok = os.path.basename( uploaded_file ) == suc.REPOSITORY_DEPENDENCY_DEFINITION_FILENAME + ok = os.path.basename( uploaded_file ) == rt_util.REPOSITORY_DEPENDENCY_DEFINITION_FILENAME elif repository.type == rt_util.TOOL_DEPENDENCY_DEFINITION: - ok = os.path.basename( uploaded_file ) == suc.TOOL_DEPENDENCY_DEFINITION_FILENAME + ok = os.path.basename( uploaded_file ) == rt_util.TOOL_DEPENDENCY_DEFINITION_FILENAME else: ok = os.path.basename( uploaded_file ) not in commit_util.UNDESIRABLE_FILES if ok: @@ -351,7 +352,7 @@ undesirable_files_removed += 1 if ok: uploaded_file_name = os.path.abspath( os.path.join( root, uploaded_file ) ) - if os.path.split( uploaded_file_name )[ -1 ] == suc.REPOSITORY_DEPENDENCY_DEFINITION_FILENAME: + if os.path.split( uploaded_file_name )[ -1 ] == rt_util.REPOSITORY_DEPENDENCY_DEFINITION_FILENAME: # Inspect the contents of the file to see if changeset_revision values are missing and # if so, set them appropriately. altered, root_elem, error_message = \ @@ -363,7 +364,7 @@ elif altered: tmp_filename = xml_util.create_and_write_tmp_file( root_elem ) shutil.move( tmp_filename, uploaded_file_name ) - elif os.path.split( uploaded_file_name )[ -1 ] == suc.TOOL_DEPENDENCY_DEFINITION_FILENAME: + elif os.path.split( uploaded_file_name )[ -1 ] == rt_util.TOOL_DEPENDENCY_DEFINITION_FILENAME: # Inspect the contents of the file to see if changeset_revision values are missing and if so, set them appropriately. altered, root_elem, error_message = commit_util.handle_tool_dependencies_definition( trans, uploaded_file_name ) if error_message: @@ -420,7 +421,7 @@ uploaded_file.close() for filename in filenames_in_archive: uploaded_file_name = os.path.join( full_path, filename ) - if os.path.split( uploaded_file_name )[ -1 ] == suc.REPOSITORY_DEPENDENCY_DEFINITION_FILENAME: + if os.path.split( uploaded_file_name )[ -1 ] == rt_util.REPOSITORY_DEPENDENCY_DEFINITION_FILENAME: # Inspect the contents of the file to see if changeset_revision values are missing and if so, set them appropriately. altered, root_elem, error_message = commit_util.handle_repository_dependencies_definition( trans, uploaded_file_name, @@ -430,7 +431,7 @@ elif altered: tmp_filename = xml_util.create_and_write_tmp_file( root_elem ) shutil.move( tmp_filename, uploaded_file_name ) - elif os.path.split( uploaded_file_name )[ -1 ] == suc.TOOL_DEPENDENCY_DEFINITION_FILENAME: + elif os.path.split( uploaded_file_name )[ -1 ] == rt_util.TOOL_DEPENDENCY_DEFINITION_FILENAME: # Inspect the contents of the file to see if changeset_revision values are missing and if so, set them appropriately. altered, root_elem, error_message = commit_util.handle_tool_dependencies_definition( trans, uploaded_file_name ) if error_message: diff -r e3c0070b64bae25215e9a30822f6e7488e938969 -r d0979f737213cbc8e000a46f90a7cd5501c709d1 lib/galaxy/webapps/tool_shed/framework/middleware/hg.py --- a/lib/galaxy/webapps/tool_shed/framework/middleware/hg.py +++ b/lib/galaxy/webapps/tool_shed/framework/middleware/hg.py @@ -131,7 +131,7 @@ # We possibly found an altered file entry. filename, change_list = entry if filename and isinstance( filename, str ): - if filename == suc.REPOSITORY_DEPENDENCY_DEFINITION_FILENAME: + if filename == rt_util.REPOSITORY_DEPENDENCY_DEFINITION_FILENAME: # Make sure the any complex repository dependency definitions contain valid <repository> tags. is_valid, error_msg = commit_util.repository_tags_are_valid( filename, change_list ) if not is_valid: @@ -150,7 +150,7 @@ # We possibly found an altered file entry. filename, change_list = entry if filename and isinstance( filename, str ): - if filename == suc.TOOL_DEPENDENCY_DEFINITION_FILENAME: + if filename == rt_util.TOOL_DEPENDENCY_DEFINITION_FILENAME: # Make sure the any complex repository dependency definitions contain valid <repository> tags. is_valid, error_msg = commit_util.repository_tags_are_valid( filename, change_list ) if not is_valid: @@ -171,8 +171,8 @@ # We possibly found an altered file entry. filename, change_list = entry if filename and isinstance( filename, str ): - if filename in [ suc.REPOSITORY_DEPENDENCY_DEFINITION_FILENAME, - suc.TOOL_DEPENDENCY_DEFINITION_FILENAME ]: + if filename in [ rt_util.REPOSITORY_DEPENDENCY_DEFINITION_FILENAME, + rt_util.TOOL_DEPENDENCY_DEFINITION_FILENAME ]: # We check both files since tool dependency definitions files can contain complex # repository dependency definitions. is_valid, error_msg = commit_util.repository_tags_are_valid( filename, change_list ) diff -r e3c0070b64bae25215e9a30822f6e7488e938969 -r d0979f737213cbc8e000a46f90a7cd5501c709d1 lib/tool_shed/repository_types/util.py --- a/lib/tool_shed/repository_types/util.py +++ b/lib/tool_shed/repository_types/util.py @@ -3,9 +3,11 @@ log = logging.getLogger( __name__ ) -UNRESTRICTED = 'unrestricted' +REPOSITORY_DEPENDENCY_DEFINITION_FILENAME = 'repository_dependencies.xml' REPOSITORY_SUITE_DEFINITION = 'repository_suite_definition' TOOL_DEPENDENCY_DEFINITION = 'tool_dependency_definition' +TOOL_DEPENDENCY_DEFINITION_FILENAME = 'tool_dependencies.xml' +UNRESTRICTED = 'unrestricted' types = [ UNRESTRICTED, TOOL_DEPENDENCY_DEFINITION, REPOSITORY_SUITE_DEFINITION ] @@ -31,3 +33,22 @@ else: repository_type_select_field.add_option( option_label, option_value, selected=selected ) return repository_type_select_field + +def generate_message_for_repository_type_change( app, repository ): + message = '' + if repository.can_change_type_to( app, REPOSITORY_SUITE_DEFINITION ): + repository_suite_definition_type_class = \ + app.repository_types_registry.get_class_by_label( REPOSITORY_SUITE_DEFINITION ) + message += "This repository currently contains a single file named <b>%s</b>. If the intent of this repository is " % \ + REPOSITORY_DEPENDENCY_DEFINITION_FILENAME + message += "to define relationships to a collection of repositories that contain related Galaxy utilities with " + message += "no plans to add additional files, consider setting its type to <b>%s</b>.<br/>" % \ + repository_suite_definition_type_class.label + elif repository.can_change_type_to( app, TOOL_DEPENDENCY_DEFINITION ): + tool_dependency_definition_type_class = \ + app.repository_types_registry.get_class_by_label( TOOL_DEPENDENCY_DEFINITION ) + message += "This repository currently contains a single file named <b>%s</b>. If additional files will " % \ + TOOL_DEPENDENCY_DEFINITION_FILENAME + message += "not be added to this repository, consider setting its type to <b>%s</b>.<br/>" % \ + tool_dependency_definition_type_class.label + return message diff -r e3c0070b64bae25215e9a30822f6e7488e938969 -r d0979f737213cbc8e000a46f90a7cd5501c709d1 lib/tool_shed/util/commit_util.py --- a/lib/tool_shed/util/commit_util.py +++ b/lib/tool_shed/util/commit_util.py @@ -51,10 +51,10 @@ if member.name in [ 'hgrc' ]: message = "Uploaded archives cannot contain hgrc files." return False, message - if repository.type == rt_util.REPOSITORY_SUITE_DEFINITION and member.name != suc.REPOSITORY_DEPENDENCY_DEFINITION_FILENAME: + if repository.type == rt_util.REPOSITORY_SUITE_DEFINITION and member.name != rt_util.REPOSITORY_DEPENDENCY_DEFINITION_FILENAME: message = 'Repositories of type <b>Repsoitory suite definition</b> can contain only a single file named <b>repository_dependencies.xml</b>.' return False, message - if repository.type == rt_util.TOOL_DEPENDENCY_DEFINITION and member.name != suc.TOOL_DEPENDENCY_DEFINITION_FILENAME: + if repository.type == rt_util.TOOL_DEPENDENCY_DEFINITION and member.name != rt_util.TOOL_DEPENDENCY_DEFINITION_FILENAME: message = 'Repositories of type <b>Tool dependency definition</b> can contain only a single file named <b>tool_dependencies.xml</b>.' return False, message return True, '' diff -r e3c0070b64bae25215e9a30822f6e7488e938969 -r d0979f737213cbc8e000a46f90a7cd5501c709d1 lib/tool_shed/util/export_util.py --- a/lib/tool_shed/util/export_util.py +++ b/lib/tool_shed/util/export_util.py @@ -7,6 +7,7 @@ from time import gmtime from time import strftime import tool_shed.util.shed_util_common as suc +import tool_shed.repository_types.util as rt_util from galaxy import eggs from galaxy import web from galaxy.util.odict import odict @@ -71,7 +72,9 @@ ordered_repositories = [] ordered_changeset_revisions = [] if repository: - repository_metadata = suc.get_current_repository_metadata_for_changeset_revision( trans, repository, changeset_revision ) + repository_metadata = suc.get_current_repository_metadata_for_changeset_revision( trans.app, + repository, + changeset_revision ) if repository_metadata: ordered_repository_ids = [ repository_id ] ordered_repositories = [ repository ] @@ -147,17 +150,19 @@ full_path = os.path.join( root, name ) relative_path = full_path.replace( work_dir, '' ).lstrip( '/' ) # See if we have a repository dependencies defined. - if name == suc.REPOSITORY_DEPENDENCY_DEFINITION_FILENAME: + if name == rt_util.REPOSITORY_DEPENDENCY_DEFINITION_FILENAME: # Eliminate the toolshed, and changeset_revision attributes from all <repository> tags. - altered, root_elem, error_message = commit_util.handle_repository_dependencies_definition( trans, full_path, unpopulate=True ) + altered, root_elem, error_message = \ + commit_util.handle_repository_dependencies_definition( trans, full_path, unpopulate=True ) if error_message: return None, error_message if altered: tmp_filename = xml_util.create_and_write_tmp_file( root_elem, use_indent=True ) shutil.move( tmp_filename, full_path ) - elif name == suc.TOOL_DEPENDENCY_DEFINITION_FILENAME: + elif name == rt_util.TOOL_DEPENDENCY_DEFINITION_FILENAME: # Eliminate the toolshed, and changeset_revision attributes from all <repository> tags. - altered, root_elem, error_message = commit_util.handle_tool_dependencies_definition( trans, full_path, unpopulate=True ) + altered, root_elem, error_message = \ + commit_util.handle_tool_dependencies_definition( trans, full_path, unpopulate=True ) if error_message: return None, error_message if altered: @@ -167,7 +172,8 @@ repository_archive.close() return repository_archive, error_message -def generate_repository_archive_filename( tool_shed_url, name, owner, changeset_revision, file_type, export_repository_dependencies=False, use_tmp_archive_dir=False ): +def generate_repository_archive_filename( tool_shed_url, name, owner, changeset_revision, file_type, + export_repository_dependencies=False, use_tmp_archive_dir=False ): tool_shed = remove_protocol_from_tool_shed_url( tool_shed_url ) file_type_str = suc.get_file_type_str( changeset_revision, file_type ) if export_repository_dependencies: @@ -200,7 +206,9 @@ description, repository_clone_url, changeset_revision, ctx_rev, repository_owner, repository_dependencies, tool_dependencies = \ suc.get_repo_info_tuple_contents( repo_info_tup ) repository = suc.get_repository_by_name_and_owner( trans.app, repository_name, repository_owner ) - repository_metadata = suc.get_current_repository_metadata_for_changeset_revision( trans, repository, changeset_revision ) + repository_metadata = suc.get_current_repository_metadata_for_changeset_revision( trans.app, + repository, + changeset_revision ) if repository_metadata: return repository, repository_metadata.changeset_revision return None, None diff -r e3c0070b64bae25215e9a30822f6e7488e938969 -r d0979f737213cbc8e000a46f90a7cd5501c709d1 lib/tool_shed/util/import_util.py --- a/lib/tool_shed/util/import_util.py +++ b/lib/tool_shed/util/import_util.py @@ -12,6 +12,7 @@ from tool_shed.util import metadata_util from tool_shed.util import xml_util import tool_shed.util.shed_util_common as suc +import tool_shed.repository_types.util as rt_util from galaxy import eggs eggs.require( 'mercurial' ) @@ -240,7 +241,7 @@ archive.close() for filename in filenames_in_archive: uploaded_file_name = os.path.join( full_path, filename ) - if os.path.split( uploaded_file_name )[ -1 ] == suc.REPOSITORY_DEPENDENCY_DEFINITION_FILENAME: + if os.path.split( uploaded_file_name )[ -1 ] == rt_util.REPOSITORY_DEPENDENCY_DEFINITION_FILENAME: # Inspect the contents of the file to see if changeset_revision values are missing and if so, set them appropriately. altered, root_elem, error_message = commit_util.handle_repository_dependencies_definition( trans, uploaded_file_name, @@ -251,7 +252,7 @@ if altered: tmp_filename = xml_util.create_and_write_tmp_file( root_elem ) shutil.move( tmp_filename, uploaded_file_name ) - elif os.path.split( uploaded_file_name )[ -1 ] == suc.TOOL_DEPENDENCY_DEFINITION_FILENAME: + elif os.path.split( uploaded_file_name )[ -1 ] == rt_util.TOOL_DEPENDENCY_DEFINITION_FILENAME: # Inspect the contents of the file to see if changeset_revision values are missing and if so, set them appropriately. altered, root_elem, error_message = commit_util.handle_tool_dependencies_definition( trans, uploaded_file_name ) if error_message: diff -r e3c0070b64bae25215e9a30822f6e7488e938969 -r d0979f737213cbc8e000a46f90a7cd5501c709d1 lib/tool_shed/util/metadata_util.py --- a/lib/tool_shed/util/metadata_util.py +++ b/lib/tool_shed/util/metadata_util.py @@ -39,8 +39,8 @@ SUBSET_VALUES = [ EQUAL, SUBSET ] NOT_TOOL_CONFIGS = [ suc.DATATYPES_CONFIG_FILENAME, - suc.REPOSITORY_DEPENDENCY_DEFINITION_FILENAME, - suc.TOOL_DEPENDENCY_DEFINITION_FILENAME, + rt_util.REPOSITORY_DEPENDENCY_DEFINITION_FILENAME, + rt_util.TOOL_DEPENDENCY_DEFINITION_FILENAME, suc.REPOSITORY_DATA_MANAGER_CONFIG_FILENAME ] def add_tool_versions( trans, id, repository_metadata, changeset_revisions ): @@ -49,7 +49,12 @@ tool_versions_dict = {} for tool_dict in metadata.get( 'tools', [] ): # We have at least 2 changeset revisions to compare tool guids and tool ids. - parent_id = get_parent_id( trans, id, tool_dict[ 'id' ], tool_dict[ 'version' ], tool_dict[ 'guid' ], changeset_revisions ) + parent_id = get_parent_id( trans.app, + id, + tool_dict[ 'id' ], + tool_dict[ 'version' ], + tool_dict[ 'guid' ], + changeset_revisions ) tool_versions_dict[ tool_dict[ 'guid' ] ] = parent_id if tool_versions_dict: repository_metadata.tool_versions = tool_versions_dict @@ -662,7 +667,7 @@ dirs.remove( '.hg' ) for name in files: # See if we have a repository dependencies defined. - if name == suc.REPOSITORY_DEPENDENCY_DEFINITION_FILENAME: + if name == rt_util.REPOSITORY_DEPENDENCY_DEFINITION_FILENAME: path_to_repository_dependencies_config = os.path.join( root, name ) metadata_dict, error_message = \ generate_repository_dependency_metadata( app, @@ -1048,17 +1053,23 @@ repository = trans.sa_session.query( trans.model.Repository ).get( decoded_repository_id ) repo = hg.repository( hg_util.get_configured_ui(), repository.repo_path( trans.app ) ) if downloadable: - changeset_revision = suc.get_latest_downloadable_changeset_revision( trans.app, repository, repo ) + changeset_revision = suc.get_latest_downloadable_changeset_revision( trans.app, + repository, + repo ) else: - changeset_revision = suc.get_latest_changeset_revision( trans, repository, repo ) - return suc.get_repository_metadata_by_changeset_revision( trans.app, trans.security.encode_id( repository.id ), changeset_revision ) + changeset_revision = suc.get_latest_changeset_revision( trans.app, + repository, + repo ) + return suc.get_repository_metadata_by_changeset_revision( trans.app, + trans.security.encode_id( repository.id ), + changeset_revision ) -def get_parent_id( trans, id, old_id, version, guid, changeset_revisions ): +def get_parent_id( app, id, old_id, version, guid, changeset_revisions ): parent_id = None # Compare from most recent to oldest. changeset_revisions.reverse() for changeset_revision in changeset_revisions: - repository_metadata = suc.get_repository_metadata_by_changeset_revision( trans.app, id, changeset_revision ) + repository_metadata = suc.get_repository_metadata_by_changeset_revision( app, id, changeset_revision ) metadata = repository_metadata.metadata tools_dicts = metadata.get( 'tools', [] ) for tool_dict in tools_dicts: @@ -1709,7 +1720,7 @@ tool_versions_dict[ tool_dict[ 'guid' ] ] = tool_dict[ 'id' ] else: for tool_dict in tool_dicts: - parent_id = get_parent_id( trans, + parent_id = get_parent_id( trans.app, id, tool_dict[ 'id' ], tool_dict[ 'version' ], diff -r e3c0070b64bae25215e9a30822f6e7488e938969 -r d0979f737213cbc8e000a46f90a7cd5501c709d1 lib/tool_shed/util/shed_util_common.py --- a/lib/tool_shed/util/shed_util_common.py +++ b/lib/tool_shed/util/shed_util_common.py @@ -39,8 +39,6 @@ MAX_DISPLAY_SIZE = 32768 DATATYPES_CONFIG_FILENAME = 'datatypes_conf.xml' REPOSITORY_DATA_MANAGER_CONFIG_FILENAME = 'data_manager_conf.xml' -REPOSITORY_DEPENDENCY_DEFINITION_FILENAME = 'repository_dependencies.xml' -TOOL_DEPENDENCY_DEFINITION_FILENAME = 'tool_dependencies.xml' new_repo_email_alert_template = """ Sharable link: ${sharable_link} @@ -129,9 +127,9 @@ tool_dependencies_select_field.add_option( option_label, option_value ) return tool_dependencies_select_field -def changeset_is_malicious( trans, id, changeset_revision, **kwd ): +def changeset_is_malicious( app, id, changeset_revision, **kwd ): """Check the malicious flag in repository metadata for a specified change set""" - repository_metadata = get_repository_metadata_by_changeset_revision( trans.app, id, changeset_revision ) + repository_metadata = get_repository_metadata_by_changeset_revision( app, id, changeset_revision ) if repository_metadata: return repository_metadata.malicious return False @@ -258,25 +256,6 @@ components_list = [ toolshed, name, owner, changeset_revision, prior_installation_required, only_if_compiling_contained_td ] return components_list -def generate_message_for_repository_type_change( trans, repository ): - message = '' - if repository.can_change_type_to( trans.app, rt_util.REPOSITORY_SUITE_DEFINITION ): - repository_suite_definition_type_class = \ - trans.app.repository_types_registry.get_class_by_label( rt_util.REPOSITORY_SUITE_DEFINITION ) - message += "This repository currently contains a single file named <b>%s</b>. If the intent of this repository is " % \ - REPOSITORY_DEPENDENCY_DEFINITION_FILENAME - message += "to define relationships to a collection of repositories that contain related Galaxy utilities with " - message += "no plans to add additional files, consider setting its type to <b>%s</b>.<br/>" % \ - repository_suite_definition_type_class.label - elif repository.can_change_type_to( trans.app, rt_util.TOOL_DEPENDENCY_DEFINITION ): - tool_dependency_definition_type_class = \ - trans.app.repository_types_registry.get_class_by_label( rt_util.TOOL_DEPENDENCY_DEFINITION ) - message += "This repository currently contains a single file named <b>%s</b>. If additional files will " % \ - TOOL_DEPENDENCY_DEFINITION_FILENAME - message += "not be added to this repository, consider setting its type to <b>%s</b>.<br/>" % \ - tool_dependency_definition_type_class.label - return message - def generate_repository_info_elem( tool_shed, repository_name, changeset_revision, owner, parent_elem=None, **kwd ): """Create and return an ElementTree repository info Element.""" if parent_elem is None: @@ -306,7 +285,7 @@ parent_elem=parent_elem, **kwd ) -def generate_sharable_link_for_repository_in_tool_shed( trans, repository, changeset_revision=None ): +def generate_sharable_link_for_repository_in_tool_shed( repository, changeset_revision=None ): """Generate the URL for sharing a repository that is in the tool shed.""" base_url = url_for( '/', qualified=True ).rstrip( '/' ) protocol, base = base_url.split( '://' ) @@ -486,16 +465,23 @@ return manifest_ctx, ctx_file return None, None -def get_current_repository_metadata_for_changeset_revision( trans, repository, changeset_revision ): - encoded_repository_id = trans.security.encode_id( repository.id ) - repository_metadata = get_repository_metadata_by_changeset_revision( trans.app, encoded_repository_id, changeset_revision ) +def get_current_repository_metadata_for_changeset_revision( app, repository, changeset_revision ): + encoded_repository_id = app.security.encode_id( repository.id ) + repository_metadata = get_repository_metadata_by_changeset_revision( app, + encoded_repository_id, + changeset_revision ) if repository_metadata: return repository_metadata - # The installable changeset_revision may have been changed because it was "moved ahead" in the repository changelog. - repo = hg.repository( hg_util.get_configured_ui(), repository.repo_path( trans.app ) ) - updated_changeset_revision = get_next_downloadable_changeset_revision( repository, repo, after_changeset_revision=changeset_revision ) + # The installable changeset_revision may have been changed because it was "moved ahead" + # in the repository changelog. + repo = hg.repository( hg_util.get_configured_ui(), repository.repo_path( app ) ) + updated_changeset_revision = get_next_downloadable_changeset_revision( repository, + repo, + after_changeset_revision=changeset_revision ) if updated_changeset_revision: - repository_metadata = get_repository_metadata_by_changeset_revision( trans.app, encoded_repository_id, updated_changeset_revision ) + repository_metadata = get_repository_metadata_by_changeset_revision( app, + encoded_repository_id, + updated_changeset_revision ) if repository_metadata: return repository_metadata return None @@ -608,9 +594,11 @@ """Get a tool shed repository record from the Galaxy database defined by the id.""" return trans.install_model.context.query( trans.install_model.ToolShedRepository ).get( trans.security.decode_id( id ) ) -def get_latest_changeset_revision( trans, repository, repo ): - repository_tip = repository.tip( trans.app ) - repository_metadata = get_repository_metadata_by_changeset_revision( trans.app, trans.security.encode_id( repository.id ), repository_tip ) +def get_latest_changeset_revision( app, repository, repo ): + repository_tip = repository.tip( app ) + repository_metadata = get_repository_metadata_by_changeset_revision( app, + app.security.encode_id( repository.id ), + repository_tip ) if repository_metadata and repository_metadata.downloadable: return repository_tip changeset_revisions = get_ordered_metadata_changeset_revisions( repository, repo, downloadable=False ) @@ -1403,7 +1391,7 @@ """ repo_dir = repository.repo_path( trans.app ) repo = hg.repository( hg_util.get_configured_ui(), repo_dir ) - sharable_link = generate_sharable_link_for_repository_in_tool_shed( trans, repository, changeset_revision=None ) + sharable_link = generate_sharable_link_for_repository_in_tool_shed( repository, changeset_revision=None ) smtp_server = trans.app.config.smtp_server if smtp_server and ( new_repo_alert or repository.email_alerts ): # Send email alert to users that want them. diff -r e3c0070b64bae25215e9a30822f6e7488e938969 -r d0979f737213cbc8e000a46f90a7cd5501c709d1 templates/webapps/tool_shed/common/repository_actions_menu.mako --- a/templates/webapps/tool_shed/common/repository_actions_menu.mako +++ b/templates/webapps/tool_shed/common/repository_actions_menu.mako @@ -27,7 +27,7 @@ else: is_new = False - if changeset_is_malicious( trans, trans.security.encode_id( repository.id ), repository.tip( trans.app ) ): + if changeset_is_malicious( trans.app, trans.security.encode_id( repository.id ), repository.tip( trans.app ) ): is_malicious = True else: is_malicious = False diff -r e3c0070b64bae25215e9a30822f6e7488e938969 -r d0979f737213cbc8e000a46f90a7cd5501c709d1 templates/webapps/tool_shed/repository/common.mako --- a/templates/webapps/tool_shed/repository/common.mako +++ b/templates/webapps/tool_shed/repository/common.mako @@ -208,7 +208,7 @@ <%def name="render_sharable_str( repository, changeset_revision=None )"><% from tool_shed.util.shed_util_common import generate_sharable_link_for_repository_in_tool_shed - sharable_link = generate_sharable_link_for_repository_in_tool_shed( trans, repository, changeset_revision=changeset_revision ) + sharable_link = generate_sharable_link_for_repository_in_tool_shed( repository, changeset_revision=changeset_revision ) %> ${sharable_link} </%def> diff -r e3c0070b64bae25215e9a30822f6e7488e938969 -r d0979f737213cbc8e000a46f90a7cd5501c709d1 templates/webapps/tool_shed/repository/manage_repository.mako --- a/templates/webapps/tool_shed/repository/manage_repository.mako +++ b/templates/webapps/tool_shed/repository/manage_repository.mako @@ -22,7 +22,7 @@ else: is_deprecated = False - if changeset_is_malicious( trans, trans.security.encode_id( repository.id ), repository.tip( trans.app ) ): + if changeset_is_malicious( trans.app, trans.security.encode_id( repository.id ), repository.tip( trans.app ) ): is_malicious = True else: is_malicious = False Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: 2 new changesets
by commits-noreply＠bitbucket.org 20 May '14

20 May '14

2 new commits in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/20227293c9fe/ Changeset: 20227293c9fe Branch: next-stable User: greg Date: 2014-05-20 23:27:00 Summary: Fix for rendering the metadata about a tool contained in a repository installed from the Tool Shed into Galaxy. Affected #: 1 file diff -r e49feb93943da7c6a1ccb84d6dff4518742282a8 -r 20227293c9fe496cf0995f7c194969e785d5ff1e templates/webapps/tool_shed/repository/common.mako --- a/templates/webapps/tool_shed/repository/common.mako +++ b/templates/webapps/tool_shed/repository/common.mako @@ -883,13 +883,14 @@ %else: <td style="padding-left: ${pad+20}px;"> %if tool.repository_id: - <div style="float:left;" class="menubutton split popup" id="tool-${encoded_id}-popup"> - <a class="view-info" href="${h.url_for( controller='repository', action='display_tool', repository_id=trans.security.encode_id( tool.repository_id ), tool_config=tool.tool_config, changeset_revision=tool.changeset_revision, render_repository_actions_for=render_repository_actions_for )}">${tool.name | h}</a> - </div> - <div popupmenu="tool-${encoded_id}-popup"> - <a class="action-button" href="${h.url_for( controller='repository', action='view_tool_metadata', repository_id=trans.security.encode_id( tool.repository_id ), changeset_revision=tool.changeset_revision, tool_id=tool.tool_id, render_repository_actions_for=render_repository_actions_for )}">View tool metadata</a> - </div> - %if trans.webapp.name == 'galaxy': + %if trans.webapp.name == 'tool_shed': + <div style="float:left;" class="menubutton split popup" id="tool-${encoded_id}-popup"> + <a class="view-info" href="${h.url_for( controller='repository', action='display_tool', repository_id=trans.security.encode_id( tool.repository_id ), tool_config=tool.tool_config, changeset_revision=tool.changeset_revision, render_repository_actions_for=render_repository_actions_for )}">${tool.name | h}</a> + </div> + <div popupmenu="tool-${encoded_id}-popup"> + <a class="action-button" href="${h.url_for( controller='repository', action='view_tool_metadata', repository_id=trans.security.encode_id( tool.repository_id ), changeset_revision=tool.changeset_revision, tool_id=tool.tool_id, render_repository_actions_for=render_repository_actions_for )}">View tool metadata</a> + </div> + %elif trans.webapp.name == 'galaxy': %if tool.repository_installation_status == trans.install_model.ToolShedRepository.installation_status.INSTALLED: <a class="action-button" href="${h.url_for( controller='admin_toolshed', action='view_tool_metadata', repository_id=trans.security.encode_id( tool.repository_id ), changeset_revision=tool.changeset_revision, tool_id=tool.tool_id )}">${tool.name | h}</a> %else: https://bitbucket.org/galaxy/galaxy-central/commits/e3c0070b64ba/ Changeset: e3c0070b64ba User: greg Date: 2014-05-20 23:27:29 Summary: Merged next-stable Affected #: 1 file diff -r b8e434a784960287fc998c9e0a7f11512dc0533c -r e3c0070b64bae25215e9a30822f6e7488e938969 templates/webapps/tool_shed/repository/common.mako --- a/templates/webapps/tool_shed/repository/common.mako +++ b/templates/webapps/tool_shed/repository/common.mako @@ -883,13 +883,14 @@ %else: <td style="padding-left: ${pad+20}px;"> %if tool.repository_id: - <div style="float:left;" class="menubutton split popup" id="tool-${encoded_id}-popup"> - <a class="view-info" href="${h.url_for( controller='repository', action='display_tool', repository_id=trans.security.encode_id( tool.repository_id ), tool_config=tool.tool_config, changeset_revision=tool.changeset_revision, render_repository_actions_for=render_repository_actions_for )}">${tool.name | h}</a> - </div> - <div popupmenu="tool-${encoded_id}-popup"> - <a class="action-button" href="${h.url_for( controller='repository', action='view_tool_metadata', repository_id=trans.security.encode_id( tool.repository_id ), changeset_revision=tool.changeset_revision, tool_id=tool.tool_id, render_repository_actions_for=render_repository_actions_for )}">View tool metadata</a> - </div> - %if trans.webapp.name == 'galaxy': + %if trans.webapp.name == 'tool_shed': + <div style="float:left;" class="menubutton split popup" id="tool-${encoded_id}-popup"> + <a class="view-info" href="${h.url_for( controller='repository', action='display_tool', repository_id=trans.security.encode_id( tool.repository_id ), tool_config=tool.tool_config, changeset_revision=tool.changeset_revision, render_repository_actions_for=render_repository_actions_for )}">${tool.name | h}</a> + </div> + <div popupmenu="tool-${encoded_id}-popup"> + <a class="action-button" href="${h.url_for( controller='repository', action='view_tool_metadata', repository_id=trans.security.encode_id( tool.repository_id ), changeset_revision=tool.changeset_revision, tool_id=tool.tool_id, render_repository_actions_for=render_repository_actions_for )}">View tool metadata</a> + </div> + %elif trans.webapp.name == 'galaxy': %if tool.repository_installation_status == trans.install_model.ToolShedRepository.installation_status.INSTALLED: <a class="action-button" href="${h.url_for( controller='admin_toolshed', action='view_tool_metadata', repository_id=trans.security.encode_id( tool.repository_id ), changeset_revision=tool.changeset_revision, tool_id=tool.tool_id )}">${tool.name | h}</a> %else: Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: greg: Add a repository registry to the Tool Shed to keep in-memory information about repositories. This will significantly improve some page rendering in the tool shed, specifically the current main Categories splash page. This registry will provide filtered certification information for the planned Tool Shed splash page which will be rendered with Javascript. The Tool Shed's install and test framework still needs to be enhanced to update this registry during install
by commits-noreply＠bitbucket.org 20 May '14

20 May '14

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/b8e434a78496/ Changeset: b8e434a78496 User: greg Date: 2014-05-20 23:18:14 Summary: Add a repository registry to the Tool Shed to keep in-memory information about repositories. This will significantly improve some page rendering in the tool shed, specifically the current main Categories splash page. This registry will provide filtered certification information for the planned Tool Shed splash page which will be rendered with Javascript. The Tool Shed's install and test framework still needs to be enhanced to update this registry during install and test runs. Affected #: 20 files diff -r 172e1713535c24cc6c07caa5f90494b68827b638 -r b8e434a784960287fc998c9e0a7f11512dc0533c lib/galaxy/webapps/tool_shed/api/repositories.py --- a/lib/galaxy/webapps/tool_shed/api/repositories.py +++ b/lib/galaxy/webapps/tool_shed/api/repositories.py @@ -127,7 +127,7 @@ action='show', id=encoded_repository_id ) # Get the repository_metadata information. - repository_metadata = suc.get_repository_metadata_by_changeset_revision( trans, + repository_metadata = suc.get_repository_metadata_by_changeset_revision( trans.app, encoded_repository_id, changeset_revision ) if repository_metadata is None: @@ -136,7 +136,7 @@ repo_dir = repository.repo_path( trans.app ) repo = hg.repository( hg_util.get_configured_ui(), repo_dir ) new_changeset_revision = suc.get_next_downloadable_changeset_revision( repository, repo, changeset_revision ) - repository_metadata = suc.get_repository_metadata_by_changeset_revision( trans, + repository_metadata = suc.get_repository_metadata_by_changeset_revision( trans.app, encoded_repository_id, new_changeset_revision ) changeset_revision = new_changeset_revision diff -r 172e1713535c24cc6c07caa5f90494b68827b638 -r b8e434a784960287fc998c9e0a7f11512dc0533c lib/galaxy/webapps/tool_shed/api/repository_revisions.py --- a/lib/galaxy/webapps/tool_shed/api/repository_revisions.py +++ b/lib/galaxy/webapps/tool_shed/api/repository_revisions.py @@ -164,7 +164,7 @@ continue repository_dependency_id = trans.security.encode_id( repository_dependency.id ) repository_dependency_repository_metadata = \ - suc.get_repository_metadata_by_changeset_revision( trans, repository_dependency_id, changeset_revision ) + suc.get_repository_metadata_by_changeset_revision( trans.app, repository_dependency_id, changeset_revision ) if repository_dependency_repository_metadata is None: # The changeset_revision column in the repository_metadata table has been updated with a new # value value, so find the changeset_revision to which we need to update. @@ -174,7 +174,7 @@ repo, changeset_revision ) repository_dependency_repository_metadata = \ - suc.get_repository_metadata_by_changeset_revision( trans, + suc.get_repository_metadata_by_changeset_revision( trans.app, repository_dependency_id, new_changeset_revision ) if repository_dependency_repository_metadata is None: diff -r 172e1713535c24cc6c07caa5f90494b68827b638 -r b8e434a784960287fc998c9e0a7f11512dc0533c lib/galaxy/webapps/tool_shed/app.py --- a/lib/galaxy/webapps/tool_shed/app.py +++ b/lib/galaxy/webapps/tool_shed/app.py @@ -9,7 +9,8 @@ from galaxy.util.dbkeys import GenomeBuilds from galaxy.web import security from galaxy.tags.tag_handler import CommunityTagHandler -from tool_shed.grids.util import RepositoryGridFilterManager +from tool_shed.grids.repository_grid_filter_manager import RepositoryGridFilterManager +import tool_shed.repository_registry import tool_shed.repository_types.registry @@ -64,6 +65,8 @@ # Let the Tool Shed's HgwebConfigManager know where the hgweb.config file is located. self.hgweb_config_manager = self.model.hgweb_config_manager self.hgweb_config_manager.hgweb_config_dir = self.config.hgweb_config_dir + # Initialize the repository registry. + self.repository_registry = tool_shed.repository_registry.Registry( self ) print >> sys.stderr, "Tool shed hgweb.config file is: ", self.hgweb_config_manager.hgweb_config def shutdown( self ): diff -r 172e1713535c24cc6c07caa5f90494b68827b638 -r b8e434a784960287fc998c9e0a7f11512dc0533c lib/galaxy/webapps/tool_shed/controllers/repository.py --- a/lib/galaxy/webapps/tool_shed/controllers/repository.py +++ b/lib/galaxy/webapps/tool_shed/controllers/repository.py @@ -920,7 +920,7 @@ # Return the same value for changeset_revision and latest_changeset_revision. url += latest_changeset_revision else: - repository_metadata = suc.get_repository_metadata_by_changeset_revision( trans, + repository_metadata = suc.get_repository_metadata_by_changeset_revision( trans.app, trans.security.encode_id( repository.id ), changeset_revision ) if repository_metadata: @@ -945,7 +945,7 @@ latest_changeset_revision = changeset_hash break else: - repository_metadata = suc.get_repository_metadata_by_changeset_revision( trans, + repository_metadata = suc.get_repository_metadata_by_changeset_revision( trans.app, trans.security.encode_id( repository.id ), changeset_hash ) if repository_metadata: @@ -1189,7 +1189,7 @@ repositories_archive_path, file_name = os.path.split( repositories_archive.name ) suc.remove_dir( repositories_archive_path ) return opened_archive - repository_metadata = suc.get_repository_metadata_by_changeset_revision( trans, repository_id, changeset_revision ) + repository_metadata = suc.get_repository_metadata_by_changeset_revision( trans.app, repository_id, changeset_revision ) metadata = repository_metadata.metadata # Get a dictionary of all repositories upon which the contents of the current repository_metadata record depend. repository_dependencies = \ @@ -1454,7 +1454,7 @@ owner = kwd.get( 'owner', None ) changeset_revision = kwd.get( 'changeset_revision', None ) repository = suc.get_repository_by_name_and_owner( trans.app, name, owner ) - repository_metadata = suc.get_repository_metadata_by_changeset_revision( trans, + repository_metadata = suc.get_repository_metadata_by_changeset_revision( trans.app, trans.security.encode_id( repository.id ), changeset_revision ) has_galaxy_utilities_dict = has_galaxy_utilities( repository_metadata ) @@ -1500,7 +1500,7 @@ changeset_hash = str( repo.changectx( changeset ) ) ctx = hg_util.get_changectx_for_changeset( repo, changeset_hash ) if update_to_changeset_hash: - update_to_repository_metadata = suc.get_repository_metadata_by_changeset_revision( trans, + update_to_repository_metadata = suc.get_repository_metadata_by_changeset_revision( trans.app, trans.security.encode_id( repository.id ), changeset_hash ) if update_to_repository_metadata: @@ -1606,7 +1606,7 @@ repository = repository_metadata.repository repo_dir = repository.repo_path( trans.app ) repo = hg.repository( hg_util.get_configured_ui(), repo_dir ) - latest_downloadable_changeset_revsion = suc.get_latest_downloadable_changeset_revision( trans, repository, repo ) + latest_downloadable_changeset_revsion = suc.get_latest_downloadable_changeset_revision( trans.app, repository, repo ) if repository_metadata.changeset_revision == latest_downloadable_changeset_revsion: # We'll display only the test run for the latest installable revision in the rss feed. tool_test_results = repository_metadata.tool_test_results @@ -1674,7 +1674,7 @@ repository = suc.get_repository_by_name_and_owner( trans.app, repository_name, repository_owner ) if repository: repo = hg.repository( hg_util.get_configured_ui(), repository.repo_path( trans.app ) ) - return suc.get_latest_downloadable_changeset_revision( trans, repository, repo ) + return suc.get_latest_downloadable_changeset_revision( trans.app, repository, repo ) return suc.INITIAL_CHANGELOG_HASH @web.json @@ -1690,7 +1690,7 @@ repository = suc.get_repository_by_name_and_owner( trans.app, repository_name, repository_owner ) if repository: repository_metadata = \ - suc.get_repository_metadata_by_changeset_revision( trans, + suc.get_repository_metadata_by_changeset_revision( trans.app, trans.security.encode_id( repository.id ), changeset_revision ) if repository_metadata: @@ -1862,7 +1862,7 @@ tool_version_dicts = [] for changeset in repo.changelog: current_changeset_revision = str( repo.changectx( changeset ) ) - repository_metadata = suc.get_repository_metadata_by_changeset_revision( trans, + repository_metadata = suc.get_repository_metadata_by_changeset_revision( trans.app, trans.security.encode_id( repository.id ), current_changeset_revision ) if repository_metadata and repository_metadata.tool_versions: @@ -1884,14 +1884,14 @@ repository_clone_url = common_util.generate_clone_url_for_repository_in_tool_shed( trans, repository ) repo_dir = repository.repo_path( trans.app ) repo = hg.repository( hg_util.get_configured_ui(), repo_dir ) - repository_metadata = suc.get_repository_metadata_by_changeset_revision( trans, repository_id, changeset_revision ) + repository_metadata = suc.get_repository_metadata_by_changeset_revision( trans.app, repository_id, changeset_revision ) if not repository_metadata: # The received changeset_revision is no longer associated with metadata, so get the next changeset_revision in the repository # changelog that is associated with metadata. changeset_revision = suc.get_next_downloadable_changeset_revision( repository, repo, after_changeset_revision=changeset_revision ) - repository_metadata = suc.get_repository_metadata_by_changeset_revision( trans, repository_id, changeset_revision ) + repository_metadata = suc.get_repository_metadata_by_changeset_revision( trans.app, repository_id, changeset_revision ) ctx = hg_util.get_changectx_for_changeset( repo, changeset_revision ) repo_info_dict = repository_util.create_repo_info_dict( trans=trans, repository_clone_url=repository_clone_url, @@ -2233,7 +2233,7 @@ trans.sa_session.flush() message += "The repository information has been updated." elif kwd.get( 'skip_tool_tests_button', False ): - repository_metadata = suc.get_repository_metadata_by_changeset_revision( trans, id, changeset_revision ) + repository_metadata = suc.get_repository_metadata_by_changeset_revision( trans.app, id, changeset_revision ) skip_tool_test = repository_metadata.skip_tool_tests if skip_tool_test: # Handle the mapper behavior. @@ -2327,7 +2327,7 @@ skip_tool_test = None repository_dependencies = None if changeset_revision != suc.INITIAL_CHANGELOG_HASH: - repository_metadata = suc.get_repository_metadata_by_changeset_revision( trans, id, changeset_revision ) + repository_metadata = suc.get_repository_metadata_by_changeset_revision( trans.app, id, changeset_revision ) if repository_metadata: revision_label = hg_util.get_revision_label( trans, repository, changeset_revision, include_date=False ) metadata = repository_metadata.metadata @@ -2338,7 +2338,7 @@ previous_changeset_revision = \ suc.get_previous_metadata_changeset_revision( repository, repo, changeset_revision, downloadable=False ) if previous_changeset_revision != suc.INITIAL_CHANGELOG_HASH: - repository_metadata = suc.get_repository_metadata_by_changeset_revision( trans, id, previous_changeset_revision ) + repository_metadata = suc.get_repository_metadata_by_changeset_revision( trans.app, id, previous_changeset_revision ) if repository_metadata: revision_label = hg_util.get_revision_label( trans, repository, previous_changeset_revision, include_date=False ) metadata = repository_metadata.metadata @@ -2439,7 +2439,7 @@ changeset_revision = kwd.get( 'changeset_revision', repository.tip( trans.app ) ) metadata = None if changeset_revision != suc.INITIAL_CHANGELOG_HASH: - repository_metadata = suc.get_repository_metadata_by_changeset_revision( trans, id, changeset_revision ) + repository_metadata = suc.get_repository_metadata_by_changeset_revision( trans.app, id, changeset_revision ) if repository_metadata: metadata = repository_metadata.metadata else: @@ -2453,7 +2453,7 @@ changeset_revision, downloadable=False ) if previous_changeset_revision != suc.INITIAL_CHANGELOG_HASH: - repository_metadata = suc.get_repository_metadata_by_changeset_revision( trans, + repository_metadata = suc.get_repository_metadata_by_changeset_revision( trans.app, id, previous_changeset_revision ) if repository_metadata: @@ -2545,7 +2545,7 @@ repo_dir = repository.repo_path( trans.app ) repo = hg.repository( hg_util.get_configured_ui(), repo_dir ) changeset_revision = kwd.get( 'changeset_revision', repository.tip( trans.app ) ) - repository_metadata = suc.get_repository_metadata_by_changeset_revision( trans, repository_id, changeset_revision ) + repository_metadata = suc.get_repository_metadata_by_changeset_revision( trans.app, repository_id, changeset_revision ) if repository_metadata: repository_metadata_id = trans.security.encode_id( repository_metadata.id ), metadata = repository_metadata.metadata @@ -2857,7 +2857,7 @@ def set_malicious( self, trans, id, ctx_str, **kwd ): malicious = kwd.get( 'malicious', '' ) if kwd.get( 'malicious_button', False ): - repository_metadata = suc.get_repository_metadata_by_changeset_revision( trans, id, ctx_str ) + repository_metadata = suc.get_repository_metadata_by_changeset_revision( trans.app, id, ctx_str ) malicious_checked = CheckboxField.is_checked( malicious ) repository_metadata.malicious = malicious_checked trans.sa_session.add( repository_metadata ) @@ -2970,7 +2970,7 @@ changeset_revision = kwd.get( 'changeset_revision', None ) repository = suc.get_repository_by_name_and_owner( trans.app, name, owner ) if repository: - repository_metadata = suc.get_repository_metadata_by_changeset_revision( trans, + repository_metadata = suc.get_repository_metadata_by_changeset_revision( trans.app, trans.security.encode_id( repository.id ), changeset_revision ) repo_dir = repository.repo_path( trans.app ) @@ -3070,7 +3070,7 @@ changesets = [] for changeset in repo.changelog: ctx = repo.changectx( changeset ) - if suc.get_repository_metadata_by_changeset_revision( trans, id, str( ctx ) ): + if suc.get_repository_metadata_by_changeset_revision( trans.app, id, str( ctx ) ): has_metadata = True else: has_metadata = False @@ -3225,7 +3225,7 @@ add_id_to_name=False, downloadable=False ) revision_label = hg_util.get_revision_label( trans, repository, changeset_revision, include_date=False ) - repository_metadata = suc.get_repository_metadata_by_changeset_revision( trans, id, changeset_revision ) + repository_metadata = suc.get_repository_metadata_by_changeset_revision( trans.app, id, changeset_revision ) if repository_metadata: metadata = repository_metadata.metadata # Get a dictionary of all repositories upon which the contents of the current repository_metadata record depend. @@ -3291,7 +3291,7 @@ guid = None original_tool_data_path = trans.app.config.tool_data_path revision_label = hg_util.get_revision_label( trans, repository, changeset_revision, include_date=False ) - repository_metadata = suc.get_repository_metadata_by_changeset_revision( trans, repository_id, changeset_revision ) + repository_metadata = suc.get_repository_metadata_by_changeset_revision( trans.app, repository_id, changeset_revision ) if repository_metadata: repository_metadata_id = trans.security.encode_id( repository_metadata.id ) metadata = repository_metadata.metadata diff -r 172e1713535c24cc6c07caa5f90494b68827b638 -r b8e434a784960287fc998c9e0a7f11512dc0533c lib/galaxy/webapps/tool_shed/controllers/repository_review.py --- a/lib/galaxy/webapps/tool_shed/controllers/repository_review.py +++ b/lib/galaxy/webapps/tool_shed/controllers/repository_review.py @@ -155,7 +155,7 @@ **kwd ) ) # A review can be initially performed only on an installable revision of a repository, so make sure we have metadata associated # with the received changeset_revision. - repository_metadata = suc.get_repository_metadata_by_changeset_revision( trans, repository_id, changeset_revision ) + repository_metadata = suc.get_repository_metadata_by_changeset_revision( trans.app, repository_id, changeset_revision ) if repository_metadata: metadata = repository_metadata.metadata if metadata: @@ -485,7 +485,7 @@ repository_reviews = review_util.get_reviews_by_repository_id_changeset_revision( trans, repository_id, changeset_revision ) # Determine if the current user can add a review to this revision. can_add_review = trans.user not in [ repository_review.user for repository_review in repository_reviews ] - repository_metadata = suc.get_repository_metadata_by_changeset_revision( trans, repository_id, changeset_revision ) + repository_metadata = suc.get_repository_metadata_by_changeset_revision( trans.app, repository_id, changeset_revision ) if repository_metadata: repository_metadata_reviews = util.listify( repository_metadata.reviews ) else: diff -r 172e1713535c24cc6c07caa5f90494b68827b638 -r b8e434a784960287fc998c9e0a7f11512dc0533c lib/tool_shed/galaxy_install/repository_util.py --- a/lib/tool_shed/galaxy_install/repository_util.py +++ b/lib/tool_shed/galaxy_install/repository_util.py @@ -59,7 +59,7 @@ repository = suc.get_repository_by_name_and_owner( trans.app, repository_name, repository_owner ) if trans.webapp.name == 'tool_shed': # We're in the tool shed. - repository_metadata = suc.get_repository_metadata_by_changeset_revision( trans, + repository_metadata = suc.get_repository_metadata_by_changeset_revision( trans.app, trans.security.encode_id( repository.id ), changeset_revision ) if repository_metadata: @@ -189,7 +189,7 @@ repo_dir = repository.repo_path( trans.app ) repo = hg.repository( hg_util.get_configured_ui(), repo_dir ) repository_clone_url = common_util.generate_clone_url_for_repository_in_tool_shed( trans, repository ) - repository_metadata = suc.get_repository_metadata_by_changeset_revision( trans, + repository_metadata = suc.get_repository_metadata_by_changeset_revision( trans.app, repository_id, changeset_revision ) if not repository_metadata: @@ -200,7 +200,7 @@ suc.get_next_downloadable_changeset_revision( repository,repo, changeset_revision ) if next_downloadable_changeset_revision: repository_metadata = \ - suc.get_repository_metadata_by_changeset_revision( trans, repository_id, next_downloadable_changeset_revision ) + suc.get_repository_metadata_by_changeset_revision( trans.app, repository_id, next_downloadable_changeset_revision ) if repository_metadata: # For now, we'll always assume that we'll get repository_metadata, but if we discover our assumption # is not valid we'll have to enhance the callers to handle repository_metadata values of None in the diff -r 172e1713535c24cc6c07caa5f90494b68827b638 -r b8e434a784960287fc998c9e0a7f11512dc0533c lib/tool_shed/galaxy_install/tool_dependencies/recipe/tag_handler.py --- a/lib/tool_shed/galaxy_install/tool_dependencies/recipe/tag_handler.py +++ b/lib/tool_shed/galaxy_install/tool_dependencies/recipe/tag_handler.py @@ -552,4 +552,3 @@ remove_from_disk=False ) tool_dependencies.append( tool_dependency ) return tool_dependencies - diff -r 172e1713535c24cc6c07caa5f90494b68827b638 -r b8e434a784960287fc998c9e0a7f11512dc0533c lib/tool_shed/grids/repository_grid_filter_manager.py --- /dev/null +++ b/lib/tool_shed/grids/repository_grid_filter_manager.py @@ -0,0 +1,50 @@ +import logging +import os + +from galaxy.util.bunch import Bunch + +log = logging.getLogger( __name__ ) + + +class RepositoryGridFilterManager( object ): + """Provides filtered views of the many Tool SHed repository grids.""" + + filters = Bunch( CERTIFIED_LEVEL_ONE = 'certified_level_one', + CERTIFIED_LEVEL_TWO = 'certified_level_two', + CERTIFIED_LEVEL_ONE_SUITES = 'certified_level_one_suites', + CERTIFIED_LEVEL_TWO_SUITES = 'certified_level_two_suites', + SUITES = 'suites' ) + + def get_grid_title( self, trans, trailing_string='', default='' ): + filter = self.get_filter( trans ) + if filter == self.filters.CERTIFIED_LEVEL_ONE: + return "Certified 1 Repositories %s" % trailing_string + if filter == self.filters.CERTIFIED_LEVEL_TWO: + return "Certified 2 Repositories %s" % trailing_string + if filter == self.filters.CERTIFIED_LEVEL_ONE_SUITES: + return "Certified 1 Repository Suites %s" % trailing_string + if filter == self.filters.CERTIFIED_LEVEL_TWO_SUITES: + return "Certified 2 Repository Suites %s" % trailing_string + if filter == self.filters.SUITES: + return "Repository Suites %s" % trailing_string + return "%s" % default + + def get_filter( self, trans ): + filter = trans.get_cookie( name='toolshedrepogridfilter' ) + return filter or None + + def is_valid_filter( self, filter ): + if filter is None: + return True + for valid_key, valid_filter in self.filters.items(): + if filter == valid_filter: + return True + return False + + def set_filter( self, trans, **kwd ): + # Set a session cookie value with the selected filter. + filter = kwd.get( 'filter', None ) + if filter is not None and self.is_valid_filter( filter ): + trans.set_cookie( value=filter, name='toolshedrepogridfilter' ) + # if the filter is not valid, expire the cookie. + trans.set_cookie( value=filter,name='toolshedrepogridfilter', age=-1 ) diff -r 172e1713535c24cc6c07caa5f90494b68827b638 -r b8e434a784960287fc998c9e0a7f11512dc0533c lib/tool_shed/grids/repository_grids.py --- a/lib/tool_shed/grids/repository_grids.py +++ b/lib/tool_shed/grids/repository_grids.py @@ -42,30 +42,17 @@ class RepositoriesColumn( grids.TextColumn ): def get_value( self, trans, grid, category ): - # TODO: we should probably keep an in-memory register to improve speed here. - if category.repositories: - viewable_repositories = 0 - for rca in category.repositories: - repository = rca.repository - filter = trans.app.repository_grid_filter_manager.get_filter( trans ) - if filter == trans.app.repository_grid_filter_manager.filters.CERTIFIED_LEVEL_ONE: - if not repository.deprecated: - is_level_one_certified_tuple = metadata_util.is_level_one_certified( trans, repository ) - latest_installable_changeset_revision, is_level_one_certified = is_level_one_certified_tuple - if is_level_one_certified: - viewable_repositories += 1 - elif filter == trans.app.repository_grid_filter_manager.filters.CERTIFIED_LEVEL_ONE_SUITES: - if repository.type == rt_util.REPOSITORY_SUITE_DEFINITION and not repository.deprecated: - is_level_one_certified_tuple = metadata_util.is_level_one_certified( trans, repository ) - latest_installable_changeset_revision, is_level_one_certified = is_level_one_certified_tuple - if is_level_one_certified: - viewable_repositories += 1 - else: - # The value filter is None. - if not repository.deleted and not repository.deprecated: - viewable_repositories += 1 - return viewable_repositories - return 0 + category_name = str( category.name ) + filter = trans.app.repository_grid_filter_manager.get_filter( trans ) + if filter == trans.app.repository_grid_filter_manager.filters.CERTIFIED_LEVEL_ONE: + return trans.app.repository_registry.certified_level_one_viewable_repositories_and_suites_by_category.get( category_name, 0 ) + elif filter == trans.app.repository_grid_filter_manager.filters.CERTIFIED_LEVEL_ONE_SUITES: + return trans.app.repository_registry.certified_level_one_viewable_suites_by_category.get( category_name, 0 ) + elif filter == trans.app.repository_grid_filter_manager.filters.SUITES: + return trans.app.repository_registry.viewable_suites_by_category.get( category_name, 0 ) + else: + # The value filter is None. + return trans.app.repository_registry.viewable_repositories_and_suites_by_category.get( category_name, 0 ) title = "Categories" model_class = model.Category @@ -325,19 +312,17 @@ def build_initial_query( self, trans, **kwd ): filter = trans.app.repository_grid_filter_manager.get_filter( trans ) if filter == trans.app.repository_grid_filter_manager.filters.CERTIFIED_LEVEL_ONE: - clause_list = get_certified_level_one_clause_list( trans ) return trans.sa_session.query( model.Repository ) \ .join( model.RepositoryMetadata.table ) \ - .filter( or_( *clause_list ) ) \ + .filter( or_( *trans.app.repository_registry.certified_level_one_clause_list ) ) \ .join( model.User.table ) \ .outerjoin( model.RepositoryCategoryAssociation.table ) \ .outerjoin( model.Category.table ) if filter == trans.app.repository_grid_filter_manager.filters.CERTIFIED_LEVEL_ONE_SUITES: - clause_list = get_certified_level_one_clause_list( trans ) return trans.sa_session.query( model.Repository ) \ .filter( model.Repository.type == rt_util.REPOSITORY_SUITE_DEFINITION ) \ .join( model.RepositoryMetadata.table ) \ - .filter( or_( *clause_list ) ) \ + .filter( or_( *trans.app.repository_registry.certified_level_one_clause_list ) ) \ .join( model.User.table ) \ .outerjoin( model.RepositoryCategoryAssociation.table ) \ .outerjoin( model.Category.table ) @@ -536,21 +521,19 @@ decoded_user_id = trans.security.decode_id( kwd[ 'user_id' ] ) filter = trans.app.repository_grid_filter_manager.get_filter( trans ) if filter == trans.app.repository_grid_filter_manager.filters.CERTIFIED_LEVEL_ONE: - clause_list = get_certified_level_one_clause_list( trans ) return trans.sa_session.query( model.Repository ) \ .filter( model.Repository.table.c.user_id == decoded_user_id ) \ .join( model.RepositoryMetadata.table ) \ - .filter( or_( *clause_list ) ) \ + .filter( or_( *trans.app.repository_registry.certified_level_one_clause_list ) ) \ .join( model.User.table ) \ .outerjoin( model.RepositoryCategoryAssociation.table ) \ .outerjoin( model.Category.table ) if filter == trans.app.repository_grid_filter_manager.filters.CERTIFIED_LEVEL_ONE_SUITES: - clause_list = get_certified_level_one_clause_list( trans ) return trans.sa_session.query( model.Repository ) \ .filter( and_( model.Repository.type == rt_util.REPOSITORY_SUITE_DEFINITION, model.Repository.table.c.user_id == decoded_user_id ) ) \ .join( model.RepositoryMetadata.table ) \ - .filter( or_( *clause_list ) ) \ + .filter( or_( *trans.app.repository_registry.certified_level_one_clause_list ) ) \ .join( model.User.table ) \ .outerjoin( model.RepositoryCategoryAssociation.table ) \ .outerjoin( model.Category.table ) @@ -602,32 +585,30 @@ category_id = kwd.get( 'id', None ) filter = trans.app.repository_grid_filter_manager.get_filter( trans ) if filter == trans.app.repository_grid_filter_manager.filters.CERTIFIED_LEVEL_ONE: - clause_list = get_certified_level_one_clause_list( trans ) if category_id: category = suc.get_category( trans, category_id ) if category: return trans.sa_session.query( model.Repository ) \ .join( model.RepositoryMetadata.table ) \ - .filter( or_( *clause_list ) ) \ + .filter( or_( *trans.app.repository_registry.certified_level_one_clause_list ) ) \ .join( model.User.table ) \ .outerjoin( model.RepositoryCategoryAssociation.table ) \ .outerjoin( model.Category.table ) \ .filter( model.Category.table.c.name == category.name ) return trans.sa_session.query( model.Repository ) \ .join( model.RepositoryMetadata.table ) \ - .filter( or_( *clause_list ) ) \ + .filter( or_( *trans.app.repository_registry.certified_level_one_clause_list ) ) \ .join( model.User.table ) \ .outerjoin( model.RepositoryCategoryAssociation.table ) \ .outerjoin( model.Category.table ) if filter == trans.app.repository_grid_filter_manager.filters.CERTIFIED_LEVEL_ONE_SUITES: - clause_list = get_certified_level_one_clause_list( trans ) if category_id: category = suc.get_category( trans, category_id ) if category: return trans.sa_session.query( model.Repository ) \ .filter( model.Repository.type == rt_util.REPOSITORY_SUITE_DEFINITION ) \ .join( model.RepositoryMetadata.table ) \ - .filter( or_( *clause_list ) ) \ + .filter( or_( *trans.app.repository_registry.certified_level_one_clause_list ) ) \ .join( model.User.table ) \ .outerjoin( model.RepositoryCategoryAssociation.table ) \ .outerjoin( model.Category.table ) \ @@ -635,7 +616,7 @@ return trans.sa_session.query( model.Repository ) \ .filter( model.Repository.type == rt_util.REPOSITORY_SUITE_DEFINITION ) \ .join( model.RepositoryMetadata.table ) \ - .filter( or_( *clause_list ) ) \ + .filter( or_( *trans.app.repository_registry.certified_level_one_clause_list ) ) \ .join( model.User.table ) \ .outerjoin( model.RepositoryCategoryAssociation.table ) \ .outerjoin( model.Category.table ) @@ -763,7 +744,8 @@ for repository in trans.sa_session.query( model.Repository ) \ .filter( and_( model.Repository.table.c.deprecated == False, model.Repository.table.c.deleted == False ) ): - changeset_revision = filter_by_latest_downloadable_changeset_revision_that_has_missing_tool_test_components( trans, repository ) + changeset_revision = \ + grids_util.filter_by_latest_downloadable_changeset_revision_that_has_missing_tool_test_components( trans, repository ) if changeset_revision: revision_clause_list.append( model.RepositoryMetadata.table.c.changeset_revision == changeset_revision ) if revision_clause_list: @@ -805,7 +787,8 @@ .filter( and_( model.Repository.table.c.deprecated == False, model.Repository.table.c.deleted == False ) ) \ .filter( or_( *user_clause_list ) ): - changeset_revision = filter_by_latest_downloadable_changeset_revision_that_has_missing_tool_test_components( trans, repository ) + changeset_revision = \ + grids_util.filter_by_latest_downloadable_changeset_revision_that_has_missing_tool_test_components( trans, repository ) if changeset_revision: revision_clause_list.append( model.RepositoryMetadata.table.c.changeset_revision == changeset_revision ) if revision_clause_list: @@ -850,7 +833,8 @@ for repository in trans.sa_session.query( model.Repository ) \ .filter( and_( model.Repository.table.c.deprecated == False, model.Repository.table.c.deleted == False ) ): - changeset_revision = filter_by_latest_downloadable_changeset_revision_that_has_test_install_errors( trans, repository ) + changeset_revision = \ + grids_util.filter_by_latest_downloadable_changeset_revision_that_has_test_install_errors( trans, repository ) if changeset_revision: revision_clause_list.append( model.RepositoryMetadata.table.c.changeset_revision == changeset_revision ) if revision_clause_list: @@ -892,7 +876,8 @@ .filter( and_( model.Repository.table.c.deprecated == False, model.Repository.table.c.deleted == False ) ) \ .filter( or_( *user_clause_list ) ): - changeset_revision = filter_by_latest_downloadable_changeset_revision_that_has_test_install_errors( trans, repository ) + changeset_revision = \ + grids_util.filter_by_latest_downloadable_changeset_revision_that_has_test_install_errors( trans, repository ) if changeset_revision: revision_clause_list.append( model.RepositoryMetadata.table.c.changeset_revision == changeset_revision ) if revision_clause_list: @@ -937,7 +922,8 @@ for repository in trans.sa_session.query( model.Repository ) \ .filter( and_( model.Repository.table.c.deprecated == False, model.Repository.table.c.deleted == False ) ): - changeset_revision = filter_by_latest_downloadable_changeset_revision_with_skip_tests_checked( trans, repository ) + changeset_revision = \ + grids_util.filter_by_latest_downloadable_changeset_revision_with_skip_tests_checked( trans, repository ) if changeset_revision: revision_clause_list.append( model.RepositoryMetadata.table.c.changeset_revision == changeset_revision ) if revision_clause_list: @@ -979,7 +965,8 @@ .filter( and_( model.Repository.table.c.deprecated == False, model.Repository.table.c.deleted == False ) ) \ .filter( or_( *user_clause_list ) ): - changeset_revision = filter_by_latest_downloadable_changeset_revision_with_skip_tests_checked( trans, repository ) + changeset_revision = \ + grids_util.filter_by_latest_downloadable_changeset_revision_with_skip_tests_checked( trans, repository ) if changeset_revision: revision_clause_list.append( model.RepositoryMetadata.table.c.changeset_revision == changeset_revision ) if revision_clause_list: @@ -1056,7 +1043,8 @@ for repository in trans.sa_session.query( model.Repository ) \ .filter( and_( model.Repository.table.c.deprecated == False, model.Repository.table.c.deleted == False ) ): - changeset_revision = filter_by_latest_downloadable_changeset_revision_that_has_failing_tool_tests( trans, repository ) + changeset_revision = \ + grids_util.filter_by_latest_downloadable_changeset_revision_that_has_failing_tool_tests( trans, repository ) if changeset_revision: revision_clause_list.append( model.RepositoryMetadata.table.c.changeset_revision == changeset_revision ) if revision_clause_list: @@ -1098,7 +1086,8 @@ .filter( and_( model.Repository.table.c.deprecated == False, model.Repository.table.c.deleted == False ) ) \ .filter( or_( *user_clause_list ) ): - changeset_revision = filter_by_latest_downloadable_changeset_revision_that_has_failing_tool_tests( trans, repository ) + changeset_revision = \ + grids_util.filter_by_latest_downloadable_changeset_revision_that_has_failing_tool_tests( trans, repository ) if changeset_revision: revision_clause_list.append( model.RepositoryMetadata.table.c.changeset_revision == changeset_revision ) if revision_clause_list: @@ -1144,7 +1133,8 @@ for repository in trans.sa_session.query( model.Repository ) \ .filter( and_( model.Repository.table.c.deprecated == False, model.Repository.table.c.deleted == False ) ): - changeset_revision = filter_by_latest_downloadable_changeset_revision_that_has_no_failing_tool_tests( trans, repository ) + changeset_revision = \ + grids_util.filter_by_latest_downloadable_changeset_revision_that_has_no_failing_tool_tests( trans, repository ) if changeset_revision: revision_clause_list.append( model.RepositoryMetadata.table.c.changeset_revision == changeset_revision ) if revision_clause_list: @@ -1187,7 +1177,8 @@ .filter( and_( model.Repository.table.c.deprecated == False, model.Repository.table.c.deleted == False ) ) \ .filter( or_( *user_clause_list ) ): - changeset_revision = filter_by_latest_downloadable_changeset_revision_that_has_no_failing_tool_tests( trans, repository ) + changeset_revision = \ + grids_util.filter_by_latest_downloadable_changeset_revision_that_has_no_failing_tool_tests( trans, repository ) if changeset_revision: revision_clause_list.append( model.RepositoryMetadata.table.c.changeset_revision == changeset_revision ) if revision_clause_list: @@ -1216,7 +1207,8 @@ # At the time this grid is displayed we know that the received repository will have invalid tools in its latest changeset revision # that has associated metadata. val = '' - repository_metadata = get_latest_repository_metadata_if_it_includes_invalid_tools( trans, repository ) + repository_metadata = \ + grids_util.get_latest_repository_metadata_if_it_includes_invalid_tools( trans, repository ) metadata = repository_metadata.metadata invalid_tools = metadata.get( 'invalid_tools', [] ) if invalid_tools: @@ -1251,7 +1243,8 @@ for repository in trans.sa_session.query( model.Repository ) \ .filter( and_( model.Repository.table.c.deprecated == False, model.Repository.table.c.deleted == False ) ): - changeset_revision = filter_by_latest_metadata_changeset_revision_that_has_invalid_tools( trans, repository ) + changeset_revision = \ + grids_util.filter_by_latest_metadata_changeset_revision_that_has_invalid_tools( trans, repository ) if changeset_revision: revision_clause_list.append( model.RepositoryMetadata.table.c.changeset_revision == changeset_revision ) if revision_clause_list: @@ -1293,7 +1286,8 @@ .filter( and_( model.Repository.table.c.deprecated == False, model.Repository.table.c.deleted == False ) ) \ .filter( or_( *user_clause_list ) ): - changeset_revision = filter_by_latest_metadata_changeset_revision_that_has_invalid_tools( trans, repository ) + changeset_revision = \ + grids_util.filter_by_latest_metadata_changeset_revision_that_has_invalid_tools( trans, repository ) if changeset_revision: revision_clause_list.append( model.RepositoryMetadata.table.c.changeset_revision == changeset_revision ) if revision_clause_list: @@ -1478,16 +1472,21 @@ required_repository = suc.get_repository_by_name_and_owner( trans.app, name, owner ) if required_repository and not required_repository.deleted: required_repository_id = trans.security.encode_id( required_repository.id ) - required_repository_metadata = metadata_util.get_repository_metadata_by_repository_id_changeset_revision( trans, - required_repository_id, - changeset_revision ) + required_repository_metadata = \ + metadata_util.get_repository_metadata_by_repository_id_changeset_revision( trans, + required_repository_id, + changeset_revision ) if not required_repository_metadata: repo_dir = required_repository.repo_path( trans.app ) repo = hg.repository( hg_util.get_configured_ui(), repo_dir ) - updated_changeset_revision = suc.get_next_downloadable_changeset_revision( required_repository, repo, changeset_revision ) - required_repository_metadata = metadata_util.get_repository_metadata_by_repository_id_changeset_revision( trans, - required_repository_id, - updated_changeset_revision ) + updated_changeset_revision = \ + suc.get_next_downloadable_changeset_revision( required_repository, + repo, + changeset_revision ) + required_repository_metadata = \ + metadata_util.get_repository_metadata_by_repository_id_changeset_revision( trans, + required_repository_id, + updated_changeset_revision ) required_repository_metadata_id = trans.security.encode_id( required_repository_metadata.id ) rd_str += '<a href="browse_repository_dependencies?operation=view_or_manage_repository&id=%s">' % ( required_repository_metadata_id ) rd_str += 'Repository <b>%s</b> revision <b>%s</b> owned by <b>%s</b>' % ( escape_html( rd_tup[ 1 ] ), escape_html( rd_tup[ 3 ] ), escape_html( rd_tup[ 2 ] ) ) @@ -1741,31 +1740,17 @@ class RepositoriesColumn( grids.TextColumn ): def get_value( self, trans, grid, category ): - # TODO: We should probably keep in in-memory register for speed improvements. - if category.repositories: - viewable_repositories = 0 - for rca in category.repositories: - repository = rca.repository - filter = trans.app.repository_grid_filter_manager.get_filter( trans ) - if filter == trans.app.repository_grid_filter_manager.filters.CERTIFIED_LEVEL_ONE: - if not repository.deprecated and repository.downloadable_revisions: - is_level_one_certified_tuple = metadata_util.is_level_one_certified( trans, repository ) - latest_installable_changeset_revision, is_level_one_certified = is_level_one_certified_tuple - if is_level_one_certified: - viewable_repositories += 1 - if filter == trans.app.repository_grid_filter_manager.filters.CERTIFIED_LEVEL_ONE_SUITES: - if repository.type == rt_util.REPOSITORY_SUITE_DEFINITION and \ - not repository.deprecated and repository.downloadable_revisions: - is_level_one_certified_tuple = metadata_util.is_level_one_certified( trans, repository ) - latest_installable_changeset_revision, is_level_one_certified = is_level_one_certified_tuple - if is_level_one_certified: - viewable_repositories += 1 - else: - # The value of filter is None. - if not repository.deleted and not repository.deprecated and repository.downloadable_revisions: - viewable_repositories += 1 - return viewable_repositories - return 0 + category_name = str( category.name ) + filter = trans.app.repository_grid_filter_manager.get_filter( trans ) + if filter == trans.app.repository_grid_filter_manager.filters.CERTIFIED_LEVEL_ONE: + return trans.app.repository_registry.certified_level_one_viewable_repositories_and_suites_by_category.get( category_name, 0 ) + elif filter == trans.app.repository_grid_filter_manager.filters.CERTIFIED_LEVEL_ONE_SUITES: + return trans.app.repository_registry.certified_level_one_viewable_suites_by_category.get( category_name, 0 ) + elif filter == trans.app.repository_grid_filter_manager.filters.SUITES: + return trans.app.repository_registry.viewable_valid_suites_by_category.get( category_name, 0 ) + else: + # The value filter is None. + return trans.app.repository_registry.viewable_valid_repositories_and_suites_by_category.get( category_name, 0 ) title = "Categories of Valid Repositories" model_class = model.Category @@ -1869,21 +1854,19 @@ # The user is browsing categories of valid repositories, so filter the request by the received id, # which is a category id. if filter == trans.app.repository_grid_filter_manager.filters.CERTIFIED_LEVEL_ONE: - clause_list = get_certified_level_one_clause_list( trans ) return trans.sa_session.query( model.Repository ) \ .join( model.RepositoryMetadata.table ) \ - .filter( or_( *clause_list ) ) \ + .filter( or_( *trans.app.repository_registry.certified_level_one_clause_list ) ) \ .join( model.User.table ) \ .join( model.RepositoryCategoryAssociation.table ) \ .join( model.Category.table ) \ .filter( and_( model.Category.table.c.id == trans.security.decode_id( kwd[ 'id' ] ), model.RepositoryMetadata.table.c.downloadable == True ) ) if filter == trans.app.repository_grid_filter_manager.filters.CERTIFIED_LEVEL_ONE_SUITES: - clause_list = get_certified_level_one_clause_list( trans ) return trans.sa_session.query( model.Repository ) \ .filter( model.Repository.type == rt_util.REPOSITORY_SUITE_DEFINITION ) \ .join( model.RepositoryMetadata.table ) \ - .filter( or_( *clause_list ) ) \ + .filter( or_( *trans.app.repository_registry.certified_level_one_clause_list ) ) \ .join( model.User.table ) \ .join( model.RepositoryCategoryAssociation.table ) \ .join( model.Category.table ) \ @@ -1902,20 +1885,18 @@ model.RepositoryMetadata.table.c.downloadable == True ) ) # The user performed a free text search on the ValidCategoryGrid. if filter == trans.app.repository_grid_filter_manager.filters.CERTIFIED_LEVEL_ONE: - clause_list = get_certified_level_one_clause_list( trans ) return trans.sa_session.query( model.Repository ) \ .join( model.RepositoryMetadata.table ) \ - .filter( or_( *clause_list ) ) \ + .filter( or_( *trans.app.repository_registry.certified_level_one_clause_list ) ) \ .join( model.User.table ) \ .outerjoin( model.RepositoryCategoryAssociation.table ) \ .outerjoin( model.Category.table ) \ .filter( model.RepositoryMetadata.table.c.downloadable == True ) if filter == trans.app.repository_grid_filter_manager.filters.CERTIFIED_LEVEL_ONE_SUITES: - clause_list = get_certified_level_one_clause_list( trans ) return trans.sa_session.query( model.Repository ) \ .filter( model.Repository.type == rt_util.REPOSITORY_SUITE_DEFINITION ) \ .join( model.RepositoryMetadata.table ) \ - .filter( or_( *clause_list ) ) \ + .filter( or_( *trans.app.repository_registry.certified_level_one_clause_list ) ) \ .join( model.User.table ) \ .outerjoin( model.RepositoryCategoryAssociation.table ) \ .outerjoin( model.Category.table ) \ @@ -1930,206 +1911,3 @@ .outerjoin( model.RepositoryCategoryAssociation.table ) \ .outerjoin( model.Category.table ) \ .filter( model.RepositoryMetadata.table.c.downloadable == True ) - -# ------ utility methods ------------------- - -def filter_by_latest_downloadable_changeset_revision_that_has_failing_tool_tests( trans, repository ): - """ - Inspect the latest downloadable changeset revision for the received repository to see if it - includes at least 1 tool that has at least 1 failing test. This will filter out repositories - of type repository_suite_definition and tool_dependency_definition. - """ - repository_metadata = get_latest_downloadable_repository_metadata_if_it_includes_tools( trans, repository ) - if repository_metadata is not None \ - and has_been_tested( repository_metadata ) \ - and not repository_metadata.missing_test_components \ - and not repository_metadata.tools_functionally_correct \ - and not repository_metadata.test_install_error: - return repository_metadata.changeset_revision - return None - -def filter_by_latest_downloadable_changeset_revision_that_has_missing_tool_test_components( trans, repository ): - """ - Inspect the latest downloadable changeset revision for the received repository to see if it - includes tools that are either missing functional tests or functional test data. If the - changset revision includes tools but is missing tool test components, return the changeset - revision hash. This will filter out repositories of type repository_suite_definition and - tool_dependency_definition. - """ - repository_metadata = get_latest_downloadable_repository_metadata_if_it_includes_tools( trans, repository ) - if repository_metadata is not None \ - and has_been_tested( repository_metadata ) \ - and repository_metadata.missing_test_components: - return repository_metadata.changeset_revision - return None - -def filter_by_latest_downloadable_changeset_revision_that_has_no_failing_tool_tests( trans, repository ): - """ - Inspect the latest downloadable changeset revision for the received repository to see if it - includes tools with no failing tests. This will filter out repositories of type repository_suite_definition - and tool_dependency_definition. - """ - repository_metadata = get_latest_downloadable_repository_metadata_if_it_includes_tools( trans, repository ) - if repository_metadata is not None \ - and has_been_tested( repository_metadata ) \ - and not repository_metadata.missing_test_components \ - and repository_metadata.tools_functionally_correct: - return repository_metadata.changeset_revision - return None - -def filter_by_latest_metadata_changeset_revision_that_has_invalid_tools( trans, repository ): - """ - Inspect the latest changeset revision with associated metadata for the received repository - to see if it has invalid tools. This will filter out repositories of type repository_suite_definition - and tool_dependency_definition. - """ - repository_metadata = get_latest_repository_metadata_if_it_includes_invalid_tools( trans, repository ) - if repository_metadata is not None: - return repository_metadata.changeset_revision - return None - -def filter_by_latest_downloadable_changeset_revision_that_has_test_install_errors( trans, repository ): - """ - Inspect the latest downloadable changeset revision for the received repository to see if - it has tool test installation errors. This will return repositories of type unrestricted - as well as types repository_suite_definition and tool_dependency_definition. - """ - repository_metadata = get_latest_downloadable_repository_metadata_if_it_has_test_install_errors( trans, repository ) - # Filter further by eliminating repositories that are missing test components. - if repository_metadata is not None \ - and has_been_tested( repository_metadata ) \ - and not repository_metadata.missing_test_components: - return repository_metadata.changeset_revision - return None - -def filter_by_latest_downloadable_changeset_revision_with_skip_tests_checked( trans, repository ): - """ - Inspect the latest downloadable changeset revision for the received repository to see if skip tests - is checked. This will return repositories of type unrestricted as well as types repository_suite_definition - and tool_dependency_definition. - """ - repository_metadata = get_latest_downloadable_repository_metadata( trans, repository ) - # The skip_tool_tests attribute is a SkipToolTest table mapping backref to the RepositoryMetadata table. - if repository_metadata is not None and repository_metadata.skip_tool_tests: - return repository_metadata.changeset_revision - return None - -def get_certified_level_one_clause_list( trans ): - clause_list = [] - for repository in trans.sa_session.query( model.Repository ) \ - .filter( and_( model.Repository.table.c.deleted == False, - model.Repository.table.c.deprecated == False ) ): - is_level_one_certified_tuple = metadata_util.is_level_one_certified( trans, repository ) - latest_installable_changeset_revision, is_level_one_certified = is_level_one_certified_tuple - if is_level_one_certified: - clause_list.append( "%s=%d and %s='%s'" % ( model.RepositoryMetadata.table.c.repository_id, - repository.id, - model.RepositoryMetadata.table.c.changeset_revision, - latest_installable_changeset_revision ) ) - return clause_list - -def get_latest_downloadable_repository_metadata( trans, repository ): - """ - Return the latest downloadable repository_metadata record for the received repository. This will - return repositories of type unrestricted as well as types repository_suite_definition and - tool_dependency_definition. - """ - encoded_repository_id = trans.security.encode_id( repository.id ) - repo = hg.repository( hg_util.get_configured_ui(), repository.repo_path( trans.app ) ) - tip_ctx = str( repo.changectx( repo.changelog.tip() ) ) - repository_metadata = None - try: - repository_metadata = suc.get_repository_metadata_by_changeset_revision( trans, encoded_repository_id, tip_ctx ) - if repository_metadata is not None and repository_metadata.downloadable: - return repository_metadata - return None - except: - latest_downloadable_revision = suc.get_previous_metadata_changeset_revision( repository, repo, tip_ctx, downloadable=True ) - if latest_downloadable_revision == suc.INITIAL_CHANGELOG_HASH: - return None - repository_metadata = suc.get_repository_metadata_by_changeset_revision( trans, - encoded_repository_id, - latest_downloadable_revision ) - if repository_metadata is not None and repository_metadata.downloadable: - return repository_metadata - return None - -def get_latest_downloadable_repository_metadata_if_it_includes_tools( trans, repository ): - """ - Return the latest downloadable repository_metadata record for the received repository if its - includes_tools attribute is True. This will filter out repositories of type repository_suite_definition - and tool_dependency_definition. - """ - repository_metadata = get_latest_downloadable_repository_metadata( trans, repository ) - if repository_metadata is not None and repository_metadata.includes_tools: - return repository_metadata - return None - -def get_latest_downloadable_repository_metadata_if_it_has_test_install_errors( trans, repository ): - """ - Return the latest downloadable repository_metadata record for the received repository if its - test_install_error attribute is True. This will return repositories of type unrestricted as - well as types repository_suite_definition and tool_dependency_definition. - """ - repository_metadata = get_latest_downloadable_repository_metadata( trans, repository ) - if repository_metadata is not None \ - and has_been_tested( repository_metadata ) \ - and repository_metadata.test_install_error: - return repository_metadata - return None - -def get_latest_repository_metadata( trans, repository ): - """ - Return the latest repository_metadata record for the received repository if it exists. This will - return repositories of type unrestricted as well as types repository_suite_definition and - tool_dependency_definition. - """ - encoded_repository_id = trans.security.encode_id( repository.id ) - repo = hg.repository( hg_util.get_configured_ui(), repository.repo_path( trans.app ) ) - tip_ctx = str( repo.changectx( repo.changelog.tip() ) ) - try: - repository_metadata = suc.get_repository_metadata_by_changeset_revision( trans, encoded_repository_id, tip_ctx ) - return repository_metadata - except: - latest_downloadable_revision = suc.get_previous_metadata_changeset_revision( repository, repo, tip_ctx, downloadable=False ) - if latest_downloadable_revision == suc.INITIAL_CHANGELOG_HASH: - return None - repository_metadata = suc.get_repository_metadata_by_changeset_revision( trans, - encoded_repository_id, - latest_downloadable_revision ) - return repository_metadata - -def get_latest_repository_metadata_if_it_includes_invalid_tools( trans, repository ): - """ - Return the latest repository_metadata record for the received repository that contains invalid - tools if one exists. This will filter out repositories of type repository_suite_definition and - tool_dependency_definition. - """ - repository_metadata = get_latest_repository_metadata( trans, repository ) - if repository_metadata is not None: - metadata = repository_metadata.metadata - if metadata is not None and 'invalid_tools' in metadata: - return repository_metadata - return None - -def has_been_tested( repository_metadata ): - """ - Return True if the received repository_metadata record'd tool_test_results column was populated by - the Tool Shed's install and test framework. - """ - tool_test_results = repository_metadata.tool_test_results - if tool_test_results is None: - return False - # The install and test framework's preparation scripts will populate the tool_test_results column - # with something like this: - # [{"test_environment": - # {"time_tested": "2014-05-15 16:15:18", - # "tool_shed_database_version": 22, - # "tool_shed_mercurial_version": "2.2.3", - # "tool_shed_revision": "13459:9a1415f8108f"} - # }] - tool_test_results = listify( tool_test_results ) - for test_results_dict in tool_test_results: - if len( test_results_dict ) > 1: - return True - return False diff -r 172e1713535c24cc6c07caa5f90494b68827b638 -r b8e434a784960287fc998c9e0a7f11512dc0533c lib/tool_shed/grids/util.py --- a/lib/tool_shed/grids/util.py +++ b/lib/tool_shed/grids/util.py @@ -1,53 +1,13 @@ import logging import os -import tool_shed.util.shed_util_common as suc + +from galaxy.util import listify +from galaxy.web.form_builder import SelectField + from tool_shed.util import hg_util -from galaxy.web.form_builder import SelectField -from galaxy.util.bunch import Bunch log = logging.getLogger( __name__ ) - -class RepositoryGridFilterManager( object ): - """Provides filtered views of the many Tool SHed repository grids.""" - - filters = Bunch( CERTIFIED_LEVEL_ONE = 'certified_level_one', - CERTIFIED_LEVEL_TWO = 'certified_level_two', - CERTIFIED_LEVEL_ONE_SUITES = 'certified_level_one_suites', - CERTIFIED_LEVEL_TWO_SUITES = 'certified_level_two_suites' ) - - def get_grid_title( self, trans, trailing_string='', default='' ): - filter = self.get_filter( trans ) - if filter == self.filters.CERTIFIED_LEVEL_ONE: - return "Certified 1 Repositories %s" % trailing_string - if filter == self.filters.CERTIFIED_LEVEL_TWO: - return "Certified 2 Repositories %s" % trailing_string - if filter == self.filters.CERTIFIED_LEVEL_ONE_SUITES: - return "Certified 1 Repository Suites %s" % trailing_string - if filter == self.filters.CERTIFIED_LEVEL_TWO_SUITES: - return "Certified 2 Repository Suites %s" % trailing_string - return "%s" % default - - def get_filter( self, trans ): - filter = trans.get_cookie( name='toolshedrepogridfilter' ) - return filter or None - - def is_valid_filter( self, filter ): - if filter is None: - return True - for valid_key, valid_filter in self.filters.items(): - if filter == valid_filter: - return True - return False - - def set_filter( self, trans, **kwd ): - # Set a session cookie value with the selected filter. - filter = kwd.get( 'filter', None ) - if filter is not None and self.is_valid_filter( filter ): - trans.set_cookie( value=filter, name='toolshedrepogridfilter' ) - # if the filter is not valid, expire the cookie. - trans.set_cookie( value=filter,name='toolshedrepogridfilter', age=-1 ) - def build_approved_select_field( trans, name, selected_value=None, for_component=True ): options = [ ( 'No', trans.model.ComponentReview.approved_states.NO ), ( 'Yes', trans.model.ComponentReview.approved_states.YES ) ] @@ -119,3 +79,190 @@ selected = selected_value and option_tup[ 1 ] == selected_value select_field.add_option( option_tup[ 0 ], option_tup[ 1 ], selected=selected ) return select_field + +def filter_by_latest_downloadable_changeset_revision_that_has_failing_tool_tests( trans, repository ): + """ + Inspect the latest downloadable changeset revision for the received repository to see if it + includes at least 1 tool that has at least 1 failing test. This will filter out repositories + of type repository_suite_definition and tool_dependency_definition. + """ + repository_metadata = get_latest_downloadable_repository_metadata_if_it_includes_tools( trans, repository ) + if repository_metadata is not None \ + and has_been_tested( repository_metadata ) \ + and not repository_metadata.missing_test_components \ + and not repository_metadata.tools_functionally_correct \ + and not repository_metadata.test_install_error: + return repository_metadata.changeset_revision + return None + +def filter_by_latest_downloadable_changeset_revision_that_has_missing_tool_test_components( trans, repository ): + """ + Inspect the latest downloadable changeset revision for the received repository to see if it + includes tools that are either missing functional tests or functional test data. If the + changset revision includes tools but is missing tool test components, return the changeset + revision hash. This will filter out repositories of type repository_suite_definition and + tool_dependency_definition. + """ + repository_metadata = get_latest_downloadable_repository_metadata_if_it_includes_tools( trans, repository ) + if repository_metadata is not None \ + and has_been_tested( repository_metadata ) \ + and repository_metadata.missing_test_components: + return repository_metadata.changeset_revision + return None + +def filter_by_latest_downloadable_changeset_revision_that_has_no_failing_tool_tests( trans, repository ): + """ + Inspect the latest downloadable changeset revision for the received repository to see if it + includes tools with no failing tests. This will filter out repositories of type repository_suite_definition + and tool_dependency_definition. + """ + repository_metadata = get_latest_downloadable_repository_metadata_if_it_includes_tools( trans, repository ) + if repository_metadata is not None \ + and has_been_tested( repository_metadata ) \ + and not repository_metadata.missing_test_components \ + and repository_metadata.tools_functionally_correct: + return repository_metadata.changeset_revision + return None + +def filter_by_latest_metadata_changeset_revision_that_has_invalid_tools( trans, repository ): + """ + Inspect the latest changeset revision with associated metadata for the received repository + to see if it has invalid tools. This will filter out repositories of type repository_suite_definition + and tool_dependency_definition. + """ + repository_metadata = get_latest_repository_metadata_if_it_includes_invalid_tools( trans, repository ) + if repository_metadata is not None: + return repository_metadata.changeset_revision + return None + +def filter_by_latest_downloadable_changeset_revision_that_has_test_install_errors( trans, repository ): + """ + Inspect the latest downloadable changeset revision for the received repository to see if + it has tool test installation errors. This will return repositories of type unrestricted + as well as types repository_suite_definition and tool_dependency_definition. + """ + repository_metadata = get_latest_downloadable_repository_metadata_if_it_has_test_install_errors( trans, repository ) + # Filter further by eliminating repositories that are missing test components. + if repository_metadata is not None \ + and has_been_tested( repository_metadata ) \ + and not repository_metadata.missing_test_components: + return repository_metadata.changeset_revision + return None + +def filter_by_latest_downloadable_changeset_revision_with_skip_tests_checked( trans, repository ): + """ + Inspect the latest downloadable changeset revision for the received repository to see if skip tests + is checked. This will return repositories of type unrestricted as well as types repository_suite_definition + and tool_dependency_definition. + """ + repository_metadata = get_latest_downloadable_repository_metadata( trans, repository ) + # The skip_tool_tests attribute is a SkipToolTest table mapping backref to the RepositoryMetadata table. + if repository_metadata is not None and repository_metadata.skip_tool_tests: + return repository_metadata.changeset_revision + return None + +def get_latest_downloadable_repository_metadata( trans, repository ): + """ + Return the latest downloadable repository_metadata record for the received repository. This will + return repositories of type unrestricted as well as types repository_suite_definition and + tool_dependency_definition. + """ + encoded_repository_id = trans.security.encode_id( repository.id ) + repo = hg.repository( hg_util.get_configured_ui(), repository.repo_path( trans.app ) ) + tip_ctx = str( repo.changectx( repo.changelog.tip() ) ) + repository_metadata = None + try: + repository_metadata = suc.get_repository_metadata_by_changeset_revision( trans.app, encoded_repository_id, tip_ctx ) + if repository_metadata is not None and repository_metadata.downloadable: + return repository_metadata + return None + except: + latest_downloadable_revision = suc.get_previous_metadata_changeset_revision( repository, repo, tip_ctx, downloadable=True ) + if latest_downloadable_revision == suc.INITIAL_CHANGELOG_HASH: + return None + repository_metadata = suc.get_repository_metadata_by_changeset_revision( trans.app, + encoded_repository_id, + latest_downloadable_revision ) + if repository_metadata is not None and repository_metadata.downloadable: + return repository_metadata + return None + +def get_latest_downloadable_repository_metadata_if_it_includes_tools( trans, repository ): + """ + Return the latest downloadable repository_metadata record for the received repository if its + includes_tools attribute is True. This will filter out repositories of type repository_suite_definition + and tool_dependency_definition. + """ + repository_metadata = get_latest_downloadable_repository_metadata( trans, repository ) + if repository_metadata is not None and repository_metadata.includes_tools: + return repository_metadata + return None + +def get_latest_downloadable_repository_metadata_if_it_has_test_install_errors( trans, repository ): + """ + Return the latest downloadable repository_metadata record for the received repository if its + test_install_error attribute is True. This will return repositories of type unrestricted as + well as types repository_suite_definition and tool_dependency_definition. + """ + repository_metadata = get_latest_downloadable_repository_metadata( trans, repository ) + if repository_metadata is not None \ + and has_been_tested( repository_metadata ) \ + and repository_metadata.test_install_error: + return repository_metadata + return None + +def get_latest_repository_metadata( trans, repository ): + """ + Return the latest repository_metadata record for the received repository if it exists. This will + return repositories of type unrestricted as well as types repository_suite_definition and + tool_dependency_definition. + """ + encoded_repository_id = trans.security.encode_id( repository.id ) + repo = hg.repository( hg_util.get_configured_ui(), repository.repo_path( trans.app ) ) + tip_ctx = str( repo.changectx( repo.changelog.tip() ) ) + try: + repository_metadata = suc.get_repository_metadata_by_changeset_revision( trans.app, encoded_repository_id, tip_ctx ) + return repository_metadata + except: + latest_downloadable_revision = suc.get_previous_metadata_changeset_revision( repository, repo, tip_ctx, downloadable=False ) + if latest_downloadable_revision == suc.INITIAL_CHANGELOG_HASH: + return None + repository_metadata = suc.get_repository_metadata_by_changeset_revision( trans.app, + encoded_repository_id, + latest_downloadable_revision ) + return repository_metadata + +def get_latest_repository_metadata_if_it_includes_invalid_tools( trans, repository ): + """ + Return the latest repository_metadata record for the received repository that contains invalid + tools if one exists. This will filter out repositories of type repository_suite_definition and + tool_dependency_definition. + """ + repository_metadata = get_latest_repository_metadata( trans, repository ) + if repository_metadata is not None: + metadata = repository_metadata.metadata + if metadata is not None and 'invalid_tools' in metadata: + return repository_metadata + return None + +def has_been_tested( repository_metadata ): + """ + Return True if the received repository_metadata record'd tool_test_results column was populated by + the Tool Shed's install and test framework. + """ + tool_test_results = repository_metadata.tool_test_results + if tool_test_results is None: + return False + # The install and test framework's preparation scripts will populate the tool_test_results column + # with something like this: + # [{"test_environment": + # {"time_tested": "2014-05-15 16:15:18", + # "tool_shed_database_version": 22, + # "tool_shed_mercurial_version": "2.2.3", + # "tool_shed_revision": "13459:9a1415f8108f"} + # }] + tool_test_results = listify( tool_test_results ) + for test_results_dict in tool_test_results: + if len( test_results_dict ) > 1: + return True + return False diff -r 172e1713535c24cc6c07caa5f90494b68827b638 -r b8e434a784960287fc998c9e0a7f11512dc0533c lib/tool_shed/repository_registry.py --- /dev/null +++ b/lib/tool_shed/repository_registry.py @@ -0,0 +1,202 @@ +import os +import logging + +import tool_shed.repository_types.util as rt_util + +from galaxy.model.orm import and_ +from galaxy.model.orm import or_ +from galaxy.webapps.tool_shed import model + +from tool_shed.util import hg_util +from tool_shed.util import metadata_util +from tool_shed.util import shed_util_common as suc + +log = logging.getLogger( __name__ ) + + +class Registry( object ): + + def __init__( self, app ): + self.app = app + self.certified_level_one_clause_list = self.get_certified_level_one_clause_list() + # The following lists contain tuples like ( repository.name, repository.user.username, changeset_revision ) + # where the changeset_revision entry is always the latest installable changeset_revision.. + self.certified_level_one_repository_and_suite_tuples = [] + self.certified_level_one_suite_tuples = [] + # These category dictionaries contain entries where the key is the category and the value is the integer count + # of viewable repositories within that category. + self.certified_level_one_viewable_repositories_and_suites_by_category = {} + self.certified_level_one_viewable_suites_by_category = {} + self.certified_level_two_repository_and_suite_tuples = [] + self.certified_level_two_suite_tuples = [] + self.certified_level_two_viewable_repositories_and_suites_by_category = {} + self.certified_level_two_viewable_suites_by_category = {} + self.repository_and_suite_tuples = [] + self.suite_tuples = [] + self.viewable_repositories_and_suites_by_category = {} + self.viewable_suites_by_category = {} + self.viewable_valid_repositories_and_suites_by_category = {} + self.viewable_valid_suites_by_category = {} + self.load_viewable_repositories_and_suites_by_category() + self.load_repository_and_suite_tuples() + + def get_certified_level_one_clause_list( self ): + certified_level_one_tuples = [] + clause_list = [] + for repository in self.sa_session.query( model.Repository ) \ + .filter( and_( model.Repository.table.c.deleted == False, + model.Repository.table.c.deprecated == False ) ): + certified_level_one_tuple = self.get_certified_level_one_tuple( repository ) + latest_installable_changeset_revision, is_level_one_certified = certified_level_one_tuple + if is_level_one_certified: + certified_level_one_tuples.append( certified_level_one_tuple ) + clause_list.append( "%s=%d and %s='%s'" % ( model.RepositoryMetadata.table.c.repository_id, + repository.id, + model.RepositoryMetadata.table.c.changeset_revision, + latest_installable_changeset_revision ) ) + return clause_list + + def get_certified_level_one_tuple( self, repository ): + """ + Return True if the latest installable changeset_revision of the received repository is level one certified. + """ + if repository.deleted: + return ( None, False ) + repo = hg_util.get_repo_for_repository( self.app, repository ) + # Get the latest installable changeset revision since that is all that is currently configured for testing. + latest_installable_changeset_revision = suc.get_latest_downloadable_changeset_revision( self.app, repository, repo ) + if latest_installable_changeset_revision not in [ None, suc.INITIAL_CHANGELOG_HASH ]: + encoded_repository_id = self.app.security.encode_id( repository.id ) + repository_metadata = suc.get_repository_metadata_by_changeset_revision( self.app, + encoded_repository_id, + latest_installable_changeset_revision ) + if repository_metadata: + # Filter out repository revisions that have not been tested. + if repository_metadata.time_last_tested is not None and repository_metadata.tool_test_results is not None: + if repository.type in [ rt_util.REPOSITORY_SUITE_DEFINITION, rt_util.TOOL_DEPENDENCY_DEFINITION ]: + # Look in the tool_test_results dictionary for installation errors. + try: + tool_test_results_dict = repository_metadata.tool_test_results[ 0 ] + except Exception, e: + message = 'Error attempting to retrieve install and test results for repository %s:\n' % str( repository.name ) + message += '%s' % str( e ) + log.exception( message ) + return ( latest_installable_changeset_revision, False ) + if 'installation_errors' in tool_test_results_dict: + return ( latest_installable_changeset_revision, False ) + return ( latest_installable_changeset_revision, True ) + else: + # We have a repository with type Unrestricted. + if repository_metadata.includes_tools: + if repository_metadata.tools_functionally_correct: + return ( latest_installable_changeset_revision, True ) + return ( latest_installable_changeset_revision, False ) + else: + # Look in the tool_test_results dictionary for installation errors. + try: + tool_test_results_dict = repository_metadata.tool_test_results[ 0 ] + except Exception, e: + message = 'Error attempting to retrieve install and test results for repository %s:\n' % str( repository.name ) + message += '%s' % str( e ) + log.exception( message ) + return ( latest_installable_changeset_revision, False ) + if 'installation_errors' in tool_test_results_dict: + return ( latest_installable_changeset_revision, False ) + return ( latest_installable_changeset_revision, True ) + else: + # No test results. + return ( latest_installable_changeset_revision, False ) + else: + # No repository_metadata. + return ( latest_installable_changeset_revision, False ) + else: + # No installable changeset_revision. + return ( None, False ) + + def is_level_one_certified( self, repository_metadata ): + repository = repository_metadata.repository + if repository.deprecated or repository.deleted: + return False + tuple = ( str( repository.name ), str( repository.user.username ), str( repository_metadata.changeset_revision ) ) + if repository.type in [ rt_util.REPOSITORY_SUITE_DEFINITION ]: + return tuple in self.certified_level_one_suite_tuples + else: + return tuple in self.certified_level_one_repository_and_suite_tuples + + def load_repository_and_suite_tuples( self ): + # Load self.certified_level_one_repository_and_suite_tuples and self.certified_level_one_suite_tuples. + for repository in self.sa_session.query( model.Repository ) \ + .join( model.RepositoryMetadata.table ) \ + .filter( or_( *self.certified_level_one_clause_list ) ) \ + .join( model.User.table ): + name = str( repository.name ) + owner = str( repository.user.username ) + tip_changeset_hash = repository.tip( self.app ) + certified_level_one_tuple = ( name, owner, tip_changeset_hash ) + if repository.type == rt_util.REPOSITORY_SUITE_DEFINITION: + self.certified_level_one_suite_tuples.append( certified_level_one_tuple ) + else: + self.certified_level_one_repository_and_suite_tuples.append( certified_level_one_tuple ) + # Load self.repository_and_suite_tuples and self.suite_tuples + for repository in self.sa_session.query( model.Repository ) \ + .filter( and_( model.Repository.table.c.deleted == False, + model.Repository.table.c.deprecated == False ) ) \ + .join( model.User.table ): + name = str( repository.name ) + owner = str( repository.user.username ) + for repository_metadata in repository.metadata_revisions: + changeset_revision = str( repository_metadata.changeset_revision ) + tuple = ( name, owner, changeset_revision ) + self.repository_and_suite_tuples.append( tuple ) + if repository.type == rt_util.REPOSITORY_SUITE_DEFINITION: + self.suite_tuples.append( tuple ) + + def load_viewable_repositories_and_suites_by_category( self ): + # Clear all dictionaries just in case they were previously loaded. + self.certified_level_one_viewable_repositories_and_suites_by_category = {} + self.certified_level_one_viewable_suites_by_category = {} + self.certified_level_two_viewable_repositories_and_suites_by_category = {} + self.certified_level_two_viewable_suites_by_category = {} + self.viewable_repositories_and_suites_by_category = {} + self.viewable_suites_by_category = {} + self.viewable_valid_repositories_and_suites_by_category = {} + self.viewable_valid_suites_by_category = {} + for category in self.sa_session.query( model.Category ): + if category not in self.certified_level_one_viewable_repositories_and_suites_by_category: + self.certified_level_one_viewable_repositories_and_suites_by_category[ str( category.name ) ] = 0 + if category not in self.certified_level_one_viewable_suites_by_category: + self.certified_level_one_viewable_suites_by_category[ str( category.name ) ] = 0 + if category not in self.viewable_repositories_and_suites_by_category: + self.viewable_repositories_and_suites_by_category[ str( category.name ) ] = 0 + if category not in self.viewable_suites_by_category: + self.viewable_suites_by_category[ str( category.name ) ] = 0 + if category not in self.viewable_valid_repositories_and_suites_by_category: + self.viewable_valid_repositories_and_suites_by_category[ str( category.name ) ] = 0 + if category not in self.viewable_valid_suites_by_category: + self.viewable_valid_suites_by_category[ str( category.name ) ] = 0 + for rca in category.repositories: + repository = rca.repository + if repository.downloadable_revisions: + is_valid = True + else: + is_valid = False + encoded_repository_id = self.app.security.encode_id( repository.id ) + tip_changeset_hash = repository.tip( self.app ) + repository_metadata = suc.get_repository_metadata_by_changeset_revision( self.app, + encoded_repository_id, + tip_changeset_hash ) + self.viewable_repositories_and_suites_by_category[ str( category.name ) ] += 1 + if is_valid: + self.viewable_valid_repositories_and_suites_by_category[ str( category.name ) ] += 1 + if repository.type in [ rt_util.REPOSITORY_SUITE_DEFINITION ]: + self.viewable_suites_by_category[ str( category.name ) ] += 1 + if is_valid: + self.viewable_valid_suites_by_category[ str( category.name ) ] += 1 + if self.is_level_one_certified( repository_metadata ): + self.certified_level_one_viewable_repositories_and_suites_by_category[ str( category.name ) ] += 1 + if repository.type in [ rt_util.REPOSITORY_SUITE_DEFINITION ]: + self.certified_level_one_viewable_suites_by_category[ str( category.name ) ] += 1 + + @property + def sa_session( self ): + return self.app.model.context.current diff -r 172e1713535c24cc6c07caa5f90494b68827b638 -r b8e434a784960287fc998c9e0a7f11512dc0533c lib/tool_shed/repository_types/util.py --- a/lib/tool_shed/repository_types/util.py +++ b/lib/tool_shed/repository_types/util.py @@ -7,7 +7,7 @@ REPOSITORY_SUITE_DEFINITION = 'repository_suite_definition' TOOL_DEPENDENCY_DEFINITION = 'tool_dependency_definition' -types = [ UNRESTRICTED, TOOL_DEPENDENCY_DEFINITION ] +types = [ UNRESTRICTED, TOOL_DEPENDENCY_DEFINITION, REPOSITORY_SUITE_DEFINITION ] def build_repository_type_select_field( trans, repository=None, name='repository_type' ): """Called from the Tool Shed to generate the current list of supported repository types.""" diff -r 172e1713535c24cc6c07caa5f90494b68827b638 -r b8e434a784960287fc998c9e0a7f11512dc0533c lib/tool_shed/util/commit_util.py --- a/lib/tool_shed/util/commit_util.py +++ b/lib/tool_shed/util/commit_util.py @@ -344,7 +344,7 @@ if repository: repo_dir = repository.repo_path( trans.app ) repo = hg.repository( hg_util.get_configured_ui(), repo_dir ) - lastest_installable_changeset_revision = suc.get_latest_downloadable_changeset_revision( trans, repository, repo ) + lastest_installable_changeset_revision = suc.get_latest_downloadable_changeset_revision( trans.app, repository, repo ) if lastest_installable_changeset_revision != suc.INITIAL_CHANGELOG_HASH: elem.attrib[ 'changeset_revision' ] = lastest_installable_changeset_revision revised = True diff -r 172e1713535c24cc6c07caa5f90494b68827b638 -r b8e434a784960287fc998c9e0a7f11512dc0533c lib/tool_shed/util/export_util.py --- a/lib/tool_shed/util/export_util.py +++ b/lib/tool_shed/util/export_util.py @@ -223,7 +223,7 @@ received repository_id. """ repository = suc.get_repository_in_tool_shed( trans, repository_id ) - repository_metadata = suc.get_repository_metadata_by_changeset_revision( trans, repository_id, changeset_revision ) + repository_metadata = suc.get_repository_metadata_by_changeset_revision( trans.app, repository_id, changeset_revision ) # Get a dictionary of all repositories upon which the contents of the current repository_metadata record depend. toolshed_base_url = str( web.url_for( '/', qualified=True ) ).rstrip( '/' ) repository_dependencies = \ diff -r 172e1713535c24cc6c07caa5f90494b68827b638 -r b8e434a784960287fc998c9e0a7f11512dc0533c lib/tool_shed/util/hg_util.py --- a/lib/tool_shed/util/hg_util.py +++ b/lib/tool_shed/util/hg_util.py @@ -69,12 +69,15 @@ ctx_date = date.strftime( "%Y-%m-%d" ) return ctx_date +def get_repo_for_repository( app, repository ): + return hg.repository( get_configured_ui(), repository.repo_path( app ) ) + def get_revision_label( trans, repository, changeset_revision, include_date=True, include_hash=True ): """ Return a string consisting of the human read-able changeset rev and the changeset revision string which includes the revision date if the receive include_date is True. """ - repo = hg.repository( get_configured_ui(), repository.repo_path( trans.app ) ) + repo = get_repo_for_repository( trans.app, repository ) ctx = get_changectx_for_changeset( repo, changeset_revision ) if ctx: return get_revision_label_from_ctx( ctx, include_date=include_date, include_hash=include_hash ) diff -r 172e1713535c24cc6c07caa5f90494b68827b638 -r b8e434a784960287fc998c9e0a7f11512dc0533c lib/tool_shed/util/import_util.py --- a/lib/tool_shed/util/import_util.py +++ b/lib/tool_shed/util/import_util.py @@ -32,7 +32,7 @@ repository = suc.get_repository_by_name_and_owner( trans.app, name, owner ) # Do not allow the repository to be automatically installed if population resulted in errors. tip_changeset_revision = repository.tip( trans.app ) - repository_metadata = suc.get_repository_metadata_by_changeset_revision( trans, + repository_metadata = suc.get_repository_metadata_by_changeset_revision( trans.app, trans.security.encode_id( repository.id ), tip_changeset_revision ) if repository_metadata: diff -r 172e1713535c24cc6c07caa5f90494b68827b638 -r b8e434a784960287fc998c9e0a7f11512dc0533c lib/tool_shed/util/metadata_util.py --- a/lib/tool_shed/util/metadata_util.py +++ b/lib/tool_shed/util/metadata_util.py @@ -307,7 +307,7 @@ downloadable = True else: downloadable = False - repository_metadata = suc.get_repository_metadata_by_changeset_revision( trans, id, changeset_revision ) + repository_metadata = suc.get_repository_metadata_by_changeset_revision( trans.app, id, changeset_revision ) if repository_metadata: # A repository metadata record already exists with the received changeset_revision, so we don't need to # check the skip_tool_test table. @@ -1048,17 +1048,17 @@ repository = trans.sa_session.query( trans.model.Repository ).get( decoded_repository_id ) repo = hg.repository( hg_util.get_configured_ui(), repository.repo_path( trans.app ) ) if downloadable: - changeset_revision = suc.get_latest_downloadable_changeset_revision( trans, repository, repo ) + changeset_revision = suc.get_latest_downloadable_changeset_revision( trans.app, repository, repo ) else: changeset_revision = suc.get_latest_changeset_revision( trans, repository, repo ) - return suc.get_repository_metadata_by_changeset_revision( trans, trans.security.encode_id( repository.id ), changeset_revision ) + return suc.get_repository_metadata_by_changeset_revision( trans.app, trans.security.encode_id( repository.id ), changeset_revision ) def get_parent_id( trans, id, old_id, version, guid, changeset_revisions ): parent_id = None # Compare from most recent to oldest. changeset_revisions.reverse() for changeset_revision in changeset_revisions: - repository_metadata = suc.get_repository_metadata_by_changeset_revision( trans, id, changeset_revision ) + repository_metadata = suc.get_repository_metadata_by_changeset_revision( trans.app, id, changeset_revision ) metadata = repository_metadata.metadata tools_dicts = metadata.get( 'tools', [] ) for tool_dict in tools_dicts: @@ -1093,11 +1093,11 @@ def get_repository_metadata_by_repository_id_changeset_revision( trans, id, changeset_revision, metadata_only=False ): """Get a specified metadata record for a specified repository in the tool shed.""" if metadata_only: - repository_metadata = suc.get_repository_metadata_by_changeset_revision( trans, id, changeset_revision ) + repository_metadata = suc.get_repository_metadata_by_changeset_revision( trans.app, id, changeset_revision ) if repository_metadata and repository_metadata.metadata: return repository_metadata.metadata return None - return suc.get_repository_metadata_by_changeset_revision( trans, id, changeset_revision ) + return suc.get_repository_metadata_by_changeset_revision( trans.app, id, changeset_revision ) def get_repository_metadata_revisions_for_review( repository, reviewed=True ): repository_metadata_revisions = [] @@ -1337,63 +1337,6 @@ return True return False -def is_level_one_certified( trans, repository ): - """ - Return True if the latest installable changeset_revision of the received repository is level one certified. - """ - if repository.deleted: - return ( None, False ) - repo = hg.repository( hg_util.get_configured_ui(), repository.repo_path( trans.app ) ) - # Get the latest installable changeset revision since that is all that is currently configured for testing. - latest_installable_changeset_revision = suc.get_latest_downloadable_changeset_revision( trans, repository, repo ) - if latest_installable_changeset_revision not in [ None, suc.INITIAL_CHANGELOG_HASH ]: - encoded_repository_id = trans.security.encode_id( repository.id ) - repository_metadata = suc.get_repository_metadata_by_changeset_revision( trans, - encoded_repository_id, - latest_installable_changeset_revision ) - if repository_metadata: - # Filter out repository revisions that have not been tested. - if repository_metadata.time_last_tested is not None and repository_metadata.tool_test_results is not None: - if repository.type in [ rt_util.REPOSITORY_SUITE_DEFINITION, rt_util.TOOL_DEPENDENCY_DEFINITION ]: - # Look in the tool_test_results dictionary for installation errors. - try: - tool_test_results_dict = repository_metadata.tool_test_results[ 0 ] - except Exception, e: - message = 'Error attempting to retrieve install and test results for repository %s:\n' % str( repository.name ) - message += '%s' % str( e ) - log.exception( message ) - return ( latest_installable_changeset_revision, False ) - if 'installation_errors' in tool_test_results_dict: - return ( latest_installable_changeset_revision, False ) - return ( latest_installable_changeset_revision, True ) - else: - # We have a repository with type Unrestricted. - if repository_metadata.includes_tools: - if repository_metadata.tools_functionally_correct: - return ( latest_installable_changeset_revision, True ) - return ( latest_installable_changeset_revision, False ) - else: - # Look in the tool_test_results dictionary for installation errors. - try: - tool_test_results_dict = repository_metadata.tool_test_results[ 0 ] - except Exception, e: - message = 'Error attempting to retrieve install and test results for repository %s:\n' % str( repository.name ) - message += '%s' % str( e ) - log.exception( message ) - return ( latest_installable_changeset_revision, False ) - if 'installation_errors' in tool_test_results_dict: - return ( latest_installable_changeset_revision, False ) - return ( latest_installable_changeset_revision, True ) - else: - # No test results. - return ( latest_installable_changeset_revision, False ) - else: - # No repository_metadata. - return ( latest_installable_changeset_revision, False ) - else: - # No installable changeset_revision. - return ( None, False ) - def new_datatypes_metadata_required( trans, repository_metadata, metadata_dict ): """ Compare the last saved metadata for each datatype in the repository with the new metadata in metadata_dict to determine if a new @@ -1746,7 +1689,7 @@ changeset_revisions_that_contain_tools = [] for changeset in repo.changelog: changeset_revision = str( repo.changectx( changeset ) ) - repository_metadata = suc.get_repository_metadata_by_changeset_revision( trans, id, changeset_revision ) + repository_metadata = suc.get_repository_metadata_by_changeset_revision( trans.app, id, changeset_revision ) if repository_metadata: metadata = repository_metadata.metadata if metadata: @@ -1756,7 +1699,7 @@ # If a repository includes tools, build a dictionary of { 'tool id' : 'parent tool id' } pairs for each tool in each changeset revision. for index, changeset_revision in enumerate( changeset_revisions_that_contain_tools ): tool_versions_dict = {} - repository_metadata = suc.get_repository_metadata_by_changeset_revision( trans, id, changeset_revision ) + repository_metadata = suc.get_repository_metadata_by_changeset_revision( trans.app, id, changeset_revision ) metadata = repository_metadata.metadata tool_dicts = metadata[ 'tools' ] if index == 0: @@ -2038,7 +1981,7 @@ changeset_revisions = [] for changeset in repo.changelog: changeset_revision = str( repo.changectx( changeset ) ) - if suc.get_repository_metadata_by_changeset_revision( trans, encoded_id, changeset_revision ): + if suc.get_repository_metadata_by_changeset_revision( trans.app, encoded_id, changeset_revision ): changeset_revisions.append( changeset_revision ) add_tool_versions( trans, encoded_id, repository_metadata, changeset_revisions ) elif len( repo ) == 1 and not invalid_file_tups: diff -r 172e1713535c24cc6c07caa5f90494b68827b638 -r b8e434a784960287fc998c9e0a7f11512dc0533c lib/tool_shed/util/readme_util.py --- a/lib/tool_shed/util/readme_util.py +++ b/lib/tool_shed/util/readme_util.py @@ -27,7 +27,7 @@ can_use_disk_files = True else: repo = hg.repository( hg_util.get_configured_ui(), repository.repo_path( trans.app ) ) - latest_downloadable_changeset_revision = suc.get_latest_downloadable_changeset_revision( trans, repository, repo ) + latest_downloadable_changeset_revision = suc.get_latest_downloadable_changeset_revision( trans.app, repository, repo ) can_use_disk_files = changeset_revision == latest_downloadable_changeset_revision readme_files_dict = {} if metadata: diff -r 172e1713535c24cc6c07caa5f90494b68827b638 -r b8e434a784960287fc998c9e0a7f11512dc0533c lib/tool_shed/util/shed_util_common.py --- a/lib/tool_shed/util/shed_util_common.py +++ b/lib/tool_shed/util/shed_util_common.py @@ -131,7 +131,7 @@ def changeset_is_malicious( trans, id, changeset_revision, **kwd ): """Check the malicious flag in repository metadata for a specified change set""" - repository_metadata = get_repository_metadata_by_changeset_revision( trans, id, changeset_revision ) + repository_metadata = get_repository_metadata_by_changeset_revision( trans.app, id, changeset_revision ) if repository_metadata: return repository_metadata.malicious return False @@ -488,14 +488,14 @@ def get_current_repository_metadata_for_changeset_revision( trans, repository, changeset_revision ): encoded_repository_id = trans.security.encode_id( repository.id ) - repository_metadata = get_repository_metadata_by_changeset_revision( trans, encoded_repository_id, changeset_revision ) + repository_metadata = get_repository_metadata_by_changeset_revision( trans.app, encoded_repository_id, changeset_revision ) if repository_metadata: return repository_metadata # The installable changeset_revision may have been changed because it was "moved ahead" in the repository changelog. repo = hg.repository( hg_util.get_configured_ui(), repository.repo_path( trans.app ) ) updated_changeset_revision = get_next_downloadable_changeset_revision( repository, repo, after_changeset_revision=changeset_revision ) if updated_changeset_revision: - repository_metadata = get_repository_metadata_by_changeset_revision( trans, encoded_repository_id, updated_changeset_revision ) + repository_metadata = get_repository_metadata_by_changeset_revision( trans.app, encoded_repository_id, updated_changeset_revision ) if repository_metadata: return repository_metadata return None @@ -610,7 +610,7 @@ def get_latest_changeset_revision( trans, repository, repo ): repository_tip = repository.tip( trans.app ) - repository_metadata = get_repository_metadata_by_changeset_revision( trans, trans.security.encode_id( repository.id ), repository_tip ) + repository_metadata = get_repository_metadata_by_changeset_revision( trans.app, trans.security.encode_id( repository.id ), repository_tip ) if repository_metadata and repository_metadata.downloadable: return repository_tip changeset_revisions = get_ordered_metadata_changeset_revisions( repository, repo, downloadable=False ) @@ -618,9 +618,9 @@ return changeset_revisions[ -1 ] return INITIAL_CHANGELOG_HASH -def get_latest_downloadable_changeset_revision( trans, repository, repo ): - repository_tip = repository.tip( trans.app ) - repository_metadata = get_repository_metadata_by_changeset_revision( trans, trans.security.encode_id( repository.id ), repository_tip ) +def get_latest_downloadable_changeset_revision( app, repository, repo ): + repository_tip = repository.tip( app ) + repository_metadata = get_repository_metadata_by_changeset_revision( app, app.security.encode_id( repository.id ), repository_tip ) if repository_metadata and repository_metadata.downloadable: return repository_tip changeset_revisions = get_ordered_metadata_changeset_revisions( repository, repo, downloadable=True ) @@ -1076,21 +1076,22 @@ """Get a repository on the tool shed side from the database via id.""" return trans.sa_session.query( trans.model.Repository ).get( trans.security.decode_id( id ) ) -def get_repository_metadata_by_changeset_revision( trans, id, changeset_revision ): +def get_repository_metadata_by_changeset_revision( app, id, changeset_revision ): """Get metadata for a specified repository change set from the database.""" # Make sure there are no duplicate records, and return the single unique record for the changeset_revision. # Duplicate records were somehow created in the past. The cause of this issue has been resolved, but we'll # leave this method as is for a while longer to ensure all duplicate records are removed. - all_metadata_records = trans.sa_session.query( trans.model.RepositoryMetadata ) \ - .filter( and_( trans.model.RepositoryMetadata.table.c.repository_id == trans.security.decode_id( id ), - trans.model.RepositoryMetadata.table.c.changeset_revision == changeset_revision ) ) \ - .order_by( trans.model.RepositoryMetadata.table.c.update_time.desc() ) \ - .all() + sa_session = app.model.context.current + all_metadata_records = sa_session.query( app.model.RepositoryMetadata ) \ + .filter( and_( app.model.RepositoryMetadata.table.c.repository_id == app.security.decode_id( id ), + app.model.RepositoryMetadata.table.c.changeset_revision == changeset_revision ) ) \ + .order_by( app.model.RepositoryMetadata.table.c.update_time.desc() ) \ + .all() if len( all_metadata_records ) > 1: # Delete all records older than the last one updated. for repository_metadata in all_metadata_records[ 1: ]: - trans.sa_session.delete( repository_metadata ) - trans.sa_session.flush() + sa_session.delete( repository_metadata ) + sa_session.flush() return all_metadata_records[ 0 ] elif all_metadata_records: return all_metadata_records[ 0 ] diff -r 172e1713535c24cc6c07caa5f90494b68827b638 -r b8e434a784960287fc998c9e0a7f11512dc0533c lib/tool_shed/util/tool_util.py --- a/lib/tool_shed/util/tool_util.py +++ b/lib/tool_shed/util/tool_util.py @@ -546,7 +546,7 @@ current_child_guid = guid for changeset in suc.reversed_upper_bounded_changelog( repo, repository_metadata.changeset_revision ): ctx = repo.changectx( changeset ) - rm = suc.get_repository_metadata_by_changeset_revision( trans, repository_id, str( ctx ) ) + rm = suc.get_repository_metadata_by_changeset_revision( trans.app, repository_id, str( ctx ) ) if rm: parent_guid = rm.tool_versions.get( current_child_guid, None ) if parent_guid: @@ -558,7 +558,7 @@ repository_metadata.changeset_revision, repository.tip( trans.app ) ): ctx = repo.changectx( changeset ) - rm = suc.get_repository_metadata_by_changeset_revision( trans, repository_id, str( ctx ) ) + rm = suc.get_repository_metadata_by_changeset_revision( trans.app, repository_id, str( ctx ) ) if rm: tool_versions = rm.tool_versions for child_guid, parent_guid in tool_versions.items(): Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: greg: Only load the tool panel if in Galaxy.
by commits-noreply＠bitbucket.org 20 May '14

20 May '14

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/172e1713535c/ Changeset: 172e1713535c User: greg Date: 2014-05-20 23:13:26 Summary: Only load the tool panel if in Galaxy. Affected #: 1 file diff -r c5e14473dfb79328f077d81a6cafd9c8a07ffe23 -r 172e1713535c24cc6c07caa5f90494b68827b638 lib/galaxy/tools/__init__.py --- a/lib/galaxy/tools/__init__.py +++ b/lib/galaxy/tools/__init__.py @@ -145,7 +145,7 @@ self.init_tools( config_filename ) except: log.exception( "Error loading tools defined in config %s", config_filename ) - if self.integrated_tool_panel_config_has_contents: + if self.app.name == 'galaxy' and self.integrated_tool_panel_config_has_contents: # Load self.tool_panel based on the order in self.integrated_tool_panel. self.load_tool_panel() if app.config.update_integrated_tool_panel: Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: jmchilton: Document install_database_connection in universe_wsgi.ini.
by commits-noreply＠bitbucket.org 20 May '14

20 May '14

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/c5e14473dfb7/ Changeset: c5e14473dfb7 User: jmchilton Date: 2014-05-20 23:07:43 Summary: Document install_database_connection in universe_wsgi.ini. Affected #: 1 file diff -r 622d4d3ed3bdb34edeca089a0775cdff86114798 -r c5e14473dfb79328f077d81a6cafd9c8a07ffe23 universe_wsgi.ini.sample --- a/universe_wsgi.ini.sample +++ b/universe_wsgi.ini.sample @@ -118,6 +118,14 @@ # 'galaxy.model.orm.logging_connection_proxy' #database_query_profiling_proxy = False +# By default, Galaxy will use the same database to track user data and +# tool shed install data. There are many situtations in which it is +# valuable to seperate these - for instance bootstrapping fresh Galaxy +# instances with pretested installs. The following optin can be used to +# separate the tool shed install database (all other options listed above +# but prefixed with install_ are also available). +#install_database_connection = sqlite:///./database/universe.sqlite?isolation_level=IMMEDIATE + # -- Files and directories # Path where genome builds are stored. This defaults to tool-data/genome Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: 3 new changesets
by commits-noreply＠bitbucket.org 20 May '14

20 May '14

3 new commits in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/ff42e7cdcd97/ Changeset: ff42e7cdcd97 Branch: next-stable User: jmchilton Date: 2014-05-20 22:31:14 Summary: Enable job metrics by default. They should work with all stock Galaxy runners now. Affected #: 1 file diff -r b814c23924470203ff3f37b28066a118e75bf348 -r ff42e7cdcd9716324c7b423135f24b44d327168c run.sh --- a/run.sh +++ b/run.sh @@ -19,6 +19,7 @@ data_manager_conf.xml.sample shed_data_manager_conf.xml.sample openid_conf.xml.sample + job_metrics_conf.xml.sample universe_wsgi.ini.sample tool-data/shared/ncbi/builds.txt.sample tool-data/shared/ensembl/builds.txt.sample https://bitbucket.org/galaxy/galaxy-central/commits/e49feb93943d/ Changeset: e49feb93943d Branch: next-stable User: jmchilton Date: 2014-05-20 22:31:14 Summary: Fix up job_conf.xml env element logging. Now works with all runners. Affected #: 1 file diff -r ff42e7cdcd9716324c7b423135f24b44d327168c -r e49feb93943da7c6a1ccb84d6dff4518742282a8 job_conf.xml.sample_advanced --- a/job_conf.xml.sample_advanced +++ b/job_conf.xml.sample_advanced @@ -104,12 +104,11 @@ </destination><destination id="remote_cluster" runner="drmaa" tags="longjobs"/><destination id="java_cluster" runner="drmaa"> - <env id="_JAVA_OPTIONS">-Xmx=6GB</env><env id="ANOTHER_OPTION" raw="true">'5'</env> https://bitbucket.org/galaxy/galaxy-central/commits/622d4d3ed3bd/ Changeset: 622d4d3ed3bd User: jmchilton Date: 2014-05-20 22:31:43 Summary: Merge next-stable. Affected #: 2 files diff -r 70400d62e299e041b18e934788fc34ad7bfc2ddb -r 622d4d3ed3bdb34edeca089a0775cdff86114798 job_conf.xml.sample_advanced --- a/job_conf.xml.sample_advanced +++ b/job_conf.xml.sample_advanced @@ -104,12 +104,11 @@ </destination><destination id="remote_cluster" runner="drmaa" tags="longjobs"/><destination id="java_cluster" runner="drmaa"> - <env id="_JAVA_OPTIONS">-Xmx=6GB</env><env id="ANOTHER_OPTION" raw="true">'5'</env> diff -r 70400d62e299e041b18e934788fc34ad7bfc2ddb -r 622d4d3ed3bdb34edeca089a0775cdff86114798 run.sh --- a/run.sh +++ b/run.sh @@ -19,6 +19,7 @@ data_manager_conf.xml.sample shed_data_manager_conf.xml.sample openid_conf.xml.sample + job_metrics_conf.xml.sample universe_wsgi.ini.sample tool-data/shared/ncbi/builds.txt.sample tool-data/shared/ensembl/builds.txt.sample Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: 2 new changesets
by commits-noreply＠bitbucket.org 20 May '14

20 May '14

2 new commits in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/b814c2392447/ Changeset: b814c2392447 Branch: next-stable User: dannon Date: 2014-05-20 18:06:07 Summary: Clarify default option (no quotes) for amqp_internal_connection Affected #: 1 file diff -r 8b66d43bc2d1e2119baa7589b6721e360840948e -r b814c23924470203ff3f37b28066a118e75bf348 universe_wsgi.ini.sample --- a/universe_wsgi.ini.sample +++ b/universe_wsgi.ini.sample @@ -831,7 +831,7 @@ # will automatically create and use a separate sqlite database located in your # <galaxy>/database folder (indicated in the commented out line below). -#amqp_internal_connection = "sqlite:///./database/control.sqlite?isolation_level=IMMEDIATE" +#amqp_internal_connection = sqlite:///./database/control.sqlite?isolation_level=IMMEDIATE https://bitbucket.org/galaxy/galaxy-central/commits/70400d62e299/ Changeset: 70400d62e299 User: dannon Date: 2014-05-20 18:06:21 Summary: Merge next-stable. Affected #: 1 file diff -r 512424a01563a2427fc0ff6225fde2737edc6900 -r 70400d62e299e041b18e934788fc34ad7bfc2ddb universe_wsgi.ini.sample --- a/universe_wsgi.ini.sample +++ b/universe_wsgi.ini.sample @@ -831,7 +831,7 @@ # will automatically create and use a separate sqlite database located in your # <galaxy>/database folder (indicated in the commented out line below). -#amqp_internal_connection = "sqlite:///./database/control.sqlite?isolation_level=IMMEDIATE" +#amqp_internal_connection = sqlite:///./database/control.sqlite?isolation_level=IMMEDIATE Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: 2 new changesets
by commits-noreply＠bitbucket.org 20 May '14

20 May '14

2 new commits in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/8b66d43bc2d1/ Changeset: 8b66d43bc2d1 Branch: next-stable User: dannon Date: 2014-05-20 17:55:24 Summary: When using AMQP, if the AMQP server goes down this will now more nicely catch the Connection refused error in send_control_task Affected #: 1 file diff -r 2ed7d87b4ddda2f32eb9d6fc8c05d74667842fef -r 8b66d43bc2d1e2119baa7589b6721e360840948e lib/galaxy/queue_worker.py --- a/lib/galaxy/queue_worker.py +++ b/lib/galaxy/queue_worker.py @@ -70,11 +70,16 @@ 'kwargs': kwargs} if noop_self: payload['noop'] = trans.app.config.server_name - c = Connection(trans.app.config.amqp_internal_connection) - with producers[c].acquire(block=True) as producer: - producer.publish(payload, exchange=galaxy.queues.galaxy_exchange, - declare=[galaxy.queues.galaxy_exchange] + galaxy.queues.all_control_queues_for_declare(trans.app.config), - routing_key='control') + try: + c = Connection(trans.app.config.amqp_internal_connection) + with producers[c].acquire(block=True) as producer: + producer.publish(payload, exchange=galaxy.queues.galaxy_exchange, + declare=[galaxy.queues.galaxy_exchange] + galaxy.queues.all_control_queues_for_declare(trans.app.config), + routing_key='control') + except Exception: + # This is likely connection refused. + # TODO Use the specific Exception above. + log.exception("Error sending control task: %s." % payload) # Tasks -- to be reorganized into a separate module as appropriate. This is https://bitbucket.org/galaxy/galaxy-central/commits/512424a01563/ Changeset: 512424a01563 User: dannon Date: 2014-05-20 17:55:54 Summary: Merge next-stable. Affected #: 1 file diff -r 37509e01ee5db4b0552537c77f3771b9671e5dae -r 512424a01563a2427fc0ff6225fde2737edc6900 lib/galaxy/queue_worker.py --- a/lib/galaxy/queue_worker.py +++ b/lib/galaxy/queue_worker.py @@ -70,11 +70,16 @@ 'kwargs': kwargs} if noop_self: payload['noop'] = trans.app.config.server_name - c = Connection(trans.app.config.amqp_internal_connection) - with producers[c].acquire(block=True) as producer: - producer.publish(payload, exchange=galaxy.queues.galaxy_exchange, - declare=[galaxy.queues.galaxy_exchange] + galaxy.queues.all_control_queues_for_declare(trans.app.config), - routing_key='control') + try: + c = Connection(trans.app.config.amqp_internal_connection) + with producers[c].acquire(block=True) as producer: + producer.publish(payload, exchange=galaxy.queues.galaxy_exchange, + declare=[galaxy.queues.galaxy_exchange] + galaxy.queues.all_control_queues_for_declare(trans.app.config), + routing_key='control') + except Exception: + # This is likely connection refused. + # TODO Use the specific Exception above. + log.exception("Error sending control task: %s." % payload) # Tasks -- to be reorganized into a separate module as appropriate. This is Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0