1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/d1dd573cdce2/ Changeset: d1dd573cdce2 User: greg Date: 2014-04-24 21:45:55 Summary: Move some hg utility functions out of shed_util_common and into hg_util. Affected #: 12 files diff -r 203623b931507dcd182e2521911ab036021faf61 -r d1dd573cdce2bd122cbb0e98e947dc7992fe3ed0 lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py --- a/lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py +++ b/lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py @@ -1855,7 +1855,7 @@ repo = hg.repository( hg_util.get_configured_ui(), path=repo_files_dir ) repository_clone_url = os.path.join( tool_shed_url, 'repos', owner, name ) repository_util.pull_repository( repo, repository_clone_url, latest_ctx_rev ) - suc.update_repository( repo, latest_ctx_rev ) + hg_util.update_repository( repo, latest_ctx_rev ) # Remove old Data Manager entries if repository.includes_data_managers: data_manager_util.remove_from_data_manager( trans.app, repository ) diff -r 203623b931507dcd182e2521911ab036021faf61 -r d1dd573cdce2bd122cbb0e98e947dc7992fe3ed0 lib/galaxy/webapps/tool_shed/controllers/hg.py --- a/lib/galaxy/webapps/tool_shed/controllers/hg.py +++ b/lib/galaxy/webapps/tool_shed/controllers/hg.py @@ -2,7 +2,7 @@ from galaxy import web from galaxy.web.base.controller import BaseUIController from tool_shed.util.shed_util_common import get_repository_by_name_and_owner -from tool_shed.util.shed_util_common import update_repository +from tool_shed.util.hg_util import update_repository from tool_shed.util.metadata_util import set_repository_metadata from galaxy import eggs diff -r 203623b931507dcd182e2521911ab036021faf61 -r d1dd573cdce2bd122cbb0e98e947dc7992fe3ed0 lib/galaxy/webapps/tool_shed/controllers/repository.py --- a/lib/galaxy/webapps/tool_shed/controllers/repository.py +++ b/lib/galaxy/webapps/tool_shed/controllers/repository.py @@ -722,7 +722,7 @@ repository = suc.get_repository_in_tool_shed( trans, id ) repo = hg.repository( hg_util.get_configured_ui(), repository.repo_path( trans.app ) ) # Update repository files for browsing. - suc.update_repository( repo ) + hg_util.update_repository( repo ) changeset_revision = repository.tip( trans.app ) metadata = metadata_util.get_repository_metadata_by_repository_id_changeset_revision( trans, id, changeset_revision, metadata_only=True ) repository_type_select_field = rt_util.build_repository_type_select_field( trans, repository=repository ) @@ -2743,7 +2743,7 @@ commands.commit( repo.ui, repo, repo_dir, user=trans.user.username, message=commit_message ) suc.handle_email_alerts( trans, repository ) # Update the repository files for browsing. - suc.update_repository( repo ) + hg_util.update_repository( repo ) # Get the new repository tip. repo = hg.repository( hg_util.get_configured_ui(), repo_dir ) if tip == repository.tip( trans.app ): @@ -3117,7 +3117,7 @@ else: ctx_child = None diffs = [] - options_dict = suc.get_mercurial_default_options_dict( 'diff' ) + options_dict = hg_util.get_mercurial_default_options_dict( 'diff' ) # Not quite sure if the following settings make any difference, but with a combination of them and the size check on each # diff, we don't run out of memory when viewing the changelog of the cisortho2 repository on the test tool shed. options_dict[ 'maxfile' ] = suc.MAXDIFFSIZE @@ -3252,7 +3252,11 @@ else: message += malicious_error status = 'error' - containers_dict = container_util.build_repository_containers_for_tool_shed( trans, repository, changeset_revision, repository_dependencies, repository_metadata ) + containers_dict = container_util.build_repository_containers_for_tool_shed( trans, + repository, + changeset_revision, + repository_dependencies, + repository_metadata ) repository_type_select_field = rt_util.build_repository_type_select_field( trans, repository=repository ) heads = suc.get_repository_heads( repo ) return trans.fill_template( '/webapps/tool_shed/repository/view_repository.mako', diff -r 203623b931507dcd182e2521911ab036021faf61 -r d1dd573cdce2bd122cbb0e98e947dc7992fe3ed0 lib/galaxy/webapps/tool_shed/controllers/upload.py --- a/lib/galaxy/webapps/tool_shed/controllers/upload.py +++ b/lib/galaxy/webapps/tool_shed/controllers/upload.py @@ -226,7 +226,7 @@ admin_only=admin_only ) if ok: # Update the repository files for browsing. - suc.update_repository( repo ) + hg_util.update_repository( repo ) # Get the new repository tip. if tip == repository.tip( trans.app ): message = 'No changes to repository. ' diff -r 203623b931507dcd182e2521911ab036021faf61 -r d1dd573cdce2bd122cbb0e98e947dc7992fe3ed0 lib/tool_shed/galaxy_install/install_manager.py --- a/lib/tool_shed/galaxy_install/install_manager.py +++ b/lib/tool_shed/galaxy_install/install_manager.py @@ -2,6 +2,7 @@ Manage automatic installation of tools configured in the xxx.xml files in ~/scripts/migrate_tools (e.g., 0002_tools.xml). All of the tools were at some point included in the Galaxy distribution, but are now hosted in the main Galaxy tool shed. """ +import json import os import shutil import tempfile @@ -9,12 +10,11 @@ import logging from galaxy import util from galaxy.tools import ToolSection -from galaxy.util.json import from_json_string -from galaxy.util.json import to_json_string import tool_shed.util.shed_util_common as suc from tool_shed.util import common_install_util from tool_shed.util import common_util from tool_shed.util import datatype_util +from tool_shed.util import hg_util from tool_shed.util import metadata_util from tool_shed.util import tool_dependency_util from tool_shed.util import tool_util @@ -505,8 +505,10 @@ tool_shed_repository.owner, tool_shed_repository.installed_changeset_revision ) if not cloned_ok: - suc.update_tool_shed_repository_status( self.app, tool_shed_repository, self.app.install_model.ToolShedRepository.installation_status.CLONING ) - cloned_ok, error_message = suc.clone_repository( repository_clone_url, os.path.abspath( install_dir ), ctx_rev ) + suc.update_tool_shed_repository_status( self.app, + tool_shed_repository, + self.app.install_model.ToolShedRepository.installation_status.CLONING ) + cloned_ok, error_message = hg_util.clone_repository( repository_clone_url, os.path.abspath( install_dir ), ctx_rev ) if cloned_ok and not is_installed: self.handle_repository_contents( tool_shed_repository=tool_shed_repository, repository_clone_url=repository_clone_url, @@ -525,7 +527,7 @@ ( self.tool_shed_url, tool_shed_repository.name, self.repository_owner, tool_shed_repository.installed_changeset_revision ) text = common_util.tool_shed_get( self.app, self.tool_shed_url, url ) if text: - tool_version_dicts = from_json_string( text ) + tool_version_dicts = json.loads( text ) tool_util.handle_tool_versions( self.app, tool_version_dicts, tool_shed_repository ) else: # Set the tool versions since they seem to be missing for this repository in the tool shed. diff -r 203623b931507dcd182e2521911ab036021faf61 -r d1dd573cdce2bd122cbb0e98e947dc7992fe3ed0 lib/tool_shed/galaxy_install/repository_util.py --- a/lib/tool_shed/galaxy_install/repository_util.py +++ b/lib/tool_shed/galaxy_install/repository_util.py @@ -560,14 +560,17 @@ if isinstance( repo_info_dict, basestring ): repo_info_dict = encoding_util.tool_shed_decode( repo_info_dict ) # Clone each repository to the configured location. - suc.update_tool_shed_repository_status( trans.app, tool_shed_repository, trans.install_model.ToolShedRepository.installation_status.CLONING ) + suc.update_tool_shed_repository_status( trans.app, + tool_shed_repository, + trans.install_model.ToolShedRepository.installation_status.CLONING ) repo_info_tuple = repo_info_dict[ tool_shed_repository.name ] description, repository_clone_url, changeset_revision, ctx_rev, repository_owner, repository_dependencies, tool_dependencies = repo_info_tuple - relative_clone_dir = suc.generate_tool_shed_repository_install_dir( repository_clone_url, tool_shed_repository.installed_changeset_revision ) + relative_clone_dir = suc.generate_tool_shed_repository_install_dir( repository_clone_url, + tool_shed_repository.installed_changeset_revision ) clone_dir = os.path.join( tool_path, relative_clone_dir ) relative_install_dir = os.path.join( relative_clone_dir, tool_shed_repository.name ) install_dir = os.path.join( tool_path, relative_install_dir ) - cloned_ok, error_message = suc.clone_repository( repository_clone_url, os.path.abspath( install_dir ), ctx_rev ) + cloned_ok, error_message = hg_util.clone_repository( repository_clone_url, os.path.abspath( install_dir ), ctx_rev ) if cloned_ok: if reinstalling: # Since we're reinstalling the repository we need to find the latest changeset revision to which it can be updated. @@ -577,7 +580,7 @@ if current_ctx_rev != ctx_rev: repo = hg.repository( hg_util.get_configured_ui(), path=os.path.abspath( install_dir ) ) pull_repository( repo, repository_clone_url, current_changeset_revision ) - suc.update_repository( repo, ctx_rev=current_ctx_rev ) + hg_util.update_repository( repo, ctx_rev=current_ctx_rev ) handle_repository_contents( trans, tool_shed_repository=tool_shed_repository, tool_path=tool_path, diff -r 203623b931507dcd182e2521911ab036021faf61 -r d1dd573cdce2bd122cbb0e98e947dc7992fe3ed0 lib/tool_shed/scripts/check_repositories_for_functional_tests.py --- a/lib/tool_shed/scripts/check_repositories_for_functional_tests.py +++ b/lib/tool_shed/scripts/check_repositories_for_functional_tests.py @@ -112,7 +112,7 @@ repo_dir = repository.repo_path( app ) repo = hg.repository( hg_util.get_configured_ui(), repo_dir ) work_dir = tempfile.mkdtemp( prefix="tmp-toolshed-cafr" ) - cloned_ok, error_message = suc.clone_repository( repo_dir, work_dir, changeset_revision ) + cloned_ok, error_message = hg_util.clone_repository( repo_dir, work_dir, changeset_revision ) if cloned_ok: # Iterate through all the directories in the cloned changeset revision and determine whether there's a # directory named test-data. If this directory is not present update the metadata record for the changeset diff -r 203623b931507dcd182e2521911ab036021faf61 -r d1dd573cdce2bd122cbb0e98e947dc7992fe3ed0 lib/tool_shed/util/export_util.py --- a/lib/tool_shed/util/export_util.py +++ b/lib/tool_shed/util/export_util.py @@ -39,7 +39,7 @@ def archive_repository_revision( trans, ui, repository, archive_dir, changeset_revision ): '''Create an un-versioned archive of a repository.''' repo = hg.repository( hg_util.get_configured_ui(), repository.repo_path( trans.app ) ) - options_dict = suc.get_mercurial_default_options_dict( 'archive' ) + options_dict = hg_util.get_mercurial_default_options_dict( 'archive' ) options_dict[ 'rev' ] = changeset_revision error_message = '' return_code = None diff -r 203623b931507dcd182e2521911ab036021faf61 -r d1dd573cdce2bd122cbb0e98e947dc7992fe3ed0 lib/tool_shed/util/hg_util.py --- a/lib/tool_shed/util/hg_util.py +++ b/lib/tool_shed/util/hg_util.py @@ -3,6 +3,7 @@ from time import gmtime from time import strftime +from galaxy.util import listify from galaxy import eggs eggs.require( 'mercurial' ) @@ -11,6 +12,26 @@ from mercurial import hg from mercurial import ui +log = logging.getLogger( __name__ ) + +def clone_repository( repository_clone_url, repository_file_dir, ctx_rev ): + """ + Clone the repository up to the specified changeset_revision. No subsequent revisions will be + present in the cloned repository. + """ + try: + commands.clone( get_configured_ui(), + str( repository_clone_url ), + dest=str( repository_file_dir ), + pull=True, + noupdate=False, + rev=listify( str( ctx_rev ) ) ) + return True, None + except Exception, e: + error_message = 'Error cloning repository: %s' % str( e ) + log.debug( error_message ) + return False, error_message + def get_changectx_for_changeset( repo, changeset_revision, **kwd ): """Retrieve a specified changectx from a repository.""" for changeset in repo.changelog: @@ -28,7 +49,19 @@ # quiet = True _ui.setconfig( 'ui', 'quiet', True ) return _ui - + +def get_mercurial_default_options_dict( command, command_table=None, **kwd ): + '''Borrowed from repoman - get default parameters for a mercurial command.''' + if command_table is None: + command_table = commands.table + possible = cmdutil.findpossible( command, command_table ) + if len( possible ) != 1: + raise Exception, 'unable to find mercurial command "%s"' % command + default_options_dict = dict( ( r[ 1 ].replace( '-', '_' ), r[ 2 ] ) for r in possible[ possible.keys()[ 0 ] ][ 1 ][ 1 ] ) + for option in kwd: + default_options_dict[ option ] = kwd[ option ] + return default_options_dict + def get_readable_ctx_date( ctx ): """Convert the date of the changeset (the received ctx) to a human-readable date.""" t, tz = ctx.date() @@ -106,3 +139,21 @@ rev = '-1' label = "-1:%s" % changeset_revision return rev, label + +def update_repository( repo, ctx_rev=None ): + """ + Update the cloned repository to changeset_revision. It is critical that the installed repository is updated to the desired + changeset_revision before metadata is set because the process for setting metadata uses the repository files on disk. + """ + # TODO: We may have files on disk in the repo directory that aren't being tracked, so they must be removed. + # The codes used to show the status of files are as follows. + # M = modified + # A = added + # R = removed + # C = clean + # ! = deleted, but still tracked + # ? = not tracked + # I = ignored + # It would be nice if we could use mercurial's purge extension to remove untracked files. The problem is that + # purging is not supported by the mercurial API. + commands.update( get_configured_ui(), repo, rev=ctx_rev ) diff -r 203623b931507dcd182e2521911ab036021faf61 -r d1dd573cdce2bd122cbb0e98e947dc7992fe3ed0 lib/tool_shed/util/metadata_util.py --- a/lib/tool_shed/util/metadata_util.py +++ b/lib/tool_shed/util/metadata_util.py @@ -1799,7 +1799,7 @@ current_changeset_revision = str( repo.changectx( changeset ) ) ctx = repo.changectx( changeset ) log.debug( "Cloning repository changeset revision: %s", str( ctx.rev() ) ) - cloned_ok, error_message = suc.clone_repository( repository_clone_url, work_dir, str( ctx.rev() ) ) + cloned_ok, error_message = hg_util.clone_repository( repository_clone_url, work_dir, str( ctx.rev() ) ) if cloned_ok: log.debug( "Generating metadata for changset revision: %s", str( ctx.rev() ) ) current_metadata_dict, invalid_tups = generate_metadata_for_changeset_revision( app=trans.app, @@ -1811,7 +1811,8 @@ resetting_all_metadata_on_repository=True, updating_installed_repository=False, persist=False ) - # We'll only display error messages for the repository tip (it may be better to display error messages for each installable changeset revision). + # We'll only display error messages for the repository tip (it may be better to display error + # messages for each installable changeset revision). if current_changeset_revision == repository.tip( trans.app ): invalid_file_tups.extend( invalid_tups ) if current_metadata_dict: diff -r 203623b931507dcd182e2521911ab036021faf61 -r d1dd573cdce2bd122cbb0e98e947dc7992fe3ed0 lib/tool_shed/util/shed_util_common.py --- a/lib/tool_shed/util/shed_util_common.py +++ b/lib/tool_shed/util/shed_util_common.py @@ -25,10 +25,7 @@ from galaxy import eggs eggs.require( 'mercurial' ) -from mercurial import cmdutil -from mercurial import commands from mercurial import hg -from mercurial import ui eggs.require( 'markupsafe' ) import markupsafe @@ -139,15 +136,6 @@ return repository_metadata.malicious return False -def changeset_is_valid( app, repository, changeset_revision ): - """Make sure a changeset hash is valid for a specified repository.""" - repo = hg.repository( hg_util.get_configured_ui(), repository.repo_path( app ) ) - for changeset in repo.changelog: - changeset_hash = str( repo.changectx( changeset ) ) - if changeset_revision == changeset_hash: - return True - return False - def check_or_update_tool_shed_status_for_installed_repository( trans, repository ): updated = False tool_shed_status_dict = get_tool_shed_status_for_installed_repository( trans.app, repository ) @@ -162,21 +150,6 @@ ok = False return ok, updated -def clone_repository( repository_clone_url, repository_file_dir, ctx_rev ): - """Clone the repository up to the specified changeset_revision. No subsequent revisions will be present in the cloned repository.""" - try: - commands.clone( hg_util.get_configured_ui(), - str( repository_clone_url ), - dest=str( repository_file_dir ), - pull=True, - noupdate=False, - rev=util.listify( str( ctx_rev ) ) ) - return True, None - except Exception, e: - error_message = 'Error cloning repository: %s' % str( e ) - log.debug( error_message ) - return False, error_message - def config_elems_to_xml_file( app, config_elems, config_filename, tool_path ): """Persist the current in-memory list of config_elems to a file named by the value of config_filename.""" fd, filename = tempfile.mkstemp( prefix="tmp-toolshed-cetxf" ) @@ -655,18 +628,6 @@ return changeset_revisions[ -1 ] return INITIAL_CHANGELOG_HASH -def get_mercurial_default_options_dict( command, command_table=None, **kwd ): - '''Borrowed from repoman - get default parameters for a mercurial command.''' - if command_table is None: - command_table = commands.table - possible = cmdutil.findpossible( command, command_table ) - if len( possible ) != 1: - raise Exception, 'unable to find mercurial command "%s"' % command - default_options_dict = dict( ( r[ 1 ].replace( '-', '_' ), r[ 2 ] ) for r in possible[ possible.keys()[ 0 ] ][ 1 ][ 1 ] ) - for option in kwd: - default_options_dict[ option ] = kwd[ option ] - return default_options_dict - def get_named_tmpfile_from_ctx( ctx, filename, dir ): """Return a named temporary file created from a specified file with a given name included in a repository changeset revision.""" filename = strip_path( filename ) @@ -1784,24 +1745,6 @@ config_elems.append( elem ) config_elems_to_xml_file( app, config_elems, shed_tool_conf, tool_path ) -def update_repository( repo, ctx_rev=None ): - """ - Update the cloned repository to changeset_revision. It is critical that the installed repository is updated to the desired - changeset_revision before metadata is set because the process for setting metadata uses the repository files on disk. - """ - # TODO: We may have files on disk in the repo directory that aren't being tracked, so they must be removed. - # The codes used to show the status of files are as follows. - # M = modified - # A = added - # R = removed - # C = clean - # ! = deleted, but still tracked - # ? = not tracked - # I = ignored - # It would be nice if we could use mercurial's purge extension to remove untracked files. The problem is that - # purging is not supported by the mercurial API. - commands.update( hg_util.get_configured_ui(), repo, rev=ctx_rev ) - def update_tool_shed_repository_status( app, tool_shed_repository, status, error_message=None ): """Update the status of a tool shed repository in the process of being installed into Galaxy.""" context = app.install_model.context diff -r 203623b931507dcd182e2521911ab036021faf61 -r d1dd573cdce2bd122cbb0e98e947dc7992fe3ed0 test/tool_shed/base/twilltestcase.py --- a/test/tool_shed/base/twilltestcase.py +++ b/test/tool_shed/base/twilltestcase.py @@ -19,6 +19,7 @@ from galaxy.web import security from tool_shed.util.encoding_util import tool_shed_encode from tool_shed.util import shed_util_common as suc +from tool_shed.util import hg_util from tool_shed.util import xml_util from galaxy import eggs @@ -260,7 +261,7 @@ def clone_repository( self, repository, destination_path ): url = '%s/repos/%s/%s' % ( self.url, repository.user.username, repository.name ) - success, message = suc.clone_repository( url, destination_path, self.get_repository_tip( repository ) ) + success, message = hg_util.clone_repository( url, destination_path, self.get_repository_tip( repository ) ) assert success is True, message def commit_and_push( self, repository, hgrepo, options, username, password ): Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.