commit/galaxy-central: greg: Add a dependency manager with tag attribute handler classes to manage the process of populating and unpopulating the toolshed and changeset_revision attributes for <repository> tag sets when uploading and importing / exporting repositories from the tool shed.
1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/63919647ca84/ Changeset: 63919647ca84 User: greg Date: 2014-06-21 20:15:20 Summary: Add a dependency manager with tag attribute handler classes to manage the process of populating and unpopulating the toolshed and changeset_revision attributes for <repository> tag sets when uploading and importing / exporting repositories from the tool shed. Affected #: 9 files diff -r e832e466471403d54d788193c938f3c4d139142c -r 63919647ca8424a28042ccb79ea2db6e0c68d507 lib/galaxy/webapps/tool_shed/controllers/upload.py --- a/lib/galaxy/webapps/tool_shed/controllers/upload.py +++ b/lib/galaxy/webapps/tool_shed/controllers/upload.py @@ -8,13 +8,14 @@ from galaxy import util from galaxy import web from galaxy.datatypes import checkers +from tool_shed.dependencies import dependency_manager import tool_shed.repository_types.util as rt_util -import tool_shed.util.shed_util_common as suc from tool_shed.util import basic_util from tool_shed.util import commit_util from tool_shed.util import hg_util from tool_shed.util import metadata_util from tool_shed.util import repository_dependency_util +from tool_shed.util import shed_util_common as suc from tool_shed.util import tool_dependency_util from tool_shed.util import tool_util from tool_shed.util import xml_util @@ -95,6 +96,8 @@ uploaded_file_filename = os.path.split( file_data.filename )[ -1 ] isempty = os.path.getsize( os.path.abspath( uploaded_file_name ) ) == 0 if uploaded_file or uploaded_directory: + rdah = dependency_manager.RepositoryDependencyAttributeHandler( trans.app, unpopulate=False ) + tdah = dependency_manager.ToolDependencyAttributeHandler( trans.app, unpopulate=False ) ok = True isgzip = False isbz2 = False @@ -124,6 +127,8 @@ if istar: ok, message, files_to_remove, content_alert_str, undesirable_dirs_removed, undesirable_files_removed = \ self.upload_tar( trans, + rdah, + tdah, repository, tar, uploaded_file, @@ -134,6 +139,8 @@ elif uploaded_directory: ok, message, files_to_remove, content_alert_str, undesirable_dirs_removed, undesirable_files_removed = \ self.upload_directory( trans, + rdah, + tdah, repository, uploaded_directory, upload_point, @@ -164,12 +171,9 @@ full_path = os.path.abspath( os.path.join( repo_dir, uploaded_file_filename ) ) # Move some version of the uploaded file to the load_point within the repository hierarchy. if uploaded_file_filename in [ rt_util.REPOSITORY_DEPENDENCY_DEFINITION_FILENAME ]: - # Inspect the contents of the file to see if changeset_revision values are missing and if so, - # set them appropriately. - altered, root_elem, error_message = \ - commit_util.handle_repository_dependencies_definition( trans.app, - uploaded_file_name, - unpopulate=False ) + # Inspect the contents of the file to see if toolshed or changeset_revision attributes + # are missing and if so, set them appropriately. + altered, root_elem, error_message = rdah.handle_tag_attributes( uploaded_file_name ) if error_message: ok = False message = error_message @@ -179,12 +183,10 @@ shutil.move( tmp_filename, full_path ) else: shutil.move( uploaded_file_name, full_path ) - elif uploaded_file_filename in [ rt_util.REPOSITORY_DEPENDENCY_DEFINITION_FILENAME, - rt_util.TOOL_DEPENDENCY_DEFINITION_FILENAME ]: + elif uploaded_file_filename in [ rt_util.TOOL_DEPENDENCY_DEFINITION_FILENAME ]: # Inspect the contents of the file to see if changeset_revision values are # missing and if so, set them appropriately. - altered, root_elem, error_message = \ - commit_util.handle_tool_dependencies_definition( trans.app, uploaded_file_name ) + altered, root_elem, error_message = tdah.handle_tag_attributes( uploaded_file_name ) if error_message: ok = False message = error_message @@ -333,7 +335,7 @@ message=message, status=status ) - def upload_directory( self, trans, repository, uploaded_directory, upload_point, remove_repo_files_not_in_tar, + def upload_directory( self, trans, rdah, tdah, repository, uploaded_directory, upload_point, remove_repo_files_not_in_tar, commit_message, new_repo_alert ): repo_dir = repository.repo_path( trans.app ) repo = hg_util.get_repo_for_repository( trans.app, repository=None, repo_path=repo_dir, create=False ) @@ -364,20 +366,18 @@ if ok: uploaded_file_name = os.path.abspath( os.path.join( root, uploaded_file ) ) if os.path.split( uploaded_file_name )[ -1 ] == rt_util.REPOSITORY_DEPENDENCY_DEFINITION_FILENAME: - # Inspect the contents of the file to see if changeset_revision values are missing and - # if so, set them appropriately. - altered, root_elem, error_message = \ - commit_util.handle_repository_dependencies_definition( trans.app, - uploaded_file_name, - unpopulate=False ) + # Inspect the contents of the file to see if toolshed or changeset_revision + # attributes are missing and if so, set them appropriately. + altered, root_elem, error_message = rdah.handle_tag_attributes( uploaded_file_name ) if error_message: return False, error_message, [], '', [], [] elif altered: tmp_filename = xml_util.create_and_write_tmp_file( root_elem ) shutil.move( tmp_filename, uploaded_file_name ) elif os.path.split( uploaded_file_name )[ -1 ] == rt_util.TOOL_DEPENDENCY_DEFINITION_FILENAME: - # Inspect the contents of the file to see if changeset_revision values are missing and if so, set them appropriately. - altered, root_elem, error_message = commit_util.handle_tool_dependencies_definition( trans.app, uploaded_file_name ) + # Inspect the contents of the file to see if toolshed or changeset_revision + # attributes are missing and if so, set them appropriately. + altered, root_elem, error_message = tdah.handle_tag_attributes( uploaded_file_name ) if error_message: return False, error_message, [], '', [], [] if altered: @@ -406,7 +406,8 @@ undesirable_dirs_removed, undesirable_files_removed ) - def upload_tar( self, trans, repository, tar, uploaded_file, upload_point, remove_repo_files_not_in_tar, commit_message, new_repo_alert ): + def upload_tar( self, trans, rdah, tdah, repository, tar, uploaded_file, upload_point, remove_repo_files_not_in_tar, + commit_message, new_repo_alert ): # Upload a tar archive of files. repo_dir = repository.repo_path( trans.app ) repo = hg_util.get_repo_for_repository( trans.app, repository=None, repo_path=repo_dir, create=False ) @@ -442,18 +443,18 @@ for filename in filenames_in_archive: uploaded_file_name = os.path.join( full_path, filename ) if os.path.split( uploaded_file_name )[ -1 ] == rt_util.REPOSITORY_DEPENDENCY_DEFINITION_FILENAME: - # Inspect the contents of the file to see if changeset_revision values are missing and if so, set them appropriately. - altered, root_elem, error_message = commit_util.handle_repository_dependencies_definition( trans.app, - uploaded_file_name, - unpopulate=False ) + # Inspect the contents of the file to see if toolshed or changeset_revision attributes + # are missing and if so, set them appropriately. + altered, root_elem, error_message = rdah.handle_tag_attributes( uploaded_file_name ) if error_message: return False, error_message, [], '', [], [] elif altered: tmp_filename = xml_util.create_and_write_tmp_file( root_elem ) shutil.move( tmp_filename, uploaded_file_name ) elif os.path.split( uploaded_file_name )[ -1 ] == rt_util.TOOL_DEPENDENCY_DEFINITION_FILENAME: - # Inspect the contents of the file to see if changeset_revision values are missing and if so, set them appropriately. - altered, root_elem, error_message = commit_util.handle_tool_dependencies_definition( trans.app, uploaded_file_name ) + # Inspect the contents of the file to see if toolshed or changeset_revision + # attributes are missing and if so, set them appropriately. + altered, root_elem, error_message = tdah.handle_tag_attributes( uploaded_file_name ) if error_message: return False, error_message, [], '', [], [] if altered: diff -r e832e466471403d54d788193c938f3c4d139142c -r 63919647ca8424a28042ccb79ea2db6e0c68d507 lib/tool_shed/dependencies/dependency_manager.py --- /dev/null +++ b/lib/tool_shed/dependencies/dependency_manager.py @@ -0,0 +1,202 @@ +import copy +import logging + +from galaxy.util import asbool +from galaxy.util.odict import odict +from galaxy.web import url_for + +from tool_shed.dependencies import tag_attribute_handler +from tool_shed.repository_types.util import REPOSITORY_DEPENDENCY_DEFINITION_FILENAME +from tool_shed.repository_types.util import TOOL_DEPENDENCY_DEFINITION_FILENAME +from tool_shed.util import hg_util +from tool_shed.util import shed_util_common as suc +from tool_shed.util import xml_util +log = logging.getLogger( __name__ ) + + +class RepositoryDependencyAttributeHandler( object ): + + def __init__( self, app, unpopulate ): + self.app = app + self.file_name = REPOSITORY_DEPENDENCY_DEFINITION_FILENAME + self.unpopulate = unpopulate + + def check_tag_attributes( self, elem ): + # <repository name="molecule_datatypes" owner="test" /> + error_message = '' + name = elem.get( 'name' ) + if not name: + error_message += 'The tag is missing the required name attribute. ' + owner = elem.get( 'owner' ) + if not owner: + error_message += 'The tag is missing the required owner attribute. ' + log.debug( error_message ) + return error_message + + def handle_complex_dependency_elem( self, parent_elem, elem_index, elem ): + """ + Populate or unpopulate the toolshed and changeset_revision attributes of a + <repository> tag that defines a complex repository dependency. + """ + # <repository name="package_eigen_2_0" owner="test" prior_installation_required="True" /> + altered, new_elem, error_message = self.handle_elem( elem ) + if error_message: + error_message += ' The %s file contains an invalid <repository> tag.' % TOOL_DEPENDENCY_DEFINITION_FILENAME + return altered, new_elem, error_message + + def handle_elem( self, elem ): + """Populate or unpopulate the changeset_revision and toolshed attributes of repository tags.""" + # <repository name="molecule_datatypes" owner="test" changeset_revision="1a070566e9c6" /> + # <repository changeset_revision="xxx" name="package_xorg_macros_1_17_1" owner="test" toolshed="yyy"> + # <package name="xorg_macros" version="1.17.1" /> + # </repository> + error_message = '' + name = elem.get( 'name' ) + owner = elem.get( 'owner' ) + # The name and owner attributes are always required, so if either are missing, return the error message. + if not name or not owner: + error_message = self.check_tag_attributes( elem ) + return False, elem, error_message + altered = False + toolshed = elem.get( 'toolshed' ) + changeset_revision = elem.get( 'changeset_revision' ) + # Over a short period of time a bug existed which caused the prior_installation_required attribute + # to be set to False and included in the <repository> tag when a repository was exported along with + # its dependencies. The following will eliminate this problematic attribute upon import. + prior_installation_required = elem.get( 'prior_installation_required' ) + if prior_installation_required is not None and not asbool( prior_installation_required ): + del elem.attrib[ 'prior_installation_required' ] + sub_elems = [ child_elem for child_elem in list( elem ) ] + if len( sub_elems ) > 0: + # At this point, a <repository> tag will point only to a package. + # <package name="xorg_macros" version="1.17.1" /> + # Coerce the list to an odict(). + sub_elements = odict() + packages = [] + for sub_elem in sub_elems: + sub_elem_type = sub_elem.tag + sub_elem_name = sub_elem.get( 'name' ) + sub_elem_version = sub_elem.get( 'version' ) + if sub_elem_type and sub_elem_name and sub_elem_version: + packages.append( ( sub_elem_name, sub_elem_version ) ) + sub_elements[ 'packages' ] = packages + else: + # Set to None. + sub_elements = None + if self.unpopulate: + # We're exporting the repository, so eliminate all toolshed and changeset_revision attributes + # from the <repository> tag. + if toolshed or changeset_revision: + attributes = odict() + attributes[ 'name' ] = name + attributes[ 'owner' ] = owner + prior_installation_required = elem.get( 'prior_installation_required' ) + if asbool( prior_installation_required ): + attributes[ 'prior_installation_required' ] = 'True' + new_elem = xml_util.create_element( 'repository', attributes=attributes, sub_elements=sub_elements ) + altered = True + return altered, new_elem, error_message + # From here on we're populating the toolshed and changeset_revision attributes if necessary. + if not toolshed: + # Default the setting to the current tool shed. + toolshed = str( url_for( '/', qualified=True ) ).rstrip( '/' ) + elem.attrib[ 'toolshed' ] = toolshed + altered = True + if not changeset_revision: + # Populate the changeset_revision attribute with the latest installable metadata revision for + # the defined repository. We use the latest installable revision instead of the latest metadata + # revision to ensure that the contents of the revision are valid. + repository = suc.get_repository_by_name_and_owner( self.app, name, owner ) + if repository: + repo = hg_util.get_repo_for_repository( self.app, + repository=repository, + repo_path=None, + create=False ) + lastest_installable_changeset_revision = \ + suc.get_latest_downloadable_changeset_revision( self.app, repository, repo ) + if lastest_installable_changeset_revision != hg_util.INITIAL_CHANGELOG_HASH: + elem.attrib[ 'changeset_revision' ] = lastest_installable_changeset_revision + altered = True + else: + error_message = 'Invalid latest installable changeset_revision %s ' % \ + str( lastest_installable_changeset_revision ) + error_message += 'retrieved for repository %s owned by %s. ' % ( str( name ), str( owner ) ) + else: + error_message = 'Unable to locate repository with name %s and owner %s. ' % ( str( name ), str( owner ) ) + return altered, elem, error_message + + def handle_sub_elem( self, parent_elem, elem_index, elem ): + """ + Populate or unpopulate the toolshed and changeset_revision attributes for each of + the following tag sets. + <action type="set_environment_for_install"> + <action type="setup_r_environment"> + <action type="setup_ruby_environment"> + """ + sub_elem_altered = False + error_message = '' + for sub_index, sub_elem in enumerate( elem ): + # Make sure to skip comments and tags that are not <repository>. + if sub_elem.tag == 'repository': + altered, new_sub_elem, message = self.handle_elem( sub_elem ) + if message: + error_message += 'The %s file contains an invalid <repository> tag. %s' % \ + ( TOOL_DEPENDENCY_DEFINITION_FILENAME, message ) + if altered: + if not sub_elem_altered: + sub_elem_altered = True + elem[ sub_index ] = new_sub_elem + if sub_elem_altered: + parent_elem[ elem_index ] = elem + return sub_elem_altered, parent_elem, error_message + + def handle_tag_attributes( self, config ): + """ + Populate or unpopulate the toolshed and changeset_revision attributes of a + <repository> tag. Populating will occur when a dependency definition file + is being uploaded to the repository, while unpopulating will occur when the + repository is being exported. + """ + # Make sure we're looking at a valid repository_dependencies.xml file. + tree, error_message = xml_util.parse_xml( config ) + if tree is None: + return False, None, error_message + root = tree.getroot() + root_altered = False + new_root = copy.deepcopy( root ) + for index, elem in enumerate( root ): + if elem.tag == 'repository': + # <repository name="molecule_datatypes" owner="test" changeset_revision="1a070566e9c6" /> + altered, new_elem, error_message = self.handle_elem( elem ) + if error_message: + error_message = 'The %s file contains an invalid <repository> tag. %s' % ( self.file_name, error_message ) + return False, None, error_message + if altered: + if not root_altered: + root_altered = True + new_root[ index ] = new_elem + return root_altered, new_root, error_message + +class ToolDependencyAttributeHandler( object ): + + def __init__( self, app, unpopulate ): + self.app = app + self.file_name = TOOL_DEPENDENCY_DEFINITION_FILENAME + self.unpopulate = unpopulate + + def handle_tag_attributes( self, tool_dependencies_config ): + """ + Populate or unpopulate the tooshed and changeset_revision attributes of each <repository> + tag defined within a tool_dependencies.xml file. + """ + rdah = RepositoryDependencyAttributeHandler( self.app, self.unpopulate ) + tah = tag_attribute_handler.TagAttributeHandler( self.app, rdah, self.unpopulate ) + altered = False + error_message = '' + # Make sure we're looking at a valid tool_dependencies.xml file. + tree, error_message = xml_util.parse_xml( tool_dependencies_config ) + if tree is None: + return False, None, error_message + root = tree.getroot() + altered, new_root, error_message = tah.process_config( root ) + return altered, new_root, error_message diff -r e832e466471403d54d788193c938f3c4d139142c -r 63919647ca8424a28042ccb79ea2db6e0c68d507 lib/tool_shed/dependencies/tag_attribute_handler.py --- /dev/null +++ b/lib/tool_shed/dependencies/tag_attribute_handler.py @@ -0,0 +1,199 @@ +import copy +import logging + +log = logging.getLogger( __name__ ) + + +class TagAttributeHandler( object ): + + def __init__( self, app, rdd, unpopulate ): + self.app = app + self.altered = False + self.rdd = rdd + self.unpopulate = unpopulate + + def process_action_tag_set( self, elem, message ): + # Here we're inside of an <actions> tag set. See http://localhost:9009/view/devteam/package_r_2_11_0 . + # <action> + # <repository name="package_readline_6_2" owner="devteam"> + # <package name="readline" version="6.2" /> + # </repository> + # </action> + elem_altered = False + new_elem = copy.deepcopy( elem ) + for sub_index, sub_elem in enumerate( elem ): + altered = False + error_message = '' + if sub_elem.tag == 'repository': + altered, new_sub_elem, error_message = \ + self.process_repository_tag_set( parent_elem=elem, + elem_index=sub_index, + elem=sub_elem, + message=message ) + if error_message: + message += error_message + if altered: + if not self.altered: + self.altered = True + if not elem_altered: + elem_altered = True + new_elem[ sub_index ] = new_sub_elem + return elem_altered, new_elem, message + + def process_actions_tag_set( self, elem, message ): + # <actions> + # <package name="libgtextutils" version="0.6"> + # <repository name="package_libgtextutils_0_6" owner="test" prior_installation_required="True" /> + # </package> + from tool_shed.util import xml_util + elem_altered = False + new_elem = copy.deepcopy( elem ) + for sub_index, sub_elem in enumerate( elem ): + altered = False + error_message = '' + if sub_elem.tag == 'package': + altered, new_sub_elem, error_message = self.process_package_tag_set( elem=sub_elem, + message=message ) + elif sub_elem.tag == 'action': + # <action type="set_environment_for_install"> + # <repository name="package_readline_6_2" owner="devteam""> + # <package name="readline" version="6.2" /> + # </repository> + # </action> + altered, new_sub_elem, error_message = self.process_action_tag_set( elem=sub_elem, + message=message ) + else: + # Inspect the sub elements of elem to locate all <repository> tags and + # populate them with toolshed and changeset_revision attributes if necessary. + altered, new_sub_elem, error_message = self.rdd.handle_sub_elem( parent_elem=elem, + elem_index=sub_index, + elem=sub_elem ) + if error_message: + message += error_message + if altered: + if not self.altered: + self.altered = True + if not elem_altered: + elem_altered = True + new_elem[ sub_index ] = new_sub_elem + return elem_altered, new_elem, message + + def process_actions_group_tag_set( self, elem, message, skip_actions_tags=False ): + # Inspect all entries in the <actions_group> tag set, skipping <actions> + # tag sets that define os and architecture attributes. We want to inspect + # only the last <actions> tag set contained within the <actions_group> tag + # set to see if a complex repository dependency is defined. + elem_altered = False + new_elem = copy.deepcopy( elem ) + for sub_index, sub_elem in enumerate( elem ): + altered = False + error_message = '' + if sub_elem.tag == 'actions': + if skip_actions_tags: + # Skip all actions tags that include os or architecture attributes. + system = sub_elem.get( 'os' ) + architecture = sub_elem.get( 'architecture' ) + if system or architecture: + continue + altered, new_sub_elem, error_message = \ + self.process_actions_tag_set( elem=sub_elem, + message=message ) + if error_message: + message += error_message + if altered: + if not self.altered: + self.altered = True + if not elem_altered: + elem_altered = True + new_elem[ sub_index ] = new_sub_elem + return elem_altered, new_elem, message + + def process_config( self, root ): + error_message = '' + new_root = copy.deepcopy( root ) + if root.tag == 'tool_dependency': + for elem_index, elem in enumerate( root ): + altered = False + if elem.tag == 'package': + # <package name="eigen" version="2.0.17"> + altered, new_elem, error_message = \ + self.process_package_tag_set( elem=elem, + message=error_message ) + if altered: + if not self.altered: + self.altered = True + new_root[ elem_index ] = new_elem + else: + error_message = "Invalid tool_dependencies.xml file." + return self.altered, new_root, error_message + + def process_install_tag_set( self, elem, message ): + # <install version="1.0"> + elem_altered = False + new_elem = copy.deepcopy( elem ) + for sub_index, sub_elem in enumerate( elem ): + altered = False + error_message = '' + if sub_elem.tag == 'actions_group': + altered, new_sub_elem, error_message = \ + self.process_actions_group_tag_set( elem=sub_elem, + message=message, + skip_actions_tags=True ) + elif sub_elem.tag == 'actions': + altered, new_sub_elem, error_message = \ + self.process_actions_tag_set( elem=sub_elem, + message=message ) + else: + package_name = elem.get( 'name', '' ) + package_version = elem.get( 'version', '' ) + error_message += 'Version %s of the %s package cannot be installed because ' % \ + ( str( package_version ), str( package_name ) ) + error_message += 'the recipe for installing the package is missing either an ' + error_message += '<actions> tag set or an <actions_group> tag set.' + if error_message: + message += error_message + if altered: + if not self.altered: + self.altered = True + if not elem_altered: + elem_altered = True + new_elem[ sub_index ] = new_sub_elem + return elem_altered, new_elem, message + + def process_package_tag_set( self, elem, message ): + elem_altered = False + new_elem = copy.deepcopy( elem ) + for sub_index, sub_elem in enumerate( elem ): + altered = False + error_message = '' + if sub_elem.tag == 'install': + altered, new_sub_elem, error_message = \ + self.process_install_tag_set( elem=sub_elem, + message=message ) + elif sub_elem.tag == 'repository': + altered, new_sub_elem, error_message = \ + self.process_repository_tag_set( parent_elem=elem, + elem_index=sub_index, + elem=sub_elem, + message=message ) + if error_message: + message += error_message + if altered: + if not self.altered: + self.altered = True + if not elem_altered: + elem_altered = True + new_elem[ sub_index ] = new_sub_elem + return elem_altered, new_elem, message + + def process_repository_tag_set( self, parent_elem, elem_index, elem, message ): + # We have a complex repository dependency. + altered, new_elem, error_message = self.rdd.handle_complex_dependency_elem( parent_elem=parent_elem, + elem_index=elem_index, + elem=elem ) + if error_message: + message += error_message + if altered: + if not self.altered: + self.altered = True + return altered, new_elem, message diff -r e832e466471403d54d788193c938f3c4d139142c -r 63919647ca8424a28042ccb79ea2db6e0c68d507 lib/tool_shed/galaxy_install/tool_dependencies/recipe/tag_handler.py --- a/lib/tool_shed/galaxy_install/tool_dependencies/recipe/tag_handler.py +++ b/lib/tool_shed/galaxy_install/tool_dependencies/recipe/tag_handler.py @@ -20,7 +20,7 @@ class RecipeTag( object ): """Abstract class that defines a standard format for handling recipe tags when installing packages.""" - def process_tag_set( tool_shed_repository, tool_dependency, package_elem, package_name, package_version, + def process_tag_set( self, tool_shed_repository, tool_dependency, package_elem, package_name, package_version, from_tool_migration_manager=False, tool_dependency_db_records=None ): raise "Unimplemented Method" diff -r e832e466471403d54d788193c938f3c4d139142c -r 63919647ca8424a28042ccb79ea2db6e0c68d507 lib/tool_shed/util/commit_util.py --- a/lib/tool_shed/util/commit_util.py +++ b/lib/tool_shed/util/commit_util.py @@ -5,14 +5,10 @@ import shutil import tempfile from galaxy.datatypes import checkers -from galaxy.util import asbool -from galaxy.util.odict import odict -from galaxy.web import url_for -import tool_shed.util.shed_util_common as suc from tool_shed.util import basic_util from tool_shed.util import hg_util +from tool_shed.util import shed_util_common as suc from tool_shed.util import tool_util -from tool_shed.util import xml_util import tool_shed.repository_types.util as rt_util log = logging.getLogger( __name__ ) @@ -48,8 +44,8 @@ def check_file_contents_for_email_alerts( app ): """ - See if any admin users have chosen to receive email alerts when a repository is updated. If so, the file contents of the update must be - checked for inappropriate content. + See if any admin users have chosen to receive email alerts when a repository is updated. + If so, the file contents of the update must be checked for inappropriate content. """ sa_session = app.model.context.current admin_users = app.config.get( "admin_users", "" ).split( "," ) @@ -71,8 +67,9 @@ def get_change_lines_in_file_for_tag( tag, change_dict ): """ - The received change_dict is the jsonified version of the changes to a file in a changeset being pushed to the tool shed from the command line. - This method cleans and returns appropriate lines for inspection. + The received change_dict is the jsonified version of the changes to a file in a + changeset being pushed to the Tool Shed from the command line. This method cleans + and returns appropriate lines for inspection. """ cleaned_lines = [] data_list = change_dict.get( 'data', [] ) @@ -108,7 +105,9 @@ return upload_point def handle_bz2( repository, uploaded_file_name ): - fd, uncompressed = tempfile.mkstemp( prefix='repo_%d_upload_bunzip2_' % repository.id, dir=os.path.dirname( uploaded_file_name ), text=False ) + fd, uncompressed = tempfile.mkstemp( prefix='repo_%d_upload_bunzip2_' % repository.id, + dir=os.path.dirname( uploaded_file_name ), + text=False ) bzipped_file = bz2.BZ2File( uploaded_file_name, 'rb' ) while 1: try: @@ -125,22 +124,6 @@ bzipped_file.close() shutil.move( uncompressed, uploaded_file_name ) -def handle_complex_repository_dependency_elem( app, elem, sub_elem_index, sub_elem, sub_elem_altered, altered, unpopulate=False ): - """ - Populate or unpopulate the toolshed and changeset_revision attributes of a <repository> tag that defines - a complex repository dependency. - """ - # <repository name="package_eigen_2_0" owner="test" prior_installation_required="True" /> - revised, repository_elem, error_message = handle_repository_dependency_elem( app, sub_elem, unpopulate=unpopulate ) - if error_message: - error_message = 'The tool_dependencies.xml file contains an invalid <repository> tag. %s' % error_message - if revised: - elem[ sub_elem_index ] = repository_elem - sub_elem_altered = True - if not altered: - altered = True - return altered, sub_elem_altered, elem, error_message - def handle_directory_changes( app, host, username, repository, full_path, filenames_in_archive, remove_repo_files_not_in_tar, new_repo_alert, commit_message, undesirable_dirs_removed, undesirable_files_removed ): repo = hg_util.get_repo_for_repository( app, repository=repository, repo_path=None, create=False ) @@ -218,20 +201,10 @@ admin_only=admin_only ) return True, '', files_to_remove, content_alert_str, undesirable_dirs_removed, undesirable_files_removed -def handle_missing_repository_attribute( elem ): - # <repository name="molecule_datatypes" owner="test" /> - error_message = '' - name = elem.get( 'name' ) - if not name: - error_message += 'The tag is missing the required name attribute. ' - owner = elem.get( 'owner' ) - if not owner: - error_message += 'The tag is missing the required owner attribute. ' - log.debug( error_message ) - return error_message - def handle_gzip( repository, uploaded_file_name ): - fd, uncompressed = tempfile.mkstemp( prefix='repo_%d_upload_gunzip_' % repository.id, dir=os.path.dirname( uploaded_file_name ), text=False ) + fd, uncompressed = tempfile.mkstemp( prefix='repo_%d_upload_gunzip_' % repository.id, + dir=os.path.dirname( uploaded_file_name ), + text=False ) gzipped_file = gzip.GzipFile( uploaded_file_name, 'rb' ) while 1: try: @@ -248,249 +221,6 @@ gzipped_file.close() shutil.move( uncompressed, uploaded_file_name ) -def handle_repository_dependencies_definition( app, repository_dependencies_config, unpopulate=False ): - """ - Populate or unpopulate the toolshed and changeset_revision attributes of a <repository> tag. Populating will occur when a - dependency definition file is being uploaded to the repository, while depopulating will occur when the repository is being - exported. - """ - altered = False - # Make sure we're looking at a valid repository_dependencies.xml file. - tree, error_message = xml_util.parse_xml( repository_dependencies_config ) - if tree is None: - return False, None, error_message - root = tree.getroot() - if root.tag == 'repositories': - for index, elem in enumerate( root ): - if elem.tag == 'repository': - # <repository name="molecule_datatypes" owner="test" changeset_revision="1a070566e9c6" /> - revised, elem, error_message = handle_repository_dependency_elem( app, elem, unpopulate=unpopulate ) - if error_message: - error_message = 'The repository_dependencies.xml file contains an invalid <repository> tag. %s' % error_message - return False, None, error_message - if revised: - root[ index ] = elem - if not altered: - altered = True - return altered, root, error_message - return False, None, error_message - -def handle_repository_dependency_elem( app, elem, unpopulate=False ): - """Populate or unpopulate repository tags.""" - # <repository name="molecule_datatypes" owner="test" changeset_revision="1a070566e9c6" /> - # <repository changeset_revision="xxx" name="package_xorg_macros_1_17_1" owner="test" toolshed="yyy"> - # <package name="xorg_macros" version="1.17.1" /> - # </repository> - error_message = '' - name = elem.get( 'name' ) - owner = elem.get( 'owner' ) - # The name and owner attributes are always required, so if either are missing, return the error message. - if not name or not owner: - error_message = handle_missing_repository_attribute( elem ) - return False, elem, error_message - revised = False - toolshed = elem.get( 'toolshed' ) - changeset_revision = elem.get( 'changeset_revision' ) - # Over a short period of time a bug existed which caused the prior_installation_required attribute - # to be set to False and included in the <repository> tag when a repository was exported along with - # its dependencies. The following will eliminate this problematic attribute upon import. - prior_installation_required = elem.get( 'prior_installation_required' ) - if prior_installation_required is not None and not asbool( prior_installation_required ): - del elem.attrib[ 'prior_installation_required' ] - sub_elems = [ child_elem for child_elem in list( elem ) ] - if len( sub_elems ) > 0: - # At this point, a <repository> tag will point only to a package. - # <package name="xorg_macros" version="1.17.1" /> - # Coerce the list to an odict(). - sub_elements = odict() - packages = [] - for sub_elem in sub_elems: - sub_elem_type = sub_elem.tag - sub_elem_name = sub_elem.get( 'name' ) - sub_elem_version = sub_elem.get( 'version' ) - if sub_elem_type and sub_elem_name and sub_elem_version: - packages.append( ( sub_elem_name, sub_elem_version ) ) - sub_elements[ 'packages' ] = packages - else: - # Set to None. - sub_elements = None - if unpopulate: - # We're exporting the repository, so eliminate all toolshed and changeset_revision attributes from the - # <repository> tag. - if toolshed or changeset_revision: - attributes = odict() - attributes[ 'name' ] = name - attributes[ 'owner' ] = owner - prior_installation_required = elem.get( 'prior_installation_required' ) - if asbool( prior_installation_required ): - attributes[ 'prior_installation_required' ] = 'True' - elem = xml_util.create_element( 'repository', attributes=attributes, sub_elements=sub_elements ) - revised = True - return revised, elem, error_message - # From here on we're populating the toolshed and changeset_revisions if necessary. - if not toolshed: - # Default the setting to the current tool shed. - toolshed = str( url_for( '/', qualified=True ) ).rstrip( '/' ) - elem.attrib[ 'toolshed' ] = toolshed - revised = True - if not changeset_revision: - # Populate the changeset_revision attribute with the latest installable metadata revision for the defined repository. - # We use the latest installable revision instead of the latest metadata revision to ensure that the contents of the - # revision are valid. - repository = suc.get_repository_by_name_and_owner( app, name, owner ) - if repository: - repo = hg_util.get_repo_for_repository( app, repository=repository, repo_path=None, create=False ) - lastest_installable_changeset_revision = suc.get_latest_downloadable_changeset_revision( app, repository, repo ) - if lastest_installable_changeset_revision != hg_util.INITIAL_CHANGELOG_HASH: - elem.attrib[ 'changeset_revision' ] = lastest_installable_changeset_revision - revised = True - else: - error_message = 'Invalid latest installable changeset_revision %s ' % str( lastest_installable_changeset_revision ) - error_message += 'retrieved for repository %s owned by %s. ' % ( str( name ), str( owner ) ) - else: - error_message = 'Unable to locate repository with name %s and owner %s. ' % ( str( name ), str( owner ) ) - return revised, elem, error_message - -def handle_repository_dependency_sub_elem( app, package_altered, altered, actions_elem, action_index, action_elem, unpopulate=False ): - """ - Populate or unpopulate the toolshed and changeset_revision attributes for each of the following tag sets. - <action type="set_environment_for_install"> - <action type="setup_r_environment"> - <action type="setup_ruby_environment"> - """ - error_message = '' - for repo_index, repo_elem in enumerate( action_elem ): - # Make sure to skip comments and tags that are not <repository>. - if repo_elem.tag == 'repository': - revised, repository_elem, message = handle_repository_dependency_elem( app, repo_elem, unpopulate=unpopulate ) - if message: - error_message += 'The tool_dependencies.xml file contains an invalid <repository> tag. %s' % message - if revised: - action_elem[ repo_index ] = repository_elem - package_altered = True - if not altered: - altered = True - if package_altered: - actions_elem[ action_index ] = action_elem - return package_altered, altered, actions_elem, error_message - -def handle_tool_dependencies_definition( app, tool_dependencies_config, unpopulate=False ): - """ - Populate or unpopulate the tooshed and changeset_revision attributes of each <repository> - tag defined within a tool_dependencies.xml file. - """ - altered = False - error_message = '' - # Make sure we're looking at a valid tool_dependencies.xml file. - tree, error_message = xml_util.parse_xml( tool_dependencies_config ) - if tree is None: - return False, None, error_message - root = tree.getroot() - if root.tag == 'tool_dependency': - package_altered = False - for root_index, root_elem in enumerate( root ): - # <package name="eigen" version="2.0.17"> - package_altered = False - if root_elem.tag == 'package': - for package_index, package_elem in enumerate( root_elem ): - if package_elem.tag == 'repository': - # We have a complex repository dependency. - altered, package_altered, root_elem, message = \ - handle_complex_repository_dependency_elem( app, - root_elem, - package_index, - package_elem, - package_altered, - altered, - unpopulate=unpopulate ) - if message: - error_message += message - if package_altered: - root[ root_index ] = root_elem - elif package_elem.tag == 'install': - # <install version="1.0"> - for actions_index, actions_elem in enumerate( package_elem ): - if actions_elem.tag == 'actions_group': - # Inspect all entries in the <actions_group> tag set, skipping <actions> - # tag sets that define os and architecture attributes. We want to inspect - # only the last <actions> tag set contained within the <actions_group> tag - # set to see if a complex repository dependency is defined. - for actions_group_index, actions_group_elem in enumerate( actions_elem ): - if actions_group_elem.tag == 'actions': - # Skip all actions tags that include os or architecture attributes. - system = actions_group_elem.get( 'os' ) - architecture = actions_group_elem.get( 'architecture' ) - if system or architecture: - continue - # ... - # <actions> - # <package name="libgtextutils" version="0.6"> - # <repository name="package_libgtextutils_0_6" owner="test" prior_installation_required="True" /> - # </package> - # ... - for last_actions_index, last_actions_elem in enumerate( actions_group_elem ): - last_actions_package_altered = False - if last_actions_elem.tag == 'package': - last_actions_elem_package_elem_altered = False - for last_actions_elem_package_index, last_actions_elem_package_elem in enumerate( last_actions_elem ): - if last_actions_elem_package_elem.tag == 'repository': - # We have a complex repository dependency. - last_actions_package_altered, last_actions_elem_package_elem_altered, last_actions_elem, message = \ - handle_complex_repository_dependency_elem( app, - last_actions_elem, - last_actions_elem_package_index, - last_actions_elem_package_elem, - last_actions_elem_package_elem_altered, - last_actions_package_altered, - unpopulate=unpopulate ) - if not altered and last_actions_package_altered: - altered = True - if message: - error_message += message - if last_actions_elem_package_elem_altered: - actions_group_elem[ last_actions_index ] = last_actions_elem - else: - # Inspect the sub elements of last_actions_elem to locate all <repository> tags and - # populate them with toolshed and changeset_revision attributes if necessary. - last_actions_package_altered, altered, last_actions_elem, message = \ - handle_repository_dependency_sub_elem( app, - last_actions_package_altered, - altered, - actions_group_elem, - last_actions_index, - last_actions_elem, - unpopulate=unpopulate ) - if message: - error_message += message - elif actions_elem.tag == 'actions': - # We are not in an <actions_group> tag set, so we must be in an <actions> tag set. - for action_index, action_elem in enumerate( actions_elem ): - # Inspect the sub elements of last_actions_elem to locate all <repository> tags - # and populate them with toolshed and changeset_revision attributes if necessary. - package_altered, altered, actions_elem, message = \ - handle_repository_dependency_sub_elem( app, - package_altered, - altered, - actions_elem, - action_index, - action_elem, - unpopulate=unpopulate ) - if message: - error_message += message - else: - package_name = root_elem.get( 'name', '' ) - package_version = root_elem.get( 'version', '' ) - error_message += 'Version %s of the %s package cannot be installed because ' % \ - ( str( package_version ), str( package_name ) ) - error_message += 'the recipe for installing the package is missing either an ' - error_message += '<actions> tag set or an <actions_group> tag set.' - if package_altered: - package_elem[ actions_index ] = actions_elem - if package_altered: - root[ root_index ] = root_elem - return altered, root, error_message - return False, None, error_message - def repository_tag_is_valid( filename, line ): """ Checks changes made to <repository> tags in a dependency definition file being pushed to the diff -r e832e466471403d54d788193c938f3c4d139142c -r 63919647ca8424a28042ccb79ea2db6e0c68d507 lib/tool_shed/util/export_util.py --- a/lib/tool_shed/util/export_util.py +++ b/lib/tool_shed/util/export_util.py @@ -6,19 +6,19 @@ import threading from time import gmtime from time import strftime -import tool_shed.util.shed_util_common as suc import tool_shed.repository_types.util as rt_util from galaxy import eggs from galaxy import web from galaxy.util.odict import odict +from tool_shed.dependencies import dependency_manager from tool_shed.util import basic_util from tool_shed.util import commit_util from tool_shed.util import common_util from tool_shed.util import encoding_util from tool_shed.util import hg_util from tool_shed.util import repository_dependency_util +from tool_shed.util import shed_util_common as suc from tool_shed.util import xml_util - from tool_shed.galaxy_install.repository_dependencies.repository_dependency_manager import RepositoryDependencyManager eggs.require( 'mercurial' ) @@ -129,6 +129,8 @@ return repositories_archive, error_messages def generate_repository_archive( trans, work_dir, tool_shed_url, repository, changeset_revision, file_type ): + rdah = dependency_manager.RepositoryDependencyAttributeHandler( trans.app, unpopulate=True ) + tdah = dependency_manager.ToolDependencyAttributeHandler( trans.app, unpopulate=True ) file_type_str = get_file_type_str( changeset_revision, file_type ) file_name = '%s-%s' % ( repository.name, file_type_str ) return_code, error_message = archive_repository_revision( trans, ui, repository, work_dir, changeset_revision ) @@ -151,8 +153,7 @@ # See if we have a repository dependencies defined. if name == rt_util.REPOSITORY_DEPENDENCY_DEFINITION_FILENAME: # Eliminate the toolshed, and changeset_revision attributes from all <repository> tags. - altered, root_elem, error_message = \ - commit_util.handle_repository_dependencies_definition( trans.app, full_path, unpopulate=True ) + altered, root_elem, error_message = rdah.handle_tag_attributes( full_path ) if error_message: return None, error_message if altered: @@ -160,8 +161,7 @@ shutil.move( tmp_filename, full_path ) elif name == rt_util.TOOL_DEPENDENCY_DEFINITION_FILENAME: # Eliminate the toolshed, and changeset_revision attributes from all <repository> tags. - altered, root_elem, error_message = \ - commit_util.handle_tool_dependencies_definition( trans.app, full_path, unpopulate=True ) + altered, root_elem, error_message = tdah.handle_tag_attributes( full_path ) if error_message: return None, error_message if altered: diff -r e832e466471403d54d788193c938f3c4d139142c -r 63919647ca8424a28042ccb79ea2db6e0c68d507 lib/tool_shed/util/import_util.py --- a/lib/tool_shed/util/import_util.py +++ b/lib/tool_shed/util/import_util.py @@ -5,13 +5,14 @@ import tempfile import urllib from galaxy import util +from tool_shed.dependencies import dependency_manager from tool_shed.util import commit_util from tool_shed.util import encoding_util from tool_shed.util import hg_util from tool_shed.util import metadata_util from tool_shed.util import repository_maintenance_util +from tool_shed.util import shed_util_common as suc from tool_shed.util import xml_util -import tool_shed.util.shed_util_common as suc import tool_shed.repository_types.util as rt_util log = logging.getLogger( __name__ ) @@ -267,6 +268,8 @@ def import_repository_archive( app, host, user, repository, repository_archive_dict ): """Import a repository archive contained within a repository capsule.""" + rdah = dependency_manager.RepositoryDependencyAttributeHandler( app, unpopulate=False ) + tdah = dependency_manager.ToolDependencyAttributeHandler( app, unpopulate=False ) archive_file_name = repository_archive_dict.get( 'archive_file_name', None ) capsule_file_name = repository_archive_dict[ 'capsule_file_name' ] encoded_file_path = repository_archive_dict[ 'encoded_file_path' ] @@ -303,9 +306,9 @@ for filename in filenames_in_archive: uploaded_file_name = os.path.join( full_path, filename ) if os.path.split( uploaded_file_name )[ -1 ] == rt_util.REPOSITORY_DEPENDENCY_DEFINITION_FILENAME: - # Inspect the contents of the file to see if changeset_revision values are missing and if so, set them appropriately. - altered, root_elem, error_message = \ - commit_util.handle_repository_dependencies_definition( app, uploaded_file_name, unpopulate=False ) + # Inspect the contents of the file to see if toolshed or changeset_revision attributes + # are missing and if so, set them appropriately. + altered, root_elem, error_message = rdah.handle_tag_attributes( uploaded_file_name ) if error_message: results_dict[ 'ok' ] = False results_dict[ 'error_message' ] += error_message @@ -313,8 +316,9 @@ tmp_filename = xml_util.create_and_write_tmp_file( root_elem ) shutil.move( tmp_filename, uploaded_file_name ) elif os.path.split( uploaded_file_name )[ -1 ] == rt_util.TOOL_DEPENDENCY_DEFINITION_FILENAME: - # Inspect the contents of the file to see if changeset_revision values are missing and if so, set them appropriately. - altered, root_elem, error_message = commit_util.handle_tool_dependencies_definition( app, uploaded_file_name ) + # Inspect the contents of the file to see if toolshed or changeset_revision + # attributes are missing and if so, set them appropriately. + altered, root_elem, error_message = tdah.handle_tag_attributes( uploaded_file_name ) if error_message: results_dict[ 'ok' ] = False results_dict[ 'error_message' ] += error_message diff -r e832e466471403d54d788193c938f3c4d139142c -r 63919647ca8424a28042ccb79ea2db6e0c68d507 test/tool_shed/functional/test_0470_tool_dependency_repository_type.py --- a/test/tool_shed/functional/test_0470_tool_dependency_repository_type.py +++ b/test/tool_shed/functional/test_0470_tool_dependency_repository_type.py @@ -169,7 +169,9 @@ commit_message='Populate package_x11_client_1_5_proto_7_0_0470 with tool dependency definitions.', strings_displayed=[], strings_not_displayed=[] ) - assert len( package_x11_repository.metadata_revisions ) == 1, 'package_x11_client_1_5_proto_7_0_0470 has incorrect number of metadata revisions' + assert len( package_x11_repository.metadata_revisions ) == 1, \ + 'package_x11_client_1_5_proto_7_0_0470 has incorrect number of metadata revisions, expected 1 but found %d' % \ + len( package_x11_repository.metadata_revisions ) def test_0025_upload_updated_tool_dependency_to_package_emboss( self ): '''Upload a new tool_dependencies.xml to package_emboss_5_0_0_0470.''' @@ -189,7 +191,9 @@ commit_message='Populate package_emboss_5_0_0_0470 with tool dependency definitions.', strings_displayed=[], strings_not_displayed=[] ) - assert len( package_emboss_repository.metadata_revisions ) == 2, 'package_emboss_5_0_0_0470 has incorrect number of metadata revisions' + assert len( package_emboss_repository.metadata_revisions ) == 2, \ + 'package_emboss_5_0_0_0470 has incorrect number of metadata revisions, expected 2 but found %d' % \ + len( package_emboss_repository.metadata_revisions ) def test_0030_upload_updated_tool_dependency_to_emboss_5_repository( self ): '''Upload a new tool_dependencies.xml to emboss_5_0470.''' Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.
participants (1)
-
commits-noreply＠bitbucket.org