April 2013 - galaxy-commits - lists.galaxyproject.org

commit/galaxy-central: guerler: grep/select tool: fixed tab-delimiter matching for GNU version of grep
by commits-noreply＠bitbucket.org 02 Apr '13

02 Apr '13

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/4ed80ec3f309/ Changeset: 4ed80ec3f309 User: guerler Date: 2013-04-02 21:18:22 Summary: grep/select tool: fixed tab-delimiter matching for GNU version of grep Affected #: 1 file diff -r 715dc4f4c8e218f50d08eb81dc1b3ef0453860a5 -r 4ed80ec3f309f5c8e1c9e735666a2b9b682ff388 tools/filters/grep.py --- a/tools/filters/grep.py +++ b/tools/filters/grep.py @@ -17,6 +17,7 @@ import re import string import commands +from subprocess import Popen, PIPE from tempfile import NamedTemporaryFile # This function is exceedingly useful, perhaps package for reuse? @@ -34,89 +35,100 @@ args = sys.argv[1:] try: - opts = getopts(args) + opts = getopts(args) except IndexError: - print "Usage:" - print " -i Input file" - print " -o Output file" - print " -pattern RegEx pattern" - print " -v true or false (Invert match)" - return 0 + print "Usage:" + print " -i Input file" + print " -o Output file" + print " -pattern RegEx pattern" + print " -v true or false (Invert match)" + return 0 outputfile = opts.get("-o") if outputfile == None: - print "No output file specified." - return -1 + print "No output file specified." + return -1 inputfile = opts.get("-i") if inputfile == None: - print "No input file specified." - return -2 + print "No input file specified." + return -2 invert = opts.get("-v") if invert == None: - print "Match style (Invert or normal) not specified." - return -3 + print "Match style (Invert or normal) not specified." + return -3 pattern = opts.get("-pattern") if pattern == None: - print "RegEx pattern not specified." - return -4 + print "RegEx pattern not specified." + return -4 # All inputs have been specified at this point, now validate. # replace if input has been escaped, remove sq # characters that are allowed but need to be escaped - mapped_chars = { '>' :'__gt__', - '<' :'__lt__', - '\'' :'__sq__', - '"' :'__dq__', - '[' :'__ob__', - ']' :'__cb__', - '{' :'__oc__', - '}' :'__cc__' - } + mapped_chars = { '>' :'__gt__', + '<' :'__lt__', + '\'':'__sq__', + '"' :'__dq__', + '[' :'__ob__', + ']' :'__cb__', + '{' :'__oc__', + '}' :'__cc__' + } - #with new sanitizing we only need to replace for single quote, but this needs to remain for backwards compatibility + # with new sanitizing we only need to replace for single quote, + # but this needs to remain for backwards compatibility for key, value in mapped_chars.items(): pattern = pattern.replace(value, key) - - fileRegEx = re.compile("^[A-Za-z0-9./\-_]+$") #why? - invertRegEx = re.compile("(true)|(false)") #why? + # match filename and invert flag + fileRegEx = re.compile("^[A-Za-z0-9./\-_]+$") + invertRegEx = re.compile("(true)|(false)") + + # verify that filename and inversion flag are in the correct format if not fileRegEx.match(outputfile): - print "Illegal output filename." - return -5 + print "Illegal output filename." + return -5 if not fileRegEx.match(inputfile): - print "Illegal input filename." - return -6 + print "Illegal input filename." + return -6 if not invertRegEx.match(invert): - print "Illegal invert option." - return -7 + print "Illegal invert option." + return -7 # invert grep search? if invert == "true": - invertflag = " -v" + invertflag = "-v" print "Not matching pattern: %s" % pattern else: invertflag = "" print "Matching pattern: %s" % pattern + + # set version flag + versionflag = "-P" - #Create temp file holding pattern - #By using a file to hold the pattern, we don't have worry about sanitizing grep commandline and can include single quotes in pattern + # MacOS 10.8.2 does not support -P option for perl-regex anymore + versionmatch = Popen("grep -V | grep 'BSD'", shell=True, stdout=PIPE).communicate()[0]; + if versionmatch: + versionflag = "-E" + + # create temp file holding pattern + # by using a file to hold the pattern, we don't have worry about sanitizing grep commandline and can include single quotes in pattern pattern_file_name = NamedTemporaryFile().name open( pattern_file_name, 'w' ).write( pattern ) - #generate grep command - commandline = "grep -E %s -f %s %s > %s" % ( invertflag, pattern_file_name, inputfile, outputfile ) + # generate grep command + commandline = "grep %s %s -f %s %s > %s" % ( versionflag, invertflag, pattern_file_name, inputfile, outputfile ) - #run grep + # run grep errorcode, stdout = commands.getstatusoutput(commandline) - #remove temp pattern file + # remove temp pattern file os.unlink( pattern_file_name ) - #return error code + # return error code return errorcode if __name__ == "__main__": Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: greg: Add baseline support for ordering the installation of repository dependencies by supporting an attribute named prior_installation_required in repository dependency definitions. If this attribute is set to True in the repository dependency definition for a certain repository, then attempts will be made to ensure the repository dependency is installed prior to the repository being installed into Galaxy. The Galaxy installation process has not yet been enhanced to
by commits-noreply＠bitbucket.org 02 Apr '13

02 Apr '13

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/715dc4f4c8e2/ Changeset: 715dc4f4c8e2 User: greg Date: 2013-04-02 21:03:00 Summary: Add baseline support for ordering the installation of repository dependencies by supporting an attribute named prior_installation_required in repository dependency definitions. If this attribute is set to True in the repository dependency definition for a certain repository, then attempts will be made to ensure the repository dependency is installed prior to the repository being installed into Galaxy. The Galaxy installation process has not yet been enhanced to leverage this feature, but the repsitory dependency framework in the tool shed itself is now using it. Affected #: 9 files diff -r d62c5edf1ef265000740cac2949f90c9b175db6c -r 715dc4f4c8e218f50d08eb81dc1b3ef0453860a5 lib/galaxy/model/__init__.py --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -3210,6 +3210,31 @@ return 'repository_dependencies' in self.metadata return False @property + def requires_prior_installation_of( self ): + """ + Return a list of repository dependency tuples like (tool_shed, name, owner, changeset_revision, prior_installation_required) for this + repository's repository dependencies where prior_installation_required is True. By definition, repository dependencies are required to + be installed in order for this repository to function correctly. However, those repository dependencies that are defined for this + repository with prior_installation_required set to True place them in a special category in that the required repositories must be + installed before this repository is installed. Among other things, this enables these "special" repository dependencies to include + information that enables the successful intallation of this repository. + """ + required_rd_tups_that_must_be_installed = [] + if self.has_repository_dependencies: + rd_tups = self.metadata[ 'repository_dependencies' ][ 'repository_dependencies' ] + for rd_tup in rd_tups: + if len( rd_tup ) == 4: + # Metadata should have been reset on this installed tool_shed_repository, but it wasn't. + tool_shed, name, owner, changeset_revision = rd_tup + # Default prior_installation_required to False. + prior_installation_required = False + elif len( rd_tup ) == 5: + tool_shed, name, owner, changeset_revision, prior_installation_required = rd_tup + prior_installation_required = util.asbool( str( prior_installation_required ) ) + if prior_installation_required: + required_rd_tups_that_must_be_installed.append( ( tool_shed, name, owner, changeset_revision, prior_installation_required ) ) + return required_rd_tups_that_must_be_installed + @property def includes_data_managers( self ): if self.metadata: return bool( len( self.metadata.get( 'data_manager', {} ).get( 'data_managers', {} ) ) ) diff -r d62c5edf1ef265000740cac2949f90c9b175db6c -r 715dc4f4c8e218f50d08eb81dc1b3ef0453860a5 lib/galaxy/webapps/tool_shed/controllers/repository.py --- a/lib/galaxy/webapps/tool_shed/controllers/repository.py +++ b/lib/galaxy/webapps/tool_shed/controllers/repository.py @@ -242,7 +242,8 @@ return trans.response.send_redirect( web.url_for( controller='repository', action='view_or_manage_repository', **kwd ) ) - if 'user_id' not in kwd: + user_id = kwd.get( 'user_id', None ) + if user_id is None: # The received id is the repository id, so we need to get the id of the user that uploaded the repository. repository_id = kwd.get( 'id', None ) if repository_id: @@ -256,8 +257,9 @@ action='view_or_manage_repository', id=trans.security.encode_id( repository.id ), changeset_revision=selected_changeset_revision ) ) - user = suc.get_user( trans, kwd[ 'user_id' ] ) - self.repositories_by_user_grid.title = "Repositories owned by %s" % user.username + if user_id: + user = suc.get_user( trans, user_id ) + self.repositories_by_user_grid.title = "Repositories owned by %s" % user.username return self.repositories_by_user_grid( trans, **kwd ) @web.expose @@ -1192,7 +1194,7 @@ encoded_repository_ids = [] changeset_revisions = [] for required_repository_tup in decoded_required_repository_tups: - tool_shed, name, owner, changeset_revision = required_repository_tup + tool_shed, name, owner, changeset_revision, prior_installation_required = suc.parse_repository_dependency_tuple( required_repository_tup ) repository = suc.get_repository_by_name_and_owner( trans.app, name, owner ) encoded_repository_ids.append( trans.security.encode_id( repository.id ) ) changeset_revisions.append( changeset_revision ) diff -r d62c5edf1ef265000740cac2949f90c9b175db6c -r 715dc4f4c8e218f50d08eb81dc1b3ef0453860a5 lib/galaxy/webapps/tool_shed/util/container_util.py --- a/lib/galaxy/webapps/tool_shed/util/container_util.py +++ b/lib/galaxy/webapps/tool_shed/util/container_util.py @@ -1,13 +1,19 @@ -import os, logging, threading +import logging +import os +import threading +from galaxy.util import asbool from tool_shed.util import readme_util +import tool_shed.util.shed_util_common as suc log = logging.getLogger( __name__ ) # String separator STRSEP = '__ESEP__' + class Folder( object ): """Container object.""" + def __init__( self, id=None, key=None, label=None, parent=None ): self.id = id self.key = key @@ -26,40 +32,49 @@ self.repository_dependencies = [] self.readme_files = [] self.workflows = [] + def contains_folder( self, folder ): for index, contained_folder in enumerate( self.folders ): if folder == contained_folder: return index, contained_folder return 0, None + def contains_repository_dependency( self, repository_dependency ): listified_repository_dependency = repository_dependency.listify for contained_repository_dependency in self.repository_dependencies: if contained_repository_dependency.listify == listified_repository_dependency: return True return False + def remove_repository_dependency( self, repository_dependency ): listified_repository_dependency = repository_dependency.listify for contained_repository_dependency in self.repository_dependencies: if contained_repository_dependency.listify == listified_repository_dependency: self.repository_dependencies.remove( contained_repository_dependency ) + def to_repository_dependency( self, repository_dependency_id ): - toolshed, name, owner, changeset_revision = self.key.split( STRSEP ) + toolshed, name, owner, changeset_revision, prior_installation_required = suc.parse_repository_dependency_tuple( self.key.split( STRSEP ) ) return RepositoryDependency( id=repository_dependency_id, toolshed=toolshed, repository_name=name, repository_owner=owner, - changeset_revision=changeset_revision ) + changeset_revision=changeset_revision, + prior_installation_required=prior_installation_required ) + class DataManager( object ): """Data Manager object""" + def __init__( self, id=None, name=None, version=None, data_tables=None ): self.id = id self.name = name self.version = version self.data_tables = data_tables + class Datatype( object ): """Datatype object""" + def __init__( self, id=None, extension=None, type=None, mimetype=None, subclass=None, converters=None, display_app_containers=None ): self.id = id self.extension = extension @@ -69,25 +84,32 @@ self.converters = converters self.display_app_containers = display_app_containers + class InvalidDataManager( object ): - """Data Manager object""" + """Invalid data Manager object""" + def __init__( self, id=None, index=None, error=None ): self.id = id self.index = index self.error = error + class InvalidRepositoryDependency( object ): """Invalid repository dependency definition object""" - def __init__( self, id=None, toolshed=None, repository_name=None, repository_owner=None, changeset_revision=None, error=None ): + + def __init__( self, id=None, toolshed=None, repository_name=None, repository_owner=None, changeset_revision=None, prior_installation_required=False, error=None ): self.id = id self.toolshed = toolshed self.repository_name = repository_name self.repository_owner = repository_owner self.changeset_revision = changeset_revision + self.prior_installation_required = prior_installation_required self.error = error + class InvalidTool( object ): """Invalid tool object""" + def __init__( self, id=None, tool_config=None, repository_id=None, changeset_revision=None, repository_installation_status=None ): self.id = id self.tool_config = tool_config @@ -95,8 +117,10 @@ self.changeset_revision = changeset_revision self.repository_installation_status = repository_installation_status + class InvalidToolDependency( object ): """Invalid tool dependency definition object""" + def __init__( self, id=None, name=None, version=None, type=None, error=None ): self.id = id self.name = name @@ -104,29 +128,38 @@ self.type = type self.error = error + class ReadMe( object ): """Readme text object""" + def __init__( self, id=None, name=None, text=None ): self.id = id self.name = name self.text = text + class RepositoryDependency( object ): """Repository dependency object""" - def __init__( self, id=None, toolshed=None, repository_name=None, repository_owner=None, changeset_revision=None, installation_status=None, tool_shed_repository_id=None ): + + def __init__( self, id=None, toolshed=None, repository_name=None, repository_owner=None, changeset_revision=None, prior_installation_required=False, + installation_status=None, tool_shed_repository_id=None ): self.id = id self.toolshed = toolshed self.repository_name = repository_name self.repository_owner = repository_owner self.changeset_revision = changeset_revision + self.prior_installation_required = prior_installation_required self.installation_status = installation_status self.tool_shed_repository_id = tool_shed_repository_id + @property def listify( self ): - return [ self.toolshed, self.repository_name, self.repository_owner, self.changeset_revision ] + return [ self.toolshed, self.repository_name, self.repository_owner, self.changeset_revision, asbool( str( self.prior_installation_required ) ) ] + class Tool( object ): """Tool object""" + def __init__( self, id=None, tool_config=None, tool_id=None, name=None, description=None, version=None, requirements=None, repository_id=None, changeset_revision=None, repository_installation_status=None ): self.id = id @@ -140,8 +173,10 @@ self.changeset_revision = changeset_revision self.repository_installation_status = repository_installation_status + class ToolDependency( object ): """Tool dependency object""" + def __init__( self, id=None, name=None, version=None, type=None, install_dir=None, readme=None, installation_status=None, repository_id=None, tool_dependency_id=None, is_orphan=None ): self.id = id @@ -154,12 +189,15 @@ self.repository_id = repository_id self.tool_dependency_id = tool_dependency_id self.is_orphan = is_orphan + @property def listify( self ): return [ self.name, self.version, self.type ] + class Workflow( object ): """Workflow object.""" + def __init__( self, id=None, workflow_name=None, steps=None, format_version=None, annotation=None, repository_metadata_id=None, repository_id=None ): # When rendered in the tool shed, repository_metadata_id will have a value and repository_id will be None. When rendered in Galaxy, repository_id # will have a value and repository_metadata_id will be None. @@ -334,8 +372,9 @@ for invalid_repository_dependency in invalid_repository_dependencies: folder_id += 1 invalid_repository_dependency_id += 1 - toolshed, name, owner, changeset_revision, error = invalid_repository_dependency - key = generate_repository_dependencies_key_for_repository( toolshed, name, owner, changeset_revision ) + toolshed, name, owner, changeset_revision, prior_installation_required, error = \ + suc.parse_repository_dependency_tuple( invalid_repository_dependency, contains_error=True ) + key = generate_repository_dependencies_key_for_repository( toolshed, name, owner, changeset_revision, prior_installation_required ) label = "Repository <b>%s</b> revision <b>%s</b> owned by <b>%s</b>" % ( name, changeset_revision, owner ) folder = Folder( id=folder_id, key=key, @@ -346,6 +385,7 @@ repository_name=name, repository_owner=owner, changeset_revision=changeset_revision, + prior_installation_required=prior_installation_required, error=error ) folder.invalid_repository_dependencies.append( ird ) invalid_repository_dependencies_folder.folders.append( folder ) @@ -691,7 +731,6 @@ data_managers = metadata['data_manager'].get( 'invalid_data_managers', None ) folder_id, data_managers_root_folder = build_invalid_data_managers_folder( trans, folder_id, data_managers, error_messages, label="Invalid Data Managers" ) containers_dict[ 'invalid_data_managers' ] = data_managers_root_folder - except Exception, e: log.debug( "Exception in build_repository_containers_for_tool_shed: %s" % str( e ) ) finally: @@ -948,23 +987,30 @@ return cast_empty_repository_dependency_folders( sub_folder, repository_dependency_id ) return folder, repository_dependency_id -def generate_repository_dependencies_folder_label_from_key( repository_name, repository_owner, changeset_revision, key ): +def generate_repository_dependencies_folder_label_from_key( repository_name, repository_owner, changeset_revision, prior_installation_required, key ): """Return a repository dependency label based on the repository dependency key.""" - if key_is_current_repositorys_key( repository_name, repository_owner, changeset_revision, key ): + if key_is_current_repositorys_key( repository_name, repository_owner, changeset_revision, prior_installation_required, key ): label = 'Repository dependencies' else: - label = "Repository <b>%s</b> revision <b>%s</b> owned by <b>%s</b>" % ( repository_name, changeset_revision, repository_owner ) + if prior_installation_required: + prior_installation_required_str = " <i>(prior install required)</i>" + else: + prior_installation_required_str = "" + label = "Repository <b>%s</b> revision <b>%s</b> owned by <b>%s</b>%s" % \ + ( repository_name, changeset_revision, repository_owner, prior_installation_required_str ) return label -def generate_repository_dependencies_key_for_repository( toolshed_base_url, repository_name, repository_owner, changeset_revision ): +def generate_repository_dependencies_key_for_repository( toolshed_base_url, repository_name, repository_owner, changeset_revision, prior_installation_required ): # FIXME: assumes tool shed is current tool shed since repository dependencies across tool sheds is not yet supported. - return '%s%s%s%s%s%s%s' % ( str( toolshed_base_url ).rstrip( '/' ), - STRSEP, - str( repository_name ), - STRSEP, - str( repository_owner ), - STRSEP, - str( changeset_revision ) ) + return '%s%s%s%s%s%s%s%s%s' % ( str( toolshed_base_url ).rstrip( '/' ), + STRSEP, + str( repository_name ), + STRSEP, + str( repository_owner ), + STRSEP, + str( changeset_revision ), + STRSEP, + str( prior_installation_required ) ) def generate_tool_dependencies_key( name, version, type ): return '%s%s%s%s%s' % ( str( name ), STRSEP, str( version ), STRSEP, str( type ) ) @@ -983,12 +1029,22 @@ repository_name = items[ 1 ] repository_owner = items[ 2 ] changeset_revision = items[ 3 ] - return toolshed_base_url, repository_name, repository_owner, changeset_revision + if len( items ) == 5: + prior_installation_required = asbool( str( items[ 4 ] ) ) + else: + # Metadata should have been reset on the repository that contains the definition for this repository_dependency. In the meantime we'll + # default the prior_installation_required to False. + prior_installation_required = False + return toolshed_base_url, repository_name, repository_owner, changeset_revision, prior_installation_required def handle_repository_dependencies_container_entry( trans, repository_dependencies_folder, rd_key, rd_value, folder_id, repository_dependency_id, folder_keys ): - toolshed, repository_name, repository_owner, changeset_revision = get_components_from_key( rd_key ) + toolshed, repository_name, repository_owner, changeset_revision, prior_installation_required = get_components_from_key( rd_key ) folder = get_folder( repository_dependencies_folder, rd_key ) - label = generate_repository_dependencies_folder_label_from_key( repository_name, repository_owner, changeset_revision, repository_dependencies_folder.key ) + label = generate_repository_dependencies_folder_label_from_key( repository_name, + repository_owner, + changeset_revision, + prior_installation_required, + repository_dependencies_folder.key ) if folder: if rd_key not in folder_keys: folder_id += 1 @@ -1013,10 +1069,18 @@ for repository_dependency in rd_value: if trans.webapp.name == 'galaxy': if len( repository_dependency ) == 6: - # We have two extra items in the tuple, repository.id and repository.status. + # Metadata should have been reset on this installed repository, but it wasn't. tool_shed_repository_id = repository_dependency[ 4 ] installation_status = repository_dependency[ 5 ] - repository_dependency = repository_dependency[ 0:4 ] + tool_shed, name, owner, changeset_revision = repository_dependency[ 0:4 ] + # Default prior_installation_required to False. + prior_installation_required = False + repository_dependency = [ tool_shed, name, owner, changeset_revision, prior_installation_required ] + elif len( repository_dependency ) == 7: + # We have a repository dependency tuple that includes a prior_installation_required value. + tool_shed_repository_id = repository_dependency[ 5 ] + installation_status = repository_dependency[ 6 ] + repository_dependency = repository_dependency[ 0:5 ] else: tool_shed_repository_id = None installation_status = 'unknown' @@ -1025,13 +1089,15 @@ installation_status = None can_create_dependency = not is_subfolder_of( sub_folder, repository_dependency ) if can_create_dependency: - toolshed, repository_name, repository_owner, changeset_revision = repository_dependency + toolshed, repository_name, repository_owner, changeset_revision, prior_installation_required = \ + suc.parse_repository_dependency_tuple( repository_dependency ) repository_dependency_id += 1 repository_dependency = RepositoryDependency( id=repository_dependency_id, toolshed=toolshed, repository_name=repository_name, repository_owner=repository_owner, changeset_revision=changeset_revision, + prior_installation_required=prior_installation_required, installation_status=installation_status, tool_shed_repository_id=tool_shed_repository_id ) # Insert the repository_dependency into the folder. @@ -1039,16 +1105,20 @@ return repository_dependencies_folder, folder_id, repository_dependency_id def is_subfolder_of( folder, repository_dependency ): - toolshed, repository_name, repository_owner, changeset_revision = repository_dependency - key = generate_repository_dependencies_key_for_repository( toolshed, repository_name, repository_owner, changeset_revision ) + toolshed, repository_name, repository_owner, changeset_revision, prior_installation_required = \ + suc.parse_repository_dependency_tuple( repository_dependency ) + key = generate_repository_dependencies_key_for_repository( toolshed, repository_name, repository_owner, changeset_revision, prior_installation_required ) for sub_folder in folder.folders: if key == sub_folder.key: return True return False -def key_is_current_repositorys_key( repository_name, repository_owner, changeset_revision, key ): - toolshed_base_url, key_name, key_owner, key_changeset_revision = get_components_from_key( key ) - return repository_name == key_name and repository_owner == key_owner and changeset_revision == key_changeset_revision +def key_is_current_repositorys_key( repository_name, repository_owner, changeset_revision, prior_installation_required, key ): + toolshed_base_url, key_name, key_owner, key_changeset_revision, key_prior_installation_required = get_components_from_key( key ) + return repository_name == key_name and \ + repository_owner == key_owner and \ + changeset_revision == key_changeset_revision and \ + prior_installation_required == key_prior_installation_required def populate_repository_dependencies_container( trans, repository_dependencies_folder, repository_dependencies, folder_id, repository_dependency_id ): folder_keys = repository_dependencies.keys() diff -r d62c5edf1ef265000740cac2949f90c9b175db6c -r 715dc4f4c8e218f50d08eb81dc1b3ef0453860a5 lib/tool_shed/galaxy_install/repository_util.py --- a/lib/tool_shed/galaxy_install/repository_util.py +++ b/lib/tool_shed/galaxy_install/repository_util.py @@ -478,7 +478,8 @@ old_container_repository_dependencies_folder.id = folder_id folder_id += 1 # Generate the label by retrieving the repository name. - toolshed, name, owner, changeset_revision = container_util.get_components_from_key( old_container_repository_dependencies_folder.key ) + toolshed, name, owner, changeset_revision, prior_installation_required = \ + container_util.get_components_from_key( old_container_repository_dependencies_folder.key ) old_container_repository_dependencies_folder.label = str( name ) repository_dependencies_folder.folders.append( old_container_repository_dependencies_folder ) # Merge tool_dependencies. diff -r d62c5edf1ef265000740cac2949f90c9b175db6c -r 715dc4f4c8e218f50d08eb81dc1b3ef0453860a5 lib/tool_shed/util/common_install_util.py --- a/lib/tool_shed/util/common_install_util.py +++ b/lib/tool_shed/util/common_install_util.py @@ -138,14 +138,15 @@ installed_repository_dependencies = {} has_repository_dependencies = repository.has_repository_dependencies if has_repository_dependencies: + # The repository dependencies container will include only the immediate repository dependencies of this repository, so the container + # will be only a single level in depth. metadata = repository.metadata installed_rd_tups = [] missing_rd_tups = [] - # The repository dependencies container will include only the immediate repository dependencies of this repository, so - # the container will be only a single level in depth. - for rd in repository.repository_dependencies: - rd_tup = [ rd.tool_shed, rd.name, rd.owner, rd.changeset_revision, rd.id, rd.status ] - if rd.status == trans.model.ToolShedRepository.installation_status.INSTALLED: + for tsr in repository.repository_dependencies: + prior_installation_required = suc.set_prior_installation_required( repository, tsr ) + rd_tup = [ tsr.tool_shed, tsr.name, tsr.owner, tsr.changeset_revision, prior_installation_required, tsr.id, tsr.status ] + if tsr.status == trans.model.ToolShedRepository.installation_status.INSTALLED: installed_rd_tups.append( rd_tup ) else: missing_rd_tups.append( rd_tup ) @@ -153,12 +154,13 @@ # Get the description from the metadata in case it has a value. repository_dependencies = metadata.get( 'repository_dependencies', {} ) description = repository_dependencies.get( 'description', None ) - # We need to add a root_key entry to one or both of installed_repository_dependencies dictionary and the - # missing_repository_dependencies dictionary for proper display parsing. + # We need to add a root_key entry to one or both of installed_repository_dependencies dictionary and the missing_repository_dependencies + # dictionary for proper display parsing. root_key = container_util.generate_repository_dependencies_key_for_repository( repository.tool_shed, repository.name, repository.owner, - repository.installed_changeset_revision ) + repository.installed_changeset_revision, + prior_installation_required ) if installed_rd_tups: installed_repository_dependencies[ 'root_key' ] = root_key installed_repository_dependencies[ root_key ] = installed_rd_tups @@ -189,7 +191,7 @@ if key in [ 'description', 'root_key' ]: continue for rd_tup in rd_tups: - tool_shed, name, owner, changeset_revision = rd_tup + tool_shed, name, owner, changeset_revision, prior_installation_required = suc.parse_repository_dependency_tuple( rd_tup ) # Updates to installed repository revisions may have occurred, so make sure to locate the appropriate repository revision if one exists. # We need to create a temporary repo_info_tuple that includes the correct repository owner which we get from the current rd_tup. The current # tuple looks like: ( description, repository_clone_url, changeset_revision, ctx_rev, repository_owner, repository_dependencies, installed_td ) @@ -197,7 +199,7 @@ tmp_repo_info_tuple = ( None, tmp_clone_url, changeset_revision, None, owner, None, None ) repository, current_changeset_revision = suc.repository_was_previously_installed( trans, tool_shed, name, tmp_repo_info_tuple ) if repository: - new_rd_tup = [ tool_shed, name, owner, changeset_revision, repository.id, repository.status ] + new_rd_tup = [ tool_shed, name, owner, changeset_revision, prior_installation_required, repository.id, repository.status ] if repository.status == trans.model.ToolShedRepository.installation_status.INSTALLED: if new_rd_tup not in installed_rd_tups: installed_rd_tups.append( new_rd_tup ) @@ -205,7 +207,7 @@ if new_rd_tup not in missing_rd_tups: missing_rd_tups.append( new_rd_tup ) else: - new_rd_tup = [ tool_shed, name, owner, changeset_revision, None, 'Never installed' ] + new_rd_tup = [ tool_shed, name, owner, changeset_revision, prior_installation_required, None, 'Never installed' ] if new_rd_tup not in missing_rd_tups: missing_rd_tups.append( new_rd_tup ) if installed_rd_tups: @@ -261,8 +263,8 @@ for key, val in repository_dependencies.items(): if key in [ 'root_key', 'description' ]: continue - toolshed, name, owner, changeset_revision = container_util.get_components_from_key( key ) - components_list = [ toolshed, name, owner, changeset_revision ] + toolshed, name, owner, changeset_revision, prior_installation_required = container_util.get_components_from_key( key ) + components_list = [ toolshed, name, owner, changeset_revision, prior_installation_required ] if components_list not in required_repository_tups: required_repository_tups.append( components_list ) for components_list in val: @@ -272,6 +274,8 @@ # The value of required_repository_tups is a list of tuples, so we need to encode it. encoded_required_repository_tups = [] for required_repository_tup in required_repository_tups: + # Convert every item in required_repository_tup to a string. + required_repository_tup = [ str( item ) for item in required_repository_tup ] encoded_required_repository_tups.append( encoding_util.encoding_sep.join( required_repository_tup ) ) encoded_required_repository_str = encoding_util.encoding_sep2.join( encoded_required_repository_tups ) encoded_required_repository_str = encoding_util.tool_shed_encode( encoded_required_repository_str ) diff -r d62c5edf1ef265000740cac2949f90c9b175db6c -r 715dc4f4c8e218f50d08eb81dc1b3ef0453860a5 lib/tool_shed/util/metadata_util.py --- a/lib/tool_shed/util/metadata_util.py +++ b/lib/tool_shed/util/metadata_util.py @@ -169,18 +169,19 @@ def compare_repository_dependencies( ancestor_repository_dependencies, current_repository_dependencies ): """Determine if ancestor_repository_dependencies is the same as or a subset of current_repository_dependencies.""" - # The list of repository_dependencies looks something like: [["http://localhost:9009", "emboss_datatypes", "test", "ab03a2a5f407"]]. + # The list of repository_dependencies looks something like: [["http://localhost:9009", "emboss_datatypes", "test", "ab03a2a5f407", False]]. # Create a string from each tuple in the list for easier comparison. if len( ancestor_repository_dependencies ) <= len( current_repository_dependencies ): for ancestor_tup in ancestor_repository_dependencies: - ancestor_tool_shed, ancestor_repository_name, ancestor_repository_owner, ancestor_changeset_revision = ancestor_tup + ancestor_tool_shed, ancestor_repository_name, ancestor_repository_owner, ancestor_changeset_revision, ancestor_prior_installation_required = ancestor_tup found_in_current = False for current_tup in current_repository_dependencies: - current_tool_shed, current_repository_name, current_repository_owner, current_changeset_revision = current_tup + current_tool_shed, current_repository_name, current_repository_owner, current_changeset_revision, current_prior_installation_required = current_tup if current_tool_shed == ancestor_tool_shed and \ current_repository_name == ancestor_repository_name and \ current_repository_owner == ancestor_repository_owner and \ - current_changeset_revision == ancestor_changeset_revision: + current_changeset_revision == ancestor_changeset_revision and \ + current_prior_installation_required == ancestor_prior_installation_required: found_in_current = True break if not found_in_current: @@ -704,23 +705,23 @@ xml_is_valid = False if xml_is_valid: invalid_repository_dependencies_dict = dict( description=root.get( 'description' ) ) - invalid_repository_dependencies_tups = [] + invalid_repository_dependency_tups = [] valid_repository_dependencies_dict = dict( description=root.get( 'description' ) ) - valid_repository_dependencies_tups = [] + valid_repository_dependency_tups = [] for repository_elem in root.findall( 'repository' ): - repository_dependencies_tup, repository_dependency_is_valid, error_message = handle_repository_elem( app, repository_elem ) + repository_dependency_tup, repository_dependency_is_valid, error_message = handle_repository_elem( app, repository_elem ) if repository_dependency_is_valid: - valid_repository_dependencies_tups.append( repository_dependencies_tup ) + valid_repository_dependency_tups.append( repository_dependency_tup ) else: # Append the error_message to the repository dependencies tuple. - toolshed, name, owner, changeset_revision = repository_dependencies_tup - repository_dependencies_tup = ( toolshed, name, owner, changeset_revision, error_message ) - invalid_repository_dependencies_tups.append( repository_dependencies_tup ) - if invalid_repository_dependencies_tups: - invalid_repository_dependencies_dict[ 'repository_dependencies' ] = invalid_repository_dependencies_tups + toolshed, name, owner, changeset_revision, prior_installation_required = repository_dependency_tup + repository_dependency_tup = ( toolshed, name, owner, changeset_revision, prior_installation_required, error_message ) + invalid_repository_dependency_tups.append( repository_dependency_tup ) + if invalid_repository_dependency_tups: + invalid_repository_dependencies_dict[ 'repository_dependencies' ] = invalid_repository_dependency_tups metadata_dict[ 'invalid_repository_dependencies' ] = invalid_repository_dependencies_dict - if valid_repository_dependencies_tups: - valid_repository_dependencies_dict[ 'repository_dependencies' ] = valid_repository_dependencies_tups + if valid_repository_dependency_tups: + valid_repository_dependencies_dict[ 'repository_dependencies' ] = valid_repository_dependency_tups metadata_dict[ 'repository_dependencies' ] = valid_repository_dependencies_dict return metadata_dict, error_message @@ -765,8 +766,8 @@ # We have an invalid complex repository dependency, so mark the tool dependency as invalid. tool_dependency_is_valid = False # Append the error message to the invalid repository dependency tuple. - toolshed, name, owner, changeset_revision = repository_dependency_tup - repository_dependency_tup = ( toolshed, name, owner, changeset_revision, message ) + toolshed, name, owner, changeset_revision, prior_installation_required = repository_dependency_tup + repository_dependency_tup = ( toolshed, name, owner, changeset_revision, prior_installation_required, message ) invalid_repository_dependency_tups.append( repository_dependency_tup ) error_message = '%s %s' % ( error_message, message ) elif elem.tag == 'set_environment': @@ -1044,11 +1045,12 @@ sa_session = app.model.context.current is_valid = True error_message = '' - toolshed = repository_elem.attrib[ 'toolshed' ] - name = repository_elem.attrib[ 'name' ] - owner = repository_elem.attrib[ 'owner' ] - changeset_revision = repository_elem.attrib[ 'changeset_revision' ] - repository_dependencies_tup = ( toolshed, name, owner, changeset_revision ) + toolshed = repository_elem.get( 'toolshed' ) + name = repository_elem.get( 'name' ) + owner = repository_elem.get( 'owner' ) + changeset_revision = repository_elem.get( 'changeset_revision' ) + prior_installation_required = repository_elem.get( 'prior_installation_required', False ) + repository_dependency_tup = ( toolshed, name, owner, changeset_revision, prior_installation_required ) user = None repository = None if app.name == 'galaxy': @@ -1058,20 +1060,16 @@ # repository_elem, we know it is valid. repository = suc.get_repository_for_dependency_relationship( app, toolshed, name, owner, changeset_revision ) if repository: - return repository_dependencies_tup, is_valid, error_message + return repository_dependency_tup, is_valid, error_message else: # Send a request to the tool shed to retrieve appropriate additional changeset revisions with which the repository may have been installed. - #try: - # Hopefully the tool shed is accessible. text = get_updated_changeset_revisions_from_tool_shed( toolshed, name, owner, changeset_revision ) - #except: - # text = None if text: updated_changeset_revisions = util.listify( text ) for updated_changeset_revision in updated_changeset_revisions: repository = suc.get_repository_for_dependency_relationship( app, toolshed, name, owner, updated_changeset_revision ) if repository: - return repository_dependencies_tup, is_valid, error_message + return repository_dependency_tup, is_valid, error_message # We'll currently default to setting the repository dependency definition as invalid if an installed repository cannot be found. # This may not be ideal because the tool shed may have simply been inaccessible when metadata was being generated for the installed # tool shed repository. @@ -1079,7 +1077,7 @@ ( toolshed, name, owner, changeset_revision ) log.debug( error_message ) is_valid = False - return repository_dependencies_tup, is_valid, error_message + return repository_dependency_tup, is_valid, error_message else: # We're in the tool shed. if suc.tool_shed_is_this_tool_shed( toolshed ): @@ -1093,7 +1091,7 @@ error_message += "because the owner is invalid. " log.debug( error_message ) is_valid = False - return repository_dependencies_tup, is_valid, error_message + return repository_dependency_tup, is_valid, error_message try: repository = sa_session.query( app.model.Repository ) \ .filter( and_( app.model.Repository.table.c.name == name, @@ -1105,7 +1103,7 @@ error_message += "because the name is invalid. " log.debug( error_message ) is_valid = False - return repository_dependencies_tup, is_valid, error_message + return repository_dependency_tup, is_valid, error_message # Find the specified changeset revision in the repository's changelog to see if it's valid. found = False repo = hg.repository( suc.get_configured_ui(), repository.repo_path( app ) ) @@ -1120,15 +1118,15 @@ error_message += "because the changeset revision is invalid. " log.debug( error_message ) is_valid = False - return repository_dependencies_tup, is_valid, error_message + return repository_dependency_tup, is_valid, error_message else: # Repository dependencies are currently supported within a single tool shed. error_message = "Repository dependencies are currently supported only within the same tool shed. Ignoring repository dependency definition " error_message += "for tool shed %s, name %s, owner %s, changeset revision %s. " % ( toolshed, name, owner, changeset_revision ) log.debug( error_message ) is_valid = False - return repository_dependencies_tup, is_valid, error_message - return repository_dependencies_tup, is_valid, error_message + return repository_dependency_tup, is_valid, error_message + return repository_dependency_tup, is_valid, error_message def is_downloadable( metadata_dict ): # NOTE: although repository README files are considered Galaxy utilities, they have no effect on determining if a revision is installable. @@ -1851,13 +1849,15 @@ repository_dependencies_dict = metadata.get( 'invalid_repository_dependencies', None ) for repository_dependency_tup in repository_dependency_tups: if is_valid: - tool_shed, name, owner, changeset_revision = repository_dependency_tup + tool_shed, name, owner, changeset_revision, prior_installation_required = repository_dependency_tup else: - tool_shed, name, owner, changeset_revision, error_message = repository_dependency_tup + tool_shed, name, owner, changeset_revision, prior_installation_required, error_message = repository_dependency_tup + prior_installation_required = util.asbool( str( prior_installation_required ) ) rd_key = container_util.generate_repository_dependencies_key_for_repository( toolshed_base_url=tool_shed, repository_name=name, repository_owner=owner, - changeset_revision=changeset_revision ) + changeset_revision=changeset_revision, + prior_installation_required=prior_installation_required ) if repository_dependencies_dict: if rd_key in repository_dependencies_dict: repository_dependencies = repository_dependencies_dict[ rd_key ] diff -r d62c5edf1ef265000740cac2949f90c9b175db6c -r 715dc4f4c8e218f50d08eb81dc1b3ef0453860a5 lib/tool_shed/util/repository_dependency_util.py --- a/lib/tool_shed/util/repository_dependency_util.py +++ b/lib/tool_shed/util/repository_dependency_util.py @@ -33,7 +33,8 @@ if key in [ 'root_key', 'description' ]: continue dependent_repository = None - dependent_toolshed, dependent_name, dependent_owner, dependent_changeset_revision = container_util.get_components_from_key( key ) + dependent_toolshed, dependent_name, dependent_owner, dependent_changeset_revision, dependent_prior_installation_required = \ + container_util.get_components_from_key( key ) for tsr in tool_shed_repositories: # Get the the tool_shed_repository defined by name, owner and changeset_revision. This is the repository that will be # dependent upon each of the tool shed repositories contained in val. @@ -47,7 +48,8 @@ # Process each repository_dependency defined for the current dependent repository. for repository_dependency_components_list in val: required_repository = None - rd_toolshed, rd_name, rd_owner, rd_changeset_revision = repository_dependency_components_list + rd_toolshed, rd_name, rd_owner, rd_changeset_revision, prior_installation_required = \ + suc.parse_repository_dependency_tuple( repository_dependency_components_list ) # Get the the tool_shed_repository defined by rd_name, rd_owner and rd_changeset_revision. This is the repository that will be # required by the current dependent_repository. # TODO: Check tool_shed_repository.tool_shed as well when repository dependencies across tool sheds is supported. @@ -244,7 +246,8 @@ if invalid_repository_dependencies_dict: invalid_repository_dependencies = invalid_repository_dependencies_dict[ 'invalid_repository_dependencies' ] for repository_dependency_tup in invalid_repository_dependencies: - toolshed, name, owner, changeset_revision, error = repository_dependency_tup + toolshed, name, owner, changeset_revision, prior_installation_required, error = \ + suc.parse_repository_dependency_tuple( repository_dependency_tup, contains_error=True ) if error: message = '%s ' % str( error ) return message @@ -253,7 +256,8 @@ return container_util.generate_repository_dependencies_key_for_repository( toolshed_base_url=toolshed_base_url, repository_name=repository.name, repository_owner=repository.user.username, - changeset_revision=repository_metadata.changeset_revision ) + changeset_revision=repository_metadata.changeset_revision, + prior_installation_required=False ) def get_repository_dependencies_for_changeset_revision( trans, repository, repository_metadata, toolshed_base_url, key_rd_dicts_to_be_processed=None, all_repository_dependencies=None, @@ -320,7 +324,7 @@ for key_rd_dict in key_rd_dicts: key = key_rd_dict.keys()[ 0 ] repository_dependency = key_rd_dict[ key ] - rd_toolshed, rd_name, rd_owner, rd_changeset_revision = repository_dependency + rd_toolshed, rd_name, rd_owner, rd_changeset_revision, rd_prior_installation_required = suc.parse_repository_dependency_tuple( repository_dependency ) if suc.tool_shed_is_this_tool_shed( rd_toolshed ): repository = suc.get_repository_by_name_and_owner( trans.app, rd_name, rd_owner ) if repository: @@ -342,16 +346,17 @@ changeset_revision ) if repository_metadata: new_key_rd_dict = {} - new_key_rd_dict[ key ] = [ rd_toolshed, rd_name, rd_owner, repository_metadata.changeset_revision ] + new_key_rd_dict[ key ] = [ rd_toolshed, rd_name, rd_owner, repository_metadata.changeset_revision, rd_prior_installation_required ] # We have the updated changset revision. updated_key_rd_dicts.append( new_key_rd_dict ) else: - toolshed, repository_name, repository_owner, repository_changeset_revision = container_util.get_components_from_key( key ) + toolshed, repository_name, repository_owner, repository_changeset_revision, prior_installation_required = \ + container_util.get_components_from_key( key ) message = "The revision %s defined for repository %s owned by %s is invalid, so repository dependencies defined for repository %s will be ignored." % \ ( str( rd_changeset_revision ), str( rd_name ), str( rd_owner ), str( repository_name ) ) log.debug( message ) else: - toolshed, repository_name, repository_owner, repository_changeset_revision = container_util.get_components_from_key( key ) + toolshed, repository_name, repository_owner, repository_changeset_revision, prior_installation_required = container_util.get_components_from_key( key ) message = "The revision %s defined for repository %s owned by %s is invalid, so repository dependencies defined for repository %s will be ignored." % \ ( str( rd_changeset_revision ), str( rd_name ), str( rd_owner ), str( repository_name ) ) log.debug( message ) @@ -397,7 +402,7 @@ def handle_key_rd_dicts_for_repository( trans, current_repository_key, repository_key_rd_dicts, key_rd_dicts_to_be_processed, handled_key_rd_dicts, circular_repository_dependencies ): key_rd_dict = repository_key_rd_dicts.pop( 0 ) repository_dependency = key_rd_dict[ current_repository_key ] - toolshed, name, owner, changeset_revision = repository_dependency + toolshed, name, owner, changeset_revision, prior_installation_required = suc.parse_repository_dependency_tuple( repository_dependency ) if suc.tool_shed_is_this_tool_shed( toolshed ): required_repository = suc.get_repository_by_name_and_owner( trans.app, name, owner ) required_repository_metadata = metadata_util.get_repository_metadata_by_repository_id_changeset_revision( trans, @@ -444,7 +449,7 @@ circular_repository_dependencies=circular_repository_dependencies ) def in_all_repository_dependencies( repository_key, repository_dependency, all_repository_dependencies ): - """Return True if { repository_key :repository_dependency } is in all_repository_dependencies.""" + """Return True if { repository_key : repository_dependency } is in all_repository_dependencies.""" for key, val in all_repository_dependencies.items(): if key != repository_key: continue @@ -455,7 +460,7 @@ def in_circular_repository_dependencies( repository_key_rd_dict, circular_repository_dependencies ): """ Return True if any combination of a circular dependency tuple is the key : value pair defined in the received repository_key_rd_dict. This - means that each circular dependency tuple is converted into the key : value pair for vomparision. + means that each circular dependency tuple is converted into the key : value pair for comparison. """ for tup in circular_repository_dependencies: rd_0, rd_1 = tup @@ -468,7 +473,8 @@ return False def initialize_all_repository_dependencies( current_repository_key, repository_dependencies_dict, all_repository_dependencies ): - # Initialize the all_repository_dependencies dictionary. It's safe to assume that current_repository_key in this case will have a value. + """Initialize the all_repository_dependencies dictionary.""" + # It's safe to assume that current_repository_key in this case will have a value. all_repository_dependencies[ 'root_key' ] = current_repository_key all_repository_dependencies[ current_repository_key ] = [] # Store the value of the 'description' key only once, the first time through this recursive method. @@ -477,6 +483,7 @@ return all_repository_dependencies def in_key_rd_dicts( key_rd_dict, key_rd_dicts ): + """Return True if key_rd_dict is contained in the list of key_rd_dicts.""" k = key_rd_dict.keys()[ 0 ] v = key_rd_dict[ k ] for key_rd_dict in key_rd_dicts: @@ -513,7 +520,7 @@ return False def merge_missing_repository_dependencies_to_installed_container( containers_dict ): - """ Merge the list of missing repository dependencies into the list of installed repository dependencies.""" + """Merge the list of missing repository dependencies into the list of installed repository dependencies.""" missing_rd_container_root = containers_dict.get( 'missing_repository_dependencies', None ) if missing_rd_container_root: # The missing_rd_container_root will be a root folder containing a single sub_folder. @@ -542,6 +549,11 @@ def populate_repository_dependency_objects_for_processing( trans, current_repository_key, repository_dependencies_dict, key_rd_dicts_to_be_processed, handled_key_rd_dicts, circular_repository_dependencies, all_repository_dependencies ): + """ + The process that discovers all repository dependencies for a specified repository's changeset revision uses this method to populate the following items + for the current processing loop: filtered_current_repository_key_rd_dicts, key_rd_dicts_to_be_processed, handled_key_rd_dicts, all_repository_dependencies. + Each processing loop may discover more repository dependencies, so this method is repeatedly called until all repository dependencies have been discovered. + """ current_repository_key_rd_dicts = [] filtered_current_repository_key_rd_dicts = [] for rd in repository_dependencies_dict[ 'repository_dependencies' ]: @@ -602,6 +614,7 @@ return valid_repository_dependencies def remove_from_key_rd_dicts( key_rd_dict, key_rd_dicts ): + """Eliminate the key_rd_dict from the list of key_rd_dicts if it is contained in the list.""" k = key_rd_dict.keys()[ 0 ] v = key_rd_dict[ k ] clean_key_rd_dicts = [] @@ -618,11 +631,11 @@ clean_key_rd_dicts = [] key = key_rd_dicts[ 0 ].keys()[ 0 ] repository_tup = key.split( container_util.STRSEP ) - rd_toolshed, rd_name, rd_owner, rd_changeset_revision = repository_tup + rd_toolshed, rd_name, rd_owner, rd_changeset_revision, rd_prior_installation_required = suc.parse_repository_dependency_tuple( repository_tup ) for key_rd_dict in key_rd_dicts: k = key_rd_dict.keys()[ 0 ] repository_dependency = key_rd_dict[ k ] - toolshed, name, owner, changeset_revision = repository_dependency + toolshed, name, owner, changeset_revision, prior_installation_required = suc.parse_repository_dependency_tuple( repository_dependency ) if rd_toolshed == toolshed and rd_name == name and rd_owner == owner: log.debug( "Removing repository dependency for repository %s owned by %s since it refers to a revision within itself." % ( name, owner ) ) else: @@ -631,44 +644,9 @@ clean_key_rd_dicts.append( new_key_rd_dict ) return clean_key_rd_dicts -def repository_dependencies_have_tool_dependencies( trans, repository_dependencies ): - rd_tups_processed = [] - for key, rd_tups in repository_dependencies.items(): - if key in [ 'root_key', 'description' ]: - continue - rd_tup = container_util.get_components_from_key( key ) - if rd_tup not in rd_tups_processed: - toolshed, name, owner, changeset_revision = rd_tup - repository = suc.get_repository_by_name_and_owner( trans.app, name, owner ) - repository_metadata = metadata_util.get_repository_metadata_by_repository_id_changeset_revision( trans, - trans.security.encode_id( repository.id ), - changeset_revision ) - if repository_metadata: - metadata = repository_metadata.metadata - if metadata: - if 'tool_dependencies' in metadata: - return True - rd_tups_processed.append( rd_tup ) - for rd_tup in rd_tups: - if rd_tup not in rd_tups_processed: - toolshed, name, owner, changeset_revision = rd_tup - repository = suc.get_repository_by_name_and_owner( trans.app, name, owner ) - repository_metadata = metadata_util.get_repository_metadata_by_repository_id_changeset_revision( trans, - trans.security.encode_id( repository.id ), - changeset_revision ) - if repository_metadata: - metadata = repository_metadata.metadata - if metadata: - if 'tool_dependencies' in metadata: - return True - rd_tups_processed.append( rd_tup ) - return False - def get_repository_dependency_as_key( repository_dependency ): - return container_util.generate_repository_dependencies_key_for_repository( repository_dependency[ 0 ], - repository_dependency[ 1 ], - repository_dependency[ 2 ], - repository_dependency[ 3 ] ) + tool_shed, name, owner, changeset_revision, prior_installation_required = suc.parse_repository_dependency_tuple( repository_dependency ) + return container_util.generate_repository_dependencies_key_for_repository( tool_shed, name, owner, changeset_revision, prior_installation_required ) def get_repository_dependency_by_repository_id( trans, decoded_repository_id ): return trans.sa_session.query( trans.model.RepositoryDependency ) \ diff -r d62c5edf1ef265000740cac2949f90c9b175db6c -r 715dc4f4c8e218f50d08eb81dc1b3ef0453860a5 lib/tool_shed/util/shed_util_common.py --- a/lib/tool_shed/util/shed_util_common.py +++ b/lib/tool_shed/util/shed_util_common.py @@ -272,8 +272,8 @@ def generate_clone_url_from_repo_info_tup( repo_info_tup ): """Generate teh URL for cloning a repositoyr given a tuple of toolshed, name, owner, changeset_revision.""" - # Example tuple: ['http://localhost:9009', 'blast_datatypes', 'test', '461a4216e8ab'] - toolshed, name, owner, changeset_revision = repo_info_tup + # Example tuple: ['http://localhost:9009', 'blast_datatypes', 'test', '461a4216e8ab', False] + toolshed, name, owner, changeset_revision, prior_installation_required = parse_repository_dependency_tuple( repo_info_tup ) # Don't include the changeset_revision in clone urls. return url_join( toolshed, 'repos', owner, name ) @@ -1019,6 +1019,28 @@ folder_contents.append( node ) return folder_contents +def parse_repository_dependency_tuple( repository_dependency_tuple, contains_error=False ): + if contains_error: + if len( repository_dependency_tuple ) == 5: + # Metadata should have been reset on the repository containing this repository_dependency definition. + tool_shed, name, owner, changeset_revision, error = repository_dependency_tuple + # Default prior_installation_required to False. + prior_installation_required = False + elif len( repository_dependency_tuple ) == 6: + toolshed, name, owner, changeset_revision, prior_installation_required, error = repository_dependency_tuple + prior_installation_required = util.asbool( str( prior_installation_required ) ) + return toolshed, name, owner, changeset_revision, prior_installation_required, error + else: + if len( repository_dependency_tuple ) == 4: + # Metadata should have been reset on the repository containing this repository_dependency definition. + tool_shed, name, owner, changeset_revision = repository_dependency_tuple + # Default prior_installation_required to False. + prior_installation_required = False + elif len( repository_dependency_tuple ) == 5: + tool_shed, name, owner, changeset_revision, prior_installation_required = repository_dependency_tuple + prior_installation_required = util.asbool( str( prior_installation_required ) ) + return tool_shed, name, owner, changeset_revision, prior_installation_required + def remove_dir( dir ): """Attempt to remove a directory from disk.""" if os.path.exists( dir ): @@ -1097,6 +1119,19 @@ trans.sa_session.add( repository ) trans.sa_session.flush() +def set_prior_installation_required( repository, required_repository ): + """Return True if the received required_repository must be installed before the received repository.""" + required_repository_tup = [ required_repository.tool_shed, required_repository.name, required_repository.owner, required_repository.changeset_revision ] + # Get the list of repository dependency tuples associated with the received repository where prior_installation_required is True. + required_rd_tups_that_must_be_installed = repository.requires_prior_installation_of + for required_rd_tup in required_rd_tups_that_must_be_installed: + # Repository dependency tuples in metadata include a prior_installation_required value, so strip it for comparision. + partial_required_rd_tup = required_rd_tup[ 0:4 ] + if partial_required_rd_tup == required_repository_tup: + # Return the boolean value of prior_installation_required, which defaults to False. + return required_rd_tup[ 4 ] + return False + def strip_path( fpath ): """Attempt to strip the path from a file name.""" if not fpath: diff -r d62c5edf1ef265000740cac2949f90c9b175db6c -r 715dc4f4c8e218f50d08eb81dc1b3ef0453860a5 templates/webapps/tool_shed/repository/common.mako --- a/templates/webapps/tool_shed/repository/common.mako +++ b/templates/webapps/tool_shed/repository/common.mako @@ -513,8 +513,12 @@ else: installation_status = None repository_name = str( repository_dependency.repository_name ) + repository_owner = str( repository_dependency.repository_owner ) changeset_revision = str( repository_dependency.changeset_revision ) - repository_owner = str( repository_dependency.repository_owner ) + if prior_installation_required: + prior_installation_required_str = " <i>(prior install required)</i>" + else: + prior_installation_required_str = "" if trans.webapp.name == 'galaxy': if row_is_header: @@ -550,7 +554,7 @@ </${cell_type}> %else: <td style="padding-left: ${pad+20}px;"> - Repository <b>${repository_name | h}</b> revision <b>${changeset_revision | h}</b> owned by <b>${repository_owner | h}</b> + Repository <b>${repository_name | h}</b> revision <b>${changeset_revision | h}</b> owned by <b>${repository_owner | h}</b>${prior_installation_required_str} </td> %endif </tr> Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: 3 new changesets
by commits-noreply＠bitbucket.org 02 Apr '13

02 Apr '13

3 new commits in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/d6ab0870547c/ Changeset: d6ab0870547c User: dannon Date: 2013-04-02 20:41:10 Summary: Library controller code cleanup, imports and spacing. Affected #: 1 file diff -r f03401b61bc6825131b163b9bb8cf1acf8ac48c3 -r d6ab0870547c5342b101d2a97513da699ef4ae39 lib/galaxy/webapps/galaxy/controllers/library.py --- a/lib/galaxy/webapps/galaxy/controllers/library.py +++ b/lib/galaxy/webapps/galaxy/controllers/library.py @@ -1,24 +1,27 @@ -import urllib, urllib2 +import logging +from galaxy import model, util +from galaxy import web +from galaxy.model.orm import and_, not_, or_ +from galaxy.web.base.controller import BaseUIController +from galaxy.web.framework.helpers import grids +from library_common import get_comptypes, lucene_search, whoosh_search -from galaxy.web.base.controller import * -from galaxy.web.framework.helpers import time_ago, iff, grids -from galaxy.model.orm import * -from galaxy.datatypes import sniff -from galaxy import model, util -from galaxy.util.odict import odict -from library_common import get_comptypes, lucene_search, whoosh_search log = logging.getLogger( __name__ ) + class LibraryListGrid( grids.Grid ): + class NameColumn( grids.TextColumn ): def get_value( self, trans, grid, library ): return library.name + class DescriptionColumn( grids.TextColumn ): def get_value( self, trans, grid, library ): if library.description: return library.description return '' + # Grid definition title = "Data Libraries" model_class = model.Library @@ -45,8 +48,10 @@ num_rows_per_page = 50 preserve_state = False use_paging = True + def build_initial_query( self, trans, **kwargs ): return trans.sa_session.query( self.model_class ).filter( self.model_class.table.c.deleted == False ) + def apply_query_filter( self, trans, query, **kwd ): current_user_role_ids = [ role.id for role in trans.get_current_user_roles() ] library_access_action = trans.app.security_agent.permitted_actions.LIBRARY_ACCESS.action @@ -65,8 +70,10 @@ # public libraries and restricted libraries accessible by the current user. return query.filter( or_( not_( trans.model.Library.table.c.id.in_( restricted_library_ids ) ), trans.model.Library.table.c.id.in_( accessible_restricted_library_ids ) ) ) + + class Library( BaseUIController ): - + library_list_grid = LibraryListGrid() @web.expose https://bitbucket.org/galaxy/galaxy-central/commits/8a84382c5c33/ Changeset: 8a84382c5c33 User: dannon Date: 2013-04-02 20:41:55 Summary: Trim trailing whitespace. Affected #: 1 file diff -r d6ab0870547c5342b101d2a97513da699ef4ae39 -r 8a84382c5c339eee6c866301bae613bb4a2b0595 lib/galaxy/webapps/galaxy/controllers/library.py --- a/lib/galaxy/webapps/galaxy/controllers/library.py +++ b/lib/galaxy/webapps/galaxy/controllers/library.py @@ -38,8 +38,8 @@ attach_popup=False, filterable="advanced" ), ] - columns.append( grids.MulticolFilterColumn( "search dataset name, info, message, dbkey", - cols_to_filter=[ columns[0], columns[1] ], + columns.append( grids.MulticolFilterColumn( "search dataset name, info, message, dbkey", + cols_to_filter=[ columns[0], columns[1] ], key="free-text-search", visible=False, filterable="standard" ) ) @@ -62,11 +62,11 @@ .filter( and_( trans.model.LibraryPermissions.table.c.action == library_access_action, trans.model.LibraryPermissions.table.c.role_id.in_( current_user_role_ids ) ) ) ] if not trans.user: - # Filter to get only public libraries, a library whose id + # Filter to get only public libraries, a library whose id # is not in restricted_library_ids is a public library return query.filter( not_( trans.model.Library.table.c.id.in_( restricted_library_ids ) ) ) else: - # Filter to get libraries accessible by the current user, get both + # Filter to get libraries accessible by the current user, get both # public libraries and restricted libraries accessible by the current user. return query.filter( or_( not_( trans.model.Library.table.c.id.in_( restricted_library_ids ) ), trans.model.Library.table.c.id.in_( accessible_restricted_library_ids ) ) ) https://bitbucket.org/galaxy/galaxy-central/commits/d62c5edf1ef2/ Changeset: d62c5edf1ef2 User: dannon Date: 2013-04-02 20:47:12 Summary: Merge Affected #: 6 files This diff is so big that we needed to truncate the remainder. Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: carlfeberhard: That went well. (rollback changes to backbone, backbone-relational, and handlebars.runtime to not break every possible thing)
by commits-noreply＠bitbucket.org 02 Apr '13

02 Apr '13

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/23fd5251f5ec/ Changeset: 23fd5251f5ec User: carlfeberhard Date: 2013-04-02 20:41:29 Summary: That went well. (rollback changes to backbone, backbone-relational, and handlebars.runtime to not break every possible thing) Affected #: 6 files This diff is so big that we needed to truncate the remainder. Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: dannon: api/users import cleanup
by commits-noreply＠bitbucket.org 02 Apr '13

02 Apr '13

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/f03401b61bc6/ Changeset: f03401b61bc6 User: dannon Date: 2013-04-02 20:36:50 Summary: api/users import cleanup Affected #: 1 file diff -r 8effd49e233eccfb0cc4f11cc1ff67da13cbb655 -r f03401b61bc6825131b163b9bb8cf1acf8ac48c3 lib/galaxy/webapps/galaxy/api/users.py --- a/lib/galaxy/webapps/galaxy/api/users.py +++ b/lib/galaxy/webapps/galaxy/api/users.py @@ -2,10 +2,9 @@ API operations on User objects. """ import logging +from paste.httpexceptions import HTTPBadRequest, HTTPNotImplemented +from galaxy import util, web from galaxy.web.base.controller import BaseAPIController, url_for -from galaxy import web, util -from elementtree.ElementTree import XML -from paste.httpexceptions import * log = logging.getLogger( __name__ ) Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: dannon: api/histories import cleanup, remove unused Params cruft.
by commits-noreply＠bitbucket.org 02 Apr '13

02 Apr '13

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/8effd49e233e/ Changeset: 8effd49e233e User: dannon Date: 2013-04-02 20:35:18 Summary: api/histories import cleanup, remove unused Params cruft. Affected #: 1 file diff -r c554d8795650fc155b2b2610bed3fe4f1e31c1f1 -r 8effd49e233eccfb0cc4f11cc1ff67da13cbb655 lib/galaxy/webapps/galaxy/api/histories.py --- a/lib/galaxy/webapps/galaxy/api/histories.py +++ b/lib/galaxy/webapps/galaxy/api/histories.py @@ -1,19 +1,11 @@ """ API operations on a history. """ -import logging, os, string, shutil, urllib, re, socket -from cgi import escape, FieldStorage -from galaxy import util, datatypes, jobs, web, util +import logging +from galaxy import web, util from galaxy.web.base.controller import BaseAPIController, UsesHistoryMixin from galaxy.web import url_for -from galaxy.util.sanitize_html import sanitize_html -from galaxy.model.orm import * -import galaxy.datatypes -from galaxy.util.bunch import Bunch - -import pkg_resources -pkg_resources.require( "Routes" ) -import routes +from galaxy.model.orm import desc log = logging.getLogger( __name__ ) @@ -59,7 +51,6 @@ Displays information about a history. """ history_id = id - params = util.Params( kwd ) deleted = util.string_as_bool( deleted ) # try to load the history, by most_recently_used or the given id @@ -91,7 +82,6 @@ POST /api/histories Creates a new history. """ - params = util.Params( payload ) hist_name = None if payload.get( 'name', None ): hist_name = util.restore_text( payload['name'] ) Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: carlfeberhard: update js libs: underscore 1.3.1 -> 1.4.4, backbone 0.9.2 -> 1.0, backbone-relational 0.5.0 -> 0.8.0, handlebars 1.0.beta.6 -> 1.0.rc1
by commits-noreply＠bitbucket.org 02 Apr '13

02 Apr '13

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/c554d8795650/ Changeset: c554d8795650 User: carlfeberhard Date: 2013-04-02 20:27:23 Summary: update js libs: underscore 1.3.1 -> 1.4.4, backbone 0.9.2 -> 1.0, backbone-relational 0.5.0 -> 0.8.0, handlebars 1.0.beta.6 -> 1.0.rc1 Affected #: 8 files This diff is so big that we needed to truncate the remainder. Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: carlfeberhard: pack_scripts.py: have yuicompressor use utf-8 as the charset; remove unused packed scripts; update d3.js from 2.9.2 to 3.1.4
by commits-noreply＠bitbucket.org 02 Apr '13

02 Apr '13

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/1207f1c95e70/ Changeset: 1207f1c95e70 User: carlfeberhard Date: 2013-04-02 20:20:55 Summary: pack_scripts.py: have yuicompressor use utf-8 as the charset; remove unused packed scripts; update d3.js from 2.9.2 to 3.1.4 Affected #: 8 files This diff is so big that we needed to truncate the remainder. Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: 2 new changesets
by commits-noreply＠bitbucket.org 02 Apr '13

02 Apr '13

2 new commits in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/1c92aed71881/ Changeset: 1c92aed71881 Branch: stable User: inithello Date: 2013-04-02 19:34:25 Summary: Enable scrolling for the left panel in the tool shed. Fix for determining if a repository dependency can be added to a container. Affected #: 3 files diff -r 9264cf7148c08082d0706c0cf4b74f6a9332dc4d -r 1c92aed71881e3a15648e3be9478a69f76c50498 lib/tool_shed/util/repository_dependency_util.py --- a/lib/tool_shed/util/repository_dependency_util.py +++ b/lib/tool_shed/util/repository_dependency_util.py @@ -85,10 +85,9 @@ for kr_dict in key_rd_dicts: key = kr_dict.keys()[ 0 ] if key == k: - val = kr_dict[ key ] - for repository_dependency in val: - if repository_dependency[ 0:3 ] == partial_rd: - return False + repository_dependency = kr_dict[ key ] + if repository_dependency[ 0:3 ] == partial_rd: + return False return True def create_repository_dependency_objects( trans, tool_path, tool_shed_url, repo_info_dicts, reinstalling=False, install_repository_dependencies=False, diff -r 9264cf7148c08082d0706c0cf4b74f6a9332dc4d -r 1c92aed71881e3a15648e3be9478a69f76c50498 templates/webapps/tool_shed/admin/index.mako --- a/templates/webapps/tool_shed/admin/index.mako +++ b/templates/webapps/tool_shed/admin/index.mako @@ -40,7 +40,7 @@ <div class="unified-panel-header" unselectable="on"><div class='unified-panel-header-inner'>Administration</div></div> - <div class="page-container" style="padding: 10px;"> + <div class="unified-panel-body" style="padding: 10px; overflow: auto;"><div class="toolMenu"><div class="toolSectionList"><div class="toolSectionTitle"> diff -r 9264cf7148c08082d0706c0cf4b74f6a9332dc4d -r 1c92aed71881e3a15648e3be9478a69f76c50498 templates/webapps/tool_shed/index.mako --- a/templates/webapps/tool_shed/index.mako +++ b/templates/webapps/tool_shed/index.mako @@ -41,7 +41,7 @@ <div class="unified-panel-header" unselectable="on"><div class='unified-panel-header-inner'>${trans.app.shed_counter.valid_tools | h} valid tools on ${trans.app.shed_counter.generation_time | h}</div></div> - <div class="page-container" style="padding: 10px;"> + <div class="unified-panel-body" style="padding: 10px; overflow: auto;"><div class="toolMenu"><div class="toolSectionList"> %if user_id or repository_id: https://bitbucket.org/galaxy/galaxy-central/commits/f9d8a99135ce/ Changeset: f9d8a99135ce User: inithello Date: 2013-04-02 19:34:53 Summary: Pull in changes from stable branch. Affected #: 3 files diff -r fe538afce7d41adec8c60f197c226e92702225f4 -r f9d8a99135ce4be3ae55459ecf177a382fe4c93a lib/tool_shed/util/repository_dependency_util.py --- a/lib/tool_shed/util/repository_dependency_util.py +++ b/lib/tool_shed/util/repository_dependency_util.py @@ -85,10 +85,9 @@ for kr_dict in key_rd_dicts: key = kr_dict.keys()[ 0 ] if key == k: - val = kr_dict[ key ] - for repository_dependency in val: - if repository_dependency[ 0:3 ] == partial_rd: - return False + repository_dependency = kr_dict[ key ] + if repository_dependency[ 0:3 ] == partial_rd: + return False return True def create_repository_dependency_objects( trans, tool_path, tool_shed_url, repo_info_dicts, reinstalling=False, install_repository_dependencies=False, diff -r fe538afce7d41adec8c60f197c226e92702225f4 -r f9d8a99135ce4be3ae55459ecf177a382fe4c93a templates/webapps/tool_shed/admin/index.mako --- a/templates/webapps/tool_shed/admin/index.mako +++ b/templates/webapps/tool_shed/admin/index.mako @@ -40,7 +40,7 @@ <div class="unified-panel-header" unselectable="on"><div class='unified-panel-header-inner'>Administration</div></div> - <div class="page-container" style="padding: 10px;"> + <div class="unified-panel-body" style="padding: 10px; overflow: auto;"><div class="toolMenu"><div class="toolSectionList"><div class="toolSectionTitle"> diff -r fe538afce7d41adec8c60f197c226e92702225f4 -r f9d8a99135ce4be3ae55459ecf177a382fe4c93a templates/webapps/tool_shed/index.mako --- a/templates/webapps/tool_shed/index.mako +++ b/templates/webapps/tool_shed/index.mako @@ -41,7 +41,7 @@ <div class="unified-panel-header" unselectable="on"><div class='unified-panel-header-inner'>${trans.app.shed_counter.valid_tools | h} valid tools on ${trans.app.shed_counter.generation_time | h}</div></div> - <div class="page-container" style="padding: 10px;"> + <div class="unified-panel-body" style="padding: 10px; overflow: auto;"><div class="toolMenu"><div class="toolSectionList"> %if user_id or repository_id: Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: inithello: Fix for adding new entries to a tool data table that is already present in app.tool_data_tables.
by commits-noreply＠bitbucket.org 02 Apr '13

02 Apr '13

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/fe538afce7d4/ Changeset: fe538afce7d4 User: inithello Date: 2013-04-02 17:47:38 Summary: Fix for adding new entries to a tool data table that is already present in app.tool_data_tables. Affected #: 1 file diff -r eea87f210c46b6bc9cc7e82e0f12c13bfa6f1e5e -r fe538afce7d41adec8c60f197c226e92702225f4 lib/galaxy/tools/data/__init__.py --- a/lib/galaxy/tools/data/__init__.py +++ b/lib/galaxy/tools/data/__init__.py @@ -59,6 +59,11 @@ if table.name not in self.data_tables: self.data_tables[ table.name ] = table log.debug( "Loaded tool data table '%s'", table.name ) + else: + for table_row in table.data: + # FIXME: This does not account for an entry with the same unique build ID, but a different path. + if table_row not in self.data_tables[ table.name ].data: + self.data_tables[ table.name ].data.append( table_row ) return table_elems def add_new_entries_from_config_file( self, config_filename, tool_data_path, shed_tool_data_table_config, persist=False ): """ Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0