1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/ea3da2000fa7/ changeset: ea3da2000fa7 user: greg date: 2013-01-30 22:29:37 summary: Support for installation and administration of complex repository dependencies in Galaxy. affected #: 7 files diff -r 618c34d9b2eab385e092c9ca2d7dd07b9ea31024 -r ea3da2000fa733a2d3ff5d5629dec58d3573645c lib/galaxy/model/__init__.py --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -3153,6 +3153,11 @@ def can_reinstall_or_activate( self ): return self.deleted @property + def has_readme_files( self ): + if self.metadata: + return 'readme_files' in self.metadata + return False + @property def has_repository_dependencies( self ): if self.metadata: return 'repository_dependencies' in self.metadata @@ -3176,11 +3181,6 @@ def in_error_state( self ): return self.status == self.installation_status.ERROR @property - def has_readme_files( self ): - if self.metadata: - return 'readme_files' in self.metadata - return False - @property def repository_dependencies( self ): required_repositories = [] for rrda in self.required_repositories: diff -r 618c34d9b2eab385e092c9ca2d7dd07b9ea31024 -r ea3da2000fa733a2d3ff5d5629dec58d3573645c lib/galaxy/tool_shed/tool_dependencies/fabric_util.py --- a/lib/galaxy/tool_shed/tool_dependencies/fabric_util.py +++ b/lib/galaxy/tool_shed/tool_dependencies/fabric_util.py @@ -50,6 +50,7 @@ install_dir = actions_dict[ 'install_dir' ] package_name = actions_dict[ 'package_name' ] actions = actions_dict.get( 'actions', None ) + filtered_actions = [] if actions: with make_tmp_dir() as work_dir: with lcd( work_dir ): @@ -57,6 +58,8 @@ # are currently only two supported processes; download_by_url and clone via a "shell_command" action type. action_type, action_dict = actions[ 0 ] if action_type == 'download_by_url': + # Eliminate the download_by_url action so remaining actions can be processed correctly. + filtered_actions = actions[ 1: ] url = action_dict[ 'url' ] if 'target_filename' in action_dict: downloaded_filename = action_dict[ 'target_filename' ] @@ -75,15 +78,24 @@ dir = work_dir elif action_type == 'shell_command': # <action type="shell_command">git clone --recursive git://github.com/ekg/freebayes.git</action> + # Eliminate the shell_command clone action so remaining actions can be processed correctly. + filtered_actions = actions[ 1: ] return_code = handle_command( app, tool_dependency, install_dir, action_dict[ 'command' ] ) if return_code: return dir = package_name + else: + # We're handling a complex repository dependency where we only have a set_environment tag set. + # <action type="set_environment"> + # <environment_variable name="PATH" action="prepend_to">$INSTALL_DIR/bin</environment_variable> + # </action> + filtered_actions = [ a for a in actions ] + dir = install_dir if not os.path.exists( dir ): os.makedirs( dir ) # The package has been down-loaded, so we can now perform all of the actions defined for building it. with lcd( dir ): - for action_tup in actions[ 1: ]: + for action_tup in filtered_actions: action_type, action_dict = action_tup current_dir = os.path.abspath( os.path.join( work_dir, dir ) ) if action_type == 'make_directory': @@ -93,6 +105,8 @@ source_dir=os.path.join( action_dict[ 'source_directory' ] ), destination_dir=os.path.join( action_dict[ 'destination_directory' ] ) ) elif action_type == 'move_file': + # TODO: Remove this hack that resets current_dir so that the pre-compiled bwa binary can be found. + # current_dir = '/Users/gvk/workspaces_2008/bwa/bwa-0.5.9' common_util.move_file( current_dir=current_dir, source=os.path.join( action_dict[ 'source' ] ), destination_dir=os.path.join( action_dict[ 'destination' ] ) ) diff -r 618c34d9b2eab385e092c9ca2d7dd07b9ea31024 -r ea3da2000fa733a2d3ff5d5629dec58d3573645c lib/galaxy/tool_shed/tool_dependencies/install_util.py --- a/lib/galaxy/tool_shed/tool_dependencies/install_util.py +++ b/lib/galaxy/tool_shed/tool_dependencies/install_util.py @@ -1,8 +1,9 @@ -import sys, os, subprocess, tempfile +import sys, os, subprocess, tempfile, urllib2 import common_util import fabric_util from galaxy.tool_shed import encoding_util from galaxy.model.orm import and_ +from galaxy.web import url_for from galaxy import eggs import pkg_resources @@ -11,6 +12,9 @@ from elementtree import ElementTree, ElementInclude from elementtree.ElementTree import Element, SubElement +def clean_tool_shed_url( base_url ): + protocol, base = base_url.split( '://' ) + return base.rstrip( '/' ) def create_or_update_tool_dependency( app, tool_shed_repository, name, version, type, status, set_status=True ): # Called from Galaxy (never the tool shed) when a new repository is being installed or when an uninstalled repository is being reinstalled. sa_session = app.model.context.current @@ -28,6 +32,64 @@ sa_session.add( tool_dependency ) sa_session.flush() return tool_dependency +def create_temporary_tool_dependencies_config( tool_shed_url, name, owner, changeset_revision ): + """Make a call to the tool shed to get the required repository's tool_dependencies.xml file.""" + url = url_join( tool_shed_url, + 'repository/get_tool_dependencies_config_contents?name=%s&owner=%s&changeset_revision=%s' % \ + ( name, owner, changeset_revision ) ) + response = urllib2.urlopen( url ) + text = response.read() + response.close() + if text: + # Write the contents to a temporary file on disk so it can be reloaded and parsed. + fh = tempfile.NamedTemporaryFile( 'wb' ) + tmp_filename = fh.name + fh.close() + fh = open( tmp_filename, 'wb' ) + fh.write( text ) + fh.close() + return tmp_filename + else: + message = "Unable to retrieve required tool_dependencies.xml file from the tool shed for revision " + message += "%s of installed repository %s owned by %s." % ( str( changeset_revision ), str( name ), str( owner ) ) + raise Exception( message ) + return None +def get_absolute_path_to_file_in_repository( repo_files_dir, file_name ): + """Return the absolute path to a specified disk file contained in a repository.""" + stripped_file_name = strip_path( file_name ) + file_path = None + for root, dirs, files in os.walk( repo_files_dir ): + if root.find( '.hg' ) < 0: + for name in files: + if name == stripped_file_name: + return os.path.abspath( os.path.join( root, name ) ) + return file_path +def get_tool_shed_repository_by_tool_shed_name_owner_changeset_revision( app, tool_shed_url, name, owner, changeset_revision ): + sa_session = app.model.context.current + tool_shed = clean_tool_shed_url( tool_shed_url ) + tool_shed_repository = sa_session.query( app.model.ToolShedRepository ) \ + .filter( and_( app.model.ToolShedRepository.table.c.tool_shed == tool_shed, + app.model.ToolShedRepository.table.c.name == name, + app.model.ToolShedRepository.table.c.owner == owner, + app.model.ToolShedRepository.table.c.changeset_revision == changeset_revision ) ) \ + .first() + if tool_shed_repository: + return tool_shed_repository + # The tool_shed_repository must have been updated to a newer changeset revision than the one defined in the repository_dependencies.xml file, + # so call the tool shed to get all appropriate newer changeset revisions. + text = get_updated_changeset_revisions_from_tool_shed( tool_shed_url, name, owner, changeset_revision ) + if text: + changeset_revisions = listify( text ) + for changeset_revision in changeset_revisions: + tool_shed_repository = sa_session.query( app.model.ToolShedRepository ) \ + .filter( and_( app.model.ToolShedRepository.table.c.tool_shed == tool_shed, + app.model.ToolShedRepository.table.c.name == name, + app.model.ToolShedRepository.table.c.owner == owner, + app.model.ToolShedRepository.table.c.changeset_revision == changeset_revision ) ) \ + .first() + if tool_shed_repository: + return tool_shed_repository + return None def get_tool_dependency_by_name_type_repository( app, repository, name, type ): sa_session = app.model.context.current return sa_session.query( app.model.ToolDependency ) \ @@ -43,23 +105,83 @@ app.model.ToolDependency.table.c.version == version, app.model.ToolDependency.table.c.type == type ) ) \ .first() -def get_tool_dependency_install_dir( app, repository, type, name, version ): - if type == 'package': +def get_tool_dependency_install_dir( app, repository_name, repository_owner, repository_changeset_revision, tool_dependency_type, tool_dependency_name, + tool_dependency_version ): + if tool_dependency_type == 'package': return os.path.abspath( os.path.join( app.config.tool_dependency_dir, - name, - version, - repository.owner, - repository.name, - repository.installed_changeset_revision ) ) - if type == 'set_environment': + tool_dependency_name, + tool_dependency_version, + repository_owner, + repository_name, + repository_changeset_revision ) ) + if tool_dependency_type == 'set_environment': return os.path.abspath( os.path.join( app.config.tool_dependency_dir, 'environment_settings', - name, - repository.owner, - repository.name, - repository.installed_changeset_revision ) ) + tool_dependency_name, + repository_owner, + repository_name, + repository_changeset_revision ) ) def get_tool_shed_repository_install_dir( app, tool_shed_repository ): return os.path.abspath( tool_shed_repository.repo_files_directory( app ) ) +def get_updated_changeset_revisions_from_tool_shed( tool_shed_url, name, owner, changeset_revision ): + """Get all appropriate newer changeset revisions for the repository defined by the received tool_shed_url / name / owner combination.""" + url = url_join( tool_shed_url, + 'repository/updated_changeset_revisions?name=%s&owner=%s&changeset_revision=%s' % ( name, owner, changeset_revision ) ) + response = urllib2.urlopen( url ) + text = response.read() + response.close() + return text +def handle_set_environment_entry_for_package( app, install_dir, tool_shed_repository, package_name, package_version, elem ): + action_dict = {} + actions = [] + for package_elem in elem: + if package_elem.tag == 'install': + # Create the tool_dependency record in the database. + tool_dependency = create_or_update_tool_dependency( app=app, + tool_shed_repository=tool_shed_repository, + name=package_name, + version=package_version, + type='package', + status=app.model.ToolDependency.installation_status.INSTALLING, + set_status=True ) + # Get the installation method version from a tag like: <install version="1.0"> + package_install_version = package_elem.get( 'version', '1.0' ) + if package_install_version == '1.0': + # Since the required tool dependency is installed for a repository dependency, all we need to do + # is inspect the <actions> tag set to find the <action type="set_environment"> tag. + for actions_elem in package_elem: + for action_elem in actions_elem: + action_type = action_elem.get( 'type', 'shell_command' ) + if action_type == 'set_environment': + # <action type="set_environment"> + # <environment_variable name="PYTHONPATH" action="append_to">$INSTALL_DIR/lib/python</environment_variable> + # <environment_variable name="PATH" action="prepend_to">$INSTALL_DIR/bin</environment_variable> + # </action> + env_var_dicts = [] + for env_elem in action_elem: + if env_elem.tag == 'environment_variable': + env_var_dict = common_util.create_env_var_dict( env_elem, tool_dependency_install_dir=install_dir ) + if env_var_dict: + env_var_dicts.append( env_var_dict ) + if env_var_dicts: + action_dict[ env_elem.tag ] = env_var_dicts + actions.append( ( action_type, action_dict ) ) + return tool_dependency, actions + return None, actions +def install_and_build_package_via_fabric( app, tool_dependency, actions_dict ): + sa_session = app.model.context.current + try: + # There is currently only one fabric method. + fabric_util.install_and_build_package( app, tool_dependency, actions_dict ) + except Exception, e: + tool_dependency.status = app.model.ToolDependency.installation_status.ERROR + tool_dependency.error_message = str( e ) + sa_session.add( tool_dependency ) + sa_session.flush() + if tool_dependency.status != app.model.ToolDependency.installation_status.ERROR: + tool_dependency.status = app.model.ToolDependency.installation_status.INSTALLED + sa_session.add( tool_dependency ) + sa_session.flush() def install_package( app, elem, tool_shed_repository, tool_dependencies=None ): # The value of tool_dependencies is a partial or full list of ToolDependency records associated with the tool_shed_repository. sa_session = app.model.context.current @@ -69,18 +191,101 @@ package_version = elem.get( 'version', None ) if package_name and package_version: if tool_dependencies: - install_dir = get_tool_dependency_install_dir( app, - repository=tool_shed_repository, - type='package', - name=package_name, - version=package_version ) + # Get the installation directory for tool dependencies that will be installed for the received tool_shed_repository. + install_dir = get_tool_dependency_install_dir( app=app, + repository_name=tool_shed_repository.name, + repository_owner=tool_shed_repository.owner, + repository_changeset_revision=tool_shed_repository.installed_changeset_revision, + tool_dependency_type='package', + tool_dependency_name=package_name, + tool_dependency_version=package_version ) if not os.path.exists( install_dir ): for package_elem in elem: - if package_elem.tag == 'install': + if package_elem.tag == 'repository': + # We have a complex repository dependency definition. + tool_shed = package_elem.attrib[ 'toolshed' ] + required_repository_name = package_elem.attrib[ 'name' ] + required_repository_owner = package_elem.attrib[ 'owner' ] + required_repository_changeset_revision = package_elem.attrib[ 'changeset_revision' ] + required_repository = get_tool_shed_repository_by_tool_shed_name_owner_changeset_revision( app, + tool_shed, + required_repository_name, + required_repository_owner, + required_repository_changeset_revision ) + tmp_filename = None + if required_repository: + # Set this repository's tool dependency env.sh file with a path to the required repository's installed tool dependency package. + # We can get everything we need from the discovered installed required_repository. + if required_repository.status in [ app.model.ToolShedRepository.installation_status.DEACTIVATED, + app.model.ToolShedRepository.installation_status.INSTALLED ]: + # Define the installation directory for the required tool dependency in the required repository. + required_repository_package_install_dir = \ + get_tool_dependency_install_dir( app=app, + repository_name=required_repository.name, + repository_owner=required_repository.owner, + repository_changeset_revision=required_repository.installed_changeset_revision, + tool_dependency_type='package', + tool_dependency_name=package_name, + tool_dependency_version=package_version ) + assert os.path.exists( required_repository_package_install_dir ), \ + 'Missing required tool dependency directory %s' % str( required_repository_package_install_dir ) + repo_files_dir = required_repository.repo_files_directory( app ) + tool_dependencies_config = get_absolute_path_to_file_in_repository( repo_files_dir, 'tool_dependencies.xml' ) + if tool_dependencies_config: + config_to_use = tool_dependencies_config + else: + message = "Unable to locate required tool_dependencies.xml file for revision %s of installed repository %s owned by %s." % \ + ( str( required_repository.changeset_revision ), str( required_repository.name ), str( required_repository.owner ) ) + raise Exception( message ) + else: + # Make a call to the tool shed to get the changeset revision to which the current value of required_repository_changeset_revision + # should be updated if it's not current. + text = get_updated_changeset_revisions_from_tool_shed( tool_shed_url=tool_shed, + name=required_repository_name, + owner=required_repository_owner, + changeset_revision=required_repository_changeset_revision ) + if text: + updated_changeset_revisions = listify( text ) + # The list of changeset revisions is in reverse order, so the newest will be first. + required_repository_changeset_revision = updated_changeset_revisions[ 0 ] + # Define the installation directory for the required tool dependency in the required repository. + required_repository_package_install_dir = \ + get_tool_dependency_install_dir( app=app, + repository_name=required_repository_name, + repository_owner=required_repository_owner, + repository_changeset_revision=required_repository_changeset_revision, + tool_dependency_type='package', + tool_dependency_name=package_name, + tool_dependency_version=package_version ) + # Make a call to the tool shed to get the required repository's tool_dependencies.xml file. + tmp_filename = create_temporary_tool_dependencies_config( tool_shed, + required_repository_name, + required_repository_owner, + required_repository_changeset_revision ) + config_to_use = tmp_filename + tool_dependency, actions_dict = populate_actions_dict( app=app, + dependent_install_dir=install_dir, + required_install_dir=required_repository_package_install_dir, + tool_shed_repository=tool_shed_repository, + package_name=package_name, + package_version=package_version, + tool_dependencies_config=config_to_use ) + if tmp_filename: + try: + os.remove( tmp_filename ) + except: + pass + # Install and build the package via fabric. + install_and_build_package_via_fabric( app, tool_dependency, actions_dict ) + else: + message = "Unable to locate required tool shed repository named %s owned by %s with revision %s." % \ + ( str( name ), str( owner ), str( changeset_revision ) ) + raise Exception( message ) + elif package_elem.tag == 'install': # <install version="1.0"> package_install_version = package_elem.get( 'version', '1.0' ) - tool_dependency = create_or_update_tool_dependency( app, - tool_shed_repository, + tool_dependency = create_or_update_tool_dependency( app=app, + tool_shed_repository=tool_shed_repository, name=package_name, version=package_version, type='package', @@ -168,7 +373,7 @@ if env_elem.tag == 'environment_variable': env_var_dict = common_util.create_env_var_dict( env_elem, tool_dependency_install_dir=install_dir ) if env_var_dict: - env_var_dicts.append( env_var_dict ) + env_var_dicts.append( env_var_dict ) if env_var_dicts: action_dict[ env_elem.tag ] = env_var_dicts else: @@ -183,18 +388,56 @@ # run_proprietary_fabric_method( app, elem, proprietary_fabfile_path, install_dir, package_name=package_name ) raise Exception( 'Tool dependency installation using proprietary fabric scripts is not yet supported.' ) else: - try: - # There is currently only one fabric method. - fabric_util.install_and_build_package( app, tool_dependency, actions_dict ) - except Exception, e: - tool_dependency.status = app.model.ToolDependency.installation_status.ERROR - tool_dependency.error_message = str( e ) - sa_session.add( tool_dependency ) - sa_session.flush() - if tool_dependency.status != app.model.ToolDependency.installation_status.ERROR: - tool_dependency.status = app.model.ToolDependency.installation_status.INSTALLED - sa_session.add( tool_dependency ) - sa_session.flush() + install_and_build_package_via_fabric( app, tool_dependency, actions_dict ) +def listify( item ): + """ + Make a single item a single item list, or return a list if passed a + list. Passing a None returns an empty list. + """ + if not item: + return [] + elif isinstance( item, list ): + return item + elif isinstance( item, basestring ) and item.count( ',' ): + return item.split( ',' ) + else: + return [ item ] +def populate_actions_dict( app, dependent_install_dir, required_install_dir, tool_shed_repository, package_name, package_version, tool_dependencies_config ): + """ + Populate an actions dictionary that can be sent to fabric_util.install_and_build_package. This method handles the scenario where a tool_dependencies.xml + file defines a complex repository dependency. In this case, the tool dependency package will be installed in a separate repository and the tool dependency + defined for the dependent repository will use an environment_variable setting defined in it's env.sh file to locate the required package. This method + basically does what the install_via_fabric method does, but restricts it's activity to the <action type="set_environment"> tag set within the required + repository's tool_dependencies.xml file. + """ + sa_session = app.model.context.current + if not os.path.exists( dependent_install_dir ): + os.makedirs( dependent_install_dir ) + actions_dict = dict( install_dir=dependent_install_dir ) + if package_name: + actions_dict[ 'package_name' ] = package_name + tool_dependency = None + action_dict = {} + if tool_dependencies_config: + required_td_tree = parse_xml( tool_dependencies_config ) + required_td_root = required_td_tree.getroot() + for required_td_elem in required_td_root: + # Find the appropriate package name and version. + if required_td_elem.tag == 'package': + # <package name="bwa" version="0.5.9"> + required_td_package_name = required_td_elem.get( 'name', None ) + required_td_package_version = required_td_elem.get( 'version', None ) + if required_td_package_name==package_name and required_td_package_version==package_version: + tool_dependency, actions = handle_set_environment_entry_for_package( app=app, + install_dir=required_install_dir, + tool_shed_repository=tool_shed_repository, + package_name=package_name, + package_version=package_version, + elem=required_td_elem ) + if actions: + actions_dict[ 'actions' ] = actions + break + return tool_dependency, actions_dict def run_proprietary_fabric_method( app, elem, proprietary_fabfile_path, install_dir, package_name=None, **kwd ): """ TODO: Handle this using the fabric api. @@ -248,6 +491,10 @@ tmp_stderr = open( tmp_name, 'rb' ) message = '%s\n' % str( tmp_stderr.read() ) tmp_stderr.close() + try: + os.remove( tmp_name ) + except: + pass return returncode, message def set_environment( app, elem, tool_shed_repository ): """ @@ -258,6 +505,11 @@ <environment_variable name="R_SCRIPT_PATH" action="set_to">$REPOSITORY_INSTALL_DIR</environment_variable></set_environment> """ + # TODO: Add support for a repository dependency definition within this tool dependency type's tag set. This should look something like + # the following. See the implementation of support for this in the tool dependency package type's method above. + # <set_environment version="1.0"> + # <repository toolshed="<tool shed>" name="<repository name>" owner="<repository owner>" changeset_revision="<changeset revision>" /> + # </set_environment> sa_session = app.model.context.current tool_dependency = None env_var_version = elem.get( 'version', '1.0' ) @@ -267,18 +519,20 @@ env_var_name = env_var_elem.get( 'name', None ) env_var_action = env_var_elem.get( 'action', None ) if env_var_name and env_var_action: - install_dir = get_tool_dependency_install_dir( app, - repository=tool_shed_repository, - type='set_environment', - name=env_var_name, - version=None ) + install_dir = get_tool_dependency_install_dir( app=app, + repository_name=tool_shed_repository.name, + repository_owner=tool_shed_repository.owner, + repository_changeset_revision=tool_shed_repository.installed_changeset_revision, + tool_dependency_type='set_environment', + tool_dependency_name=env_var_name, + tool_dependency_version=None ) tool_shed_repository_install_dir = get_tool_shed_repository_install_dir( app, tool_shed_repository ) env_var_dict = common_util.create_env_var_dict( env_var_elem, tool_shed_repository_install_dir=tool_shed_repository_install_dir ) if env_var_dict: if not os.path.exists( install_dir ): os.makedirs( install_dir ) - tool_dependency = create_or_update_tool_dependency( app, - tool_shed_repository, + tool_dependency = create_or_update_tool_dependency( app=app, + tool_shed_repository=tool_shed_repository, name=env_var_name, version=None, type='set_environment', @@ -294,3 +548,22 @@ sa_session.add( tool_dependency ) sa_session.flush() print 'Environment variable ', env_var_name, 'set in', install_dir +def strip_path( fpath ): + if not fpath: + return fpath + try: + file_path, file_name = os.path.split( fpath ) + except: + file_name = fpath + return file_name +def parse_xml( file_name ): + """Returns a parsed xml tree.""" + tree = ElementTree.parse( file_name ) + root = tree.getroot() + ElementInclude.include( root ) + return tree +def url_join( *args ): + parts = [] + for arg in args: + parts.append( arg.strip( '/' ) ) + return '/'.join( parts ) diff -r 618c34d9b2eab385e092c9ca2d7dd07b9ea31024 -r ea3da2000fa733a2d3ff5d5629dec58d3573645c lib/galaxy/util/shed_util.py --- a/lib/galaxy/util/shed_util.py +++ b/lib/galaxy/util/shed_util.py @@ -414,7 +414,7 @@ cleaned_repository_clone_url = suc.clean_repository_clone_url( repository_clone_url ) if not owner: owner = get_repository_owner( cleaned_repository_clone_url ) - tool_shed = cleaned_repository_clone_url.split( 'repos' )[ 0 ].rstrip( '/' ) + tool_shed = cleaned_repository_clone_url.split( '/repos/' )[ 0 ].rstrip( '/' ) for guid, tool_section_dicts in tool_panel_dict.items(): for tool_section_dict in tool_section_dicts: tool_section = None @@ -484,20 +484,6 @@ tool_section_dicts = generate_tool_section_dicts( tool_config=file_name, tool_sections=tool_sections ) tool_panel_dict[ guid ] = tool_section_dicts return tool_panel_dict -def generate_tool_path( repository_clone_url, changeset_revision ): - """ - Generate a tool path that guarantees repositories with the same name will always be installed - in different directories. The tool path will be of the form: - <tool shed url>/repos/<repository owner>/<repository name>/<installed changeset revision> - http://test@bx.psu.edu:9009/repos/test/filter - """ - tmp_url = suc.clean_repository_clone_url( repository_clone_url ) - # Now tmp_url is something like: bx.psu.edu:9009/repos/some_username/column - items = tmp_url.split( 'repos' ) - tool_shed_url = items[ 0 ] - repo_path = items[ 1 ] - tool_shed_url = suc.clean_tool_shed_url( tool_shed_url ) - return suc.url_join( tool_shed_url, 'repos', repo_path, changeset_revision ) def generate_tool_section_dicts( tool_config=None, tool_sections=None ): tool_section_dicts = [] if tool_config is None: @@ -529,6 +515,18 @@ else: tool_section = None return tool_section +def generate_tool_shed_repository_install_dir( repository_clone_url, changeset_revision ): + """ + Generate a repository installation directory that guarantees repositories with the same name will always be installed in different directories. + The tool path will be of the form: <tool shed url>/repos/<repository owner>/<repository name>/<installed changeset revision> + """ + tmp_url = suc.clean_repository_clone_url( repository_clone_url ) + # Now tmp_url is something like: bx.psu.edu:9009/repos/some_username/column + items = tmp_url.split( '/repos/' ) + tool_shed_url = items[ 0 ] + repo_path = items[ 1 ] + tool_shed_url = suc.clean_tool_shed_url( tool_shed_url ) + return suc.url_join( tool_shed_url, 'repos', repo_path, changeset_revision ) def get_config( config_file, repo, ctx, dir ): """Return the latest version of config_filename from the repository manifest.""" config_file = suc.strip_path( config_file ) @@ -821,14 +819,14 @@ readme_files_dict = json.from_json_string( raw_text ) return readme_files_dict def get_repository_owner( cleaned_repository_url ): - items = cleaned_repository_url.split( 'repos' ) + items = cleaned_repository_url.split( '/repos/' ) repo_path = items[ 1 ] if repo_path.startswith( '/' ): repo_path = repo_path.replace( '/', '', 1 ) return repo_path.lstrip( '/' ).split( '/' )[ 0 ] def get_repository_owner_from_clone_url( repository_clone_url ): tmp_url = suc.clean_repository_clone_url( repository_clone_url ) - tool_shed = tmp_url.split( 'repos' )[ 0 ].rstrip( '/' ) + tool_shed = tmp_url.split( '/repos/' )[ 0 ].rstrip( '/' ) return get_repository_owner( tmp_url ) def get_required_repo_info_dicts( tool_shed_url, repo_info_dicts ): """ diff -r 618c34d9b2eab385e092c9ca2d7dd07b9ea31024 -r ea3da2000fa733a2d3ff5d5629dec58d3573645c lib/galaxy/util/shed_util_common.py --- a/lib/galaxy/util/shed_util_common.py +++ b/lib/galaxy/util/shed_util_common.py @@ -1202,7 +1202,10 @@ app.config.tool_data_table_config_path = original_tool_data_table_config_path return metadata_dict, invalid_file_tups def generate_package_dependency_metadata( app, elem, tool_dependencies_dict ): - """The value of package_name must match the value of the "package" type in the tool config's <requirements> tag set.""" + """ + Generate the metadata for a tool dependencies package defined for a repository. The value of package_name must match the value of the "package" + type in the tool config's <requirements> tag set. This method is called from both Galaxy and the tool shed. + """ repository_dependency_tup = [] requirements_dict = {} error_message = '' @@ -1217,9 +1220,9 @@ requirements_dict[ 'readme' ] = sub_elem.text elif sub_elem.tag == 'repository': # We have a complex repository dependency. - current_rd_tups, error_message = handle_repository_elem_for_tool_shed( app=app, - repository_elem=sub_elem, - repository_dependencies_tups=None ) + current_rd_tups, error_message = handle_repository_elem( app=app, + repository_elem=sub_elem, + repository_dependencies_tups=None ) repository_dependency_tup = current_rd_tups[ 0 ] if requirements_dict: dependency_key = '%s/%s' % ( package_name, package_version ) @@ -1274,7 +1277,7 @@ is_valid = False if is_valid: for repository_elem in root.findall( 'repository' ): - current_rd_tups, error_message = handle_repository_elem_for_tool_shed( app, repository_elem, repository_dependencies_tups ) + current_rd_tups, error_message = handle_repository_elem( app, repository_elem, repository_dependencies_tups ) if error_message: log.debug( error_message ) return metadata_dict, error_message @@ -1477,7 +1480,7 @@ metadata_dict[ 'workflows' ] = [ ( relative_path, exported_workflow_dict ) ] return metadata_dict def get_absolute_path_to_file_in_repository( repo_files_dir, file_name ): - """Return the absolute path to a specified disk file containe in a repository.""" + """Return the absolute path to a specified disk file contained in a repository.""" stripped_file_name = strip_path( file_name ) file_path = None for root, dirs, files in os.walk( repo_files_dir ): @@ -1677,8 +1680,8 @@ return None def get_next_downloadable_changeset_revision( repository, repo, after_changeset_revision ): """ - Return the installable changeset_revision in the repository changelog after to the changeset to which after_changeset_revision - refers. If there isn't one, return None. + Return the installable changeset_revision in the repository changelog after the changeset to which after_changeset_revision refers. If there + isn't one, return None. """ changeset_revisions = get_ordered_downloadable_changeset_revisions( repository, repo ) if len( changeset_revisions ) == 1: @@ -2157,7 +2160,7 @@ .first() def get_tool_shed_from_clone_url( repository_clone_url ): tmp_url = clean_repository_clone_url( repository_clone_url ) - return tmp_url.split( 'repos' )[ 0 ].rstrip( '/' ) + return tmp_url.split( '/repos/' )[ 0 ].rstrip( '/' ) def get_updated_changeset_revisions_for_repository_dependencies( trans, key_rd_dicts ): updated_key_rd_dicts = [] for key_rd_dict in key_rd_dicts: @@ -2411,6 +2414,67 @@ all_repository_dependencies=all_repository_dependencies, handled_key_rd_dicts=handled_key_rd_dicts, circular_repository_dependencies=circular_repository_dependencies ) +def handle_repository_elem( app, repository_elem, repository_dependencies_tups ): + """ + Process the received repository_elem which is a <repository> tag either from a repository_dependencies.xml file or a tool_dependencies.xml file. + If the former, we're generating repository dependencies metadata for a repository in the tool shed. If the latter, we're generating package + dependency metadata with in Galaxy or the tool shed. + """ + if repository_dependencies_tups is None: + new_rd_tups = [] + else: + new_rd_tups = [ rdt for rdt in repository_dependencies_tups ] + error_message = '' + sa_session = app.model.context.current + toolshed = repository_elem.attrib[ 'toolshed' ] + name = repository_elem.attrib[ 'name' ] + owner = repository_elem.attrib[ 'owner' ] + changeset_revision = repository_elem.attrib[ 'changeset_revision' ] + user = None + repository = None + if app.name == 'galaxy': + # We're in Galaxy. + try: + repository = sa_session.query( app.model.ToolShedRepository ) \ + .filter( and_( app.model.ToolShedRepository.table.c.name == name, + app.model.ToolShedRepository.table.c.owner == owner ) ) \ + .first() + except: + error_message = "Invalid name %s or owner %s defined for repository. Repository dependencies will be ignored." % ( name, owner ) + log.debug( error_message ) + return new_rd_tups, error_message + repository_dependencies_tup = ( toolshed, name, owner, changeset_revision ) + if repository_dependencies_tup not in new_rd_tups: + new_rd_tups.append( repository_dependencies_tup ) + else: + # We're in the tool shed. + if tool_shed_is_this_tool_shed( toolshed ): + try: + user = sa_session.query( app.model.User ) \ + .filter( app.model.User.table.c.username == owner ) \ + .one() + except Exception, e: + error_message = "Invalid owner %s defined for repository %s. Repository dependencies will be ignored." % ( owner, name ) + log.debug( error_message ) + return new_rd_tups, error_message + try: + repository = sa_session.query( app.model.Repository ) \ + .filter( and_( app.model.Repository.table.c.name == name, + app.model.Repository.table.c.user_id == user.id ) ) \ + .first() + except: + error_message = "Invalid name %s or owner %s defined for repository. Repository dependencies will be ignored." % ( name, owner ) + log.debug( error_message ) + return new_rd_tups, error_message + repository_dependencies_tup = ( toolshed, name, owner, changeset_revision ) + if repository_dependencies_tup not in new_rd_tups: + new_rd_tups.append( repository_dependencies_tup ) + else: + # Repository dependencies are currentlhy supported within a single tool shed. + error_message = "Invalid tool shed %s defined for repository %s. " % ( toolshed, name ) + error_message += "Repository dependencies are currently supported within a single tool shed, so your definition will be ignored." + log.debug( error_message ) + return new_rd_tups, error_message def handle_sample_files_and_load_tool_from_disk( trans, repo_files_dir, tool_config_filepath, work_dir ): # Copy all sample files from disk to a temporary directory since the sample files may be in multiple directories. message = '' @@ -2487,54 +2551,6 @@ if is_orphan_in_tool_shed: return True return False -def handle_repository_elem_for_tool_shed( app, repository_elem, repository_dependencies_tups ): - if repository_dependencies_tups is None: - new_rd_tups = [] - else: - new_rd_tups = [ rdt for rdt in repository_dependencies_tups ] - error_message = '' - sa_session = app.model.context.current - toolshed = repository_elem.attrib[ 'toolshed' ] - name = repository_elem.attrib[ 'name' ] - owner = repository_elem.attrib[ 'owner' ] - changeset_revision = repository_elem.attrib[ 'changeset_revision' ] - user = None - repository = None - if tool_shed_is_this_tool_shed( toolshed ): - try: - user = sa_session.query( app.model.User ) \ - .filter( app.model.User.table.c.username == owner ) \ - .one() - except Exception, e: - error_message = "Invalid owner %s defined for repository %s. Repository dependencies will be ignored." % ( owner, name ) - log.debug( error_message ) - return new_rd_tups, error_message - if user: - try: - repository = sa_session.query( app.model.Repository ) \ - .filter( and_( app.model.Repository.table.c.name == name, - app.model.Repository.table.c.user_id == user.id ) ) \ - .first() - except: - error_message = "Invalid name %s or owner %s defined for repository. Repository dependencies will be ignored." % ( name, owner ) - log.debug( error_message ) - return new_rd_tups, error_message - if repository: - repository_dependencies_tup = ( toolshed, name, owner, changeset_revision ) - if repository_dependencies_tup not in new_rd_tups: - new_rd_tups.append( repository_dependencies_tup ) - else: - error_message = "Invalid name %s or owner %s defined for repository. Repository dependencies will be ignored." % ( name, owner ) - log.debug( error_message ) - else: - error_message = "Invalid owner %s defined for owner of repository %s. Repository dependencies will be ignored." % ( owner, name ) - log.debug( error_message ) - else: - # Repository dependencies are currentlhy supported within a single tool shed. - error_message = "Invalid tool shed %s defined for repository %s. " % ( toolshed, name ) - error_message += "Repository dependencies are currently supported within a single tool shed, so your definition will be ignored." - log.debug( error_message ) - return new_rd_tups, error_message def has_previous_repository_reviews( trans, repository, changeset_revision ): """Determine if a repository has a changeset revision review prior to the received changeset revision.""" repo = hg.repository( get_configured_ui(), repository.repo_path( trans.app ) ) @@ -2588,7 +2604,22 @@ return True return False def is_downloadable( metadata_dict ): - return 'datatypes' in metadata_dict or 'repository_dependencies' in metadata_dict or 'tools' in metadata_dict or 'workflows' in metadata_dict + if 'datatypes' in metadata_dict: + # We have proprietary datatypes. + return True + if 'repository_dependencies' in metadata_dict: + # We have repository_dependencies. + return True + if 'tools' in metadata_dict: + # We have tools. + return True + if 'tool_dependencies' in metadata_dict: + # We have tool dependencies, and perhaps only tool dependencies! + return True + if 'workflows' in metadata_dict: + # We have exported workflows. + return True + return False def initialize_all_repository_dependencies( current_repository_key, repository_dependencies_dict, all_repository_dependencies ): # Initialize the all_repository_dependencies dictionary. It's safe to assume that current_repository_key in this case will have a value. all_repository_dependencies[ 'root_key' ] = current_repository_key @@ -3322,7 +3353,7 @@ return ''.join( translated ) return text def tool_shed_from_repository_clone_url( repository_clone_url ): - return clean_repository_clone_url( repository_clone_url ).split( 'repos' )[ 0 ].rstrip( '/' ) + return clean_repository_clone_url( repository_clone_url ).split( '/repos/' )[ 0 ].rstrip( '/' ) def tool_shed_is_this_tool_shed( toolshed_base_url ): return toolshed_base_url.rstrip( '/' ) == str( url_for( '/', qualified=True ) ).rstrip( '/' ) def translate_string( raw_text, to_html=True ): diff -r 618c34d9b2eab385e092c9ca2d7dd07b9ea31024 -r ea3da2000fa733a2d3ff5d5629dec58d3573645c lib/galaxy/webapps/community/controllers/repository.py --- a/lib/galaxy/webapps/community/controllers/repository.py +++ b/lib/galaxy/webapps/community/controllers/repository.py @@ -1466,7 +1466,7 @@ return repo_info_dict @web.expose def get_tool_dependencies( self, trans, **kwd ): - """Handle a request from a Galaxy instance.""" + """Handle a request from a Galaxy instance to get the tool_dependencies entry from the metadata for a specified changeset revision.""" params = util.Params( kwd ) name = params.get( 'name', None ) owner = params.get( 'owner', None ) @@ -1481,6 +1481,26 @@ return encoding_util.tool_shed_encode( tool_dependencies ) return '' @web.expose + def get_tool_dependencies_config_contents( self, trans, **kwd ): + """Handle a request from a Galaxy instance to get the tool_dependencies.xml file contents for a specified changeset revision.""" + params = util.Params( kwd ) + name = params.get( 'name', None ) + owner = params.get( 'owner', None ) + changeset_revision = params.get( 'changeset_revision', None ) + repository = suc.get_repository_by_name_and_owner( trans, name, owner ) + # TODO: We're currently returning the tool_dependencies.xml file that is available on disk. We need to enhance this process + # to retrieve older versions of the tool-dependencies.xml file from the repository manafest. + repo_dir = repository.repo_path( trans.app ) + # Get the tool_dependencies.xml file from disk. + tool_dependencies_config = suc.get_config_from_disk( 'tool_dependencies.xml', repo_dir ) + # Return the encoded contents of the tool_dependencies.xml file. + if tool_dependencies_config: + tool_dependencies_config_file = open( tool_dependencies_config, 'rb' ) + contents = tool_dependencies_config_file.read() + tool_dependencies_config_file.close() + return contents + return '' + @web.expose def get_tool_versions( self, trans, **kwd ): """ For each valid /downloadable change set (up to the received changeset_revision) in the repository's change log, append the change @@ -1505,7 +1525,7 @@ return '' @web.json def get_updated_repository_information( self, trans, name, owner, changeset_revision, **kwd ): - """Generate a disctionary that contains the information about a repository that is necessary for installing it into a local Galaxy instance.""" + """Generate a dictionary that contains the information about a repository that is necessary for installing it into a local Galaxy instance.""" repository = suc.get_repository_by_name_and_owner( trans, name, owner ) repository_id = trans.security.encode_id( repository.id ) repository_clone_url = suc.generate_clone_url_for_repository_in_tool_shed( trans, repository ) @@ -2079,7 +2099,7 @@ repository = suc.get_repository_by_name_and_owner( trans, name, owner ) repo_dir = repository.repo_path( trans.app ) repo = hg.repository( suc.get_configured_ui(), repo_dir ) - # Get the lower bound changeset revision + # Get the lower bound changeset revision. lower_bound_changeset_revision = suc.get_previous_downloadable_changset_revision( repository, repo, changeset_revision ) # Build the list of changeset revision hashes. changeset_hashes = [] @@ -2404,6 +2424,35 @@ if list: return ','.join( list ) return '' + @web.expose + def updated_changeset_revisions( self, trans, **kwd ): + """ + Handle a request from a local Galaxy instance to retrieve the lsit of changeset revisions to which an installed repository can be updated. This + method will return a string of comma-separated changeset revision hashes for all available updates to the received changeset revision. Among + other things , this method handles the scenario where an installed tool shed repository's tool_dependency definition file defines a changeset + revision for a complex repository dependency that is outdated. In other words, a defined changeset revision is older than the current changeset + revision for the required repository, making it impossible to discover the repository without knowledge of revisions to which it could have been + updated. + """ + params = util.Params( kwd ) + name = params.get( 'name', None ) + owner = params.get( 'owner', None ) + changeset_revision = params.get( 'changeset_revision', None ) + repository = suc.get_repository_by_name_and_owner( trans, name, owner ) + repo_dir = repository.repo_path( trans.app ) + repo = hg.repository( suc.get_configured_ui(), repo_dir ) + # Get the upper bound changeset revision. + upper_bound_changeset_revision = suc.get_next_downloadable_changeset_revision( repository, repo, changeset_revision ) + # Build the list of changeset revision hashes defining each available update up to, but excluding, upper_bound_changeset_revision. + changeset_hashes = [] + for changeset in suc.reversed_lower_upper_bounded_changelog( repo, changeset_revision, upper_bound_changeset_revision ): + # Make sure to exclude upper_bound_changeset_revision. + if changeset != upper_bound_changeset_revision: + changeset_hashes.append( str( repo.changectx( changeset ) ) ) + if changeset_hashes: + changeset_hashes_str = ','.join( changeset_hashes ) + return changeset_hashes_str + return '' def __validate_repository_name( self, name, user ): # Repository names must be unique for each user, must be at least four characters # in length and must contain only lower-case letters, numbers, and the '_' character. diff -r 618c34d9b2eab385e092c9ca2d7dd07b9ea31024 -r ea3da2000fa733a2d3ff5d5629dec58d3573645c lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py --- a/lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py +++ b/lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py @@ -890,7 +890,7 @@ shed_util.update_tool_shed_repository_status( trans.app, tool_shed_repository, trans.model.ToolShedRepository.installation_status.CLONING ) repo_info_tuple = repo_info_dict[ tool_shed_repository.name ] description, repository_clone_url, changeset_revision, ctx_rev, repository_owner, repository_dependencies, tool_dependencies = repo_info_tuple - relative_clone_dir = shed_util.generate_tool_path( repository_clone_url, tool_shed_repository.installed_changeset_revision ) + relative_clone_dir = shed_util.generate_tool_shed_repository_install_dir( repository_clone_url, tool_shed_repository.installed_changeset_revision ) clone_dir = os.path.join( tool_path, relative_clone_dir ) relative_install_dir = os.path.join( relative_clone_dir, tool_shed_repository.name ) install_dir = os.path.join( tool_path, relative_install_dir ) @@ -1416,7 +1416,8 @@ install_tool_dependencies = CheckboxField.is_checked( kwd.get( 'install_tool_dependencies', '' ) ) shed_tool_conf, tool_path, relative_install_dir = suc.get_tool_panel_config_tool_path_install_dir( trans.app, tool_shed_repository ) repository_clone_url = suc.generate_clone_url_for_installed_repository( trans.app, tool_shed_repository ) - clone_dir = os.path.join( tool_path, shed_util.generate_tool_path( repository_clone_url, tool_shed_repository.installed_changeset_revision ) ) + clone_dir = os.path.join( tool_path, shed_util.generate_tool_shed_repository_install_dir( repository_clone_url, + tool_shed_repository.installed_changeset_revision ) ) relative_install_dir = os.path.join( clone_dir, tool_shed_repository.name ) tool_shed_url = suc.get_url_from_repository_tool_shed( trans.app, tool_shed_repository ) tool_section = None Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.