1 new changeset in galaxy-central: http://bitbucket.org/galaxy/galaxy-central/changeset/b13ab23c7d8e/ changeset: b13ab23c7d8e user: greg date: 2011-09-20 16:38:30 summary: Add baseline support for using Fabric scripts for automatically installing tool dependencies on a local Galaxy instance when repositories are installed from a Galaxy tool shed, and persist the url to the local Galaxy instance in a cookie named toolshedgalaxyurl when installing a tool shed repository to a local Galaxy instance. affected #: 12 files (-1 bytes) --- a/lib/galaxy/config.py Mon Sep 19 16:07:53 2011 -0400 +++ b/lib/galaxy/config.py Tue Sep 20 10:38:30 2011 -0400 @@ -119,9 +119,12 @@ self.ftp_upload_site = kwargs.get( 'ftp_upload_site', None ) self.allow_library_path_paste = kwargs.get( 'allow_library_path_paste', False ) self.disable_library_comptypes = kwargs.get( 'disable_library_comptypes', '' ).lower().split( ',' ) - # Location for dependencies + # Location for tool dependencies. if 'tool_dependency_dir' in kwargs: self.tool_dependency_dir = resolve_path( kwargs.get( "tool_dependency_dir" ), self.root ) + # Setting the following flag to true will ultimately cause tool dependencies + # to be located in the shell environment and used by the job that is executing + # the tool. self.use_tool_dependencies = True else: self.tool_dependency_dir = None --- a/lib/galaxy/datatypes/assembly.py Mon Sep 19 16:07:53 2011 -0400 +++ b/lib/galaxy/datatypes/assembly.py Tue Sep 20 10:38:30 2011 -0400 @@ -143,7 +143,6 @@ def __init__( self, **kwd ): Html.__init__( self, **kwd ) - log.debug( "Velvet log info %s" % 'JJ __init__') self.add_composite_file( 'Sequences', mimetype = 'text/html', description = 'Sequences', substitute_name_with_metadata = None, is_binary = False ) self.add_composite_file( 'Roadmaps', mimetype = 'text/html', description = 'Roadmaps', substitute_name_with_metadata = None, is_binary = False ) self.add_composite_file( 'Log', mimetype = 'text/html', description = 'Log', optional = 'True', substitute_name_with_metadata = None, is_binary = False ) --- a/lib/galaxy/tools/__init__.py Mon Sep 19 16:07:53 2011 -0400 +++ b/lib/galaxy/tools/__init__.py Tue Sep 20 10:38:30 2011 -0400 @@ -227,12 +227,11 @@ id = self.app.security.decode_id( workflow_id ) stored = self.app.model.context.query( self.app.model.StoredWorkflow ).get( id ) return stored.latest_workflow - def init_dependency_manager( self ): - self.dependency_manager = None if self.app.config.use_tool_dependencies: self.dependency_manager = DependencyManager( [ self.app.config.tool_dependency_dir ] ) - + else: + self.dependency_manager = None @property def sa_session( self ): """ @@ -341,12 +340,17 @@ """ Represents an external requirement that must be available for the tool to run (for example, a program, package, or library). Requirements can - optionally assert a specific version + optionally assert a specific version, or reference a command to execute a + fabric script. If fabric is used, the type is 'fabfile' and the version + attribute is not used since the fabric script includes all necessary + information for automatic dependency installation. """ - def __init__( self ): - self.name = None - self.type = None - self.version = None + def __init__( self, name=None, type=None, version=None, fabfile=None, method=None ): + self.name = name + self.type = type + self.version = version + self.fabfile = fabfile + self.method = method class Tool: """ @@ -882,10 +886,21 @@ self.requirements """ for requirement_elem in requirements_elem.findall( 'requirement' ): - requirement = ToolRequirement() - requirement.name = util.xml_text( requirement_elem ) - requirement.type = requirement_elem.get( "type", "package" ) - requirement.version = requirement_elem.get( "version" ) + name = util.xml_text( requirement_elem ) + type = requirement_elem.get( "type", "package" ) + if type == 'fabfile': + # The fabric script will include all necessary information for + # automatically installing the tool dependencies. + fabfile = requirement_elem.get( "fabfile" ) + method = requirement_elem.get( "method" ) + version = None + else: + # For backward compatibility, requirements tag sets should not require the + # use of a fabric script. + version = requirement_elem.get( "version" ) + fabfile = None + method = None + requirement = ToolRequirement( name=name, type=type, version=version, fabfile=fabfile, method=method ) self.requirements.append( requirement ) def check_workflow_compatible( self ): --- a/lib/galaxy/tools/deps/__init__.py Mon Sep 19 16:07:53 2011 -0400 +++ b/lib/galaxy/tools/deps/__init__.py Tue Sep 20 10:38:30 2011 -0400 @@ -18,11 +18,10 @@ and should each contain a file 'env.sh' which can be sourced to make the dependency available in the current shell environment. """ - def __init__( self, base_paths=[] ): """ - Create a new dependency manager looking for packages under the - paths listed in `base_paths`. + Create a new dependency manager looking for packages under the paths listed + in `base_paths`. The default base path is app.config.tool_dependency_dir. """ self.base_paths = [] for base_path in base_paths: @@ -31,19 +30,17 @@ if not os.path.isdir( base_path ): log.warn( "Path '%s' is not directory, ignoring", base_path ) self.base_paths.append( os.path.abspath( base_path ) ) - def find_dep( self, name, version=None ): """ Attempt to find a dependency named `name` at version `version`. If version is None, return the "default" version as determined using a symbolic link (if found). Returns a triple of: - env_script, base_path, real_version + env_script, base_path, real_version """ if version is None: return self._find_dep_default( name ) else: return self._find_dep_versioned( name, version ) - def _find_dep_versioned( self, name, version ): for base_path in self.base_paths: path = os.path.join( base_path, name, version ) @@ -52,7 +49,6 @@ return script, path, version else: return None, None, None - def _find_dep_default( self, name ): version = None for base_path in self.base_paths: @@ -65,5 +61,3 @@ return script, real_path, real_version else: return None, None, None - - --- a/lib/galaxy/web/base/controller.py Mon Sep 19 16:07:53 2011 -0400 +++ b/lib/galaxy/web/base/controller.py Tue Sep 20 10:38:30 2011 -0400 @@ -2749,10 +2749,11 @@ @web.expose @web.require_admin def browse_tool_shed( self, trans, **kwd ): - tool_shed_name = kwd[ 'tool_shed_name' ] tool_shed_url = kwd[ 'tool_shed_url' ] galaxy_url = trans.request.host - url = '%s/repository/browse_downloadable_repositories?tool_shed_name=%s&galaxy_url=%s&webapp=community' % ( tool_shed_url, tool_shed_name, galaxy_url ) + # Set the galayurl cookie so we can get back here from the remote tool shed. + trans.set_cookie( galaxy_url, name='toolshedgalaxyurl' ) + url = '%s/repository/browse_downloadable_repositories?webapp=community' % ( tool_shed_url ) return trans.response.send_redirect( url ) @web.expose @web.require_admin @@ -2784,8 +2785,8 @@ status = 'error' else: os.makedirs( clone_dir ) + log.debug( 'Cloning %s...' % repository_clone_url ) cmd = 'hg clone %s' % repository_clone_url - log.debug( 'Cloning: %s' % repository_clone_url ) tmp_name = tempfile.NamedTemporaryFile().name tmp_stderr = open( tmp_name, 'wb' ) os.chdir( clone_dir ) @@ -2795,7 +2796,7 @@ tmp_stderr.close() if returncode == 0: repo_files_dir = os.path.join( clone_dir, repository_name ) - log.debug( 'Updating cloned repository to revision: %s' % changeset_revision ) + log.debug( 'Updating cloned repository to revision "%s"...' % changeset_revision ) cmd = 'hg update -r %s' % changeset_revision tmp_name = tempfile.NamedTemporaryFile().name tmp_stderr = open( tmp_name, 'wb' ) @@ -2831,6 +2832,39 @@ # We have an inavlid .xml file, so not a tool config. log.debug( "Ignoring invalid tool config (%s). Error: %s" % ( str( relative_path ), str( e ) ) ) if repository_tools_tups: + # Look at each tool to see if it includes a "requirement" that refers to + # a fabric script. For those that do, execute the fabric script to install + # tool dependencies. + for repository_tools_tup in repository_tools_tups: + tup_path, repository_tool = repository_tools_tup + for requirement in repository_tool.requirements: + if requirement.type == 'fabfile': + log.debug( 'Executing fabric script to install dependencies for tool "%s"...' % repository_tool.name ) + fabfile = requirement.fabfile + method = requirement.method + # Find the relative path to the fabfile. + relative_fabfile_path = None + for root, dirs, files in os.walk( repo_files_dir ): + for name in files: + if name == fabfile: + relative_fabfile_path = os.path.join( root, name ) + break + if relative_fabfile_path: + # cmd will look something like: fab -f fabfile.py install_bowtie + cmd = 'fab -f %s %s' % ( relative_fabfile_path, method ) + tmp_name = tempfile.NamedTemporaryFile().name + tmp_stderr = open( tmp_name, 'wb' ) + os.chdir( repo_files_dir ) + proc = subprocess.Popen( cmd, shell=True, stderr=tmp_stderr.fileno() ) + returncode = proc.wait() + os.chdir( current_working_dir ) + tmp_stderr.close() + if returncode != 0: + # TODO: do something more here than logging the problem. + tmp_stderr = open( tmp_name, 'rb' ) + error = tmp_stderr.read() + tmp_stderr.close() + log.debug( 'Problem installing dependencies for tool "%s"\n%s' % ( repository_tool.name, error ) ) # Generate an in-memory tool conf section that includes the new tools. new_tool_section = self.__generate_tool_panel_section( repository_name, repository_clone_url, --- a/lib/galaxy/webapps/community/controllers/common.py Mon Sep 19 16:07:53 2011 -0400 +++ b/lib/galaxy/webapps/community/controllers/common.py Tue Sep 20 10:38:30 2011 -0400 @@ -171,9 +171,21 @@ repository = get_repository( trans, id ) tool_requirements = [] for tr in tool.requirements: - requirement_dict = dict( name=tr.name, - type=tr.type, - version=tr.version ) + name=tr.name + type=tr.type + if type == 'fabfile': + version = None + fabfile = tr.fabfile + method = tr.method + else: + version = tr.version + fabfile = None + method = None + requirement_dict = dict( name=name, + type=type, + version=version, + fabfile=fabfile, + method=method ) tool_requirements.append( requirement_dict ) tool_tests = [] if tool.tests: @@ -335,13 +347,17 @@ trans.sa_session.flush() else: message = "Changeset revision '%s' includes no tools or exported workflows for which metadata can be set." % str( changeset_revision ) - status = "done" + status = "error" else: # change_set is None message = "Repository does not include changeset revision '%s'." % str( changeset_revision ) status = 'error' if invalid_files: - message = "Metadata cannot be defined for change set revision '%s'. Correct the following problems and reset metadata.<br/>" % str( changeset_revision ) + if metadata_dict: + message = "Metadata was defined for some items in change set revision '%s'. " % str( changeset_revision ) + message += "If the following files are Galaxy tool configs or exported Galaxy workflows, correct the problems and reset metadata.<br/>" + else: + message = "Metadata cannot be defined for change set revision '%s'. Correct the following problems and reset metadata.<br/>" % str( changeset_revision ) for itc_tup in invalid_files: tool_file = itc_tup[0] exception_msg = itc_tup[1] @@ -527,7 +543,7 @@ ToolClass = Tool return ToolClass( config_file, root, trans.app ) return None -def build_changeset_revision_select_field( trans, repository, selected_value=None, add_id_to_name=True, galaxy_url=None ): +def build_changeset_revision_select_field( trans, repository, selected_value=None, add_id_to_name=True ): """ Build a SelectField whose options are the changeset_revision strings of all downloadable_revisions of the received repository. @@ -541,15 +557,9 @@ options.append( ( revision_label, changeset_revision ) ) refresh_on_change_values.append( changeset_revision ) if add_id_to_name: - if galaxy_url: - name = '%s_changeset_revision_%d' % ( galaxy_url, repository.id ) - else: - name = 'changeset_revision_%d' % repository.id + name = 'changeset_revision_%d' % repository.id else: - if galaxy_url: - name = '%s_changeset_revision' % galaxy_url - else: - name = 'changeset_revision' + name = 'changeset_revision' select_field = SelectField( name=name, refresh_on_change=True, refresh_on_change_values=refresh_on_change_values ) --- a/lib/galaxy/webapps/community/controllers/repository.py Mon Sep 19 16:07:53 2011 -0400 +++ b/lib/galaxy/webapps/community/controllers/repository.py Tue Sep 20 10:38:30 2011 -0400 @@ -122,7 +122,6 @@ return 'yes' return '' # Grid definition - galaxy_url = None title = "Repositories" model_class = model.Repository template='/webapps/community/repository/grid.mako' @@ -192,12 +191,11 @@ Display a SelectField whose options are the changeset_revision strings of all downloadable_revisions of this repository. """ - select_field = build_changeset_revision_select_field( trans, - repository, - galaxy_url=grid.galaxy_url ) + select_field = build_changeset_revision_select_field( trans, repository ) if len( select_field.options ) > 1: return select_field.get_html() return repository.revision + title = "Downloadable repositories" columns = [ RepositoryListGrid.NameColumn( "Name", key="name", @@ -264,9 +262,7 @@ return self.category_list_grid( trans, **kwd ) @web.expose def browse_downloadable_repositories( self, trans, **kwd ): - tool_shed_name = kwd.get( 'tool_shed_name', None ) repository_id = kwd.get( 'id', None ) - galaxy_url = kwd.get( 'galaxy_url', None ) if 'operation' in kwd: operation = kwd[ 'operation' ].lower() if operation == "preview_tools_in_changeset": @@ -274,8 +270,7 @@ return trans.response.send_redirect( web.url_for( controller='repository', action='preview_tools_in_changeset', repository_id=repository_id, - changeset_revision=repository.tip, - galaxy_url=galaxy_url ) ) + changeset_revision=repository.tip ) ) # The changeset_revision_select_field in the RepositoryListGrid performs a refresh_on_change # which sends in request parameters like changeset_revison_1, changeset_revision_2, etc. One @@ -292,31 +287,10 @@ return trans.response.send_redirect( web.url_for( controller='repository', action='preview_tools_in_changeset', repository_id=trans.security.encode_id( repository.id ), - changeset_revision=v, - galaxy_url=galaxy_url ) ) - elif k.find( changset_revision_str ) > 0: - # Keys look like: 'localhost:8763_changeset_revision_3' and values: '4ef2cf631604'. - items = k.split( '_%s' % changset_revision_str ) - galaxy_url = items[0] - repository_id = trans.security.encode_id( int( items[1] ) ) - repository = get_repository( trans, repository_id ) - if repository.tip != v: - return trans.response.send_redirect( web.url_for( controller='repository', - action='preview_tools_in_changeset', - repository_id=trans.security.encode_id( repository.id ), - changeset_revision=v, - galaxy_url=galaxy_url ) ) - - if tool_shed_name: - title = "%s downloadable repositories" % tool_shed_name - else: - title = "Downloadable repositories" - self.downloadable_repository_list_grid.title = title - self.downloadable_repository_list_grid.galaxy_url = galaxy_url + changeset_revision=v ) ) url_args = dict( action='browse_downloadable_repositories', operation='preview_tools_in_changeset', - repository_id=repository_id, - galaxy_url=galaxy_url ) + repository_id=repository_id ) self.downloadable_repository_list_grid.operations = [ grids.GridOperation( "Preview and install tools", url_args=url_args, allow_multiple=False, @@ -329,7 +303,6 @@ params = util.Params( kwd ) message = util.restore_text( params.get( 'message', '' ) ) status = params.get( 'status', 'done' ) - galaxy_url = util.restore_text( params.get( 'galaxy_url', '' ) ) repository = get_repository( trans, repository_id ) changeset_revision = util.restore_text( params.get( 'changeset_revision', repository.tip ) ) repository_metadata = get_repository_metadata_by_changeset_revision( trans, repository_id, changeset_revision ) @@ -348,7 +321,6 @@ revision_label=revision_label, changeset_revision_select_field=changeset_revision_select_field, metadata=metadata, - galaxy_url=galaxy_url, display_for_install=True, message=message, status=status ) @@ -357,7 +329,7 @@ params = util.Params( kwd ) message = util.restore_text( params.get( 'message', '' ) ) status = params.get( 'status', 'done' ) - galaxy_url = util.restore_text( params.get( 'galaxy_url', '' ) ) + galaxy_url = trans.get_cookie( name='toolshedgalaxyurl' ) repository = get_repository( trans, repository_id ) changeset_revision = util.restore_text( params.get( 'changeset_revision', repository.tip ) ) # Redirect back to local Galaxy to perform install. @@ -1232,7 +1204,6 @@ params = util.Params( kwd ) message = util.restore_text( params.get( 'message', '' ) ) status = params.get( 'status', 'done' ) - galaxy_url = util.restore_text( params.get( 'galaxy_url', '' ) ) display_for_install = util.string_as_bool( params.get( 'display_for_install', False ) ) repository = get_repository( trans, repository_id ) repo = hg.repository( get_configured_ui(), repository.repo_path ) @@ -1274,7 +1245,6 @@ tool_state=tool_state, is_malicious=is_malicious, display_for_install=display_for_install, - galaxy_url=galaxy_url, message=message, status=status ) except Exception, e: --- a/templates/admin/select_tool_panel_section.mako Mon Sep 19 16:07:53 2011 -0400 +++ b/templates/admin/select_tool_panel_section.mako Tue Sep 20 10:38:30 2011 -0400 @@ -16,7 +16,7 @@ or concerns. </div><br/> -<div class="errormessage"> +<div class="warningmessage"> Installation may take a while, depending upon the size of the repository contents. Wait until a message is displayed in your browser after clicking the <b>Install</b> button below. </div> --- a/templates/webapps/community/repository/common.mako Mon Sep 19 16:07:53 2011 -0400 +++ b/templates/webapps/community/repository/common.mako Tue Sep 20 10:38:30 2011 -0400 @@ -83,7 +83,7 @@ hg clone <a href="${clone_str}">${clone_str}</a></%def> -<%def name="render_repository_tools_and_workflows( metadata, can_set_metadata=False, display_for_install=False, galaxy_url=None )"> +<%def name="render_repository_tools_and_workflows( metadata, can_set_metadata=False, display_for_install=False )"> %if metadata or can_set_metadata: <p/><div class="toolForm"> @@ -111,12 +111,12 @@ <tr><td><div style="float: left; margin-left: 1px;" class="menubutton split popup" id="tool-${tool_dict[ 'id' ].replace( ' ', '_' )}-popup"> - <a class="view-info" href="${h.url_for( controller='repository', action='display_tool', repository_id=trans.security.encode_id( repository.id ), tool_config=tool_dict[ 'tool_config' ], changeset_revision=changeset_revision, display_for_install=display_for_install, galaxy_url=galaxy_url )}"> + <a class="view-info" href="${h.url_for( controller='repository', action='display_tool', repository_id=trans.security.encode_id( repository.id ), tool_config=tool_dict[ 'tool_config' ], changeset_revision=changeset_revision, display_for_install=display_for_install )}"> ${tool_dict[ 'name' ]} </a></div><div popupmenu="tool-${tool_dict[ 'id' ].replace( ' ', '_' )}-popup"> - <a class="action-button" href="${h.url_for( controller='repository', action='view_tool_metadata', repository_id=trans.security.encode_id( repository.id ), changeset_revision=changeset_revision, tool_id=tool_dict[ 'id' ], display_for_install=display_for_install, galaxy_url=galaxy_url )}">View tool metadata</a> + <a class="action-button" href="${h.url_for( controller='repository', action='view_tool_metadata', repository_id=trans.security.encode_id( repository.id ), changeset_revision=changeset_revision, tool_id=tool_dict[ 'id' ], display_for_install=display_for_install )}">View tool metadata</a></div></td><td>${tool_dict[ 'description' ]}</td> --- a/templates/webapps/community/repository/preview_tools_in_changeset.mako Mon Sep 19 16:07:53 2011 -0400 +++ b/templates/webapps/community/repository/preview_tools_in_changeset.mako Tue Sep 20 10:38:30 2011 -0400 @@ -64,7 +64,7 @@ <br/><br/><ul class="manage-table-actions"> - <a class="action-button" href="${h.url_for( controller='repository', action='install_repository_revision', repository_id=trans.security.encode_id( repository.id ), webapp='community', changeset_revision=changeset_revision, galaxy_url=galaxy_url )}">Install to local Galaxy</a> + <a class="action-button" href="${h.url_for( controller='repository', action='install_repository_revision', repository_id=trans.security.encode_id( repository.id ), webapp='community', changeset_revision=changeset_revision )}">Install to local Galaxy</a></ul> %if message: @@ -75,7 +75,7 @@ <div class="toolFormTitle">Repository ${repository.name}</div><div class="toolFormBody"> %if len( changeset_revision_select_field.options ) > 1: - <form name="change_revision" id="change_revision" action="${h.url_for( controller='repository', action='preview_tools_in_changeset', repository_id=trans.security.encode_id( repository.id ), galaxy_url=galaxy_url )}" method="post" > + <form name="change_revision" id="change_revision" action="${h.url_for( controller='repository', action='preview_tools_in_changeset', repository_id=trans.security.encode_id( repository.id ) )}" method="post" ><div class="form-row"><% if changeset_revision == repository.tip: @@ -98,4 +98,4 @@ </div></div><p/> -${render_repository_tools_and_workflows( metadata, display_for_install=display_for_install, galaxy_url=galaxy_url )} +${render_repository_tools_and_workflows( metadata, display_for_install=display_for_install )} --- a/templates/webapps/community/repository/tool_form.mako Mon Sep 19 16:07:53 2011 -0400 +++ b/templates/webapps/community/repository/tool_form.mako Tue Sep 20 10:38:30 2011 -0400 @@ -111,7 +111,7 @@ <br/><br/><ul class="manage-table-actions"> %if display_for_install: - <a class="action-button" href="${h.url_for( controller='repository', action='install_repository_revision', repository_id=trans.security.encode_id( repository.id ), webapp='community', changeset_revision=changeset_revision, galaxy_url=galaxy_url )}">Install to local Galaxy</a> + <a class="action-button" href="${h.url_for( controller='repository', action='install_repository_revision', repository_id=trans.security.encode_id( repository.id ), webapp='community', changeset_revision=changeset_revision )}">Install to local Galaxy</a> %else: %if is_new: <a class="action-button" href="${h.url_for( controller='upload', action='upload', repository_id=trans.security.encode_id( repository.id ), webapp='community' )}">Upload files to repository</a> --- a/templates/webapps/galaxy/admin/index.mako Mon Sep 19 16:07:53 2011 -0400 +++ b/templates/webapps/galaxy/admin/index.mako Tue Sep 20 10:38:30 2011 -0400 @@ -70,7 +70,7 @@ <div class="toolSectionBody"><div class="toolSectionBg"> %for name, url in trans.app.tool_shed_registry.tool_sheds.items(): - <div class="toolTitle"><a href="${h.url_for( controller='admin', action='browse_tool_shed', tool_shed_name=name, tool_shed_url=url )}" target="galaxy_main">${name}</a></div> + <div class="toolTitle"><a href="${h.url_for( controller='admin', action='browse_tool_shed', tool_shed_url=url )}" target="galaxy_main">${name}</a></div> %endfor </div></div> Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.