1 new changeset in galaxy-central: http://bitbucket.org/galaxy/galaxy-central/changeset/a0f021288c57/ changeset: a0f021288c57 user: greg date: 2011-10-17 18:22:27 summary: Enhance automatic tool shed repository installation to enable installation of multiple repositories using a multi-select frid layout instead of installing just a single repository at a time. affected #: 5 files (-1 bytes) --- a/lib/galaxy/web/controllers/admin.py Fri Oct 14 15:20:12 2011 -0400 +++ b/lib/galaxy/web/controllers/admin.py Mon Oct 17 12:22:27 2011 -0400 @@ -688,8 +688,8 @@ @web.require_admin def browse_tool_shed( self, trans, **kwd ): tool_shed_url = kwd[ 'tool_shed_url' ] - galaxy_url = trans.request.host - url = '%s/repository/browse_downloadable_repositories?galaxy_url=%s&webapp=community' % ( tool_shed_url, galaxy_url ) + trans.set_cookie( trans.request.host, name='toolshedgalaxyurl' ) + url = '%s/repository/browse_downloadable_repositories?webapp=galaxy' % tool_shed_url return trans.response.send_redirect( url ) @web.expose @web.require_admin @@ -706,11 +706,7 @@ params = util.Params( kwd ) message = util.restore_text( params.get( 'message', '' ) ) status = params.get( 'status', 'done' ) - tool_shed_url = kwd[ 'tool_shed_url' ] - name = kwd[ 'name' ] - description = kwd[ 'description' ] - changeset_revision = kwd[ 'changeset_revision' ] - repository_clone_url = kwd[ 'repository_clone_url' ] + tool_shed_url = trans.get_cookie( name='galaxytoolshedurl' ) if kwd.get( 'select_tool_panel_section_button', False ): shed_tool_conf = kwd[ 'shed_tool_conf' ] # Get the tool path. @@ -720,94 +716,104 @@ if 'tool_panel_section' in kwd: section_key = 'section_%s' % kwd[ 'tool_panel_section' ] tool_section = trans.app.toolbox.tool_panel[ section_key ] + # Get the number of repositories to clone from a cookie. + num_repos_to_clone = int( trans.get_cookie( name='numrepostoclone' ) ) # Clone the repository to the configured location. current_working_dir = os.getcwd() - clone_dir = os.path.join( tool_path, self.__generate_tool_path( repository_clone_url, changeset_revision ) ) - if os.path.exists( clone_dir ): - # Repository and revision has already been cloned. - # TODO: implement the ability to re-install or revert an existing repository. - message = 'Revision <b>%s</b> of repository <b>%s</b> has already been installed. Updating an existing repository is not yet supported.' % \ - ( changeset_revision, name ) - status = 'error' - else: - os.makedirs( clone_dir ) - log.debug( 'Cloning %s...' % repository_clone_url ) - cmd = 'hg clone %s' % repository_clone_url - tmp_name = tempfile.NamedTemporaryFile().name - tmp_stderr = open( tmp_name, 'wb' ) - os.chdir( clone_dir ) - proc = subprocess.Popen( args=cmd, shell=True, stderr=tmp_stderr.fileno() ) - returncode = proc.wait() - os.chdir( current_working_dir ) - tmp_stderr.close() - if returncode == 0: - # Add a new record to the tool_shed_repository table. - tool_shed_repository = self.__create_tool_shed_repository( trans, - name, - description, - changeset_revision, - repository_clone_url ) - # Update the cloned repository to changeset_revision. - repo_files_dir = os.path.join( clone_dir, name ) - log.debug( 'Updating cloned repository to revision "%s"...' % changeset_revision ) - cmd = 'hg update -r %s' % changeset_revision + for index in range( 0, num_repos_to_clone ): + cookie_name = 'toolshedrepository%i' % index + clone_repo_info_str = trans.get_cookie( name=cookie_name ) + clone_repo_info_items = clone_repo_info_str.split( '&' ) + name = clone_repo_info_items[ 0 ] + description = clone_repo_info_items[ 1 ] + repository_clone_url = clone_repo_info_items[ 2 ] + changeset_revision = clone_repo_info_items[ 3 ] + clone_dir = os.path.join( tool_path, self.__generate_tool_path( repository_clone_url, changeset_revision ) ) + if os.path.exists( clone_dir ): + # Repository and revision has already been cloned. + # TODO: implement the ability to re-install or revert an existing repository. + message += 'Revision <b>%s</b> of repository <b>%s</b> has already been installed. Updating an existing repository is not yet supported.<br/>' % \ + ( changeset_revision, name ) + status = 'error' + else: + os.makedirs( clone_dir ) + log.debug( 'Cloning %s...' % repository_clone_url ) + cmd = 'hg clone %s' % repository_clone_url tmp_name = tempfile.NamedTemporaryFile().name tmp_stderr = open( tmp_name, 'wb' ) - os.chdir( repo_files_dir ) - proc = subprocess.Popen( cmd, shell=True, stderr=tmp_stderr.fileno() ) + os.chdir( clone_dir ) + proc = subprocess.Popen( args=cmd, shell=True, stderr=tmp_stderr.fileno() ) returncode = proc.wait() os.chdir( current_working_dir ) tmp_stderr.close() if returncode == 0: - sample_files, repository_tools_tups = self.__get_repository_tools_and_sample_files( trans, tool_path, repo_files_dir ) - if repository_tools_tups: - # Handle missing data table entries for tool parameters that are dynamically generated select lists. - repository_tools_tups = self.__handle_missing_data_table_entry( trans, tool_path, sample_files, repository_tools_tups ) - # Handle missing index files for tool parameters that are dynamically generated select lists. - repository_tools_tups = self.__handle_missing_index_file( trans, tool_path, sample_files, repository_tools_tups ) - # Handle tools that use fabric scripts to install dependencies. - self.__handle_tool_dependencies( current_working_dir, repo_files_dir, repository_tools_tups ) - # Generate an in-memory tool conf section that includes the new tools. - new_tool_section = self.__generate_tool_panel_section( name, - repository_clone_url, + # Add a new record to the tool_shed_repository table. + tool_shed_repository = self.__create_tool_shed_repository( trans, + name, + description, changeset_revision, - tool_section, - repository_tools_tups ) - # Create a temporary file to persist the in-memory tool section - # TODO: Figure out how to do this in-memory using xml.etree. - tmp_name = tempfile.NamedTemporaryFile().name - persisted_new_tool_section = open( tmp_name, 'wb' ) - persisted_new_tool_section.write( new_tool_section ) - persisted_new_tool_section.close() - # Parse the persisted tool panel section - tree = ElementTree.parse( tmp_name ) - root = tree.getroot() - ElementInclude.include( root ) - # Load the tools in the section into the tool panel. - trans.app.toolbox.load_section_tag_set( root, trans.app.toolbox.tool_panel, tool_path ) - # Remove the temporary file - try: - os.unlink( tmp_name ) - except: - pass - # Append the new section to the shed_tool_config file. - self.__add_shed_tool_conf_entry( trans, shed_tool_conf, new_tool_section ) - if trans.app.toolbox_search.enabled: - # If search support for tools is enabled, index the new installed tools. - trans.app.toolbox_search = ToolBoxSearch( trans.app.toolbox ) - message = 'Revision <b>%s</b> of repository <b>%s</b> has been installed in tool panel section <b>%s</b>.' % \ - ( changeset_revision, name, tool_section.name ) - return trans.show_ok_message( message ) + repository_clone_url ) + # Update the cloned repository to changeset_revision. + repo_files_dir = os.path.join( clone_dir, name ) + log.debug( 'Updating cloned repository to revision "%s"...' % changeset_revision ) + cmd = 'hg update -r %s' % changeset_revision + tmp_name = tempfile.NamedTemporaryFile().name + tmp_stderr = open( tmp_name, 'wb' ) + os.chdir( repo_files_dir ) + proc = subprocess.Popen( cmd, shell=True, stderr=tmp_stderr.fileno() ) + returncode = proc.wait() + os.chdir( current_working_dir ) + tmp_stderr.close() + if returncode == 0: + sample_files, repository_tools_tups = self.__get_repository_tools_and_sample_files( trans, tool_path, repo_files_dir ) + if repository_tools_tups: + # Handle missing data table entries for tool parameters that are dynamically generated select lists. + repository_tools_tups = self.__handle_missing_data_table_entry( trans, tool_path, sample_files, repository_tools_tups ) + # Handle missing index files for tool parameters that are dynamically generated select lists. + repository_tools_tups = self.__handle_missing_index_file( trans, tool_path, sample_files, repository_tools_tups ) + # Handle tools that use fabric scripts to install dependencies. + self.__handle_tool_dependencies( current_working_dir, repo_files_dir, repository_tools_tups ) + # Generate an in-memory tool conf section that includes the new tools. + new_tool_section = self.__generate_tool_panel_section( name, + repository_clone_url, + changeset_revision, + tool_section, + repository_tools_tups ) + # Create a temporary file to persist the in-memory tool section + # TODO: Figure out how to do this in-memory using xml.etree. + tmp_name = tempfile.NamedTemporaryFile().name + persisted_new_tool_section = open( tmp_name, 'wb' ) + persisted_new_tool_section.write( new_tool_section ) + persisted_new_tool_section.close() + # Parse the persisted tool panel section + tree = ElementTree.parse( tmp_name ) + root = tree.getroot() + ElementInclude.include( root ) + # Load the tools in the section into the tool panel. + trans.app.toolbox.load_section_tag_set( root, trans.app.toolbox.tool_panel, tool_path ) + # Remove the temporary file + try: + os.unlink( tmp_name ) + except: + pass + # Append the new section to the shed_tool_config file. + self.__add_shed_tool_conf_entry( trans, shed_tool_conf, new_tool_section ) + if trans.app.toolbox_search.enabled: + # If search support for tools is enabled, index the new installed tools. + trans.app.toolbox_search = ToolBoxSearch( trans.app.toolbox ) + message += 'Revision <b>%s</b> of repository <b>%s</b> has been loaded into tool panel section <b>%s</b>.<br/>' % \ + ( changeset_revision, name, tool_section.name ) + #return trans.show_ok_message( message ) + else: + tmp_stderr = open( tmp_name, 'rb' ) + message += '%s<br/>' % tmp_stderr.read() + tmp_stderr.close() + status = 'error' else: tmp_stderr = open( tmp_name, 'rb' ) - message = tmp_stderr.read() + message += '%s<br/>' % tmp_stderr.read() tmp_stderr.close() status = 'error' - else: - tmp_stderr = open( tmp_name, 'rb' ) - message = tmp_stderr.read() - tmp_stderr.close() - status = 'error' else: message = 'Choose the section in your tool panel to contain the installed tools.' status = 'error' @@ -819,11 +825,6 @@ shed_tool_conf_select_field = None tool_panel_section_select_field = build_tool_panel_section_select_field( trans ) return trans.fill_template( '/admin/select_tool_panel_section.mako', - tool_shed_url=tool_shed_url, - name=name, - description=description, - changeset_revision=changeset_revision, - repository_clone_url=repository_clone_url, shed_tool_conf=shed_tool_conf, shed_tool_conf_select_field=shed_tool_conf_select_field, tool_panel_section_select_field=tool_panel_section_select_field, @@ -835,11 +836,12 @@ params = util.Params( kwd ) repository_id = params.get( 'id', None ) repository = get_repository( trans, repository_id ) - galaxy_url = trans.request.host + trans.set_cookie( trans.request.host, name='toolshedgalaxyurl' ) # Send a request to the relevant tool shed to see if there are any updates. # TODO: support https in the following url. - url = 'http://%s/repository/check_for_updates?galaxy_url=%s&name=%s&owner=%s&changeset_revision=%s&webapp=community' % \ - ( repository.tool_shed, galaxy_url, repository.name, repository.owner, repository.changeset_revision ) + # TODO: Set cookies for name, owner, changeset-revision instead of sending in request + url = 'http://%s/repository/check_for_updates?name=%s&owner=%s&changeset_revision=%s&webapp=community' % \ + ( repository.tool_shed, repository.name, repository.owner, repository.changeset_revision ) return trans.response.send_redirect( url ) @web.expose @web.require_admin --- a/lib/galaxy/web/controllers/workflow.py Fri Oct 14 15:20:12 2011 -0400 +++ b/lib/galaxy/web/controllers/workflow.py Mon Oct 17 12:22:27 2011 -0400 @@ -1150,17 +1150,20 @@ if missing_tool_tups: if trans.user_is_admin(): # A required tool is not available in the local Galaxy instance. - # TODO: It would sure be nice to be able to redirec to a mako tempalte here that displays a nice - # page including the links to the configured tool shed instead of this stupd message, but trying + # TODO: It would sure be nice to be able to redirect to a mako template here that displays a nice + # page including the links to the configured tool sheds instead of this message, but trying # to get the panels back is a nightmare since workflow eliminates the Galaxy panels. Someone - #involved in workflow development needs to figure out what it will take to get the Galaxy panels back... + # involved in workflow development needs to figure out what it will take to be able to switch + # back and forth between Galaxy (with panels ) and the workflow view (without panels ), having + # the Galaxy panels displayed whenever in Galaxy. + trans.set_cookie( trans.request.host, name='toolshedgalaxyurl' ) galaxy_url = trans.request.host message += "The workflow requires the following tools that are not available in this Galaxy instance." message += "You can likely install the required tools from one of the Galaxy tool sheds listed below.<br/><br/>" for tool_shed_name, tool_shed_url in trans.app.tool_shed_registry.tool_sheds.items(): if tool_shed_url.endswith( '/' ): tool_shed_url = tool_shed_url.rstrip( '/' ) - url = '%s/repository/find_tools?galaxy_url=%s&webapp=community' % ( tool_shed_url, galaxy_url ) + url = '%s/repository/find_tools?galaxy_url=%s&webapp=galaxy' % ( tool_shed_url, galaxy_url ) for missing_tool_tup in missing_tool_tups: missing_tool_id = missing_tool_tup[0] url += '&tool_id=%s' % missing_tool_id --- a/lib/galaxy/webapps/community/controllers/repository.py Fri Oct 14 15:20:12 2011 -0400 +++ b/lib/galaxy/webapps/community/controllers/repository.py Mon Oct 17 12:22:27 2011 -0400 @@ -284,8 +284,9 @@ # Grid definition title = "Repositories with required tools" operations = [ grids.GridOperation( "Preview and install tools", - url_args = dict( action='find_tools', - operation='install_tools' ), + url_args = dict( controller='repository', + action='find_tools', + webapp='community' ), allow_multiple=True, async_compatible=False ) ] @@ -383,6 +384,7 @@ params = util.Params( kwd ) message = util.restore_text( params.get( 'message', '' ) ) status = params.get( 'status', 'done' ) + webapp = params.get( 'webapp', 'galaxy' ) # Set the toolshedgalaxyurl cookie so we can get back # to the calling local Galaxy instance. galaxy_url = kwd.get( 'galaxy_url', None ) @@ -390,11 +392,11 @@ trans.set_cookie( galaxy_url, name='toolshedgalaxyurl' ) if 'operation' in kwd: operation = kwd[ 'operation' ].lower() + is_admin = trans.user_is_admin() if operation == "view_or_manage_repository": repository_metadata = get_repository_metadata_by_id( trans, kwd[ 'id' ] ) repository_id = trans.security.encode_id( repository_metadata.repository.id ) repository = get_repository( trans, repository_id ) - is_admin = trans.user_is_admin() # The received id is a RepositoryMetadata.id, so we have to get the repository id. kwd[ 'id' ] = repository_id kwd[ 'changeset_revision' ] = repository_metadata.changeset_revision @@ -407,7 +409,17 @@ action='view_repository', **kwd ) ) if operation == "preview and install tools": - pass + galaxy_url = trans.get_cookie( name='toolshedgalaxyurl' ) + trans.set_cookie( trans.request.host, name='galaxytoolshedurl' ) + # TODO: support https in the following url. + url = 'http://%s/admin/install_tool_shed_repository' % ( galaxy_url ) + repos_to_clone = util.listify( kwd[ 'id' ] ) + num_repos_to_clone = len( repos_to_clone ) + trans.set_cookie( num_repos_to_clone, name='numrepostoclone' ) + for index in range( 0, num_repos_to_clone ): + repository_metadata_id = repos_to_clone[ index ] + self._set_clone_repository_cookie( trans, index, repository_metadata_id ) + return trans.response.send_redirect( url ) tool_ids = [ tid.lower() for tid in util.listify( kwd.get( 'tool_id', '' ) ) ] tool_names = [ tn.lower() for tn in util.listify( kwd.get( 'tool_name', '' ) ) ] tool_versions = [ tv.lower() for tv in util.listify( kwd.get( 'tool_version', '' ) ) ] @@ -499,6 +511,7 @@ else: tool_version = '' return trans.fill_template( '/webapps/community/repository/find_tools.mako', + webapp=webapp, tool_id=tool_id, tool_name=tool_name, tool_version=tool_version, @@ -535,6 +548,16 @@ found = ( tool_version == tool_dict_tool_version and tool_name == tool_dict_tool_name ) or \ ( not exact_matches_checked and tool_dict_tool_version.find( tool_version ) >= 0 and tool_dict_tool_name.find( tool_name ) >= 0 ) return found + def _set_clone_repository_cookie( self, trans, index, repository_metadata_id ): + cookie_name = 'toolshedrepository%i' % index + repository_metadata = get_repository_metadata_by_id( trans, repository_metadata_id ) + repository = get_repository( trans, trans.security.encode_id( repository_metadata.repository_id ) ) + repository_id = trans.security.encode_id( repository.id ) + changeset_revision = repository_metadata.changeset_revision + # Redirect back to local Galaxy to perform install. + repository_clone_url = generate_clone_url( trans, repository_id ) + cookie_value = '%s&%s&%s&%s' % ( repository.name, repository.description, repository_clone_url, changeset_revision ) + trans.set_cookie( cookie_value, name=cookie_name ) @web.expose def preview_tools_in_changeset( self, trans, repository_id, **kwd ): params = util.Params( kwd ) @@ -570,11 +593,11 @@ repository = get_repository( trans, repository_id ) changeset_revision = util.restore_text( params.get( 'changeset_revision', repository.tip ) ) # Redirect back to local Galaxy to perform install. - tool_shed_url = trans.request.host + trans.set_cookie( trans.request.host, name='galaxytoolshedurl' ) repository_clone_url = generate_clone_url( trans, repository_id ) # TODO: support https in the following url. - url = 'http://%s/admin/install_tool_shed_repository?tool_shed_url=%s&name=%s&description=%s&repository_clone_url=%s&changeset_revision=%s' % \ - ( galaxy_url, tool_shed_url, repository.name, repository.description, repository_clone_url, changeset_revision ) + url = 'http://%s/admin/install_tool_shed_repository?name=%s&description=%s&repository_clone_url=%s&changeset_revision=%s' % \ + ( galaxy_url, repository.name, repository.description, repository_clone_url, changeset_revision ) return trans.response.send_redirect( url ) @web.expose def check_for_updates( self, trans, **kwd ): @@ -588,14 +611,11 @@ owner = params.get( 'owner', None ) changeset_revision = params.get( 'changeset_revision', None ) webapp = params.get( 'webapp', None ) - tool_shed_url = trans.request.host + trans.set_cookie( trans.request.host, name='galaxytoolshedurl' ) # Start building up the url to redirect back to the calling Galaxy instance. # TODO: support https in the following url. - url = 'http://%s/admin/update_to_changeset_revision?tool_shed_url=%s' % ( galaxy_url, tool_shed_url ) + url = 'http://%s/admin/update_to_changeset_revision' % galaxy_url repository = get_repository_by_name_and_owner( trans, name, owner ) - #if error: - # url += '&message=%s&status=error' % message - #else: url += '&name=%s&owner=%s&changeset_revision=%s&latest_changeset_revision=' % \ ( repository.name, repository.user.username, changeset_revision ) if changeset_revision == repository.tip: --- a/templates/admin/select_tool_panel_section.mako Fri Oct 14 15:20:12 2011 -0400 +++ b/templates/admin/select_tool_panel_section.mako Mon Oct 17 12:22:27 2011 -0400 @@ -23,9 +23,9 @@ <br/><div class="toolForm"> - <div class="toolFormTitle">Install tools from repository '${name}'</div> + <div class="toolFormTitle">Load tools into tool panel</div><div class="toolFormBody"> - <form name="select_tool_panel_section" id="select_tool_panel_section" action="${h.url_for( controller='admin', action='install_tool_shed_repository', tool_shed_url=tool_shed_url, name=name, description=description, changeset_revision=changeset_revision, repository_clone_url=repository_clone_url )}" method="post" > + <form name="select_tool_panel_section" id="select_tool_panel_section" action="${h.url_for( controller='admin', action='install_tool_shed_repository' )}" method="post" > %if shed_tool_conf_select_field: <div class="form-row"><label>Shed tool configuration file:</label> --- a/templates/webapps/community/repository/find_tools.mako Fri Oct 14 15:20:12 2011 -0400 +++ b/templates/webapps/community/repository/find_tools.mako Mon Oct 17 12:22:27 2011 -0400 @@ -23,7 +23,7 @@ strings if these types of strings are entered in more than one field. </div><div style="clear: both"></div> - <form name="find_tools" id="find_tools" action="${h.url_for( controller='repository', action='find_tools' )}" method="post" > + <form name="find_tools" id="find_tools" action="${h.url_for( controller='repository', action='find_tools', webapp=webapp )}" method="post" ><div class="form-row"><label>Tool id:</label><input name="tool_id" type="textfield" value="${tool_id}" size="40"/> Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.