1 new changeset in galaxy-central: http://bitbucket.org/galaxy/galaxy-central/changeset/b68bbdc8dd14/ changeset: b68bbdc8dd14 user: greg date: 2011-07-21 15:43:49 summary: Several tool shed enhancements and fixes: 1) Add the ability to add an entry for a tool to the tool_data_table_conf.xml file in real time. This allows metadata to be generated for tools that use this feature. 2) Add a method to configure settings to the mercurial ui, and configure it such that all message output from mercurial are not displayed (we're in quiet mode). 3) Make sure that mercurial commits performed within exception blocks account for the user performing the commit. affected #: 6 files (7.5 KB) --- a/lib/galaxy/util/__init__.py Wed Jul 20 09:01:37 2011 -0400 +++ b/lib/galaxy/util/__init__.py Thu Jul 21 09:43:49 2011 -0400 @@ -127,7 +127,8 @@ '@' : '__at__', '\n' : '__cn__', '\r' : '__cr__', - '\t' : '__tc__' + '\t' : '__tc__', + '#' : '__pd__' } def restore_text(text): --- a/lib/galaxy/webapps/community/controllers/common.py Wed Jul 20 09:01:37 2011 -0400 +++ b/lib/galaxy/webapps/community/controllers/common.py Thu Jul 21 09:43:49 2011 -0400 @@ -81,7 +81,7 @@ status = 'done' repository = get_repository( trans, id ) repo_dir = repository.repo_path - repo = hg.repository( ui.ui(), repo_dir ) + repo = hg.repository( get_configured_ui(), repo_dir ) change_set = get_change_set( trans, repo, change_set_revision ) invalid_files = [] flush_needed = False @@ -214,10 +214,6 @@ if invalid_files: message = "Metadata cannot be defined for change set revision '%s'. Correct the following problems and reset metadata.<br/>" % str( change_set_revision ) for itc_tup in invalid_files: - # Handle the special case where a tool depends on a missing xxx.loc file by telliing - # the user to upload xxx.loc.sample to the repository so that it can be copied to - # ~/tool-data/xxx.loc. In this case, itc_tup[1] will be a message looking something like: - # [Errno 2] No such file or directory: '/Users/gvk/central/tool-data/blast2go.loc' tool_file = itc_tup[0] exception_msg = itc_tup[1] if exception_msg.find( 'No such file or directory' ) >= 0: @@ -226,10 +222,28 @@ missing_file = missing_file_items[-1].rstrip( '\'' ) correction_msg = "This file refers to a missing file <b>%s</b>. " % str( missing_file ) if exception_msg.find( '.loc' ) >= 0: + # Handle the special case where a tool depends on a missing xxx.loc file by telliing + # the user to upload xxx.loc.sample to the repository so that it can be copied to + # ~/tool-data/xxx.loc. In this case, exception_msg will look something like: + # [Errno 2] No such file or directory: '/Users/gvk/central/tool-data/blast2go.loc' sample_loc_file = '%s.sample' % str( missing_file ) correction_msg += "Upload a file named <b>%s</b> to the repository to correct this error." % sample_loc_file else: correction_msg += "Upload a file named <b>%s</b> to the repository to correct this error." % missing_file + elif exception_msg.find( 'Data table named' ) >= 0: + # Handle the special case where the tool requires an entry in the tool_data_table.conf file. + # In this case, exception_msg will look something like: + # Data table named 'tmap_indexes' is required by tool but not configured + exception_items = exception_msg.split() + name_attr = exception_items[3].lstrip( '\'' ).rstrip( '\'' ) + message += "<b>%s</b> - This tool requires an entry in the tool_data_table_conf.xml file. " % tool_file + message += "Complete and <b>Save</b> the form below to resolve this issue.<br/>" + return trans.response.send_redirect( web.url_for( controller='repository', + action='add_tool_data_table_entry', + name_attr=name_attr, + repository_id=id, + message=message, + status='error' ) ) else: correction_msg = exception_msg message += "<b>%s</b> - %s<br/>" % ( tool_file, correction_msg ) @@ -257,12 +271,21 @@ if not ( os.path.exists( os.path.join( tool_data_path, loc_file ) ) or os.path.exists( os.path.join( tool_data_path, sample_loc_file ) ) ): shutil.copy( os.path.abspath( filename ), os.path.join( tool_data_path, sample_loc_file ) ) shutil.copy( os.path.abspath( filename ), os.path.join( tool_data_path, loc_file ) ) +def get_configured_ui(): + # Configure any desired ui settings. + _ui = ui.ui() + # The following will suppress all messages. This is + # the same as adding the following setting to the repo + # hgrc file' [ui] section: + # quiet = True + _ui.setconfig( 'ui', 'quiet', True ) + return _ui def get_user( trans, id ): """Get a user from the database""" return trans.sa_session.query( trans.model.User ).get( trans.security.decode_id( id ) ) def handle_email_alerts( trans, repository ): repo_dir = repository.repo_path - repo = hg.repository( ui.ui(), repo_dir ) + repo = hg.repository( get_configured_ui(), repo_dir ) smtp_server = trans.app.config.smtp_server if smtp_server and repository.email_alerts: # Send email alert to users that want them. @@ -299,17 +322,19 @@ util.send_mail( frm, to, subject, body, trans.app.config ) except Exception, e: log.exception( "An error occurred sending a tool shed repository update alert by email." ) -def update_for_browsing( repository, current_working_dir, commit_message='' ): - # Make a copy of a repository's files for browsing. +def update_for_browsing( trans, repository, current_working_dir, commit_message='' ): + # Make a copy of a repository's files for browsing, remove from disk all files that + # are not tracked, and commit all added, modified or removed files that have not yet + # been committed. repo_dir = repository.repo_path - repo = hg.repository( ui.ui(), repo_dir ) + repo = hg.repository( get_configured_ui(), repo_dir ) # The following will delete the disk copy of only the files in the repository. #os.system( 'hg update -r null > /dev/null 2>&1' ) repo.ui.pushbuffer() commands.status( repo.ui, repo, all=True ) status_and_file_names = repo.ui.popbuffer().strip().split( "\n" ) # status_and_file_names looks something like: - # ['? MY_README_AGAIN', '? galaxy_tmap_tool/tmap-0.0.9.tar.gz', '? dna_filtering.py', 'C filtering.py', 'C filtering.xml'] + # ['? README', '? tmap_tool/tmap-0.0.9.tar.gz', '? dna_filtering.py', 'C filtering.py', 'C filtering.xml'] # The codes used to show the status of files are: # M = modified # A = added @@ -345,7 +370,7 @@ if not commit_message: commit_message = 'Committed changes to: %s' % ', '.join( files_to_commit ) repo.dirstate.write() - repo.commit( text=commit_message ) + repo.commit( user=trans.user.username, text=commit_message ) os.chdir( repo_dir ) os.system( 'hg update > /dev/null 2>&1' ) os.chdir( current_working_dir ) --- a/lib/galaxy/webapps/community/controllers/repository.py Wed Jul 20 09:01:37 2011 -0400 +++ b/lib/galaxy/webapps/community/controllers/repository.py Thu Jul 21 09:43:49 2011 -0400 @@ -312,7 +312,7 @@ if not os.path.exists( repository_path ): os.makedirs( repository_path ) # Create the local repository - repo = hg.repository( ui.ui(), repository_path, create=True ) + repo = hg.repository( get_configured_ui(), repository_path, create=True ) # Add an entry in the hgweb.config file for the local repository # This enables calls to repository.repo_path self.__add_hgweb_config_entry( trans, repository, repository_path ) @@ -406,7 +406,7 @@ # push_ssl = False # Since we support both http and https, we set push_ssl to False to override # the default (which is True) in the mercurial api. - repo = hg.repository( ui.ui(), path=repository.repo_path ) + repo = hg.repository( get_configured_ui(), path=repository.repo_path ) fp = repo.opener( 'hgrc', 'wb' ) fp.write( '[paths]\n' ) fp.write( 'default = .\n' ) @@ -423,9 +423,10 @@ status = params.get( 'status', 'done' ) commit_message = util.restore_text( params.get( 'commit_message', 'Deleted selected files' ) ) repository = get_repository( trans, id ) - repo = hg.repository( ui.ui(), repository.repo_path ) + repo = hg.repository( get_configured_ui(), repository.repo_path ) current_working_dir = os.getcwd() - update_for_browsing( repository, current_working_dir, commit_message=commit_message ) + # Update repository files for browsing. + update_for_browsing( trans, repository, current_working_dir, commit_message=commit_message ) return trans.fill_template( '/webapps/community/repository/browse_repository.mako', repo=repo, repository=repository, @@ -440,7 +441,7 @@ commit_message = util.restore_text( params.get( 'commit_message', 'Deleted selected files' ) ) repository = get_repository( trans, id ) repo_dir = repository.repo_path - repo = hg.repository( ui.ui(), repo_dir ) + repo = hg.repository( get_configured_ui(), repo_dir ) selected_files_to_delete = util.restore_text( params.get( 'selected_files_to_delete', '' ) ) if params.get( 'select_files_to_delete_button', False ): if selected_files_to_delete: @@ -450,7 +451,7 @@ tip = repository.tip for selected_file in selected_files_to_delete: repo_file = os.path.abspath( selected_file ) - commands.remove( repo.ui, repo, repo_file ) + commands.remove( repo.ui, repo, repo_file, force=True ) # Commit the change set. if not commit_message: commit_message = 'Deleted selected files' @@ -461,12 +462,12 @@ # tool shed environment, it occasionally throws a "TypeError: array item must be char" # exception. If this happens, we'll try the following. repo.dirstate.write() - repo.commit( text=commit_message ) + repo.commit( user=trans.user.username, text=commit_message ) handle_email_alerts( trans, repository ) # Update the repository files for browsing. - update_for_browsing( repository, current_working_dir, commit_message=commit_message ) + update_for_browsing( trans, repository, current_working_dir, commit_message=commit_message ) # Get the new repository tip. - repo = hg.repository( ui.ui(), repo_dir ) + repo = hg.repository( get_configured_ui(), repo_dir ) if tip != repository.tip: message = "The selected files were deleted from the repository." else: @@ -495,7 +496,7 @@ message = util.restore_text( params.get( 'message', '' ) ) status = params.get( 'status', 'done' ) repository = get_repository( trans, id ) - repo = hg.repository( ui.ui(), repository.repo_path ) + repo = hg.repository( get_configured_ui(), repository.repo_path ) avg_rating, num_ratings = self.get_ave_item_rating_data( trans.sa_session, repository, webapp_model=trans.model ) display_reviews = util.string_as_bool( params.get( 'display_reviews', False ) ) alerts = params.get( 'alerts', '' ) @@ -545,7 +546,7 @@ status = params.get( 'status', 'done' ) repository = get_repository( trans, id ) repo_dir = repository.repo_path - repo = hg.repository( ui.ui(), repo_dir ) + repo = hg.repository( get_configured_ui(), repo_dir ) repo_name = util.restore_text( params.get( 'repo_name', repository.name ) ) description = util.restore_text( params.get( 'description', repository.description ) ) long_description = util.restore_text( params.get( 'long_description', repository.long_description ) ) @@ -673,7 +674,7 @@ message = util.restore_text( params.get( 'message', '' ) ) status = params.get( 'status', 'done' ) repository = get_repository( trans, id ) - repo = hg.repository( ui.ui(), repository.repo_path ) + repo = hg.repository( get_configured_ui(), repository.repo_path ) changesets = [] for changeset in repo.changelog: ctx = repo.changectx( changeset ) @@ -701,7 +702,7 @@ message = util.restore_text( params.get( 'message', '' ) ) status = params.get( 'status', 'done' ) repository = get_repository( trans, id ) - repo = hg.repository( ui.ui(), repository.repo_path ) + repo = hg.repository( get_configured_ui(), repository.repo_path ) ctx = get_change_set( trans, repo, ctx_str ) if ctx is None: message = "Repository does not include changeset revision '%s'." % str( ctx_str ) @@ -745,7 +746,7 @@ message='Select a repository to rate', status='error' ) ) repository = get_repository( trans, id ) - repo = hg.repository( ui.ui(), repository.repo_path ) + repo = hg.repository( get_configured_ui(), repository.repo_path ) if repository.user == trans.user: return trans.response.send_redirect( web.url_for( controller='repository', action='browse_repositories', @@ -815,6 +816,98 @@ message=message, status=status ) ) @web.expose + def add_tool_data_table_entry( self, trans, name_attr, repository_id, **kwd ): + params = util.Params( kwd ) + message = util.restore_text( params.get( 'message', '' ) ) + status = params.get( 'status', 'done' ) + comment_char = util.restore_text( params.get( 'comment_char', '#' ) ) + loc_filename = util.restore_text( params.get( 'loc_filename', '' ) ) + repository = get_repository( trans, repository_id ) + repo = hg.repository( get_configured_ui(), repository.repo_path ) + column_fields = self.__get_column_fields( **kwd ) + if params.get( 'add_field_button', False ): + # Add a field + field_index = len( column_fields ) + 1 + field_tup = ( '%i_field_name' % field_index, '' ) + column_fields.append( field_tup ) + elif params.get( 'remove_button', False ): + # Delete a field - find the index of the field to be removed from the remove button label + index = int( kwd[ 'remove_button' ].split( ' ' )[2] ) - 1 + tup_to_remove = column_fields[ index ] + column_fields.remove( tup_to_remove ) + # Re-number field tups + new_column_fields = [] + for field_index, old_field_tup in enumerate( column_fields ): + name = '%i_field_name' % ( field_index + 1 ) + value = old_field_tup[1] + new_column_fields.append( ( name, value ) ) + column_fields = new_column_fields + elif params.get( 'add_tool_data_table_entry_button', False ): + # Add an entry to the end of the tool_data_table_conf.xml file + tdt_config = "%s/tool_data_table_conf.xml" % trans.app.config.root + if os.path.exists( tdt_config ): + # Make a backup of the file since we're going to be changing it. + today = date.today() + backup_date = today.strftime( "%Y_%m_%d" ) + tdt_config_copy = '%s/tool_data_table_conf.xml_%s_backup' % ( trans.app.config.root, backup_date ) + shutil.copy( os.path.abspath( tdt_config ), os.path.abspath( tdt_config_copy ) ) + # Generate the string of column names + column_names = ', '.join( [ column_tup[1] for column_tup in column_fields ] ) + # Write each line of the tool_data_table_conf.xml file, except the last line to a temp file. + fh = tempfile.NamedTemporaryFile( 'wb' ) + tmp_filename = fh.name + fh.close() + new_tdt_config = open( tmp_filename, 'wb' ) + for i, line in enumerate( open( tdt_config, 'rb' ) ): + if line.startswith( '</tables>' ): + break + new_tdt_config.write( line ) + new_tdt_config.write( ' <!-- Location of %s files -->\n' % name_attr ) + new_tdt_config.write( ' <table name="%s" comment_char="%s">\n' % ( name_attr, comment_char ) ) + new_tdt_config.write( ' <columns>%s</columns>\n' % column_names ) + new_tdt_config.write( ' <file path="tool-data/%s" />\n' % loc_filename ) + new_tdt_config.write( ' </table>\n' ) + # Now write the last line of the file + new_tdt_config.write( '</tables>\n' ) + new_tdt_config.close() + shutil.move( tmp_filename, os.path.abspath( tdt_config ) ) + # Reload the tool_data_table_conf entries + trans.app.tool_data_tables = galaxy.tools.data.ToolDataTableManager( trans.app.config.tool_data_table_config_path ) + message = "The new entry has been added to the tool_data_table_conf.xml file, so click the <b>Reset metadata</b> button below." + # TODO: what if ~/tool-data/<loc_filename> doesn't exist? We need to figure out how to + # force the user to upload it's sample to the repository in order to generate metadata. + return trans.response.send_redirect( web.url_for( controller='repository', + action='manage_repository', + id=repository_id, + message=message, + status=status ) ) + return trans.fill_template( '/webapps/community/repository/add_tool_data_table_entry.mako', + name_attr=name_attr, + repository=repository, + comment_char=comment_char, + loc_filename=loc_filename, + column_fields=column_fields, + message=message, + status=status ) + def __get_column_fields( self, **kwd ): + ''' + Return a dictionary of the user-entered form fields representing columns + in the location file. + ''' + params = util.Params( kwd ) + column_fields = [] + index = 0 + while True: + name = '%i_field_name' % ( index + 1 ) + if kwd.has_key( name ): + value = util.restore_text( params.get( name, '' ) ) + field_tup = ( name, value ) + index += 1 + column_fields.append( field_tup ) + else: + break + return column_fields + @web.expose def display_tool( self, trans, repository_id, tool_config, **kwd ): params = util.Params( kwd ) message = util.restore_text( params.get( 'message', '' ) ) --- a/lib/galaxy/webapps/community/controllers/upload.py Wed Jul 20 09:01:37 2011 -0400 +++ b/lib/galaxy/webapps/community/controllers/upload.py Thu Jul 21 09:43:49 2011 -0400 @@ -27,7 +27,7 @@ repository_id = params.get( 'repository_id', '' ) repository = get_repository( trans, repository_id ) repo_dir = repository.repo_path - repo = hg.repository( ui.ui(), repo_dir ) + repo = hg.repository( get_configured_ui(), repo_dir ) uncompress_file = util.string_as_bool( params.get( 'uncompress_file', 'true' ) ) remove_repo_files_not_in_tar = util.string_as_bool( params.get( 'remove_repo_files_not_in_tar', 'true' ) ) uploaded_file = None @@ -87,7 +87,6 @@ # Move the uploaded file to the load_point within the repository hierarchy. shutil.move( uploaded_file_name, full_path ) commands.add( repo.ui, repo, full_path ) - """ try: commands.commit( repo.ui, repo, full_path, user=trans.user.username, message=commit_message ) except Exception, e: @@ -95,17 +94,15 @@ # tool shed environment, it occasionally throws a "TypeError: array item must be char" # exception. If this happens, we'll try the following. repo.dirstate.write() - repo.commit( text=commit_message ) - """ + repo.commit( user=trans.user.username, text=commit_message ) if full_path.endswith( '.loc.sample' ): # Handle the special case where a xxx.loc.sample file is # being uploaded by copying it to ~/tool-data/xxx.loc. copy_sample_loc_file( trans, full_path ) handle_email_alerts( trans, repository ) if ok: - # Update the repository files for browsing, a by-product of doing this - # is eliminating unwanted files from the repository directory. - update_for_browsing( repository, current_working_dir, commit_message=commit_message ) + # Update the repository files for browsing. + update_for_browsing( trans, repository, current_working_dir, commit_message=commit_message ) # Get the new repository tip. if tip != repository.tip: if ( isgzip or isbz2 ) and uncompress_file: @@ -148,7 +145,7 @@ def upload_tar( self, trans, repository, tar, uploaded_file, upload_point, remove_repo_files_not_in_tar, commit_message ): # Upload a tar archive of files. repo_dir = repository.repo_path - repo = hg.repository( ui.ui(), repo_dir ) + repo = hg.repository( get_configured_ui(), repo_dir ) files_to_remove = [] ok, message = self.__check_archive( tar ) if not ok: @@ -185,7 +182,7 @@ for repo_file in files_to_remove: # Remove files in the repository (relative to the upload point) # that are not in the uploaded archive. - commands.remove( repo.ui, repo, repo_file ) + commands.remove( repo.ui, repo, repo_file, force=True ) for filename_in_archive in filenames_in_archive: commands.add( repo.ui, repo, filename_in_archive ) if filename_in_archive.endswith( '.loc.sample' ): @@ -199,7 +196,7 @@ # tool shed environment, it occasionally throws a "TypeError: array item must be char" # exception. If this happens, we'll try the following. repo.dirstate.write() - repo.commit( text=commit_message ) + repo.commit( user=trans.user.username, text=commit_message ) handle_email_alerts( trans, repository ) return True, '', files_to_remove def uncompress( self, repository, uploaded_file_name, uploaded_file_filename, isgzip, isbz2 ): --- a/templates/webapps/community/repository/create_repository.mako Wed Jul 20 09:01:37 2011 -0400 +++ b/templates/webapps/community/repository/create_repository.mako Thu Jul 21 09:43:49 2011 -0400 @@ -17,15 +17,15 @@ <div class="toolForm"><div class="toolFormTitle">Create Repository</div><div class="toolFormBody"> - <form name="create_repository_form" id="create_repository_form" action="${h.url_for( action='create_repository' )}" method="post" > + <form name="create_repository_form" id="create_repository_form" action="${h.url_for( controller='repository', action='create_repository' )}" method="post" ><div class="form-row"><label>Name:</label> - <input name="name" type="textfield" value="${name}" size=40"/> + <input name="name" type="textfield" value="${name}" size="40"/><div style="clear: both"></div></div><div class="form-row"><label>Synopsis:</label> - <input name="description" type="textfield" value="${description}" size=80"/> + <input name="description" type="textfield" value="${description}" size="80"/><div style="clear: both"></div></div><div class="form-row"> Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.