[galaxy-commits] commit/galaxy-central: greg: Change many tool shed methods to use the mercurial api. Fix 2 bugs introduced when email alerts were added - unauthenticated users can once again browse categories / repositories.

1 new changeset in galaxy-central: http://bitbucket.org/galaxy/galaxy-central/changeset/dca79f6fb1ec/ changeset: dca79f6fb1ec user: greg date: 2011-06-28 16:24:49 summary: Change many tool shed methods to use the mercurial api. Fix 2 bugs introduced when email alerts were added - unauthenticated users can once again browse categories / repositories. affected #: 11 files (14.7 KB) --- a/.hgignore Mon Jun 27 17:25:32 2011 -0400 +++ b/.hgignore Tue Jun 28 10:24:49 2011 -0400 @@ -13,6 +13,7 @@ # Database stuff database/beaker_sessions +database/community_files database/compiled_templates database/files database/pbs --- a/lib/galaxy/webapps/community/controllers/common.py Mon Jun 27 17:25:32 2011 -0400 +++ b/lib/galaxy/webapps/community/controllers/common.py Tue Jun 28 10:24:49 2011 -0400 @@ -1,15 +1,13 @@ -import os, tarfile, tempfile, shutil, string, socket +import os, string, socket, logging from time import strftime from datetime import * from galaxy.util.json import from_json_string, to_json_string from galaxy.web.base.controller import * from galaxy.webapps.community import model from galaxy.model.orm import * -from galaxy.web.framework.helpers import time_ago, iff, grids -from galaxy.web.form_builder import SelectField from galaxy.model.item_attrs import UsesItemRatings from mercurial import hg, ui -import logging + log = logging.getLogger( __name__ ) email_alert_template = """ @@ -81,42 +79,9 @@ def get_user( trans, id ): """Get a user from the database""" return trans.sa_session.query( trans.model.User ).get( trans.security.decode_id( id ) ) -def hg_add( trans, current_working_dir, cloned_repo_dir ): - # Add files to a cloned repository. If they're already tracked, this should do nothing. - os.chdir( cloned_repo_dir ) - os.system( 'hg add > /dev/null 2>&1' ) - os.chdir( current_working_dir ) -def hg_clone( trans, repository, current_working_dir ): - # Make a clone of a repository in a temporary location. - repo_dir = repository.repo_path - tmp_dir = tempfile.mkdtemp() - tmp_archive_dir = os.path.join( tmp_dir, 'tmp_archive_dir' ) - if not os.path.exists( tmp_archive_dir ): - os.makedirs( tmp_archive_dir ) - cmd = "hg clone %s > /dev/null 2>&1" % os.path.abspath( repo_dir ) - os.chdir( tmp_archive_dir ) - os.system( cmd ) - os.chdir( current_working_dir ) - cloned_repo_dir = os.path.join( tmp_archive_dir, 'repo_%d' % repository.id ) - return tmp_dir, cloned_repo_dir -def hg_commit( commit_message, current_working_dir, cloned_repo_dir ): - # Commit a change set to a cloned repository. - if not commit_message: - commit_message = "No commit message" - os.chdir( cloned_repo_dir ) - os.system( "hg commit -m '%s' > /dev/null 2>&1" % commit_message ) - os.chdir( current_working_dir ) -def hg_push( trans, repository, current_working_dir, cloned_repo_dir ): - # Push a change set from a cloned repository to a master repository. +def handle_email_alerts( trans, repository ): repo_dir = repository.repo_path repo = hg.repository( ui.ui(), repo_dir ) - # We want these change sets to be associated with the owner of the repository, so we'll - # set the HGUSER environment variable accordingly. - os.environ[ 'HGUSER' ] = trans.user.username - cmd = "hg push %s > /dev/null 2>&1" % os.path.abspath( repo_dir ) - os.chdir( cloned_repo_dir ) - os.system( cmd ) - os.chdir( current_working_dir ) smtp_server = trans.app.config.smtp_server if smtp_server and repository.email_alerts: # Send email alert to users that want them. @@ -152,20 +117,23 @@ try: util.send_mail( frm, to, subject, body, trans.app.config ) except Exception, e: - log.exception( "An error occurred sending a tool shed repository update alert by email." ) -def hg_remove( file_path, current_working_dir, cloned_repo_dir ): - # Remove a file path from a cloned repository. Since mercurial doesn't track - # directories (only files), directories are automatically removed when they - # become empty. - abs_file_path = os.path.join( cloned_repo_dir, file_path ) - if os.path.exists( abs_file_path ): - cmd = 'hg remove %s > /dev/null 2>&1' % file_path - os.chdir( cloned_repo_dir ) - os.system( cmd ) - os.chdir( current_working_dir ) + log.exception( "An error occurred sending a tool shed repository update alert by email." ) def update_for_browsing( repository, current_working_dir ): # Make a copy of a repository's files for browsing. repo_dir = repository.repo_path os.chdir( repo_dir ) os.system( 'hg update > /dev/null 2>&1' ) os.chdir( current_working_dir ) + """ + # TODO: the following is useful if the repository files somehow include missing or + # untracked files. If this happens, we can enhance the following to clean things up. + # We're not currently doing any cleanup though since so far none of the repositories + # have problematic files for browsing. + # Get the tip change set. + repo = hg.repository( ui.ui(), repo_dir ) + for changeset in repo.changelog: + ctx = repo.changectx( changeset ) + ctx_parent = ctx.parents()[0] + break + modified, added, removed, deleted, unknown, ignored, clean = repo.status( node1=ctx_parent.node(), node2=ctx.node() ) + """ --- a/lib/galaxy/webapps/community/controllers/repository.py Mon Jun 27 17:25:32 2011 -0400 +++ b/lib/galaxy/webapps/community/controllers/repository.py Tue Jun 28 10:24:49 2011 -0400 @@ -12,7 +12,7 @@ from galaxy.util.json import from_json_string, to_json_string from galaxy.model.orm import * from common import * -from mercurial import hg, ui, patch +from mercurial import hg, ui, patch, commands log = logging.getLogger( __name__ ) @@ -117,7 +117,7 @@ model.User.table.c.email == column_filter ) ) class EmailAlertsColumn( grids.TextColumn ): def get_value( self, trans, grid, repository ): - if repository.email_alerts and trans.user.email in from_json_string( repository.email_alerts ): + if trans.user and repository.email_alerts and trans.user.email in from_json_string( repository.email_alerts ): return 'yes' return '' # Grid definition @@ -373,53 +373,18 @@ # allow_push = test # name = convert_characters1 # push_ssl = False - # Upon repository creation, only the owner can push to it ( allow_push setting ), - # and since we support both http and https, we set push_ssl to False to override + # Since we support both http and https, we set push_ssl to False to override # the default (which is True) in the mercurial api. - hgrc_file = os.path.abspath( os.path.join( repository.repo_path, ".hg", "hgrc" ) ) - output = open( hgrc_file, 'w' ) - output.write( '[web]\n' ) - output.write( 'allow_push = %s\n' % repository.user.username ) - output.write( 'name = %s\n' % repository.name ) - output.write( 'push_ssl = false\n' ) - output.flush() - output.close() - def __get_allow_push( self, repository ): - # TODO: Use the mercurial api to handle this - hgrc_file = os.path.abspath( os.path.join( repository.repo_path, ".hg", "hgrc" ) ) - config = ConfigParser.ConfigParser() - config.read( hgrc_file ) - for option in config.options( "web" ): - if option == 'allow_push': - return config.get( "web", option ) - raise Exception( "Repository %s missing allow_push entry under the [web] option in it's hgrc file." % repository.name ) - def __set_allow_push( self, repository, usernames, remove_auth='' ): - """ - # TODO: Use the mercurial api to handle this, something like the following: - items = repo.ui.configitems( section, untrusted=False ) - push_section = repo.ui.config( 'hgrc', 'allow_push' ) - for XXX (in name you want to add): - repo.ui.updateconfig( section=extensions_section, name='XXX', value='YYY' ) - """ - hgrc_file = os.path.abspath( os.path.join( repository.repo_path, ".hg", "hgrc" ) ) - fh, fn = tempfile.mkstemp() - for i, line in enumerate( open( hgrc_file ) ): - if line.startswith( 'allow_push' ): - value = line.split( ' = ' )[1].rstrip( '\n' ) - if remove_auth: - current_usernames = value.split( ',' ) - new_usernames = [] - for current_username in current_usernames: - if current_username != remove_auth: - new_usernames.append( current_username ) - new_usernames = ','.join( new_usernames ) - line = 'allow_push = %s\n' % new_usernames - else: - value = '%s,%s\n' % ( value, usernames ) - line = 'allow_push = %s' % value - os.write( fh, line ) - os.close( fh ) - shutil.move( fn, hgrc_file ) + repo = hg.repository( ui.ui(), path=repository.repo_path ) + fp = repo.opener( 'hgrc', 'wb' ) + fp.write( '[paths]\n' ) + fp.write( 'default = .\n' ) + fp.write( 'default-push = .\n' ) + fp.write( '[web]\n' ) + fp.write( 'allow_push = %s\n' % repository.user.username ) + fp.write( 'name = %s\n' % repository.name ) + fp.write( 'push_ssl = false\n' ) + fp.close() @web.expose def browse_repository( self, trans, id, **kwd ): params = util.Params( kwd ) @@ -428,14 +393,8 @@ commit_message = util.restore_text( params.get( 'commit_message', 'Deleted selected files' ) ) repository = get_repository( trans, id ) repo = hg.repository( ui.ui(), repository.repo_path ) - # Our current support for browsing a repository requires copies of the - # repository files to be in the repository root directory. We do the - # following to ensure the latest files are being browsed. current_working_dir = os.getcwd() - repo_dir = repository.repo_path - os.chdir( repo_dir ) - os.system( 'hg update > /dev/null 2>&1' ) - os.chdir( current_working_dir ) + update_for_browsing( repository, current_working_dir ) return trans.fill_template( '/webapps/community/repository/browse_repository.mako', repo=repo, repository=repository, @@ -449,9 +408,8 @@ status = params.get( 'status', 'done' ) commit_message = util.restore_text( params.get( 'commit_message', 'Deleted selected files' ) ) repository = get_repository( trans, id ) - _ui = ui.ui() repo_dir = repository.repo_path - repo = hg.repository( _ui, repo_dir ) + repo = hg.repository( ui.ui(), repo_dir ) selected_files_to_delete = util.restore_text( params.get( 'selected_files_to_delete', '' ) ) if params.get( 'select_files_to_delete_button', False ): if selected_files_to_delete: @@ -459,25 +417,19 @@ current_working_dir = os.getcwd() # Get the current repository tip. tip = repo[ 'tip' ] - # Clone the repository to a temporary location. - tmp_dir, cloned_repo_dir = hg_clone( trans, repository, current_working_dir ) - # Delete the selected files from the repository. - cloned_repo = hg.repository( _ui, cloned_repo_dir ) for selected_file in selected_files_to_delete: - selected_file_path = selected_file.split( 'repo_%d' % repository.id )[ 1 ].lstrip( '/' ) - hg_remove( selected_file_path, current_working_dir, cloned_repo_dir ) + repo_file = os.path.abspath( selected_file ) + commands.remove( repo.ui, repo, repo_file ) # Commit the change set. if not commit_message: commit_message = 'Deleted selected files' - hg_commit( commit_message, current_working_dir, cloned_repo_dir ) - # Push the change set from the cloned repository to the master repository. - hg_push( trans, repository, current_working_dir, cloned_repo_dir ) - # Remove the temporary directory containing the cloned repository. - shutil.rmtree( tmp_dir ) + # Commit the changes. + commands.commit( repo.ui, repo, repo_dir, message=commit_message ) + handle_email_alerts( trans, repository ) # Update the repository files for browsing. update_for_browsing( repository, current_working_dir ) # Get the new repository tip. - repo = hg.repository( _ui, repo_dir ) + repo = hg.repository( ui.ui(), repo_dir ) if tip != repo[ 'tip' ]: message = "The selected files were deleted from the repository." else: @@ -508,7 +460,7 @@ else: email_alerts = [] user = trans.user - if params.get( 'receive_email_alerts_button', False ): + if user and params.get( 'receive_email_alerts_button', False ): flush_needed = False if alerts_checked: if user.email not in email_alerts: @@ -523,7 +475,7 @@ if flush_needed: trans.sa_session.add( repository ) trans.sa_session.flush() - checked = alerts_checked or user.email in email_alerts + checked = alerts_checked or ( user and user.email in email_alerts ) alerts_check_box = CheckboxField( 'alerts', checked=checked ) return trans.fill_template( '/webapps/community/repository/view_repository.mako', repo=repo, @@ -592,7 +544,7 @@ user = trans.sa_session.query( trans.model.User ).get( trans.security.decode_id( user_id ) ) usernames.append( user.username ) usernames = ','.join( usernames ) - self.__set_allow_push( repository, usernames, remove_auth=remove_auth ) + repository.set_allow_push( usernames, remove_auth=remove_auth ) elif params.get( 'receive_email_alerts_button', False ): flush_needed = False if alerts_checked: @@ -610,7 +562,10 @@ trans.sa_session.flush() if error: status = 'error' - current_allow_push_list = self.__get_allow_push( repository ).split( ',' ) + if repository.allow_push: + current_allow_push_list = repository.allow_push.split( ',' ) + else: + current_allow_push_list = [] allow_push_select_field = self.__build_allow_push_select_field( trans, current_allow_push_list ) checked = alerts_checked or user.email in email_alerts alerts_check_box = CheckboxField( 'alerts', checked=checked ) --- a/lib/galaxy/webapps/community/controllers/upload.py Mon Jun 27 17:25:32 2011 -0400 +++ b/lib/galaxy/webapps/community/controllers/upload.py Tue Jun 28 10:24:49 2011 -0400 @@ -2,8 +2,8 @@ from galaxy.web.base.controller import * from galaxy.model.orm import * from galaxy.datatypes.checkers import * -from common import get_categories, get_repository, hg_add, hg_clone, hg_commit, hg_push, hg_remove, update_for_browsing -from mercurial import hg, ui +from common import * +from mercurial import hg, ui, commands log = logging.getLogger( __name__ ) @@ -31,30 +31,10 @@ uncompress_file = util.string_as_bool( params.get( 'uncompress_file', 'true' ) ) remove_repo_files_not_in_tar = util.string_as_bool( params.get( 'remove_repo_files_not_in_tar', 'true' ) ) uploaded_file = None - upload_point = params.get( 'upload_point', None ) - if upload_point is not None: - # The value of upload_point will be something like: database/community_files/000/repo_12/1.bed - if os.path.exists( upload_point ): - if os.path.isfile( upload_point ): - # Get the parent directory - upload_point, not_needed = os.path.split( upload_point ) - # Now the value of uplaod_point will be something like: database/community_files/000/repo_12/ - upload_point = upload_point.split( 'repo_%d' % repository.id )[ 1 ] - if upload_point: - upload_point = upload_point.lstrip( '/' ) - upload_point = upload_point.rstrip( '/' ) - # Now the value of uplaod_point will be something like: / - if upload_point == '/': - upload_point = None - else: - # Must have been an error selecting something that didn't exist, so default to repository root - # TODO: throw an exception???? - upload_point = None - else: - # Default to repository root - upload_point = None + upload_point = self.__get_upload_point( repository, **kwd ) + # Get the current repository tip. + tip = repo[ 'tip' ] if params.get( 'upload_button', False ): - ctx = repo.changectx( "tip" ) current_working_dir = os.getcwd() file_data = params.get( 'file_data', '' ) if file_data == '': @@ -66,12 +46,6 @@ uploaded_file_name = uploaded_file.name uploaded_file_filename = file_data.filename if uploaded_file: - # Our current support for browsing repo contents requires a copy of the - # repository files in the repo root directory. To eliminate these copies, - # we update the repo, passing the "-r null" flag. - os.chdir( repo_dir ) - os.system( 'hg update -r null > /dev/null 2>&1' ) - os.chdir( current_working_dir ) isgzip = False isbz2 = False if uncompress_file: @@ -79,10 +53,9 @@ if not isgzip: isbz2 = is_bz2( uploaded_file_name ) ok = True - files_to_commit = [] # Determine what we have - a single file or an archive try: - if uncompress_file: + if ( isgzip or isbz2 ) and uncompress_file: # Open for reading with transparent compression. tar = tarfile.open( uploaded_file_name, 'r:*' ) else: @@ -91,125 +64,52 @@ except tarfile.ReadError, e: tar = None istar = False - if repository.is_new: - if istar: - # We have an archive ( a tarball ) in a new repository. - ok, message = self.__check_archive( tar ) - if ok: - tar.extractall( path=repo_dir ) - tar.close() - uploaded_file.close() - for root, dirs, files in os.walk( repo_dir, topdown=False ): - # Don't visit .hg directories and don't include hgrc files in commit. - if not root.find( '.hg' ) >= 0 and not root.find( 'hgrc' ) >= 0: - if '.hg' in dirs: - # Don't visit .hg directories - dirs.remove( '.hg' ) - if 'hgrc' in files: - # Don't include hgrc files in commit - should be impossible - # since we don't visit .hg dirs, but just in case... - files.remove( 'hgrc' ) - for name in files: - relative_root = root.split( 'repo_%d' % repository.id )[ 1 ].lstrip ( '/' ) - if upload_point is not None: - file_path = os.path.join( relative_root, upload_point, name ) - else: - file_path = os.path.join( relative_root, name ) - # Check if the file is tracked and make it tracked if not. - repo_contains = file_path in [ i for i in ctx.manifest() ] - if not repo_contains: - # Add the file to the dirstate - repo.dirstate.add( file_path ) - files_to_commit.append( file_path ) - else: - tar.close() + if istar: + ok, message, files_to_remove = self.upload_tar( trans, + repository, + tar, + uploaded_file, + upload_point, + remove_repo_files_not_in_tar, + commit_message ) + else: + if ( isgzip or isbz2 ) and uncompress_file: + uploaded_file_filename = self.uncompress( repository, uploaded_file_name, uploaded_file_filename, isgzip, isbz2 ) + if upload_point is not None: + full_path = os.path.abspath( os.path.join( repo_dir, upload_point, uploaded_file_filename ) ) else: - # We have a single file in a new repository. - if uncompress_file and ( isgzip or isbz2 ): - uploaded_file_filename = self.uncompress( repository, uploaded_file_name, uploaded_file_filename, isgzip, isbz2 ) - if upload_point is not None: - full_path = os.path.abspath( os.path.join( upload_point, uploaded_file_filename ) ) - file_path = os.path.join( upload_point, uploaded_file_filename ) - else: - full_path = os.path.abspath( os.path.join( repo_dir, uploaded_file_filename ) ) - file_path = os.path.join( uploaded_file_filename ) - shutil.move( uploaded_file_name, full_path ) - repo.dirstate.add( file_path ) - files_to_commit.append( file_path ) - else: - # We have a repository that is not new (it contains files). - if uncompress_file and ( isgzip or isbz2 ): - uploaded_file_filename = self.uncompress( repository, uploaded_file_name, uploaded_file_filename, isgzip, isbz2 ) - # Get the current repository tip. - tip = repo[ 'tip' ] - # Clone the repository to a temporary location. - tmp_dir, cloned_repo_dir = hg_clone( trans, repository, current_working_dir ) - # Move the uploaded files to the upload_point within the cloned repository. - files_to_remove = self.__move_to_upload_point( repository, - upload_point, - uploaded_file, - uploaded_file_name, - uploaded_file_filename, - cloned_repo_dir, - istar, - tar, - remove_repo_files_not_in_tar ) - if remove_repo_files_not_in_tar and files_to_remove: - # Remove files in the repository (relative to the upload point) - # that are not in the uploaded archive. - for repo_file in files_to_remove: - hg_remove( repo_file, current_working_dir, cloned_repo_dir ) - # Add the files in the uploaded archive to the cloned repository. - hg_add( trans, current_working_dir, cloned_repo_dir ) - # Commit the files to the cloned repository. - if not commit_message: - commit_message = 'Uploaded' - hg_commit( commit_message, current_working_dir, cloned_repo_dir ) - # Push the changes from the cloned repository to the master repository. - hg_push( trans, repository, current_working_dir, cloned_repo_dir ) - # Remove the temporary directory containing the cloned repository. - shutil.rmtree( tmp_dir ) - # Update the repository files for browsing. + full_path = os.path.abspath( os.path.join( repo_dir, uploaded_file_filename ) ) + # Move the uploaded file to the load_point within the repository hierarchy. + shutil.move( uploaded_file_name, full_path ) + commands.add( repo.ui, repo, full_path ) + commands.commit( repo.ui, repo, full_path, message=commit_message ) + handle_email_alerts( trans, repository ) + if ok: + # Update the repository files for browsing, a by-product of doing this + # is eliminating unwanted files from the repository directory. update_for_browsing( repository, current_working_dir ) # Get the new repository tip. repo = hg.repository( ui.ui(), repo_dir ) if tip != repo[ 'tip' ]: - if uncompress_file: + if ( isgzip or isbz2 ) and uncompress_file: uncompress_str = ' uncompressed and ' else: uncompress_str = ' ' message = "The file '%s' has been successfully%suploaded to the repository." % ( uploaded_file_filename, uncompress_str ) - if istar and remove_repo_files_not_in_tar: - message += " %d files were removed from the repository." % len( files_to_remove ) + if istar and remove_repo_files_not_in_tar and files_to_remove: + if upload_point is not None: + message += " %d files were removed from the repository relative to the selected upload point '%s'." % ( len( files_to_remove ), upload_point ) + else: + message += " %d files were removed from the repository root." % len( files_to_remove ) else: - message = 'No changes to repository.' + message = 'No changes to repository.' trans.response.send_redirect( web.url_for( controller='repository', action='browse_repository', commit_message='Deleted selected files', message=message, id=trans.security.encode_id( repository.id ) ) ) - if ok: - if files_to_commit: - repo.dirstate.write() - repo.commit( text=commit_message ) - os.chdir( repo_dir ) - os.system( 'hg update > /dev/null 2>&1' ) - os.chdir( current_working_dir ) - if uncompress_file: - uncompress_str = ' uncompressed and ' - else: - uncompress_str = ' ' - message = "The file '%s' has been successfully%suploaded to the repository." % ( uploaded_file_filename, uncompress_str ) - trans.response.send_redirect( web.url_for( controller='repository', - action='browse_repository', - commit_message='Deleted selected files', - message=message, - id=trans.security.encode_id( repository.id ) ) ) else: status = 'error' - os.chdir( repo_dir ) - os.system( 'hg update > /dev/null 2>&1' ) - os.chdir( current_working_dir ) selected_categories = [ trans.security.decode_id( id ) for id in category_ids ] return trans.fill_template( '/webapps/community/repository/upload.mako', repository=repository, @@ -218,6 +118,53 @@ remove_repo_files_not_in_tar=remove_repo_files_not_in_tar, message=message, status=status ) + def upload_tar( self, trans, repository, tar, uploaded_file, upload_point, remove_repo_files_not_in_tar, commit_message ): + # Upload a tar archive of files. + repo_dir = repository.repo_path + repo = hg.repository( ui.ui(), repo_dir ) + files_to_remove = [] + ok, message = self.__check_archive( tar ) + if not ok: + tar.close() + uploaded_file.close() + return ok, message, files_to_remove + else: + if upload_point is not None: + full_path = os.path.abspath( os.path.join( repo_dir, upload_point ) ) + else: + full_path = os.path.abspath( repo_dir ) + filenames_in_archive = [ tarinfo_obj.name for tarinfo_obj in tar.getmembers() ] + filenames_in_archive = [ os.path.join( full_path, name ) for name in filenames_in_archive ] + # Extract the uploaded tar to the load_point within the repository hierarchy. + tar.extractall( path=full_path ) + tar.close() + uploaded_file.close() + if remove_repo_files_not_in_tar and not repository.is_new: + # We have a repository that is not new (it contains files), so discover + # those files that are in the repository, but not in the uploaded archive. + for root, dirs, files in os.walk( full_path ): + if not root.find( '.hg' ) >= 0 and not root.find( 'hgrc' ) >= 0: + if '.hg' in dirs: + # Don't visit .hg directories - should be impossible since we don't + # allow uploaded archives that contain .hg dirs, but just in case... + dirs.remove( '.hg' ) + if 'hgrc' in files: + # Don't include hgrc files in commit. + files.remove( 'hgrc' ) + for name in files: + full_name = os.path.join( root, name ) + if full_name not in filenames_in_archive: + files_to_remove.append( full_name ) + for repo_file in files_to_remove: + # Remove files in the repository (relative to the upload point) + # that are not in the uploaded archive. + commands.remove( repo.ui, repo, repo_file ) + for filename_in_archive in filenames_in_archive: + commands.add( repo.ui, repo, filename_in_archive ) + # Commit the changes. + commands.commit( repo.ui, repo, full_path, message=commit_message ) + handle_email_alerts( trans, repository ) + return True, '', files_to_remove def uncompress( self, repository, uploaded_file_name, uploaded_file_filename, isgzip, isbz2 ): if isgzip: self.__handle_gzip( repository, uploaded_file_name ) @@ -259,44 +206,26 @@ os.close( fd ) bzipped_file.close() shutil.move( uncompressed, uploaded_file_name ) - def __move_to_upload_point( self, repository, upload_point, uploaded_file, uploaded_file_name, - uploaded_file_filename, cloned_repo_dir, istar, tar, remove_repo_files_not_in_tar ): - files_to_remove = [] + def __get_upload_point( self, repository, **kwd ): + upload_point = kwd.get( 'upload_point', None ) if upload_point is not None: - if istar: - full_path = os.path.abspath( os.path.join( cloned_repo_dir, upload_point ) ) + # The value of upload_point will be something like: database/community_files/000/repo_12/1.bed + if os.path.exists( upload_point ): + if os.path.isfile( upload_point ): + # Get the parent directory + upload_point, not_needed = os.path.split( upload_point ) + # Now the value of uplaod_point will be something like: database/community_files/000/repo_12/ + upload_point = upload_point.split( 'repo_%d' % repository.id )[ 1 ] + if upload_point: + upload_point = upload_point.lstrip( '/' ) + upload_point = upload_point.rstrip( '/' ) + # Now the value of uplaod_point will be something like: / + if upload_point == '/': + upload_point = None else: - full_path = os.path.abspath( os.path.join( cloned_repo_dir, upload_point, uploaded_file_filename ) ) - else: - if istar: - full_path = os.path.abspath( os.path.join( cloned_repo_dir ) ) - else: - full_path = os.path.abspath( os.path.join( cloned_repo_dir, uploaded_file_filename ) ) - if istar: - if remove_repo_files_not_in_tar: - # Discover those files that are in the repository, but not in the uploaded archive - filenames_in_archive = [ tarinfo_obj.name for tarinfo_obj in tar.getmembers() ] - for root, dirs, files in os.walk( full_path ): - relative_dir = root.split( 'repo_%d' % repository.id )[1].lstrip( '/' ) - if not root.find( '.hg' ) >= 0 and not root.find( 'hgrc' ) >= 0: - if '.hg' in dirs: - # Don't visit .hg directories - dirs.remove( '.hg' ) - if 'hgrc' in files: - # Don't include hgrc files in commit - should be impossible - # since we don't visit .hg dirs, but just in case... - files.remove( 'hgrc' ) - for name in files: - if name not in filenames_in_archive: - files_to_remove.append( os.path.join( relative_dir, name ) ) - # Extract the uploaded tarball to the load_point within the cloned repository hierarchy - tar.extractall( path=full_path ) - tar.close() - uploaded_file.close() - else: - # Move the uploaded file to the load_point within the cloned repository hierarchy - shutil.move( uploaded_file_name, full_path ) - return files_to_remove + # Must have been an error selecting something that didn't exist, so default to repository root + upload_point = None + return upload_point def __check_archive( self, archive ): for member in archive.getmembers(): # Allow regular files and directories only --- a/lib/galaxy/webapps/community/model/__init__.py Mon Jun 27 17:25:32 2011 -0400 +++ b/lib/galaxy/webapps/community/model/__init__.py Tue Jun 28 10:24:49 2011 -0400 @@ -5,6 +5,7 @@ the relationship cardinalities are obvious (e.g. prefer Dataset to Data) """ import os.path, os, errno, sys, codecs, operator, tempfile, logging, tarfile, mimetypes, ConfigParser +from galaxy import util from galaxy.util.bunch import Bunch from galaxy.util.hash_util import * from galaxy.web.form_builder import * @@ -115,163 +116,30 @@ repo = hg.repository( ui.ui(), self.repo_path ) tip_ctx = repo.changectx( repo.changelog.tip() ) return tip_ctx.rev() < 0 -class Tool( object ): - file_path = '/tmp' # Overridden in mapping.__init__() - states = Bunch( NEW = 'new', - ERROR = 'error', - DELETED = 'deleted', - WAITING = 'waiting', - APPROVED = 'approved', - REJECTED = 'rejected', - ARCHIVED = 'archived' ) - def __init__( self, guid=None, tool_id=None, name=None, description=None, user_description=None, - category=None, version=None, user_id=None, external_filename=None, suite=False ): - self.guid = guid - self.tool_id = tool_id - self.name = name or "Unnamed tool" - self.description = description - self.user_description = user_description - self.version = version or "1.0.0" - self.user_id = user_id - self.external_filename = external_filename - self.deleted = False - self.__extension = None - self.suite = suite - def get_file_name( self ): - if not self.external_filename: - assert self.id is not None, "ID must be set before filename used (commit the object)" - dir = os.path.join( self.file_path, 'tools', *directory_hash_id( self.id ) ) - # Create directory if it does not exist - if not os.path.exists( dir ): - os.makedirs( dir ) - # Return filename inside hashed directory - filename = os.path.join( dir, "tool_%d.dat" % self.id ) + @property + def allow_push( self ): + repo = hg.repository( ui.ui(), self.repo_path ) + return repo.ui.config( 'web', 'allow_push' ) + def set_allow_push( self, usernames, remove_auth='' ): + allow_push = util.listify( self.allow_push ) + if remove_auth: + allow_push.remove( remove_auth ) else: - filename = self.external_filename - # Make filename absolute - return os.path.abspath( filename ) - def set_file_name( self, filename ): - if not filename: - self.external_filename = None - else: - self.external_filename = filename - file_name = property( get_file_name, set_file_name ) - def create_from_datatype( self, datatype_bunch ): - # TODO: ensure guid is unique and generate a new one if not. - self.guid = datatype_bunch.guid - self.tool_id = datatype_bunch.id - self.name = datatype_bunch.name - self.description = datatype_bunch.description - self.version = datatype_bunch.version - self.user_id = datatype_bunch.user.id - self.suite = datatype_bunch.suite - @property - def state( self ): - latest_event = self.latest_event - if latest_event: - return latest_event.state - return None - @property - def latest_event( self ): - if self.events: - events = [ tea.event for tea in self.events ] - # Get the last event that occurred ( events mapper is sorted descending ) - return events[0] - return None - # Tool states - @property - def is_new( self ): - return self.state == self.states.NEW - @property - def is_error( self ): - return self.state == self.states.ERROR - @property - def is_deleted( self ): - return self.state == self.states.DELETED - @property - def is_waiting( self ): - return self.state == self.states.WAITING - @property - def is_approved( self ): - return self.state == self.states.APPROVED - @property - def is_rejected( self ): - return self.state == self.states.REJECTED - @property - def is_archived( self ): - return self.state == self.states.ARCHIVED - def get_state_message( self ): - if self.is_suite: - label = 'tool suite' - else: - label = 'tool' - if self.is_new: - return '<font color="red"><b><i>This is an unsubmitted version of this %s</i></b></font>' % label - if self.is_error: - return '<font color="red"><b><i>This %s is in an error state</i></b></font>' % label - if self.is_deleted: - return '<font color="red"><b><i>This is a deleted version of this %s</i></b></font>' % label - if self.is_waiting: - return '<font color="red"><b><i>This version of this %s is awaiting administrative approval</i></b></font>' % label - if self.is_approved: - return '<b><i>This is the latest approved version of this %s</i></b>' % label - if self.is_rejected: - return '<font color="red"><b><i>This version of this %s has been rejected by an administrator</i></b></font>' % label - if self.is_archived: - return '<font color="red"><b><i>This is an archived version of this %s</i></b></font>' % label - @property - def extension( self ): - # if instantiated via a query, this unmapped property won't exist - if '_Tool__extension' not in dir( self ): - self.__extension = None - if self.__extension is None: - head = open( self.file_name, 'rb' ).read( 4 ) - try: - assert head[:3] == 'BZh' - assert int( head[-1] ) in range( 0, 10 ) - self.__extension = 'tar.bz2' - except AssertionError: - pass - if self.__extension is None: - try: - assert head[:2] == '\037\213' - self.__extension = 'tar.gz' - except: - pass - if self.__extension is None: - self.__extension = 'tar' - return self.__extension - @property - def is_suite( self ): - return self.suite - @property - def label( self ): - if self.is_suite: - return 'tool suite' - else: - return 'tool' - @property - def type( self ): - # Hack - if self.is_suite: - return 'toolsuite' - return 'tool' - @property - def download_file_name( self ): - return '%s_%s.%s' % ( self.tool_id, self.version, self.extension ) - @property - def mimetype( self ): - return mimetypes.guess_type( self.download_file_name )[0] - -class Event( object ): - def __init__( self, state=None, comment='' ): - self.state = state - self.comment = comment - -class ToolEventAssociation( object ): - def __init__( self, tool=None, event=None ): - self.tool = tool - self.event = event + for username in util.listify( usernames ): + if username not in allow_push: + allow_push.append( username ) + allow_push = '%s\n' % ','.join( allow_push ) + repo = hg.repository( ui.ui(), path=self.repo_path ) + # Why doesn't the following work? + #repo.ui.setconfig( 'web', 'allow_push', allow_push ) + lines = repo.opener( 'hgrc', 'rb' ).readlines() + fp = repo.opener( 'hgrc', 'wb' ) + for line in lines: + if line.startswith( 'allow_push' ): + fp.write( 'allow_push = %s' % allow_push ) + else: + fp.write( line ) + fp.close() class ItemRatingAssociation( object ): def __init__( self, id=None, user=None, item=None, rating=0, comment='' ): @@ -284,10 +152,6 @@ """ Set association's item. """ pass -class ToolRatingAssociation( ItemRatingAssociation ): - def set_item( self, tool ): - self.tool = tool - class RepositoryRatingAssociation( ItemRatingAssociation ): def set_item( self, repository ): self.repository = repository @@ -298,11 +162,6 @@ self.description = description self.deleted = deleted -class ToolCategoryAssociation( object ): - def __init__( self, tool=None, category=None ): - self.tool = tool - self.category = category - class RepositoryCategoryAssociation( object ): def __init__( self, repository=None, category=None ): self.repository = repository @@ -326,12 +185,6 @@ self.user_tname = user_tname self.value = None self.user_value = None - -class ToolTagAssociation ( ItemTagAssociation ): - pass - -class ToolAnnotationAssociation( object ): - pass ## ---- Utility methods ------------------------------------------------------- def sort_by_attr( seq, attr ): --- a/templates/webapps/community/repository/browse_repository.mako Mon Jun 27 17:25:32 2011 -0400 +++ b/templates/webapps/community/repository/browse_repository.mako Tue Jun 28 10:24:49 2011 -0400 @@ -9,7 +9,7 @@ can_push = trans.app.security_agent.can_push( trans.user, repository ) can_upload = can_push can_browse_contents = not is_new - can_rate = repository.user != trans.user + can_rate = trans.user and repository.user != trans.user can_manage = repository.user == trans.user can_view_change_log = not is_new %> --- a/templates/webapps/community/repository/manage_repository.mako Mon Jun 27 17:25:32 2011 -0400 +++ b/templates/webapps/community/repository/manage_repository.mako Tue Jun 28 10:24:49 2011 -0400 @@ -9,7 +9,7 @@ can_push = trans.app.security_agent.can_push( trans.user, repository ) can_upload = can_push can_browse_contents = not is_new - can_rate = not is_new and repository.user != trans.user + can_rate = not is_new and trans.user and repository.user != trans.user can_view_change_log = not is_new if can_push: browse_label = 'Browse or delete repository files' --- a/templates/webapps/community/repository/upload.mako Mon Jun 27 17:25:32 2011 -0400 +++ b/templates/webapps/community/repository/upload.mako Tue Jun 28 10:24:49 2011 -0400 @@ -93,7 +93,7 @@ Supported compression types are gz and bz2. If <b>Yes</b> is selected, the uploaded file will be uncompressed. However, if the uploaded file is an archive that contains compressed files, the contained files will not be uncompressed. For example, if the uploaded compressed file is some_file.tar.gz, some_file.tar will be uncompressed and extracted, but if - some_file.tar contains 4.bed.gz, the contained file 4.bed.gz will not be uncompressed. + some_file.tar contains some_contained_file.gz, the contained file will not be uncompressed. </div></div><div class="form-row"> @@ -105,7 +105,7 @@ yes_selected = '' no_selected = 'selected' %> - <label>Remove files in the repository (relative to the upload point) that are not in the uploaded archive?</label> + <label>Remove files in the repository (relative to the root or selected upload point) that are not in the uploaded archive?</label><div class="form-row-input"><select name="remove_repo_files_not_in_tar"><option value="true" ${yes_selected}>Yes @@ -114,9 +114,9 @@ </div><div class="toolParamHelp" style="clear: both;"> This selection pertains only to uploaded tar archives, not to single file uploads. If <b>Yes</b> is selected, files - that exist in the repository (relative to the upload point) but that are not in the uploaded archive will be removed - from the repository. Otherwise, all existing repository files will remain and the uploaded archive files will be added - to the repository. + that exist in the repository (relative to the root or selected upload point) but that are not in the uploaded archive + will be removed from the repository. Otherwise, all existing repository files will remain and the uploaded archive + files will be added to the repository. </div></div><div class="form-row"> @@ -141,8 +141,9 @@ </div><input type="hidden" id="upload_point" name="upload_point" value=""/><div class="toolParamHelp" style="clear: both;"> - Select a location within the repository to upload your files by clicking a check box next to the location. If a location - is not selected, files will be uploaded to the repository root. + Select a location within the repository to upload your files by clicking a check box next to the location. The + selected location is considered the upload point. If a location is not selected, the upload point will be the + repository root. </div><div style="clear: both"></div></div> --- a/templates/webapps/community/repository/view_changelog.mako Mon Jun 27 17:25:32 2011 -0400 +++ b/templates/webapps/community/repository/view_changelog.mako Tue Jun 28 10:24:49 2011 -0400 @@ -9,7 +9,7 @@ can_browse_contents = not is_new can_manage = trans.user == repository.user can_push = trans.app.security_agent.can_push( trans.user, repository ) - can_rate = repository.user != trans.user + can_rate = trans.user and repository.user != trans.user can_upload = can_push if can_push: browse_label = 'Browse or delete repository files' --- a/templates/webapps/community/repository/view_changeset.mako Mon Jun 27 17:25:32 2011 -0400 +++ b/templates/webapps/community/repository/view_changeset.mako Tue Jun 28 10:24:49 2011 -0400 @@ -7,7 +7,7 @@ from galaxy.web.framework.helpers import time_ago is_new = repository.is_new can_browse_contents = not is_new - can_rate = repository.user != trans.user + can_rate = trans.user and repository.user != trans.user can_manage = trans.user == repository.user can_push = trans.app.security_agent.can_push( trans.user, repository ) can_view_change_log = not is_new --- a/templates/webapps/community/repository/view_repository.mako Mon Jun 27 17:25:32 2011 -0400 +++ b/templates/webapps/community/repository/view_repository.mako Tue Jun 28 10:24:49 2011 -0400 @@ -7,7 +7,7 @@ from galaxy.web.framework.helpers import time_ago is_new = repository.is_new can_push = trans.app.security_agent.can_push( trans.user, repository ) - can_rate = not is_new and repository.user != trans.user + can_rate = not is_new and trans.user and repository.user != trans.user can_upload = can_push can_browse_contents = not is_new can_view_change_log = not is_new Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.