1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/d94ef8154c63/ Changeset: d94ef8154c63 User: dannon Date: 2014-10-15 20:30:56+00:00 Summary: Merged in jmchilton/galaxy-central-fork-1 (pull request #530) Implement creating tool shed repository revisions via API tarball upload. Affected #: 5 files diff -r 9728fcd5167c2aceb8e0b72d3d8b9dc6fd5435b7 -r d94ef8154c63145b0e2928b099017324097d4352 lib/galaxy/tools/actions/upload_common.py --- a/lib/galaxy/tools/actions/upload_common.py +++ b/lib/galaxy/tools/actions/upload_common.py @@ -30,7 +30,7 @@ f.file.close() upload_dataset['file_data'] = dict( filename=f.filename, local_filename=local_filename ) - elif type( f ) == dict and 'filename' and 'local_filename' not in f: + elif type( f ) == dict and 'local_filename' not in f: raise Exception( 'Uploaded file was encoded in a way not understood by Galaxy.' ) if upload_dataset['url_paste'] and upload_dataset['url_paste'].strip() != '': upload_dataset['url_paste'], is_multi_byte = datatypes.sniff.stream_to_file( StringIO.StringIO( upload_dataset['url_paste'] ), prefix="strio_url_paste_" ) diff -r 9728fcd5167c2aceb8e0b72d3d8b9dc6fd5435b7 -r d94ef8154c63145b0e2928b099017324097d4352 lib/galaxy/webapps/tool_shed/api/repositories.py --- a/lib/galaxy/webapps/tool_shed/api/repositories.py +++ b/lib/galaxy/webapps/tool_shed/api/repositories.py @@ -2,11 +2,15 @@ import logging import os import tarfile +import StringIO import tempfile from time import strftime +from cgi import FieldStorage + from galaxy import util from galaxy import web +from galaxy.datatypes import checkers from galaxy.model.orm import and_ from galaxy.web.base.controller import BaseAPIController from galaxy.web.base.controller import HTTPBadRequest @@ -16,10 +20,14 @@ from tool_shed.metadata import repository_metadata_manager from tool_shed.repository_types import util as rt_util +from tool_shed.dependencies import attribute_handlers + from tool_shed.util import basic_util +from tool_shed.util import commit_util from tool_shed.util import encoding_util from tool_shed.util import hg_util from tool_shed.util import repository_util +from tool_shed.util import repository_content_util from tool_shed.util import shed_util_common as suc from tool_shed.util import tool_util @@ -568,3 +576,108 @@ action='show', id=trans.security.encode_id( repository.id ) ) return repository_dict + + @web.expose_api + def create_changeset_revision( self, trans, id, payload, **kwd ): + """ + POST /api/repositories/{encoded_repository_id}/changeset_revision + + Create a new tool shed repository commit - leaving PUT on parent + resource open for updating meta-attirbutes of the repository (and + Galaxy doesn't allow PUT multipart data anyway + https://trello.com/c/CQwmCeG6). + + :param id: the encoded id of the Repository object + + The following parameters may be included in the payload. + :param commit_message: hg commit message for update. + """ + + # Example URL: http://localhost:9009/api/repositories/f9cad7b01a472135 + rdah = attribute_handlers.RepositoryDependencyAttributeHandler( trans.app, unpopulate=False ) + tdah = attribute_handlers.ToolDependencyAttributeHandler( trans.app, unpopulate=False ) + + repository = suc.get_repository_in_tool_shed( trans.app, id ) + repo_dir = repository.repo_path( trans.app ) + repo = hg_util.get_repo_for_repository( trans.app, repository=None, repo_path=repo_dir, create=False ) + + upload_point = commit_util.get_upload_point( repository, **kwd ) + tip = repository.tip( trans.app ) + + file_data = payload.get('file') + # Code stolen from gx's upload_common.py + if isinstance( file_data, FieldStorage ): + assert not isinstance( file_data.file, StringIO.StringIO ) + assert file_data.file.name != '<fdopen>' + local_filename = util.mkstemp_ln( file_data.file.name, 'upload_file_data_' ) + file_data.file.close() + file_data = dict( filename=file_data.filename, + local_filename=local_filename ) + elif type( file_data ) == dict and 'local_filename' not in file_data: + raise Exception( 'Uploaded file was encoded in a way not understood.' ) + + commit_message = kwd.get( 'commit_message', 'Uploaded' ) + + uploaded_file = open(file_data['local_filename'], 'rb') + uploaded_file_name = file_data['local_filename'] + + isgzip = False + isbz2 = False + isgzip = checkers.is_gzip( uploaded_file_name ) + if not isgzip: + isbz2 = checkers.is_bz2( uploaded_file_name ) + if ( isgzip or isbz2 ): + # Open for reading with transparent compression. + tar = tarfile.open( uploaded_file_name, 'r:*' ) + else: + tar = tarfile.open( uploaded_file_name ) + + new_repo_alert = False + remove_repo_files_not_in_tar = True + + ok, message, files_to_remove, content_alert_str, undesirable_dirs_removed, undesirable_files_removed = \ + repository_content_util.upload_tar( + trans, + rdah, + tdah, + repository, + tar, + uploaded_file, + upload_point, + remove_repo_files_not_in_tar, + commit_message, + new_repo_alert + ) + if ok: + # Update the repository files for browsing. + hg_util.update_repository( repo ) + # Get the new repository tip. + if tip == repository.tip( trans.app ): + trans.response.status = 400 + message = 'No changes to repository.' + ok = False + else: + rmm = repository_metadata_manager.RepositoryMetadataManager( app=trans.app, + user=trans.user, + repository=repository ) + status, error_message = \ + rmm.set_repository_metadata_due_to_new_tip( trans.request.host, + content_alert_str=content_alert_str, + **kwd ) + if error_message: + ok = False + trans.response.status = 500 + message = error_message + else: + trans.response.status = 500 + + if not ok: + return { + "err_msg": message, + "content_alert": content_alert_str + } + else: + return { + "message": message, + "content_alert": content_alert_str + } diff -r 9728fcd5167c2aceb8e0b72d3d8b9dc6fd5435b7 -r d94ef8154c63145b0e2928b099017324097d4352 lib/galaxy/webapps/tool_shed/buildapp.py --- a/lib/galaxy/webapps/tool_shed/buildapp.py +++ b/lib/galaxy/webapps/tool_shed/buildapp.py @@ -119,6 +119,12 @@ name_prefix='user_', path_prefix='/api', parent_resources=dict( member_name='user', collection_name='users' ) ) + webapp.mapper.connect( 'repository_create_changeset_revision', + '/api/repositories/:id/changeset_revision', + controller='repositories', + action='create_changeset_revision', + conditions=dict( method=[ "POST" ] ) ) + webapp.finalize_config() # Wrap the webapp in some useful middleware if kwargs.get( 'middleware', True ): diff -r 9728fcd5167c2aceb8e0b72d3d8b9dc6fd5435b7 -r d94ef8154c63145b0e2928b099017324097d4352 lib/galaxy/webapps/tool_shed/controllers/upload.py --- a/lib/galaxy/webapps/tool_shed/controllers/upload.py +++ b/lib/galaxy/webapps/tool_shed/controllers/upload.py @@ -20,6 +20,7 @@ from tool_shed.util import commit_util from tool_shed.util import hg_util from tool_shed.util import shed_util_common as suc +from tool_shed.util import repository_content_util from tool_shed.util import xml_util from galaxy import eggs @@ -129,16 +130,18 @@ istar = False if istar: ok, message, files_to_remove, content_alert_str, undesirable_dirs_removed, undesirable_files_removed = \ - self.upload_tar( trans, - rdah, - tdah, - repository, - tar, - uploaded_file, - upload_point, - remove_repo_files_not_in_tar, - commit_message, - new_repo_alert ) + repository_content_util.upload_tar( + trans, + rdah, + tdah, + repository, + tar, + uploaded_file, + upload_point, + remove_repo_files_not_in_tar, + commit_message, + new_repo_alert + ) elif uploaded_directory: ok, message, files_to_remove, content_alert_str, undesirable_dirs_removed, undesirable_files_removed = \ self.upload_directory( trans, @@ -407,69 +410,3 @@ commit_message, undesirable_dirs_removed, undesirable_files_removed ) - - def upload_tar( self, trans, rdah, tdah, repository, tar, uploaded_file, upload_point, remove_repo_files_not_in_tar, - commit_message, new_repo_alert ): - # Upload a tar archive of files. - repo_dir = repository.repo_path( trans.app ) - repo = hg_util.get_repo_for_repository( trans.app, repository=None, repo_path=repo_dir, create=False ) - undesirable_dirs_removed = 0 - undesirable_files_removed = 0 - ok, message = commit_util.check_archive( repository, tar ) - if not ok: - tar.close() - uploaded_file.close() - return ok, message, [], '', undesirable_dirs_removed, undesirable_files_removed - else: - if upload_point is not None: - full_path = os.path.abspath( os.path.join( repo_dir, upload_point ) ) - else: - full_path = os.path.abspath( repo_dir ) - filenames_in_archive = [] - for tarinfo_obj in tar.getmembers(): - ok = os.path.basename( tarinfo_obj.name ) not in commit_util.UNDESIRABLE_FILES - if ok: - for file_path_item in tarinfo_obj.name.split( '/' ): - if file_path_item in commit_util.UNDESIRABLE_DIRS: - undesirable_dirs_removed += 1 - ok = False - break - else: - undesirable_files_removed += 1 - if ok: - filenames_in_archive.append( tarinfo_obj.name ) - # Extract the uploaded tar to the load_point within the repository hierarchy. - tar.extractall( path=full_path ) - tar.close() - uploaded_file.close() - for filename in filenames_in_archive: - uploaded_file_name = os.path.join( full_path, filename ) - if os.path.split( uploaded_file_name )[ -1 ] == rt_util.REPOSITORY_DEPENDENCY_DEFINITION_FILENAME: - # Inspect the contents of the file to see if toolshed or changeset_revision attributes - # are missing and if so, set them appropriately. - altered, root_elem, error_message = rdah.handle_tag_attributes( uploaded_file_name ) - if error_message: - return False, error_message, [], '', [], [] - elif altered: - tmp_filename = xml_util.create_and_write_tmp_file( root_elem ) - shutil.move( tmp_filename, uploaded_file_name ) - elif os.path.split( uploaded_file_name )[ -1 ] == rt_util.TOOL_DEPENDENCY_DEFINITION_FILENAME: - # Inspect the contents of the file to see if toolshed or changeset_revision - # attributes are missing and if so, set them appropriately. - altered, root_elem, error_message = tdah.handle_tag_attributes( uploaded_file_name ) - if error_message: - return False, error_message, [], '', [], [] - if altered: - tmp_filename = xml_util.create_and_write_tmp_file( root_elem ) - shutil.move( tmp_filename, uploaded_file_name ) - return commit_util.handle_directory_changes( trans.app, - trans.request.host, - trans.user.username, - repository, - full_path, - filenames_in_archive, - remove_repo_files_not_in_tar, - new_repo_alert, - commit_message, - undesirable_dirs_removed, - undesirable_files_removed ) diff -r 9728fcd5167c2aceb8e0b72d3d8b9dc6fd5435b7 -r d94ef8154c63145b0e2928b099017324097d4352 lib/tool_shed/util/repository_content_util.py --- /dev/null +++ b/lib/tool_shed/util/repository_content_util.py @@ -0,0 +1,75 @@ +import os +import shutil + +from tool_shed.util import commit_util +from tool_shed.util import hg_util +from tool_shed.util import xml_util + +import tool_shed.repository_types.util as rt_util + + +def upload_tar( trans, rdah, tdah, repository, tar, uploaded_file, upload_point, remove_repo_files_not_in_tar, + commit_message, new_repo_alert ): + # Upload a tar archive of files. + repo_dir = repository.repo_path( trans.app ) + hg_util.get_repo_for_repository( trans.app, repository=None, repo_path=repo_dir, create=False ) + undesirable_dirs_removed = 0 + undesirable_files_removed = 0 + ok, message = commit_util.check_archive( repository, tar ) + if not ok: + tar.close() + uploaded_file.close() + return ok, message, [], '', undesirable_dirs_removed, undesirable_files_removed + else: + if upload_point is not None: + full_path = os.path.abspath( os.path.join( repo_dir, upload_point ) ) + else: + full_path = os.path.abspath( repo_dir ) + filenames_in_archive = [] + for tarinfo_obj in tar.getmembers(): + ok = os.path.basename( tarinfo_obj.name ) not in commit_util.UNDESIRABLE_FILES + if ok: + for file_path_item in tarinfo_obj.name.split( '/' ): + if file_path_item in commit_util.UNDESIRABLE_DIRS: + undesirable_dirs_removed += 1 + ok = False + break + else: + undesirable_files_removed += 1 + if ok: + filenames_in_archive.append( tarinfo_obj.name ) + # Extract the uploaded tar to the load_point within the repository hierarchy. + tar.extractall( path=full_path ) + tar.close() + uploaded_file.close() + for filename in filenames_in_archive: + uploaded_file_name = os.path.join( full_path, filename ) + if os.path.split( uploaded_file_name )[ -1 ] == rt_util.REPOSITORY_DEPENDENCY_DEFINITION_FILENAME: + # Inspect the contents of the file to see if toolshed or changeset_revision attributes + # are missing and if so, set them appropriately. + altered, root_elem, error_message = rdah.handle_tag_attributes( uploaded_file_name ) + if error_message: + return False, error_message, [], '', [], [] + elif altered: + tmp_filename = xml_util.create_and_write_tmp_file( root_elem ) + shutil.move( tmp_filename, uploaded_file_name ) + elif os.path.split( uploaded_file_name )[ -1 ] == rt_util.TOOL_DEPENDENCY_DEFINITION_FILENAME: + # Inspect the contents of the file to see if toolshed or changeset_revision + # attributes are missing and if so, set them appropriately. + altered, root_elem, error_message = tdah.handle_tag_attributes( uploaded_file_name ) + if error_message: + return False, error_message, [], '', [], [] + if altered: + tmp_filename = xml_util.create_and_write_tmp_file( root_elem ) + shutil.move( tmp_filename, uploaded_file_name ) + return commit_util.handle_directory_changes( trans.app, + trans.request.host, + trans.user.username, + repository, + full_path, + filenames_in_archive, + remove_repo_files_not_in_tar, + new_repo_alert, + commit_message, + undesirable_dirs_removed, + undesirable_files_removed ) Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.