1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/1d05c839c1ea/ Changeset: 1d05c839c1ea User: greg Date: 2013-07-15 17:28:41 Summary: Only allow a single file named tool_dependencies.xml to be committed to a repository in the tool shed whose type is tool_dependency_definition. Affected #: 9 files diff -r 320d6f095b810d8c338c5ad5dbc04d6685505c5e -r 1d05c839c1ea7a83112a98734a6887f7ce419567 lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py --- a/lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py +++ b/lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py @@ -436,7 +436,7 @@ message = '' tool_shed_repository = tool_dependencies[ 0 ].tool_shed_repository # Get the tool_dependencies.xml file from the repository. - tool_dependencies_config = suc.get_config_from_disk( 'tool_dependencies.xml', tool_shed_repository.repo_path( trans.app ) ) + tool_dependencies_config = suc.get_config_from_disk( suc.TOOL_DEPENDENCY_DEFINITION_FILENAME, tool_shed_repository.repo_path( trans.app ) ) installed_tool_dependencies = common_install_util.handle_tool_dependencies( app=trans.app, tool_shed_repository=tool_shed_repository, tool_dependencies_config=tool_dependencies_config, diff -r 320d6f095b810d8c338c5ad5dbc04d6685505c5e -r 1d05c839c1ea7a83112a98734a6887f7ce419567 lib/galaxy/webapps/tool_shed/controllers/repository.py --- a/lib/galaxy/webapps/tool_shed/controllers/repository.py +++ b/lib/galaxy/webapps/tool_shed/controllers/repository.py @@ -1656,7 +1656,7 @@ # to retrieve older versions of the tool-dependencies.xml file from the repository manafest. repo_dir = repository.repo_path( trans.app ) # Get the tool_dependencies.xml file from disk. - tool_dependencies_config = suc.get_config_from_disk( 'tool_dependencies.xml', repo_dir ) + tool_dependencies_config = suc.get_config_from_disk( suc.TOOL_DEPENDENCY_DEFINITION_FILENAME, repo_dir ) # Return the encoded contents of the tool_dependencies.xml file. if tool_dependencies_config: tool_dependencies_config_file = open( tool_dependencies_config, 'rb' ) diff -r 320d6f095b810d8c338c5ad5dbc04d6685505c5e -r 1d05c839c1ea7a83112a98734a6887f7ce419567 lib/galaxy/webapps/tool_shed/controllers/upload.py --- a/lib/galaxy/webapps/tool_shed/controllers/upload.py +++ b/lib/galaxy/webapps/tool_shed/controllers/upload.py @@ -8,6 +8,7 @@ from galaxy import util from galaxy import web from galaxy.datatypes import checkers +import tool_shed.repository_types.util as rt_util import tool_shed.util.shed_util_common as suc from tool_shed.util import commit_util from tool_shed.util import metadata_util @@ -123,50 +124,54 @@ else: if ( isgzip or isbz2 ) and uncompress_file: uploaded_file_filename = commit_util.uncompress( repository, uploaded_file_name, uploaded_file_filename, isgzip, isbz2 ) - if upload_point is not None: - full_path = os.path.abspath( os.path.join( repo_dir, upload_point, uploaded_file_filename ) ) - else: - full_path = os.path.abspath( os.path.join( repo_dir, uploaded_file_filename ) ) - # Move some version of the uploaded file to the load_point within the repository hierarchy. - if uploaded_file_filename in [ 'repository_dependencies.xml' ]: - # Inspect the contents of the file to see if changeset_revision values are missing and if so, set them appropriately. - altered, root_elem = commit_util.handle_repository_dependencies_definition( trans, uploaded_file_name ) - if altered: - tmp_filename = xml_util.create_and_write_tmp_file( root_elem ) - shutil.move( tmp_filename, full_path ) + if repository.type == rt_util.TOOL_DEPENDENCY_DEFINITION and uploaded_file_filename != suc.TOOL_DEPENDENCY_DEFINITION_FILENAME: + ok = False + message = 'Repositories of type <b>Tool dependency definition</b> can only contain a single file named <b>tool_dependencies.xml</b>.' + if ok: + if upload_point is not None: + full_path = os.path.abspath( os.path.join( repo_dir, upload_point, uploaded_file_filename ) ) + else: + full_path = os.path.abspath( os.path.join( repo_dir, uploaded_file_filename ) ) + # Move some version of the uploaded file to the load_point within the repository hierarchy. + if uploaded_file_filename in [ suc.REPOSITORY_DEPENDENCY_DEFINITION_FILENAME ]: + # Inspect the contents of the file to see if changeset_revision values are missing and if so, set them appropriately. + altered, root_elem = commit_util.handle_repository_dependencies_definition( trans, uploaded_file_name ) + if altered: + tmp_filename = xml_util.create_and_write_tmp_file( root_elem ) + shutil.move( tmp_filename, full_path ) + else: + shutil.move( uploaded_file_name, full_path ) + elif uploaded_file_filename in [ suc.TOOL_DEPENDENCY_DEFINITION_FILENAME ]: + # Inspect the contents of the file to see if it defines a complex repository dependency definition whose changeset_revision values + # are missing and if so, set them appropriately. + altered, root_elem = commit_util.handle_tool_dependencies_definition( trans, uploaded_file_name ) + if altered: + tmp_filename = xml_util.create_and_write_tmp_file( root_elem ) + shutil.move( tmp_filename, full_path ) + else: + shutil.move( uploaded_file_name, full_path ) else: shutil.move( uploaded_file_name, full_path ) - elif uploaded_file_filename in [ 'tool_dependencies.xml' ]: - # Inspect the contents of the file to see if it defines a complex repository dependency definition whose changeset_revision values - # are missing and if so, set them appropriately. - altered, root_elem = commit_util.handle_tool_dependencies_definition( trans, uploaded_file_name ) - if altered: - tmp_filename = xml_util.create_and_write_tmp_file( root_elem ) - shutil.move( tmp_filename, full_path ) + # See if any admin users have chosen to receive email alerts when a repository is updated. If so, check every uploaded file to ensure + # content is appropriate. + check_contents = commit_util.check_file_contents_for_email_alerts( trans ) + if check_contents and os.path.isfile( full_path ): + content_alert_str = commit_util.check_file_content_for_html_and_images( full_path ) else: - shutil.move( uploaded_file_name, full_path ) - else: - shutil.move( uploaded_file_name, full_path ) - # See if any admin users have chosen to receive email alerts when a repository is updated. If so, check every uploaded file to ensure - # content is appropriate. - check_contents = commit_util.check_file_contents_for_email_alerts( trans ) - if check_contents and os.path.isfile( full_path ): - content_alert_str = commit_util.check_file_content_for_html_and_images( full_path ) - else: - content_alert_str = '' - commands.add( repo.ui, repo, full_path ) - # Convert from unicode to prevent "TypeError: array item must be char" - full_path = full_path.encode( 'ascii', 'replace' ) - commands.commit( repo.ui, repo, full_path, user=trans.user.username, message=commit_message ) - if full_path.endswith( 'tool_data_table_conf.xml.sample' ): - # Handle the special case where a tool_data_table_conf.xml.sample file is being uploaded by parsing the file and adding new entries - # to the in-memory trans.app.tool_data_tables dictionary. - error, error_message = tool_util.handle_sample_tool_data_table_conf_file( trans.app, full_path ) - if error: - message = '%s<br/>%s' % ( message, error_message ) - # See if the content of the change set was valid. - admin_only = len( repository.downloadable_revisions ) != 1 - suc.handle_email_alerts( trans, repository, content_alert_str=content_alert_str, new_repo_alert=new_repo_alert, admin_only=admin_only ) + content_alert_str = '' + commands.add( repo.ui, repo, full_path ) + # Convert from unicode to prevent "TypeError: array item must be char" + full_path = full_path.encode( 'ascii', 'replace' ) + commands.commit( repo.ui, repo, full_path, user=trans.user.username, message=commit_message ) + if full_path.endswith( 'tool_data_table_conf.xml.sample' ): + # Handle the special case where a tool_data_table_conf.xml.sample file is being uploaded by parsing the file and adding new entries + # to the in-memory trans.app.tool_data_tables dictionary. + error, error_message = tool_util.handle_sample_tool_data_table_conf_file( trans.app, full_path ) + if error: + message = '%s<br/>%s' % ( message, error_message ) + # See if the content of the change set was valid. + admin_only = len( repository.downloadable_revisions ) != 1 + suc.handle_email_alerts( trans, repository, content_alert_str=content_alert_str, new_repo_alert=new_repo_alert, admin_only=admin_only ) if ok: # Update the repository files for browsing. suc.update_repository( repo ) @@ -255,7 +260,10 @@ for root, dirs, files in os.walk( uploaded_directory ): for uploaded_file in files: relative_path = os.path.normpath( os.path.join( os.path.relpath( root, uploaded_directory ), uploaded_file ) ) - ok = os.path.basename( uploaded_file ) not in commit_util.UNDESIRABLE_FILES + if repository.type == rt_util.TOOL_DEPENDENCY_DEFINITION: + ok = os.path.basename( uploaded_file ) == suc.TOOL_DEPENDENCY_DEFINITION_FILENAME + else: + ok = os.path.basename( uploaded_file ) not in commit_util.UNDESIRABLE_FILES if ok: for file_path_item in relative_path.split( '/' ): if file_path_item in commit_util.UNDESIRABLE_DIRS: @@ -264,20 +272,20 @@ break else: undesirable_files_removed += 1 - uploaded_file_name = os.path.abspath( os.path.join( root, uploaded_file ) ) - if os.path.split( uploaded_file_name )[ -1 ] == 'repository_dependencies.xml': - # Inspect the contents of the file to see if changeset_revision values are missing and if so, set them appropriately. - altered, root_elem = commit_util.handle_repository_dependencies_definition( trans, uploaded_file_name ) - if altered: - tmp_filename = xml_util.create_and_write_tmp_file( root_elem ) - shutil.move( tmp_filename, uploaded_file_name ) - elif os.path.split( uploaded_file_name )[ -1 ] == 'tool_dependencies.xml': - # Inspect the contents of the file to see if changeset_revision values are missing and if so, set them appropriately. - altered, root_elem = commit_util.handle_tool_dependencies_definition( trans, uploaded_file_name ) - if altered: - tmp_filename = xml_util.create_and_write_tmp_file( root_elem ) - shutil.move( tmp_filename, uploaded_file_name ) if ok: + uploaded_file_name = os.path.abspath( os.path.join( root, uploaded_file ) ) + if os.path.split( uploaded_file_name )[ -1 ] == suc.REPOSITORY_DEPENDENCY_DEFINITION_FILENAME: + # Inspect the contents of the file to see if changeset_revision values are missing and if so, set them appropriately. + altered, root_elem = commit_util.handle_repository_dependencies_definition( trans, uploaded_file_name ) + if altered: + tmp_filename = xml_util.create_and_write_tmp_file( root_elem ) + shutil.move( tmp_filename, uploaded_file_name ) + elif os.path.split( uploaded_file_name )[ -1 ] == suc.TOOL_DEPENDENCY_DEFINITION_FILENAME: + # Inspect the contents of the file to see if changeset_revision values are missing and if so, set them appropriately. + altered, root_elem = commit_util.handle_tool_dependencies_definition( trans, uploaded_file_name ) + if altered: + tmp_filename = xml_util.create_and_write_tmp_file( root_elem ) + shutil.move( tmp_filename, uploaded_file_name ) repo_path = os.path.join( full_path, relative_path ) repo_basedir = os.path.normpath( os.path.join( repo_path, os.path.pardir ) ) if not os.path.exists( repo_basedir ): @@ -298,7 +306,7 @@ repo = hg.repository( suc.get_configured_ui(), repo_dir ) undesirable_dirs_removed = 0 undesirable_files_removed = 0 - ok, message = commit_util.check_archive( tar ) + ok, message = commit_util.check_archive( repository, tar ) if not ok: tar.close() uploaded_file.close() @@ -327,13 +335,13 @@ uploaded_file.close() for filename in filenames_in_archive: uploaded_file_name = os.path.join( full_path, filename ) - if os.path.split( uploaded_file_name )[ -1 ] == 'repository_dependencies.xml': + if os.path.split( uploaded_file_name )[ -1 ] == suc.REPOSITORY_DEPENDENCY_DEFINITION_FILENAME: # Inspect the contents of the file to see if changeset_revision values are missing and if so, set them appropriately. altered, root_elem = commit_util.handle_repository_dependencies_definition( trans, uploaded_file_name ) if altered: tmp_filename = xml_util.create_and_write_tmp_file( root_elem ) shutil.move( tmp_filename, uploaded_file_name ) - elif os.path.split( uploaded_file_name )[ -1 ] == 'tool_dependencies.xml': + elif os.path.split( uploaded_file_name )[ -1 ] == suc.TOOL_DEPENDENCY_DEFINITION_FILENAME: # Inspect the contents of the file to see if changeset_revision values are missing and if so, set them appropriately. altered, root_elem = commit_util.handle_tool_dependencies_definition( trans, uploaded_file_name ) if altered: diff -r 320d6f095b810d8c338c5ad5dbc04d6685505c5e -r 1d05c839c1ea7a83112a98734a6887f7ce419567 lib/tool_shed/galaxy_install/install_manager.py --- a/lib/tool_shed/galaxy_install/install_manager.py +++ b/lib/tool_shed/galaxy_install/install_manager.py @@ -269,7 +269,7 @@ self.app.sa_session.add( tool_shed_repository ) self.app.sa_session.flush() work_dir = tempfile.mkdtemp( prefix="tmp-toolshed-hrc" ) - datatypes_config = suc.get_config_from_disk( 'datatypes_conf.xml', repo_install_dir ) + datatypes_config = suc.get_config_from_disk( suc.DATATYPES_CONFIG_FILENAME, repo_install_dir ) # Load proprietary data types required by tools. The value of override is not important here since the Galaxy server will be started # after this installation completes. converter_path, display_path = datatype_util.alter_config_and_load_prorietary_datatypes( self.app, datatypes_config, repo_install_dir, override=False ) #repo_install_dir was relative_install_dir diff -r 320d6f095b810d8c338c5ad5dbc04d6685505c5e -r 1d05c839c1ea7a83112a98734a6887f7ce419567 lib/tool_shed/galaxy_install/repository_util.py --- a/lib/tool_shed/galaxy_install/repository_util.py +++ b/lib/tool_shed/galaxy_install/repository_util.py @@ -448,7 +448,7 @@ files_dir = relative_install_dir if shed_config_dict.get( 'tool_path' ): files_dir = os.path.join( shed_config_dict[ 'tool_path' ], files_dir ) - datatypes_config = suc.get_config_from_disk( 'datatypes_conf.xml', files_dir ) + datatypes_config = suc.get_config_from_disk( suc.DATATYPES_CONFIG_FILENAME, files_dir ) # Load data types required by tools. converter_path, display_path = datatype_util.alter_config_and_load_prorietary_datatypes( trans.app, datatypes_config, files_dir, override=False ) if converter_path or display_path: diff -r 320d6f095b810d8c338c5ad5dbc04d6685505c5e -r 1d05c839c1ea7a83112a98734a6887f7ce419567 lib/tool_shed/util/commit_util.py --- a/lib/tool_shed/util/commit_util.py +++ b/lib/tool_shed/util/commit_util.py @@ -11,6 +11,7 @@ from tool_shed.util import tool_util from tool_shed.util import xml_util from galaxy import eggs +import tool_shed.repository_types.util as rt_util eggs.require( 'mercurial' ) from mercurial import commands @@ -22,7 +23,7 @@ UNDESIRABLE_DIRS = [ '.hg', '.svn', '.git', '.cvs' ] UNDESIRABLE_FILES = [ '.hg_archival.txt', 'hgrc', '.DS_Store' ] -def check_archive( archive ): +def check_archive( repository, archive ): for member in archive.getmembers(): # Allow regular files and directories only if not ( member.isdir() or member.isfile() or member.islnk() ): @@ -35,6 +36,9 @@ if member.name in [ 'hgrc' ]: message = "Uploaded archives cannot contain hgrc files." return False, message + if repository.type == rt_util.TOOL_DEPENDENCY_DEFINITION and member.name != suc.TOOL_DEPENDENCY_DEFINITION_FILENAME: + message = 'Repositories of type <b>Tool dependency definition</b> can contain only a single file named <b>tool_dependencies.xml</b>.' + return False, message return True, '' def check_file_contents_for_email_alerts( trans ): diff -r 320d6f095b810d8c338c5ad5dbc04d6685505c5e -r 1d05c839c1ea7a83112a98734a6887f7ce419567 lib/tool_shed/util/datatype_util.py --- a/lib/tool_shed/util/datatype_util.py +++ b/lib/tool_shed/util/datatype_util.py @@ -166,7 +166,7 @@ # Load proprietary datatypes and return information needed for loading proprietary datatypes converters and display applications later. metadata = repository.metadata repository_dict = None - datatypes_config = suc.get_config_from_disk( 'datatypes_conf.xml', relative_install_dir ) + datatypes_config = suc.get_config_from_disk( suc.DATATYPES_CONFIG_FILENAME, relative_install_dir ) if datatypes_config: converter_path, display_path = alter_config_and_load_prorietary_datatypes( app, datatypes_config, relative_install_dir, deactivate=deactivate ) if converter_path or display_path: diff -r 320d6f095b810d8c338c5ad5dbc04d6685505c5e -r 1d05c839c1ea7a83112a98734a6887f7ce419567 lib/tool_shed/util/metadata_util.py --- a/lib/tool_shed/util/metadata_util.py +++ b/lib/tool_shed/util/metadata_util.py @@ -36,8 +36,10 @@ SUBSET = 'subset' SUBSET_VALUES = [ EQUAL, SUBSET ] -REPOSITORY_DATA_MANAGER_CONFIG_FILENAME = 'data_manager_conf.xml' -NOT_TOOL_CONFIGS = [ 'datatypes_conf.xml', 'repository_dependencies.xml', 'tool_dependencies.xml', REPOSITORY_DATA_MANAGER_CONFIG_FILENAME ] +NOT_TOOL_CONFIGS = [ suc.DATATYPES_CONFIG_FILENAME, + suc.REPOSITORY_DEPENDENCY_DEFINITION_FILENAME, + suc.TOOL_DEPENDENCY_DEFINITION_FILENAME, + suc.REPOSITORY_DATA_MANAGER_CONFIG_FILENAME ] def add_tool_versions( trans, id, repository_metadata, changeset_revisions ): # Build a dictionary of { 'tool id' : 'parent tool id' } pairs for each tool in repository_metadata. @@ -604,7 +606,7 @@ app.config.tool_data_path = work_dir #FIXME: Thread safe? app.config.tool_data_table_config_path = work_dir # Handle proprietary datatypes, if any. - datatypes_config = suc.get_config_from_disk( 'datatypes_conf.xml', files_dir ) + datatypes_config = suc.get_config_from_disk( suc.DATATYPES_CONFIG_FILENAME, files_dir ) if datatypes_config: metadata_dict = generate_datatypes_metadata( app, repository, repository_clone_url, files_dir, datatypes_config, metadata_dict ) # Get the relative path to all sample files included in the repository for storage in the repository's metadata. @@ -632,7 +634,7 @@ dirs.remove( '.hg' ) for name in files: # See if we have a repository dependencies defined. - if name == 'repository_dependencies.xml': + if name == suc.REPOSITORY_DEPENDENCY_DEFINITION_FILENAME: path_to_repository_dependencies_config = os.path.join( root, name ) metadata_dict, error_message = generate_repository_dependency_metadata( app, path_to_repository_dependencies_config, metadata_dict ) if error_message: @@ -700,7 +702,7 @@ metadata_dict = generate_data_manager_metadata( app, repository, files_dir, - suc.get_config_from_disk( REPOSITORY_DATA_MANAGER_CONFIG_FILENAME, files_dir ), + suc.get_config_from_disk( suc.REPOSITORY_DATA_MANAGER_CONFIG_FILENAME, files_dir ), metadata_dict, shed_config_dict=shed_config_dict ) diff -r 320d6f095b810d8c338c5ad5dbc04d6685505c5e -r 1d05c839c1ea7a83112a98734a6887f7ce419567 lib/tool_shed/util/shed_util_common.py --- a/lib/tool_shed/util/shed_util_common.py +++ b/lib/tool_shed/util/shed_util_common.py @@ -35,6 +35,11 @@ MAX_DISPLAY_SIZE = 32768 VALID_CHARS = set( string.letters + string.digits + "'\"-=_.()/+*^,:?!#[]%\\$@;{}&<>" ) +DATATYPES_CONFIG_FILENAME = 'datatypes_conf.xml' +REPOSITORY_DATA_MANAGER_CONFIG_FILENAME = 'data_manager_conf.xml' +REPOSITORY_DEPENDENCY_DEFINITION_FILENAME = 'repository_dependencies.xml' +TOOL_DEPENDENCY_DEFINITION_FILENAME = 'tool_dependencies.xml' + new_repo_email_alert_template = """ Sharable link: ${sharable_link} Repository name: ${repository_name} Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.