1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/24437efed65c/ Changeset: 24437efed65c User: greg Date: 2014-07-18 22:38:57 Summary: Add a ToolValidator to validate tools contained in tool shed repositories. Affected #: 6 files diff -r cd5d9bf32d865657208b0b657b08c9142b357e25 -r 24437efed65c4db4b22b25df6b6d307ac5dcb2ff lib/galaxy/webapps/tool_shed/controllers/repository.py --- a/lib/galaxy/webapps/tool_shed/controllers/repository.py +++ b/lib/galaxy/webapps/tool_shed/controllers/repository.py @@ -20,6 +20,7 @@ from tool_shed.galaxy_install import dependency_display from tool_shed.metadata import repository_metadata_manager +from tool_shed.tools import tool_validator from tool_shed.util import basic_util from tool_shed.util import common_util @@ -1164,10 +1165,10 @@ message = kwd.get( 'message', '' ) status = kwd.get( 'status', 'done' ) render_repository_actions_for = kwd.get( 'render_repository_actions_for', 'tool_shed' ) - repository, tool, message = tool_util.load_tool_from_changeset_revision( trans.app, - repository_id, - changeset_revision, - tool_config ) + tv = tool_validator.ToolValidator( trans.app ) + repository, tool, message = tv.load_tool_from_changeset_revision( repository_id, + changeset_revision, + tool_config ) if message: status = 'error' tool_state = tool_util.new_state( trans, tool, invalid=False ) @@ -2147,18 +2148,17 @@ message = kwd.get( 'message', '' ) status = kwd.get( 'status', 'error' ) render_repository_actions_for = kwd.get( 'render_repository_actions_for', 'tool_shed' ) - repository, tool, error_message = tool_util.load_tool_from_changeset_revision( trans.app, - repository_id, - changeset_revision, - tool_config ) + tv = tool_validator.ToolValidator( trans.app ) + repository, tool, error_message = tv.load_tool_from_changeset_revision( repository_id, + changeset_revision, + tool_config ) tool_state = tool_util.new_state( trans, tool, invalid=True ) invalid_file_tups = [] if tool: - invalid_file_tups = tool_util.check_tool_input_params( trans.app, - repository.repo_path( trans.app ), - tool_config, - tool, - [] ) + invalid_file_tups = tv.check_tool_input_params( repository.repo_path( trans.app ), + tool_config, + tool, + [] ) if invalid_file_tups: message = tool_util.generate_message_for_invalid_tools( trans.app, invalid_file_tups, @@ -3397,6 +3397,7 @@ metadata = repository_metadata.metadata if metadata: if 'tools' in metadata: + tv = tool_validator.ToolValidator( trans.app ) for tool_metadata_dict in metadata[ 'tools' ]: if tool_metadata_dict[ 'id' ] == tool_id: work_dir = tempfile.mkdtemp() @@ -3404,29 +3405,26 @@ guid = tool_metadata_dict[ 'guid' ] full_path_to_tool_config = os.path.abspath( relative_path_to_tool_config ) full_path_to_dir, tool_config_filename = os.path.split( full_path_to_tool_config ) - can_use_disk_file = tool_util.can_use_tool_config_disk_file( trans.app, - repository, - repo, - full_path_to_tool_config, - changeset_revision ) + can_use_disk_file = tv.can_use_tool_config_disk_file( repository, + repo, + full_path_to_tool_config, + changeset_revision ) if can_use_disk_file: trans.app.config.tool_data_path = work_dir tool, valid, message, sample_files = \ - tool_util.handle_sample_files_and_load_tool_from_disk( trans.app, - repo_files_dir, - repository_id, - full_path_to_tool_config, - work_dir ) + tv.handle_sample_files_and_load_tool_from_disk( repo_files_dir, + repository_id, + full_path_to_tool_config, + work_dir ) if message: status = 'error' else: tool, message, sample_files = \ - tool_util.handle_sample_files_and_load_tool_from_tmp_config( trans.app, - repo, - repository_id, - changeset_revision, - tool_config_filename, - work_dir ) + tv.handle_sample_files_and_load_tool_from_tmp_config( repo, + repository_id, + changeset_revision, + tool_config_filename, + work_dir ) if message: status = 'error' basic_util.remove_dir( work_dir ) diff -r cd5d9bf32d865657208b0b657b08c9142b357e25 -r 24437efed65c4db4b22b25df6b6d307ac5dcb2ff lib/tool_shed/metadata/metadata_generator.py --- a/lib/tool_shed/metadata/metadata_generator.py +++ b/lib/tool_shed/metadata/metadata_generator.py @@ -10,6 +10,7 @@ from galaxy.web import url_for from tool_shed.repository_types import util as rt_util +from tool_shed.tools import tool_validator from tool_shed.util import basic_util from tool_shed.util import common_util @@ -128,7 +129,7 @@ log.debug( 'Loaded Data Manager tool_files: %s' % ( tool_file ) ) return metadata_dict - def generate_datatypes_metadata( self, repository, repository_clone_url, repository_files_dir, datatypes_config, + def generate_datatypes_metadata( self, tv, repository, repository_clone_url, repository_files_dir, datatypes_config, metadata_dict ): """Update the received metadata_dict with information from the parsed datatypes_config.""" tree, error_message = xml_util.parse_xml( datatypes_config ) @@ -178,7 +179,7 @@ tool_config_path = hg_util.get_config_from_disk( tool_config, repository_files_dir ) full_path = os.path.abspath( tool_config_path ) tool, valid, error_message = \ - tool_util.load_tool_from_config( self.app, self.app.security.encode_id( repository.id ), full_path ) + tv.load_tool_from_config( self.app.security.encode_id( repository.id ), full_path ) if tool is None: guid = None else: @@ -245,6 +246,7 @@ tool_data_table_conf.xml.sample file, in which case the entries should ultimately be persisted to the file referred to by self.app.config.shed_tool_data_table_config. """ + tv = tool_validator.ToolValidator( self.app ) if shed_config_dict is None: shed_config_dict = {} if updating_installed_repository: @@ -290,7 +292,8 @@ # Handle proprietary datatypes, if any. datatypes_config = hg_util.get_config_from_disk( suc.DATATYPES_CONFIG_FILENAME, files_dir ) if datatypes_config: - metadata_dict = self.generate_datatypes_metadata( repository, + metadata_dict = self.generate_datatypes_metadata( tv, + repository, repository_clone_url, files_dir, datatypes_config, @@ -361,20 +364,18 @@ is_tool = element_tree_root.tag == 'tool' if is_tool: tool, valid, error_message = \ - tool_util.load_tool_from_config( self.app, - self.app.security.encode_id( repository.id ), - full_path ) + tv.load_tool_from_config( self.app.security.encode_id( repository.id ), + full_path ) if tool is None: if not valid: invalid_tool_configs.append( name ) invalid_file_tups.append( ( name, error_message ) ) else: invalid_files_and_errors_tups = \ - tool_util.check_tool_input_params( self.app, - files_dir, - name, - tool, - sample_file_copy_paths ) + tv.check_tool_input_params( files_dir, + name, + tool, + sample_file_copy_paths ) can_set_metadata = True for tup in invalid_files_and_errors_tups: if name in tup: diff -r cd5d9bf32d865657208b0b657b08c9142b357e25 -r 24437efed65c4db4b22b25df6b6d307ac5dcb2ff lib/tool_shed/tools/tool_validator.py --- /dev/null +++ b/lib/tool_shed/tools/tool_validator.py @@ -0,0 +1,289 @@ +import filecmp +import logging +import os +import tempfile + +from galaxy.tools import parameters +from galaxy.tools.parameters import dynamic_options + +from tool_shed.util import basic_util +from tool_shed.util import hg_util +from tool_shed.util import shed_util_common as suc +from tool_shed.util import tool_util + +log = logging.getLogger( __name__ ) + + +class ToolValidator( object ): + + def __init__( self, app ): + self.app = app + + def can_use_tool_config_disk_file( self, repository, repo, file_path, changeset_revision ): + """ + Determine if repository's tool config file on disk can be used. This method + is restricted to tool config files since, with the exception of tool config + files, multiple files with the same name will likely be in various directories + in the repository and we're comparing file names only (not relative paths). + """ + if not file_path or not os.path.exists( file_path ): + # The file no longer exists on disk, so it must have been deleted at some previous + # point in the change log. + return False + if changeset_revision == repository.tip( self.app ): + return True + file_name = basic_util.strip_path( file_path ) + latest_version_of_file = \ + self.get_latest_tool_config_revision_from_repository_manifest( repo, file_name, changeset_revision ) + can_use_disk_file = filecmp.cmp( file_path, latest_version_of_file ) + try: + os.unlink( latest_version_of_file ) + except: + pass + return can_use_disk_file + + def check_tool_input_params( self, repo_dir, tool_config_name, tool, sample_files ): + """ + Check all of the tool's input parameters, looking for any that are dynamically + generated using external data files to make sure the files exist. + """ + invalid_files_and_errors_tups = [] + correction_msg = '' + for input_param in tool.input_params: + if isinstance( input_param, parameters.basic.SelectToolParameter ) and input_param.is_dynamic: + # If the tool refers to .loc files or requires an entry in the tool_data_table_conf.xml, + # make sure all requirements exist. + options = input_param.dynamic_options or input_param.options + if options and isinstance( options, dynamic_options.DynamicOptions ): + if options.tool_data_table or options.missing_tool_data_table_name: + # Make sure the repository contains a tool_data_table_conf.xml.sample file. + sample_tool_data_table_conf = hg_util.get_config_from_disk( 'tool_data_table_conf.xml.sample', repo_dir ) + if sample_tool_data_table_conf: + error, correction_msg = \ + tool_util.handle_sample_tool_data_table_conf_file( self.app, sample_tool_data_table_conf ) + if error: + invalid_files_and_errors_tups.append( ( 'tool_data_table_conf.xml.sample', correction_msg ) ) + else: + options.missing_tool_data_table_name = None + else: + correction_msg = "This file requires an entry in the tool_data_table_conf.xml file. " + correction_msg += "Upload a file named tool_data_table_conf.xml.sample to the repository " + correction_msg += "that includes the required entry to correct this error.<br/>" + invalid_tup = ( tool_config_name, correction_msg ) + if invalid_tup not in invalid_files_and_errors_tups: + invalid_files_and_errors_tups.append( invalid_tup ) + if options.index_file or options.missing_index_file: + # Make sure the repository contains the required xxx.loc.sample file. + index_file = options.index_file or options.missing_index_file + index_file_name = basic_util.strip_path( index_file ) + sample_found = False + for sample_file in sample_files: + sample_file_name = basic_util.strip_path( sample_file ) + if sample_file_name == '%s.sample' % index_file_name: + options.index_file = index_file_name + options.missing_index_file = None + if options.tool_data_table: + options.tool_data_table.missing_index_file = None + sample_found = True + break + if not sample_found: + correction_msg = "This file refers to a file named <b>%s</b>. " % str( index_file_name ) + correction_msg += "Upload a file named <b>%s.sample</b> to the repository to correct this error." % \ + str( index_file_name ) + invalid_files_and_errors_tups.append( ( tool_config_name, correction_msg ) ) + return invalid_files_and_errors_tups + + def concat_messages( self, msg1, msg2 ): + if msg1: + if msg2: + message = '%s %s' % ( msg1, msg2 ) + else: + message = msg1 + elif msg2: + message = msg2 + else: + message = '' + return message + + def copy_disk_sample_files_to_dir( self, repo_files_dir, dest_path ): + """ + Copy all files currently on disk that end with the .sample extension to the + directory to which dest_path refers. + """ + sample_files = [] + for root, dirs, files in os.walk( repo_files_dir ): + if root.find( '.hg' ) < 0: + for name in files: + if name.endswith( '.sample' ): + relative_path = os.path.join( root, name ) + tool_util.copy_sample_file( self.app, relative_path, dest_path=dest_path ) + sample_files.append( name ) + return sample_files + + def get_latest_tool_config_revision_from_repository_manifest( self, repo, filename, changeset_revision ): + """ + Get the latest revision of a tool config file named filename from the repository + manifest up to the value of changeset_revision. This method is restricted to tool_config + files rather than any file since it is likely that, with the exception of tool config + files, multiple files will have the same name in various directories within the repository. + """ + stripped_filename = basic_util.strip_path( filename ) + for changeset in hg_util.reversed_upper_bounded_changelog( repo, changeset_revision ): + manifest_ctx = repo.changectx( changeset ) + for ctx_file in manifest_ctx.files(): + ctx_file_name = basic_util.strip_path( ctx_file ) + if ctx_file_name == stripped_filename: + try: + fctx = manifest_ctx[ ctx_file ] + except LookupError: + # The ctx_file may have been moved in the change set. For example, + # 'ncbi_blastp_wrapper.xml' was moved to 'tools/ncbi_blast_plus/ncbi_blastp_wrapper.xml', + # so keep looking for the file until we find the new location. + continue + fh = tempfile.NamedTemporaryFile( 'wb', prefix="tmp-toolshed-gltcrfrm" ) + tmp_filename = fh.name + fh.close() + fh = open( tmp_filename, 'wb' ) + fh.write( fctx.data() ) + fh.close() + return tmp_filename + return None + + def handle_sample_files_and_load_tool_from_disk( self, repo_files_dir, repository_id, tool_config_filepath, work_dir ): + """ + Copy all sample files from disk to a temporary directory since the sample files may + be in multiple directories. + """ + message = '' + sample_files = self.copy_disk_sample_files_to_dir( repo_files_dir, work_dir ) + if sample_files: + if 'tool_data_table_conf.xml.sample' in sample_files: + # Load entries into the tool_data_tables if the tool requires them. + tool_data_table_config = os.path.join( work_dir, 'tool_data_table_conf.xml' ) + error, message = tool_util.handle_sample_tool_data_table_conf_file( self.app, tool_data_table_config ) + tool, valid, message2 = self.load_tool_from_config( repository_id, tool_config_filepath ) + message = self.concat_messages( message, message2 ) + return tool, valid, message, sample_files + + def handle_sample_files_and_load_tool_from_tmp_config( self, repo, repository_id, changeset_revision, + tool_config_filename, work_dir ): + tool = None + message = '' + ctx = hg_util.get_changectx_for_changeset( repo, changeset_revision ) + # We're not currently doing anything with the returned list of deleted_sample_files here. It is + # intended to help handle sample files that are in the manifest, but have been deleted from disk. + sample_files, deleted_sample_files = get_list_of_copied_sample_files( repo, ctx, dir=work_dir ) + if sample_files: + self.app.config.tool_data_path = work_dir + if 'tool_data_table_conf.xml.sample' in sample_files: + # Load entries into the tool_data_tables if the tool requires them. + tool_data_table_config = os.path.join( work_dir, 'tool_data_table_conf.xml' ) + if tool_data_table_config: + error, message = tool_util.handle_sample_tool_data_table_conf_file( self.app, tool_data_table_config ) + if error: + log.debug( message ) + manifest_ctx, ctx_file = hg_util.get_ctx_file_path_from_manifest( tool_config_filename, repo, changeset_revision ) + if manifest_ctx and ctx_file: + tool, message2 = self.load_tool_from_tmp_config( repo, repository_id, manifest_ctx, ctx_file, work_dir ) + message = self.concat_messages( message, message2 ) + return tool, message, sample_files + + def load_tool_from_changeset_revision( self, repository_id, changeset_revision, tool_config_filename ): + """ + Return a loaded tool whose tool config file name (e.g., filtering.xml) is the value + of tool_config_filename. The value of changeset_revision is a valid (downloadable) + changeset revision. The tool config will be located in the repository manifest between + the received valid changeset revision and the first changeset revision in the repository, + searching backwards. + """ + original_tool_data_path = self.app.config.tool_data_path + repository = suc.get_repository_in_tool_shed( self.app, repository_id ) + repo_files_dir = repository.repo_path( self.app ) + repo = hg_util.get_repo_for_repository( self.app, repository=None, repo_path=repo_files_dir, create=False ) + message = '' + tool = None + can_use_disk_file = False + tool_config_filepath = suc.get_absolute_path_to_file_in_repository( repo_files_dir, tool_config_filename ) + work_dir = tempfile.mkdtemp( prefix="tmp-toolshed-ltfcr" ) + can_use_disk_file = self.can_use_tool_config_disk_file( repository, + repo, + tool_config_filepath, + changeset_revision ) + if can_use_disk_file: + self.app.config.tool_data_path = work_dir + tool, valid, message, sample_files = \ + self.handle_sample_files_and_load_tool_from_disk( repo_files_dir, + repository_id, + tool_config_filepath, + work_dir ) + if tool is not None: + invalid_files_and_errors_tups = \ + self.check_tool_input_params( repo_files_dir, + tool_config_filename, + tool, + sample_files ) + if invalid_files_and_errors_tups: + message2 = tool_util.generate_message_for_invalid_tools( self.app, + invalid_files_and_errors_tups, + repository, + metadata_dict=None, + as_html=True, + displaying_invalid_tool=True ) + message = self.concat_messages( message, message2 ) + else: + tool, message, sample_files = \ + self.handle_sample_files_and_load_tool_from_tmp_config( repo, + repository_id, + changeset_revision, + tool_config_filename, + work_dir ) + basic_util.remove_dir( work_dir ) + self.app.config.tool_data_path = original_tool_data_path + # Reset the tool_data_tables by loading the empty tool_data_table_conf.xml file. + tool_util.reset_tool_data_tables( self.app ) + return repository, tool, message + + def load_tool_from_config( self, repository_id, full_path ): + try: + tool = self.app.toolbox.load_tool( full_path, repository_id=repository_id ) + valid = True + error_message = None + except KeyError, e: + tool = None + valid = False + error_message = 'This file requires an entry for "%s" in the tool_data_table_conf.xml file. Upload a file ' % str( e ) + error_message += 'named tool_data_table_conf.xml.sample to the repository that includes the required entry to correct ' + error_message += 'this error. ' + except Exception, e: + tool = None + valid = False + error_message = str( e ) + return tool, valid, error_message + + def load_tool_from_tmp_config( self, repo, repository_id, ctx, ctx_file, work_dir ): + tool = None + message = '' + tmp_tool_config = hg_util.get_named_tmpfile_from_ctx( ctx, ctx_file, work_dir ) + if tmp_tool_config: + element_tree, error_message = xml_util.parse_xml( tmp_tool_config ) + if element_tree is None: + return tool, message + element_tree_root = element_tree.getroot() + # Look for code files required by the tool config. + tmp_code_files = [] + for code_elem in element_tree_root.findall( 'code' ): + code_file_name = code_elem.get( 'file' ) + tmp_code_file_name = hg_util.copy_file_from_manifest( repo, ctx, code_file_name, work_dir ) + if tmp_code_file_name: + tmp_code_files.append( tmp_code_file_name ) + tool, valid, message = self.load_tool_from_config( repository_id, tmp_tool_config ) + for tmp_code_file in tmp_code_files: + try: + os.unlink( tmp_code_file ) + except: + pass + try: + os.unlink( tmp_tool_config ) + except: + pass + return tool, message diff -r cd5d9bf32d865657208b0b657b08c9142b357e25 -r 24437efed65c4db4b22b25df6b6d307ac5dcb2ff lib/tool_shed/util/tool_util.py --- a/lib/tool_shed/util/tool_util.py +++ b/lib/tool_shed/util/tool_util.py @@ -1,14 +1,11 @@ -import filecmp import logging import os import shutil -import tempfile import galaxy.tools from galaxy import util from galaxy.datatypes import checkers from galaxy.model.orm import and_ from galaxy.tools import parameters -from galaxy.tools.parameters import dynamic_options from galaxy.tools.search import ToolBoxSearch from galaxy.util.expressions import ExpressionContext from galaxy.web.form_builder import SelectField @@ -112,107 +109,6 @@ select_field.add_option( option_tup[ 0 ], option_tup[ 1 ] ) return select_field -def can_use_tool_config_disk_file( app, repository, repo, file_path, changeset_revision ): - """ - Determine if repository's tool config file on disk can be used. This method - is restricted to tool config files since, with the exception of tool config - files, multiple files with the same name will likely be in various directories - in the repository and we're comparing file names only (not relative paths). - """ - if not file_path or not os.path.exists( file_path ): - # The file no longer exists on disk, so it must have been deleted at some previous - # point in the change log. - return False - if changeset_revision == repository.tip( app ): - return True - file_name = basic_util.strip_path( file_path ) - latest_version_of_file = get_latest_tool_config_revision_from_repository_manifest( repo, - file_name, - changeset_revision ) - can_use_disk_file = filecmp.cmp( file_path, latest_version_of_file ) - try: - os.unlink( latest_version_of_file ) - except: - pass - return can_use_disk_file - -def check_tool_input_params( app, repo_dir, tool_config_name, tool, sample_files ): - """ - Check all of the tool's input parameters, looking for any that are dynamically - generated using external data files to make sure the files exist. - """ - invalid_files_and_errors_tups = [] - correction_msg = '' - for input_param in tool.input_params: - if isinstance( input_param, parameters.basic.SelectToolParameter ) and input_param.is_dynamic: - # If the tool refers to .loc files or requires an entry in the tool_data_table_conf.xml, - # make sure all requirements exist. - options = input_param.dynamic_options or input_param.options - if options and isinstance( options, dynamic_options.DynamicOptions ): - if options.tool_data_table or options.missing_tool_data_table_name: - # Make sure the repository contains a tool_data_table_conf.xml.sample file. - sample_tool_data_table_conf = hg_util.get_config_from_disk( 'tool_data_table_conf.xml.sample', repo_dir ) - if sample_tool_data_table_conf: - error, correction_msg = handle_sample_tool_data_table_conf_file( app, sample_tool_data_table_conf ) - if error: - invalid_files_and_errors_tups.append( ( 'tool_data_table_conf.xml.sample', correction_msg ) ) - else: - options.missing_tool_data_table_name = None - else: - correction_msg = "This file requires an entry in the tool_data_table_conf.xml file. " - correction_msg += "Upload a file named tool_data_table_conf.xml.sample to the repository " - correction_msg += "that includes the required entry to correct this error.<br/>" - invalid_tup = ( tool_config_name, correction_msg ) - if invalid_tup not in invalid_files_and_errors_tups: - invalid_files_and_errors_tups.append( invalid_tup ) - if options.index_file or options.missing_index_file: - # Make sure the repository contains the required xxx.loc.sample file. - index_file = options.index_file or options.missing_index_file - index_file_name = basic_util.strip_path( index_file ) - sample_found = False - for sample_file in sample_files: - sample_file_name = basic_util.strip_path( sample_file ) - if sample_file_name == '%s.sample' % index_file_name: - options.index_file = index_file_name - options.missing_index_file = None - if options.tool_data_table: - options.tool_data_table.missing_index_file = None - sample_found = True - break - if not sample_found: - correction_msg = "This file refers to a file named <b>%s</b>. " % str( index_file_name ) - correction_msg += "Upload a file named <b>%s.sample</b> to the repository to correct this error." % \ - str( index_file_name ) - invalid_files_and_errors_tups.append( ( tool_config_name, correction_msg ) ) - return invalid_files_and_errors_tups - -def concat_messages( msg1, msg2 ): - if msg1: - if msg2: - message = '%s %s' % ( msg1, msg2 ) - else: - message = msg1 - elif msg2: - message = msg2 - else: - message = '' - return message - -def copy_disk_sample_files_to_dir( app, repo_files_dir, dest_path ): - """ - Copy all files currently on disk that end with the .sample extension to the - directory to which dest_path refers. - """ - sample_files = [] - for root, dirs, files in os.walk( repo_files_dir ): - if root.find( '.hg' ) < 0: - for name in files: - if name.endswith( '.sample' ): - relative_path = os.path.join( root, name ) - copy_sample_file( app, relative_path, dest_path=dest_path ) - sample_files.append( name ) - return sample_files - def copy_sample_file( app, filename, dest_path=None ): """ Copy xxx.sample to dest_path/xxx.sample and dest_path/xxx. The default value for dest_path @@ -445,35 +341,6 @@ break return headers -def get_latest_tool_config_revision_from_repository_manifest( repo, filename, changeset_revision ): - """ - Get the latest revision of a tool config file named filename from the repository - manifest up to the value of changeset_revision. This method is restricted to tool_config - files rather than any file since it is likely that, with the exception of tool config - files, multiple files will have the same name in various directories within the repository. - """ - stripped_filename = basic_util.strip_path( filename ) - for changeset in hg_util.reversed_upper_bounded_changelog( repo, changeset_revision ): - manifest_ctx = repo.changectx( changeset ) - for ctx_file in manifest_ctx.files(): - ctx_file_name = basic_util.strip_path( ctx_file ) - if ctx_file_name == stripped_filename: - try: - fctx = manifest_ctx[ ctx_file ] - except LookupError: - # The ctx_file may have been moved in the change set. For example, - # 'ncbi_blastp_wrapper.xml' was moved to 'tools/ncbi_blast_plus/ncbi_blastp_wrapper.xml', - # so keep looking for the file until we find the new location. - continue - fh = tempfile.NamedTemporaryFile( 'wb', prefix="tmp-toolshed-gltcrfrm" ) - tmp_filename = fh.name - fh.close() - fh = open( tmp_filename, 'wb' ) - fh.write( fctx.data() ) - fh.close() - return tmp_filename - return None - def get_list_of_copied_sample_files( repo, ctx, dir ): """ Find all sample files (files in the repository with the special .sample extension) @@ -664,45 +531,6 @@ repository_tools_tups[ index ] = ( tup_path, guid, repository_tool ) return repository_tools_tups, sample_files_copied -def handle_sample_files_and_load_tool_from_disk( app, repo_files_dir, repository_id, tool_config_filepath, work_dir ): - """ - Copy all sample files from disk to a temporary directory since the sample files may - be in multiple directories. - """ - message = '' - sample_files = copy_disk_sample_files_to_dir( app, repo_files_dir, work_dir ) - if sample_files: - if 'tool_data_table_conf.xml.sample' in sample_files: - # Load entries into the tool_data_tables if the tool requires them. - tool_data_table_config = os.path.join( work_dir, 'tool_data_table_conf.xml' ) - error, message = handle_sample_tool_data_table_conf_file( app, tool_data_table_config ) - tool, valid, message2 = load_tool_from_config( app, repository_id, tool_config_filepath ) - message = concat_messages( message, message2 ) - return tool, valid, message, sample_files - -def handle_sample_files_and_load_tool_from_tmp_config( app, repo, repository_id, changeset_revision, - tool_config_filename, work_dir ): - tool = None - message = '' - ctx = hg_util.get_changectx_for_changeset( repo, changeset_revision ) - # We're not currently doing anything with the returned list of deleted_sample_files here. It is - # intended to help handle sample files that are in the manifest, but have been deleted from disk. - sample_files, deleted_sample_files = get_list_of_copied_sample_files( repo, ctx, dir=work_dir ) - if sample_files: - app.config.tool_data_path = work_dir - if 'tool_data_table_conf.xml.sample' in sample_files: - # Load entries into the tool_data_tables if the tool requires them. - tool_data_table_config = os.path.join( work_dir, 'tool_data_table_conf.xml' ) - if tool_data_table_config: - error, message = handle_sample_tool_data_table_conf_file( app, tool_data_table_config ) - if error: - log.debug( message ) - manifest_ctx, ctx_file = hg_util.get_ctx_file_path_from_manifest( tool_config_filename, repo, changeset_revision ) - if manifest_ctx and ctx_file: - tool, message2 = load_tool_from_tmp_config( app, repo, repository_id, manifest_ctx, ctx_file, work_dir ) - message = concat_messages( message, message2 ) - return tool, message, sample_files - def handle_sample_tool_data_table_conf_file( app, filename, persist=False ): """ Parse the incoming filename and add new entries to the in-memory @@ -911,107 +739,6 @@ # Default to copying the file if none of the above are true. return True -def load_tool_from_changeset_revision( app, repository_id, changeset_revision, tool_config_filename ): - """ - Return a loaded tool whose tool config file name (e.g., filtering.xml) is the value - of tool_config_filename. The value of changeset_revision is a valid (downloadable) - changset revision. The tool config will be located in the repository manifest between - the received valid changeset revision and the first changeset revision in the repository, - searching backwards. - """ - original_tool_data_path = app.config.tool_data_path - repository = suc.get_repository_in_tool_shed( app, repository_id ) - repo_files_dir = repository.repo_path( app ) - repo = hg_util.get_repo_for_repository( app, repository=None, repo_path=repo_files_dir, create=False ) - message = '' - tool = None - can_use_disk_file = False - tool_config_filepath = suc.get_absolute_path_to_file_in_repository( repo_files_dir, tool_config_filename ) - work_dir = tempfile.mkdtemp( prefix="tmp-toolshed-ltfcr" ) - can_use_disk_file = can_use_tool_config_disk_file( app, - repository, - repo, - tool_config_filepath, - changeset_revision ) - if can_use_disk_file: - app.config.tool_data_path = work_dir - tool, valid, message, sample_files = handle_sample_files_and_load_tool_from_disk( app, - repo_files_dir, - repository_id, - tool_config_filepath, - work_dir ) - if tool is not None: - invalid_files_and_errors_tups = check_tool_input_params( app, - repo_files_dir, - tool_config_filename, - tool, - sample_files ) - if invalid_files_and_errors_tups: - message2 = generate_message_for_invalid_tools( app, - invalid_files_and_errors_tups, - repository, - metadata_dict=None, - as_html=True, - displaying_invalid_tool=True ) - message = concat_messages( message, message2 ) - else: - tool, message, sample_files = handle_sample_files_and_load_tool_from_tmp_config( app, - repo, - repository_id, - changeset_revision, - tool_config_filename, - work_dir ) - basic_util.remove_dir( work_dir ) - app.config.tool_data_path = original_tool_data_path - # Reset the tool_data_tables by loading the empty tool_data_table_conf.xml file. - reset_tool_data_tables( app ) - return repository, tool, message - -def load_tool_from_config( app, repository_id, full_path ): - try: - tool = app.toolbox.load_tool( full_path, repository_id=repository_id ) - valid = True - error_message = None - except KeyError, e: - tool = None - valid = False - error_message = 'This file requires an entry for "%s" in the tool_data_table_conf.xml file. Upload a file ' % str( e ) - error_message += 'named tool_data_table_conf.xml.sample to the repository that includes the required entry to correct ' - error_message += 'this error. ' - except Exception, e: - tool = None - valid = False - error_message = str( e ) - return tool, valid, error_message - -def load_tool_from_tmp_config( app, repo, repository_id, ctx, ctx_file, work_dir ): - tool = None - message = '' - tmp_tool_config = hg_util.get_named_tmpfile_from_ctx( ctx, ctx_file, work_dir ) - if tmp_tool_config: - element_tree, error_message = xml_util.parse_xml( tmp_tool_config ) - if element_tree is None: - return tool, message - element_tree_root = element_tree.getroot() - # Look for code files required by the tool config. - tmp_code_files = [] - for code_elem in element_tree_root.findall( 'code' ): - code_file_name = code_elem.get( 'file' ) - tmp_code_file_name = hg_util.copy_file_from_manifest( repo, ctx, code_file_name, work_dir ) - if tmp_code_file_name: - tmp_code_files.append( tmp_code_file_name ) - tool, valid, message = load_tool_from_config( app, repository_id, tmp_tool_config ) - for tmp_code_file in tmp_code_files: - try: - os.unlink( tmp_code_file ) - except: - pass - try: - os.unlink( tmp_tool_config ) - except: - pass - return tool, message - def new_state( trans, tool, invalid=False ): """Create a new `DefaultToolState` for the received tool. Only inputs on the first page will be initialized.""" state = galaxy.tools.DefaultToolState() diff -r cd5d9bf32d865657208b0b657b08c9142b357e25 -r 24437efed65c4db4b22b25df6b6d307ac5dcb2ff lib/tool_shed/util/workflow_util.py --- a/lib/tool_shed/util/workflow_util.py +++ b/lib/tool_shed/util/workflow_util.py @@ -1,5 +1,6 @@ import logging import os + import galaxy.tools import galaxy.tools.parameters import galaxy.webapps.galaxy.controllers.workflow @@ -9,9 +10,12 @@ from galaxy.workflow.modules import InputDataModule from galaxy.workflow.modules import ToolModule from galaxy.workflow.modules import WorkflowModuleFactory -import tool_shed.util.shed_util_common as suc + +from tool_shed.tools import tool_validator + from tool_shed.util import encoding_util from tool_shed.util import metadata_util +from tool_shed.util import shed_util_common as suc from tool_shed.util import tool_util eggs.require( "SVGFig" ) @@ -57,14 +61,14 @@ self.tool_id = tool_id self.tool = None self.errors = None + self.tv = tool_validator.ToolValidator( trans.app ) if trans.webapp.name == 'tool_shed': # We're in the tool shed. for tool_dict in tools_metadata: if self.tool_id in [ tool_dict[ 'id' ], tool_dict[ 'guid' ] ]: - repository, self.tool, message = tool_util.load_tool_from_changeset_revision( trans.app, - repository_id, - changeset_revision, - tool_dict[ 'tool_config' ] ) + repository, self.tool, message = self.tv.load_tool_from_changeset_revision( repository_id, + changeset_revision, + tool_dict[ 'tool_config' ] ) if message and self.tool is None: self.errors = 'unavailable' break Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.