1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/changeset/9ff18845d24b/ changeset: 9ff18845d24b user: greg date: 2012-08-30 16:42:55 summary: Enhancements, fixes and code cleanup related to setting metadata and loading and displaying tools in the tool shed. affected #: 4 files diff -r 32030702cf48527a17a648e828ec6658bf96841c -r 9ff18845d24b092a9297d1ec4a803b7e0e5fae4a lib/galaxy/util/shed_util.py --- a/lib/galaxy/util/shed_util.py +++ b/lib/galaxy/util/shed_util.py @@ -334,7 +334,7 @@ noupdate=False, rev=util.listify( str( ctx_rev ) ) ) def copy_sample_file( app, filename, dest_path=None ): - """Copy xxx.loc.sample to dest_path/xxx.loc.sample and dest_path/xxx.loc. The default value for dest_path is ~/tool-data.""" + """Copy xxx.sample to dest_path/xxx.sample and dest_path/xxx. The default value for dest_path is ~/tool-data.""" if dest_path is None: dest_path = os.path.abspath( app.config.tool_data_path ) sample_file_name = strip_path( filename ) @@ -573,13 +573,13 @@ if datatypes_config: metadata_dict = generate_datatypes_metadata( datatypes_config, metadata_dict ) # Get the relative path to all sample files included in the repository for storage in the repository's metadata. - sample_files = get_sample_files_from_disk( repository_files_dir=files_dir, - relative_install_dir=relative_install_dir, - resetting_all_metadata_on_repository=resetting_all_metadata_on_repository ) - if sample_files: - metadata_dict[ 'sample_files' ] = sample_files + sample_file_metadata_paths, sample_file_copy_paths = get_sample_files_from_disk( repository_files_dir=files_dir, + relative_install_dir=relative_install_dir, + resetting_all_metadata_on_repository=resetting_all_metadata_on_repository ) + if sample_file_metadata_paths: + metadata_dict[ 'sample_files' ] = sample_file_metadata_paths # Copy all sample files included in the repository to a single directory location so we can load tools that depend on them. - for sample_file in sample_files: + for sample_file in sample_file_copy_paths: copy_sample_file( app, sample_file, dest_path=work_dir ) # If the list of sample files includes a tool_data_table_conf.xml.sample file, laad it's table elements into memory. relative_path, filename = os.path.split( sample_file ) @@ -608,21 +608,9 @@ print "Error parsing %s", full_path, ", exception: ", str( e ) is_tool = False if is_tool: - try: - tool = app.toolbox.load_tool( full_path ) - except KeyError, e: - tool = None - invalid_tool_configs.append( name ) - error_message = 'This file requires an entry for "%s" in the tool_data_table_conf.xml file. Upload a file ' % str( e ) - error_message += 'named tool_data_table_conf.xml.sample to the repository that includes the required entry to correct ' - error_message += 'this error. ' - invalid_file_tups.append( ( name, error_message ) ) - except Exception, e: - tool = None - invalid_tool_configs.append( name ) - invalid_file_tups.append( ( name, str( e ) ) ) + tool, valid, error_message = load_tool_from_config( app, full_path ) if tool is not None: - invalid_files_and_errors_tups = check_tool_input_params( app, files_dir, name, tool, sample_files, webapp=webapp ) + invalid_files_and_errors_tups = check_tool_input_params( app, files_dir, name, tool, sample_file_metadata_paths, webapp=webapp ) can_set_metadata = True for tup in invalid_files_and_errors_tups: if name in tup: @@ -1077,7 +1065,8 @@ if resetting_all_metadata_on_repository: # Keep track of the location where the repository is temporarily cloned so that we can strip it when setting metadata. work_dir = repository_files_dir - sample_files = [] + sample_file_metadata_paths = [] + sample_file_copy_paths = [] for root, dirs, files in os.walk( repository_files_dir ): if root.find( '.hg' ) < 0: for name in files: @@ -1088,10 +1077,15 @@ if stripped_path_to_sample_file.startswith( '/' ): stripped_path_to_sample_file = stripped_path_to_sample_file[ 1: ] relative_path_to_sample_file = os.path.join( relative_install_dir, stripped_path_to_sample_file ) + if os.path.exists( relative_path_to_sample_file ): + sample_file_copy_paths.append( relative_path_to_sample_file ) + else: + sample_file_copy_paths.append( full_path_to_sample_file ) else: relative_path_to_sample_file = os.path.join( root, name ) - sample_files.append( relative_path_to_sample_file ) - return sample_files + sample_file_copy_paths.append( relative_path_to_sample_file ) + sample_file_metadata_paths.append( relative_path_to_sample_file ) + return sample_file_metadata_paths, sample_file_copy_paths def get_shed_tool_conf_dict( app, shed_tool_conf ): """ Return the in-memory version of the shed_tool_conf file, which is stored in the config_elems entry @@ -1404,6 +1398,22 @@ def load_installed_display_applications( app, installed_repository_dict, deactivate=False ): # Load or deactivate proprietary datatype display applications app.datatypes_registry.load_display_applications( installed_repository_dict=installed_repository_dict, deactivate=deactivate ) +def load_tool_from_config( app, full_path ): + try: + tool = app.toolbox.load_tool( full_path ) + valid = True + error_message = None + except KeyError, e: + tool = None + valid = False + error_message = 'This file requires an entry for "%s" in the tool_data_table_conf.xml file. Upload a file ' % str( e ) + error_message += 'named tool_data_table_conf.xml.sample to the repository that includes the required entry to correct ' + error_message += 'this error. ' + except Exception, e: + tool = None + valid = False + error_message = str( e ) + return tool, valid, error_message def open_repository_files_folder( trans, folder_path ): try: files_list = get_repository_files( trans, folder_path ) diff -r 32030702cf48527a17a648e828ec6658bf96841c -r 9ff18845d24b092a9297d1ec4a803b7e0e5fae4a lib/galaxy/webapps/community/controllers/common.py --- a/lib/galaxy/webapps/community/controllers/common.py +++ b/lib/galaxy/webapps/community/controllers/common.py @@ -1,13 +1,14 @@ -import os, string, socket, logging, simplejson, binascii, tempfile +import os, string, socket, logging, simplejson, binascii, tempfile, filecmp from time import strftime from datetime import * from galaxy.datatypes.checkers import * from galaxy.tools import * from galaxy.util.json import from_json_string, to_json_string from galaxy.util.hash_util import * -from galaxy.util.shed_util import clone_repository, generate_metadata_for_changeset_revision, get_changectx_for_changeset, get_config_from_disk -from galaxy.util.shed_util import get_configured_ui, get_named_tmpfile_from_ctx, handle_sample_tool_data_table_conf_file, INITIAL_CHANGELOG_HASH -from galaxy.util.shed_util import reset_tool_data_tables, reversed_upper_bounded_changelog, strip_path +from galaxy.util.shed_util import check_tool_input_params, clone_repository, copy_sample_file, generate_metadata_for_changeset_revision +from galaxy.util.shed_util import get_changectx_for_changeset, get_config_from_disk, get_configured_ui, get_named_tmpfile_from_ctx +from galaxy.util.shed_util import handle_sample_tool_data_table_conf_file, INITIAL_CHANGELOG_HASH, load_tool_from_config, reset_tool_data_tables +from galaxy.util.shed_util import reversed_upper_bounded_changelog, strip_path from galaxy.web.base.controller import * from galaxy.webapps.community import model from galaxy.model.orm import * @@ -105,6 +106,42 @@ trans.sa_session.flush() return item_rating +def add_tool_versions( trans, id, repository_metadata, changeset_revisions ): + # Build a dictionary of { 'tool id' : 'parent tool id' } pairs for each tool in repository_metadata. + metadata = repository_metadata.metadata + tool_versions_dict = {} + for tool_dict in metadata.get( 'tools', [] ): + # We have at least 2 changeset revisions to compare tool guids and tool ids. + parent_id = get_parent_id( trans, + id, + tool_dict[ 'id' ], + tool_dict[ 'version' ], + tool_dict[ 'guid' ], + changeset_revisions ) + tool_versions_dict[ tool_dict[ 'guid' ] ] = parent_id + if tool_versions_dict: + repository_metadata.tool_versions = tool_versions_dict + trans.sa_session.add( repository_metadata ) + trans.sa_session.flush() +def can_use_tool_config_disk_file( trans, repository, repo, file_path, changeset_revision ): + """ + Determine if repository's tool config file on disk can be used. This method is restricted to tool config files since, with the + exception of tool config files, multiple files with the same name will likely be in various directories in the repository and we're + comparing file names only (not relative paths). + """ + if not file_path or not os.path.exists( file_path ): + # The file no longer exists on disk, so it must have been deleted at some previous point in the change log. + return False + if changeset_revision == repository.tip: + return True + file_name = strip_path( file_path ) + latest_version_of_file = get_latest_tool_config_revision_from_repository_manifest( repo, file_name, changeset_revision ) + can_use_disk_file = filecmp.cmp( file_path, latest_version_of_file ) + try: + os.unlink( latest_version_of_file ) + except: + pass + return can_use_disk_file def changeset_is_malicious( trans, id, changeset_revision, **kwd ): """Check the malicious flag in repository metadata for a specified change set""" repository_metadata = get_repository_metadata_by_changeset_revision( trans, id, changeset_revision ) @@ -221,6 +258,16 @@ else: return 'subset' return 'not equal and not subset' +def copy_disk_sample_files_to_dir( trans, repo_files_dir, dest_path ): + sample_files = [] + for root, dirs, files in os.walk( repo_files_dir ): + if root.find( '.hg' ) < 0: + for name in files: + if name.endswith( '.sample' ): + relative_path = os.path.join( root, name ) + copy_sample_file( trans.app, relative_path, dest_path=dest_path ) + sample_files_copied.append( name ) + return sample_files def copy_file_from_disk( filename, repo_dir, dir ): file_path = None found = False @@ -240,9 +287,7 @@ tmp_filename = None return tmp_filename def copy_file_from_manifest( repo, ctx, filename, dir ): - """ - Copy the latest version of the file named filename from the repository manifest to the directory to which dir refers. - """ + """Copy the latest version of the file named filename from the repository manifest to the directory to which dir refers.""" for changeset in reversed_upper_bounded_changelog( repo, ctx ): changeset_ctx = repo.changectx( changeset ) fctx = get_file_context_from_ctx( changeset_ctx, filename ) @@ -277,7 +322,7 @@ return '%s://%s%s/repos/%s/%s' % ( protocol, username, base, repository.user.username, repository.name ) else: return '%s/repos/%s/%s' % ( base_url, repository.user.username, repository.name ) -def generate_message_for_invalid_tools( invalid_file_tups, repository, metadata_dict, as_html=True ): +def generate_message_for_invalid_tools( invalid_file_tups, repository, metadata_dict, as_html=True, displaying_invalid_tool=False ): if as_html: new_line = '<br/>' bold_start = '<b>' @@ -287,12 +332,13 @@ bold_start = '' bold_end = '' message = '' - if metadata_dict: - message += "Metadata was defined for some items in revision '%s'. " % str( repository.tip ) - message += "Correct the following problems if necessary and reset metadata.%s" % new_line - else: - message += "Metadata cannot be defined for revision '%s' so this revision cannot be automatically " % str( repository.tip ) - message += "installed into a local Galaxy instance. Correct the following problems and reset metadata.%s" % new_line + if not displaying_invalid_tool: + if metadata_dict: + message += "Metadata was defined for some items in revision '%s'. " % str( repository.tip ) + message += "Correct the following problems if necessary and reset metadata.%s" % new_line + else: + message += "Metadata cannot be defined for revision '%s' so this revision cannot be automatically " % str( repository.tip ) + message += "installed into a local Galaxy instance. Correct the following problems and reset metadata.%s" % new_line for itc_tup in invalid_file_tups: tool_file, exception_msg = itc_tup if exception_msg.find( 'No such file or directory' ) >= 0: @@ -322,6 +368,19 @@ repository.name, tool.id, tool.version ) +def get_absolute_path_to_file_in_repository( repo_files_dir, file_name ): + file_path = None + found = False + for root, dirs, files in os.walk( repo_files_dir ): + if root.find( '.hg' ) < 0: + for name in files: + if name == file_name: + file_path = os.path.abspath( os.path.join( root, name ) ) + found = True + break + if found: + break + return file_path def get_category( trans, id ): """Get a category from the database""" return trans.sa_session.query( trans.model.Category ).get( trans.security.decode_id( id ) ) @@ -353,12 +412,44 @@ if deleted: return 'DELETED' return None +def get_ctx_file_path_from_manifest( filename, repo, changeset_revision ): + """Get the ctx file path for the latest revision of filename from the repository manifest up to the value of changeset_revision.""" + stripped_filename = strip_path( filename ) + for changeset in reversed_upper_bounded_changelog( repo, changeset_revision ): + manifest_changeset_revision = str( repo.changectx( changeset ) ) + manifest_ctx = repo.changectx( changeset ) + for ctx_file in manifest_ctx.files(): + ctx_file_name = strip_path( ctx_file ) + if ctx_file_name == stripped_filename: + return manifest_ctx, ctx_file + return None, None def get_latest_repository_metadata( trans, decoded_repository_id ): """Get last metadata defined for a specified repository from the database""" return trans.sa_session.query( trans.model.RepositoryMetadata ) \ .filter( trans.model.RepositoryMetadata.table.c.repository_id == decoded_repository_id ) \ .order_by( trans.model.RepositoryMetadata.table.c.id.desc() ) \ .first() +def get_latest_tool_config_revision_from_repository_manifest( repo, filename, changeset_revision ): + """ + Get the latest revision of a tool config file named filename from the repository manifest up to the value of changeset_revision. + This method is restricted to tool_config files rather than any file since it is likely that, with the exception of tool config files, + multiple files will have the same name in various directories within the repository. + """ + stripped_filename = strip_path( filename ) + for changeset in reversed_upper_bounded_changelog( repo, changeset_revision ): + manifest_ctx = repo.changectx( changeset ) + for ctx_file in manifest_ctx.files(): + ctx_file_name = strip_path( ctx_file ) + if ctx_file_name == stripped_filename: + fctx = manifest_ctx[ ctx_file ] + fh = tempfile.NamedTemporaryFile( 'wb' ) + tmp_filename = fh.name + fh.close() + fh = open( tmp_filename, 'wb' ) + fh.write( fctx.data() ) + fh.close() + return tmp_filename + return None def get_list_of_copied_sample_files( repo, ctx, dir ): """ Find all sample files (files in the repository with the special .sample extension) in the reversed repository manifest up to ctx. Copy @@ -451,8 +542,8 @@ def get_repository_metadata_by_changeset_revision( trans, id, changeset_revision ): """Get metadata for a specified repository change set from the database""" # Make sure there are no duplicate records, and return the single unique record for the changeset_revision. Duplicate records were somehow - # creatd in the past. This may or may not be resolved, so when it is confirmed that the cause of duplicate records has been corrected, tweak - # this method accordingly. + # created in the past. The cause of this issue has been resolved, but we'll leave this method as is for a while longer to ensure all duplicate + # records are removed. all_metadata_records = trans.sa_session.query( trans.model.RepositoryMetadata ) \ .filter( and_( trans.model.RepositoryMetadata.table.c.repository_id == trans.security.decode_id( id ), trans.model.RepositoryMetadata.table.c.changeset_revision == changeset_revision ) ) \ @@ -577,90 +668,61 @@ log.exception( "An error occurred sending a tool shed repository update alert by email." ) def is_downloadable( metadata_dict ): return 'datatypes' in metadata_dict or 'tools' in metadata_dict or 'workflows' in metadata_dict -def load_tool( trans, config_file ): - """Load a single tool from the file named by `config_file` and return an instance of `Tool`.""" - # Parse XML configuration file and get the root element - tree = util.parse_xml( config_file ) - root = tree.getroot() - if root.tag == 'tool': - # Allow specifying a different tool subclass to instantiate - if root.find( "type" ) is not None: - type_elem = root.find( "type" ) - module = type_elem.get( 'module', 'galaxy.tools' ) - cls = type_elem.get( 'class' ) - mod = __import__( module, globals(), locals(), [cls] ) - ToolClass = getattr( mod, cls ) - elif root.get( 'tool_type', None ) is not None: - ToolClass = tool_types.get( root.get( 'tool_type' ) ) - else: - ToolClass = Tool - return ToolClass( config_file, root, trans.app ) - return None def load_tool_from_changeset_revision( trans, repository_id, changeset_revision, tool_config_filename ): """ Return a loaded tool whose tool config file name (e.g., filtering.xml) is the value of tool_config_filename. The value of changeset_revision is a valid (downloadable) changset revision. The tool config will be located in the repository manifest between the received valid changeset revision and the first changeset revision in the repository, searching backwards. """ - def load_from_tmp_config( toolbox, ctx, ctx_file, work_dir ): - tool = None - message = '' - tmp_tool_config = get_named_tmpfile_from_ctx( ctx, ctx_file, work_dir ) - if tmp_tool_config: - element_tree = util.parse_xml( tmp_tool_config ) - element_tree_root = element_tree.getroot() - # Look for code files required by the tool config. - tmp_code_files = [] - for code_elem in element_tree_root.findall( 'code' ): - code_file_name = code_elem.get( 'file' ) - tmp_code_file_name = copy_file_from_manifest( repo, ctx, code_file_name, work_dir ) - if tmp_code_file_name: - tmp_code_files.append( tmp_code_file_name ) - try: - tool = toolbox.load_tool( tmp_tool_config ) - except KeyError, e: - message = '<b>%s</b> - This file requires an entry for %s in the tool_data_table_conf.xml file. ' % ( tool_config_filename, str( e ) ) - message += 'Upload a file named tool_data_table_conf.xml.sample to the repository that includes the required entry to correct this error. ' - except Exception, e: - message = 'Error loading tool: %s. ' % str( e ) - for tmp_code_file in tmp_code_files: - try: - os.unlink( tmp_code_file ) - except: - pass - try: - os.unlink( tmp_tool_config ) - except: - pass - return tool, message original_tool_data_path = trans.app.config.tool_data_path - tool_config_filename = strip_path( tool_config_filename ) repository = get_repository( trans, repository_id ) repo_files_dir = repository.repo_path repo = hg.repository( get_configured_ui(), repo_files_dir ) - ctx = get_changectx_for_changeset( repo, changeset_revision ) message = '' + tool = None + can_use_disk_file = False + tool_config_filepath = get_absolute_path_to_file_in_repository( repo_files_dir, tool_config_filename ) work_dir = tempfile.mkdtemp() - sample_files, deleted_sample_files = get_list_of_copied_sample_files( repo, ctx, dir=work_dir ) - if sample_files: - trans.app.config.tool_data_path = work_dir - if 'tool_data_table_conf.xml.sample' in sample_files: - # Load entries into the tool_data_tables if the tool requires them. - tool_data_table_config = os.path.join( work_dir, 'tool_data_table_conf.xml' ) - error, correction_msg = handle_sample_tool_data_table_conf_file( trans.app, tool_data_table_config ) - found = False - # Get the latest revision of the tool config from the repository manifest up to the value of changeset_revision. - for changeset in reversed_upper_bounded_changelog( repo, changeset_revision ): - manifest_changeset_revision = str( repo.changectx( changeset ) ) - manifest_ctx = repo.changectx( changeset ) - for ctx_file in manifest_ctx.files(): - ctx_file_name = strip_path( ctx_file ) - if ctx_file_name == tool_config_filename: - found = True - break - if found: - tool, message = load_from_tmp_config( trans.app.toolbox, manifest_ctx, ctx_file, work_dir ) - break + can_use_disk_file = can_use_tool_config_disk_file( trans, repository, repo, tool_config_filepath, changeset_revision ) + if can_use_disk_file: + # Copy all sample files from disk to a temporary directory since the sample files may be in multiple directories. + sample_files = copy_disk_sample_files_to_dir( trans, repo_files_dir, work_dir ) + if sample_files: + trans.app.config.tool_data_path = work_dir + if 'tool_data_table_conf.xml.sample' in sample_files: + # Load entries into the tool_data_tables if the tool requires them. + tool_data_table_config = os.path.join( work_dir, 'tool_data_table_conf.xml' ) + error, correction_msg = handle_sample_tool_data_table_conf_file( trans.app, tool_data_table_config ) + tool, valid, message = load_tool_from_config( trans.app, tool_config_filepath ) + if tool is not None: + invalid_files_and_errors_tups = check_tool_input_params( trans.app, + repo_files_dir, + tool_config_filename, + tool, + sample_files, + webapp='community' ) + if invalid_files_and_errors_tups: + message = generate_message_for_invalid_tools( invalid_files_and_errors_tups, + repository, + metadata_dict=None, + as_html=True, + displaying_invalid_tool=True ) + status = 'error' + else: + # The desired version of the tool config is no longer on disk, so create a temporary work environment and copy the tool config and dependent files. + ctx = get_changectx_for_changeset( repo, changeset_revision ) + # We're not currently doing anything with the returned list of deleted_sample_files here. It is intended to help handle sample files that are in + # the manifest, but have been deleted from disk. + sample_files, deleted_sample_files = get_list_of_copied_sample_files( repo, ctx, dir=work_dir ) + if sample_files: + trans.app.config.tool_data_path = work_dir + if 'tool_data_table_conf.xml.sample' in sample_files: + # Load entries into the tool_data_tables if the tool requires them. + tool_data_table_config = os.path.join( work_dir, 'tool_data_table_conf.xml' ) + error, correction_msg = handle_sample_tool_data_table_conf_file( trans.app, tool_data_table_config ) + manifest_ctx, ctx_file = get_ctx_file_path_from_manifest( tool_config_filename, repo, changeset_revision ) + if manifest_ctx and ctx_file: + tool, message = load_tool_from_tmp_config( trans, manifest_ctx, ctx_file, work_dir ) try: shutil.rmtree( work_dir ) except: @@ -669,50 +731,32 @@ trans.app.config.tool_data_path = original_tool_data_path # Reset the tool_data_tables by loading the empty tool_data_table_conf.xml file. reset_tool_data_tables( trans.app ) + return repository, tool, message +def load_tool_from_tmp_config( trans, ctx, ctx_file, work_dir ): + tool = None + message = '' + tmp_tool_config = get_named_tmpfile_from_ctx( ctx, ctx_file, work_dir ) + if tmp_tool_config: + element_tree = util.parse_xml( tmp_tool_config ) + element_tree_root = element_tree.getroot() + # Look for code files required by the tool config. + tmp_code_files = [] + for code_elem in element_tree_root.findall( 'code' ): + code_file_name = code_elem.get( 'file' ) + tmp_code_file_name = copy_file_from_manifest( repo, ctx, code_file_name, work_dir ) + if tmp_code_file_name: + tmp_code_files.append( tmp_code_file_name ) + tool, valid, message = load_tool_from_config( trans.app, tmp_tool_config ) + for tmp_code_file in tmp_code_files: + try: + os.unlink( tmp_code_file ) + except: + pass + try: + os.unlink( tmp_tool_config ) + except: + pass return tool, message -def load_tool_from_tmp_directory( trans, repo, repo_dir, ctx, filename, dir ): - is_tool_config = False - tool = None - valid = False - error_message = '' - tmp_config = get_named_tmpfile_from_ctx( ctx, filename, dir ) - if tmp_config: - if not ( check_binary( tmp_config ) or check_image( tmp_config ) or check_gzip( tmp_config )[ 0 ] - or check_bz2( tmp_config )[ 0 ] or check_zip( tmp_config ) ): - try: - # Make sure we're looking at a tool config and not a display application config or something else. - element_tree = util.parse_xml( tmp_config ) - element_tree_root = element_tree.getroot() - is_tool_config = element_tree_root.tag == 'tool' - except Exception, e: - log.debug( "Error parsing %s, exception: %s" % ( tmp_config, str( e ) ) ) - is_tool_config = False - if is_tool_config: - # Load entries into the tool_data_tables if the tool requires them. - tool_data_table_config = copy_file_from_manifest( repo, ctx, 'tool_data_table_conf.xml.sample', dir ) - if tool_data_table_config: - error, correction_msg = handle_sample_tool_data_table_conf_file( trans.app, tool_data_table_config ) - # Look for code files required by the tool config. The directory to which dir refers should be removed by the caller. - for code_elem in element_tree_root.findall( 'code' ): - code_file_name = code_elem.get( 'file' ) - if not os.path.exists( os.path.join( dir, code_file_name ) ): - tmp_code_file_name = copy_file_from_disk( code_file_name, repo_dir, dir ) - if tmp_code_file_name is None: - tmp_code_file_name = copy_file_from_manifest( repo, ctx, code_file_name, dir ) - try: - tool = load_tool( trans, tmp_config ) - valid = True - except KeyError, e: - valid = False - error_message = 'This file requires an entry for "%s" in the tool_data_table_conf.xml file. Upload a file ' % str( e ) - error_message += 'named tool_data_table_conf.xml.sample to the repository that includes the required entry to correct ' - error_message += 'this error. ' - except Exception, e: - valid = False - error_message = str( e ) - # Reset the tool_data_tables by loading the empty tool_data_table_conf.xml file. - reset_tool_data_tables( trans.app ) - return is_tool_config, valid, tool, error_message def new_tool_metadata_required( trans, repository, metadata_dict ): """ Compare the last saved metadata for each tool in the repository with the new metadata in metadata_dict to determine if a new repository_metadata @@ -895,23 +939,6 @@ Set metadata using the repository's current disk files, returning specific error messages (if any) to alert the repository owner that the changeset has problems. """ - def add_tool_versions( trans, id, repository_metadata, changeset_revisions ): - # Build a dictionary of { 'tool id' : 'parent tool id' } pairs for each tool in repository_metadata. - metadata = repository_metadata.metadata - tool_versions_dict = {} - for tool_dict in metadata.get( 'tools', [] ): - # We have at least 2 changeset revisions to compare tool guids and tool ids. - parent_id = get_parent_id( trans, - id, - tool_dict[ 'id' ], - tool_dict[ 'version' ], - tool_dict[ 'guid' ], - changeset_revisions ) - tool_versions_dict[ tool_dict[ 'guid' ] ] = parent_id - if tool_versions_dict: - repository_metadata.tool_versions = tool_versions_dict - trans.sa_session.add( repository_metadata ) - trans.sa_session.flush() message = '' status = 'done' encoded_id = trans.security.encode_id( repository.id ) diff -r 32030702cf48527a17a648e828ec6658bf96841c -r 9ff18845d24b092a9297d1ec4a803b7e0e5fae4a lib/galaxy/webapps/community/controllers/repository.py --- a/lib/galaxy/webapps/community/controllers/repository.py +++ b/lib/galaxy/webapps/community/controllers/repository.py @@ -9,9 +9,9 @@ from galaxy.web.framework.helpers import time_ago, iff, grids from galaxy.util.json import from_json_string, to_json_string from galaxy.model.orm import * -from galaxy.util.shed_util import create_repo_info_dict, get_changectx_for_changeset, get_configured_ui, get_repository_file_contents, NOT_TOOL_CONFIGS -from galaxy.util.shed_util import open_repository_files_folder, reversed_lower_upper_bounded_changelog, reversed_upper_bounded_changelog, strip_path -from galaxy.util.shed_util import to_html_escaped, update_repository, url_join +from galaxy.util.shed_util import create_repo_info_dict, get_changectx_for_changeset, get_configured_ui, get_repository_file_contents, load_tool_from_config +from galaxy.util.shed_util import NOT_TOOL_CONFIGS, open_repository_files_folder, reversed_lower_upper_bounded_changelog, reversed_upper_bounded_changelog +from galaxy.util.shed_util import strip_path, to_html_escaped, update_repository, url_join from galaxy.tool_shed.encoding_util import * from common import * @@ -251,10 +251,13 @@ # TODO: improve performance by adding a db table associating users with repositories for which they have write access. username = kwd[ 'username' ] clause_list = [] - for repository in trans.sa_session.query( self.model_class ): - allow_push_usernames = repository.allow_push.split( ',' ) - if username in allow_push_usernames: - clause_list.append( self.model_class.table.c.id == repository.id ) + for repository in trans.sa_session.query( self.model_class ) \ + .filter( self.model_class.table.c.deleted == False ): + allow_push = repository.allow_push + if allow_push: + allow_push_usernames = allow_push.split( ',' ) + if username in allow_push_usernames: + clause_list.append( self.model_class.table.c.id == repository.id ) if clause_list: return trans.sa_session.query( self.model_class ) \ .filter( or_( *clause_list ) ) \ @@ -912,8 +915,7 @@ message = util.restore_text( params.get( 'message', '' ) ) status = params.get( 'status', 'done' ) webapp = get_webapp( trans, **kwd ) - repository = get_repository( trans, repository_id ) - tool, message = load_tool_from_changeset_revision( trans, repository_id, changeset_revision, tool_config ) + repository, tool, message = load_tool_from_changeset_revision( trans, repository_id, changeset_revision, tool_config ) tool_state = self.__new_state( trans ) is_malicious = changeset_is_malicious( trans, repository_id, repository.tip ) try: @@ -1402,14 +1404,12 @@ return trans.response.send_redirect( url ) @web.expose def load_invalid_tool( self, trans, repository_id, tool_config, changeset_revision, **kwd ): - # FIXME: loading an invalid tool should display an appropriate message as to why the tool is invalid. This worked until recently. params = util.Params( kwd ) message = util.restore_text( params.get( 'message', '' ) ) status = params.get( 'status', 'error' ) webapp = get_webapp( trans, **kwd ) repository_clone_url = generate_clone_url( trans, repository_id ) - repository = get_repository( trans, repository_id ) - tool, error_message = load_tool_from_changeset_revision( trans, repository_id, changeset_revision, tool_config ) + repository, tool, error_message = load_tool_from_changeset_revision( trans, repository_id, changeset_revision, tool_config ) tool_state = self.__new_state( trans ) is_malicious = changeset_is_malicious( trans, repository_id, repository.tip ) try: @@ -2220,6 +2220,8 @@ status = params.get( 'status', 'done' ) webapp = get_webapp( trans, **kwd ) repository = get_repository( trans, repository_id ) + repo_files_dir = repository.repo_path + repo = hg.repository( get_configured_ui(), repo_files_dir ) tool_metadata_dict = {} tool_lineage = [] tool = None @@ -2230,12 +2232,18 @@ if 'tools' in metadata: for tool_metadata_dict in metadata[ 'tools' ]: if tool_metadata_dict[ 'id' ] == tool_id: + relative_path_to_tool_config = tool_metadata_dict[ 'tool_config' ] guid = tool_metadata_dict[ 'guid' ] - try: - # We may be attempting to load a tool that no longer exists in the repository tip. - tool = load_tool( trans, os.path.abspath( tool_metadata_dict[ 'tool_config' ] ) ) - except: - tool = None + full_path = os.path.abspath( relative_path_to_tool_config ) + can_use_disk_file = can_use_tool_config_disk_file( trans, repository, repo, full_path, changeset_revision ) + if can_use_disk_file: + tool, valid, message = load_tool_from_config( trans.app, full_path ) + else: + # We're attempting to load a tool using a config that no longer exists on disk. + work_dir = tempfile.mkdtemp() + manifest_ctx, ctx_file = get_ctx_file_path_from_manifest( relative_path_to_tool_config, repo, changeset_revision ) + if manifest_ctx and ctx_file: + tool, message = load_tool_from_tmp_config( trans, manifest_ctx, ctx_file, work_dir ) break if guid: tool_lineage = self.get_versions_of_tool( trans, repository, repository_metadata, guid ) diff -r 32030702cf48527a17a648e828ec6658bf96841c -r 9ff18845d24b092a9297d1ec4a803b7e0e5fae4a lib/galaxy/webapps/community/controllers/workflow.py --- a/lib/galaxy/webapps/community/controllers/workflow.py +++ b/lib/galaxy/webapps/community/controllers/workflow.py @@ -48,7 +48,7 @@ self.errors = None for tool_dict in tools_metadata: if self.tool_id in [ tool_dict[ 'id' ], tool_dict[ 'guid' ] ]: - self.tool, message = load_tool_from_changeset_revision( trans, repository_id, changeset_revision, tool_dict[ 'tool_config' ] ) + repository, self.tool, message = load_tool_from_changeset_revision( trans, repository_id, changeset_revision, tool_dict[ 'tool_config' ] ) if message and self.tool is None: self.errors = 'unavailable' break Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.