[galaxy-commits] commit/galaxy-central: greg: Enhance sample file handling for installed tool shed repositories to handle more than just .loc.sample files in the tool-data directory.

1 new commit in galaxy-central:

https://bitbucket.org/galaxy/galaxy-central/changeset/ea9f3c25a415/ changeset: ea9f3c25a415 user: greg date: 2012-05-03 19:37:33 summary: Enhance sample file handling for installed tool shed repositories to handle more than just .loc.sample files in the tool-data directory. affected #: 4 files

diff -r 71dfeecdd019554b5cccc047546ae26ed7b50afa -r ea9f3c25a41576216f3c328e3333ebdb55250ed1 lib/galaxy/tool_shed/install_manager.py --- a/lib/galaxy/tool_shed/install_manager.py +++ b/lib/galaxy/tool_shed/install_manager.py @@ -156,9 +156,15 @@ # Handle missing data table entries for tool parameters that are dynamically generated select lists. repository_tools_tups = handle_missing_data_table_entry( self.app, self.tool_path, sample_files, repository_tools_tups ) # Handle missing index files for tool parameters that are dynamically generated select lists. - repository_tools_tups = handle_missing_index_file( self.app, self.tool_path, sample_files, repository_tools_tups ) - # Handle tools that use fabric scripts to install dependencies. - handle_tool_dependencies( current_working_dir, relative_install_dir, repository_tools_tups ) + repository_tools_tups, sample_files_copied = handle_missing_index_file( self.app, self.tool_path, sample_files, repository_tools_tups ) + # Copy remaining sample files included in the repository to the ~/tool-data directory of the local Galaxy instance. + copy_sample_files( self.app, sample_files, sample_files_copied=sample_files_copied ) + if 'tool_dependencies_config' in metadata_dict: + # Install tool dependencies. + status, message = handle_tool_dependencies( self.app, repository_clone_url, metadata_dict[ 'tool_dependencies_config' ] ) + if status != 'ok' and message: + print 'The following error occurred while installing tool dependencies:' + print message add_to_tool_panel( self.app, repository_name, repository_clone_url, @@ -215,7 +221,7 @@ ctx_rev ) if 'tools' in metadata_dict: # Get the tool_versions from the tool shed for each tool in the installed change set. - url = '%s/repository/get_tool_versions?name=%s&owner=%s&changeset_revision=%s&webapp=galaxy' % \ + url = '%s/repository/get_tool_versions?name=%s&owner=%s&changeset_revision=%s&webapp=galaxy&no_reset=true' % \ ( tool_shed_url, tool_shed_repository.name, self.repository_owner, changeset_revision ) response = urllib2.urlopen( url ) text = response.read()

diff -r 71dfeecdd019554b5cccc047546ae26ed7b50afa -r ea9f3c25a41576216f3c328e3333ebdb55250ed1 lib/galaxy/util/shed_util.py --- a/lib/galaxy/util/shed_util.py +++ b/lib/galaxy/util/shed_util.py @@ -1,4 +1,4 @@ -import os, tempfile, shutil, subprocess, logging, string, urllib2 +import sys, os, tempfile, shutil, subprocess, logging, string, urllib2 from datetime import date, datetime, timedelta from time import strftime, gmtime from galaxy import util @@ -7,6 +7,9 @@ from galaxy.tools.search import ToolBoxSearch from galaxy.model.orm import *

+from galaxy import eggs +import pkg_resources + pkg_resources.require( 'mercurial' ) from mercurial import ui, commands

@@ -215,7 +218,6 @@ # The value of proprietary_path must be an absolute path due to job_working_directory. elem.attrib[ 'proprietary_path' ] = os.path.abspath( relative_head ) elem.attrib[ 'proprietary_datatype_module' ] = proprietary_datatype_module - sniffers = datatypes_config_root.find( 'sniffers' ) else: sniffers = None @@ -270,8 +272,7 @@ return tool_shed_url.rstrip( '/' ) def clone_repository( repository_clone_url, repository_file_dir, ctx_rev ): """ - Clone the repository up to the specified changeset_revision. No subsequent revisions will be present - in the cloned repository. + Clone the repository up to the specified changeset_revision. No subsequent revisions will be present in the cloned repository. """ commands.clone( get_configured_ui(), repository_clone_url, @@ -279,17 +280,28 @@ pull=True, noupdate=False, rev=[ ctx_rev ] ) -def copy_sample_loc_file( app, filename ): - """Copy xxx.loc.sample to ~/tool-data/xxx.loc.sample and ~/tool-data/xxx.loc""" - head, sample_loc_file = os.path.split( filename ) - loc_file = sample_loc_file.replace( '.sample', '' ) - tool_data_path = os.path.abspath( app.config.tool_data_path ) +def copy_sample_file( app, filename, dest_path=None ): + """ + Copy xxx.loc.sample to dest_path/xxx.loc.sample and dest_path/xxx.loc. The default value for dest_path is ~/tool-data. + """ + if dest_path is None: + dest_path = os.path.abspath( app.config.tool_data_path ) + sample_file_path, sample_file_name = os.path.split( filename ) + copied_file = sample_file_name.replace( '.sample', '' ) # It's ok to overwrite the .sample version of the file. - shutil.copy( os.path.abspath( filename ), os.path.join( tool_data_path, sample_loc_file ) ) - # Only create the .loc file if it does not yet exist. We don't - # overwrite it in case it contains stuff proprietary to the local instance. - if not os.path.exists( os.path.join( tool_data_path, loc_file ) ): - shutil.copy( os.path.abspath( filename ), os.path.join( tool_data_path, loc_file ) ) + shutil.copy( os.path.abspath( filename ), os.path.join( dest_path, sample_file_name ) ) + # Only create the .loc file if it does not yet exist. We don't overwrite it in case it contains stuff proprietary to the local instance. + if not os.path.exists( os.path.join( dest_path, copied_file ) ): + shutil.copy( os.path.abspath( filename ), os.path.join( dest_path, copied_file ) ) +def copy_sample_files( app, sample_files, sample_files_copied=None, dest_path=None ): + """ + Copy all files to dest_path in the local Galaxy environment that have not already been copied. Those that have been copied + are contained in sample_files_copied. The default value for dest_path is ~/tool-data. + """ + sample_files_copied = util.listify( sample_files_copied ) + for filename in sample_files: + if filename not in sample_files_copied: + copy_sample_file( app, filename, dest_path=dest_path ) def create_repository_dict_for_proprietary_datatypes( tool_shed, name, owner, installed_changeset_revision, tool_dicts, converter_path=None, display_path=None ): return dict( tool_shed=tool_shed, repository_name=name, @@ -336,7 +348,7 @@ tool_shed_url = get_url_from_repository_tool_shed( trans.app, repository ) return '%s/repos/%s/%s' % ( tool_shed_url, repository.owner, repository.name ) def generate_datatypes_metadata( datatypes_config, metadata_dict ): - """Update the received metadata_dict with changes that have been applied to the received datatypes_config.""" + """Update the received metadata_dict with information from the parsed datatypes_config.""" tree = ElementTree.parse( datatypes_config ) root = tree.getroot() ElementInclude.include( root ) @@ -454,21 +466,9 @@ # Handle tool.requirements. tool_requirements = [] for tr in tool.requirements: - name=tr.name - type=tr.type - if type == 'fabfile': - version = None - fabfile = tr.fabfile - method = tr.method - else: - version = tr.version - fabfile = None - method = None - requirement_dict = dict( name=name, - type=type, - version=version, - fabfile=fabfile, - method=method ) + requirement_dict = dict( name=tr.name, + type=tr.type, + version=tr.version ) tool_requirements.append( requirement_dict ) # Handle tool.tests. tool_tests = [] @@ -732,7 +732,7 @@ break return converter_path, display_path def get_ctx_rev( tool_shed_url, name, owner, changeset_revision ): - url = '%s/repository/get_ctx_rev?name=%s&owner=%s&changeset_revision=%s&webapp=galaxy' % ( tool_shed_url, name, owner, changeset_revision ) + url = '%s/repository/get_ctx_rev?name=%s&owner=%s&changeset_revision=%s&webapp=galaxy&no_reset=true' % ( tool_shed_url, name, owner, changeset_revision ) response = urllib2.urlopen( url ) ctx_rev = response.read() response.close() @@ -893,27 +893,27 @@ return repository_tools_tups def handle_missing_index_file( app, tool_path, sample_files, repository_tools_tups ): """Inspect each tool to see if it has any input parameters that are dynamically generated select lists that depend on a .loc file.""" - missing_files_handled = [] + sample_files_copied = [] for index, repository_tools_tup in enumerate( repository_tools_tups ): tup_path, guid, repository_tool = repository_tools_tup params_with_missing_index_file = repository_tool.params_with_missing_index_file for param in params_with_missing_index_file: options = param.options - missing_head, missing_tail = os.path.split( options.missing_index_file ) - if missing_tail not in missing_files_handled: + missing_file_path, missing_file_name = os.path.split( options.missing_index_file ) + if missing_file_name not in sample_files_copied: # The repository must contain the required xxx.loc.sample file. for sample_file in sample_files: - sample_head, sample_tail = os.path.split( sample_file ) - if sample_tail == '%s.sample' % missing_tail: - copy_sample_loc_file( app, sample_file ) + sample_file_path, sample_file_name = os.path.split( sample_file ) + if sample_file_name == '%s.sample' % missing_file_name: + copy_sample_file( app, sample_file ) if options.tool_data_table and options.tool_data_table.missing_index_file: options.tool_data_table.handle_found_index_file( options.missing_index_file ) - missing_files_handled.append( missing_tail ) + sample_files_copied.append( options.missing_index_file ) break # Reload the tool into the local list of repository_tools_tups. repository_tool = app.toolbox.load_tool( os.path.join( tool_path, tup_path ), guid=guid ) repository_tools_tups[ index ] = ( tup_path, guid, repository_tool ) - return repository_tools_tups + return repository_tools_tups, sample_files_copied def handle_sample_tool_data_table_conf_file( app, filename ): """ Parse the incoming filename and add new entries to the in-memory @@ -1038,7 +1038,8 @@ # Load or deactivate proprietary datatype display applications app.datatypes_registry.load_display_applications( installed_repository_dict=repository_dict, deactivate=deactivate ) def load_repository_contents( trans, repository_name, description, owner, changeset_revision, ctx_rev, tool_path, repository_clone_url, - relative_install_dir, current_working_dir, tool_shed=None, tool_section=None, shed_tool_conf=None ): + relative_install_dir, current_working_dir, tool_shed=None, tool_section=None, shed_tool_conf=None, + install_tool_dependencies=False ): """Generate the metadata for the installed tool shed repository, among other things.""" # It is critical that the installed repository is updated to the desired changeset_revision before metadata is set because the # process for setting metadata uses the repository files on disk. This method is called when an admin is installing a new repository @@ -1063,8 +1064,9 @@ # Handle missing data table entries for tool parameters that are dynamically generated select lists. repository_tools_tups = handle_missing_data_table_entry( trans.app, tool_path, sample_files, repository_tools_tups ) # Handle missing index files for tool parameters that are dynamically generated select lists. - repository_tools_tups = handle_missing_index_file( trans.app, tool_path, sample_files, repository_tools_tups ) - # Handle tools that use fabric scripts to install dependencies. + repository_tools_tups, sample_files_copied = handle_missing_index_file( trans.app, tool_path, sample_files, repository_tools_tups ) + # Copy remaining sample files included in the repository to the ~/tool-data directory of the local Galaxy instance. + copy_sample_files( trans.app, sample_files, sample_files_copied=sample_files_copied ) handle_tool_dependencies( current_working_dir, relative_install_dir, repository_tools_tups ) add_to_tool_panel( app=trans.app, repository_name=repository_name,

diff -r 71dfeecdd019554b5cccc047546ae26ed7b50afa -r ea9f3c25a41576216f3c328e3333ebdb55250ed1 lib/galaxy/webapps/community/controllers/common.py --- a/lib/galaxy/webapps/community/controllers/common.py +++ b/lib/galaxy/webapps/community/controllers/common.py @@ -5,7 +5,7 @@ from galaxy.tools import * from galaxy.util.json import from_json_string, to_json_string from galaxy.util.hash_util import * -from galaxy.util.shed_util import copy_sample_loc_file, get_configured_ui, generate_datatypes_metadata, generate_tool_metadata, generate_workflow_metadata +from galaxy.util.shed_util import copy_sample_file, get_configured_ui, generate_datatypes_metadata, generate_tool_metadata, generate_workflow_metadata from galaxy.util.shed_util import handle_sample_tool_data_table_conf_file, to_html_escaped, to_html_str, update_repository from galaxy.web.base.controller import * from galaxy.webapps.community import model @@ -209,13 +209,13 @@ if options.index_file or options.missing_index_file: # Make sure the repository contains the required xxx.loc.sample file. index_file = options.index_file or options.missing_index_file - index_head, index_tail = os.path.split( index_file ) + index_file_path, index_file_name = os.path.split( index_file ) sample_found = False for sample_file in sample_files: - sample_head, sample_tail = os.path.split( sample_file ) - if sample_tail == '%s.sample' % index_tail: - copy_sample_loc_file( trans.app, sample_file ) - options.index_file = index_tail + sample_file_path, sample_file_name = os.path.split( sample_file ) + if sample_file_name == '%s.sample' % index_file_name: + copy_sample_file( trans.app, sample_file ) + options.index_file = index_file_name options.missing_index_file = None if options.tool_data_table: options.tool_data_table.missing_index_file = None @@ -224,7 +224,7 @@ if not sample_found: can_set_metadata = False correction_msg = "This file refers to a file named <b>%s</b>. " % str( index_file ) - correction_msg += "Upload a file named <b>%s.sample</b> to the repository to correct this error." % str( index_tail ) + correction_msg += "Upload a file named <b>%s.sample</b> to the repository to correct this error." % str( index_file_name ) invalid_files.append( ( name, correction_msg ) ) return can_set_metadata, invalid_files def new_tool_metadata_required( trans, id, metadata_dict ): @@ -283,8 +283,7 @@ else: # There is no saved repository metadata, so we need to create a new repository_metadata table record. return True - # The received metadata_dict includes no metadata for workflows, so a new repository_metadata table - # record is not needed. + # The received metadata_dict includes no metadata for workflows, so a new repository_metadata table record is not needed. return False def generate_metadata_for_repository_tip( trans, id, ctx, changeset_revision, repo, repo_dir ): """

diff -r 71dfeecdd019554b5cccc047546ae26ed7b50afa -r ea9f3c25a41576216f3c328e3333ebdb55250ed1 lib/galaxy/webapps/community/controllers/upload.py --- a/lib/galaxy/webapps/community/controllers/upload.py +++ b/lib/galaxy/webapps/community/controllers/upload.py @@ -139,9 +139,8 @@ if error: message = '%s<br/>%s' % ( message, error_message ) if full_path.endswith( '.loc.sample' ): - # Handle the special case where a xxx.loc.sample file is - # being uploaded by copying it to ~/tool-data/xxx.loc. - copy_sample_loc_file( trans.app, full_path ) + # Handle the special case where a xxx.loc.sample file is being uploaded by copying it to ~/tool-data/xxx.loc. + copy_sample_file( trans.app, full_path ) # See if the content of the change set was valid. admin_only = len( repository.downloadable_revisions ) != 1 handle_email_alerts( trans, repository, content_alert_str=content_alert_str, new_repo_alert=new_repo_alert, admin_only=admin_only ) @@ -295,9 +294,8 @@ if error: return False, message, files_to_remove, content_alert_str if filename_in_archive.endswith( '.loc.sample' ): - # Handle the special case where a xxx.loc.sample file is - # being uploaded by copying it to ~/tool-data/xxx.loc. - copy_sample_loc_file( trans.app, filename_in_archive ) + # Handle the special case where a xxx.loc.sample file is being uploaded by copying it to ~/tool-data/xxx.loc. + copy_sample_file( trans.app, filename_in_archive ) try: commands.commit( repo.ui, repo, full_path, user=trans.user.username, message=commit_message ) except Exception, e:

Repository URL: https://bitbucket.org/galaxy/galaxy-central/

--

This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.