- galaxy-commits - lists.galaxyproject.org

commit/galaxy-central: 2 new changesets
by commits-noreply＠bitbucket.org 28 May '14

28 May '14

2 new commits in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/f8c6577665c3/ Changeset: f8c6577665c3 User: jgoecks Date: 2014-05-28 23:20:15 Summary: Enhance TabularChunkedView to load first chunk from server if it's not already available. This enables TabularChunkedView to be loaded dynamically rather than relying on bootstrapped data. Affected #: 1 file diff -r 31740dffaf9ae4e345fd2d4de4ab44ed133dd567 -r f8c6577665c383645e934629784ece516b498d9b static/scripts/mvc/data.js --- a/static/scripts/mvc/data.js +++ b/static/scripts/mvc/data.js @@ -145,17 +145,24 @@ header_row.append('<th>' + column_names.join('</th><th>') + '</th>'); } - // Add first chunk. - var first_chunk = this.model.get('first_data_chunk'); + // Render first chunk. + var self = this, + first_chunk = this.model.get('first_data_chunk'); if (first_chunk) { + // First chunk is bootstrapped, so render now. this._renderChunk(first_chunk); } + else { + // No bootstrapping, so get first chunk and then render. + $.when(self.model.get_next_chunk()).then(function(result) { + self._renderChunk(result); + }); + } // -- Show new chunks during scrolling. -- - var self = this, - // Flag to ensure that only one chunk is loaded at a time. - loading_chunk = false; + // Flag to ensure that only one chunk is loaded at a time. + var loading_chunk = false; // Set up chunk loading when scrolling using the scrolling element. this.scroll_elt.scroll(function() { https://bitbucket.org/galaxy/galaxy-central/commits/85aa79e9ab9a/ Changeset: 85aa79e9ab9a User: jgoecks Date: 2014-05-28 23:21:04 Summary: Automated merge Affected #: 19 files diff -r f8c6577665c383645e934629784ece516b498d9b -r 85aa79e9ab9a87a5b5fea0ed1039aeb9a5c23642 lib/galaxy/dataset_collections/__init__.py --- a/lib/galaxy/dataset_collections/__init__.py +++ b/lib/galaxy/dataset_collections/__init__.py @@ -138,6 +138,18 @@ changed = self._set_from_dict( trans, dataset_collection_instance, payload ) return changed + def copy( + self, + trans, + parent, # PRECONDITION: security checks on ability to add to parent occurred during load. + source, + encoded_source_id, + ): + assert source == "hdca" # for now + source_hdca = self.__get_history_collection_instance( trans, encoded_source_id ) + parent.add_dataset_collection( source_hdca.copy() ) + return source_hdca + def _set_from_dict( self, trans, dataset_collection_instance, new_data ): # Blatantly stolen from UsesHistoryDatasetAssociationMixin.set_hda_from_dict. diff -r f8c6577665c383645e934629784ece516b498d9b -r 85aa79e9ab9a87a5b5fea0ed1039aeb9a5c23642 lib/galaxy/webapps/galaxy/api/history_contents.py --- a/lib/galaxy/webapps/galaxy/api/history_contents.py +++ b/lib/galaxy/webapps/galaxy/api/history_contents.py @@ -292,9 +292,28 @@ return hda_dict def __create_dataset_collection( self, trans, history, payload, **kwd ): - create_params = api_payload_to_create_params( payload ) + source = kwd.get("source", "new_collection") service = trans.app.dataset_collections_service - dataset_collection_instance = service.create( trans, parent=history, **create_params ) + if source == "new_collection": + create_params = api_payload_to_create_params( payload ) + dataset_collection_instance = service.create( + trans, + parent=history, + **create_params + ) + elif source == "hdca": + content = payload.get( 'content', None ) + if content is None: + raise exceptions.RequestParameterMissingException( "'content' id of target to copy is missing" ) + dataset_collection_instance = service.copy( + trans=trans, + parent=history, + source="hdca", + encoded_source_id=content, + ) + else: + message = "Invalid 'source' parameter in request %s" % source + raise exceptions.RequestParameterInvalidException(message) return self.__collection_dict( trans, dataset_collection_instance, view="element" ) @expose_api_anonymous diff -r f8c6577665c383645e934629784ece516b498d9b -r 85aa79e9ab9a87a5b5fea0ed1039aeb9a5c23642 lib/galaxy/webapps/tool_shed/controllers/repository.py --- a/lib/galaxy/webapps/tool_shed/controllers/repository.py +++ b/lib/galaxy/webapps/tool_shed/controllers/repository.py @@ -3120,12 +3120,12 @@ options_dict = hg_util.get_mercurial_default_options_dict( 'diff' ) # Not quite sure if the following settings make any difference, but with a combination of them and the size check on each # diff, we don't run out of memory when viewing the changelog of the cisortho2 repository on the test tool shed. - options_dict[ 'maxfile' ] = suc.MAXDIFFSIZE - options_dict[ 'maxtotal' ] = suc.MAXDIFFSIZE + options_dict[ 'maxfile' ] = basic_util.MAXDIFFSIZE + options_dict[ 'maxtotal' ] = basic_util.MAXDIFFSIZE diffopts = mdiff.diffopts( **options_dict ) for diff in patch.diff( repo, node1=ctx_parent.node(), node2=ctx.node(), opts=diffopts ): - if len( diff ) > suc.MAXDIFFSIZE: - diff = util.shrink_string_by_size( diff, suc.MAXDIFFSIZE ) + if len( diff ) > basic_util.MAXDIFFSIZE: + diff = util.shrink_string_by_size( diff, basic_util.MAXDIFFSIZE ) diffs.append( basic_util.to_html_string( diff ) ) modified, added, removed, deleted, unknown, ignored, clean = repo.status( node1=ctx_parent.node(), node2=ctx.node() ) anchors = modified + added + removed + deleted + unknown + ignored + clean diff -r f8c6577665c383645e934629784ece516b498d9b -r 85aa79e9ab9a87a5b5fea0ed1039aeb9a5c23642 lib/tool_shed/galaxy_install/install_manager.py --- a/lib/tool_shed/galaxy_install/install_manager.py +++ b/lib/tool_shed/galaxy_install/install_manager.py @@ -1,5 +1,7 @@ import logging import os +import sys +import traceback from galaxy import eggs @@ -11,7 +13,6 @@ from tool_shed.util import tool_dependency_util -from tool_shed.galaxy_install.tool_dependencies import td_common_util from tool_shed.galaxy_install.tool_dependencies.recipe.env_file_builder import EnvFileBuilder from tool_shed.galaxy_install.tool_dependencies.recipe.install_environment import InstallEnvironment from tool_shed.galaxy_install.tool_dependencies.recipe.recipe_manager import StepManager @@ -25,6 +26,10 @@ class InstallManager( object ): + def format_traceback( self ): + ex_type, ex, tb = sys.exc_info() + return ''.join( traceback.format_tb( tb ) ) + def get_tool_shed_repository_install_dir( self, app, tool_shed_repository ): return os.path.abspath( tool_shed_repository.repo_files_directory( app ) ) @@ -112,7 +117,7 @@ log.exception( 'Error installing tool dependency %s version %s.', str( tool_dependency.name ), str( tool_dependency.version ) ) # Since there was an installation error, update the tool dependency status to Error. The remove_installation_path option must # be left False here. - error_message = '%s\n%s' % ( td_common_util.format_traceback(), str( e ) ) + error_message = '%s\n%s' % ( self.format_traceback(), str( e ) ) tool_dependency = tool_dependency_util.handle_tool_dependency_installation_error( app, tool_dependency, error_message, diff -r f8c6577665c383645e934629784ece516b498d9b -r 85aa79e9ab9a87a5b5fea0ed1039aeb9a5c23642 lib/tool_shed/galaxy_install/tool_dependencies/env_manager.py --- /dev/null +++ b/lib/tool_shed/galaxy_install/tool_dependencies/env_manager.py @@ -0,0 +1,147 @@ +import logging +import os +import sys +from tool_shed.util import common_util +import tool_shed.util.shed_util_common as suc + +log = logging.getLogger( __name__ ) + + +class EnvManager( object ): + + def __init__( self, app ): + self.app = app + + def create_env_var_dict( self, elem, install_environment ): + env_var_name = elem.get( 'name', 'PATH' ) + env_var_action = elem.get( 'action', 'prepend_to' ) + env_var_text = None + tool_dependency_install_dir = install_environment.install_dir + tool_shed_repository_install_dir = install_environment.tool_shed_repository_install_dir + if elem.text and elem.text.find( 'REPOSITORY_INSTALL_DIR' ) >= 0: + if tool_shed_repository_install_dir and elem.text.find( '$REPOSITORY_INSTALL_DIR' ) != -1: + env_var_text = elem.text.replace( '$REPOSITORY_INSTALL_DIR', tool_shed_repository_install_dir ) + return dict( name=env_var_name, action=env_var_action, value=env_var_text ) + else: + env_var_text = elem.text.replace( '$REPOSITORY_INSTALL_DIR', tool_dependency_install_dir ) + return dict( name=env_var_name, action=env_var_action, value=env_var_text ) + if elem.text and elem.text.find( 'INSTALL_DIR' ) >= 0: + if tool_dependency_install_dir: + env_var_text = elem.text.replace( '$INSTALL_DIR', tool_dependency_install_dir ) + return dict( name=env_var_name, action=env_var_action, value=env_var_text ) + else: + env_var_text = elem.text.replace( '$INSTALL_DIR', tool_shed_repository_install_dir ) + return dict( name=env_var_name, action=env_var_action, value=env_var_text ) + if elem.text: + # Allow for environment variables that contain neither REPOSITORY_INSTALL_DIR nor INSTALL_DIR + # since there may be command line parameters that are tuned for a Galaxy instance. Allowing them + # to be set in one location rather than being hard coded into each tool config is the best approach. + # For example: + # <environment_variable name="GATK2_SITE_OPTIONS" action="set_to"> + # "--num_threads 4 --num_cpu_threads_per_data_thread 3 --phone_home STANDARD" + # </environment_variable> + return dict( name=env_var_name, action=env_var_action, value=elem.text) + return None + + def get_env_shell_file_path( self, installation_directory ): + env_shell_file_name = 'env.sh' + default_location = os.path.abspath( os.path.join( installation_directory, env_shell_file_name ) ) + if os.path.exists( default_location ): + return default_location + for root, dirs, files in os.walk( installation_directory ): + for name in files: + if name == env_shell_file_name: + return os.path.abspath( os.path.join( root, name ) ) + return None + + def get_env_shell_file_paths( self, elem ): + # Currently only the following tag set is supported. + # <repository toolshed="http://localhost:9009/" name="package_numpy_1_7" owner="test" changeset_revision="c84c6a8be056"> + # <package name="numpy" version="1.7.1" /> + # </repository> + env_shell_file_paths = [] + toolshed = elem.get( 'toolshed', None ) + repository_name = elem.get( 'name', None ) + repository_owner = elem.get( 'owner', None ) + changeset_revision = elem.get( 'changeset_revision', None ) + if toolshed and repository_name and repository_owner and changeset_revision: + # The protocol is not stored, but the port is if it exists. + toolshed = common_util.remove_protocol_from_tool_shed_url( toolshed ) + repository = suc.get_repository_for_dependency_relationship( self.app, + toolshed, + repository_name, + repository_owner, + changeset_revision ) + if repository: + for sub_elem in elem: + tool_dependency_type = sub_elem.tag + tool_dependency_name = sub_elem.get( 'name' ) + tool_dependency_version = sub_elem.get( 'version' ) + if tool_dependency_type and tool_dependency_name and tool_dependency_version: + # Get the tool_dependency so we can get its installation directory. + tool_dependency = None + for tool_dependency in repository.tool_dependencies: + if tool_dependency.type == tool_dependency_type and \ + tool_dependency.name == tool_dependency_name and \ + tool_dependency.version == tool_dependency_version: + break + if tool_dependency: + tool_dependency_key = '%s/%s' % ( tool_dependency_name, tool_dependency_version ) + installation_directory = tool_dependency.installation_directory( self.app ) + env_shell_file_path = self.get_env_shell_file_path( installation_directory ) + if env_shell_file_path: + env_shell_file_paths.append( env_shell_file_path ) + else: + error_message = "Skipping tool dependency definition because unable to locate env.sh file for tool dependency " + error_message += "type %s, name %s, version %s for repository %s" % \ + ( str( tool_dependency_type ), str( tool_dependency_name ), str( tool_dependency_version ), str( repository.name ) ) + log.debug( error_message ) + continue + else: + error_message = "Skipping tool dependency definition because unable to locate tool dependency " + error_message += "type %s, name %s, version %s for repository %s" % \ + ( str( tool_dependency_type ), str( tool_dependency_name ), str( tool_dependency_version ), str( repository.name ) ) + log.debug( error_message ) + continue + else: + error_message = "Skipping invalid tool dependency definition: type %s, name %s, version %s." % \ + ( str( tool_dependency_type ), str( tool_dependency_name ), str( tool_dependency_version ) ) + log.debug( error_message ) + continue + else: + error_message = "Skipping set_environment_for_install definition because unable to locate required installed tool shed repository: " + error_message += "toolshed %s, name %s, owner %s, changeset_revision %s." % \ + ( str( toolshed ), str( repository_name ), str( repository_owner ), str( changeset_revision ) ) + log.debug( error_message ) + else: + error_message = "Skipping invalid set_environment_for_install definition: toolshed %s, name %s, owner %s, changeset_revision %s." % \ + ( str( toolshed ), str( repository_name ), str( repository_owner ), str( changeset_revision ) ) + log.debug( error_message ) + return env_shell_file_paths + + def get_env_shell_file_paths_from_setup_environment_elem( self, all_env_shell_file_paths, elem, action_dict ): + """ + Parse an XML tag set to discover all child repository dependency tags and define the path to an env.sh file associated + with the repository (this requires the repository dependency to be in an installed state). The received action_dict + will be updated with these discovered paths and returned to the caller. This method handles tool dependency definition + tag sets <setup_r_environment>, <setup_ruby_environment> and <setup_perl_environment>. + """ + # An example elem is: + # <action type="setup_perl_environment"> + # <repository name="package_perl_5_18" owner="iuc"> + # <package name="perl" version="5.18.1" /> + # </repository> + # <repository name="package_expat_2_1" owner="iuc" prior_installation_required="True"> + # <package name="expat" version="2.1.0" /> + # </repository> + # <package>http://search.cpan.org/CPAN/authors/id/T/TO/TODDR/XML-Parser-2.41.tar.gz</package> + # <package>http://search.cpan.org/CPAN/authors/id/L/LD/LDS/CGI.pm-3.43.tar.gz</package> + # </action> + for action_elem in elem: + if action_elem.tag == 'repository': + env_shell_file_paths = self.get_env_shell_file_paths( action_elem ) + all_env_shell_file_paths.extend( env_shell_file_paths ) + if all_env_shell_file_paths: + action_dict[ 'env_shell_file_paths' ] = all_env_shell_file_paths + action_dict[ 'action_shell_file_paths' ] = env_shell_file_paths + return action_dict diff -r f8c6577665c383645e934629784ece516b498d9b -r 85aa79e9ab9a87a5b5fea0ed1039aeb9a5c23642 lib/tool_shed/galaxy_install/tool_dependencies/recipe/install_environment.py --- a/lib/tool_shed/galaxy_install/tool_dependencies/recipe/install_environment.py +++ b/lib/tool_shed/galaxy_install/tool_dependencies/recipe/install_environment.py @@ -24,11 +24,13 @@ from galaxy.util import shrink_string_by_size from galaxy.util import unicodify -from tool_shed.galaxy_install.tool_dependencies import td_common_util from tool_shed.galaxy_install.tool_dependencies.recipe import asynchronous_reader +from tool_shed.util import basic_util + log = logging.getLogger( __name__ ) + class InstallEnvironment( object ): """Object describing the environment built up as part of the process of building and installing a package.""" @@ -44,7 +46,7 @@ self.tool_shed_repository_install_dir = tool_shed_repository_install_dir def __call__( self ): - with settings( warn_only=True, **td_common_util.get_env_var_values( self ) ): + with settings( warn_only=True, **basic_util.get_env_var_values( self ) ): with prefix( self.__setup_environment() ): yield @@ -125,7 +127,7 @@ context = app.install_model.context command = str( cmd ) output = self.handle_complex_command( command ) - self.log_results( cmd, output, os.path.join( self.install_dir, td_common_util.INSTALLATION_LOG ) ) + self.log_results( cmd, output, os.path.join( self.install_dir, basic_util.INSTALLATION_LOG ) ) stdout = output.stdout stderr = output.stderr if len( stdout ) > DATABASE_MAX_STRING_SIZE: @@ -214,18 +216,18 @@ # Sleep a bit before asking the readers again. time.sleep( .1 ) current_wait_time = time.time() - start_timer - if stdout_queue.empty() and stderr_queue.empty() and current_wait_time > td_common_util.NO_OUTPUT_TIMEOUT: + if stdout_queue.empty() and stderr_queue.empty() and current_wait_time > basic_util.NO_OUTPUT_TIMEOUT: err_msg = "\nShutting down process id %s because it generated no output for the defined timeout period of %.1f seconds.\n" % \ - ( pid, td_common_util.NO_OUTPUT_TIMEOUT ) + ( pid, basic_util.NO_OUTPUT_TIMEOUT ) stderr_reader.lines.append( err_msg ) process_handle.kill() break thread_lock.release() # Wait until each of the threads we've started terminate. The following calls will block each thread # until it terminates either normally, through an unhandled exception, or until the timeout occurs. - stdio_thread.join( td_common_util.NO_OUTPUT_TIMEOUT ) - stdout_reader.join( td_common_util.NO_OUTPUT_TIMEOUT ) - stderr_reader.join( td_common_util.NO_OUTPUT_TIMEOUT ) + stdio_thread.join( basic_util.NO_OUTPUT_TIMEOUT ) + stdout_reader.join( basic_util.NO_OUTPUT_TIMEOUT ) + stderr_reader.join( basic_util.NO_OUTPUT_TIMEOUT ) # Close subprocess' file descriptors. error = self.close_file_descriptor( process_handle.stdout ) error = self.close_file_descriptor( process_handle.stderr ) diff -r f8c6577665c383645e934629784ece516b498d9b -r 85aa79e9ab9a87a5b5fea0ed1039aeb9a5c23642 lib/tool_shed/galaxy_install/tool_dependencies/recipe/step_handler.py --- a/lib/tool_shed/galaxy_install/tool_dependencies/recipe/step_handler.py +++ b/lib/tool_shed/galaxy_install/tool_dependencies/recipe/step_handler.py @@ -4,12 +4,17 @@ import stat from string import Template import sys +import tarfile +import time +import urllib2 +import zipfile from galaxy.util import asbool from galaxy.util.template import fill_template +from tool_shed.util import basic_util from tool_shed.util import tool_dependency_util -from tool_shed.galaxy_install.tool_dependencies import td_common_util +from tool_shed.galaxy_install.tool_dependencies.env_manager import EnvManager # TODO: eliminate the use of fabric here. from galaxy import eggs @@ -26,6 +31,152 @@ VIRTUALENV_URL = 'https://pypi.python.org/packages/source/v/virtualenv/virtualenv-1.9.1.tar.gz' +class CompressedFile( object ): + + def __init__( self, file_path, mode='r' ): + if tarfile.is_tarfile( file_path ): + self.file_type = 'tar' + elif zipfile.is_zipfile( file_path ) and not file_path.endswith( '.jar' ): + self.file_type = 'zip' + self.file_name = os.path.splitext( os.path.basename( file_path ) )[ 0 ] + if self.file_name.endswith( '.tar' ): + self.file_name = os.path.splitext( self.file_name )[ 0 ] + self.type = self.file_type + method = 'open_%s' % self.file_type + if hasattr( self, method ): + self.archive = getattr( self, method )( file_path, mode ) + else: + raise NameError( 'File type %s specified, no open method found.' % self.file_type ) + + def extract( self, path ): + '''Determine the path to which the archive should be extracted.''' + contents = self.getmembers() + extraction_path = path + if len( contents ) == 1: + # The archive contains a single file, return the extraction path. + if self.isfile( contents[ 0 ] ): + extraction_path = os.path.join( path, self.file_name ) + if not os.path.exists( extraction_path ): + os.makedirs( extraction_path ) + self.archive.extractall( extraction_path ) + else: + # Get the common prefix for all the files in the archive. If the common prefix ends with a slash, + # or self.isdir() returns True, the archive contains a single directory with the desired contents. + # Otherwise, it contains multiple files and/or directories at the root of the archive. + common_prefix = os.path.commonprefix( [ self.getname( item ) for item in contents ] ) + if len( common_prefix ) >= 1 and not common_prefix.endswith( os.sep ) and self.isdir( self.getmember( common_prefix ) ): + common_prefix += os.sep + if common_prefix.endswith( os.sep ): + self.archive.extractall( os.path.join( path ) ) + extraction_path = os.path.join( path, common_prefix ) + else: + extraction_path = os.path.join( path, self.file_name ) + if not os.path.exists( extraction_path ): + os.makedirs( extraction_path ) + self.archive.extractall( os.path.join( extraction_path ) ) + return os.path.abspath( extraction_path ) + + def getmembers_tar( self ): + return self.archive.getmembers() + + def getmembers_zip( self ): + return self.archive.infolist() + + def getname_tar( self, item ): + return item.name + + def getname_zip( self, item ): + return item.filename + + def getmember( self, name ): + for member in self.getmembers(): + if self.getname( member ) == name: + return member + + def getmembers( self ): + return getattr( self, 'getmembers_%s' % self.type )() + + def getname( self, member ): + return getattr( self, 'getname_%s' % self.type )( member ) + + def isdir( self, member ): + return getattr( self, 'isdir_%s' % self.type )( member ) + + def isdir_tar( self, member ): + return member.isdir() + + def isdir_zip( self, member ): + if member.filename.endswith( os.sep ): + return True + return False + + def isfile( self, member ): + if not self.isdir( member ): + return True + return False + + def open_tar( self, filepath, mode ): + return tarfile.open( filepath, mode, errorlevel=0 ) + + def open_zip( self, filepath, mode ): + return zipfile.ZipFile( filepath, mode ) + + def zipfile_ok( self, path_to_archive ): + """ + This function is a bit pedantic and not functionally necessary. It checks whether there is + no file pointing outside of the extraction, because ZipFile.extractall() has some potential + security holes. See python zipfile documentation for more details. + """ + basename = os.path.realpath( os.path.dirname( path_to_archive ) ) + zip_archive = zipfile.ZipFile( path_to_archive ) + for member in zip_archive.namelist(): + member_path = os.path.realpath( os.path.join( basename, member ) ) + if not member_path.startswith( basename ): + return False + return True + + +class Download( object ): + + def url_download( self, install_dir, downloaded_file_name, download_url, extract=True ): + file_path = os.path.join( install_dir, downloaded_file_name ) + src = None + dst = None + # Set a timer so we don't sit here forever. + start_time = time.time() + try: + src = urllib2.urlopen( download_url ) + dst = open( file_path, 'wb' ) + while True: + chunk = src.read( basic_util.CHUNK_SIZE ) + if chunk: + dst.write( chunk ) + else: + break + time_taken = time.time() - start_time + if time_taken > basic_util.NO_OUTPUT_TIMEOUT: + err_msg = 'Downloading from URL %s took longer than the defined timeout period of %.1f seconds.' % \ + ( str( download_url ), basic_util.NO_OUTPUT_TIMEOUT ) + raise Exception( err_msg ) + except Exception, e: + err_msg = err_msg = 'Error downloading from URL\n%s:\n%s' % ( str( download_url ), str( e ) ) + raise Exception( err_msg ) + finally: + if src: + src.close() + if dst: + dst.close() + if extract: + if tarfile.is_tarfile( file_path ) or ( zipfile.is_zipfile( file_path ) and not file_path.endswith( '.jar' ) ): + archive = CompressedFile( file_path ) + extraction_path = archive.extract( install_dir ) + else: + extraction_path = os.path.abspath( install_dir ) + else: + extraction_path = os.path.abspath( install_dir ) + return extraction_path + + class RecipeStep( object ): """Abstract class that defines a standard format for handling recipe steps when installing packages.""" @@ -42,6 +193,22 @@ def __init__( self ): self.type = 'assert_directory_executable' + def assert_directory_executable( self, full_path ): + """ + Return True if a symbolic link or directory exists and is executable, but if + full_path is a file, return False. + """ + if full_path is None: + return False + if os.path.isfile( full_path ): + return False + if os.path.isdir( full_path ): + # Make sure the owner has execute permission on the directory. + # See http://docs.python.org/2/library/stat.html + if stat.S_IXUSR & os.stat( full_path )[ stat.ST_MODE ] == 64: + return True + return False + def execute_step( self, app, tool_dependency, package_name, actions, action_dict, filtered_actions, env_file_builder, install_environment, work_dir, current_dir=None, initial_download=False ): """ @@ -53,7 +220,7 @@ full_path = action_dict[ 'full_path' ] else: full_path = os.path.join( current_dir, action_dict[ 'full_path' ] ) - if not td_common_util.assert_directory_executable( full_path=full_path ): + if not self.assert_directory_executable( full_path=full_path ): status = app.install_model.ToolDependency.installation_status.ERROR error_message = 'The path %s is not a directory or is not executable by the owner.' % str( full_path ) tool_dependency = tool_dependency_util.set_tool_dependency_attributes( app, @@ -66,7 +233,7 @@ def prepare_step( self, app, tool_dependency, action_elem, action_dict, install_environment, is_binary_download ): # <action type="assert_executable">$INSTALL_DIR/mira/my_file</action> if action_elem.text: - action_dict[ 'full_path' ] = td_common_util.evaluate_template( action_elem.text, install_environment ) + action_dict[ 'full_path' ] = basic_util.evaluate_template( action_elem.text, install_environment ) return action_dict @@ -75,6 +242,18 @@ def __init__( self ): self.type = 'assert_directory_exists' + def assert_directory_exists( self, full_path ): + """ + Return True if a symbolic link or directory exists, but if full_path is a file, + return False. """ + if full_path is None: + return False + if os.path.isfile( full_path ): + return False + if os.path.isdir( full_path ): + return True + return False + def execute_step( self, app, tool_dependency, package_name, actions, action_dict, filtered_actions, env_file_builder, install_environment, work_dir, current_dir=None, initial_download=False ): """ @@ -86,7 +265,7 @@ full_path = action_dict[ 'full_path' ] else: full_path = os.path.join( current_dir, action_dict[ 'full_path' ] ) - if not td_common_util.assert_directory_exists( full_path=full_path ): + if not self.assert_directory_exists( full_path=full_path ): status = app.install_model.ToolDependency.installation_status.ERROR error_message = 'The path %s is not a directory or does not exist.' % str( full_path ) tool_dependency = tool_dependency_util.set_tool_dependency_attributes( app, @@ -99,7 +278,7 @@ def prepare_step( self, app, tool_dependency, action_elem, action_dict, install_environment, is_binary_download ): # <action type="make_directory">$INSTALL_DIR/mira</action> if action_elem.text: - action_dict[ 'full_path' ] = td_common_util.evaluate_template( action_elem.text, install_environment ) + action_dict[ 'full_path' ] = basic_util.evaluate_template( action_elem.text, install_environment ) return action_dict @@ -108,6 +287,22 @@ def __init__( self ): self.type = 'assert_file_executable' + def assert_file_executable( self, full_path ): + """ + Return True if a symbolic link or file exists and is executable, but if full_path + is a directory, return False. + """ + if full_path is None: + return False + if os.path.isdir( full_path ): + return False + if os.path.exists( full_path ): + # Make sure the owner has execute permission on the file. + # See http://docs.python.org/2/library/stat.html + if stat.S_IXUSR & os.stat( full_path )[ stat.ST_MODE ] == 64: + return True + return False + def execute_step( self, app, tool_dependency, package_name, actions, action_dict, filtered_actions, env_file_builder, install_environment, work_dir, current_dir=None, initial_download=False ): """ @@ -119,7 +314,7 @@ full_path = action_dict[ 'full_path' ] else: full_path = os.path.join( current_dir, action_dict[ 'full_path' ] ) - if not td_common_util.assert_file_executable( full_path=full_path ): + if not self.assert_file_executable( full_path=full_path ): status = app.install_model.ToolDependency.installation_status.ERROR error_message = 'The path %s is not a file or is not executable by the owner.' % str( full_path ) tool_dependency = tool_dependency_util.set_tool_dependency_attributes( app, @@ -132,7 +327,7 @@ def prepare_step( self, app, tool_dependency, action_elem, action_dict, install_environment, is_binary_download ): # <action type="assert_executable">$INSTALL_DIR/mira/my_file</action> if action_elem.text: - action_dict[ 'full_path' ] = td_common_util.evaluate_template( action_elem.text, install_environment ) + action_dict[ 'full_path' ] = basic_util.evaluate_template( action_elem.text, install_environment ) return action_dict @@ -141,6 +336,19 @@ def __init__( self ): self.type = 'assert_file_exists' + def assert_file_exists( self, full_path ): + """ + Return True if a symbolic link or file exists, but if full_path is a directory, + return False. + """ + if full_path is None: + return False + if os.path.isdir( full_path ): + return False + if os.path.exists( full_path ): + return True + return False + def execute_step( self, app, tool_dependency, package_name, actions, action_dict, filtered_actions, env_file_builder, install_environment, work_dir, current_dir=None, initial_download=False ): """ @@ -152,7 +360,7 @@ full_path = action_dict[ 'full_path' ] else: full_path = os.path.join( current_dir, action_dict[ 'full_path' ] ) - if not td_common_util.assert_file_exists( full_path=full_path ): + if not self.assert_file_exists( full_path=full_path ): status = app.install_model.ToolDependency.installation_status.ERROR error_message = 'The path %s is not a file or does not exist.' % str( full_path ) tool_dependency = tool_dependency_util.set_tool_dependency_attributes( app, @@ -165,7 +373,7 @@ def prepare_step( self, app, tool_dependency, action_elem, action_dict, install_environment, is_binary_download ): # <action type="assert_on_path">$INSTALL_DIR/mira/my_file</action> if action_elem.text: - action_dict[ 'full_path' ] = td_common_util.evaluate_template( action_elem.text, install_environment ) + action_dict[ 'full_path' ] = basic_util.evaluate_template( action_elem.text, install_environment ) return action_dict @@ -187,7 +395,7 @@ pre_cmd = './configure %s && make && make install' % configure_opts else: pre_cmd = './configure --prefix=$INSTALL_DIR %s && make && make install' % configure_opts - cmd = install_environment.build_command( td_common_util.evaluate_template( pre_cmd, install_environment ) ) + cmd = install_environment.build_command( basic_util.evaluate_template( pre_cmd, install_environment ) ) return_code = install_environment.handle_command( app=app, tool_dependency=tool_dependency, cmd=cmd, @@ -199,7 +407,7 @@ def prepare_step( self, app, tool_dependency, action_elem, action_dict, install_environment, is_binary_download ): # Handle configure, make and make install allow providing configuration options if action_elem.text: - configure_opts = td_common_util.evaluate_template( action_elem.text, install_environment ) + configure_opts = basic_util.evaluate_template( action_elem.text, install_environment ) action_dict[ 'configure_opts' ] = configure_opts return action_dict @@ -274,7 +482,7 @@ received_mode = int( file_elem.get( 'mode', 600 ), base=8 ) # For added security, ensure that the setuid and setgid bits are not set. mode = received_mode & ~( stat.S_ISUID | stat.S_ISGID ) - file = td_common_util.evaluate_template( file_elem.text, install_environment ) + file = basic_util.evaluate_template( file_elem.text, install_environment ) chmod_tuple = ( file, mode ) chmod_actions.append( chmod_tuple ) if chmod_actions: @@ -282,11 +490,17 @@ return action_dict -class DownloadBinary( RecipeStep ): +class DownloadBinary( Download, RecipeStep ): def __init__( self ): self.type = 'download_binary' + def download_binary( self, url, work_dir ): + """Download a pre-compiled binary from the specified URL.""" + downloaded_filename = os.path.split( url )[ -1 ] + dir = self.url_download( work_dir, downloaded_filename, url, extract=False ) + return downloaded_filename + def filter_actions_after_binary_installation( self, actions ): '''Filter out actions that should not be processed if a binary download succeeded.''' filtered_actions = [] @@ -311,7 +525,7 @@ log.debug( 'Attempting to download from %s to %s', url, str( target_directory ) ) downloaded_filename = None try: - downloaded_filename = td_common_util.download_binary( url, work_dir ) + downloaded_filename = self.download_binary( url, work_dir ) if initial_download: # Filter out any actions that are not download_binary, chmod, or set_environment. filtered_actions = self.filter_actions_after_binary_installation( actions[ 1: ] ) @@ -338,9 +552,9 @@ full_path_to_dir = os.path.abspath( install_environment.install_dir ) else: full_path_to_dir = os.path.abspath( install_environment.install_dir ) - td_common_util.move_file( current_dir=work_dir, - source=downloaded_filename, - destination=full_path_to_dir ) + basic_util.move_file( current_dir=work_dir, + source=downloaded_filename, + destination=full_path_to_dir ) # Not sure why dir is ignored in this method, need to investigate... dir = None if initial_download: @@ -368,7 +582,7 @@ return action_dict -class DownloadByUrl( RecipeStep ): +class DownloadByUrl( Download, RecipeStep ): def __init__( self ): self.type = 'download_by_url' @@ -394,9 +608,9 @@ downloaded_filename = action_dict[ 'target_filename' ] else: downloaded_filename = os.path.split( url )[ -1 ] - dir = td_common_util.url_download( work_dir, downloaded_filename, url, extract=True ) + dir = self.url_download( work_dir, downloaded_filename, url, extract=True ) if is_binary: - log_file = os.path.join( install_environment.install_dir, td_common_util.INSTALLATION_LOG ) + log_file = os.path.join( install_environment.install_dir, basic_util.INSTALLATION_LOG ) if os.path.exists( log_file ): logfile = open( log_file, 'ab' ) else: @@ -422,7 +636,7 @@ return action_dict -class DownloadFile( RecipeStep ): +class DownloadFile( Download, RecipeStep ): def __init__( self ): self.type = 'download_file' @@ -447,7 +661,7 @@ filename = action_dict[ 'target_filename' ] else: filename = url.split( '/' )[ -1 ] - td_common_util.url_download( work_dir, filename, url ) + self.url_download( work_dir, filename, url ) if initial_download: dir = os.path.curdir return tool_dependency, filtered_actions, dir @@ -479,13 +693,17 @@ full_path = action_dict[ 'full_path' ] else: full_path = os.path.join( current_dir, action_dict[ 'full_path' ] ) - td_common_util.make_directory( full_path=full_path ) + self.make_directory( full_path=full_path ) return tool_dependency, None, None + def make_directory( self, full_path ): + if not os.path.exists( full_path ): + os.makedirs( full_path ) + def prepare_step( self, app, tool_dependency, action_elem, action_dict, install_environment, is_binary_download ): # <action type="make_directory">$INSTALL_DIR/lib/python</action> if action_elem.text: - action_dict[ 'full_path' ] = td_common_util.evaluate_template( action_elem.text, install_environment ) + action_dict[ 'full_path' ] = basic_util.evaluate_template( action_elem.text, install_environment ) return action_dict @@ -515,7 +733,7 @@ def prepare_step( self, app, tool_dependency, action_elem, action_dict, install_environment, is_binary_download ): # make; make install; allow providing make options if action_elem.text: - make_opts = td_common_util.evaluate_template( action_elem.text, install_environment ) + make_opts = basic_util.evaluate_template( action_elem.text, install_environment ) action_dict[ 'make_opts' ] = make_opts return action_dict @@ -531,18 +749,38 @@ Move a directory of files. Since this class is not used in the initial download stage, no recipe step filtering is performed here, and None values are always returned for filtered_actions and dir. """ - td_common_util.move_directory_files( current_dir=current_dir, - source_dir=os.path.join( action_dict[ 'source_directory' ] ), - destination_dir=os.path.join( action_dict[ 'destination_directory' ] ) ) + self.move_directory_files( current_dir=current_dir, + source_dir=os.path.join( action_dict[ 'source_directory' ] ), + destination_dir=os.path.join( action_dict[ 'destination_directory' ] ) ) return tool_dependency, None, None + def move_directory_files( self, current_dir, source_dir, destination_dir ): + source_directory = os.path.abspath( os.path.join( current_dir, source_dir ) ) + destination_directory = os.path.join( destination_dir ) + if not os.path.isdir( destination_directory ): + os.makedirs( destination_directory ) + symlinks = [] + regular_files = [] + for file_name in os.listdir( source_directory ): + source_file = os.path.join( source_directory, file_name ) + destination_file = os.path.join( destination_directory, file_name ) + files_tuple = ( source_file, destination_file ) + if os.path.islink( source_file ): + symlinks.append( files_tuple ) + else: + regular_files.append( files_tuple ) + for source_file, destination_file in symlinks: + shutil.move( source_file, destination_file ) + for source_file, destination_file in regular_files: + shutil.move( source_file, destination_file ) + def prepare_step( self, app, tool_dependency, action_elem, action_dict, install_environment, is_binary_download ): # <action type="move_directory_files"> # <source_directory>bin</source_directory> # <destination_directory>$INSTALL_DIR/bin</destination_directory> # </action> for move_elem in action_elem: - move_elem_text = td_common_util.evaluate_template( move_elem.text, install_environment ) + move_elem_text = basic_util.evaluate_template( move_elem.text, install_environment ) if move_elem_text: action_dict[ move_elem.tag ] = move_elem_text return action_dict @@ -559,10 +797,10 @@ Move a file on disk. Since this class is not used in the initial download stage, no recipe step filtering is performed here, and None values are always returned for filtered_actions and dir. """ - td_common_util.move_file( current_dir=current_dir, - source=os.path.join( action_dict[ 'source' ] ), - destination=os.path.join( action_dict[ 'destination' ] ), - rename_to=action_dict[ 'rename_to' ] ) + basic_util.move_file( current_dir=current_dir, + source=os.path.join( action_dict[ 'source' ] ), + destination=os.path.join( action_dict[ 'destination' ] ), + rename_to=action_dict[ 'rename_to' ] ) return tool_dependency, None, None def prepare_step( self, app, tool_dependency, action_elem, action_dict, install_environment, is_binary_download ): @@ -570,8 +808,8 @@ # <source>misc/some_file</source> # <destination>$INSTALL_DIR/bin</destination> # </action> - action_dict[ 'source' ] = td_common_util.evaluate_template( action_elem.find( 'source' ).text, install_environment ) - action_dict[ 'destination' ] = td_common_util.evaluate_template( action_elem.find( 'destination' ).text, install_environment ) + action_dict[ 'source' ] = basic_util.evaluate_template( action_elem.find( 'source' ).text, install_environment ) + action_dict[ 'destination' ] = basic_util.evaluate_template( action_elem.find( 'destination' ).text, install_environment ) action_dict[ 'rename_to' ] = action_elem.get( 'rename_to' ) return action_dict @@ -717,12 +955,12 @@ # <action type="set_environment"> # <environment_variable name="PATH" action="prepend_to">$INSTALL_DIR</environment_variable> # </action> + env_manager = EnvManager( app ) env_var_dicts = [] for env_elem in action_elem: if env_elem.tag == 'environment_variable': - env_var_dict = \ - td_common_util.create_env_var_dict( elem=env_elem, - install_environment=install_environment ) + env_var_dict = env_manager.create_env_var_dict( elem=env_elem, + install_environment=install_environment ) if env_var_dict: env_var_dicts.append( env_var_dict ) if env_var_dicts: @@ -764,16 +1002,17 @@ # the current tool dependency package. See the package_matplotlib_1_2 repository in the test tool # shed for a real-world example. all_env_shell_file_paths = [] + env_manager = EnvManager( app ) for env_elem in action_elem: if env_elem.tag == 'repository': - env_shell_file_paths = td_common_util.get_env_shell_file_paths( app, env_elem ) + env_shell_file_paths = env_manager.get_env_shell_file_paths( env_elem ) if env_shell_file_paths: all_env_shell_file_paths.extend( env_shell_file_paths ) action_dict[ 'env_shell_file_paths' ] = all_env_shell_file_paths return action_dict -class SetupPerlEnvironment( RecipeStep ): +class SetupPerlEnvironment( Download, RecipeStep ): def __init__( self ): self.type = 'setup_purl_environment' @@ -822,7 +1061,7 @@ # We assume a URL to a gem file. url = perl_package perl_package_name = url.split( '/' )[ -1 ] - dir = td_common_util.url_download( work_dir, perl_package_name, url, extract=True ) + dir = self.url_download( work_dir, perl_package_name, url, extract=True ) # Search for Build.PL or Makefile.PL (ExtUtils::MakeMaker vs. Module::Build). tmp_work_dir = os.path.join( work_dir, dir ) if os.path.exists( os.path.join( tmp_work_dir, 'Makefile.PL' ) ): @@ -836,7 +1075,7 @@ return tool_dependency, filtered_actions, dir return tool_dependency, None, None with lcd( tmp_work_dir ): - cmd = install_environment.build_command( td_common_util.evaluate_template( cmd, install_environment ) ) + cmd = install_environment.build_command( basic_util.evaluate_template( cmd, install_environment ) ) return_code = install_environment.handle_command( app=app, tool_dependency=tool_dependency, cmd=cmd, @@ -849,7 +1088,7 @@ # perl package from CPAN without version number. # cpanm should be installed with the parent perl distribution, otherwise this will not work. cmd += '''cpanm --local-lib=$INSTALL_DIR %s''' % ( perl_package ) - cmd = install_environment.build_command( td_common_util.evaluate_template( cmd, install_environment ) ) + cmd = install_environment.build_command( basic_util.evaluate_template( cmd, install_environment ) ) return_code = install_environment.handle_command( app=app, tool_dependency=tool_dependency, cmd=cmd, @@ -890,10 +1129,11 @@ # with each repository. This will potentially update the value of the 'env_shell_file_paths' entry # in action_dict. all_env_shell_file_paths = [] - action_dict = td_common_util.get_env_shell_file_paths_from_setup_environment_elem( app, - all_env_shell_file_paths, - action_elem, - action_dict ) + env_manager = EnvManager( app ) + action_dict = env_manager.get_env_shell_file_paths_from_setup_environment_elem( app, + all_env_shell_file_paths, + action_elem, + action_dict ) perl_packages = [] for env_elem in action_elem: if env_elem.tag == 'package': @@ -908,7 +1148,7 @@ return action_dict -class SetupREnvironment( RecipeStep ): +class SetupREnvironment( Download, RecipeStep ): def __init__( self ): self.type = 'setup_r_environment' @@ -947,7 +1187,7 @@ for url in action_dict[ 'r_packages' ]: filename = url.split( '/' )[ -1 ] tarball_names.append( filename ) - td_common_util.url_download( work_dir, filename, url, extract=False ) + self.url_download( work_dir, filename, url, extract=False ) dir = os.path.curdir current_dir = os.path.abspath( os.path.join( work_dir, dir ) ) with lcd( current_dir ): @@ -958,7 +1198,7 @@ cmd = r'''PATH=$PATH:$R_HOME/bin; export PATH; R_LIBS=$INSTALL_DIR; export R_LIBS; Rscript -e "install.packages(c('%s'),lib='$INSTALL_DIR', repos=NULL, dependencies=FALSE)"''' % \ ( str( tarball_name ) ) - cmd = install_environment.build_command( td_common_util.evaluate_template( cmd, install_environment ) ) + cmd = install_environment.build_command( basic_util.evaluate_template( cmd, install_environment ) ) return_code = install_environment.handle_command( app=app, tool_dependency=tool_dependency, cmd=cmd, @@ -993,10 +1233,11 @@ # associated with each repository. This will potentially update the value of the # 'env_shell_file_paths' entry in action_dict. all_env_shell_file_paths = [] - action_dict = td_common_util.get_env_shell_file_paths_from_setup_environment_elem( app, - all_env_shell_file_paths, - action_elem, - action_dict ) + env_manager = EnvManager( app ) + action_dict = env_manager.get_env_shell_file_paths_from_setup_environment_elem( app, + all_env_shell_file_paths, + action_elem, + action_dict ) r_packages = list() for env_elem in action_elem: if env_elem.tag == 'package': @@ -1006,7 +1247,7 @@ return action_dict -class SetupRubyEnvironment( RecipeStep ): +class SetupRubyEnvironment( Download, RecipeStep ): def __init__( self ): self.type = 'setup_ruby_environment' @@ -1058,7 +1299,7 @@ # We assume a URL to a gem file. url = gem gem_name = url.split( '/' )[ -1 ] - td_common_util.url_download( work_dir, gem_name, url, extract=False ) + self.url_download( work_dir, gem_name, url, extract=False ) cmd = '''PATH=$PATH:$RUBY_HOME/bin; export PATH; GEM_HOME=$INSTALL_DIR; export GEM_HOME; gem install --local %s ''' % ( gem_name ) else: @@ -1073,7 +1314,7 @@ # no version number given cmd = '''PATH=$PATH:$RUBY_HOME/bin; export PATH; GEM_HOME=$INSTALL_DIR; export GEM_HOME; gem install %s''' % ( gem ) - cmd = install_environment.build_command( td_common_util.evaluate_template( cmd, install_environment ) ) + cmd = install_environment.build_command( basic_util.evaluate_template( cmd, install_environment ) ) return_code = install_environment.handle_command( app=app, tool_dependency=tool_dependency, cmd=cmd, @@ -1114,10 +1355,11 @@ # associated with each repository. This will potentially update the value of the # 'env_shell_file_paths' entry in action_dict. all_env_shell_file_paths = [] - action_dict = td_common_util.get_env_shell_file_paths_from_setup_environment_elem( app, - all_env_shell_file_paths, - action_elem, - action_dict ) + env_manager = EnvManager( app ) + action_dict = env_manager.get_env_shell_file_paths_from_setup_environment_elem( app, + all_env_shell_file_paths, + action_elem, + action_dict ) ruby_package_tups = [] for env_elem in action_elem: if env_elem.tag == 'package': @@ -1140,7 +1382,7 @@ return action_dict -class SetupVirtualEnv( RecipeStep ): +class SetupVirtualEnv( Download, RecipeStep ): def __init__( self ): self.type = 'setup_virtualenv' @@ -1228,9 +1470,10 @@ with install_environment.make_tmp_dir() as work_dir: downloaded_filename = VIRTUALENV_URL.rsplit('/', 1)[-1] try: - dir = td_common_util.url_download( work_dir, downloaded_filename, VIRTUALENV_URL ) + dir = self.url_download( work_dir, downloaded_filename, VIRTUALENV_URL ) except: - log.error( "Failed to download virtualenv: td_common_util.url_download( '%s', '%s', '%s' ) threw an exception", work_dir, downloaded_filename, VIRTUALENV_URL ) + log.error( "Failed to download virtualenv: url_download( '%s', '%s', '%s' ) threw an exception", + work_dir, downloaded_filename, VIRTUALENV_URL ) return False full_path_to_dir = os.path.abspath( os.path.join( work_dir, dir ) ) shutil.move( full_path_to_dir, venv_dir ) @@ -1245,7 +1488,7 @@ # lxml==2.3.0</action> ## Manually specify contents of requirements.txt file to create dynamically. action_dict[ 'use_requirements_file' ] = asbool( action_elem.get( 'use_requirements_file', True ) ) - action_dict[ 'requirements' ] = td_common_util.evaluate_template( action_elem.text or 'requirements.txt', install_environment ) + action_dict[ 'requirements' ] = basic_util.evaluate_template( action_elem.text or 'requirements.txt', install_environment ) action_dict[ 'python' ] = action_elem.get( 'python', 'python' ) return action_dict @@ -1316,7 +1559,7 @@ def prepare_step( self, app, tool_dependency, action_elem, action_dict, install_environment, is_binary_download ): # <action type="shell_command">make</action> - action_elem_text = td_common_util.evaluate_template( action_elem.text, install_environment ) + action_elem_text = basic_util.evaluate_template( action_elem.text, install_environment ) if action_elem_text: action_dict[ 'command' ] = action_elem_text return action_dict @@ -1338,7 +1581,7 @@ env_vars = dict() env_vars = install_environment.environment_dict() tool_shed_repository = tool_dependency.tool_shed_repository - env_vars.update( td_common_util.get_env_var_values( install_environment ) ) + env_vars.update( basic_util.get_env_var_values( install_environment ) ) language = action_dict[ 'language' ] with settings( warn_only=True, **env_vars ): if language == 'cheetah': diff -r f8c6577665c383645e934629784ece516b498d9b -r 85aa79e9ab9a87a5b5fea0ed1039aeb9a5c23642 lib/tool_shed/galaxy_install/tool_dependencies/recipe/tag_handler.py --- a/lib/tool_shed/galaxy_install/tool_dependencies/recipe/tag_handler.py +++ b/lib/tool_shed/galaxy_install/tool_dependencies/recipe/tag_handler.py @@ -10,7 +10,7 @@ from tool_shed.util import tool_dependency_util from tool_shed.util import xml_util -from tool_shed.galaxy_install.tool_dependencies import td_common_util +from tool_shed.galaxy_install.tool_dependencies.env_manager import EnvManager from tool_shed.galaxy_install.tool_dependencies.recipe.env_file_builder import EnvFileBuilder from tool_shed.galaxy_install.tool_dependencies.recipe.install_environment import InstallEnvironment @@ -85,9 +85,9 @@ platform_info_dict = tool_dependency_util.get_platform_info_dict() if package_install_version == '1.0': # Handle tool dependency installation using a fabric method included in the Galaxy framework. - actions_elem_tuples = td_common_util.parse_package_elem( package_elem, - platform_info_dict=platform_info_dict, - include_after_install_actions=True ) + actions_elem_tuples = tool_dependency_util.parse_package_elem( package_elem, + platform_info_dict=platform_info_dict, + include_after_install_actions=True ) if not actions_elem_tuples: proceed_with_install = False error_message = 'Version %s of the %s package cannot be installed because ' % ( str( package_version ), str( package_name ) ) @@ -491,6 +491,7 @@ # <set_environment version="1.0"> # <repository toolshed="<tool shed>" name="<repository name>" owner="<repository owner>" changeset_revision="<changeset revision>" /> # </set_environment> + env_manager = EnvManager( app ) tool_dependencies = [] env_var_version = elem.get( 'version', '1.0' ) tool_shed_repository_install_dir = os.path.abspath( tool_shed_repository.repo_files_directory( app ) ) @@ -514,8 +515,8 @@ tool_dependency_version=None ) install_environment = InstallEnvironment( tool_shed_repository_install_dir=tool_shed_repository_install_dir, install_dir=install_dir ) - env_var_dict = td_common_util.create_env_var_dict( elem=env_var_elem, - install_environment=install_environment ) + env_var_dict = env_manager.create_env_var_dict( elem=env_var_elem, + install_environment=install_environment ) if env_var_dict: if not os.path.exists( install_dir ): os.makedirs( install_dir ) diff -r f8c6577665c383645e934629784ece516b498d9b -r 85aa79e9ab9a87a5b5fea0ed1039aeb9a5c23642 lib/tool_shed/galaxy_install/tool_dependencies/td_common_util.py --- a/lib/tool_shed/galaxy_install/tool_dependencies/td_common_util.py +++ /dev/null @@ -1,578 +0,0 @@ -import logging -import os -import re -import shutil -import stat -import sys -import tarfile -import time -import traceback -import urllib2 -import zipfile -from string import Template -from tool_shed.util import common_util -import tool_shed.util.shed_util_common as suc -from galaxy.datatypes import checkers - -log = logging.getLogger( __name__ ) - -# Set no activity timeout to 20 minutes. -NO_OUTPUT_TIMEOUT = 1200.0 -INSTALLATION_LOG = 'INSTALLATION.log' - - -class CompressedFile( object ): - - def __init__( self, file_path, mode='r' ): - if istar( file_path ): - self.file_type = 'tar' - elif iszip( file_path ) and not isjar( file_path ): - self.file_type = 'zip' - self.file_name = os.path.splitext( os.path.basename( file_path ) )[ 0 ] - if self.file_name.endswith( '.tar' ): - self.file_name = os.path.splitext( self.file_name )[ 0 ] - self.type = self.file_type - method = 'open_%s' % self.file_type - if hasattr( self, method ): - self.archive = getattr( self, method )( file_path, mode ) - else: - raise NameError( 'File type %s specified, no open method found.' % self.file_type ) - - def extract( self, path ): - '''Determine the path to which the archive should be extracted.''' - contents = self.getmembers() - extraction_path = path - if len( contents ) == 1: - # The archive contains a single file, return the extraction path. - if self.isfile( contents[ 0 ] ): - extraction_path = os.path.join( path, self.file_name ) - if not os.path.exists( extraction_path ): - os.makedirs( extraction_path ) - self.archive.extractall( extraction_path ) - else: - # Get the common prefix for all the files in the archive. If the common prefix ends with a slash, - # or self.isdir() returns True, the archive contains a single directory with the desired contents. - # Otherwise, it contains multiple files and/or directories at the root of the archive. - common_prefix = os.path.commonprefix( [ self.getname( item ) for item in contents ] ) - if len( common_prefix ) >= 1 and not common_prefix.endswith( os.sep ) and self.isdir( self.getmember( common_prefix ) ): - common_prefix += os.sep - if common_prefix.endswith( os.sep ): - self.archive.extractall( os.path.join( path ) ) - extraction_path = os.path.join( path, common_prefix ) - else: - extraction_path = os.path.join( path, self.file_name ) - if not os.path.exists( extraction_path ): - os.makedirs( extraction_path ) - self.archive.extractall( os.path.join( extraction_path ) ) - return os.path.abspath( extraction_path ) - - def getmembers_tar( self ): - return self.archive.getmembers() - - def getmembers_zip( self ): - return self.archive.infolist() - - def getname_tar( self, item ): - return item.name - - def getname_zip( self, item ): - return item.filename - - def getmember( self, name ): - for member in self.getmembers(): - if self.getname( member ) == name: - return member - - def getmembers( self ): - return getattr( self, 'getmembers_%s' % self.type )() - - def getname( self, member ): - return getattr( self, 'getname_%s' % self.type )( member ) - - def isdir( self, member ): - return getattr( self, 'isdir_%s' % self.type )( member ) - - def isdir_tar( self, member ): - return member.isdir() - - def isdir_zip( self, member ): - if member.filename.endswith( os.sep ): - return True - return False - - def isfile( self, member ): - if not self.isdir( member ): - return True - return False - - def open_tar( self, filepath, mode ): - return tarfile.open( filepath, mode, errorlevel=0 ) - - def open_zip( self, filepath, mode ): - return zipfile.ZipFile( filepath, mode ) - -def assert_directory_executable( full_path ): - """ - Return True if a symbolic link or directory exists and is executable, but if - full_path is a file, return False. - """ - if full_path is None: - return False - if os.path.isfile( full_path ): - return False - if os.path.isdir( full_path ): - # Make sure the owner has execute permission on the directory. - # See http://docs.python.org/2/library/stat.html - if stat.S_IXUSR & os.stat( full_path )[ stat.ST_MODE ] == 64: - return True - return False - -def assert_directory_exists( full_path ): - """ - Return True if a symbolic link or directory exists, but if full_path is a file, - return False. """ - if full_path is None: - return False - if os.path.isfile( full_path ): - return False - if os.path.isdir( full_path ): - return True - return False - -def assert_file_executable( full_path ): - """ - Return True if a symbolic link or file exists and is executable, but if full_path - is a directory, return False. - """ - if full_path is None: - return False - if os.path.isdir( full_path ): - return False - if os.path.exists( full_path ): - # Make sure the owner has execute permission on the file. - # See http://docs.python.org/2/library/stat.html - if stat.S_IXUSR & os.stat( full_path )[ stat.ST_MODE ] == 64: - return True - return False - -def assert_file_exists( full_path ): - """ - Return True if a symbolic link or file exists, but if full_path is a directory, - return False. - """ - if full_path is None: - return False - if os.path.isdir( full_path ): - return False - if os.path.exists( full_path ): - return True - return False - -def create_env_var_dict( elem, install_environment ): - env_var_name = elem.get( 'name', 'PATH' ) - env_var_action = elem.get( 'action', 'prepend_to' ) - env_var_text = None - tool_dependency_install_dir = install_environment.install_dir - tool_shed_repository_install_dir = install_environment.tool_shed_repository_install_dir - if elem.text and elem.text.find( 'REPOSITORY_INSTALL_DIR' ) >= 0: - if tool_shed_repository_install_dir and elem.text.find( '$REPOSITORY_INSTALL_DIR' ) != -1: - env_var_text = elem.text.replace( '$REPOSITORY_INSTALL_DIR', tool_shed_repository_install_dir ) - return dict( name=env_var_name, action=env_var_action, value=env_var_text ) - else: - env_var_text = elem.text.replace( '$REPOSITORY_INSTALL_DIR', tool_dependency_install_dir ) - return dict( name=env_var_name, action=env_var_action, value=env_var_text ) - if elem.text and elem.text.find( 'INSTALL_DIR' ) >= 0: - if tool_dependency_install_dir: - env_var_text = elem.text.replace( '$INSTALL_DIR', tool_dependency_install_dir ) - return dict( name=env_var_name, action=env_var_action, value=env_var_text ) - else: - env_var_text = elem.text.replace( '$INSTALL_DIR', tool_shed_repository_install_dir ) - return dict( name=env_var_name, action=env_var_action, value=env_var_text ) - if elem.text: - # Allow for environment variables that contain neither REPOSITORY_INSTALL_DIR nor INSTALL_DIR - # since there may be command line parameters that are tuned for a Galaxy instance. Allowing them - # to be set in one location rather than being hard coded into each tool config is the best approach. - # For example: - # <environment_variable name="GATK2_SITE_OPTIONS" action="set_to"> - # "--num_threads 4 --num_cpu_threads_per_data_thread 3 --phone_home STANDARD" - # </environment_variable> - return dict( name=env_var_name, action=env_var_action, value=elem.text) - return None - -def download_binary( url, work_dir ): - """Download a pre-compiled binary from the specified URL.""" - downloaded_filename = os.path.split( url )[ -1 ] - dir = url_download( work_dir, downloaded_filename, url, extract=False ) - return downloaded_filename - -def egrep_escape( text ): - """Escape ``text`` to allow literal matching using egrep.""" - regex = re.escape( text ) - # Seems like double escaping is needed for \ - regex = regex.replace( '\\\\', '\\\\\\' ) - # Triple-escaping seems to be required for $ signs - regex = regex.replace( r'\$', r'\\\$' ) - # Whereas single quotes should not be escaped - regex = regex.replace( r"\'", "'" ) - return regex - -def evaluate_template( text, install_environment ): - """ - Substitute variables defined in XML blocks from dependencies file. The value of the received - repository_install_dir is the root installation directory of the repository that contains the - tool dependency. The value of the received install_dir is the root installation directory of - the tool_dependency. - """ - return Template( text ).safe_substitute( get_env_var_values( install_environment ) ) - -def format_traceback(): - ex_type, ex, tb = sys.exc_info() - return ''.join( traceback.format_tb( tb ) ) - -def get_env_shell_file_path( installation_directory ): - env_shell_file_name = 'env.sh' - default_location = os.path.abspath( os.path.join( installation_directory, env_shell_file_name ) ) - if os.path.exists( default_location ): - return default_location - for root, dirs, files in os.walk( installation_directory ): - for name in files: - if name == env_shell_file_name: - return os.path.abspath( os.path.join( root, name ) ) - return None - -def get_env_shell_file_paths( app, elem ): - # Currently only the following tag set is supported. - # <repository toolshed="http://localhost:9009/" name="package_numpy_1_7" owner="test" changeset_revision="c84c6a8be056"> - # <package name="numpy" version="1.7.1" /> - # </repository> - env_shell_file_paths = [] - toolshed = elem.get( 'toolshed', None ) - repository_name = elem.get( 'name', None ) - repository_owner = elem.get( 'owner', None ) - changeset_revision = elem.get( 'changeset_revision', None ) - if toolshed and repository_name and repository_owner and changeset_revision: - # The protocol is not stored, but the port is if it exists. - toolshed = common_util.remove_protocol_from_tool_shed_url( toolshed ) - repository = suc.get_repository_for_dependency_relationship( app, toolshed, repository_name, repository_owner, changeset_revision ) - if repository: - for sub_elem in elem: - tool_dependency_type = sub_elem.tag - tool_dependency_name = sub_elem.get( 'name' ) - tool_dependency_version = sub_elem.get( 'version' ) - if tool_dependency_type and tool_dependency_name and tool_dependency_version: - # Get the tool_dependency so we can get its installation directory. - tool_dependency = None - for tool_dependency in repository.tool_dependencies: - if tool_dependency.type == tool_dependency_type and \ - tool_dependency.name == tool_dependency_name and \ - tool_dependency.version == tool_dependency_version: - break - if tool_dependency: - tool_dependency_key = '%s/%s' % ( tool_dependency_name, tool_dependency_version ) - installation_directory = tool_dependency.installation_directory( app ) - env_shell_file_path = get_env_shell_file_path( installation_directory ) - if env_shell_file_path: - env_shell_file_paths.append( env_shell_file_path ) - else: - error_message = "Skipping tool dependency definition because unable to locate env.sh file for tool dependency " - error_message += "type %s, name %s, version %s for repository %s" % \ - ( str( tool_dependency_type ), str( tool_dependency_name ), str( tool_dependency_version ), str( repository.name ) ) - log.debug( error_message ) - continue - else: - error_message = "Skipping tool dependency definition because unable to locate tool dependency " - error_message += "type %s, name %s, version %s for repository %s" % \ - ( str( tool_dependency_type ), str( tool_dependency_name ), str( tool_dependency_version ), str( repository.name ) ) - log.debug( error_message ) - continue - else: - error_message = "Skipping invalid tool dependency definition: type %s, name %s, version %s." % \ - ( str( tool_dependency_type ), str( tool_dependency_name ), str( tool_dependency_version ) ) - log.debug( error_message ) - continue - else: - error_message = "Skipping set_environment_for_install definition because unable to locate required installed tool shed repository: " - error_message += "toolshed %s, name %s, owner %s, changeset_revision %s." % \ - ( str( toolshed ), str( repository_name ), str( repository_owner ), str( changeset_revision ) ) - log.debug( error_message ) - else: - error_message = "Skipping invalid set_environment_for_install definition: toolshed %s, name %s, owner %s, changeset_revision %s." % \ - ( str( toolshed ), str( repository_name ), str( repository_owner ), str( changeset_revision ) ) - log.debug( error_message ) - return env_shell_file_paths - -def get_env_shell_file_paths_from_setup_environment_elem( app, all_env_shell_file_paths, elem, action_dict ): - """ - Parse an XML tag set to discover all child repository dependency tags and define the path to an env.sh file associated - with the repository (this requires the repository dependency to be in an installed state). The received action_dict - will be updated with these discovered paths and returned to the caller. This method handles tool dependency definition - tag sets <setup_r_environment>, <setup_ruby_environment> and <setup_perl_environment>. - """ - # An example elem is: - # <action type="setup_perl_environment"> - # <repository name="package_perl_5_18" owner="iuc"> - # <package name="perl" version="5.18.1" /> - # </repository> - # <repository name="package_expat_2_1" owner="iuc" prior_installation_required="True"> - # <package name="expat" version="2.1.0" /> - # </repository> - # <package>http://search.cpan.org/CPAN/authors/id/T/TO/TODDR/XML-Parser-2.41.tar.gz</package> - # <package>http://search.cpan.org/CPAN/authors/id/L/LD/LDS/CGI.pm-3.43.tar.gz</package> - # </action> - for action_elem in elem: - if action_elem.tag == 'repository': - env_shell_file_paths = get_env_shell_file_paths( app, action_elem ) - all_env_shell_file_paths.extend( env_shell_file_paths ) - if all_env_shell_file_paths: - action_dict[ 'env_shell_file_paths' ] = all_env_shell_file_paths - action_dict[ 'action_shell_file_paths' ] = env_shell_file_paths - return action_dict - -def get_env_var_values( install_environment ): - """ - Return a dictionary of values, some of which enable substitution of reserved words for the values. - The received install_enviroment object has 2 important attributes for reserved word substitution: - install_environment.tool_shed_repository_install_dir is the root installation directory of the repository - that contains the tool dependency being installed, and install_environment.install_dir is the root - installation directory of the tool dependency. - """ - env_var_dict = {} - env_var_dict[ 'REPOSITORY_INSTALL_DIR' ] = install_environment.tool_shed_repository_install_dir - env_var_dict[ 'INSTALL_DIR' ] = install_environment.install_dir - env_var_dict[ 'system_install' ] = install_environment.install_dir - # If the Python interpreter is 64bit then we can safely assume that the underlying system is also 64bit. - env_var_dict[ '__is64bit__' ] = sys.maxsize > 2**32 - return env_var_dict - -def isbz2( file_path ): - return checkers.is_bz2( file_path ) - -def isgzip( file_path ): - return checkers.is_gzip( file_path ) - -def isjar( file_path ): - return iszip( file_path ) and file_path.endswith( '.jar' ) - -def istar( file_path ): - return tarfile.is_tarfile( file_path ) - -def iszip( file_path ): - return checkers.check_zip( file_path ) - -def is_compressed( file_path ): - if isjar( file_path ): - return False - else: - return iszip( file_path ) or isgzip( file_path ) or istar( file_path ) or isbz2( file_path ) - -def make_directory( full_path ): - if not os.path.exists( full_path ): - os.makedirs( full_path ) - -def move_directory_files( current_dir, source_dir, destination_dir ): - source_directory = os.path.abspath( os.path.join( current_dir, source_dir ) ) - destination_directory = os.path.join( destination_dir ) - if not os.path.isdir( destination_directory ): - os.makedirs( destination_directory ) - symlinks = [] - regular_files = [] - for file_name in os.listdir( source_directory ): - source_file = os.path.join( source_directory, file_name ) - destination_file = os.path.join( destination_directory, file_name ) - files_tuple = ( source_file, destination_file ) - if os.path.islink( source_file ): - symlinks.append( files_tuple ) - else: - regular_files.append( files_tuple ) - for source_file, destination_file in symlinks: - shutil.move( source_file, destination_file ) - for source_file, destination_file in regular_files: - shutil.move( source_file, destination_file ) - -def move_file( current_dir, source, destination, rename_to=None ): - source_path = os.path.abspath( os.path.join( current_dir, source ) ) - source_file = os.path.basename( source_path ) - if rename_to is not None: - destination_file = rename_to - destination_directory = os.path.join( destination ) - destination_path = os.path.join( destination_directory, destination_file ) - else: - destination_directory = os.path.join( destination ) - destination_path = os.path.join( destination_directory, source_file ) - if not os.path.exists( destination_directory ): - os.makedirs( destination_directory ) - shutil.move( source_path, destination_path ) - -def parse_package_elem( package_elem, platform_info_dict=None, include_after_install_actions=True ): - """ - Parse a <package> element within a tool dependency definition and return a list of action tuples. - This method is called when setting metadata on a repository that includes a tool_dependencies.xml - file or when installing a repository that includes a tool_dependencies.xml file. If installing, - platform_info_dict must be a valid dictionary and include_after_install_actions must be True. - """ - # The actions_elem_tuples list contains <actions> tag sets (possibly inside of an <actions_group> - # tag set) to be processed in the order they are defined in the tool_dependencies.xml file. - actions_elem_tuples = [] - # The tag sets that will go into the actions_elem_list are those that install a compiled binary if - # the architecture and operating system match its defined attributes. If compiled binary is not - # installed, the first <actions> tag set [following those that have the os and architecture attributes] - # that does not have os or architecture attributes will be processed. This tag set must contain the - # recipe for downloading and compiling source. - actions_elem_list = [] - for elem in package_elem: - if elem.tag == 'actions': - # We have an <actions> tag that should not be matched against a specific combination of - # architecture and operating system. - in_actions_group = False - actions_elem_tuples.append( ( in_actions_group, elem ) ) - elif elem.tag == 'actions_group': - # We have an actions_group element, and its child <actions> elements should therefore be compared - # with the current operating system - # and processor architecture. - in_actions_group = True - # Record the number of <actions> elements so we can filter out any <action> elements that precede - # <actions> elements. - actions_elem_count = len( elem.findall( 'actions' ) ) - # Record the number of <actions> elements that have both architecture and os specified, in order - # to filter out any platform-independent <actions> elements that come before platform-specific - # <actions> elements. - platform_actions_elements = [] - for actions_elem in elem.findall( 'actions' ): - if actions_elem.get( 'architecture' ) is not None and actions_elem.get( 'os' ) is not None: - platform_actions_elements.append( actions_elem ) - platform_actions_element_count = len( platform_actions_elements ) - platform_actions_elements_processed = 0 - actions_elems_processed = 0 - # The tag sets that will go into the after_install_actions list are <action> tags instead of <actions> - # tags. These will be processed only if they are at the very end of the <actions_group> tag set (after - # all <actions> tag sets). See below for details. - after_install_actions = [] - # Inspect the <actions_group> element and build the actions_elem_list and the after_install_actions list. - for child_element in elem: - if child_element.tag == 'actions': - actions_elems_processed += 1 - system = child_element.get( 'os' ) - architecture = child_element.get( 'architecture' ) - # Skip <actions> tags that have only one of architecture or os specified, in order for the - # count in platform_actions_elements_processed to remain accurate. - if ( system and not architecture ) or ( architecture and not system ): - log.debug( 'Error: Both architecture and os attributes must be specified in an <actions> tag.' ) - continue - # Since we are inside an <actions_group> tag set, compare it with our current platform information - # and filter the <actions> tag sets that don't match. Require both the os and architecture attributes - # to be defined in order to find a match. - if system and architecture: - platform_actions_elements_processed += 1 - # If either the os or architecture do not match the platform, this <actions> tag will not be - # considered a match. Skip it and proceed with checking the next one. - if platform_info_dict: - if platform_info_dict[ 'os' ] != system or platform_info_dict[ 'architecture' ] != architecture: - continue - else: - # We must not be installing a repository into Galaxy, so determining if we can install a - # binary is not necessary. - continue - else: - # <actions> tags without both os and architecture attributes are only allowed to be specified - # after platform-specific <actions> tags. If we find a platform-independent <actions> tag before - # all platform-specific <actions> tags have been processed. - if platform_actions_elements_processed < platform_actions_element_count: - debug_msg = 'Error: <actions> tags without os and architecture attributes are only allowed ' - debug_msg += 'after all <actions> tags with os and architecture attributes have been defined. ' - debug_msg += 'Skipping the <actions> tag set with no os or architecture attributes that has ' - debug_msg += 'been defined between two <actions> tag sets that have these attributes defined. ' - log.debug( debug_msg ) - continue - # If we reach this point, it means one of two things: 1) The system and architecture attributes are - # not defined in this <actions> tag, or 2) The system and architecture attributes are defined, and - # they are an exact match for the current platform. Append the child element to the list of elements - # to process. - actions_elem_list.append( child_element ) - elif child_element.tag == 'action': - # Any <action> tags within an <actions_group> tag set must come after all <actions> tags. - if actions_elems_processed == actions_elem_count: - # If all <actions> elements have been processed, then this <action> element can be appended to the - # list of actions to execute within this group. - after_install_actions.append( child_element ) - else: - # If any <actions> elements remain to be processed, then log a message stating that <action> - # elements are not allowed to precede any <actions> elements within an <actions_group> tag set. - debug_msg = 'Error: <action> tags are only allowed at the end of an <actions_group> tag set after ' - debug_msg += 'all <actions> tags. Skipping <%s> element with type %s.' % \ - ( child_element.tag, child_element.get( 'type', 'unknown' ) ) - log.debug( debug_msg ) - continue - if platform_info_dict is None and not include_after_install_actions: - # We must be setting metadata on a repository. - if len( actions_elem_list ) >= 1: - actions_elem_tuples.append( ( in_actions_group, actions_elem_list[ 0 ] ) ) - else: - # We are processing a recipe that contains only an <actions_group> tag set for installing a binary, - # but does not include an additional recipe for installing and compiling from source. - actions_elem_tuples.append( ( in_actions_group, [] ) ) - elif platform_info_dict is not None and include_after_install_actions: - # We must be installing a repository. - if after_install_actions: - actions_elem_list.extend( after_install_actions ) - actions_elem_tuples.append( ( in_actions_group, actions_elem_list ) ) - else: - # Skip any element that is not <actions> or <actions_group> - this will skip comments, <repository> tags - # and <readme> tags. - in_actions_group = False - continue - return actions_elem_tuples - -def __shellquote( s ): - """Quote and escape the supplied string for use in shell expressions.""" - return "'" + s.replace( "'", "'\\''" ) + "'" - -def url_download( install_dir, downloaded_file_name, download_url, extract=True ): - file_path = os.path.join( install_dir, downloaded_file_name ) - src = None - dst = None - # Set a timer so we don't sit here forever. - start_time = time.time() - try: - src = urllib2.urlopen( download_url ) - dst = open( file_path, 'wb' ) - while True: - chunk = src.read( suc.CHUNK_SIZE ) - if chunk: - dst.write( chunk ) - else: - break - time_taken = time.time() - start_time - if time_taken > NO_OUTPUT_TIMEOUT: - err_msg = 'Downloading from URL %s took longer than the defined timeout period of %.1f seconds.' % \ - ( str( download_url ), NO_OUTPUT_TIMEOUT ) - raise Exception( err_msg ) - except Exception, e: - err_msg = err_msg = 'Error downloading from URL\n%s:\n%s' % ( str( download_url ), str( e ) ) - raise Exception( err_msg ) - finally: - if src: - src.close() - if dst: - dst.close() - if extract: - if istar( file_path ) or ( iszip( file_path ) and not isjar( file_path ) ): - archive = CompressedFile( file_path ) - extraction_path = archive.extract( install_dir ) - else: - extraction_path = os.path.abspath( install_dir ) - else: - extraction_path = os.path.abspath( install_dir ) - return extraction_path - -def zipfile_ok( path_to_archive ): - """ - This function is a bit pedantic and not functionally necessary. It checks whether there is no file pointing outside of the extraction, - because ZipFile.extractall() has some potential security holes. See python zipfile documentation for more details. - """ - basename = os.path.realpath( os.path.dirname( path_to_archive ) ) - zip_archive = zipfile.ZipFile( path_to_archive ) - for member in zip_archive.namelist(): - member_path = os.path.realpath( os.path.join( basename, member ) ) - if not member_path.startswith( basename ): - return False - return True diff -r f8c6577665c383645e934629784ece516b498d9b -r 85aa79e9ab9a87a5b5fea0ed1039aeb9a5c23642 lib/tool_shed/scripts/check_filesystem_for_empty_tool_dependency_installation_paths.py --- a/lib/tool_shed/scripts/check_filesystem_for_empty_tool_dependency_installation_paths.py +++ b/lib/tool_shed/scripts/check_filesystem_for_empty_tool_dependency_installation_paths.py @@ -7,7 +7,7 @@ new_path.extend( sys.path[1:] ) sys.path = new_path -from tool_shed.galaxy_install.tool_dependencies import td_common_util +from tool_shed.util.basic_util import INSTALLATION_LOG def main( args ): empty_installation_paths = [] @@ -31,13 +31,13 @@ no_files = False if len( dirs ) == 0: no_dirs = True - if len( files ) == 0 or len( files ) == 1 and td_common_util.INSTALLATION_LOG in files: + if len( files ) == 0 or len( files ) == 1 and INSTALLATION_LOG in files: no_files = True if no_files and no_dirs and root not in empty_installation_paths: empty_installation_paths.append( root ) if len( empty_installation_paths ) > 0: print 'The following %d tool dependency installation directories were found to be empty or contain only the file %s.' % \ - ( len( empty_installation_paths ), td_common_util.INSTALLATION_LOG ) + ( len( empty_installation_paths ), INSTALLATION_LOG ) if args.delete: for path in empty_installation_paths: if os.path.exists( path ): diff -r f8c6577665c383645e934629784ece516b498d9b -r 85aa79e9ab9a87a5b5fea0ed1039aeb9a5c23642 lib/tool_shed/scripts/check_s3_for_empty_tool_dependency_installation_paths.py --- a/lib/tool_shed/scripts/check_s3_for_empty_tool_dependency_installation_paths.py +++ b/lib/tool_shed/scripts/check_s3_for_empty_tool_dependency_installation_paths.py @@ -12,8 +12,7 @@ import boto -from tool_shed.galaxy_install.tool_dependencies import td_common_util - +from tool_shed.util.basic_util import INSTALLATION_LOG class BucketList( object ): @@ -77,7 +76,7 @@ # This would not be the case in a Galaxy instance, since the Galaxy admin will need to verify the contents of # the installation path in order to determine which action should be taken. elif len( tool_dependency_path_contents ) == 2 and \ - tool_dependency_path_contents[1].name.endswith( td_common_util.INSTALLATION_LOG ): + tool_dependency_path_contents[1].name.endswith( INSTALLATION_LOG ): empty_directories.append( tool_dependency_path_contents[ 0 ] ) return [ item.name for item in empty_directories ] @@ -106,7 +105,7 @@ print 'No empty installation paths found, exiting.' return 0 print 'The following %d tool dependency installation paths were found to be empty or contain only the file %s.' % \ - ( len( dependency_cleaner.empty_installation_paths ), td_common_util.INSTALLATION_LOG ) + ( len( dependency_cleaner.empty_installation_paths ), INSTALLATION_LOG ) if asbool( args.delete ): dependency_cleaner.delete_empty_installation_paths() else: diff -r f8c6577665c383645e934629784ece516b498d9b -r 85aa79e9ab9a87a5b5fea0ed1039aeb9a5c23642 lib/tool_shed/util/basic_util.py --- a/lib/tool_shed/util/basic_util.py +++ b/lib/tool_shed/util/basic_util.py @@ -1,5 +1,8 @@ import logging import os +import shutil +import sys +from string import Template from galaxy.util import unicodify @@ -10,8 +13,52 @@ log = logging.getLogger( __name__ ) +CHUNK_SIZE = 2**20 # 1Mb +INSTALLATION_LOG = 'INSTALLATION.log' +# Set no activity timeout to 20 minutes. +NO_OUTPUT_TIMEOUT = 1200.0 +MAXDIFFSIZE = 8000 MAX_DISPLAY_SIZE = 32768 +def evaluate_template( text, install_environment ): + """ + Substitute variables defined in XML blocks from dependencies file. The value of the received + repository_install_dir is the root installation directory of the repository that contains the + tool dependency. The value of the received install_dir is the root installation directory of + the tool_dependency. + """ + return Template( text ).safe_substitute( get_env_var_values( install_environment ) ) + +def get_env_var_values( install_environment ): + """ + Return a dictionary of values, some of which enable substitution of reserved words for the values. + The received install_enviroment object has 2 important attributes for reserved word substitution: + install_environment.tool_shed_repository_install_dir is the root installation directory of the repository + that contains the tool dependency being installed, and install_environment.install_dir is the root + installation directory of the tool dependency. + """ + env_var_dict = {} + env_var_dict[ 'REPOSITORY_INSTALL_DIR' ] = install_environment.tool_shed_repository_install_dir + env_var_dict[ 'INSTALL_DIR' ] = install_environment.install_dir + env_var_dict[ 'system_install' ] = install_environment.install_dir + # If the Python interpreter is 64bit then we can safely assume that the underlying system is also 64bit. + env_var_dict[ '__is64bit__' ] = sys.maxsize > 2**32 + return env_var_dict + +def move_file( current_dir, source, destination, rename_to=None ): + source_path = os.path.abspath( os.path.join( current_dir, source ) ) + source_file = os.path.basename( source_path ) + if rename_to is not None: + destination_file = rename_to + destination_directory = os.path.join( destination ) + destination_path = os.path.join( destination_directory, destination_file ) + else: + destination_directory = os.path.join( destination ) + destination_path = os.path.join( destination_directory, source_file ) + if not os.path.exists( destination_directory ): + os.makedirs( destination_directory ) + shutil.move( source_path, destination_path ) + def remove_dir( dir ): """Attempt to remove a directory from disk.""" if dir: diff -r f8c6577665c383645e934629784ece516b498d9b -r 85aa79e9ab9a87a5b5fea0ed1039aeb9a5c23642 lib/tool_shed/util/commit_util.py --- a/lib/tool_shed/util/commit_util.py +++ b/lib/tool_shed/util/commit_util.py @@ -12,6 +12,7 @@ from galaxy.util.odict import odict from galaxy.web import url_for import tool_shed.util.shed_util_common as suc +from tool_shed.util import basic_util from tool_shed.util import hg_util from tool_shed.util import tool_util from tool_shed.util import xml_util @@ -126,7 +127,7 @@ bzipped_file = bz2.BZ2File( uploaded_file_name, 'rb' ) while 1: try: - chunk = bzipped_file.read( suc.CHUNK_SIZE ) + chunk = bzipped_file.read( basic_util.CHUNK_SIZE ) except IOError: os.close( fd ) os.remove( uncompressed ) @@ -239,7 +240,7 @@ gzipped_file = gzip.GzipFile( uploaded_file_name, 'rb' ) while 1: try: - chunk = gzipped_file.read( suc.CHUNK_SIZE ) + chunk = gzipped_file.read( basic_util.CHUNK_SIZE ) except IOError, e: os.close( fd ) os.remove( uncompressed ) diff -r f8c6577665c383645e934629784ece516b498d9b -r 85aa79e9ab9a87a5b5fea0ed1039aeb9a5c23642 lib/tool_shed/util/hg_util.py --- a/lib/tool_shed/util/hg_util.py +++ b/lib/tool_shed/util/hg_util.py @@ -4,6 +4,7 @@ from datetime import datetime from time import gmtime from time import strftime +import tempfile from galaxy.util import listify from galaxy import eggs diff -r f8c6577665c383645e934629784ece516b498d9b -r 85aa79e9ab9a87a5b5fea0ed1039aeb9a5c23642 lib/tool_shed/util/metadata_util.py --- a/lib/tool_shed/util/metadata_util.py +++ b/lib/tool_shed/util/metadata_util.py @@ -22,11 +22,12 @@ from tool_shed.util import tool_dependency_util from tool_shed.util import tool_util from tool_shed.util import xml_util -from tool_shed.galaxy_install.tool_dependencies import td_common_util import tool_shed.repository_types.util as rt_util log = logging.getLogger( __name__ ) +REPOSITORY_DATA_MANAGER_CONFIG_FILENAME = 'data_manager_conf.xml' + # Repository metadata comparisons for changeset revisions. EQUAL = 'equal' NO_METADATA = 'no metadata' @@ -37,7 +38,7 @@ NOT_TOOL_CONFIGS = [ suc.DATATYPES_CONFIG_FILENAME, rt_util.REPOSITORY_DEPENDENCY_DEFINITION_FILENAME, rt_util.TOOL_DEPENDENCY_DEFINITION_FILENAME, - suc.REPOSITORY_DATA_MANAGER_CONFIG_FILENAME ] + REPOSITORY_DATA_MANAGER_CONFIG_FILENAME ] def add_tool_versions( trans, id, repository_metadata, changeset_revisions ): # Build a dictionary of { 'tool id' : 'parent tool id' } pairs for each tool in repository_metadata. @@ -750,7 +751,7 @@ metadata_dict = generate_data_manager_metadata( app, repository, files_dir, - hg_util.get_config_from_disk( suc.REPOSITORY_DATA_MANAGER_CONFIG_FILENAME, files_dir ), + hg_util.get_config_from_disk( REPOSITORY_DATA_MANAGER_CONFIG_FILENAME, files_dir ), metadata_dict, shed_config_dict=shed_config_dict ) @@ -809,10 +810,10 @@ if package_install_version == '1.0': # Complex repository dependencies can be defined within the last <actions> tag set contained in an # <actions_group> tag set. Comments, <repository> tag sets and <readme> tag sets will be skipped - # in td_common_util.parse_package_elem(). - actions_elem_tuples = td_common_util.parse_package_elem( sub_elem, - platform_info_dict=None, - include_after_install_actions=False ) + # in tool_dependency_util.parse_package_elem(). + actions_elem_tuples = tool_dependency_util.parse_package_elem( sub_elem, + platform_info_dict=None, + include_after_install_actions=False ) if actions_elem_tuples: # We now have a list of a single tuple that looks something like: # [(True, <Element 'actions' at 0x104017850>)] diff -r f8c6577665c383645e934629784ece516b498d9b -r 85aa79e9ab9a87a5b5fea0ed1039aeb9a5c23642 lib/tool_shed/util/shed_util_common.py --- a/lib/tool_shed/util/shed_util_common.py +++ b/lib/tool_shed/util/shed_util_common.py @@ -24,11 +24,8 @@ log = logging.getLogger( __name__ ) -CHUNK_SIZE = 2**20 # 1Mb MAX_CONTENT_SIZE = 1048576 -MAXDIFFSIZE = 8000 DATATYPES_CONFIG_FILENAME = 'datatypes_conf.xml' -REPOSITORY_DATA_MANAGER_CONFIG_FILENAME = 'data_manager_conf.xml' new_repo_email_alert_template = """ Sharable link: ${sharable_link} This diff is so big that we needed to truncate the remainder. Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: jmchilton: Collections: Add API ability to copy HDCA mirroring HDA operations.
by commits-noreply＠bitbucket.org 28 May '14

28 May '14

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/a06c6c99d74f/ Changeset: a06c6c99d74f User: jmchilton Date: 2014-05-28 22:28:55 Summary: Collections: Add API ability to copy HDCA mirroring HDA operations. Affected #: 3 files diff -r baeea4ce794df61d9e6531023b8e4c105b26a8bb -r a06c6c99d74f7aa902fb2c23e8c904df32a49791 lib/galaxy/dataset_collections/__init__.py --- a/lib/galaxy/dataset_collections/__init__.py +++ b/lib/galaxy/dataset_collections/__init__.py @@ -138,6 +138,18 @@ changed = self._set_from_dict( trans, dataset_collection_instance, payload ) return changed + def copy( + self, + trans, + parent, # PRECONDITION: security checks on ability to add to parent occurred during load. + source, + encoded_source_id, + ): + assert source == "hdca" # for now + source_hdca = self.__get_history_collection_instance( trans, encoded_source_id ) + parent.add_dataset_collection( source_hdca.copy() ) + return source_hdca + def _set_from_dict( self, trans, dataset_collection_instance, new_data ): # Blatantly stolen from UsesHistoryDatasetAssociationMixin.set_hda_from_dict. diff -r baeea4ce794df61d9e6531023b8e4c105b26a8bb -r a06c6c99d74f7aa902fb2c23e8c904df32a49791 lib/galaxy/webapps/galaxy/api/history_contents.py --- a/lib/galaxy/webapps/galaxy/api/history_contents.py +++ b/lib/galaxy/webapps/galaxy/api/history_contents.py @@ -292,9 +292,28 @@ return hda_dict def __create_dataset_collection( self, trans, history, payload, **kwd ): - create_params = api_payload_to_create_params( payload ) + source = kwd.get("source", "new_collection") service = trans.app.dataset_collections_service - dataset_collection_instance = service.create( trans, parent=history, **create_params ) + if source == "new_collection": + create_params = api_payload_to_create_params( payload ) + dataset_collection_instance = service.create( + trans, + parent=history, + **create_params + ) + elif source == "hdca": + content = payload.get( 'content', None ) + if content is None: + raise exceptions.RequestParameterMissingException( "'content' id of target to copy is missing" ) + dataset_collection_instance = service.copy( + trans=trans, + parent=history, + source="hdca", + encoded_source_id=content, + ) + else: + message = "Invalid 'source' parameter in request %s" % source + raise exceptions.RequestParameterInvalidException(message) return self.__collection_dict( trans, dataset_collection_instance, view="element" ) @expose_api_anonymous diff -r baeea4ce794df61d9e6531023b8e4c105b26a8bb -r a06c6c99d74f7aa902fb2c23e8c904df32a49791 test/api/test_history_contents.py --- a/test/api/test_history_contents.py +++ b/test/api/test_history_contents.py @@ -97,9 +97,7 @@ dataset_collection_response = self._post( "histories/%s/contents" % self.history_id, payload ) - self._assert_status_code_is( dataset_collection_response, 200 ) - dataset_collection = dataset_collection_response.json() - self._assert_has_keys( dataset_collection, "url", "name", "deleted" ) + dataset_collection = self.__check_create_collection_response( dataset_collection_response ) post_collection_count = self.__count_contents( type="dataset_collection" ) post_dataset_count = self.__count_contents( type="dataset" ) @@ -144,6 +142,23 @@ show_response = self.__show( hdca ) assert str( show_response.json()[ "name" ] ) == "newnameforpair" + def test_hdca_copy( self ): + hdca = self.dataset_collection_populator.create_pair_in_history( self.history_id ).json() + hdca_id = hdca[ "id" ] + second_history_id = self._new_history() + create_data = dict( + source='hdca', + content=hdca_id, + ) + create_response = self._post( "histories/%s/contents/dataset_collections" % second_history_id, create_data ) + self.__check_create_collection_response( create_response ) + + def __check_create_collection_response( self, response ): + self._assert_status_code_is( response, 200 ) + dataset_collection = response.json() + self._assert_has_keys( dataset_collection, "url", "name", "deleted", "visible", "elements" ) + return dataset_collection + def __show( self, contents ): show_response = self._get( "histories/%s/contents/%ss/%s" % ( self.history_id, contents["history_content_type"], contents[ "id" ] ) ) return show_response Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: greg: Import fix in the Tool Shed's hg_util.py module.
by commits-noreply＠bitbucket.org 28 May '14

28 May '14

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/baeea4ce794d/ Changeset: baeea4ce794d User: greg Date: 2014-05-28 22:06:46 Summary: Import fix in the Tool Shed's hg_util.py module. Affected #: 1 file diff -r b1b1e5cefca5b0193c8623a9aee7a39cd1dda55c -r baeea4ce794df61d9e6531023b8e4c105b26a8bb lib/tool_shed/util/hg_util.py --- a/lib/tool_shed/util/hg_util.py +++ b/lib/tool_shed/util/hg_util.py @@ -4,6 +4,7 @@ from datetime import datetime from time import gmtime from time import strftime +import tempfile from galaxy.util import listify from galaxy import eggs Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: greg: Eliminate the Tool Shed's td_common_util.py module by moving the functions to appropriate classes.
by commits-noreply＠bitbucket.org 28 May '14

28 May '14

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/b1b1e5cefca5/ Changeset: b1b1e5cefca5 User: greg Date: 2014-05-28 21:49:02 Summary: Eliminate the Tool Shed's td_common_util.py module by moving the functions to appropriate classes. Affected #: 15 files diff -r 31740dffaf9ae4e345fd2d4de4ab44ed133dd567 -r b1b1e5cefca5b0193c8623a9aee7a39cd1dda55c lib/galaxy/webapps/tool_shed/controllers/repository.py --- a/lib/galaxy/webapps/tool_shed/controllers/repository.py +++ b/lib/galaxy/webapps/tool_shed/controllers/repository.py @@ -3120,12 +3120,12 @@ options_dict = hg_util.get_mercurial_default_options_dict( 'diff' ) # Not quite sure if the following settings make any difference, but with a combination of them and the size check on each # diff, we don't run out of memory when viewing the changelog of the cisortho2 repository on the test tool shed. - options_dict[ 'maxfile' ] = suc.MAXDIFFSIZE - options_dict[ 'maxtotal' ] = suc.MAXDIFFSIZE + options_dict[ 'maxfile' ] = basic_util.MAXDIFFSIZE + options_dict[ 'maxtotal' ] = basic_util.MAXDIFFSIZE diffopts = mdiff.diffopts( **options_dict ) for diff in patch.diff( repo, node1=ctx_parent.node(), node2=ctx.node(), opts=diffopts ): - if len( diff ) > suc.MAXDIFFSIZE: - diff = util.shrink_string_by_size( diff, suc.MAXDIFFSIZE ) + if len( diff ) > basic_util.MAXDIFFSIZE: + diff = util.shrink_string_by_size( diff, basic_util.MAXDIFFSIZE ) diffs.append( basic_util.to_html_string( diff ) ) modified, added, removed, deleted, unknown, ignored, clean = repo.status( node1=ctx_parent.node(), node2=ctx.node() ) anchors = modified + added + removed + deleted + unknown + ignored + clean diff -r 31740dffaf9ae4e345fd2d4de4ab44ed133dd567 -r b1b1e5cefca5b0193c8623a9aee7a39cd1dda55c lib/tool_shed/galaxy_install/install_manager.py --- a/lib/tool_shed/galaxy_install/install_manager.py +++ b/lib/tool_shed/galaxy_install/install_manager.py @@ -1,5 +1,7 @@ import logging import os +import sys +import traceback from galaxy import eggs @@ -11,7 +13,6 @@ from tool_shed.util import tool_dependency_util -from tool_shed.galaxy_install.tool_dependencies import td_common_util from tool_shed.galaxy_install.tool_dependencies.recipe.env_file_builder import EnvFileBuilder from tool_shed.galaxy_install.tool_dependencies.recipe.install_environment import InstallEnvironment from tool_shed.galaxy_install.tool_dependencies.recipe.recipe_manager import StepManager @@ -25,6 +26,10 @@ class InstallManager( object ): + def format_traceback( self ): + ex_type, ex, tb = sys.exc_info() + return ''.join( traceback.format_tb( tb ) ) + def get_tool_shed_repository_install_dir( self, app, tool_shed_repository ): return os.path.abspath( tool_shed_repository.repo_files_directory( app ) ) @@ -112,7 +117,7 @@ log.exception( 'Error installing tool dependency %s version %s.', str( tool_dependency.name ), str( tool_dependency.version ) ) # Since there was an installation error, update the tool dependency status to Error. The remove_installation_path option must # be left False here. - error_message = '%s\n%s' % ( td_common_util.format_traceback(), str( e ) ) + error_message = '%s\n%s' % ( self.format_traceback(), str( e ) ) tool_dependency = tool_dependency_util.handle_tool_dependency_installation_error( app, tool_dependency, error_message, diff -r 31740dffaf9ae4e345fd2d4de4ab44ed133dd567 -r b1b1e5cefca5b0193c8623a9aee7a39cd1dda55c lib/tool_shed/galaxy_install/tool_dependencies/env_manager.py --- /dev/null +++ b/lib/tool_shed/galaxy_install/tool_dependencies/env_manager.py @@ -0,0 +1,147 @@ +import logging +import os +import sys +from tool_shed.util import common_util +import tool_shed.util.shed_util_common as suc + +log = logging.getLogger( __name__ ) + + +class EnvManager( object ): + + def __init__( self, app ): + self.app = app + + def create_env_var_dict( self, elem, install_environment ): + env_var_name = elem.get( 'name', 'PATH' ) + env_var_action = elem.get( 'action', 'prepend_to' ) + env_var_text = None + tool_dependency_install_dir = install_environment.install_dir + tool_shed_repository_install_dir = install_environment.tool_shed_repository_install_dir + if elem.text and elem.text.find( 'REPOSITORY_INSTALL_DIR' ) >= 0: + if tool_shed_repository_install_dir and elem.text.find( '$REPOSITORY_INSTALL_DIR' ) != -1: + env_var_text = elem.text.replace( '$REPOSITORY_INSTALL_DIR', tool_shed_repository_install_dir ) + return dict( name=env_var_name, action=env_var_action, value=env_var_text ) + else: + env_var_text = elem.text.replace( '$REPOSITORY_INSTALL_DIR', tool_dependency_install_dir ) + return dict( name=env_var_name, action=env_var_action, value=env_var_text ) + if elem.text and elem.text.find( 'INSTALL_DIR' ) >= 0: + if tool_dependency_install_dir: + env_var_text = elem.text.replace( '$INSTALL_DIR', tool_dependency_install_dir ) + return dict( name=env_var_name, action=env_var_action, value=env_var_text ) + else: + env_var_text = elem.text.replace( '$INSTALL_DIR', tool_shed_repository_install_dir ) + return dict( name=env_var_name, action=env_var_action, value=env_var_text ) + if elem.text: + # Allow for environment variables that contain neither REPOSITORY_INSTALL_DIR nor INSTALL_DIR + # since there may be command line parameters that are tuned for a Galaxy instance. Allowing them + # to be set in one location rather than being hard coded into each tool config is the best approach. + # For example: + # <environment_variable name="GATK2_SITE_OPTIONS" action="set_to"> + # "--num_threads 4 --num_cpu_threads_per_data_thread 3 --phone_home STANDARD" + # </environment_variable> + return dict( name=env_var_name, action=env_var_action, value=elem.text) + return None + + def get_env_shell_file_path( self, installation_directory ): + env_shell_file_name = 'env.sh' + default_location = os.path.abspath( os.path.join( installation_directory, env_shell_file_name ) ) + if os.path.exists( default_location ): + return default_location + for root, dirs, files in os.walk( installation_directory ): + for name in files: + if name == env_shell_file_name: + return os.path.abspath( os.path.join( root, name ) ) + return None + + def get_env_shell_file_paths( self, elem ): + # Currently only the following tag set is supported. + # <repository toolshed="http://localhost:9009/" name="package_numpy_1_7" owner="test" changeset_revision="c84c6a8be056"> + # <package name="numpy" version="1.7.1" /> + # </repository> + env_shell_file_paths = [] + toolshed = elem.get( 'toolshed', None ) + repository_name = elem.get( 'name', None ) + repository_owner = elem.get( 'owner', None ) + changeset_revision = elem.get( 'changeset_revision', None ) + if toolshed and repository_name and repository_owner and changeset_revision: + # The protocol is not stored, but the port is if it exists. + toolshed = common_util.remove_protocol_from_tool_shed_url( toolshed ) + repository = suc.get_repository_for_dependency_relationship( self.app, + toolshed, + repository_name, + repository_owner, + changeset_revision ) + if repository: + for sub_elem in elem: + tool_dependency_type = sub_elem.tag + tool_dependency_name = sub_elem.get( 'name' ) + tool_dependency_version = sub_elem.get( 'version' ) + if tool_dependency_type and tool_dependency_name and tool_dependency_version: + # Get the tool_dependency so we can get its installation directory. + tool_dependency = None + for tool_dependency in repository.tool_dependencies: + if tool_dependency.type == tool_dependency_type and \ + tool_dependency.name == tool_dependency_name and \ + tool_dependency.version == tool_dependency_version: + break + if tool_dependency: + tool_dependency_key = '%s/%s' % ( tool_dependency_name, tool_dependency_version ) + installation_directory = tool_dependency.installation_directory( self.app ) + env_shell_file_path = self.get_env_shell_file_path( installation_directory ) + if env_shell_file_path: + env_shell_file_paths.append( env_shell_file_path ) + else: + error_message = "Skipping tool dependency definition because unable to locate env.sh file for tool dependency " + error_message += "type %s, name %s, version %s for repository %s" % \ + ( str( tool_dependency_type ), str( tool_dependency_name ), str( tool_dependency_version ), str( repository.name ) ) + log.debug( error_message ) + continue + else: + error_message = "Skipping tool dependency definition because unable to locate tool dependency " + error_message += "type %s, name %s, version %s for repository %s" % \ + ( str( tool_dependency_type ), str( tool_dependency_name ), str( tool_dependency_version ), str( repository.name ) ) + log.debug( error_message ) + continue + else: + error_message = "Skipping invalid tool dependency definition: type %s, name %s, version %s." % \ + ( str( tool_dependency_type ), str( tool_dependency_name ), str( tool_dependency_version ) ) + log.debug( error_message ) + continue + else: + error_message = "Skipping set_environment_for_install definition because unable to locate required installed tool shed repository: " + error_message += "toolshed %s, name %s, owner %s, changeset_revision %s." % \ + ( str( toolshed ), str( repository_name ), str( repository_owner ), str( changeset_revision ) ) + log.debug( error_message ) + else: + error_message = "Skipping invalid set_environment_for_install definition: toolshed %s, name %s, owner %s, changeset_revision %s." % \ + ( str( toolshed ), str( repository_name ), str( repository_owner ), str( changeset_revision ) ) + log.debug( error_message ) + return env_shell_file_paths + + def get_env_shell_file_paths_from_setup_environment_elem( self, all_env_shell_file_paths, elem, action_dict ): + """ + Parse an XML tag set to discover all child repository dependency tags and define the path to an env.sh file associated + with the repository (this requires the repository dependency to be in an installed state). The received action_dict + will be updated with these discovered paths and returned to the caller. This method handles tool dependency definition + tag sets <setup_r_environment>, <setup_ruby_environment> and <setup_perl_environment>. + """ + # An example elem is: + # <action type="setup_perl_environment"> + # <repository name="package_perl_5_18" owner="iuc"> + # <package name="perl" version="5.18.1" /> + # </repository> + # <repository name="package_expat_2_1" owner="iuc" prior_installation_required="True"> + # <package name="expat" version="2.1.0" /> + # </repository> + # <package>http://search.cpan.org/CPAN/authors/id/T/TO/TODDR/XML-Parser-2.41.tar.gz</package> + # <package>http://search.cpan.org/CPAN/authors/id/L/LD/LDS/CGI.pm-3.43.tar.gz</package> + # </action> + for action_elem in elem: + if action_elem.tag == 'repository': + env_shell_file_paths = self.get_env_shell_file_paths( action_elem ) + all_env_shell_file_paths.extend( env_shell_file_paths ) + if all_env_shell_file_paths: + action_dict[ 'env_shell_file_paths' ] = all_env_shell_file_paths + action_dict[ 'action_shell_file_paths' ] = env_shell_file_paths + return action_dict diff -r 31740dffaf9ae4e345fd2d4de4ab44ed133dd567 -r b1b1e5cefca5b0193c8623a9aee7a39cd1dda55c lib/tool_shed/galaxy_install/tool_dependencies/recipe/install_environment.py --- a/lib/tool_shed/galaxy_install/tool_dependencies/recipe/install_environment.py +++ b/lib/tool_shed/galaxy_install/tool_dependencies/recipe/install_environment.py @@ -24,11 +24,13 @@ from galaxy.util import shrink_string_by_size from galaxy.util import unicodify -from tool_shed.galaxy_install.tool_dependencies import td_common_util from tool_shed.galaxy_install.tool_dependencies.recipe import asynchronous_reader +from tool_shed.util import basic_util + log = logging.getLogger( __name__ ) + class InstallEnvironment( object ): """Object describing the environment built up as part of the process of building and installing a package.""" @@ -44,7 +46,7 @@ self.tool_shed_repository_install_dir = tool_shed_repository_install_dir def __call__( self ): - with settings( warn_only=True, **td_common_util.get_env_var_values( self ) ): + with settings( warn_only=True, **basic_util.get_env_var_values( self ) ): with prefix( self.__setup_environment() ): yield @@ -125,7 +127,7 @@ context = app.install_model.context command = str( cmd ) output = self.handle_complex_command( command ) - self.log_results( cmd, output, os.path.join( self.install_dir, td_common_util.INSTALLATION_LOG ) ) + self.log_results( cmd, output, os.path.join( self.install_dir, basic_util.INSTALLATION_LOG ) ) stdout = output.stdout stderr = output.stderr if len( stdout ) > DATABASE_MAX_STRING_SIZE: @@ -214,18 +216,18 @@ # Sleep a bit before asking the readers again. time.sleep( .1 ) current_wait_time = time.time() - start_timer - if stdout_queue.empty() and stderr_queue.empty() and current_wait_time > td_common_util.NO_OUTPUT_TIMEOUT: + if stdout_queue.empty() and stderr_queue.empty() and current_wait_time > basic_util.NO_OUTPUT_TIMEOUT: err_msg = "\nShutting down process id %s because it generated no output for the defined timeout period of %.1f seconds.\n" % \ - ( pid, td_common_util.NO_OUTPUT_TIMEOUT ) + ( pid, basic_util.NO_OUTPUT_TIMEOUT ) stderr_reader.lines.append( err_msg ) process_handle.kill() break thread_lock.release() # Wait until each of the threads we've started terminate. The following calls will block each thread # until it terminates either normally, through an unhandled exception, or until the timeout occurs. - stdio_thread.join( td_common_util.NO_OUTPUT_TIMEOUT ) - stdout_reader.join( td_common_util.NO_OUTPUT_TIMEOUT ) - stderr_reader.join( td_common_util.NO_OUTPUT_TIMEOUT ) + stdio_thread.join( basic_util.NO_OUTPUT_TIMEOUT ) + stdout_reader.join( basic_util.NO_OUTPUT_TIMEOUT ) + stderr_reader.join( basic_util.NO_OUTPUT_TIMEOUT ) # Close subprocess' file descriptors. error = self.close_file_descriptor( process_handle.stdout ) error = self.close_file_descriptor( process_handle.stderr ) diff -r 31740dffaf9ae4e345fd2d4de4ab44ed133dd567 -r b1b1e5cefca5b0193c8623a9aee7a39cd1dda55c lib/tool_shed/galaxy_install/tool_dependencies/recipe/step_handler.py --- a/lib/tool_shed/galaxy_install/tool_dependencies/recipe/step_handler.py +++ b/lib/tool_shed/galaxy_install/tool_dependencies/recipe/step_handler.py @@ -4,12 +4,17 @@ import stat from string import Template import sys +import tarfile +import time +import urllib2 +import zipfile from galaxy.util import asbool from galaxy.util.template import fill_template +from tool_shed.util import basic_util from tool_shed.util import tool_dependency_util -from tool_shed.galaxy_install.tool_dependencies import td_common_util +from tool_shed.galaxy_install.tool_dependencies.env_manager import EnvManager # TODO: eliminate the use of fabric here. from galaxy import eggs @@ -26,6 +31,152 @@ VIRTUALENV_URL = 'https://pypi.python.org/packages/source/v/virtualenv/virtualenv-1.9.1.tar.gz' +class CompressedFile( object ): + + def __init__( self, file_path, mode='r' ): + if tarfile.is_tarfile( file_path ): + self.file_type = 'tar' + elif zipfile.is_zipfile( file_path ) and not file_path.endswith( '.jar' ): + self.file_type = 'zip' + self.file_name = os.path.splitext( os.path.basename( file_path ) )[ 0 ] + if self.file_name.endswith( '.tar' ): + self.file_name = os.path.splitext( self.file_name )[ 0 ] + self.type = self.file_type + method = 'open_%s' % self.file_type + if hasattr( self, method ): + self.archive = getattr( self, method )( file_path, mode ) + else: + raise NameError( 'File type %s specified, no open method found.' % self.file_type ) + + def extract( self, path ): + '''Determine the path to which the archive should be extracted.''' + contents = self.getmembers() + extraction_path = path + if len( contents ) == 1: + # The archive contains a single file, return the extraction path. + if self.isfile( contents[ 0 ] ): + extraction_path = os.path.join( path, self.file_name ) + if not os.path.exists( extraction_path ): + os.makedirs( extraction_path ) + self.archive.extractall( extraction_path ) + else: + # Get the common prefix for all the files in the archive. If the common prefix ends with a slash, + # or self.isdir() returns True, the archive contains a single directory with the desired contents. + # Otherwise, it contains multiple files and/or directories at the root of the archive. + common_prefix = os.path.commonprefix( [ self.getname( item ) for item in contents ] ) + if len( common_prefix ) >= 1 and not common_prefix.endswith( os.sep ) and self.isdir( self.getmember( common_prefix ) ): + common_prefix += os.sep + if common_prefix.endswith( os.sep ): + self.archive.extractall( os.path.join( path ) ) + extraction_path = os.path.join( path, common_prefix ) + else: + extraction_path = os.path.join( path, self.file_name ) + if not os.path.exists( extraction_path ): + os.makedirs( extraction_path ) + self.archive.extractall( os.path.join( extraction_path ) ) + return os.path.abspath( extraction_path ) + + def getmembers_tar( self ): + return self.archive.getmembers() + + def getmembers_zip( self ): + return self.archive.infolist() + + def getname_tar( self, item ): + return item.name + + def getname_zip( self, item ): + return item.filename + + def getmember( self, name ): + for member in self.getmembers(): + if self.getname( member ) == name: + return member + + def getmembers( self ): + return getattr( self, 'getmembers_%s' % self.type )() + + def getname( self, member ): + return getattr( self, 'getname_%s' % self.type )( member ) + + def isdir( self, member ): + return getattr( self, 'isdir_%s' % self.type )( member ) + + def isdir_tar( self, member ): + return member.isdir() + + def isdir_zip( self, member ): + if member.filename.endswith( os.sep ): + return True + return False + + def isfile( self, member ): + if not self.isdir( member ): + return True + return False + + def open_tar( self, filepath, mode ): + return tarfile.open( filepath, mode, errorlevel=0 ) + + def open_zip( self, filepath, mode ): + return zipfile.ZipFile( filepath, mode ) + + def zipfile_ok( self, path_to_archive ): + """ + This function is a bit pedantic and not functionally necessary. It checks whether there is + no file pointing outside of the extraction, because ZipFile.extractall() has some potential + security holes. See python zipfile documentation for more details. + """ + basename = os.path.realpath( os.path.dirname( path_to_archive ) ) + zip_archive = zipfile.ZipFile( path_to_archive ) + for member in zip_archive.namelist(): + member_path = os.path.realpath( os.path.join( basename, member ) ) + if not member_path.startswith( basename ): + return False + return True + + +class Download( object ): + + def url_download( self, install_dir, downloaded_file_name, download_url, extract=True ): + file_path = os.path.join( install_dir, downloaded_file_name ) + src = None + dst = None + # Set a timer so we don't sit here forever. + start_time = time.time() + try: + src = urllib2.urlopen( download_url ) + dst = open( file_path, 'wb' ) + while True: + chunk = src.read( basic_util.CHUNK_SIZE ) + if chunk: + dst.write( chunk ) + else: + break + time_taken = time.time() - start_time + if time_taken > basic_util.NO_OUTPUT_TIMEOUT: + err_msg = 'Downloading from URL %s took longer than the defined timeout period of %.1f seconds.' % \ + ( str( download_url ), basic_util.NO_OUTPUT_TIMEOUT ) + raise Exception( err_msg ) + except Exception, e: + err_msg = err_msg = 'Error downloading from URL\n%s:\n%s' % ( str( download_url ), str( e ) ) + raise Exception( err_msg ) + finally: + if src: + src.close() + if dst: + dst.close() + if extract: + if tarfile.is_tarfile( file_path ) or ( zipfile.is_zipfile( file_path ) and not file_path.endswith( '.jar' ) ): + archive = CompressedFile( file_path ) + extraction_path = archive.extract( install_dir ) + else: + extraction_path = os.path.abspath( install_dir ) + else: + extraction_path = os.path.abspath( install_dir ) + return extraction_path + + class RecipeStep( object ): """Abstract class that defines a standard format for handling recipe steps when installing packages.""" @@ -42,6 +193,22 @@ def __init__( self ): self.type = 'assert_directory_executable' + def assert_directory_executable( self, full_path ): + """ + Return True if a symbolic link or directory exists and is executable, but if + full_path is a file, return False. + """ + if full_path is None: + return False + if os.path.isfile( full_path ): + return False + if os.path.isdir( full_path ): + # Make sure the owner has execute permission on the directory. + # See http://docs.python.org/2/library/stat.html + if stat.S_IXUSR & os.stat( full_path )[ stat.ST_MODE ] == 64: + return True + return False + def execute_step( self, app, tool_dependency, package_name, actions, action_dict, filtered_actions, env_file_builder, install_environment, work_dir, current_dir=None, initial_download=False ): """ @@ -53,7 +220,7 @@ full_path = action_dict[ 'full_path' ] else: full_path = os.path.join( current_dir, action_dict[ 'full_path' ] ) - if not td_common_util.assert_directory_executable( full_path=full_path ): + if not self.assert_directory_executable( full_path=full_path ): status = app.install_model.ToolDependency.installation_status.ERROR error_message = 'The path %s is not a directory or is not executable by the owner.' % str( full_path ) tool_dependency = tool_dependency_util.set_tool_dependency_attributes( app, @@ -66,7 +233,7 @@ def prepare_step( self, app, tool_dependency, action_elem, action_dict, install_environment, is_binary_download ): # <action type="assert_executable">$INSTALL_DIR/mira/my_file</action> if action_elem.text: - action_dict[ 'full_path' ] = td_common_util.evaluate_template( action_elem.text, install_environment ) + action_dict[ 'full_path' ] = basic_util.evaluate_template( action_elem.text, install_environment ) return action_dict @@ -75,6 +242,18 @@ def __init__( self ): self.type = 'assert_directory_exists' + def assert_directory_exists( self, full_path ): + """ + Return True if a symbolic link or directory exists, but if full_path is a file, + return False. """ + if full_path is None: + return False + if os.path.isfile( full_path ): + return False + if os.path.isdir( full_path ): + return True + return False + def execute_step( self, app, tool_dependency, package_name, actions, action_dict, filtered_actions, env_file_builder, install_environment, work_dir, current_dir=None, initial_download=False ): """ @@ -86,7 +265,7 @@ full_path = action_dict[ 'full_path' ] else: full_path = os.path.join( current_dir, action_dict[ 'full_path' ] ) - if not td_common_util.assert_directory_exists( full_path=full_path ): + if not self.assert_directory_exists( full_path=full_path ): status = app.install_model.ToolDependency.installation_status.ERROR error_message = 'The path %s is not a directory or does not exist.' % str( full_path ) tool_dependency = tool_dependency_util.set_tool_dependency_attributes( app, @@ -99,7 +278,7 @@ def prepare_step( self, app, tool_dependency, action_elem, action_dict, install_environment, is_binary_download ): # <action type="make_directory">$INSTALL_DIR/mira</action> if action_elem.text: - action_dict[ 'full_path' ] = td_common_util.evaluate_template( action_elem.text, install_environment ) + action_dict[ 'full_path' ] = basic_util.evaluate_template( action_elem.text, install_environment ) return action_dict @@ -108,6 +287,22 @@ def __init__( self ): self.type = 'assert_file_executable' + def assert_file_executable( self, full_path ): + """ + Return True if a symbolic link or file exists and is executable, but if full_path + is a directory, return False. + """ + if full_path is None: + return False + if os.path.isdir( full_path ): + return False + if os.path.exists( full_path ): + # Make sure the owner has execute permission on the file. + # See http://docs.python.org/2/library/stat.html + if stat.S_IXUSR & os.stat( full_path )[ stat.ST_MODE ] == 64: + return True + return False + def execute_step( self, app, tool_dependency, package_name, actions, action_dict, filtered_actions, env_file_builder, install_environment, work_dir, current_dir=None, initial_download=False ): """ @@ -119,7 +314,7 @@ full_path = action_dict[ 'full_path' ] else: full_path = os.path.join( current_dir, action_dict[ 'full_path' ] ) - if not td_common_util.assert_file_executable( full_path=full_path ): + if not self.assert_file_executable( full_path=full_path ): status = app.install_model.ToolDependency.installation_status.ERROR error_message = 'The path %s is not a file or is not executable by the owner.' % str( full_path ) tool_dependency = tool_dependency_util.set_tool_dependency_attributes( app, @@ -132,7 +327,7 @@ def prepare_step( self, app, tool_dependency, action_elem, action_dict, install_environment, is_binary_download ): # <action type="assert_executable">$INSTALL_DIR/mira/my_file</action> if action_elem.text: - action_dict[ 'full_path' ] = td_common_util.evaluate_template( action_elem.text, install_environment ) + action_dict[ 'full_path' ] = basic_util.evaluate_template( action_elem.text, install_environment ) return action_dict @@ -141,6 +336,19 @@ def __init__( self ): self.type = 'assert_file_exists' + def assert_file_exists( self, full_path ): + """ + Return True if a symbolic link or file exists, but if full_path is a directory, + return False. + """ + if full_path is None: + return False + if os.path.isdir( full_path ): + return False + if os.path.exists( full_path ): + return True + return False + def execute_step( self, app, tool_dependency, package_name, actions, action_dict, filtered_actions, env_file_builder, install_environment, work_dir, current_dir=None, initial_download=False ): """ @@ -152,7 +360,7 @@ full_path = action_dict[ 'full_path' ] else: full_path = os.path.join( current_dir, action_dict[ 'full_path' ] ) - if not td_common_util.assert_file_exists( full_path=full_path ): + if not self.assert_file_exists( full_path=full_path ): status = app.install_model.ToolDependency.installation_status.ERROR error_message = 'The path %s is not a file or does not exist.' % str( full_path ) tool_dependency = tool_dependency_util.set_tool_dependency_attributes( app, @@ -165,7 +373,7 @@ def prepare_step( self, app, tool_dependency, action_elem, action_dict, install_environment, is_binary_download ): # <action type="assert_on_path">$INSTALL_DIR/mira/my_file</action> if action_elem.text: - action_dict[ 'full_path' ] = td_common_util.evaluate_template( action_elem.text, install_environment ) + action_dict[ 'full_path' ] = basic_util.evaluate_template( action_elem.text, install_environment ) return action_dict @@ -187,7 +395,7 @@ pre_cmd = './configure %s && make && make install' % configure_opts else: pre_cmd = './configure --prefix=$INSTALL_DIR %s && make && make install' % configure_opts - cmd = install_environment.build_command( td_common_util.evaluate_template( pre_cmd, install_environment ) ) + cmd = install_environment.build_command( basic_util.evaluate_template( pre_cmd, install_environment ) ) return_code = install_environment.handle_command( app=app, tool_dependency=tool_dependency, cmd=cmd, @@ -199,7 +407,7 @@ def prepare_step( self, app, tool_dependency, action_elem, action_dict, install_environment, is_binary_download ): # Handle configure, make and make install allow providing configuration options if action_elem.text: - configure_opts = td_common_util.evaluate_template( action_elem.text, install_environment ) + configure_opts = basic_util.evaluate_template( action_elem.text, install_environment ) action_dict[ 'configure_opts' ] = configure_opts return action_dict @@ -274,7 +482,7 @@ received_mode = int( file_elem.get( 'mode', 600 ), base=8 ) # For added security, ensure that the setuid and setgid bits are not set. mode = received_mode & ~( stat.S_ISUID | stat.S_ISGID ) - file = td_common_util.evaluate_template( file_elem.text, install_environment ) + file = basic_util.evaluate_template( file_elem.text, install_environment ) chmod_tuple = ( file, mode ) chmod_actions.append( chmod_tuple ) if chmod_actions: @@ -282,11 +490,17 @@ return action_dict -class DownloadBinary( RecipeStep ): +class DownloadBinary( Download, RecipeStep ): def __init__( self ): self.type = 'download_binary' + def download_binary( self, url, work_dir ): + """Download a pre-compiled binary from the specified URL.""" + downloaded_filename = os.path.split( url )[ -1 ] + dir = self.url_download( work_dir, downloaded_filename, url, extract=False ) + return downloaded_filename + def filter_actions_after_binary_installation( self, actions ): '''Filter out actions that should not be processed if a binary download succeeded.''' filtered_actions = [] @@ -311,7 +525,7 @@ log.debug( 'Attempting to download from %s to %s', url, str( target_directory ) ) downloaded_filename = None try: - downloaded_filename = td_common_util.download_binary( url, work_dir ) + downloaded_filename = self.download_binary( url, work_dir ) if initial_download: # Filter out any actions that are not download_binary, chmod, or set_environment. filtered_actions = self.filter_actions_after_binary_installation( actions[ 1: ] ) @@ -338,9 +552,9 @@ full_path_to_dir = os.path.abspath( install_environment.install_dir ) else: full_path_to_dir = os.path.abspath( install_environment.install_dir ) - td_common_util.move_file( current_dir=work_dir, - source=downloaded_filename, - destination=full_path_to_dir ) + basic_util.move_file( current_dir=work_dir, + source=downloaded_filename, + destination=full_path_to_dir ) # Not sure why dir is ignored in this method, need to investigate... dir = None if initial_download: @@ -368,7 +582,7 @@ return action_dict -class DownloadByUrl( RecipeStep ): +class DownloadByUrl( Download, RecipeStep ): def __init__( self ): self.type = 'download_by_url' @@ -394,9 +608,9 @@ downloaded_filename = action_dict[ 'target_filename' ] else: downloaded_filename = os.path.split( url )[ -1 ] - dir = td_common_util.url_download( work_dir, downloaded_filename, url, extract=True ) + dir = self.url_download( work_dir, downloaded_filename, url, extract=True ) if is_binary: - log_file = os.path.join( install_environment.install_dir, td_common_util.INSTALLATION_LOG ) + log_file = os.path.join( install_environment.install_dir, basic_util.INSTALLATION_LOG ) if os.path.exists( log_file ): logfile = open( log_file, 'ab' ) else: @@ -422,7 +636,7 @@ return action_dict -class DownloadFile( RecipeStep ): +class DownloadFile( Download, RecipeStep ): def __init__( self ): self.type = 'download_file' @@ -447,7 +661,7 @@ filename = action_dict[ 'target_filename' ] else: filename = url.split( '/' )[ -1 ] - td_common_util.url_download( work_dir, filename, url ) + self.url_download( work_dir, filename, url ) if initial_download: dir = os.path.curdir return tool_dependency, filtered_actions, dir @@ -479,13 +693,17 @@ full_path = action_dict[ 'full_path' ] else: full_path = os.path.join( current_dir, action_dict[ 'full_path' ] ) - td_common_util.make_directory( full_path=full_path ) + self.make_directory( full_path=full_path ) return tool_dependency, None, None + def make_directory( self, full_path ): + if not os.path.exists( full_path ): + os.makedirs( full_path ) + def prepare_step( self, app, tool_dependency, action_elem, action_dict, install_environment, is_binary_download ): # <action type="make_directory">$INSTALL_DIR/lib/python</action> if action_elem.text: - action_dict[ 'full_path' ] = td_common_util.evaluate_template( action_elem.text, install_environment ) + action_dict[ 'full_path' ] = basic_util.evaluate_template( action_elem.text, install_environment ) return action_dict @@ -515,7 +733,7 @@ def prepare_step( self, app, tool_dependency, action_elem, action_dict, install_environment, is_binary_download ): # make; make install; allow providing make options if action_elem.text: - make_opts = td_common_util.evaluate_template( action_elem.text, install_environment ) + make_opts = basic_util.evaluate_template( action_elem.text, install_environment ) action_dict[ 'make_opts' ] = make_opts return action_dict @@ -531,18 +749,38 @@ Move a directory of files. Since this class is not used in the initial download stage, no recipe step filtering is performed here, and None values are always returned for filtered_actions and dir. """ - td_common_util.move_directory_files( current_dir=current_dir, - source_dir=os.path.join( action_dict[ 'source_directory' ] ), - destination_dir=os.path.join( action_dict[ 'destination_directory' ] ) ) + self.move_directory_files( current_dir=current_dir, + source_dir=os.path.join( action_dict[ 'source_directory' ] ), + destination_dir=os.path.join( action_dict[ 'destination_directory' ] ) ) return tool_dependency, None, None + def move_directory_files( self, current_dir, source_dir, destination_dir ): + source_directory = os.path.abspath( os.path.join( current_dir, source_dir ) ) + destination_directory = os.path.join( destination_dir ) + if not os.path.isdir( destination_directory ): + os.makedirs( destination_directory ) + symlinks = [] + regular_files = [] + for file_name in os.listdir( source_directory ): + source_file = os.path.join( source_directory, file_name ) + destination_file = os.path.join( destination_directory, file_name ) + files_tuple = ( source_file, destination_file ) + if os.path.islink( source_file ): + symlinks.append( files_tuple ) + else: + regular_files.append( files_tuple ) + for source_file, destination_file in symlinks: + shutil.move( source_file, destination_file ) + for source_file, destination_file in regular_files: + shutil.move( source_file, destination_file ) + def prepare_step( self, app, tool_dependency, action_elem, action_dict, install_environment, is_binary_download ): # <action type="move_directory_files"> # <source_directory>bin</source_directory> # <destination_directory>$INSTALL_DIR/bin</destination_directory> # </action> for move_elem in action_elem: - move_elem_text = td_common_util.evaluate_template( move_elem.text, install_environment ) + move_elem_text = basic_util.evaluate_template( move_elem.text, install_environment ) if move_elem_text: action_dict[ move_elem.tag ] = move_elem_text return action_dict @@ -559,10 +797,10 @@ Move a file on disk. Since this class is not used in the initial download stage, no recipe step filtering is performed here, and None values are always returned for filtered_actions and dir. """ - td_common_util.move_file( current_dir=current_dir, - source=os.path.join( action_dict[ 'source' ] ), - destination=os.path.join( action_dict[ 'destination' ] ), - rename_to=action_dict[ 'rename_to' ] ) + basic_util.move_file( current_dir=current_dir, + source=os.path.join( action_dict[ 'source' ] ), + destination=os.path.join( action_dict[ 'destination' ] ), + rename_to=action_dict[ 'rename_to' ] ) return tool_dependency, None, None def prepare_step( self, app, tool_dependency, action_elem, action_dict, install_environment, is_binary_download ): @@ -570,8 +808,8 @@ # <source>misc/some_file</source> # <destination>$INSTALL_DIR/bin</destination> # </action> - action_dict[ 'source' ] = td_common_util.evaluate_template( action_elem.find( 'source' ).text, install_environment ) - action_dict[ 'destination' ] = td_common_util.evaluate_template( action_elem.find( 'destination' ).text, install_environment ) + action_dict[ 'source' ] = basic_util.evaluate_template( action_elem.find( 'source' ).text, install_environment ) + action_dict[ 'destination' ] = basic_util.evaluate_template( action_elem.find( 'destination' ).text, install_environment ) action_dict[ 'rename_to' ] = action_elem.get( 'rename_to' ) return action_dict @@ -717,12 +955,12 @@ # <action type="set_environment"> # <environment_variable name="PATH" action="prepend_to">$INSTALL_DIR</environment_variable> # </action> + env_manager = EnvManager( app ) env_var_dicts = [] for env_elem in action_elem: if env_elem.tag == 'environment_variable': - env_var_dict = \ - td_common_util.create_env_var_dict( elem=env_elem, - install_environment=install_environment ) + env_var_dict = env_manager.create_env_var_dict( elem=env_elem, + install_environment=install_environment ) if env_var_dict: env_var_dicts.append( env_var_dict ) if env_var_dicts: @@ -764,16 +1002,17 @@ # the current tool dependency package. See the package_matplotlib_1_2 repository in the test tool # shed for a real-world example. all_env_shell_file_paths = [] + env_manager = EnvManager( app ) for env_elem in action_elem: if env_elem.tag == 'repository': - env_shell_file_paths = td_common_util.get_env_shell_file_paths( app, env_elem ) + env_shell_file_paths = env_manager.get_env_shell_file_paths( env_elem ) if env_shell_file_paths: all_env_shell_file_paths.extend( env_shell_file_paths ) action_dict[ 'env_shell_file_paths' ] = all_env_shell_file_paths return action_dict -class SetupPerlEnvironment( RecipeStep ): +class SetupPerlEnvironment( Download, RecipeStep ): def __init__( self ): self.type = 'setup_purl_environment' @@ -822,7 +1061,7 @@ # We assume a URL to a gem file. url = perl_package perl_package_name = url.split( '/' )[ -1 ] - dir = td_common_util.url_download( work_dir, perl_package_name, url, extract=True ) + dir = self.url_download( work_dir, perl_package_name, url, extract=True ) # Search for Build.PL or Makefile.PL (ExtUtils::MakeMaker vs. Module::Build). tmp_work_dir = os.path.join( work_dir, dir ) if os.path.exists( os.path.join( tmp_work_dir, 'Makefile.PL' ) ): @@ -836,7 +1075,7 @@ return tool_dependency, filtered_actions, dir return tool_dependency, None, None with lcd( tmp_work_dir ): - cmd = install_environment.build_command( td_common_util.evaluate_template( cmd, install_environment ) ) + cmd = install_environment.build_command( basic_util.evaluate_template( cmd, install_environment ) ) return_code = install_environment.handle_command( app=app, tool_dependency=tool_dependency, cmd=cmd, @@ -849,7 +1088,7 @@ # perl package from CPAN without version number. # cpanm should be installed with the parent perl distribution, otherwise this will not work. cmd += '''cpanm --local-lib=$INSTALL_DIR %s''' % ( perl_package ) - cmd = install_environment.build_command( td_common_util.evaluate_template( cmd, install_environment ) ) + cmd = install_environment.build_command( basic_util.evaluate_template( cmd, install_environment ) ) return_code = install_environment.handle_command( app=app, tool_dependency=tool_dependency, cmd=cmd, @@ -890,10 +1129,11 @@ # with each repository. This will potentially update the value of the 'env_shell_file_paths' entry # in action_dict. all_env_shell_file_paths = [] - action_dict = td_common_util.get_env_shell_file_paths_from_setup_environment_elem( app, - all_env_shell_file_paths, - action_elem, - action_dict ) + env_manager = EnvManager( app ) + action_dict = env_manager.get_env_shell_file_paths_from_setup_environment_elem( app, + all_env_shell_file_paths, + action_elem, + action_dict ) perl_packages = [] for env_elem in action_elem: if env_elem.tag == 'package': @@ -908,7 +1148,7 @@ return action_dict -class SetupREnvironment( RecipeStep ): +class SetupREnvironment( Download, RecipeStep ): def __init__( self ): self.type = 'setup_r_environment' @@ -947,7 +1187,7 @@ for url in action_dict[ 'r_packages' ]: filename = url.split( '/' )[ -1 ] tarball_names.append( filename ) - td_common_util.url_download( work_dir, filename, url, extract=False ) + self.url_download( work_dir, filename, url, extract=False ) dir = os.path.curdir current_dir = os.path.abspath( os.path.join( work_dir, dir ) ) with lcd( current_dir ): @@ -958,7 +1198,7 @@ cmd = r'''PATH=$PATH:$R_HOME/bin; export PATH; R_LIBS=$INSTALL_DIR; export R_LIBS; Rscript -e "install.packages(c('%s'),lib='$INSTALL_DIR', repos=NULL, dependencies=FALSE)"''' % \ ( str( tarball_name ) ) - cmd = install_environment.build_command( td_common_util.evaluate_template( cmd, install_environment ) ) + cmd = install_environment.build_command( basic_util.evaluate_template( cmd, install_environment ) ) return_code = install_environment.handle_command( app=app, tool_dependency=tool_dependency, cmd=cmd, @@ -993,10 +1233,11 @@ # associated with each repository. This will potentially update the value of the # 'env_shell_file_paths' entry in action_dict. all_env_shell_file_paths = [] - action_dict = td_common_util.get_env_shell_file_paths_from_setup_environment_elem( app, - all_env_shell_file_paths, - action_elem, - action_dict ) + env_manager = EnvManager( app ) + action_dict = env_manager.get_env_shell_file_paths_from_setup_environment_elem( app, + all_env_shell_file_paths, + action_elem, + action_dict ) r_packages = list() for env_elem in action_elem: if env_elem.tag == 'package': @@ -1006,7 +1247,7 @@ return action_dict -class SetupRubyEnvironment( RecipeStep ): +class SetupRubyEnvironment( Download, RecipeStep ): def __init__( self ): self.type = 'setup_ruby_environment' @@ -1058,7 +1299,7 @@ # We assume a URL to a gem file. url = gem gem_name = url.split( '/' )[ -1 ] - td_common_util.url_download( work_dir, gem_name, url, extract=False ) + self.url_download( work_dir, gem_name, url, extract=False ) cmd = '''PATH=$PATH:$RUBY_HOME/bin; export PATH; GEM_HOME=$INSTALL_DIR; export GEM_HOME; gem install --local %s ''' % ( gem_name ) else: @@ -1073,7 +1314,7 @@ # no version number given cmd = '''PATH=$PATH:$RUBY_HOME/bin; export PATH; GEM_HOME=$INSTALL_DIR; export GEM_HOME; gem install %s''' % ( gem ) - cmd = install_environment.build_command( td_common_util.evaluate_template( cmd, install_environment ) ) + cmd = install_environment.build_command( basic_util.evaluate_template( cmd, install_environment ) ) return_code = install_environment.handle_command( app=app, tool_dependency=tool_dependency, cmd=cmd, @@ -1114,10 +1355,11 @@ # associated with each repository. This will potentially update the value of the # 'env_shell_file_paths' entry in action_dict. all_env_shell_file_paths = [] - action_dict = td_common_util.get_env_shell_file_paths_from_setup_environment_elem( app, - all_env_shell_file_paths, - action_elem, - action_dict ) + env_manager = EnvManager( app ) + action_dict = env_manager.get_env_shell_file_paths_from_setup_environment_elem( app, + all_env_shell_file_paths, + action_elem, + action_dict ) ruby_package_tups = [] for env_elem in action_elem: if env_elem.tag == 'package': @@ -1140,7 +1382,7 @@ return action_dict -class SetupVirtualEnv( RecipeStep ): +class SetupVirtualEnv( Download, RecipeStep ): def __init__( self ): self.type = 'setup_virtualenv' @@ -1228,9 +1470,10 @@ with install_environment.make_tmp_dir() as work_dir: downloaded_filename = VIRTUALENV_URL.rsplit('/', 1)[-1] try: - dir = td_common_util.url_download( work_dir, downloaded_filename, VIRTUALENV_URL ) + dir = self.url_download( work_dir, downloaded_filename, VIRTUALENV_URL ) except: - log.error( "Failed to download virtualenv: td_common_util.url_download( '%s', '%s', '%s' ) threw an exception", work_dir, downloaded_filename, VIRTUALENV_URL ) + log.error( "Failed to download virtualenv: url_download( '%s', '%s', '%s' ) threw an exception", + work_dir, downloaded_filename, VIRTUALENV_URL ) return False full_path_to_dir = os.path.abspath( os.path.join( work_dir, dir ) ) shutil.move( full_path_to_dir, venv_dir ) @@ -1245,7 +1488,7 @@ # lxml==2.3.0</action> ## Manually specify contents of requirements.txt file to create dynamically. action_dict[ 'use_requirements_file' ] = asbool( action_elem.get( 'use_requirements_file', True ) ) - action_dict[ 'requirements' ] = td_common_util.evaluate_template( action_elem.text or 'requirements.txt', install_environment ) + action_dict[ 'requirements' ] = basic_util.evaluate_template( action_elem.text or 'requirements.txt', install_environment ) action_dict[ 'python' ] = action_elem.get( 'python', 'python' ) return action_dict @@ -1316,7 +1559,7 @@ def prepare_step( self, app, tool_dependency, action_elem, action_dict, install_environment, is_binary_download ): # <action type="shell_command">make</action> - action_elem_text = td_common_util.evaluate_template( action_elem.text, install_environment ) + action_elem_text = basic_util.evaluate_template( action_elem.text, install_environment ) if action_elem_text: action_dict[ 'command' ] = action_elem_text return action_dict @@ -1338,7 +1581,7 @@ env_vars = dict() env_vars = install_environment.environment_dict() tool_shed_repository = tool_dependency.tool_shed_repository - env_vars.update( td_common_util.get_env_var_values( install_environment ) ) + env_vars.update( basic_util.get_env_var_values( install_environment ) ) language = action_dict[ 'language' ] with settings( warn_only=True, **env_vars ): if language == 'cheetah': diff -r 31740dffaf9ae4e345fd2d4de4ab44ed133dd567 -r b1b1e5cefca5b0193c8623a9aee7a39cd1dda55c lib/tool_shed/galaxy_install/tool_dependencies/recipe/tag_handler.py --- a/lib/tool_shed/galaxy_install/tool_dependencies/recipe/tag_handler.py +++ b/lib/tool_shed/galaxy_install/tool_dependencies/recipe/tag_handler.py @@ -10,7 +10,7 @@ from tool_shed.util import tool_dependency_util from tool_shed.util import xml_util -from tool_shed.galaxy_install.tool_dependencies import td_common_util +from tool_shed.galaxy_install.tool_dependencies.env_manager import EnvManager from tool_shed.galaxy_install.tool_dependencies.recipe.env_file_builder import EnvFileBuilder from tool_shed.galaxy_install.tool_dependencies.recipe.install_environment import InstallEnvironment @@ -85,9 +85,9 @@ platform_info_dict = tool_dependency_util.get_platform_info_dict() if package_install_version == '1.0': # Handle tool dependency installation using a fabric method included in the Galaxy framework. - actions_elem_tuples = td_common_util.parse_package_elem( package_elem, - platform_info_dict=platform_info_dict, - include_after_install_actions=True ) + actions_elem_tuples = tool_dependency_util.parse_package_elem( package_elem, + platform_info_dict=platform_info_dict, + include_after_install_actions=True ) if not actions_elem_tuples: proceed_with_install = False error_message = 'Version %s of the %s package cannot be installed because ' % ( str( package_version ), str( package_name ) ) @@ -491,6 +491,7 @@ # <set_environment version="1.0"> # <repository toolshed="<tool shed>" name="<repository name>" owner="<repository owner>" changeset_revision="<changeset revision>" /> # </set_environment> + env_manager = EnvManager( app ) tool_dependencies = [] env_var_version = elem.get( 'version', '1.0' ) tool_shed_repository_install_dir = os.path.abspath( tool_shed_repository.repo_files_directory( app ) ) @@ -514,8 +515,8 @@ tool_dependency_version=None ) install_environment = InstallEnvironment( tool_shed_repository_install_dir=tool_shed_repository_install_dir, install_dir=install_dir ) - env_var_dict = td_common_util.create_env_var_dict( elem=env_var_elem, - install_environment=install_environment ) + env_var_dict = env_manager.create_env_var_dict( elem=env_var_elem, + install_environment=install_environment ) if env_var_dict: if not os.path.exists( install_dir ): os.makedirs( install_dir ) diff -r 31740dffaf9ae4e345fd2d4de4ab44ed133dd567 -r b1b1e5cefca5b0193c8623a9aee7a39cd1dda55c lib/tool_shed/galaxy_install/tool_dependencies/td_common_util.py --- a/lib/tool_shed/galaxy_install/tool_dependencies/td_common_util.py +++ /dev/null @@ -1,578 +0,0 @@ -import logging -import os -import re -import shutil -import stat -import sys -import tarfile -import time -import traceback -import urllib2 -import zipfile -from string import Template -from tool_shed.util import common_util -import tool_shed.util.shed_util_common as suc -from galaxy.datatypes import checkers - -log = logging.getLogger( __name__ ) - -# Set no activity timeout to 20 minutes. -NO_OUTPUT_TIMEOUT = 1200.0 -INSTALLATION_LOG = 'INSTALLATION.log' - - -class CompressedFile( object ): - - def __init__( self, file_path, mode='r' ): - if istar( file_path ): - self.file_type = 'tar' - elif iszip( file_path ) and not isjar( file_path ): - self.file_type = 'zip' - self.file_name = os.path.splitext( os.path.basename( file_path ) )[ 0 ] - if self.file_name.endswith( '.tar' ): - self.file_name = os.path.splitext( self.file_name )[ 0 ] - self.type = self.file_type - method = 'open_%s' % self.file_type - if hasattr( self, method ): - self.archive = getattr( self, method )( file_path, mode ) - else: - raise NameError( 'File type %s specified, no open method found.' % self.file_type ) - - def extract( self, path ): - '''Determine the path to which the archive should be extracted.''' - contents = self.getmembers() - extraction_path = path - if len( contents ) == 1: - # The archive contains a single file, return the extraction path. - if self.isfile( contents[ 0 ] ): - extraction_path = os.path.join( path, self.file_name ) - if not os.path.exists( extraction_path ): - os.makedirs( extraction_path ) - self.archive.extractall( extraction_path ) - else: - # Get the common prefix for all the files in the archive. If the common prefix ends with a slash, - # or self.isdir() returns True, the archive contains a single directory with the desired contents. - # Otherwise, it contains multiple files and/or directories at the root of the archive. - common_prefix = os.path.commonprefix( [ self.getname( item ) for item in contents ] ) - if len( common_prefix ) >= 1 and not common_prefix.endswith( os.sep ) and self.isdir( self.getmember( common_prefix ) ): - common_prefix += os.sep - if common_prefix.endswith( os.sep ): - self.archive.extractall( os.path.join( path ) ) - extraction_path = os.path.join( path, common_prefix ) - else: - extraction_path = os.path.join( path, self.file_name ) - if not os.path.exists( extraction_path ): - os.makedirs( extraction_path ) - self.archive.extractall( os.path.join( extraction_path ) ) - return os.path.abspath( extraction_path ) - - def getmembers_tar( self ): - return self.archive.getmembers() - - def getmembers_zip( self ): - return self.archive.infolist() - - def getname_tar( self, item ): - return item.name - - def getname_zip( self, item ): - return item.filename - - def getmember( self, name ): - for member in self.getmembers(): - if self.getname( member ) == name: - return member - - def getmembers( self ): - return getattr( self, 'getmembers_%s' % self.type )() - - def getname( self, member ): - return getattr( self, 'getname_%s' % self.type )( member ) - - def isdir( self, member ): - return getattr( self, 'isdir_%s' % self.type )( member ) - - def isdir_tar( self, member ): - return member.isdir() - - def isdir_zip( self, member ): - if member.filename.endswith( os.sep ): - return True - return False - - def isfile( self, member ): - if not self.isdir( member ): - return True - return False - - def open_tar( self, filepath, mode ): - return tarfile.open( filepath, mode, errorlevel=0 ) - - def open_zip( self, filepath, mode ): - return zipfile.ZipFile( filepath, mode ) - -def assert_directory_executable( full_path ): - """ - Return True if a symbolic link or directory exists and is executable, but if - full_path is a file, return False. - """ - if full_path is None: - return False - if os.path.isfile( full_path ): - return False - if os.path.isdir( full_path ): - # Make sure the owner has execute permission on the directory. - # See http://docs.python.org/2/library/stat.html - if stat.S_IXUSR & os.stat( full_path )[ stat.ST_MODE ] == 64: - return True - return False - -def assert_directory_exists( full_path ): - """ - Return True if a symbolic link or directory exists, but if full_path is a file, - return False. """ - if full_path is None: - return False - if os.path.isfile( full_path ): - return False - if os.path.isdir( full_path ): - return True - return False - -def assert_file_executable( full_path ): - """ - Return True if a symbolic link or file exists and is executable, but if full_path - is a directory, return False. - """ - if full_path is None: - return False - if os.path.isdir( full_path ): - return False - if os.path.exists( full_path ): - # Make sure the owner has execute permission on the file. - # See http://docs.python.org/2/library/stat.html - if stat.S_IXUSR & os.stat( full_path )[ stat.ST_MODE ] == 64: - return True - return False - -def assert_file_exists( full_path ): - """ - Return True if a symbolic link or file exists, but if full_path is a directory, - return False. - """ - if full_path is None: - return False - if os.path.isdir( full_path ): - return False - if os.path.exists( full_path ): - return True - return False - -def create_env_var_dict( elem, install_environment ): - env_var_name = elem.get( 'name', 'PATH' ) - env_var_action = elem.get( 'action', 'prepend_to' ) - env_var_text = None - tool_dependency_install_dir = install_environment.install_dir - tool_shed_repository_install_dir = install_environment.tool_shed_repository_install_dir - if elem.text and elem.text.find( 'REPOSITORY_INSTALL_DIR' ) >= 0: - if tool_shed_repository_install_dir and elem.text.find( '$REPOSITORY_INSTALL_DIR' ) != -1: - env_var_text = elem.text.replace( '$REPOSITORY_INSTALL_DIR', tool_shed_repository_install_dir ) - return dict( name=env_var_name, action=env_var_action, value=env_var_text ) - else: - env_var_text = elem.text.replace( '$REPOSITORY_INSTALL_DIR', tool_dependency_install_dir ) - return dict( name=env_var_name, action=env_var_action, value=env_var_text ) - if elem.text and elem.text.find( 'INSTALL_DIR' ) >= 0: - if tool_dependency_install_dir: - env_var_text = elem.text.replace( '$INSTALL_DIR', tool_dependency_install_dir ) - return dict( name=env_var_name, action=env_var_action, value=env_var_text ) - else: - env_var_text = elem.text.replace( '$INSTALL_DIR', tool_shed_repository_install_dir ) - return dict( name=env_var_name, action=env_var_action, value=env_var_text ) - if elem.text: - # Allow for environment variables that contain neither REPOSITORY_INSTALL_DIR nor INSTALL_DIR - # since there may be command line parameters that are tuned for a Galaxy instance. Allowing them - # to be set in one location rather than being hard coded into each tool config is the best approach. - # For example: - # <environment_variable name="GATK2_SITE_OPTIONS" action="set_to"> - # "--num_threads 4 --num_cpu_threads_per_data_thread 3 --phone_home STANDARD" - # </environment_variable> - return dict( name=env_var_name, action=env_var_action, value=elem.text) - return None - -def download_binary( url, work_dir ): - """Download a pre-compiled binary from the specified URL.""" - downloaded_filename = os.path.split( url )[ -1 ] - dir = url_download( work_dir, downloaded_filename, url, extract=False ) - return downloaded_filename - -def egrep_escape( text ): - """Escape ``text`` to allow literal matching using egrep.""" - regex = re.escape( text ) - # Seems like double escaping is needed for \ - regex = regex.replace( '\\\\', '\\\\\\' ) - # Triple-escaping seems to be required for $ signs - regex = regex.replace( r'\$', r'\\\$' ) - # Whereas single quotes should not be escaped - regex = regex.replace( r"\'", "'" ) - return regex - -def evaluate_template( text, install_environment ): - """ - Substitute variables defined in XML blocks from dependencies file. The value of the received - repository_install_dir is the root installation directory of the repository that contains the - tool dependency. The value of the received install_dir is the root installation directory of - the tool_dependency. - """ - return Template( text ).safe_substitute( get_env_var_values( install_environment ) ) - -def format_traceback(): - ex_type, ex, tb = sys.exc_info() - return ''.join( traceback.format_tb( tb ) ) - -def get_env_shell_file_path( installation_directory ): - env_shell_file_name = 'env.sh' - default_location = os.path.abspath( os.path.join( installation_directory, env_shell_file_name ) ) - if os.path.exists( default_location ): - return default_location - for root, dirs, files in os.walk( installation_directory ): - for name in files: - if name == env_shell_file_name: - return os.path.abspath( os.path.join( root, name ) ) - return None - -def get_env_shell_file_paths( app, elem ): - # Currently only the following tag set is supported. - # <repository toolshed="http://localhost:9009/" name="package_numpy_1_7" owner="test" changeset_revision="c84c6a8be056"> - # <package name="numpy" version="1.7.1" /> - # </repository> - env_shell_file_paths = [] - toolshed = elem.get( 'toolshed', None ) - repository_name = elem.get( 'name', None ) - repository_owner = elem.get( 'owner', None ) - changeset_revision = elem.get( 'changeset_revision', None ) - if toolshed and repository_name and repository_owner and changeset_revision: - # The protocol is not stored, but the port is if it exists. - toolshed = common_util.remove_protocol_from_tool_shed_url( toolshed ) - repository = suc.get_repository_for_dependency_relationship( app, toolshed, repository_name, repository_owner, changeset_revision ) - if repository: - for sub_elem in elem: - tool_dependency_type = sub_elem.tag - tool_dependency_name = sub_elem.get( 'name' ) - tool_dependency_version = sub_elem.get( 'version' ) - if tool_dependency_type and tool_dependency_name and tool_dependency_version: - # Get the tool_dependency so we can get its installation directory. - tool_dependency = None - for tool_dependency in repository.tool_dependencies: - if tool_dependency.type == tool_dependency_type and \ - tool_dependency.name == tool_dependency_name and \ - tool_dependency.version == tool_dependency_version: - break - if tool_dependency: - tool_dependency_key = '%s/%s' % ( tool_dependency_name, tool_dependency_version ) - installation_directory = tool_dependency.installation_directory( app ) - env_shell_file_path = get_env_shell_file_path( installation_directory ) - if env_shell_file_path: - env_shell_file_paths.append( env_shell_file_path ) - else: - error_message = "Skipping tool dependency definition because unable to locate env.sh file for tool dependency " - error_message += "type %s, name %s, version %s for repository %s" % \ - ( str( tool_dependency_type ), str( tool_dependency_name ), str( tool_dependency_version ), str( repository.name ) ) - log.debug( error_message ) - continue - else: - error_message = "Skipping tool dependency definition because unable to locate tool dependency " - error_message += "type %s, name %s, version %s for repository %s" % \ - ( str( tool_dependency_type ), str( tool_dependency_name ), str( tool_dependency_version ), str( repository.name ) ) - log.debug( error_message ) - continue - else: - error_message = "Skipping invalid tool dependency definition: type %s, name %s, version %s." % \ - ( str( tool_dependency_type ), str( tool_dependency_name ), str( tool_dependency_version ) ) - log.debug( error_message ) - continue - else: - error_message = "Skipping set_environment_for_install definition because unable to locate required installed tool shed repository: " - error_message += "toolshed %s, name %s, owner %s, changeset_revision %s." % \ - ( str( toolshed ), str( repository_name ), str( repository_owner ), str( changeset_revision ) ) - log.debug( error_message ) - else: - error_message = "Skipping invalid set_environment_for_install definition: toolshed %s, name %s, owner %s, changeset_revision %s." % \ - ( str( toolshed ), str( repository_name ), str( repository_owner ), str( changeset_revision ) ) - log.debug( error_message ) - return env_shell_file_paths - -def get_env_shell_file_paths_from_setup_environment_elem( app, all_env_shell_file_paths, elem, action_dict ): - """ - Parse an XML tag set to discover all child repository dependency tags and define the path to an env.sh file associated - with the repository (this requires the repository dependency to be in an installed state). The received action_dict - will be updated with these discovered paths and returned to the caller. This method handles tool dependency definition - tag sets <setup_r_environment>, <setup_ruby_environment> and <setup_perl_environment>. - """ - # An example elem is: - # <action type="setup_perl_environment"> - # <repository name="package_perl_5_18" owner="iuc"> - # <package name="perl" version="5.18.1" /> - # </repository> - # <repository name="package_expat_2_1" owner="iuc" prior_installation_required="True"> - # <package name="expat" version="2.1.0" /> - # </repository> - # <package>http://search.cpan.org/CPAN/authors/id/T/TO/TODDR/XML-Parser-2.41.tar.gz</package> - # <package>http://search.cpan.org/CPAN/authors/id/L/LD/LDS/CGI.pm-3.43.tar.gz</package> - # </action> - for action_elem in elem: - if action_elem.tag == 'repository': - env_shell_file_paths = get_env_shell_file_paths( app, action_elem ) - all_env_shell_file_paths.extend( env_shell_file_paths ) - if all_env_shell_file_paths: - action_dict[ 'env_shell_file_paths' ] = all_env_shell_file_paths - action_dict[ 'action_shell_file_paths' ] = env_shell_file_paths - return action_dict - -def get_env_var_values( install_environment ): - """ - Return a dictionary of values, some of which enable substitution of reserved words for the values. - The received install_enviroment object has 2 important attributes for reserved word substitution: - install_environment.tool_shed_repository_install_dir is the root installation directory of the repository - that contains the tool dependency being installed, and install_environment.install_dir is the root - installation directory of the tool dependency. - """ - env_var_dict = {} - env_var_dict[ 'REPOSITORY_INSTALL_DIR' ] = install_environment.tool_shed_repository_install_dir - env_var_dict[ 'INSTALL_DIR' ] = install_environment.install_dir - env_var_dict[ 'system_install' ] = install_environment.install_dir - # If the Python interpreter is 64bit then we can safely assume that the underlying system is also 64bit. - env_var_dict[ '__is64bit__' ] = sys.maxsize > 2**32 - return env_var_dict - -def isbz2( file_path ): - return checkers.is_bz2( file_path ) - -def isgzip( file_path ): - return checkers.is_gzip( file_path ) - -def isjar( file_path ): - return iszip( file_path ) and file_path.endswith( '.jar' ) - -def istar( file_path ): - return tarfile.is_tarfile( file_path ) - -def iszip( file_path ): - return checkers.check_zip( file_path ) - -def is_compressed( file_path ): - if isjar( file_path ): - return False - else: - return iszip( file_path ) or isgzip( file_path ) or istar( file_path ) or isbz2( file_path ) - -def make_directory( full_path ): - if not os.path.exists( full_path ): - os.makedirs( full_path ) - -def move_directory_files( current_dir, source_dir, destination_dir ): - source_directory = os.path.abspath( os.path.join( current_dir, source_dir ) ) - destination_directory = os.path.join( destination_dir ) - if not os.path.isdir( destination_directory ): - os.makedirs( destination_directory ) - symlinks = [] - regular_files = [] - for file_name in os.listdir( source_directory ): - source_file = os.path.join( source_directory, file_name ) - destination_file = os.path.join( destination_directory, file_name ) - files_tuple = ( source_file, destination_file ) - if os.path.islink( source_file ): - symlinks.append( files_tuple ) - else: - regular_files.append( files_tuple ) - for source_file, destination_file in symlinks: - shutil.move( source_file, destination_file ) - for source_file, destination_file in regular_files: - shutil.move( source_file, destination_file ) - -def move_file( current_dir, source, destination, rename_to=None ): - source_path = os.path.abspath( os.path.join( current_dir, source ) ) - source_file = os.path.basename( source_path ) - if rename_to is not None: - destination_file = rename_to - destination_directory = os.path.join( destination ) - destination_path = os.path.join( destination_directory, destination_file ) - else: - destination_directory = os.path.join( destination ) - destination_path = os.path.join( destination_directory, source_file ) - if not os.path.exists( destination_directory ): - os.makedirs( destination_directory ) - shutil.move( source_path, destination_path ) - -def parse_package_elem( package_elem, platform_info_dict=None, include_after_install_actions=True ): - """ - Parse a <package> element within a tool dependency definition and return a list of action tuples. - This method is called when setting metadata on a repository that includes a tool_dependencies.xml - file or when installing a repository that includes a tool_dependencies.xml file. If installing, - platform_info_dict must be a valid dictionary and include_after_install_actions must be True. - """ - # The actions_elem_tuples list contains <actions> tag sets (possibly inside of an <actions_group> - # tag set) to be processed in the order they are defined in the tool_dependencies.xml file. - actions_elem_tuples = [] - # The tag sets that will go into the actions_elem_list are those that install a compiled binary if - # the architecture and operating system match its defined attributes. If compiled binary is not - # installed, the first <actions> tag set [following those that have the os and architecture attributes] - # that does not have os or architecture attributes will be processed. This tag set must contain the - # recipe for downloading and compiling source. - actions_elem_list = [] - for elem in package_elem: - if elem.tag == 'actions': - # We have an <actions> tag that should not be matched against a specific combination of - # architecture and operating system. - in_actions_group = False - actions_elem_tuples.append( ( in_actions_group, elem ) ) - elif elem.tag == 'actions_group': - # We have an actions_group element, and its child <actions> elements should therefore be compared - # with the current operating system - # and processor architecture. - in_actions_group = True - # Record the number of <actions> elements so we can filter out any <action> elements that precede - # <actions> elements. - actions_elem_count = len( elem.findall( 'actions' ) ) - # Record the number of <actions> elements that have both architecture and os specified, in order - # to filter out any platform-independent <actions> elements that come before platform-specific - # <actions> elements. - platform_actions_elements = [] - for actions_elem in elem.findall( 'actions' ): - if actions_elem.get( 'architecture' ) is not None and actions_elem.get( 'os' ) is not None: - platform_actions_elements.append( actions_elem ) - platform_actions_element_count = len( platform_actions_elements ) - platform_actions_elements_processed = 0 - actions_elems_processed = 0 - # The tag sets that will go into the after_install_actions list are <action> tags instead of <actions> - # tags. These will be processed only if they are at the very end of the <actions_group> tag set (after - # all <actions> tag sets). See below for details. - after_install_actions = [] - # Inspect the <actions_group> element and build the actions_elem_list and the after_install_actions list. - for child_element in elem: - if child_element.tag == 'actions': - actions_elems_processed += 1 - system = child_element.get( 'os' ) - architecture = child_element.get( 'architecture' ) - # Skip <actions> tags that have only one of architecture or os specified, in order for the - # count in platform_actions_elements_processed to remain accurate. - if ( system and not architecture ) or ( architecture and not system ): - log.debug( 'Error: Both architecture and os attributes must be specified in an <actions> tag.' ) - continue - # Since we are inside an <actions_group> tag set, compare it with our current platform information - # and filter the <actions> tag sets that don't match. Require both the os and architecture attributes - # to be defined in order to find a match. - if system and architecture: - platform_actions_elements_processed += 1 - # If either the os or architecture do not match the platform, this <actions> tag will not be - # considered a match. Skip it and proceed with checking the next one. - if platform_info_dict: - if platform_info_dict[ 'os' ] != system or platform_info_dict[ 'architecture' ] != architecture: - continue - else: - # We must not be installing a repository into Galaxy, so determining if we can install a - # binary is not necessary. - continue - else: - # <actions> tags without both os and architecture attributes are only allowed to be specified - # after platform-specific <actions> tags. If we find a platform-independent <actions> tag before - # all platform-specific <actions> tags have been processed. - if platform_actions_elements_processed < platform_actions_element_count: - debug_msg = 'Error: <actions> tags without os and architecture attributes are only allowed ' - debug_msg += 'after all <actions> tags with os and architecture attributes have been defined. ' - debug_msg += 'Skipping the <actions> tag set with no os or architecture attributes that has ' - debug_msg += 'been defined between two <actions> tag sets that have these attributes defined. ' - log.debug( debug_msg ) - continue - # If we reach this point, it means one of two things: 1) The system and architecture attributes are - # not defined in this <actions> tag, or 2) The system and architecture attributes are defined, and - # they are an exact match for the current platform. Append the child element to the list of elements - # to process. - actions_elem_list.append( child_element ) - elif child_element.tag == 'action': - # Any <action> tags within an <actions_group> tag set must come after all <actions> tags. - if actions_elems_processed == actions_elem_count: - # If all <actions> elements have been processed, then this <action> element can be appended to the - # list of actions to execute within this group. - after_install_actions.append( child_element ) - else: - # If any <actions> elements remain to be processed, then log a message stating that <action> - # elements are not allowed to precede any <actions> elements within an <actions_group> tag set. - debug_msg = 'Error: <action> tags are only allowed at the end of an <actions_group> tag set after ' - debug_msg += 'all <actions> tags. Skipping <%s> element with type %s.' % \ - ( child_element.tag, child_element.get( 'type', 'unknown' ) ) - log.debug( debug_msg ) - continue - if platform_info_dict is None and not include_after_install_actions: - # We must be setting metadata on a repository. - if len( actions_elem_list ) >= 1: - actions_elem_tuples.append( ( in_actions_group, actions_elem_list[ 0 ] ) ) - else: - # We are processing a recipe that contains only an <actions_group> tag set for installing a binary, - # but does not include an additional recipe for installing and compiling from source. - actions_elem_tuples.append( ( in_actions_group, [] ) ) - elif platform_info_dict is not None and include_after_install_actions: - # We must be installing a repository. - if after_install_actions: - actions_elem_list.extend( after_install_actions ) - actions_elem_tuples.append( ( in_actions_group, actions_elem_list ) ) - else: - # Skip any element that is not <actions> or <actions_group> - this will skip comments, <repository> tags - # and <readme> tags. - in_actions_group = False - continue - return actions_elem_tuples - -def __shellquote( s ): - """Quote and escape the supplied string for use in shell expressions.""" - return "'" + s.replace( "'", "'\\''" ) + "'" - -def url_download( install_dir, downloaded_file_name, download_url, extract=True ): - file_path = os.path.join( install_dir, downloaded_file_name ) - src = None - dst = None - # Set a timer so we don't sit here forever. - start_time = time.time() - try: - src = urllib2.urlopen( download_url ) - dst = open( file_path, 'wb' ) - while True: - chunk = src.read( suc.CHUNK_SIZE ) - if chunk: - dst.write( chunk ) - else: - break - time_taken = time.time() - start_time - if time_taken > NO_OUTPUT_TIMEOUT: - err_msg = 'Downloading from URL %s took longer than the defined timeout period of %.1f seconds.' % \ - ( str( download_url ), NO_OUTPUT_TIMEOUT ) - raise Exception( err_msg ) - except Exception, e: - err_msg = err_msg = 'Error downloading from URL\n%s:\n%s' % ( str( download_url ), str( e ) ) - raise Exception( err_msg ) - finally: - if src: - src.close() - if dst: - dst.close() - if extract: - if istar( file_path ) or ( iszip( file_path ) and not isjar( file_path ) ): - archive = CompressedFile( file_path ) - extraction_path = archive.extract( install_dir ) - else: - extraction_path = os.path.abspath( install_dir ) - else: - extraction_path = os.path.abspath( install_dir ) - return extraction_path - -def zipfile_ok( path_to_archive ): - """ - This function is a bit pedantic and not functionally necessary. It checks whether there is no file pointing outside of the extraction, - because ZipFile.extractall() has some potential security holes. See python zipfile documentation for more details. - """ - basename = os.path.realpath( os.path.dirname( path_to_archive ) ) - zip_archive = zipfile.ZipFile( path_to_archive ) - for member in zip_archive.namelist(): - member_path = os.path.realpath( os.path.join( basename, member ) ) - if not member_path.startswith( basename ): - return False - return True diff -r 31740dffaf9ae4e345fd2d4de4ab44ed133dd567 -r b1b1e5cefca5b0193c8623a9aee7a39cd1dda55c lib/tool_shed/scripts/check_filesystem_for_empty_tool_dependency_installation_paths.py --- a/lib/tool_shed/scripts/check_filesystem_for_empty_tool_dependency_installation_paths.py +++ b/lib/tool_shed/scripts/check_filesystem_for_empty_tool_dependency_installation_paths.py @@ -7,7 +7,7 @@ new_path.extend( sys.path[1:] ) sys.path = new_path -from tool_shed.galaxy_install.tool_dependencies import td_common_util +from tool_shed.util.basic_util import INSTALLATION_LOG def main( args ): empty_installation_paths = [] @@ -31,13 +31,13 @@ no_files = False if len( dirs ) == 0: no_dirs = True - if len( files ) == 0 or len( files ) == 1 and td_common_util.INSTALLATION_LOG in files: + if len( files ) == 0 or len( files ) == 1 and INSTALLATION_LOG in files: no_files = True if no_files and no_dirs and root not in empty_installation_paths: empty_installation_paths.append( root ) if len( empty_installation_paths ) > 0: print 'The following %d tool dependency installation directories were found to be empty or contain only the file %s.' % \ - ( len( empty_installation_paths ), td_common_util.INSTALLATION_LOG ) + ( len( empty_installation_paths ), INSTALLATION_LOG ) if args.delete: for path in empty_installation_paths: if os.path.exists( path ): diff -r 31740dffaf9ae4e345fd2d4de4ab44ed133dd567 -r b1b1e5cefca5b0193c8623a9aee7a39cd1dda55c lib/tool_shed/scripts/check_s3_for_empty_tool_dependency_installation_paths.py --- a/lib/tool_shed/scripts/check_s3_for_empty_tool_dependency_installation_paths.py +++ b/lib/tool_shed/scripts/check_s3_for_empty_tool_dependency_installation_paths.py @@ -12,8 +12,7 @@ import boto -from tool_shed.galaxy_install.tool_dependencies import td_common_util - +from tool_shed.util.basic_util import INSTALLATION_LOG class BucketList( object ): @@ -77,7 +76,7 @@ # This would not be the case in a Galaxy instance, since the Galaxy admin will need to verify the contents of # the installation path in order to determine which action should be taken. elif len( tool_dependency_path_contents ) == 2 and \ - tool_dependency_path_contents[1].name.endswith( td_common_util.INSTALLATION_LOG ): + tool_dependency_path_contents[1].name.endswith( INSTALLATION_LOG ): empty_directories.append( tool_dependency_path_contents[ 0 ] ) return [ item.name for item in empty_directories ] @@ -106,7 +105,7 @@ print 'No empty installation paths found, exiting.' return 0 print 'The following %d tool dependency installation paths were found to be empty or contain only the file %s.' % \ - ( len( dependency_cleaner.empty_installation_paths ), td_common_util.INSTALLATION_LOG ) + ( len( dependency_cleaner.empty_installation_paths ), INSTALLATION_LOG ) if asbool( args.delete ): dependency_cleaner.delete_empty_installation_paths() else: diff -r 31740dffaf9ae4e345fd2d4de4ab44ed133dd567 -r b1b1e5cefca5b0193c8623a9aee7a39cd1dda55c lib/tool_shed/util/basic_util.py --- a/lib/tool_shed/util/basic_util.py +++ b/lib/tool_shed/util/basic_util.py @@ -1,5 +1,8 @@ import logging import os +import shutil +import sys +from string import Template from galaxy.util import unicodify @@ -10,8 +13,52 @@ log = logging.getLogger( __name__ ) +CHUNK_SIZE = 2**20 # 1Mb +INSTALLATION_LOG = 'INSTALLATION.log' +# Set no activity timeout to 20 minutes. +NO_OUTPUT_TIMEOUT = 1200.0 +MAXDIFFSIZE = 8000 MAX_DISPLAY_SIZE = 32768 +def evaluate_template( text, install_environment ): + """ + Substitute variables defined in XML blocks from dependencies file. The value of the received + repository_install_dir is the root installation directory of the repository that contains the + tool dependency. The value of the received install_dir is the root installation directory of + the tool_dependency. + """ + return Template( text ).safe_substitute( get_env_var_values( install_environment ) ) + +def get_env_var_values( install_environment ): + """ + Return a dictionary of values, some of which enable substitution of reserved words for the values. + The received install_enviroment object has 2 important attributes for reserved word substitution: + install_environment.tool_shed_repository_install_dir is the root installation directory of the repository + that contains the tool dependency being installed, and install_environment.install_dir is the root + installation directory of the tool dependency. + """ + env_var_dict = {} + env_var_dict[ 'REPOSITORY_INSTALL_DIR' ] = install_environment.tool_shed_repository_install_dir + env_var_dict[ 'INSTALL_DIR' ] = install_environment.install_dir + env_var_dict[ 'system_install' ] = install_environment.install_dir + # If the Python interpreter is 64bit then we can safely assume that the underlying system is also 64bit. + env_var_dict[ '__is64bit__' ] = sys.maxsize > 2**32 + return env_var_dict + +def move_file( current_dir, source, destination, rename_to=None ): + source_path = os.path.abspath( os.path.join( current_dir, source ) ) + source_file = os.path.basename( source_path ) + if rename_to is not None: + destination_file = rename_to + destination_directory = os.path.join( destination ) + destination_path = os.path.join( destination_directory, destination_file ) + else: + destination_directory = os.path.join( destination ) + destination_path = os.path.join( destination_directory, source_file ) + if not os.path.exists( destination_directory ): + os.makedirs( destination_directory ) + shutil.move( source_path, destination_path ) + def remove_dir( dir ): """Attempt to remove a directory from disk.""" if dir: diff -r 31740dffaf9ae4e345fd2d4de4ab44ed133dd567 -r b1b1e5cefca5b0193c8623a9aee7a39cd1dda55c lib/tool_shed/util/commit_util.py --- a/lib/tool_shed/util/commit_util.py +++ b/lib/tool_shed/util/commit_util.py @@ -12,6 +12,7 @@ from galaxy.util.odict import odict from galaxy.web import url_for import tool_shed.util.shed_util_common as suc +from tool_shed.util import basic_util from tool_shed.util import hg_util from tool_shed.util import tool_util from tool_shed.util import xml_util @@ -126,7 +127,7 @@ bzipped_file = bz2.BZ2File( uploaded_file_name, 'rb' ) while 1: try: - chunk = bzipped_file.read( suc.CHUNK_SIZE ) + chunk = bzipped_file.read( basic_util.CHUNK_SIZE ) except IOError: os.close( fd ) os.remove( uncompressed ) @@ -239,7 +240,7 @@ gzipped_file = gzip.GzipFile( uploaded_file_name, 'rb' ) while 1: try: - chunk = gzipped_file.read( suc.CHUNK_SIZE ) + chunk = gzipped_file.read( basic_util.CHUNK_SIZE ) except IOError, e: os.close( fd ) os.remove( uncompressed ) diff -r 31740dffaf9ae4e345fd2d4de4ab44ed133dd567 -r b1b1e5cefca5b0193c8623a9aee7a39cd1dda55c lib/tool_shed/util/metadata_util.py --- a/lib/tool_shed/util/metadata_util.py +++ b/lib/tool_shed/util/metadata_util.py @@ -22,11 +22,12 @@ from tool_shed.util import tool_dependency_util from tool_shed.util import tool_util from tool_shed.util import xml_util -from tool_shed.galaxy_install.tool_dependencies import td_common_util import tool_shed.repository_types.util as rt_util log = logging.getLogger( __name__ ) +REPOSITORY_DATA_MANAGER_CONFIG_FILENAME = 'data_manager_conf.xml' + # Repository metadata comparisons for changeset revisions. EQUAL = 'equal' NO_METADATA = 'no metadata' @@ -37,7 +38,7 @@ NOT_TOOL_CONFIGS = [ suc.DATATYPES_CONFIG_FILENAME, rt_util.REPOSITORY_DEPENDENCY_DEFINITION_FILENAME, rt_util.TOOL_DEPENDENCY_DEFINITION_FILENAME, - suc.REPOSITORY_DATA_MANAGER_CONFIG_FILENAME ] + REPOSITORY_DATA_MANAGER_CONFIG_FILENAME ] def add_tool_versions( trans, id, repository_metadata, changeset_revisions ): # Build a dictionary of { 'tool id' : 'parent tool id' } pairs for each tool in repository_metadata. @@ -750,7 +751,7 @@ metadata_dict = generate_data_manager_metadata( app, repository, files_dir, - hg_util.get_config_from_disk( suc.REPOSITORY_DATA_MANAGER_CONFIG_FILENAME, files_dir ), + hg_util.get_config_from_disk( REPOSITORY_DATA_MANAGER_CONFIG_FILENAME, files_dir ), metadata_dict, shed_config_dict=shed_config_dict ) @@ -809,10 +810,10 @@ if package_install_version == '1.0': # Complex repository dependencies can be defined within the last <actions> tag set contained in an # <actions_group> tag set. Comments, <repository> tag sets and <readme> tag sets will be skipped - # in td_common_util.parse_package_elem(). - actions_elem_tuples = td_common_util.parse_package_elem( sub_elem, - platform_info_dict=None, - include_after_install_actions=False ) + # in tool_dependency_util.parse_package_elem(). + actions_elem_tuples = tool_dependency_util.parse_package_elem( sub_elem, + platform_info_dict=None, + include_after_install_actions=False ) if actions_elem_tuples: # We now have a list of a single tuple that looks something like: # [(True, <Element 'actions' at 0x104017850>)] diff -r 31740dffaf9ae4e345fd2d4de4ab44ed133dd567 -r b1b1e5cefca5b0193c8623a9aee7a39cd1dda55c lib/tool_shed/util/shed_util_common.py --- a/lib/tool_shed/util/shed_util_common.py +++ b/lib/tool_shed/util/shed_util_common.py @@ -24,11 +24,8 @@ log = logging.getLogger( __name__ ) -CHUNK_SIZE = 2**20 # 1Mb MAX_CONTENT_SIZE = 1048576 -MAXDIFFSIZE = 8000 DATATYPES_CONFIG_FILENAME = 'datatypes_conf.xml' -REPOSITORY_DATA_MANAGER_CONFIG_FILENAME = 'data_manager_conf.xml' new_repo_email_alert_template = """ Sharable link: ${sharable_link} This diff is so big that we needed to truncate the remainder. Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: carlfeberhard: HDA view: properly compile underscore template on first load, override template fn to pass localizer (_l) into template fn
by commits-noreply＠bitbucket.org 28 May '14

28 May '14

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/31740dffaf9a/ Changeset: 31740dffaf9a User: carlfeberhard Date: 2014-05-28 19:41:07 Summary: HDA view: properly compile underscore template on first load, override template fn to pass localizer (_l) into template fn Affected #: 3 files diff -r 7fd77febf6bb8aa7a6acde9c3a31429f7b1ee99f -r 31740dffaf9ae4e345fd2d4de4ab44ed133dd567 static/scripts/mvc/dataset/hda-base.js --- a/static/scripts/mvc/dataset/hda-base.js +++ b/static/scripts/mvc/dataset/hda-base.js @@ -652,7 +652,8 @@ }); //------------------------------------------------------------------------------ TEMPLATES -var skeletonTemplate = [ +//TODO: possibly break these out into a sep. module +var skeletonTemplate = _.template([ '<div class="dataset hda">', '<div class="dataset-warnings">', // error during index fetch - show error on dataset @@ -704,9 +705,9 @@ '<div class="dataset-body"></div>', '</div>' -].join( '' ); +].join( '' )); -var bodyTemplate = [ +var bodyTemplate = _.template([ '<div class="dataset-body">', '<% if( hda.body ){ %>', '<div class="dataset-summary">', @@ -799,14 +800,15 @@ '<% } %>', // end if body '</div>' -].join( '' ); +].join( '' )); HDABaseView.templates = { + // we override here in order to pass the localizer (_L) into the template scope - since we use it as a fn within skeleton : function( hdaJSON ){ - return _.template( skeletonTemplate, hdaJSON, { variable: 'hda' }); + return skeletonTemplate({ _l: _l, hda: hdaJSON }); }, body : function( hdaJSON ){ - return _.template( bodyTemplate, hdaJSON, { variable: 'hda' }); + return bodyTemplate({ _l: _l, hda: hdaJSON }); } }; diff -r 7fd77febf6bb8aa7a6acde9c3a31429f7b1ee99f -r 31740dffaf9ae4e345fd2d4de4ab44ed133dd567 static/scripts/packed/mvc/dataset/hda-base.js --- a/static/scripts/packed/mvc/dataset/hda-base.js +++ b/static/scripts/packed/mvc/dataset/hda-base.js @@ -1,1 +1,1 @@ -define(["mvc/dataset/hda-model","mvc/base-mvc","utils/localization"],function(e,b,d){var g=Backbone.View.extend(b.LoggableMixin).extend({tagName:"div",fxSpeed:"fast",_queueNewRender:function(i,j){j=(j===undefined)?(true):(j);var h=this;if(j){$(h).queue(function(k){this.$el.fadeOut(h.fxSpeed,k)})}$(h).queue(function(k){this.$el.empty().attr("class",h.className).addClass("state-"+h.model.get("state")).append(i.children());if(this.selectable){this.showSelector(0)}k()});if(j){$(h).queue(function(k){this.$el.fadeIn(h.fxSpeed,k)})}$(h).queue(function(k){this.trigger("rendered",h);if(this.model.inReadyState()){this.trigger("rendered:ready",h)}if(this.draggable){this.draggableOn()}k()})},toggleBodyVisibility:function(k,i){var h=32,j=13;if(k&&(k.type==="keydown")&&!(k.keyCode===h||k.keyCode===j)){return true}var l=this.$el.find(".dataset-body");i=(i===undefined)?(!l.is(":visible")):(i);if(i){this.expandBody()}else{this.collapseBody()}return false},showSelector:function(){if(this.selected){this.select(null,true)}this.selectable=true;this.trigger("selectable",true,this);this.$(".dataset-primary-actions").hide();this.$(".dataset-selector").show()},hideSelector:function(){this.selectable=false;this.trigger("selectable",false,this);this.$(".dataset-selector").hide();this.$(".dataset-primary-actions").show()},toggleSelector:function(){if(!this.$el.find(".dataset-selector").is(":visible")){this.showSelector()}else{this.hideSelector()}},select:function(h){this.$el.find(".dataset-selector span").removeClass("fa-square-o").addClass("fa-check-square-o");if(!this.selected){this.trigger("selected",this);this.selected=true}return false},deselect:function(h){this.$el.find(".dataset-selector span").removeClass("fa-check-square-o").addClass("fa-square-o");if(this.selected){this.trigger("de-selected",this);this.selected=false}return false},toggleSelect:function(h){if(this.selected){this.deselect(h)}else{this.select(h)}},});var c=g.extend({className:"dataset hda history-panel-hda",id:function(){return"hda-"+this.model.get("id")},initialize:function(h){if(h.logger){this.logger=this.model.logger=h.logger}this.log(this+".initialize:",h);this.defaultPrimaryActionButtonRenderers=[this._render_showParamsButton];this.linkTarget=h.linkTarget||"_blank";this.selectable=h.selectable||false;this.selected=h.selected||false;this.expanded=h.expanded||false;this.draggable=h.draggable||false;this._setUpListeners()},_setUpListeners:function(){this.model.on("change",function(i,h){if(this.model.changedAttributes().state&&this.model.inReadyState()&&this.expanded&&!this.model.hasDetails()){this.model.fetch()}else{this.render()}},this)},render:function(i){this.$el.find("[title]").tooltip("destroy");this.urls=this.model.urls();var h=this._buildNewRender();this._queueNewRender(h,i);return this},_buildNewRender:function(){var h=$(c.templates.skeleton(this.model.toJSON()));h.find(".dataset-primary-actions").append(this._render_titleButtons());h.children(".dataset-body").replaceWith(this._render_body());this._setUpBehaviors(h);return h},_setUpBehaviors:function(h){h=h||this.$el;make_popup_menus(h);h.find("[title]").tooltip({placement:"bottom"})},_render_titleButtons:function(){return[this._render_displayButton()]},_render_displayButton:function(){if((this.model.get("state")===e.HistoryDatasetAssociation.STATES.NOT_VIEWABLE)||(this.model.get("state")===e.HistoryDatasetAssociation.STATES.DISCARDED)||(!this.model.get("accessible"))){return null}var i={target:this.linkTarget,classes:"dataset-display"};if(this.model.get("purged")){i.disabled=true;i.title=d("Cannot display datasets removed from disk")}else{if(this.model.get("state")===e.HistoryDatasetAssociation.STATES.UPLOAD){i.disabled=true;i.title=d("This dataset must finish uploading before it can be viewed")}else{if(this.model.get("state")===e.HistoryDatasetAssociation.STATES.NEW){i.disabled=true;i.title=d("This dataset is not yet viewable")}else{i.title=d("View data");i.href=this.urls.display;var h=this;i.onclick=function(j){if(Galaxy.frame&&Galaxy.frame.active){Galaxy.frame.add({title:"Data Viewer: "+h.model.get("name"),type:"url",content:h.urls.display});j.preventDefault()}}}}}i.faIcon="fa-eye";return faIconButton(i)},_render_downloadButton:function(){if(this.model.get("purged")||!this.model.hasData()){return null}var i=this.urls,j=this.model.get("meta_files");if(_.isEmpty(j)){return $(['<a href="'+i.download+'" title="'+d("Download")+'" ','class="icon-btn dataset-download-btn">','<span class="fa fa-floppy-o"></span>',"</a>"].join(""))}var k="dataset-"+this.model.get("id")+"-popup",h=['<div popupmenu="'+k+'">','<a href="'+i.download+'">',d("Download dataset"),"</a>","<a>"+d("Additional files")+"</a>",_.map(j,function(l){return['<a class="action-button" href="',i.meta_download+l.file_type,'">',d("Download")," ",l.file_type,"</a>"].join("")}).join("\n"),"</div>",'<div class="icon-btn-group">','<a href="'+i.download+'" title="'+d("Download")+'" ','class="icon-btn dataset-download-btn">','<span class="fa fa-floppy-o"></span>','</a><a class="icon-btn popup" id="'+k+'">','<span class="fa fa-caret-down"></span>',"</a>","</div>"].join("\n");return $(h)},_render_showParamsButton:function(){return faIconButton({title:d("View details"),classes:"dataset-params-btn",href:this.urls.show_params,target:this.linkTarget,faIcon:"fa-info-circle"})},_render_body:function(){var i=$('<div>Error: unknown dataset state "'+this.model.get("state")+'".</div>'),h=this["_render_body_"+this.model.get("state")];if(_.isFunction(h)){i=h.call(this)}this._setUpBehaviors(i);if(this.expanded){i.show()}return i},_render_stateBodyHelper:function(h,k){k=k||[];var i=this,j=$(c.templates.body(_.extend(this.model.toJSON(),{body:h})));j.find(".dataset-actions .left").append(_.map(k,function(l){return l.call(i)}));return j},_render_body_new:function(){return this._render_stateBodyHelper("<div>"+d("This is a new dataset and not all of its data are available yet")+"</div>",this.defaultPrimaryActionButtonRenderers)},_render_body_noPermission:function(){return this._render_stateBodyHelper("<div>"+d("You do not have permission to view this dataset")+"</div>")},_render_body_discarded:function(){return this._render_stateBodyHelper("<div>"+d("The job creating this dataset was cancelled before completion")+"</div>",this.defaultPrimaryActionButtonRenderers)},_render_body_queued:function(){return this._render_stateBodyHelper("<div>"+d("This job is waiting to run")+"</div>",this.defaultPrimaryActionButtonRenderers)},_render_body_upload:function(){return this._render_stateBodyHelper("<div>"+d("This dataset is currently uploading")+"</div>")},_render_body_setting_metadata:function(){return this._render_stateBodyHelper("<div>"+d("Metadata is being auto-detected")+"</div>")},_render_body_running:function(){return this._render_stateBodyHelper("<div>"+d("This job is currently running")+"</div>",this.defaultPrimaryActionButtonRenderers)},_render_body_paused:function(){return this._render_stateBodyHelper("<div>"+d('This job is paused. Use the "Resume Paused Jobs" in the history menu to resume')+"</div>",this.defaultPrimaryActionButtonRenderers)},_render_body_error:function(){var h=['<span class="help-text">',d("An error occurred with this dataset"),":</span>",'<div class="job-error-text">',$.trim(this.model.get("misc_info")),"</div>"].join("");if(!this.model.get("purged")){h="<div>"+this.model.get("misc_blurb")+"</div>"+h}return this._render_stateBodyHelper(h,[this._render_downloadButton].concat(this.defaultPrimaryActionButtonRenderers))},_render_body_empty:function(){return this._render_stateBodyHelper("<div>"+d("No data")+": <i>"+this.model.get("misc_blurb")+"</i></div>",this.defaultPrimaryActionButtonRenderers)},_render_body_failed_metadata:function(){var h=$('<div class="warningmessagesmall"></div>').append($("<strong/>").text(d("An error occurred setting the metadata for this dataset"))),i=this._render_body_ok();i.prepend(h);return i},_render_body_ok:function(){var h=this,j=$(c.templates.body(this.model.toJSON())),i=[this._render_downloadButton].concat(this.defaultPrimaryActionButtonRenderers);j.find(".dataset-actions .left").append(_.map(i,function(k){return k.call(h)}));if(this.model.isDeletedOrPurged()){return j}return j},events:{"click .dataset-title-bar":"toggleBodyVisibility","keydown .dataset-title-bar":"toggleBodyVisibility","click .dataset-selector":"toggleSelect"},expandBody:function(){var h=this;function i(){h.$el.children(".dataset-body").replaceWith(h._render_body());h.$el.children(".dataset-body").slideDown(h.fxSpeed,function(){h.expanded=true;h.trigger("body-expanded",h.model.get("id"))})}if(this.model.inReadyState()&&!this.model.hasDetails()){this.model.fetch({silent:true}).always(function(j){h.urls=h.model.urls();i()})}else{i()}},collapseBody:function(){var h=this;this.$el.children(".dataset-body").slideUp(h.fxSpeed,function(){h.expanded=false;h.trigger("body-collapsed",h.model.get("id"))})},draggableOn:function(){this.draggable=true;this.dragStartHandler=_.bind(this._dragStartHandler,this);this.dragEndHandler=_.bind(this._dragEndHandler,this);var h=this.$el.find(".dataset-title-bar").attr("draggable",true).get(0);h.addEventListener("dragstart",this.dragStartHandler,false);h.addEventListener("dragend",this.dragEndHandler,false)},draggableOff:function(){this.draggable=false;var h=this.$el.find(".dataset-title-bar").attr("draggable",false).get(0);h.removeEventListener("dragstart",this.dragStartHandler,false);h.removeEventListener("dragend",this.dragEndHandler,false)},toggleDraggable:function(){if(this.draggable){this.draggableOff()}else{this.draggableOn()}},_dragStartHandler:function(h){this.trigger("dragstart",this);h.dataTransfer.effectAllowed="move";h.dataTransfer.setData("text",JSON.stringify(this.model.toJSON()));return false},_dragEndHandler:function(h){this.trigger("dragend",this);return false},remove:function(i){var h=this;this.$el.fadeOut(h.fxSpeed,function(){h.$el.remove();h.off();if(i){i()}})},toString:function(){var h=(this.model)?(this.model+""):("(no model)");return"HDABaseView("+h+")"}});var a=['<div class="dataset hda">','<div class="dataset-warnings">',"<% if( hda.error ){ %>",'<div class="errormessagesmall">',d("There was an error getting the data for this dataset"),":<%- hda.error %>","</div>","<% } %>","<% if( hda.deleted ){ %>","<% if( hda.purged ){ %>",'<div class="dataset-purged-msg warningmessagesmall"><strong>',d("This dataset has been deleted and removed from disk")+".","</strong></div>","<% } else { %>",'<div class="dataset-deleted-msg warningmessagesmall"><strong>',d("This dataset has been deleted")+".","</strong></div>","<% } %>","<% } %>","<% if( !hda.visible ){ %>",'<div class="dataset-hidden-msg warningmessagesmall"><strong>',d("This dataset has been hidden")+".","</strong></div>","<% } %>","</div>",'<div class="dataset-selector">','<span class="fa fa-2x fa-square-o"></span>',"</div>",'<div class="dataset-primary-actions"></div>','<div class="dataset-title-bar clear" tabindex="0">','<span class="dataset-state-icon state-icon"></span>','<div class="dataset-title">','<span class="hda-hid"><%- hda.hid %></span> ','<span class="dataset-name"><%- hda.name %></span>',"</div>","</div>",'<div class="dataset-body"></div>',"</div>"].join("");var f=['<div class="dataset-body">',"<% if( hda.body ){ %>",'<div class="dataset-summary">',"<%= hda.body %>","</div>",'<div class="dataset-actions clear">','<div class="left"></div>','<div class="right"></div>',"</div>","<% } else { %>",'<div class="dataset-summary">',"<% if( hda.misc_blurb ){ %>",'<div class="dataset-blurb">','<span class="value"><%- hda.misc_blurb %></span>',"</div>","<% } %>","<% if( hda.data_type ){ %>",'<div class="dataset-datatype">','<label class="prompt">',d("format"),"</label>",'<span class="value"><%- hda.data_type %></span>',"</div>","<% } %>","<% if( hda.metadata_dbkey ){ %>",'<div class="dataset-dbkey">','<label class="prompt">',d("database"),"</label>",'<span class="value">',"<%- hda.metadata_dbkey %>","</span>","</div>","<% } %>","<% if( hda.misc_info ){ %>",'<div class="dataset-info">','<span class="value"><%- hda.misc_info %></span>',"</div>","<% } %>","</div>",'<div class="dataset-actions clear">','<div class="left"></div>','<div class="right"></div>',"</div>","<% if( !hda.deleted ){ %>",'<div class="tags-display"></div>','<div class="annotation-display"></div>','<div class="dataset-display-applications">',"<% _.each( hda.display_apps, function( app ){ %>",'<div class="display-application">','<span class="display-application-location"><%- app.label %></span> ','<span class="display-application-links">',"<% _.each( app.links, function( link ){ %>",'<a target="<%= link.target %>" href="<%= link.href %>">',"<% print( _l( link.text ) ); %>","</a> ","<% }); %>","</span>","</div>","<% }); %>","<% _.each( hda.display_types, function( app ){ %>",'<div class="display-application">','<span class="display-application-location"><%- app.label %></span> ','<span class="display-application-links">',"<% _.each( app.links, function( link ){ %>",'<a target="<%= link.target %>" href="<%= link.href %>">',"<% print( _l( link.text ) ); %>","</a> ","<% }); %>","</span>","</div>","<% }); %>","</div>",'<div class="dataset-peek">',"<% if( hda.peek ){ %>",'<pre class="peek"><%= hda.peek %></pre>',"<% } %>","</div>","<% } %>","<% } %>","</div>"].join("");c.templates={skeleton:function(h){return _.template(a,h,{variable:"hda"})},body:function(h){return _.template(f,h,{variable:"hda"})}};return{HistoryContentBaseView:g,HDABaseView:c}}); \ No newline at end of file +define(["mvc/dataset/hda-model","mvc/base-mvc","utils/localization"],function(e,b,d){var g=Backbone.View.extend(b.LoggableMixin).extend({tagName:"div",fxSpeed:"fast",_queueNewRender:function(i,j){j=(j===undefined)?(true):(j);var h=this;if(j){$(h).queue(function(k){this.$el.fadeOut(h.fxSpeed,k)})}$(h).queue(function(k){this.$el.empty().attr("class",h.className).addClass("state-"+h.model.get("state")).append(i.children());if(this.selectable){this.showSelector(0)}k()});if(j){$(h).queue(function(k){this.$el.fadeIn(h.fxSpeed,k)})}$(h).queue(function(k){this.trigger("rendered",h);if(this.model.inReadyState()){this.trigger("rendered:ready",h)}if(this.draggable){this.draggableOn()}k()})},toggleBodyVisibility:function(k,i){var h=32,j=13;if(k&&(k.type==="keydown")&&!(k.keyCode===h||k.keyCode===j)){return true}var l=this.$el.find(".dataset-body");i=(i===undefined)?(!l.is(":visible")):(i);if(i){this.expandBody()}else{this.collapseBody()}return false},showSelector:function(){if(this.selected){this.select(null,true)}this.selectable=true;this.trigger("selectable",true,this);this.$(".dataset-primary-actions").hide();this.$(".dataset-selector").show()},hideSelector:function(){this.selectable=false;this.trigger("selectable",false,this);this.$(".dataset-selector").hide();this.$(".dataset-primary-actions").show()},toggleSelector:function(){if(!this.$el.find(".dataset-selector").is(":visible")){this.showSelector()}else{this.hideSelector()}},select:function(h){this.$el.find(".dataset-selector span").removeClass("fa-square-o").addClass("fa-check-square-o");if(!this.selected){this.trigger("selected",this);this.selected=true}return false},deselect:function(h){this.$el.find(".dataset-selector span").removeClass("fa-check-square-o").addClass("fa-square-o");if(this.selected){this.trigger("de-selected",this);this.selected=false}return false},toggleSelect:function(h){if(this.selected){this.deselect(h)}else{this.select(h)}},});var c=g.extend({className:"dataset hda history-panel-hda",id:function(){return"hda-"+this.model.get("id")},initialize:function(h){if(h.logger){this.logger=this.model.logger=h.logger}this.log(this+".initialize:",h);this.defaultPrimaryActionButtonRenderers=[this._render_showParamsButton];this.linkTarget=h.linkTarget||"_blank";this.selectable=h.selectable||false;this.selected=h.selected||false;this.expanded=h.expanded||false;this.draggable=h.draggable||false;this._setUpListeners()},_setUpListeners:function(){this.model.on("change",function(i,h){if(this.model.changedAttributes().state&&this.model.inReadyState()&&this.expanded&&!this.model.hasDetails()){this.model.fetch()}else{this.render()}},this)},render:function(i){this.$el.find("[title]").tooltip("destroy");this.urls=this.model.urls();var h=this._buildNewRender();this._queueNewRender(h,i);return this},_buildNewRender:function(){var h=$(c.templates.skeleton(this.model.toJSON()));h.find(".dataset-primary-actions").append(this._render_titleButtons());h.children(".dataset-body").replaceWith(this._render_body());this._setUpBehaviors(h);return h},_setUpBehaviors:function(h){h=h||this.$el;make_popup_menus(h);h.find("[title]").tooltip({placement:"bottom"})},_render_titleButtons:function(){return[this._render_displayButton()]},_render_displayButton:function(){if((this.model.get("state")===e.HistoryDatasetAssociation.STATES.NOT_VIEWABLE)||(this.model.get("state")===e.HistoryDatasetAssociation.STATES.DISCARDED)||(!this.model.get("accessible"))){return null}var i={target:this.linkTarget,classes:"dataset-display"};if(this.model.get("purged")){i.disabled=true;i.title=d("Cannot display datasets removed from disk")}else{if(this.model.get("state")===e.HistoryDatasetAssociation.STATES.UPLOAD){i.disabled=true;i.title=d("This dataset must finish uploading before it can be viewed")}else{if(this.model.get("state")===e.HistoryDatasetAssociation.STATES.NEW){i.disabled=true;i.title=d("This dataset is not yet viewable")}else{i.title=d("View data");i.href=this.urls.display;var h=this;i.onclick=function(j){if(Galaxy.frame&&Galaxy.frame.active){Galaxy.frame.add({title:"Data Viewer: "+h.model.get("name"),type:"url",content:h.urls.display});j.preventDefault()}}}}}i.faIcon="fa-eye";return faIconButton(i)},_render_downloadButton:function(){if(this.model.get("purged")||!this.model.hasData()){return null}var i=this.urls,j=this.model.get("meta_files");if(_.isEmpty(j)){return $(['<a href="'+i.download+'" title="'+d("Download")+'" ','class="icon-btn dataset-download-btn">','<span class="fa fa-floppy-o"></span>',"</a>"].join(""))}var k="dataset-"+this.model.get("id")+"-popup",h=['<div popupmenu="'+k+'">','<a href="'+i.download+'">',d("Download dataset"),"</a>","<a>"+d("Additional files")+"</a>",_.map(j,function(l){return['<a class="action-button" href="',i.meta_download+l.file_type,'">',d("Download")," ",l.file_type,"</a>"].join("")}).join("\n"),"</div>",'<div class="icon-btn-group">','<a href="'+i.download+'" title="'+d("Download")+'" ','class="icon-btn dataset-download-btn">','<span class="fa fa-floppy-o"></span>','</a><a class="icon-btn popup" id="'+k+'">','<span class="fa fa-caret-down"></span>',"</a>","</div>"].join("\n");return $(h)},_render_showParamsButton:function(){return faIconButton({title:d("View details"),classes:"dataset-params-btn",href:this.urls.show_params,target:this.linkTarget,faIcon:"fa-info-circle"})},_render_body:function(){var i=$('<div>Error: unknown dataset state "'+this.model.get("state")+'".</div>'),h=this["_render_body_"+this.model.get("state")];if(_.isFunction(h)){i=h.call(this)}this._setUpBehaviors(i);if(this.expanded){i.show()}return i},_render_stateBodyHelper:function(h,k){k=k||[];var i=this,j=$(c.templates.body(_.extend(this.model.toJSON(),{body:h})));j.find(".dataset-actions .left").append(_.map(k,function(l){return l.call(i)}));return j},_render_body_new:function(){return this._render_stateBodyHelper("<div>"+d("This is a new dataset and not all of its data are available yet")+"</div>",this.defaultPrimaryActionButtonRenderers)},_render_body_noPermission:function(){return this._render_stateBodyHelper("<div>"+d("You do not have permission to view this dataset")+"</div>")},_render_body_discarded:function(){return this._render_stateBodyHelper("<div>"+d("The job creating this dataset was cancelled before completion")+"</div>",this.defaultPrimaryActionButtonRenderers)},_render_body_queued:function(){return this._render_stateBodyHelper("<div>"+d("This job is waiting to run")+"</div>",this.defaultPrimaryActionButtonRenderers)},_render_body_upload:function(){return this._render_stateBodyHelper("<div>"+d("This dataset is currently uploading")+"</div>")},_render_body_setting_metadata:function(){return this._render_stateBodyHelper("<div>"+d("Metadata is being auto-detected")+"</div>")},_render_body_running:function(){return this._render_stateBodyHelper("<div>"+d("This job is currently running")+"</div>",this.defaultPrimaryActionButtonRenderers)},_render_body_paused:function(){return this._render_stateBodyHelper("<div>"+d('This job is paused. Use the "Resume Paused Jobs" in the history menu to resume')+"</div>",this.defaultPrimaryActionButtonRenderers)},_render_body_error:function(){var h=['<span class="help-text">',d("An error occurred with this dataset"),":</span>",'<div class="job-error-text">',$.trim(this.model.get("misc_info")),"</div>"].join("");if(!this.model.get("purged")){h="<div>"+this.model.get("misc_blurb")+"</div>"+h}return this._render_stateBodyHelper(h,[this._render_downloadButton].concat(this.defaultPrimaryActionButtonRenderers))},_render_body_empty:function(){return this._render_stateBodyHelper("<div>"+d("No data")+": <i>"+this.model.get("misc_blurb")+"</i></div>",this.defaultPrimaryActionButtonRenderers)},_render_body_failed_metadata:function(){var h=$('<div class="warningmessagesmall"></div>').append($("<strong/>").text(d("An error occurred setting the metadata for this dataset"))),i=this._render_body_ok();i.prepend(h);return i},_render_body_ok:function(){var h=this,j=$(c.templates.body(this.model.toJSON())),i=[this._render_downloadButton].concat(this.defaultPrimaryActionButtonRenderers);j.find(".dataset-actions .left").append(_.map(i,function(k){return k.call(h)}));if(this.model.isDeletedOrPurged()){return j}return j},events:{"click .dataset-title-bar":"toggleBodyVisibility","keydown .dataset-title-bar":"toggleBodyVisibility","click .dataset-selector":"toggleSelect"},expandBody:function(){var h=this;function i(){h.$el.children(".dataset-body").replaceWith(h._render_body());h.$el.children(".dataset-body").slideDown(h.fxSpeed,function(){h.expanded=true;h.trigger("body-expanded",h.model.get("id"))})}if(this.model.inReadyState()&&!this.model.hasDetails()){this.model.fetch({silent:true}).always(function(j){h.urls=h.model.urls();i()})}else{i()}},collapseBody:function(){var h=this;this.$el.children(".dataset-body").slideUp(h.fxSpeed,function(){h.expanded=false;h.trigger("body-collapsed",h.model.get("id"))})},draggableOn:function(){this.draggable=true;this.dragStartHandler=_.bind(this._dragStartHandler,this);this.dragEndHandler=_.bind(this._dragEndHandler,this);var h=this.$el.find(".dataset-title-bar").attr("draggable",true).get(0);h.addEventListener("dragstart",this.dragStartHandler,false);h.addEventListener("dragend",this.dragEndHandler,false)},draggableOff:function(){this.draggable=false;var h=this.$el.find(".dataset-title-bar").attr("draggable",false).get(0);h.removeEventListener("dragstart",this.dragStartHandler,false);h.removeEventListener("dragend",this.dragEndHandler,false)},toggleDraggable:function(){if(this.draggable){this.draggableOff()}else{this.draggableOn()}},_dragStartHandler:function(h){this.trigger("dragstart",this);h.dataTransfer.effectAllowed="move";h.dataTransfer.setData("text",JSON.stringify(this.model.toJSON()));return false},_dragEndHandler:function(h){this.trigger("dragend",this);return false},remove:function(i){var h=this;this.$el.fadeOut(h.fxSpeed,function(){h.$el.remove();h.off();if(i){i()}})},toString:function(){var h=(this.model)?(this.model+""):("(no model)");return"HDABaseView("+h+")"}});var a=_.template(['<div class="dataset hda">','<div class="dataset-warnings">',"<% if( hda.error ){ %>",'<div class="errormessagesmall">',d("There was an error getting the data for this dataset"),":<%- hda.error %>","</div>","<% } %>","<% if( hda.deleted ){ %>","<% if( hda.purged ){ %>",'<div class="dataset-purged-msg warningmessagesmall"><strong>',d("This dataset has been deleted and removed from disk")+".","</strong></div>","<% } else { %>",'<div class="dataset-deleted-msg warningmessagesmall"><strong>',d("This dataset has been deleted")+".","</strong></div>","<% } %>","<% } %>","<% if( !hda.visible ){ %>",'<div class="dataset-hidden-msg warningmessagesmall"><strong>',d("This dataset has been hidden")+".","</strong></div>","<% } %>","</div>",'<div class="dataset-selector">','<span class="fa fa-2x fa-square-o"></span>',"</div>",'<div class="dataset-primary-actions"></div>','<div class="dataset-title-bar clear" tabindex="0">','<span class="dataset-state-icon state-icon"></span>','<div class="dataset-title">','<span class="hda-hid"><%- hda.hid %></span> ','<span class="dataset-name"><%- hda.name %></span>',"</div>","</div>",'<div class="dataset-body"></div>',"</div>"].join(""));var f=_.template(['<div class="dataset-body">',"<% if( hda.body ){ %>",'<div class="dataset-summary">',"<%= hda.body %>","</div>",'<div class="dataset-actions clear">','<div class="left"></div>','<div class="right"></div>',"</div>","<% } else { %>",'<div class="dataset-summary">',"<% if( hda.misc_blurb ){ %>",'<div class="dataset-blurb">','<span class="value"><%- hda.misc_blurb %></span>',"</div>","<% } %>","<% if( hda.data_type ){ %>",'<div class="dataset-datatype">','<label class="prompt">',d("format"),"</label>",'<span class="value"><%- hda.data_type %></span>',"</div>","<% } %>","<% if( hda.metadata_dbkey ){ %>",'<div class="dataset-dbkey">','<label class="prompt">',d("database"),"</label>",'<span class="value">',"<%- hda.metadata_dbkey %>","</span>","</div>","<% } %>","<% if( hda.misc_info ){ %>",'<div class="dataset-info">','<span class="value"><%- hda.misc_info %></span>',"</div>","<% } %>","</div>",'<div class="dataset-actions clear">','<div class="left"></div>','<div class="right"></div>',"</div>","<% if( !hda.deleted ){ %>",'<div class="tags-display"></div>','<div class="annotation-display"></div>','<div class="dataset-display-applications">',"<% _.each( hda.display_apps, function( app ){ %>",'<div class="display-application">','<span class="display-application-location"><%- app.label %></span> ','<span class="display-application-links">',"<% _.each( app.links, function( link ){ %>",'<a target="<%= link.target %>" href="<%= link.href %>">',"<% print( _l( link.text ) ); %>","</a> ","<% }); %>","</span>","</div>","<% }); %>","<% _.each( hda.display_types, function( app ){ %>",'<div class="display-application">','<span class="display-application-location"><%- app.label %></span> ','<span class="display-application-links">',"<% _.each( app.links, function( link ){ %>",'<a target="<%= link.target %>" href="<%= link.href %>">',"<% print( _l( link.text ) ); %>","</a> ","<% }); %>","</span>","</div>","<% }); %>","</div>",'<div class="dataset-peek">',"<% if( hda.peek ){ %>",'<pre class="peek"><%= hda.peek %></pre>',"<% } %>","</div>","<% } %>","<% } %>","</div>"].join(""));c.templates={skeleton:function(h){return a({_l:d,hda:h})},body:function(h){return f({_l:d,hda:h})}};return{HistoryContentBaseView:g,HDABaseView:c}}); \ No newline at end of file diff -r 7fd77febf6bb8aa7a6acde9c3a31429f7b1ee99f -r 31740dffaf9ae4e345fd2d4de4ab44ed133dd567 static/scripts/packed/viz/trackster.js --- a/static/scripts/packed/viz/trackster.js +++ b/static/scripts/packed/viz/trackster.js @@ -1,1 +1,1 @@ -var ui=null;var view=null;var browser_router=null;require(["utils/utils","libs/jquery/jstorage","libs/jquery/jquery.event.drag","libs/jquery/jquery.event.hover","libs/jquery/jquery.mousewheel","libs/jquery/jquery-ui","libs/jquery/select2","libs/farbtastic","libs/jquery/jquery.form","libs/jquery/jquery.rating","mvc/ui"],function(a){a.cssLoadFile("static/style/jquery.rating.css");a.cssLoadFile("static/style/autocomplete_tagging.css");a.cssLoadFile("static/style/jquery-ui/smoothness/jquery-ui.css");a.cssLoadFile("static/style/library.css");a.cssLoadFile("static/style/trackster.css")});define(["libs/backbone/backbone","viz/trackster_ui"],function(b,a){var c=Backbone.View.extend({initialize:function(){ui=new a.TracksterUI(galaxy_config.root);ui.createButtonMenu();ui.buttonMenu.$el.attr("style","float: right");$("#center .unified-panel-header-inner").append(ui.buttonMenu.$el);$("#right .unified-panel-title").append("Bookmarks");$("#right .unified-panel-icons").append("<a id='add-bookmark-button' class='icon-button menu-button plus-button' href='javascript:void(0);' title='Add bookmark'></a>");$("#right-border").click(function(){view.resize_window()});force_right_panel("hide");if(galaxy_config.app.id){this.view_existing()}else{this.view_new()}},view_existing:function(){var d=galaxy_config.app.viz_config;view=ui.create_visualization({container:$("#center .unified-panel-body"),name:d.title,vis_id:d.vis_id,dbkey:d.dbkey},d.viewport,d.tracks,d.bookmarks,true);this.init_editor()},view_new:function(){var d=this;$.ajax({url:galaxy_config.root+"api/genomes?chrom_info=True",data:{},error:function(){alert("Couldn't create new browser.")},success:function(e){Galaxy.modal.show({title:"New Visualization",body:d.template_view_new(e),buttons:{Cancel:function(){window.location=galaxy_config.root+"visualization/list"},Create:function(){d.create_browser($("#new-title").val(),$("#new-dbkey").val());Galaxy.modal.hide()}}});if(galaxy_config.app.default_dbkey){$("#new-dbkey").val(galaxy_config.app.default_dbkey)}$("#new-title").focus();$("select[name='dbkey']").select2();$("#overlay").css("overflow","auto")}})},template_view_new:function(d){var f='<form id="new-browser-form" action="javascript:void(0);" method="post" onsubmit="return false;"><div class="form-row"><label for="new-title">Browser name:</label><div class="form-row-input"><input type="text" name="title" id="new-title" value="Unnamed"></input></div><div style="clear: both;"></div></div><div class="form-row"><label for="new-dbkey">Reference genome build (dbkey): </label><div class="form-row-input"><select name="dbkey" id="new-dbkey">';for(var e=0;e<d.length;e++){f+='<option value="'+d[e][1]+'">'+d[e][0]+"</option>"}f+='</select></div><div style="clear: both;"></div></div><div class="form-row">Is the build not listed here? <a href="'+galaxy_config.root+'user/dbkeys?use_panels=True">Add a Custom Build</a></div></form>';return f},create_browser:function(e,d){$(document).trigger("convert_to_values");view=ui.create_visualization({container:$("#center .unified-panel-body"),name:e,dbkey:d},galaxy_config.app.gene_region);this.init_editor();view.editor=true},init_editor:function(){$("#center .unified-panel-title").text(view.config.get_value("name")+" ("+view.dbkey+")");if(galaxy_config.app.add_dataset){$.ajax({url:galaxy_config.root+"api/datasets/"+galaxy_config.app.add_dataset,data:{hda_ldda:"hda",data_type:"track_config"},dataType:"json",success:function(d){view.add_drawable(a.object_from_template(d,view,view))}})}$("#add-bookmark-button").click(function(){var e=view.chrom+":"+view.low+"-"+view.high,d="Bookmark description";return ui.add_bookmark(e,d,true)});ui.init_keyboard_nav(view)}});return{GalaxyApp:c}}); \ No newline at end of file +var ui=null;var view=null;var browser_router=null;require(["utils/utils","libs/jquery/jquery.event.drag","libs/jquery/jquery.event.hover","libs/jquery/jquery.mousewheel","libs/jquery/jquery-ui","libs/jquery/select2","libs/farbtastic","libs/jquery/jquery.form","libs/jquery/jquery.rating","mvc/ui"],function(a){a.cssLoadFile("static/style/jquery.rating.css");a.cssLoadFile("static/style/autocomplete_tagging.css");a.cssLoadFile("static/style/jquery-ui/smoothness/jquery-ui.css");a.cssLoadFile("static/style/library.css");a.cssLoadFile("static/style/trackster.css")});define(["libs/backbone/backbone","viz/trackster_ui"],function(b,a){var c=Backbone.View.extend({initialize:function(){ui=new a.TracksterUI(galaxy_config.root);ui.createButtonMenu();ui.buttonMenu.$el.attr("style","float: right");$("#center .unified-panel-header-inner").append(ui.buttonMenu.$el);$("#right .unified-panel-title").append("Bookmarks");$("#right .unified-panel-icons").append("<a id='add-bookmark-button' class='icon-button menu-button plus-button' href='javascript:void(0);' title='Add bookmark'></a>");$("#right-border").click(function(){view.resize_window()});force_right_panel("hide");if(galaxy_config.app.id){this.view_existing()}else{this.view_new()}},view_existing:function(){var d=galaxy_config.app.viz_config;view=ui.create_visualization({container:$("#center .unified-panel-body"),name:d.title,vis_id:d.vis_id,dbkey:d.dbkey},d.viewport,d.tracks,d.bookmarks,true);this.init_editor()},view_new:function(){var d=this;$.ajax({url:galaxy_config.root+"api/genomes?chrom_info=True",data:{},error:function(){alert("Couldn't create new browser.")},success:function(e){Galaxy.modal.show({title:"New Visualization",body:d.template_view_new(e),buttons:{Cancel:function(){window.location=galaxy_config.root+"visualization/list"},Create:function(){d.create_browser($("#new-title").val(),$("#new-dbkey").val());Galaxy.modal.hide()}}});if(galaxy_config.app.default_dbkey){$("#new-dbkey").val(galaxy_config.app.default_dbkey)}$("#new-title").focus();$("select[name='dbkey']").select2();$("#overlay").css("overflow","auto")}})},template_view_new:function(d){var f='<form id="new-browser-form" action="javascript:void(0);" method="post" onsubmit="return false;"><div class="form-row"><label for="new-title">Browser name:</label><div class="form-row-input"><input type="text" name="title" id="new-title" value="Unnamed"></input></div><div style="clear: both;"></div></div><div class="form-row"><label for="new-dbkey">Reference genome build (dbkey): </label><div class="form-row-input"><select name="dbkey" id="new-dbkey">';for(var e=0;e<d.length;e++){f+='<option value="'+d[e][1]+'">'+d[e][0]+"</option>"}f+='</select></div><div style="clear: both;"></div></div><div class="form-row">Is the build not listed here? <a href="'+galaxy_config.root+'user/dbkeys?use_panels=True">Add a Custom Build</a></div></form>';return f},create_browser:function(e,d){$(document).trigger("convert_to_values");view=ui.create_visualization({container:$("#center .unified-panel-body"),name:e,dbkey:d},galaxy_config.app.gene_region);this.init_editor();view.editor=true},init_editor:function(){$("#center .unified-panel-title").text(view.config.get_value("name")+" ("+view.dbkey+")");if(galaxy_config.app.add_dataset){$.ajax({url:galaxy_config.root+"api/datasets/"+galaxy_config.app.add_dataset,data:{hda_ldda:"hda",data_type:"track_config"},dataType:"json",success:function(d){view.add_drawable(a.object_from_template(d,view,view))}})}$("#add-bookmark-button").click(function(){var e=view.chrom+":"+view.low+"-"+view.high,d="Bookmark description";return ui.add_bookmark(e,d,true)});ui.init_keyboard_nav(view)}});return{GalaxyApp:c}}); \ No newline at end of file Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: 3 new changesets
by commits-noreply＠bitbucket.org 28 May '14

28 May '14

3 new commits in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/2aafaa6a9159/ Changeset: 2aafaa6a9159 User: jmchilton Date: 2014-05-28 18:58:33 Summary: Remove invalid comment. Affected #: 1 file diff -r 23aac3b3c0e2cfe784d8c5dfd94cc3b6ccff7b7c -r 2aafaa6a915977f2f5970e01ccfbf51dc3db7c14 static/scripts/galaxy.tools.js --- a/static/scripts/galaxy.tools.js +++ b/static/scripts/galaxy.tools.js @@ -30,7 +30,7 @@ 'min_option_count': 2 // Don't show multiple select switch if only // one dataset available. }, - 'select_collection': { // NOT YET IMPLEMENTED + 'select_collection': { 'icon_class': 'fa-folder-o', 'select_by': 'Run tool in parallel across dataset collection', 'allow_remap': false https://bitbucket.org/galaxy/galaxy-central/commits/a763290e92ad/ Changeset: a763290e92ad User: jmchilton Date: 2014-05-28 18:58:33 Summary: Improvements to workflow extraction tests. Affected #: 1 file diff -r 2aafaa6a915977f2f5970e01ccfbf51dc3db7c14 -r a763290e92adb2215f4cba507e3eb9fe52618d2c test/api/test_workflows.py --- a/test/api/test_workflows.py +++ b/test/api/test_workflows.py @@ -75,11 +75,12 @@ @skip_without_tool( "cat1" ) def test_extract_from_history( self ): + # Run the simple test workflow and extract it back out from history workflow = self.workflow_populator.load_workflow( name="test_for_extract" ) workflow_request, history_id = self._setup_workflow_run( workflow ) contents_response = self._get( "histories/%s/contents" % history_id ) self._assert_status_code_is( contents_response, 200 ) - hda_ids = map( lambda c: c[ "hid" ], contents_response.json() ) + input_hids = map( lambda c: c[ "hid" ], contents_response.json() ) run_workflow_response = self._post( "workflows", data=workflow_request ) self._assert_status_code_is( run_workflow_response, 200 ) @@ -89,23 +90,16 @@ jobs_response = self._get( "jobs", data=data ) self._assert_status_code_is( jobs_response, 200 ) cat1_job_id = jobs_response.json()[ 0 ][ "id" ] - - contents_response = self._get( "history/%s/contents", data=data ) - create_from_data = dict( + downloaded_workflow = self._extract_and_download_workflow( from_history_id=history_id, - dataset_ids=dumps( hda_ids ), + dataset_ids=dumps( input_hids ), job_ids=dumps( [ cat1_job_id ] ), workflow_name="test import from history", ) - create_workflow_response = self._post( "workflows", data=create_from_data ) - self._assert_status_code_is( create_workflow_response, 200 ) - - new_workflow_id = create_workflow_response.json()[ "id" ] - download_response = self._get( "workflows/%s/download" % new_workflow_id ) - self._assert_status_code_is( download_response, 200 ) - downloaded_workflow = download_response.json() self.assertEquals( downloaded_workflow[ "name" ], "test import from history" ) assert len( downloaded_workflow[ "steps" ] ) == 3 + self._get_steps_of_type( downloaded_workflow, "data_input", expected_len=2 ) + self._get_steps_of_type( downloaded_workflow, "tool", expected_len=1 ) @skip_without_tool( "collection_paired_test" ) def test_extract_workflows_with_dataset_collections( self ): @@ -241,7 +235,8 @@ def _get_steps_of_type( self, downloaded_workflow, type, expected_len=None ): steps = [ s for s in downloaded_workflow[ "steps" ].values() if s[ "type" ] == type ] if expected_len is not None: - assert len( steps ) == expected_len + n = len( steps ) + assert n == expected_len, "Expected %d steps of type %s, found %d" % ( expected_len, type, n ) return steps @skip_without_tool( "random_lines1" ) https://bitbucket.org/galaxy/galaxy-central/commits/7fd77febf6bb/ Changeset: 7fd77febf6bb User: jmchilton Date: 2014-05-28 18:58:33 Summary: Workflow extraction test from copied datasets. Detailed some bugs I found in the test case and recorded them here https://trello.com/c/mKzLbM2P. Affected #: 1 file diff -r a763290e92adb2215f4cba507e3eb9fe52618d2c -r 7fd77febf6bb8aa7a6acde9c3a31429f7b1ee99f test/api/test_workflows.py --- a/test/api/test_workflows.py +++ b/test/api/test_workflows.py @@ -1,6 +1,7 @@ from base import api from json import dumps from json import loads +import operator import time from .helpers import WorkflowPopulator from .helpers import DatasetPopulator @@ -75,21 +76,11 @@ @skip_without_tool( "cat1" ) def test_extract_from_history( self ): + history_id = self.dataset_populator.new_history() # Run the simple test workflow and extract it back out from history - workflow = self.workflow_populator.load_workflow( name="test_for_extract" ) - workflow_request, history_id = self._setup_workflow_run( workflow ) + cat1_job_id = self.__setup_and_run_cat1_workflow( history_id=history_id ) contents_response = self._get( "histories/%s/contents" % history_id ) - self._assert_status_code_is( contents_response, 200 ) - input_hids = map( lambda c: c[ "hid" ], contents_response.json() ) - - run_workflow_response = self._post( "workflows", data=workflow_request ) - self._assert_status_code_is( run_workflow_response, 200 ) - - self.dataset_populator.wait_for_history( history_id, assert_ok=True ) - data = dict( history_id=history_id, tool_id="cat1" ) - jobs_response = self._get( "jobs", data=data ) - self._assert_status_code_is( jobs_response, 200 ) - cat1_job_id = jobs_response.json()[ 0 ][ "id" ] + input_hids = map( lambda c: c[ "hid" ], contents_response.json()[ 0:2 ] ) downloaded_workflow = self._extract_and_download_workflow( from_history_id=history_id, dataset_ids=dumps( input_hids ), @@ -97,9 +88,74 @@ workflow_name="test import from history", ) self.assertEquals( downloaded_workflow[ "name" ], "test import from history" ) + self.__assert_looks_like_cat1_example_workflow( downloaded_workflow ) + + def test_extract_with_copied_inputs( self ): + old_history_id = self.dataset_populator.new_history() + # Run the simple test workflow and extract it back out from history + self.__setup_and_run_cat1_workflow( history_id=old_history_id ) + + history_id = self.dataset_populator.new_history() + + # Bug cannot mess up hids or these don't extract correctly. See Trello card here: + # https://trello.com/c/mKzLbM2P + # # create dummy dataset to complicate hid mapping + # self.dataset_populator.new_dataset( history_id, content="dummydataset" ) + # offset = 1 + + offset = 0 + old_contents = self._get( "histories/%s/contents" % old_history_id ).json() + for old_dataset in old_contents: + payload = dict( + source="hda", + content=old_dataset["id"] + ) + response = self._post( "histories/%s/contents/datasets" % history_id, payload ) + self._assert_status_code_is( response, 200 ) + new_contents = self._get( "histories/%s/contents" % history_id ).json() + input_hids = map( lambda c: c[ "hid" ], new_contents[ (offset + 0):(offset + 2) ] ) + cat1_job_id = self.__job_id( history_id, new_contents[ (offset + 2) ][ "id" ] ) + downloaded_workflow = self._extract_and_download_workflow( + from_history_id=history_id, + dataset_ids=dumps( input_hids ), + job_ids=dumps( [ cat1_job_id ] ), + workflow_name="test import from history", + ) + self.__assert_looks_like_cat1_example_workflow( downloaded_workflow ) + + def __assert_looks_like_cat1_example_workflow( self, downloaded_workflow ): assert len( downloaded_workflow[ "steps" ] ) == 3 - self._get_steps_of_type( downloaded_workflow, "data_input", expected_len=2 ) - self._get_steps_of_type( downloaded_workflow, "tool", expected_len=1 ) + input_steps = self._get_steps_of_type( downloaded_workflow, "data_input", expected_len=2 ) + tool_step = self._get_steps_of_type( downloaded_workflow, "tool", expected_len=1 )[ 0 ] + + input1 = tool_step[ "input_connections" ][ "input1" ] + input2 = tool_step[ "input_connections" ][ "queries_0|input2" ] + + print downloaded_workflow + self.assertEquals( input_steps[ 0 ][ "id" ], input1[ "id" ] ) + self.assertEquals( input_steps[ 1 ][ "id" ], input2[ "id" ] ) + + def __setup_and_run_cat1_workflow( self, history_id ): + workflow = self.workflow_populator.load_workflow( name="test_for_extract" ) + workflow_request, history_id = self._setup_workflow_run( workflow, history_id=history_id ) + run_workflow_response = self._post( "workflows", data=workflow_request ) + self._assert_status_code_is( run_workflow_response, 200 ) + + self.dataset_populator.wait_for_history( history_id, assert_ok=True, timeout=10 ) + return self.__cat_job_id( history_id ) + + def __cat_job_id( self, history_id ): + data = dict( history_id=history_id, tool_id="cat1" ) + jobs_response = self._get( "jobs", data=data ) + self._assert_status_code_is( jobs_response, 200 ) + cat1_job_id = jobs_response.json()[ 0 ][ "id" ] + return cat1_job_id + + def __job_id( self, history_id, dataset_id ): + url = "histories/%s/contents/%s/provenance" % ( history_id, dataset_id ) + prov_response = self._get( url, data=dict( follow=False ) ) + self._assert_status_code_is( prov_response, 200 ) + return prov_response.json()[ "job_id" ] @skip_without_tool( "collection_paired_test" ) def test_extract_workflows_with_dataset_collections( self ): @@ -237,7 +293,7 @@ if expected_len is not None: n = len( steps ) assert n == expected_len, "Expected %d steps of type %s, found %d" % ( expected_len, type, n ) - return steps + return sorted( steps, key=operator.itemgetter("id") ) @skip_without_tool( "random_lines1" ) def test_run_replace_params_by_tool( self ): @@ -316,7 +372,7 @@ # renamed to 'the_new_name'. assert "the_new_name" in map( lambda hda: hda[ "name" ], contents ) - def _setup_workflow_run( self, workflow ): + def _setup_workflow_run( self, workflow, history_id=None ): uploaded_workflow_id = self.workflow_populator.create_workflow( workflow ) workflow_inputs = self._workflow_inputs( uploaded_workflow_id ) step_1 = step_2 = None @@ -326,7 +382,8 @@ step_1 = key if label == "WorkflowInput2": step_2 = key - history_id = self.dataset_populator.new_history() + if not history_id: + history_id = self.dataset_populator.new_history() hda1 = self.dataset_populator.new_dataset( history_id, content="1 2 3" ) hda2 = self.dataset_populator.new_dataset( history_id, content="4 5 6" ) workflow_request = dict( Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: greg: Handle Tool Shed category administration in the Tool Shed's repository registry.
by commits-noreply＠bitbucket.org 28 May '14

28 May '14

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/23aac3b3c0e2/ Changeset: 23aac3b3c0e2 User: greg Date: 2014-05-28 18:08:04 Summary: Handle Tool Shed category administration in the Tool Shed's repository registry. Affected #: 2 files diff -r dbc3d5c3506ed833dc453faa4f00797d95b8a8bb -r 23aac3b3c0e2cfe784d8c5dfd94cc3b6ccff7b7c lib/galaxy/webapps/tool_shed/controllers/admin.py --- a/lib/galaxy/webapps/tool_shed/controllers/admin.py +++ b/lib/galaxy/webapps/tool_shed/controllers/admin.py @@ -132,6 +132,8 @@ category = trans.app.model.Category( name=name, description=description ) trans.sa_session.add( category ) trans.sa_session.flush() + # Update the Tool Shed's repository registry. + trans.app.repository_registry.add_category_entry( category ) message = "Category '%s' has been created" % category.name status = 'done' trans.response.send_redirect( web.url_for( controller='admin', @@ -224,21 +226,32 @@ message=message, status='error' ) ) category = suc.get_category( trans, id ) + original_category_name = str( category.name ) + original_category_description = str( category.description ) if kwd.get( 'edit_category_button', False ): + flush_needed = False new_name = kwd.get( 'name', '' ).strip() new_description = kwd.get( 'description', '' ).strip() - if category.name != new_name or category.description != new_description: + if original_category_name != new_name: if not new_name: message = 'Enter a valid name' status = 'error' - elif category.name != new_name and suc.get_category_by_name( trans, new_name ): + elif original_category_name != new_name and suc.get_category_by_name( trans, new_name ): message = 'A category with that name already exists' status = 'error' else: category.name = new_name + flush_needed = True + if original_category_description != new_description: category.description = new_description + if not flus_needed: + flush_needed = True + if flush_needed: trans.sa_session.add( category ) trans.sa_session.flush() + if original_category_name != new_name: + # Update the Tool Shed's repository registry. + trans.app.repository_registry.edit_category_entry( original_category_name, new_name ) message = "The information has been saved for category '%s'" % ( category.name ) status = 'done' return trans.response.send_redirect( web.url_for( controller='admin', @@ -403,6 +416,8 @@ category.deleted = True trans.sa_session.add( category ) trans.sa_session.flush() + # Update the Tool Shed's repository registry. + trans.app.repository_registry.remove_category_entry( category ) message += " %s " % category.name else: message = "No category ids received for deleting." @@ -459,6 +474,8 @@ category.deleted = False trans.sa_session.add( category ) trans.sa_session.flush() + # Update the Tool Shed's repository registry. + trans.app.repository_registry.add_category_entry( category ) count += 1 undeleted_categories += " %s" % category.name message = "Undeleted %d categories: %s" % ( count, undeleted_categories ) diff -r dbc3d5c3506ed833dc453faa4f00797d95b8a8bb -r 23aac3b3c0e2cfe784d8c5dfd94cc3b6ccff7b7c lib/tool_shed/repository_registry.py --- a/lib/tool_shed/repository_registry.py +++ b/lib/tool_shed/repository_registry.py @@ -41,6 +41,21 @@ self.load_viewable_repositories_and_suites_by_category() self.load_repository_and_suite_tuples() + def add_category_entry( self, category ): + category_name = str( category.name ) + if category_name not in self.viewable_repositories_and_suites_by_category: + self.viewable_repositories_and_suites_by_category[ category_name ] = 0 + if category_name not in self.viewable_suites_by_category: + self.viewable_suites_by_category[ category_name ] = 0 + if category_name not in self.viewable_valid_repositories_and_suites_by_category: + self.viewable_valid_repositories_and_suites_by_category[ category_name ] = 0 + if category_name not in self.viewable_valid_suites_by_category: + self.viewable_valid_suites_by_category[ category_name ] = 0 + if category_name not in self.certified_level_one_viewable_repositories_and_suites_by_category: + self.certified_level_one_viewable_repositories_and_suites_by_category[ category_name ] = 0 + if category_name not in self.certified_level_one_viewable_suites_by_category: + self.certified_level_one_viewable_suites_by_category[ category_name ] = 0 + def add_entry( self, repository ): try: if repository: @@ -88,6 +103,44 @@ # will be corrected at next server start. log.exception( "Handled error adding entry to repository registry: %s." % str( e ) ) + def edit_category_entry( self, old_name, new_name ): + if old_name in self.viewable_repositories_and_suites_by_category: + val = self.viewable_repositories_and_suites_by_category[ old_name ] + del self.viewable_repositories_and_suites_by_category[ old_name ] + self.viewable_repositories_and_suites_by_category[ new_name ] = val + else: + self.viewable_repositories_and_suites_by_category[ new_name ] = 0 + if old_name in self.viewable_valid_repositories_and_suites_by_category: + val = self.viewable_valid_repositories_and_suites_by_category[ old_name ] + del self.viewable_valid_repositories_and_suites_by_category[ old_name ] + self.viewable_valid_repositories_and_suites_by_category[ new_name ] = val + else: + self.viewable_valid_repositories_and_suites_by_category[ new_name ] = 0 + if old_name in self.viewable_suites_by_category: + val = self.viewable_suites_by_category[ old_name ] + del self.viewable_suites_by_category[ old_name ] + self.viewable_suites_by_category[ new_name ] = val + else: + self.viewable_suites_by_category[ new_name ] = 0 + if old_name in self.viewable_valid_suites_by_category: + val = self.viewable_valid_suites_by_category[ old_name ] + del self.viewable_valid_suites_by_category[ old_name ] + self.viewable_valid_suites_by_category[ new_name ] = val + else: + self.viewable_valid_suites_by_category[ new_name ] = 0 + if old_name in self.certified_level_one_viewable_repositories_and_suites_by_category: + val = self.certified_level_one_viewable_repositories_and_suites_by_category[ old_name ] + del self.certified_level_one_viewable_repositories_and_suites_by_category[ old_name ] + self.certified_level_one_viewable_repositories_and_suites_by_category[ new_name ] = val + else: + self.certified_level_one_viewable_repositories_and_suites_by_category[ new_name ] = 0 + if old_name in self.certified_level_one_viewable_suites_by_category: + val = self.certified_level_one_viewable_suites_by_category[ old_name ] + del self.certified_level_one_viewable_suites_by_category[ old_name ] + self.certified_level_one_viewable_suites_by_category[ new_name ] = val + else: + self.certified_level_one_viewable_suites_by_category[ new_name ] = 0 + def get_certified_level_one_clause_list( self ): certified_level_one_tuples = [] clause_list = [] @@ -266,6 +319,21 @@ if repository.type in [ rt_util.REPOSITORY_SUITE_DEFINITION ]: self.certified_level_one_viewable_suites_by_category[ category_name ] += 1 + def remove_category_entry( self, category ): + catgeory_name = str( category.name ) + if catgeory_name in self.viewable_repositories_and_suites_by_category: + del self.viewable_repositories_and_suites_by_category[ catgeory_name ] + if catgeory_name in self.viewable_valid_repositories_and_suites_by_category: + del self.viewable_valid_repositories_and_suites_by_category[ catgeory_name ] + if catgeory_name in self.viewable_suites_by_category: + del self.viewable_suites_by_category[ catgeory_name ] + if catgeory_name in self.viewable_valid_suites_by_category: + del self.viewable_valid_suites_by_category[ catgeory_name ] + if catgeory_name in self.certified_level_one_viewable_repositories_and_suites_by_category: + del self.certified_level_one_viewable_repositories_and_suites_by_category[ catgeory_name ] + if catgeory_name in self.certified_level_one_viewable_suites_by_category: + del self.certified_level_one_viewable_suites_by_category[ catgeory_name ] + def remove_entry( self, repository ): try: if repository: Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: greg: Continued down-sizing of the Tool Shed's shed_util_common module by moving certain functions to more appropriate locations. Some fixes for adding and removing entries to the Tool Shed's repository registry.
by commits-noreply＠bitbucket.org 28 May '14

28 May '14

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/dbc3d5c3506e/ Changeset: dbc3d5c3506e User: greg Date: 2014-05-28 17:17:50 Summary: Continued down-sizing of the Tool Shed's shed_util_common module by moving certain functions to more appropriate locations. Some fixes for adding and removing entries to the Tool Shed's repository registry. Affected #: 36 files diff -r f6aa2d17d38430e92beb9da80f53c9989dd504ee -r dbc3d5c3506ed833dc453faa4f00797d95b8a8bb lib/galaxy/webapps/galaxy/api/tool_shed_repositories.py --- a/lib/galaxy/webapps/galaxy/api/tool_shed_repositories.py +++ b/lib/galaxy/webapps/galaxy/api/tool_shed_repositories.py @@ -11,6 +11,7 @@ from tool_shed.galaxy_install import repository_util from tool_shed.util import common_util from tool_shed.util import encoding_util +from tool_shed.util import hg_util from tool_shed.util import metadata_util from tool_shed.util import workflow_util from tool_shed.util import tool_util @@ -108,7 +109,7 @@ changeset_revisions = json.from_json_string( raw_text ) if len( changeset_revisions ) >= 1: return changeset_revisions[ -1 ] - return suc.INITIAL_CHANGELOG_HASH + return hg_util.INITIAL_CHANGELOG_HASH def __get_value_mapper( self, trans, tool_shed_repository ): value_mapper={ 'id' : trans.security.encode_id( tool_shed_repository.id ), diff -r f6aa2d17d38430e92beb9da80f53c9989dd504ee -r dbc3d5c3506ed833dc453faa4f00797d95b8a8bb lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py --- a/lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py +++ b/lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py @@ -462,8 +462,8 @@ err_msg = '' tool_shed_repository = tool_dependencies[ 0 ].tool_shed_repository # Get the tool_dependencies.xml file from the repository. - tool_dependencies_config = suc.get_config_from_disk( rt_util.TOOL_DEPENDENCY_DEFINITION_FILENAME, - tool_shed_repository.repo_path( trans.app ) ) + tool_dependencies_config = hg_util.get_config_from_disk( rt_util.TOOL_DEPENDENCY_DEFINITION_FILENAME, + tool_shed_repository.repo_path( trans.app ) ) installed_tool_dependencies = \ common_install_util.install_specified_packages( app=trans.app, tool_shed_repository=tool_shed_repository, @@ -505,7 +505,7 @@ 'repository/get_latest_downloadable_changeset_revision%s' % params ) raw_text = common_util.tool_shed_get( trans.app, tool_shed_url, url ) latest_downloadable_revision = json.from_json_string( raw_text ) - if latest_downloadable_revision == suc.INITIAL_CHANGELOG_HASH: + if latest_downloadable_revision == hg_util.INITIAL_CHANGELOG_HASH: message = 'Error retrieving the latest downloadable revision for this repository via the url <b>%s</b>.' % url status = 'error' else: diff -r f6aa2d17d38430e92beb9da80f53c9989dd504ee -r dbc3d5c3506ed833dc453faa4f00797d95b8a8bb lib/galaxy/webapps/tool_shed/api/repositories.py --- a/lib/galaxy/webapps/tool_shed/api/repositories.py +++ b/lib/galaxy/webapps/tool_shed/api/repositories.py @@ -13,6 +13,7 @@ import tool_shed.repository_types.util as rt_util import tool_shed.util.shed_util_common as suc from tool_shed.galaxy_install import repository_util +from tool_shed.util import basic_util from tool_shed.util import encoding_util from tool_shed.util import hg_util from tool_shed.util import import_util @@ -279,7 +280,7 @@ repository_status_info_dict, import_results_tups ) import_util.check_status_and_reset_downloadable( trans, import_results_tups ) - suc.remove_dir( file_path ) + basic_util.remove_dir( file_path ) # NOTE: the order of installation is defined in import_results_tups, but order will be lost # when transferred to return_dict. return_dict = {} diff -r f6aa2d17d38430e92beb9da80f53c9989dd504ee -r dbc3d5c3506ed833dc453faa4f00797d95b8a8bb lib/galaxy/webapps/tool_shed/controllers/repository.py --- a/lib/galaxy/webapps/tool_shed/controllers/repository.py +++ b/lib/galaxy/webapps/tool_shed/controllers/repository.py @@ -15,6 +15,7 @@ from galaxy.util import json from galaxy.model.orm import and_ import tool_shed.util.shed_util_common as suc +from tool_shed.util import basic_util from tool_shed.util import common_util from tool_shed.util import container_util from tool_shed.util import encoding_util @@ -1146,7 +1147,7 @@ # server account's .hgrc file to include the following setting: # [web] # allow_archive = bz2, gz, zip - file_type_str = suc.get_file_type_str( changeset_revision, file_type ) + file_type_str = export_util.get_file_type_str( changeset_revision, file_type ) repository.times_downloaded += 1 trans.sa_session.add( repository ) trans.sa_session.flush() @@ -1188,7 +1189,7 @@ # Make sure the file is removed from disk after the contents have been downloaded. os.unlink( repositories_archive.name ) repositories_archive_path, file_name = os.path.split( repositories_archive.name ) - suc.remove_dir( repositories_archive_path ) + basic_util.remove_dir( repositories_archive_path ) return opened_archive repository_metadata = suc.get_repository_metadata_by_changeset_revision( trans.app, repository_id, changeset_revision ) metadata = repository_metadata.metadata @@ -1312,7 +1313,10 @@ return self.install_matched_repository_grid( trans, **kwd ) else: kwd[ 'message' ] = "tool id: <b>%s</b><br/>tool name: <b>%s</b><br/>tool version: <b>%s</b><br/>exact matches only: <b>%s</b>" % \ - ( suc.stringify( tool_ids ), suc.stringify( tool_names ), suc.stringify( tool_versions ), str( exact_matches_checked ) ) + ( basic_util.stringify( tool_ids ), + basic_util.stringify( tool_names ), + basic_util.stringify( tool_versions ), + str( exact_matches_checked ) ) self.matched_repository_grid.title = "Repositories with matching tools" return self.matched_repository_grid( trans, **kwd ) else: @@ -1320,9 +1324,9 @@ status = "error" exact_matches_check_box = CheckboxField( 'exact_matches', checked=exact_matches_checked ) return trans.fill_template( '/webapps/tool_shed/repository/find_tools.mako', - tool_id=suc.stringify( tool_ids ), - tool_name=suc.stringify( tool_names ), - tool_version=suc.stringify( tool_versions ), + tool_id=basic_util.stringify( tool_ids ), + tool_name=basic_util.stringify( tool_names ), + tool_version=basic_util.stringify( tool_versions ), exact_matches_check_box=exact_matches_check_box, message=message, status=status ) @@ -1396,7 +1400,7 @@ return self.install_matched_repository_grid( trans, **kwd ) else: kwd[ 'message' ] = "workflow name: <b>%s</b><br/>exact matches only: <b>%s</b>" % \ - ( suc.stringify( workflow_names ), str( exact_matches_checked ) ) + ( basic_util.stringify( workflow_names ), str( exact_matches_checked ) ) self.matched_repository_grid.title = "Repositories with matching workflows" return self.matched_repository_grid( trans, **kwd ) else: @@ -1407,7 +1411,7 @@ workflow_names = [] exact_matches_check_box = CheckboxField( 'exact_matches', checked=exact_matches_checked ) return trans.fill_template( '/webapps/tool_shed/repository/find_workflows.mako', - workflow_name=suc.stringify( workflow_names ), + workflow_name=basic_util.stringify( workflow_names ), exact_matches_check_box=exact_matches_check_box, message=message, status=status ) @@ -1666,7 +1670,7 @@ if repository: repo = hg_util.get_repo_for_repository( trans.app, repository=repository, repo_path=None, create=False ) return suc.get_latest_downloadable_changeset_revision( trans.app, repository, repo ) - return suc.INITIAL_CHANGELOG_HASH + return hg_util.INITIAL_CHANGELOG_HASH @web.json def get_readme_files( self, trans, **kwd ): @@ -1832,7 +1836,7 @@ #manafest. repo_dir = repository.repo_path( trans.app ) # Get the tool_dependencies.xml file from disk. - tool_dependencies_config = suc.get_config_from_disk( rt_util.TOOL_DEPENDENCY_DEFINITION_FILENAME, repo_dir ) + tool_dependencies_config = hg_util.get_config_from_disk( rt_util.TOOL_DEPENDENCY_DEFINITION_FILENAME, repo_dir ) # Return the encoded contents of the tool_dependencies.xml file. if tool_dependencies_config: tool_dependencies_config_file = open( tool_dependencies_config, 'rb' ) @@ -1982,7 +1986,7 @@ repository_status_info_dict, import_results_tups ) import_util.check_status_and_reset_downloadable( trans, import_results_tups ) - suc.remove_dir( file_path ) + basic_util.remove_dir( file_path ) return trans.fill_template( '/webapps/tool_shed/repository/import_capsule_results.mako', export_info_dict=export_info_dict, import_results_tups=import_results_tups, @@ -2318,7 +2322,7 @@ is_malicious = False skip_tool_test = None repository_dependencies = None - if changeset_revision != suc.INITIAL_CHANGELOG_HASH: + if changeset_revision != hg_util.INITIAL_CHANGELOG_HASH: repository_metadata = suc.get_repository_metadata_by_changeset_revision( trans.app, id, changeset_revision ) if repository_metadata: revision_label = hg_util.get_revision_label( trans, repository, changeset_revision, include_date=False ) @@ -2328,8 +2332,8 @@ # There is no repository_metadata defined for the changeset_revision, so see if it was defined in a previous # changeset in the changelog. previous_changeset_revision = \ - suc.get_previous_metadata_changeset_revision( repository, repo, changeset_revision, downloadable=False ) - if previous_changeset_revision != suc.INITIAL_CHANGELOG_HASH: + metadata_util.get_previous_metadata_changeset_revision( repository, repo, changeset_revision, downloadable=False ) + if previous_changeset_revision != hg_util.INITIAL_CHANGELOG_HASH: repository_metadata = suc.get_repository_metadata_by_changeset_revision( trans.app, id, previous_changeset_revision ) if repository_metadata: revision_label = hg_util.get_revision_label( trans, repository, previous_changeset_revision, include_date=False ) @@ -2390,7 +2394,7 @@ changeset_revision, repository_dependencies, repository_metadata ) - heads = suc.get_repository_heads( repo ) + heads = hg_util.get_repository_heads( repo ) deprecated_repository_dependency_tups = \ repository_dependency_util.get_repository_dependency_tups_from_repository_metadata( trans.app, repository_metadata, @@ -2432,7 +2436,7 @@ repository = suc.get_repository_in_tool_shed( trans, id ) changeset_revision = kwd.get( 'changeset_revision', repository.tip( trans.app ) ) metadata = None - if changeset_revision != suc.INITIAL_CHANGELOG_HASH: + if changeset_revision != hg_util.INITIAL_CHANGELOG_HASH: repository_metadata = suc.get_repository_metadata_by_changeset_revision( trans.app, id, changeset_revision ) if repository_metadata: metadata = repository_metadata.metadata @@ -2441,11 +2445,11 @@ # in a previous changeset in the changelog. repo = hg_util.get_repo_for_repository( trans.app, repository=repository, repo_path=None, create=False ) previous_changeset_revision = \ - suc.get_previous_metadata_changeset_revision( repository, - repo, - changeset_revision, - downloadable=False ) - if previous_changeset_revision != suc.INITIAL_CHANGELOG_HASH: + metadata_util.get_previous_metadata_changeset_revision( repository, + repo, + changeset_revision, + downloadable=False ) + if previous_changeset_revision != hg_util.INITIAL_CHANGELOG_HASH: repository_metadata = suc.get_repository_metadata_by_changeset_revision( trans.app, id, previous_changeset_revision ) @@ -2602,10 +2606,15 @@ repository = suc.get_repository_by_name_and_owner( trans.app, name, owner ) repo = hg_util.get_repo_for_repository( trans.app, repository=repository, repo_path=None, create=False ) # Get the lower bound changeset revision. - lower_bound_changeset_revision = suc.get_previous_metadata_changeset_revision( repository, repo, changeset_revision, downloadable=True ) + lower_bound_changeset_revision = metadata_util.get_previous_metadata_changeset_revision( repository, + repo, + changeset_revision, + downloadable=True ) # Build the list of changeset revision hashes. changeset_hashes = [] - for changeset in suc.reversed_lower_upper_bounded_changelog( repo, lower_bound_changeset_revision, changeset_revision ): + for changeset in hg_util.reversed_lower_upper_bounded_changelog( repo, + lower_bound_changeset_revision, + changeset_revision ): changeset_hashes.append( str( repo.changectx( changeset ) ) ) if changeset_hashes: changeset_hashes_str = ','.join( changeset_hashes ) @@ -3117,7 +3126,7 @@ for diff in patch.diff( repo, node1=ctx_parent.node(), node2=ctx.node(), opts=diffopts ): if len( diff ) > suc.MAXDIFFSIZE: diff = util.shrink_string_by_size( diff, suc.MAXDIFFSIZE ) - diffs.append( suc.to_html_string( diff ) ) + diffs.append( basic_util.to_html_string( diff ) ) modified, added, removed, deleted, unknown, ignored, clean = repo.status( node1=ctx_parent.node(), node2=ctx.node() ) anchors = modified + added + removed + deleted + unknown + ignored + clean metadata = metadata_util.get_repository_metadata_by_repository_id_changeset_revision( trans, id, ctx_str, metadata_only=True ) @@ -3236,7 +3245,7 @@ status = 'warning' else: metadata = None - is_malicious = suc.changeset_is_malicious( trans.app, id, repository.tip( trans.app ) ) + is_malicious = metadata_util.is_malicious( trans.app, id, repository.tip( trans.app ) ) if is_malicious: if trans.app.security_agent.can_push( trans.app, trans.user, repository ): message += malicious_error_can_push @@ -3249,7 +3258,7 @@ repository_dependencies, repository_metadata ) repository_type_select_field = rt_util.build_repository_type_select_field( trans, repository=repository ) - heads = suc.get_repository_heads( repo ) + heads = hg_util.get_repository_heads( repo ) return trans.fill_template( '/webapps/tool_shed/repository/view_repository.mako', repo=repo, heads=heads, @@ -3320,7 +3329,7 @@ work_dir ) if message: status = 'error' - suc.remove_dir( work_dir ) + basic_util.remove_dir( work_dir ) break if guid: tool_lineage = tool_util.get_version_lineage_for_tool( trans, repository_id, repository_metadata, guid ) diff -r f6aa2d17d38430e92beb9da80f53c9989dd504ee -r dbc3d5c3506ed833dc453faa4f00797d95b8a8bb lib/galaxy/webapps/tool_shed/controllers/repository_review.py --- a/lib/galaxy/webapps/tool_shed/controllers/repository_review.py +++ b/lib/galaxy/webapps/tool_shed/controllers/repository_review.py @@ -470,7 +470,7 @@ metadata_revision_hashes = [ metadata_revision.changeset_revision for metadata_revision in repository.metadata_revisions ] reviewed_revision_hashes = [ review.changeset_revision for review in repository.reviews ] reviews_dict = odict() - for changeset in suc.get_reversed_changelog_changesets( repo ): + for changeset in hg_util.get_reversed_changelog_changesets( repo ): ctx = repo.changectx( changeset ) changeset_revision = str( ctx ) if changeset_revision in metadata_revision_hashes or changeset_revision in reviewed_revision_hashes: diff -r f6aa2d17d38430e92beb9da80f53c9989dd504ee -r dbc3d5c3506ed833dc453faa4f00797d95b8a8bb lib/galaxy/webapps/tool_shed/controllers/upload.py --- a/lib/galaxy/webapps/tool_shed/controllers/upload.py +++ b/lib/galaxy/webapps/tool_shed/controllers/upload.py @@ -10,6 +10,7 @@ from galaxy.datatypes import checkers import tool_shed.repository_types.util as rt_util import tool_shed.util.shed_util_common as suc +from tool_shed.util import basic_util from tool_shed.util import commit_util from tool_shed.util import hg_util from tool_shed.util import metadata_util @@ -64,9 +65,9 @@ try: commands.clone( hg_util.get_configured_ui(), repo_url, uploaded_directory ) except Exception, e: - message = 'Error uploading via mercurial clone: %s' % suc.to_html_string( str( e ) ) + message = 'Error uploading via mercurial clone: %s' % basic_util.to_html_string( str( e ) ) status = 'error' - suc.remove_dir( uploaded_directory ) + basic_util.remove_dir( uploaded_directory ) uploaded_directory = None elif url: valid_url = True @@ -296,7 +297,7 @@ # Reset the tool_data_tables by loading the empty tool_data_table_conf.xml file. tool_util.reset_tool_data_tables( trans.app ) if uploaded_directory: - suc.remove_dir( uploaded_directory ) + basic_util.remove_dir( uploaded_directory ) trans.response.send_redirect( web.url_for( controller='repository', action='browse_repository', id=repository_id, @@ -305,7 +306,7 @@ status=status ) ) else: if uploaded_directory: - suc.remove_dir( uploaded_directory ) + basic_util.remove_dir( uploaded_directory ) status = 'error' # Reset the tool_data_tables by loading the empty tool_data_table_conf.xml file. tool_util.reset_tool_data_tables( trans.app ) diff -r f6aa2d17d38430e92beb9da80f53c9989dd504ee -r dbc3d5c3506ed833dc453faa4f00797d95b8a8bb lib/tool_shed/galaxy_install/repository_util.py --- a/lib/tool_shed/galaxy_install/repository_util.py +++ b/lib/tool_shed/galaxy_install/repository_util.py @@ -9,6 +9,7 @@ from galaxy import web from galaxy.model.orm import or_ import tool_shed.util.shed_util_common as suc +from tool_shed.util import basic_util from tool_shed.util import common_util from tool_shed.util import common_install_util from tool_shed.util import container_util @@ -448,7 +449,7 @@ files_dir = relative_install_dir if shed_config_dict.get( 'tool_path' ): files_dir = os.path.join( shed_config_dict[ 'tool_path' ], files_dir ) - datatypes_config = suc.get_config_from_disk( suc.DATATYPES_CONFIG_FILENAME, files_dir ) + datatypes_config = hg_util.get_config_from_disk( suc.DATATYPES_CONFIG_FILENAME, files_dir ) # Load data types required by tools. converter_path, display_path = \ datatype_util.alter_config_and_load_prorietary_datatypes( trans.app, datatypes_config, files_dir, override=False ) @@ -620,14 +621,14 @@ tool_shed_repository, trans.install_model.ToolShedRepository.installation_status.INSTALLING_TOOL_DEPENDENCIES ) # Get the tool_dependencies.xml file from the repository. - tool_dependencies_config = suc.get_config_from_disk( 'tool_dependencies.xml', install_dir ) + tool_dependencies_config = hg_util.get_config_from_disk( 'tool_dependencies.xml', install_dir ) installed_tool_dependencies = \ common_install_util.install_specified_packages( app=trans.app, tool_shed_repository=tool_shed_repository, tool_dependencies_config=tool_dependencies_config, tool_dependencies=tool_shed_repository.tool_dependencies, from_tool_migration_manager=False ) - suc.remove_dir( work_dir ) + basic_util.remove_dir( work_dir ) suc.update_tool_shed_repository_status( trans.app, tool_shed_repository, trans.install_model.ToolShedRepository.installation_status.INSTALLED ) @@ -878,7 +879,7 @@ repository, trans.install_model.ToolShedRepository.installation_status.INSTALLING_TOOL_DEPENDENCIES ) # Get the tool_dependencies.xml file from the repository. - tool_dependencies_config = suc.get_config_from_disk( 'tool_dependencies.xml', repository.repo_path( trans.app ) ) + tool_dependencies_config = hg_util.get_config_from_disk( 'tool_dependencies.xml', repository.repo_path( trans.app ) ) installed_tool_dependencies = \ common_install_util.install_specified_packages( app=trans.app, tool_shed_repository=repository, @@ -888,7 +889,7 @@ for installed_tool_dependency in installed_tool_dependencies: if installed_tool_dependency.status in [ trans.install_model.ToolDependency.installation_status.ERROR ]: repair_dict = add_repair_dict_entry( repository.name, installed_tool_dependency.error_message ) - suc.remove_dir( work_dir ) + basic_util.remove_dir( work_dir ) suc.update_tool_shed_repository_status( trans.app, repository, trans.install_model.ToolShedRepository.installation_status.INSTALLED ) return repair_dict diff -r f6aa2d17d38430e92beb9da80f53c9989dd504ee -r dbc3d5c3506ed833dc453faa4f00797d95b8a8bb lib/tool_shed/galaxy_install/tool_dependencies/recipe/tag_handler.py --- a/lib/tool_shed/galaxy_install/tool_dependencies/recipe/tag_handler.py +++ b/lib/tool_shed/galaxy_install/tool_dependencies/recipe/tag_handler.py @@ -405,7 +405,7 @@ package_name=package_name, package_version=package_version, tool_dependencies_config=config_to_use ) - suc.remove_file( tmp_filename ) + self.remove_file( tmp_filename ) else: message = "Unable to locate required tool shed repository named %s owned by %s with revision %s." % \ ( str( required_repository_name ), str( required_repository_owner ), str( default_required_repository_changeset_revision ) ) @@ -429,6 +429,15 @@ print "Error installing tool dependency for required repository: %s" % str( rd_tool_dependency.error_message ) return tool_dependency, proceed_with_install, action_elem_tuples + def remove_file( self, file_name ): + """Attempt to remove a file from disk.""" + if file_name: + if os.path.exists( file_name ): + try: + os.remove( file_name ) + except: + pass + class SetEnvironment( RecipeTag ): diff -r f6aa2d17d38430e92beb9da80f53c9989dd504ee -r dbc3d5c3506ed833dc453faa4f00797d95b8a8bb lib/tool_shed/galaxy_install/tool_migration_manager.py --- a/lib/tool_shed/galaxy_install/tool_migration_manager.py +++ b/lib/tool_shed/galaxy_install/tool_migration_manager.py @@ -11,6 +11,7 @@ from galaxy import util from galaxy.tools import ToolSection import tool_shed.util.shed_util_common as suc +from tool_shed.util import basic_util from tool_shed.util import common_install_util from tool_shed.util import common_util from tool_shed.util import datatype_util @@ -251,13 +252,13 @@ def get_guid( self, repository_clone_url, relative_install_dir, tool_config ): if self.shed_config_dict.get( 'tool_path' ): relative_install_dir = os.path.join( self.shed_config_dict[ 'tool_path' ], relative_install_dir ) - tool_config_filename = suc.strip_path( tool_config ) + tool_config_filename = basic_util.strip_path( tool_config ) for root, dirs, files in os.walk( relative_install_dir ): if root.find( '.hg' ) < 0 and root.find( 'hgrc' ) < 0: if '.hg' in dirs: dirs.remove( '.hg' ) for name in files: - filename = suc.strip_path( name ) + filename = basic_util.strip_path( name ) if filename == tool_config_filename: full_path = str( os.path.abspath( os.path.join( root, name ) ) ) tool = self.toolbox.load_tool( full_path ) @@ -444,7 +445,7 @@ tool_shed_repository, self.app.install_model.ToolShedRepository.installation_status.INSTALLING_TOOL_DEPENDENCIES ) # Get the tool_dependencies.xml file from disk. - tool_dependencies_config = suc.get_config_from_disk( 'tool_dependencies.xml', repo_install_dir ) + tool_dependencies_config = hg_util.get_config_from_disk( 'tool_dependencies.xml', repo_install_dir ) installed_tool_dependencies = \ common_install_util.install_specified_packages( app=self.app, tool_shed_repository=tool_shed_repository, @@ -462,7 +463,7 @@ self.app.install_model.context.add( tool_shed_repository ) self.app.install_model.context.flush() work_dir = tempfile.mkdtemp( prefix="tmp-toolshed-hrc" ) - datatypes_config = suc.get_config_from_disk( suc.DATATYPES_CONFIG_FILENAME, repo_install_dir ) + datatypes_config = hg_util.get_config_from_disk( suc.DATATYPES_CONFIG_FILENAME, repo_install_dir ) # Load proprietary data types required by tools. The value of override is not important here since the Galaxy server will be started # after this installation completes. converter_path, display_path = datatype_util.alter_config_and_load_prorietary_datatypes( self.app, datatypes_config, repo_install_dir, override=False ) #repo_install_dir was relative_install_dir @@ -481,7 +482,7 @@ if display_path: # Load proprietary datatype display applications self.app.datatypes_registry.load_display_applications( installed_repository_dict=repository_dict ) - suc.remove_dir( work_dir ) + basic_util.remove_dir( work_dir ) def install_repository( self, repository_elem, tool_shed_repository, install_dependencies, is_repository_dependency=False ): """Install a single repository, loading contained tools into the tool panel.""" diff -r f6aa2d17d38430e92beb9da80f53c9989dd504ee -r dbc3d5c3506ed833dc453faa4f00797d95b8a8bb lib/tool_shed/grids/repository_grids.py --- a/lib/tool_shed/grids/repository_grids.py +++ b/lib/tool_shed/grids/repository_grids.py @@ -98,7 +98,7 @@ def get_value( self, trans, grid, repository ): """Display the current repository heads.""" repo = hg_util.get_repo_for_repository( trans.app, repository=repository, repo_path=None, create=False ) - heads = suc.get_repository_heads( repo ) + heads = hg_util.get_repository_heads( repo ) multiple_heads = len( heads ) > 1 if multiple_heads: heads_str = '<font color="red">' diff -r f6aa2d17d38430e92beb9da80f53c9989dd504ee -r dbc3d5c3506ed833dc453faa4f00797d95b8a8bb lib/tool_shed/grids/util.py --- a/lib/tool_shed/grids/util.py +++ b/lib/tool_shed/grids/util.py @@ -5,6 +5,7 @@ from galaxy.web.form_builder import SelectField from tool_shed.util import hg_util +from tool_shed.util import metadata_util from tool_shed.util import shed_util_common as suc log = logging.getLogger( __name__ ) @@ -178,8 +179,11 @@ return repository_metadata return None except: - latest_downloadable_revision = suc.get_previous_metadata_changeset_revision( repository, repo, tip_ctx, downloadable=True ) - if latest_downloadable_revision == suc.INITIAL_CHANGELOG_HASH: + latest_downloadable_revision = metadata_util.get_previous_metadata_changeset_revision( repository, + repo, + tip_ctx, + downloadable=True ) + if latest_downloadable_revision == hg_util.INITIAL_CHANGELOG_HASH: return None repository_metadata = suc.get_repository_metadata_by_changeset_revision( trans.app, encoded_repository_id, @@ -225,8 +229,11 @@ repository_metadata = suc.get_repository_metadata_by_changeset_revision( trans.app, encoded_repository_id, tip_ctx ) return repository_metadata except: - latest_downloadable_revision = suc.get_previous_metadata_changeset_revision( repository, repo, tip_ctx, downloadable=False ) - if latest_downloadable_revision == suc.INITIAL_CHANGELOG_HASH: + latest_downloadable_revision = metadata_util.get_previous_metadata_changeset_revision( repository, + repo, + tip_ctx, + downloadable=False ) + if latest_downloadable_revision == hg_util.INITIAL_CHANGELOG_HASH: return None repository_metadata = suc.get_repository_metadata_by_changeset_revision( trans.app, encoded_repository_id, diff -r f6aa2d17d38430e92beb9da80f53c9989dd504ee -r dbc3d5c3506ed833dc453faa4f00797d95b8a8bb lib/tool_shed/repository_registry.py --- a/lib/tool_shed/repository_registry.py +++ b/lib/tool_shed/repository_registry.py @@ -50,17 +50,35 @@ for rca in repository.categories: category = rca.category category_name = str( category.name ) - self.viewable_repositories_and_suites_by_category[ category_name ] += 1 + if category_name in self.viewable_repositories_and_suites_by_category: + self.viewable_repositories_and_suites_by_category[ category_name ] += 1 + else: + self.viewable_repositories_and_suites_by_category[ category_name ] = 1 if is_valid: - self.viewable_valid_repositories_and_suites_by_category[ category_name ] += 1 + if category_name in self.viewable_valid_repositories_and_suites_by_category: + self.viewable_valid_repositories_and_suites_by_category[ category_name ] += 1 + else: + self.viewable_valid_repositories_and_suites_by_category[ category_name ] = 1 if repository.type == rt_util.REPOSITORY_SUITE_DEFINITION: - self.viewable_suites_by_category[ category_name ] += 1 + if category_name in self.viewable_suites_by_category: + self.viewable_suites_by_category[ category_name ] += 1 + else: + self.viewable_suites_by_category[ category_name ] = 1 if is_valid: - self.viewable_valid_suites_by_category[ category_name ] += 1 + if category_name in self.viewable_valid_suites_by_category: + self.viewable_valid_suites_by_category[ category_name ] += 1 + else: + self.viewable_valid_suites_by_category[ category_name ] = 1 if is_level_one_certified: - self.certified_level_one_viewable_repositories_and_suites_by_category[ category_name ] += 1 + if category_name in self.certified_level_one_viewable_repositories_and_suites_by_category: + self.certified_level_one_viewable_repositories_and_suites_by_category[ category_name ] += 1 + else: + self.certified_level_one_viewable_repositories_and_suites_by_category[ category_name ] = 1 if repository.type == rt_util.REPOSITORY_SUITE_DEFINITION: - self.certified_level_one_viewable_suites_by_category[ category_name ] += 1 + if category_name in self.certified_level_one_viewable_suites_by_category: + self.certified_level_one_viewable_suites_by_category[ category_name ] += 1 + else: + self.certified_level_one_viewable_suites_by_category[ category_name ] = 1 self.load_repository_and_suite_tuple( repository ) if is_level_one_certified: self.load_certified_level_one_repository_and_suite_tuple( repository ) @@ -97,7 +115,7 @@ repo = hg_util.get_repo_for_repository( self.app, repository=repository, repo_path=None, create=False ) # Get the latest installable changeset revision since that is all that is currently configured for testing. latest_installable_changeset_revision = suc.get_latest_downloadable_changeset_revision( self.app, repository, repo ) - if latest_installable_changeset_revision not in [ None, suc.INITIAL_CHANGELOG_HASH ]: + if latest_installable_changeset_revision not in [ None, hg_util.INITIAL_CHANGELOG_HASH ]: encoded_repository_id = self.app.security.encode_id( repository.id ) repository_metadata = suc.get_repository_metadata_by_changeset_revision( self.app, encoded_repository_id, @@ -168,7 +186,7 @@ name = str( repository.name ) owner = str( repository.user.username ) tip_changeset_hash = repository.tip( self.app ) - if tip_changeset_hash != suc.INITIAL_CHANGELOG_HASH: + if tip_changeset_hash != hg_util.INITIAL_CHANGELOG_HASH: certified_level_one_tuple = ( name, owner, tip_changeset_hash ) if repository.type == rt_util.REPOSITORY_SUITE_DEFINITION: if certified_level_one_tuple not in self.certified_level_one_suite_tuples: @@ -257,17 +275,41 @@ for rca in repository.categories: category = rca.category category_name = str( category.name ) - self.viewable_repositories_and_suites_by_category[ category_name ] -= 1 + if category_name in self.viewable_repositories_and_suites_by_category: + if self.viewable_repositories_and_suites_by_category[ category_name ] > 0: + self.viewable_repositories_and_suites_by_category[ category_name ] -= 1 + else: + self.viewable_repositories_and_suites_by_category[ category_name ] = 0 if is_valid: - self.viewable_valid_repositories_and_suites_by_category[ category_name ] -= 1 + if category_name in self.viewable_valid_repositories_and_suites_by_category: + if self.viewable_valid_repositories_and_suites_by_category[ category_name ] > 0: + self.viewable_valid_repositories_and_suites_by_category[ category_name ] -= 1 + else: + self.viewable_valid_repositories_and_suites_by_category[ category_name ] = 0 if repository.type == rt_util.REPOSITORY_SUITE_DEFINITION: - self.viewable_suites_by_category[ category_name ] -= 1 + if category_name in self.viewable_suites_by_category: + if self.viewable_suites_by_category[ category_name ] > 0: + self.viewable_suites_by_category[ category_name ] -= 1 + else: + self.viewable_suites_by_category[ category_name ] = 0 if is_valid: - self.viewable_valid_suites_by_category[ category_name ] -= 1 + if category_name in self.viewable_valid_suites_by_category: + if self.viewable_valid_suites_by_category[ category_name ] > 0: + self.viewable_valid_suites_by_category[ category_name ] -= 1 + else: + self.viewable_valid_suites_by_category[ category_name ] = 0 if is_level_one_certified: - self.certified_level_one_viewable_repositories_and_suites_by_category[ category_name ] -= 1 + if category_name in self.certified_level_one_viewable_repositories_and_suites_by_category: + if self.certified_level_one_viewable_repositories_and_suites_by_category[ category_name ] > 0: + self.certified_level_one_viewable_repositories_and_suites_by_category[ category_name ] -= 1 + else: + self.certified_level_one_viewable_repositories_and_suites_by_category[ category_name ] = 0 if repository.type == rt_util.REPOSITORY_SUITE_DEFINITION: - self.certified_level_one_viewable_suites_by_category[ category_name ] -= 1 + if category_name in self.certified_level_one_viewable_suites_by_category: + if self.certified_level_one_viewable_suites_by_category[ category_name ] > 0: + self.certified_level_one_viewable_suites_by_category[ category_name ] -= 1 + else: + self.certified_level_one_viewable_suites_by_category[ category_name ] = 0 self.unload_repository_and_suite_tuple( repository ) if is_level_one_certified: self.unload_certified_level_one_repository_and_suite_tuple( repository ) @@ -286,7 +328,7 @@ name = str( repository.name ) owner = str( repository.user.username ) tip_changeset_hash = repository.tip( self.app ) - if tip_changeset_hash != suc.INITIAL_CHANGELOG_HASH: + if tip_changeset_hash != hg_util.INITIAL_CHANGELOG_HASH: certified_level_one_tuple = ( name, owner, tip_changeset_hash ) if repository.type == rt_util.REPOSITORY_SUITE_DEFINITION: if certified_level_one_tuple in self.certified_level_one_suite_tuples: diff -r f6aa2d17d38430e92beb9da80f53c9989dd504ee -r dbc3d5c3506ed833dc453faa4f00797d95b8a8bb lib/tool_shed/repository_types/repository_suite_definition.py --- a/lib/tool_shed/repository_types/repository_suite_definition.py +++ b/lib/tool_shed/repository_types/repository_suite_definition.py @@ -1,7 +1,7 @@ import logging from tool_shed.repository_types.metadata import TipOnly import tool_shed.repository_types.util as rt_util -import tool_shed.util.shed_util_common as suc +from tool_shed.util import basic_util from galaxy import eggs eggs.require( 'mercurial' ) @@ -35,7 +35,7 @@ # is named repository_dependencies.xml. files_changed_in_changeset = ctx.files() for file_path in files_changed_in_changeset: - file_name = suc.strip_path( file_path ) + file_name = basic_util.strip_path( file_path ) if file_name not in self.valid_file_names: return False return True diff -r f6aa2d17d38430e92beb9da80f53c9989dd504ee -r dbc3d5c3506ed833dc453faa4f00797d95b8a8bb lib/tool_shed/repository_types/tool_dependency_definition.py --- a/lib/tool_shed/repository_types/tool_dependency_definition.py +++ b/lib/tool_shed/repository_types/tool_dependency_definition.py @@ -1,7 +1,7 @@ import logging from tool_shed.repository_types.metadata import TipOnly import tool_shed.repository_types.util as rt_util -import tool_shed.util.shed_util_common as suc +from tool_shed.util import basic_util from galaxy import eggs eggs.require( 'mercurial' ) @@ -34,7 +34,7 @@ # Inspect all files in the changeset (in sorted order) to make sure there is only one and it is named tool_dependencies.xml. files_changed_in_changeset = ctx.files() for file_path in files_changed_in_changeset: - file_name = suc.strip_path( file_path ) + file_name = basic_util.strip_path( file_path ) if file_name not in self.valid_file_names: return False return True diff -r f6aa2d17d38430e92beb9da80f53c9989dd504ee -r dbc3d5c3506ed833dc453faa4f00797d95b8a8bb lib/tool_shed/scripts/api/common.py --- a/lib/tool_shed/scripts/api/common.py +++ b/lib/tool_shed/scripts/api/common.py @@ -8,11 +8,8 @@ new_path.extend( sys.path[ 1: ] ) sys.path = new_path -import tool_shed.util.shed_util_common as suc from tool_shed.util import common_util - -from galaxy import eggs -import pkg_resources +from tool_shed.util import hg_util def delete( api_key, url, data, return_formatted=True ): """ @@ -115,7 +112,7 @@ return None, error_message if len( changeset_revisions ) >= 1: return changeset_revisions[ -1 ], error_message - return suc.INITIAL_CHANGELOG_HASH, error_message + return hg_util.INITIAL_CHANGELOG_HASH, error_message def get_repository_dict( url, repository_dict ): """ diff -r f6aa2d17d38430e92beb9da80f53c9989dd504ee -r dbc3d5c3506ed833dc453faa4f00797d95b8a8bb lib/tool_shed/scripts/api/export.py --- a/lib/tool_shed/scripts/api/export.py +++ b/lib/tool_shed/scripts/api/export.py @@ -17,17 +17,6 @@ CHUNK_SIZE = 2**20 # 1Mb -def get_file_type_str( changeset_revision, file_type ): - if file_type == 'zip': - file_type_str = '%s.zip' % changeset_revision - elif file_type == 'bz2': - file_type_str = '%s.tar.bz2' % changeset_revision - elif file_type == 'gz': - file_type_str = '%s.tar.gz' % changeset_revision - else: - file_type_str = '' - return file_type_str - def string_as_bool( string ): if str( string ).lower() in ( 'true', 'yes', 'on' ): return True diff -r f6aa2d17d38430e92beb9da80f53c9989dd504ee -r dbc3d5c3506ed833dc453faa4f00797d95b8a8bb lib/tool_shed/scripts/api/get_filtered_repository_revisions.py --- a/lib/tool_shed/scripts/api/get_filtered_repository_revisions.py +++ b/lib/tool_shed/scripts/api/get_filtered_repository_revisions.py @@ -34,7 +34,7 @@ from galaxy.util import asbool from galaxy.util.json import from_json_string -import tool_shed.util.shed_util_common as suc +from tool_shed.util import hg_util def main( options ): base_tool_shed_url = options.tool_shed_url.rstrip( '/' ) @@ -71,12 +71,12 @@ repository_dicts.append( baseline_repository_dict ) else: # Don't test empty repositories. - changeset_revision = baseline_repository_dict.get( 'changeset_revision', suc.INITIAL_CHANGELOG_HASH ) - if changeset_revision != suc.INITIAL_CHANGELOG_HASH: + changeset_revision = baseline_repository_dict.get( 'changeset_revision', hg_util.INITIAL_CHANGELOG_HASH ) + if changeset_revision != hg_util.INITIAL_CHANGELOG_HASH: # Merge the dictionary returned from /api/repository_revisions with the detailed repository_dict and # append it to the list of repository_dicts to install and test. if latest_revision_only: - latest_revision = repository_dict.get( 'latest_revision', suc.INITIAL_CHANGELOG_HASH ) + latest_revision = repository_dict.get( 'latest_revision', hg_util.INITIAL_CHANGELOG_HASH ) if changeset_revision == latest_revision: repository_dicts.append( dict( repository_dict.items() + baseline_repository_dict.items() ) ) else: diff -r f6aa2d17d38430e92beb9da80f53c9989dd504ee -r dbc3d5c3506ed833dc453faa4f00797d95b8a8bb lib/tool_shed/util/basic_util.py --- /dev/null +++ b/lib/tool_shed/util/basic_util.py @@ -0,0 +1,57 @@ +import logging +import os + +from galaxy.util import unicodify + +from galaxy import eggs + +eggs.require( 'markupsafe' ) +import markupsafe + +log = logging.getLogger( __name__ ) + +MAX_DISPLAY_SIZE = 32768 + +def remove_dir( dir ): + """Attempt to remove a directory from disk.""" + if dir: + if os.path.exists( dir ): + try: + shutil.rmtree( dir ) + except: + pass + +def size_string( raw_text, size=MAX_DISPLAY_SIZE ): + """Return a subset of a string (up to MAX_DISPLAY_SIZE) translated to a safe string for display in a browser.""" + if raw_text and len( raw_text ) >= size: + large_str = '\nFile contents truncated because file size is larger than maximum viewing size of %s\n' % util.nice_size( size ) + raw_text = '%s%s' % ( raw_text[ 0:size ], large_str ) + return raw_text or '' + +def stringify( list ): + if list: + return ','.join( list ) + return '' + +def strip_path( fpath ): + """Attempt to strip the path from a file name.""" + if not fpath: + return fpath + try: + file_path, file_name = os.path.split( fpath ) + except: + file_name = fpath + return file_name + +def to_html_string( text ): + """Translates the characters in text to an html string""" + if text: + try: + text = unicodify( text ) + except UnicodeDecodeError, e: + return "Error decoding string: %s" % str( e ) + text = unicode( markupsafe.escape( text ) ) + text = text.replace( '\n', '<br/>' ) + text = text.replace( ' ', '    ' ) + text = text.replace( ' ', ' ' ) + return text diff -r f6aa2d17d38430e92beb9da80f53c9989dd504ee -r dbc3d5c3506ed833dc453faa4f00797d95b8a8bb lib/tool_shed/util/commit_util.py --- a/lib/tool_shed/util/commit_util.py +++ b/lib/tool_shed/util/commit_util.py @@ -345,7 +345,7 @@ if repository: repo = hg_util.get_repo_for_repository( trans.app, repository=repository, repo_path=None, create=False ) lastest_installable_changeset_revision = suc.get_latest_downloadable_changeset_revision( trans.app, repository, repo ) - if lastest_installable_changeset_revision != suc.INITIAL_CHANGELOG_HASH: + if lastest_installable_changeset_revision != hg_util.INITIAL_CHANGELOG_HASH: elem.attrib[ 'changeset_revision' ] = lastest_installable_changeset_revision revised = True else: diff -r f6aa2d17d38430e92beb9da80f53c9989dd504ee -r dbc3d5c3506ed833dc453faa4f00797d95b8a8bb lib/tool_shed/util/datatype_util.py --- a/lib/tool_shed/util/datatype_util.py +++ b/lib/tool_shed/util/datatype_util.py @@ -3,6 +3,8 @@ import tempfile from galaxy import eggs from galaxy.util import asbool +from tool_shed.util import basic_util +from tool_shed.util import hg_util from tool_shed.util import tool_util from tool_shed.util import xml_util import tool_shed.util.shed_util_common as suc @@ -127,7 +129,7 @@ for converter in elem.findall( 'converter' ): converter_config = converter.get( 'file', None ) if converter_config: - converter_config_file_name = suc.strip_path( converter_config ) + converter_config_file_name = basic_util.strip_path( converter_config ) for root, dirs, files in os.walk( relative_install_dir ): if root.find( '.hg' ) < 0: for name in files: @@ -144,7 +146,7 @@ for display_app in elem.findall( 'display' ): display_config = display_app.get( 'file', None ) if display_config: - display_config_file_name = suc.strip_path( display_config ) + display_config_file_name = basic_util.strip_path( display_config ) for root, dirs, files in os.walk( relative_install_dir ): if root.find( '.hg' ) < 0: for name in files: @@ -166,7 +168,7 @@ # Load proprietary datatypes and return information needed for loading proprietary datatypes converters and display applications later. metadata = repository.metadata repository_dict = None - datatypes_config = suc.get_config_from_disk( suc.DATATYPES_CONFIG_FILENAME, relative_install_dir ) + datatypes_config = hg_util.get_config_from_disk( suc.DATATYPES_CONFIG_FILENAME, relative_install_dir ) if datatypes_config: converter_path, display_path = alter_config_and_load_prorietary_datatypes( app, datatypes_config, relative_install_dir, deactivate=deactivate ) if converter_path or display_path: diff -r f6aa2d17d38430e92beb9da80f53c9989dd504ee -r dbc3d5c3506ed833dc453faa4f00797d95b8a8bb lib/tool_shed/util/export_util.py --- a/lib/tool_shed/util/export_util.py +++ b/lib/tool_shed/util/export_util.py @@ -11,6 +11,7 @@ from galaxy import eggs from galaxy import web from galaxy.util.odict import odict +from tool_shed.util import basic_util from tool_shed.util import commit_util from tool_shed.util import common_install_util from tool_shed.util import common_util @@ -101,7 +102,7 @@ attributes, sub_elements = get_repository_attributes_and_sub_elements( ordered_repository, archive_name ) elem = xml_util.create_element( 'repository', attributes=attributes, sub_elements=sub_elements ) exported_repository_registry.exported_repository_elems.append( elem ) - suc.remove_dir( work_dir ) + basic_util.remove_dir( work_dir ) # Keep information about the export in a file name export_info.xml in the archive. sub_elements = generate_export_elem( tool_shed_url, repository, changeset_revision, export_repository_dependencies, api ) export_elem = xml_util.create_element( 'export_info', attributes=None, sub_elements=sub_elements ) @@ -127,7 +128,7 @@ return repositories_archive, error_messages def generate_repository_archive( trans, work_dir, tool_shed_url, repository, changeset_revision, file_type ): - file_type_str = suc.get_file_type_str( changeset_revision, file_type ) + file_type_str = get_file_type_str( changeset_revision, file_type ) file_name = '%s-%s' % ( repository.name, file_type_str ) return_code, error_message = archive_repository_revision( trans, ui, repository, work_dir, changeset_revision ) if return_code: @@ -172,7 +173,7 @@ def generate_repository_archive_filename( tool_shed_url, name, owner, changeset_revision, file_type, export_repository_dependencies=False, use_tmp_archive_dir=False ): tool_shed = remove_protocol_from_tool_shed_url( tool_shed_url ) - file_type_str = suc.get_file_type_str( changeset_revision, file_type ) + file_type_str = get_file_type_str( changeset_revision, file_type ) if export_repository_dependencies: repositories_archive_filename = '%s_%s_%s_%s_%s' % ( CAPSULE_WITH_DEPENDENCIES_FILENAME, tool_shed, name, owner, file_type_str ) else: @@ -210,6 +211,17 @@ return repository, repository_metadata.changeset_revision return None, None +def get_file_type_str( changeset_revision, file_type ): + if file_type == 'zip': + file_type_str = '%s.zip' % changeset_revision + elif file_type == 'bz2': + file_type_str = '%s.tar.bz2' % changeset_revision + elif file_type == 'gz': + file_type_str = '%s.tar.gz' % changeset_revision + else: + file_type_str = '' + return file_type_str + def get_repo_info_dict_for_import( encoded_repository_id, encoded_repository_ids, repo_info_dicts ): """ The received encoded_repository_ids and repo_info_dicts are lists that contain associated elements at each diff -r f6aa2d17d38430e92beb9da80f53c9989dd504ee -r dbc3d5c3506ed833dc453faa4f00797d95b8a8bb lib/tool_shed/util/hg_util.py --- a/lib/tool_shed/util/hg_util.py +++ b/lib/tool_shed/util/hg_util.py @@ -1,4 +1,6 @@ import logging +import os + from datetime import datetime from time import gmtime from time import strftime @@ -12,8 +14,12 @@ from mercurial import hg from mercurial import ui +from tool_shed.util import basic_util + log = logging.getLogger( __name__ ) +INITIAL_CHANGELOG_HASH = '000000000000' + def clone_repository( repository_clone_url, repository_file_dir, ctx_rev ): """ Clone the repository up to the specified changeset_revision. No subsequent revisions will be @@ -32,6 +38,22 @@ log.debug( error_message ) return False, error_message +def copy_file_from_manifest( repo, ctx, filename, dir ): + """ + Copy the latest version of the file named filename from the repository manifest to the directory + to which dir refers. + """ + for changeset in reversed_upper_bounded_changelog( repo, ctx ): + changeset_ctx = repo.changectx( changeset ) + fctx = get_file_context_from_ctx( changeset_ctx, filename ) + if fctx and fctx not in [ 'DELETED' ]: + file_path = os.path.join( dir, filename ) + fh = open( file_path, 'wb' ) + fh.write( fctx.data() ) + fh.close() + return file_path + return None + def get_changectx_for_changeset( repo, changeset_revision, **kwd ): """Retrieve a specified changectx from a repository.""" for changeset in repo.changelog: @@ -40,6 +62,25 @@ return ctx return None +def get_config( config_file, repo, ctx, dir ): + """Return the latest version of config_filename from the repository manifest.""" + config_file = basic_util.strip_path( config_file ) + for changeset in reversed_upper_bounded_changelog( repo, ctx ): + changeset_ctx = repo.changectx( changeset ) + for ctx_file in changeset_ctx.files(): + ctx_file_name = basic_util.strip_path( ctx_file ) + if ctx_file_name == config_file: + return get_named_tmpfile_from_ctx( changeset_ctx, ctx_file, dir ) + return None + +def get_config_from_disk( config_file, relative_install_dir ): + for root, dirs, files in os.walk( relative_install_dir ): + if root.find( '.hg' ) < 0: + for name in files: + if name == config_file: + return os.path.abspath( os.path.join( root, name ) ) + return None + def get_configured_ui(): """Configure any desired ui settings.""" _ui = ui.ui() @@ -50,6 +91,44 @@ _ui.setconfig( 'ui', 'quiet', True ) return _ui +def get_ctx_file_path_from_manifest( filename, repo, changeset_revision ): + """ + Get the ctx file path for the latest revision of filename from the repository manifest up + to the value of changeset_revision. + """ + stripped_filename = basic_util.strip_path( filename ) + for changeset in reversed_upper_bounded_changelog( repo, changeset_revision ): + manifest_changeset_revision = str( repo.changectx( changeset ) ) + manifest_ctx = repo.changectx( changeset ) + for ctx_file in manifest_ctx.files(): + ctx_file_name = basic_util.strip_path( ctx_file ) + if ctx_file_name == stripped_filename: + return manifest_ctx, ctx_file + return None, None + +def get_file_context_from_ctx( ctx, filename ): + """Return the mercurial file context for a specified file.""" + # We have to be careful in determining if we found the correct file because multiple files with + # the same name may be in different directories within ctx if the files were moved within the change + # set. For example, in the following ctx.files() list, the former may have been moved to the latter: + # ['tmap_wrapper_0.0.19/tool_data_table_conf.xml.sample', 'tmap_wrapper_0.3.3/tool_data_table_conf.xml.sample']. + # Another scenario is that the file has been deleted. + deleted = False + filename = basic_util.strip_path( filename ) + for ctx_file in ctx.files(): + ctx_file_name = basic_util.strip_path( ctx_file ) + if filename == ctx_file_name: + try: + # If the file was moved, its destination will be returned here. + fctx = ctx[ ctx_file ] + return fctx + except LookupError, e: + # Set deleted for now, and continue looking in case the file was moved instead of deleted. + deleted = True + if deleted: + return 'DELETED' + return None + def get_mercurial_default_options_dict( command, command_table=None, **kwd ): '''Borrowed from repoman - get default parameters for a mercurial command.''' if command_table is None: @@ -62,6 +141,32 @@ default_options_dict[ option ] = kwd[ option ] return default_options_dict +def get_named_tmpfile_from_ctx( ctx, filename, dir ): + """ + Return a named temporary file created from a specified file with a given name included in a repository + changeset revision. + """ + filename = basic_util.strip_path( filename ) + for ctx_file in ctx.files(): + ctx_file_name = basic_util.strip_path( ctx_file ) + if filename == ctx_file_name: + try: + # If the file was moved, its destination file contents will be returned here. + fctx = ctx[ ctx_file ] + except LookupError, e: + # Continue looking in case the file was moved. + fctx = None + continue + if fctx: + fh = tempfile.NamedTemporaryFile( 'wb', prefix="tmp-toolshed-gntfc", dir=dir ) + tmp_filename = fh.name + fh.close() + fh = open( tmp_filename, 'wb' ) + fh.write( fctx.data() ) + fh.close() + return tmp_filename + return None + def get_readable_ctx_date( ctx ): """Convert the date of the changeset (the received ctx) to a human-readable date.""" t, tz = ctx.date() @@ -75,6 +180,18 @@ if repo_path is not None: return hg.repository( get_configured_ui(), repo_path, create=create ) +def get_repository_heads( repo ): + """Return current repository heads, which are changesets with no child changesets.""" + heads = [ repo[ h ] for h in repo.heads( None ) ] + return heads + +def get_reversed_changelog_changesets( repo ): + """Return a list of changesets in reverse order from that provided by the repository manifest.""" + reversed_changelog = [] + for changeset in repo.changelog: + reversed_changelog.insert( 0, changeset ) + return reversed_changelog + def get_revision_label( trans, repository, changeset_revision, include_date=True, include_hash=True ): """ Return a string consisting of the human read-able changeset rev and the changeset revision string @@ -146,6 +263,38 @@ label = "-1:%s" % changeset_revision return rev, label +def reversed_lower_upper_bounded_changelog( repo, excluded_lower_bounds_changeset_revision, included_upper_bounds_changeset_revision ): + """ + Return a reversed list of changesets in the repository changelog after the excluded_lower_bounds_changeset_revision, + but up to and including the included_upper_bounds_changeset_revision. The value of excluded_lower_bounds_changeset_revision + will be the value of INITIAL_CHANGELOG_HASH if no valid changesets exist before included_upper_bounds_changeset_revision. + """ + # To set excluded_lower_bounds_changeset_revision, calling methods should do the following, where the value + # of changeset_revision is a downloadable changeset_revision. + # excluded_lower_bounds_changeset_revision = \ + # metadata_util.get_previous_metadata_changeset_revision( repository, repo, changeset_revision, downloadable=? ) + if excluded_lower_bounds_changeset_revision == INITIAL_CHANGELOG_HASH: + appending_started = True + else: + appending_started = False + reversed_changelog = [] + for changeset in repo.changelog: + changeset_hash = str( repo.changectx( changeset ) ) + if appending_started: + reversed_changelog.insert( 0, changeset ) + if changeset_hash == excluded_lower_bounds_changeset_revision and not appending_started: + appending_started = True + if changeset_hash == included_upper_bounds_changeset_revision: + break + return reversed_changelog + +def reversed_upper_bounded_changelog( repo, included_upper_bounds_changeset_revision ): + """ + Return a reversed list of changesets in the repository changelog up to and including the + included_upper_bounds_changeset_revision. + """ + return reversed_lower_upper_bounded_changelog( repo, INITIAL_CHANGELOG_HASH, included_upper_bounds_changeset_revision ) + def update_repository( repo, ctx_rev=None ): """ Update the cloned repository to changeset_revision. It is critical that the installed repository is updated to the desired diff -r f6aa2d17d38430e92beb9da80f53c9989dd504ee -r dbc3d5c3506ed833dc453faa4f00797d95b8a8bb lib/tool_shed/util/metadata_util.py --- a/lib/tool_shed/util/metadata_util.py +++ b/lib/tool_shed/util/metadata_util.py @@ -13,6 +13,7 @@ import tool_shed.util.shed_util_common as suc from tool_shed.repository_types.metadata import TipOnly +from tool_shed.util import basic_util from tool_shed.util import common_util from tool_shed.util import common_install_util from tool_shed.util import container_util @@ -526,7 +527,7 @@ tool_config = sub_elem.attrib[ 'file' ] target_datatype = sub_elem.attrib[ 'target_datatype' ] # Parse the tool_config to get the guid. - tool_config_path = suc.get_config_from_disk( tool_config, repository_files_dir ) + tool_config_path = hg_util.get_config_from_disk( tool_config, repository_files_dir ) full_path = os.path.abspath( tool_config_path ) tool, valid, error_message = tool_util.load_tool_from_config( app, app.security.encode_id( repository.id ), full_path ) if tool is None: @@ -632,7 +633,7 @@ app.config.tool_data_path = work_dir #FIXME: Thread safe? app.config.tool_data_table_config_path = work_dir # Handle proprietary datatypes, if any. - datatypes_config = suc.get_config_from_disk( suc.DATATYPES_CONFIG_FILENAME, files_dir ) + datatypes_config = hg_util.get_config_from_disk( suc.DATATYPES_CONFIG_FILENAME, files_dir ) if datatypes_config: metadata_dict = generate_datatypes_metadata( app, repository, repository_clone_url, files_dir, datatypes_config, metadata_dict ) # Get the relative path to all sample files included in the repository for storage in the repository's metadata. @@ -749,14 +750,14 @@ metadata_dict = generate_data_manager_metadata( app, repository, files_dir, - suc.get_config_from_disk( suc.REPOSITORY_DATA_MANAGER_CONFIG_FILENAME, files_dir ), + hg_util.get_config_from_disk( suc.REPOSITORY_DATA_MANAGER_CONFIG_FILENAME, files_dir ), metadata_dict, shed_config_dict=shed_config_dict ) if readme_files: metadata_dict[ 'readme_files' ] = readme_files # This step must be done after metadata for tools has been defined. - tool_dependencies_config = suc.get_config_from_disk( 'tool_dependencies.xml', files_dir ) + tool_dependencies_config = hg_util.get_config_from_disk( 'tool_dependencies.xml', files_dir ) if tool_dependencies_config: metadata_dict, error_message = generate_tool_dependency_metadata( app, repository, @@ -772,7 +773,7 @@ # Reset the value of the app's tool_data_path and tool_data_table_config_path to their respective original values. app.config.tool_data_path = original_tool_data_path app.config.tool_data_table_config_path = original_tool_data_table_config_path - suc.remove_dir( work_dir ) + basic_util.remove_dir( work_dir ) return metadata_dict, invalid_file_tups def generate_package_dependency_metadata( app, elem, valid_tool_dependencies_dict, invalid_tool_dependencies_dict ): @@ -1006,7 +1007,7 @@ outputs = [] for output in ttb.outputs: name, file_name, extra = output - outputs.append( ( name, suc.strip_path( file_name ) if file_name else None ) ) + outputs.append( ( name, basic_util.strip_path( file_name ) if file_name else None ) ) if file_name not in required_files and file_name is not None: required_files.append( file_name ) test_dict = dict( name=str( ttb.name ), @@ -1078,6 +1079,29 @@ # The tool did not change through all of the changeset revisions. return old_id +def get_previous_metadata_changeset_revision( repository, repo, before_changeset_revision, downloadable=True ): + """ + Return the changeset_revision in the repository changelog that has associated metadata prior to + the changeset to which before_changeset_revision refers. If there isn't one, return the hash value + of an empty repository changelog, hg_util.INITIAL_CHANGELOG_HASH. + """ + changeset_revisions = suc.get_ordered_metadata_changeset_revisions( repository, repo, downloadable=downloadable ) + if len( changeset_revisions ) == 1: + changeset_revision = changeset_revisions[ 0 ] + if changeset_revision == before_changeset_revision: + return hg_util.INITIAL_CHANGELOG_HASH + return changeset_revision + previous_changeset_revision = None + for changeset_revision in changeset_revisions: + if changeset_revision == before_changeset_revision: + if previous_changeset_revision: + return previous_changeset_revision + else: + # Return the hash value of an empty repository changelog - note that this will not be a valid changeset revision. + return hg_util.INITIAL_CHANGELOG_HASH + else: + previous_changeset_revision = changeset_revision + def get_relative_path_to_repository_file( root, name, relative_install_dir, work_dir, shed_config_dict, resetting_all_metadata_on_repository ): if resetting_all_metadata_on_repository: full_path_to_file = os.path.join( root, name ) @@ -1344,6 +1368,13 @@ return True return False +def is_malicious( app, id, changeset_revision, **kwd ): + """Check the malicious flag in repository metadata for a specified change set revision.""" + repository_metadata = suc.get_repository_metadata_by_changeset_revision( app, id, changeset_revision ) + if repository_metadata: + return repository_metadata.malicious + return False + def new_datatypes_metadata_required( trans, repository_metadata, metadata_dict ): """ Compare the last saved metadata for each datatype in the repository with the new metadata in metadata_dict to determine if a new @@ -1821,7 +1852,7 @@ changeset_revisions.append( metadata_changeset_revision ) ancestor_changeset_revision = None ancestor_metadata_dict = None - suc.remove_dir( work_dir ) + basic_util.remove_dir( work_dir ) # Delete all repository_metadata records for this repository that do not have a changeset_revision # value in changeset_revisions. clean_repository_metadata( trans, id, changeset_revisions ) diff -r f6aa2d17d38430e92beb9da80f53c9989dd504ee -r dbc3d5c3506ed833dc453faa4f00797d95b8a8bb lib/tool_shed/util/readme_util.py --- a/lib/tool_shed/util/readme_util.py +++ b/lib/tool_shed/util/readme_util.py @@ -10,6 +10,7 @@ from galaxy.util import unicodify import tool_shed.util.shed_util_common as suc +from tool_shed.util import basic_util from tool_shed.util import common_util from tool_shed.util import hg_util @@ -46,7 +47,7 @@ log.exception( "Error reading README file '%s' from disk: %s" % ( str( relative_path_to_readme_file ), str( e ) ) ) text = None if text: - text_of_reasonable_length = suc.size_string( text ) + text_of_reasonable_length = basic_util.size_string( text ) if text_of_reasonable_length.find( '.. image:: ' ) >= 0: # Handle image display for README files that are contained in repositories in the tool shed or installed into Galaxy. lock = threading.Lock() @@ -69,17 +70,17 @@ host_url=web.url_for( '/', qualified=True ) ) text_of_reasonable_length = unicodify( text_of_reasonable_length ) else: - text_of_reasonable_length = suc.to_html_string( text_of_reasonable_length ) + text_of_reasonable_length = basic_util.to_html_string( text_of_reasonable_length ) readme_files_dict[ readme_file_name ] = text_of_reasonable_length else: # We must be in the tool shed and have an old changeset_revision, so we need to retrieve the file contents from the repository manifest. ctx = hg_util.get_changectx_for_changeset( repo, changeset_revision ) if ctx: - fctx = suc.get_file_context_from_ctx( ctx, readme_file_name ) + fctx = hg_util.get_file_context_from_ctx( ctx, readme_file_name ) if fctx and fctx not in [ 'DELETED' ]: try: text = unicodify( fctx.data() ) - readme_files_dict[ readme_file_name ] = suc.size_string( text ) + readme_files_dict[ readme_file_name ] = basic_util.size_string( text ) except Exception, e: log.exception( "Error reading README file '%s' from repository manifest: %s" % \ ( str( relative_path_to_readme_file ), str( e ) ) ) diff -r f6aa2d17d38430e92beb9da80f53c9989dd504ee -r dbc3d5c3506ed833dc453faa4f00797d95b8a8bb lib/tool_shed/util/review_util.py --- a/lib/tool_shed/util/review_util.py +++ b/lib/tool_shed/util/review_util.py @@ -55,7 +55,7 @@ repo = hg_util.get_repo_for_repository( trans.app, repository=repository, repo_path=None, create=False ) reviewed_revision_hashes = [ review.changeset_revision for review in repository.reviews ] previous_reviews_dict = odict() - for changeset in suc.reversed_upper_bounded_changelog( repo, changeset_revision ): + for changeset in hg_util.reversed_upper_bounded_changelog( repo, changeset_revision ): previous_changeset_revision = str( repo.changectx( changeset ) ) if previous_changeset_revision in reviewed_revision_hashes: previous_rev, previous_changeset_revision_label = hg_util.get_rev_label_from_changeset_revision( repo, previous_changeset_revision ) @@ -89,7 +89,7 @@ """Determine if a repository has a changeset revision review prior to the received changeset revision.""" repo = hg_util.get_repo_for_repository( trans.app, repository=repository, repo_path=None, create=False ) reviewed_revision_hashes = [ review.changeset_revision for review in repository.reviews ] - for changeset in suc.reversed_upper_bounded_changelog( repo, changeset_revision ): + for changeset in hg_util.reversed_upper_bounded_changelog( repo, changeset_revision ): previous_changeset_revision = str( repo.changectx( changeset ) ) if previous_changeset_revision in reviewed_revision_hashes: return True diff -r f6aa2d17d38430e92beb9da80f53c9989dd504ee -r dbc3d5c3506ed833dc453faa4f00797d95b8a8bb lib/tool_shed/util/shed_util_common.py --- a/lib/tool_shed/util/shed_util_common.py +++ b/lib/tool_shed/util/shed_util_common.py @@ -7,13 +7,13 @@ from galaxy import util from galaxy.util import asbool from galaxy.util import json -from galaxy.util import unicodify from galaxy.web import url_for from galaxy.web.form_builder import SelectField from galaxy.datatypes import checkers from galaxy.model.orm import and_ from galaxy.model.orm import or_ import sqlalchemy.orm.exc +from tool_shed.util import basic_util from tool_shed.util import common_util from tool_shed.util import encoding_util from tool_shed.util import hg_util @@ -22,18 +22,11 @@ from xml.etree import ElementTree as XmlET from urllib2 import HTTPError -from galaxy import eggs - -eggs.require( 'markupsafe' ) -import markupsafe - log = logging.getLogger( __name__ ) CHUNK_SIZE = 2**20 # 1Mb -INITIAL_CHANGELOG_HASH = '000000000000' MAX_CONTENT_SIZE = 1048576 MAXDIFFSIZE = 8000 -MAX_DISPLAY_SIZE = 32768 DATATYPES_CONFIG_FILENAME = 'datatypes_conf.xml' REPOSITORY_DATA_MANAGER_CONFIG_FILENAME = 'data_manager_conf.xml' @@ -124,13 +117,6 @@ tool_dependencies_select_field.add_option( option_label, option_value ) return tool_dependencies_select_field -def changeset_is_malicious( app, id, changeset_revision, **kwd ): - """Check the malicious flag in repository metadata for a specified change set""" - repository_metadata = get_repository_metadata_by_changeset_revision( app, id, changeset_revision ) - if repository_metadata: - return repository_metadata.malicious - return False - def check_or_update_tool_shed_status_for_installed_repository( trans, repository ): updated = False tool_shed_status_dict = get_tool_shed_status_for_installed_repository( trans.app, repository ) @@ -157,19 +143,6 @@ shutil.move( filename, os.path.abspath( config_filename ) ) os.chmod( config_filename, 0644 ) -def copy_file_from_manifest( repo, ctx, filename, dir ): - """Copy the latest version of the file named filename from the repository manifest to the directory to which dir refers.""" - for changeset in reversed_upper_bounded_changelog( repo, ctx ): - changeset_ctx = repo.changectx( changeset ) - fctx = get_file_context_from_ctx( changeset_ctx, filename ) - if fctx and fctx not in [ 'DELETED' ]: - file_path = os.path.join( dir, filename ) - fh = open( file_path, 'wb' ) - fh.write( fctx.data() ) - fh.close() - return file_path - return None - def create_or_update_tool_shed_repository( app, name, description, installed_changeset_revision, ctx_rev, repository_clone_url, metadata_dict, status, current_changeset_revision=None, owner='', dist_to_shed=False ): """ @@ -337,7 +310,7 @@ for tool_dict in metadata[ 'tools' ]: guid = tool_dict[ 'guid' ] tool_config = tool_dict[ 'tool_config' ] - file_name = strip_path( tool_config ) + file_name = basic_util.strip_path( tool_config ) guids_and_configs[ guid ] = file_name # Parse the shed_tool_conf file in which all of this repository's tools are defined and generate the tool_panel_dict. tree, error_message = xml_util.parse_xml( shed_tool_conf ) @@ -389,7 +362,7 @@ def get_absolute_path_to_file_in_repository( repo_files_dir, file_name ): """Return the absolute path to a specified disk file contained in a repository.""" - stripped_file_name = strip_path( file_name ) + stripped_file_name = basic_util.strip_path( file_name ) file_path = None for root, dirs, files in os.walk( repo_files_dir ): if root.find( '.hg' ) < 0: @@ -416,25 +389,6 @@ except sqlalchemy.orm.exc.NoResultFound: return None -def get_config( config_file, repo, ctx, dir ): - """Return the latest version of config_filename from the repository manifest.""" - config_file = strip_path( config_file ) - for changeset in reversed_upper_bounded_changelog( repo, ctx ): - changeset_ctx = repo.changectx( changeset ) - for ctx_file in changeset_ctx.files(): - ctx_file_name = strip_path( ctx_file ) - if ctx_file_name == config_file: - return get_named_tmpfile_from_ctx( changeset_ctx, ctx_file, dir ) - return None - -def get_config_from_disk( config_file, relative_install_dir ): - for root, dirs, files in os.walk( relative_install_dir ): - if root.find( '.hg' ) < 0: - for name in files: - if name == config_file: - return os.path.abspath( os.path.join( root, name ) ) - return None - def get_ctx_rev( app, tool_shed_url, name, owner, changeset_revision ): """ Send a request to the tool shed to retrieve the ctx_rev for a repository defined by the @@ -447,21 +401,6 @@ ctx_rev = common_util.tool_shed_get( app, tool_shed_url, url ) return ctx_rev -def get_ctx_file_path_from_manifest( filename, repo, changeset_revision ): - """ - Get the ctx file path for the latest revision of filename from the repository manifest up - to the value of changeset_revision. - """ - stripped_filename = strip_path( filename ) - for changeset in reversed_upper_bounded_changelog( repo, changeset_revision ): - manifest_changeset_revision = str( repo.changectx( changeset ) ) - manifest_ctx = repo.changectx( changeset ) - for ctx_file in manifest_ctx.files(): - ctx_file_name = strip_path( ctx_file ) - if ctx_file_name == stripped_filename: - return manifest_ctx, ctx_file - return None, None - def get_current_repository_metadata_for_changeset_revision( app, repository, changeset_revision ): encoded_repository_id = app.security.encode_id( repository.id ) repository_metadata = get_repository_metadata_by_changeset_revision( app, @@ -537,40 +476,6 @@ dependent_downloadable_revisions.append( downloadable_revision ) return dependent_downloadable_revisions -def get_file_context_from_ctx( ctx, filename ): - """Return the mercurial file context for a specified file.""" - # We have to be careful in determining if we found the correct file because multiple files with - # the same name may be in different directories within ctx if the files were moved within the change - # set. For example, in the following ctx.files() list, the former may have been moved to the latter: - # ['tmap_wrapper_0.0.19/tool_data_table_conf.xml.sample', 'tmap_wrapper_0.3.3/tool_data_table_conf.xml.sample']. - # Another scenario is that the file has been deleted. - deleted = False - filename = strip_path( filename ) - for ctx_file in ctx.files(): - ctx_file_name = strip_path( ctx_file ) - if filename == ctx_file_name: - try: - # If the file was moved, its destination will be returned here. - fctx = ctx[ ctx_file ] - return fctx - except LookupError, e: - # Set deleted for now, and continue looking in case the file was moved instead of deleted. - deleted = True - if deleted: - return 'DELETED' - return None - -def get_file_type_str( changeset_revision, file_type ): - if file_type == 'zip': - file_type_str = '%s.zip' % changeset_revision - elif file_type == 'bz2': - file_type_str = '%s.tar.bz2' % changeset_revision - elif file_type == 'gz': - file_type_str = '%s.tar.gz' % changeset_revision - else: - file_type_str = '' - return file_type_str - def get_ids_of_tool_shed_repositories_being_installed( trans, as_string=False ): installing_repository_ids = [] new_status = trans.install_model.ToolShedRepository.installation_status.NEW @@ -603,7 +508,7 @@ changeset_revisions = get_ordered_metadata_changeset_revisions( repository, repo, downloadable=False ) if changeset_revisions: return changeset_revisions[ -1 ] - return INITIAL_CHANGELOG_HASH + return hg_util.INITIAL_CHANGELOG_HASH def get_latest_downloadable_changeset_revision( app, repository, repo ): repository_tip = repository.tip( app ) @@ -613,30 +518,7 @@ changeset_revisions = get_ordered_metadata_changeset_revisions( repository, repo, downloadable=True ) if changeset_revisions: return changeset_revisions[ -1 ] - return INITIAL_CHANGELOG_HASH - -def get_named_tmpfile_from_ctx( ctx, filename, dir ): - """Return a named temporary file created from a specified file with a given name included in a repository changeset revision.""" - filename = strip_path( filename ) - for ctx_file in ctx.files(): - ctx_file_name = strip_path( ctx_file ) - if filename == ctx_file_name: - try: - # If the file was moved, its destination file contents will be returned here. - fctx = ctx[ ctx_file ] - except LookupError, e: - # Continue looking in case the file was moved. - fctx = None - continue - if fctx: - fh = tempfile.NamedTemporaryFile( 'wb', prefix="tmp-toolshed-gntfc", dir=dir ) - tmp_filename = fh.name - fh.close() - fh = open( tmp_filename, 'wb' ) - fh.write( fctx.data() ) - fh.close() - return tmp_filename - return None + return hg_util.INITIAL_CHANGELOG_HASH def get_next_downloadable_changeset_revision( repository, repo, after_changeset_revision ): """ @@ -737,28 +619,6 @@ sorted_changeset_revisions = [ str( changeset_tup[ 1 ] ) for changeset_tup in sorted_changeset_tups ] return sorted_changeset_revisions -def get_previous_metadata_changeset_revision( repository, repo, before_changeset_revision, downloadable=True ): - """ - Return the changeset_revision in the repository changelog that has associated metadata prior to the changeset to which - before_changeset_revision refers. If there isn't one, return the hash value of an empty repository changelog, INITIAL_CHANGELOG_HASH. - """ - changeset_revisions = get_ordered_metadata_changeset_revisions( repository, repo, downloadable=downloadable ) - if len( changeset_revisions ) == 1: - changeset_revision = changeset_revisions[ 0 ] - if changeset_revision == before_changeset_revision: - return INITIAL_CHANGELOG_HASH - return changeset_revision - previous_changeset_revision = None - for changeset_revision in changeset_revisions: - if changeset_revision == before_changeset_revision: - if previous_changeset_revision: - return previous_changeset_revision - else: - # Return the hash value of an empty repository changelog - note that this will not be a valid changeset revision. - return INITIAL_CHANGELOG_HASH - else: - previous_changeset_revision = changeset_revision - def get_prior_import_or_install_required_dict( trans, tsr_ids, repo_info_dicts ): """ This method is used in the Tool Shed when exporting a repository and its dependencies, and in Galaxy when a repository and its dependencies @@ -974,7 +834,7 @@ else: safe_str = '' for i, line in enumerate( open( file_path ) ): - safe_str = '%s%s' % ( safe_str, to_html_string( line ) ) + safe_str = '%s%s' % ( safe_str, basic_util.to_html_string( line ) ) # Stop reading after string is larger than MAX_CONTENT_SIZE. if len( safe_str ) > MAX_CONTENT_SIZE: large_str = \ @@ -982,13 +842,17 @@ util.nice_size( MAX_CONTENT_SIZE ) safe_str = '%s%s' % ( safe_str, large_str ) break - if len( safe_str ) > MAX_DISPLAY_SIZE: - # Eliminate the middle of the file to display a file no larger than MAX_DISPLAY_SIZE. This may not be ideal if the file is larger - # than MAX_CONTENT_SIZE. + if len( safe_str ) > basic_util.MAX_DISPLAY_SIZE: + # Eliminate the middle of the file to display a file no larger than basic_util.MAX_DISPLAY_SIZE. + # This may not be ideal if the file is larger than MAX_CONTENT_SIZE. join_by_str = \ "<br/><br/>...some text eliminated here because file size is larger than maximum viewing size of %s...<br/><br/>" % \ - util.nice_size( MAX_DISPLAY_SIZE ) - safe_str = util.shrink_string_by_size( safe_str, MAX_DISPLAY_SIZE, join_by=join_by_str, left_larger=True, beginning_on_size_error=True ) + util.nice_size( basic_util.MAX_DISPLAY_SIZE ) + safe_str = util.shrink_string_by_size( safe_str, + basic_util.MAX_DISPLAY_SIZE, + join_by=join_by_str, + left_larger=True, + beginning_on_size_error=True ) return safe_str def get_repository_files( trans, folder_path ): @@ -1023,11 +887,6 @@ # This should never be reached - raise an exception? return v, None -def get_repository_heads( repo ): - """Return current repository heads, which are changesets with no child changesets.""" - heads = [ repo[ h ] for h in repo.heads( None ) ] - return heads - def get_repository_ids_requiring_prior_import_or_install( trans, tsr_ids, repository_dependencies ): """ This method is used in the Tool Shed when exporting a repository and its dependencies, and in Galaxy when a repository and its dependencies @@ -1131,20 +990,13 @@ repository_tools_tups.append( ( relative_path, guid, tool ) ) return repository_tools_tups -def get_reversed_changelog_changesets( repo ): - """Return a list of changesets in reverse order from that provided by the repository manifest.""" - reversed_changelog = [] - for changeset in repo.changelog: - reversed_changelog.insert( 0, changeset ) - return reversed_changelog - def get_shed_tool_conf_dict( app, shed_tool_conf ): """Return the in-memory version of the shed_tool_conf file, which is stored in the config_elems entry in the shed_tool_conf_dict associated with the file.""" for index, shed_tool_conf_dict in enumerate( app.toolbox.shed_tool_confs ): if shed_tool_conf == shed_tool_conf_dict[ 'config_filename' ]: return index, shed_tool_conf_dict else: - file_name = strip_path( shed_tool_conf_dict[ 'config_filename' ] ) + file_name = basic_util.strip_path( shed_tool_conf_dict[ 'config_filename' ] ) if shed_tool_conf == file_name: return index, shed_tool_conf_dict @@ -1188,7 +1040,7 @@ if config_filename == shed_tool_conf: return shed_tool_conf_dict[ 'tool_path' ] else: - file_name = strip_path( config_filename ) + file_name = basic_util.strip_path( config_filename ) if file_name == shed_tool_conf: return shed_tool_conf_dict[ 'tool_path' ] return None @@ -1331,9 +1183,10 @@ repo = hg_util.get_repo_for_repository( trans.app, repository=repository, repo_path=None, create=False ) # Get the upper bound changeset revision. upper_bound_changeset_revision = get_next_downloadable_changeset_revision( repository, repo, changeset_revision ) - # Build the list of changeset revision hashes defining each available update up to, but excluding, upper_bound_changeset_revision. + # Build the list of changeset revision hashes defining each available update up to, but excluding + # upper_bound_changeset_revision. changeset_hashes = [] - for changeset in reversed_lower_upper_bounded_changelog( repo, changeset_revision, upper_bound_changeset_revision ): + for changeset in hg_util.reversed_lower_upper_bounded_changelog( repo, changeset_revision, upper_bound_changeset_revision ): # Make sure to exclude upper_bound_changeset_revision. if changeset != upper_bound_changeset_revision: changeset_hashes.append( str( repo.changectx( changeset ) ) ) @@ -1477,7 +1330,10 @@ return False def open_repository_files_folder( trans, folder_path ): - """Return a list of dictionaries, each of which contains information for a file or directory contained within a directory in a repository file hierarchy.""" + """ + Return a list of dictionaries, each of which contains information for a file or directory contained + within a directory in a repository file hierarchy. + """ try: files_list = get_repository_files( trans, folder_path ) except OSError, e: @@ -1499,28 +1355,6 @@ folder_contents.append( node ) return folder_contents -def pretty_print( dict=None ): - if dict is not None: - return json.to_json_string( dict, sort_keys=True, indent=4 ) - -def remove_dir( dir ): - """Attempt to remove a directory from disk.""" - if dir: - if os.path.exists( dir ): - try: - shutil.rmtree( dir ) - except: - pass - -def remove_file( file_name ): - """Attempt to remove a file from disk.""" - if file_name: - if os.path.exists( file_name ): - try: - os.remove( file_name ) - except: - pass - def repository_was_previously_installed( trans, tool_shed_url, repository_name, repo_info_tuple ): """ Find out if a repository is already installed into Galaxy - there are several scenarios where this @@ -1584,34 +1418,6 @@ trans.install_model.context.add( repository ) trans.install_model.context.flush() -def reversed_lower_upper_bounded_changelog( repo, excluded_lower_bounds_changeset_revision, included_upper_bounds_changeset_revision ): - """ - Return a reversed list of changesets in the repository changelog after the excluded_lower_bounds_changeset_revision, but up to and - including the included_upper_bounds_changeset_revision. The value of excluded_lower_bounds_changeset_revision will be the value of - INITIAL_CHANGELOG_HASH if no valid changesets exist before included_upper_bounds_changeset_revision. - """ - # To set excluded_lower_bounds_changeset_revision, calling methods should do the following, where the value of changeset_revision - # is a downloadable changeset_revision. - # excluded_lower_bounds_changeset_revision = get_previous_metadata_changeset_revision( repository, repo, changeset_revision, downloadable=? ) - if excluded_lower_bounds_changeset_revision == INITIAL_CHANGELOG_HASH: - appending_started = True - else: - appending_started = False - reversed_changelog = [] - for changeset in repo.changelog: - changeset_hash = str( repo.changectx( changeset ) ) - if appending_started: - reversed_changelog.insert( 0, changeset ) - if changeset_hash == excluded_lower_bounds_changeset_revision and not appending_started: - appending_started = True - if changeset_hash == included_upper_bounds_changeset_revision: - break - return reversed_changelog - -def reversed_upper_bounded_changelog( repo, included_upper_bounds_changeset_revision ): - """Return a reversed list of changesets in the repository changelog up to and including the included_upper_bounds_changeset_revision.""" - return reversed_lower_upper_bounded_changelog( repo, INITIAL_CHANGELOG_HASH, included_upper_bounds_changeset_revision ) - def set_image_paths( app, encoded_repository_id, text ): """ Handle tool help image display for tools that are contained in repositories in the tool shed or installed into Galaxy as well as image @@ -1666,41 +1472,6 @@ return str( required_rd_tup[ 4 ] ) return 'False' -def size_string( raw_text, size=MAX_DISPLAY_SIZE ): - """Return a subset of a string (up to MAX_DISPLAY_SIZE) translated to a safe string for display in a browser.""" - if raw_text and len( raw_text ) >= size: - large_str = '\nFile contents truncated because file size is larger than maximum viewing size of %s\n' % util.nice_size( size ) - raw_text = '%s%s' % ( raw_text[ 0:size ], large_str ) - return raw_text or '' - -def stringify( list ): - if list: - return ','.join( list ) - return '' - -def strip_path( fpath ): - """Attempt to strip the path from a file name.""" - if not fpath: - return fpath - try: - file_path, file_name = os.path.split( fpath ) - except: - file_name = fpath - return file_name - -def to_html_string( text ): - """Translates the characters in text to an html string""" - if text: - try: - text = unicodify( text ) - except UnicodeDecodeError, e: - return "Error decoding string: %s" % str( e ) - text = unicode( markupsafe.escape( text ) ) - text = text.replace( '\n', '<br/>' ) - text = text.replace( ' ', '    ' ) - text = text.replace( ' ', ' ' ) - return text - def tool_shed_from_repository_clone_url( repository_clone_url ): """Given a repository clone URL, return the tool shed that contains the repository.""" return common_util.remove_protocol_and_user_from_clone_url( repository_clone_url ).split( '/repos/' )[ 0 ].rstrip( '/' ) diff -r f6aa2d17d38430e92beb9da80f53c9989dd504ee -r dbc3d5c3506ed833dc453faa4f00797d95b8a8bb lib/tool_shed/util/tool_dependency_util.py --- a/lib/tool_shed/util/tool_dependency_util.py +++ b/lib/tool_shed/util/tool_dependency_util.py @@ -7,6 +7,8 @@ from galaxy.model.orm import or_ import tool_shed.util.shed_util_common as suc import tool_shed.repository_types.util as rt_util +from tool_shed.util import basic_util +from tool_shed.util import hg_util from tool_shed.util import xml_util from tool_shed.galaxy_install.tool_dependencies import td_common_util @@ -85,7 +87,7 @@ if shed_config_dict.get( 'tool_path' ): relative_install_dir = os.path.join( shed_config_dict.get( 'tool_path' ), relative_install_dir ) # Get the tool_dependencies.xml file from the repository. - tool_dependencies_config = suc.get_config_from_disk( 'tool_dependencies.xml', relative_install_dir ) + tool_dependencies_config = hg_util.get_config_from_disk( 'tool_dependencies.xml', relative_install_dir ) tree, error_message = xml_util.parse_xml( tool_dependencies_config ) if tree is None: return tool_dependency_objects @@ -629,7 +631,7 @@ error_message += ' prepared for re-installation.' print error_message tool_dependency.status = app.install_model.ToolDependency.installation_status.NEVER_INSTALLED - suc.remove_dir( tool_dependency_install_dir ) + basic_util.remove_dir( tool_dependency_install_dir ) can_install_tool_dependency = True sa_session.add( tool_dependency ) sa_session.flush() diff -r f6aa2d17d38430e92beb9da80f53c9989dd504ee -r dbc3d5c3506ed833dc453faa4f00797d95b8a8bb lib/tool_shed/util/tool_util.py --- a/lib/tool_shed/util/tool_util.py +++ b/lib/tool_shed/util/tool_util.py @@ -13,6 +13,7 @@ from galaxy.util.expressions import ExpressionContext from galaxy.web.form_builder import SelectField from galaxy.tools.actions.upload import UploadToolAction +from tool_shed.util import basic_util from tool_shed.util import common_util from tool_shed.util import hg_util from tool_shed.util import xml_util @@ -119,7 +120,7 @@ return False if changeset_revision == repository.tip( trans.app ): return True - file_name = suc.strip_path( file_path ) + file_name = basic_util.strip_path( file_path ) latest_version_of_file = get_latest_tool_config_revision_from_repository_manifest( repo, file_name, changeset_revision ) can_use_disk_file = filecmp.cmp( file_path, latest_version_of_file ) try: @@ -142,7 +143,7 @@ if options and isinstance( options, dynamic_options.DynamicOptions ): if options.tool_data_table or options.missing_tool_data_table_name: # Make sure the repository contains a tool_data_table_conf.xml.sample file. - sample_tool_data_table_conf = suc.get_config_from_disk( 'tool_data_table_conf.xml.sample', repo_dir ) + sample_tool_data_table_conf = hg_util.get_config_from_disk( 'tool_data_table_conf.xml.sample', repo_dir ) if sample_tool_data_table_conf: error, correction_msg = handle_sample_tool_data_table_conf_file( app, sample_tool_data_table_conf ) if error: @@ -158,10 +159,10 @@ if options.index_file or options.missing_index_file: # Make sure the repository contains the required xxx.loc.sample file. index_file = options.index_file or options.missing_index_file - index_file_name = suc.strip_path( index_file ) + index_file_name = basic_util.strip_path( index_file ) sample_found = False for sample_file in sample_files: - sample_file_name = suc.strip_path( sample_file ) + sample_file_name = basic_util.strip_path( sample_file ) if sample_file_name == '%s.sample' % index_file_name: options.index_file = index_file_name options.missing_index_file = None @@ -206,7 +207,7 @@ """ if dest_path is None: dest_path = os.path.abspath( app.config.tool_data_path ) - sample_file_name = suc.strip_path( filename ) + sample_file_name = basic_util.strip_path( filename ) copied_file = sample_file_name.replace( '.sample', '' ) full_source_path = os.path.abspath( filename ) full_destination_path = os.path.join( dest_path, sample_file_name ) @@ -312,7 +313,7 @@ {<Tool guid> : [{ tool_config : <tool_config_file>, id: <ToolSection id>, version : <ToolSection version>, name : <TooSection name>}]} """ tool_panel_dict = {} - file_name = suc.strip_path( tool_config ) + file_name = basic_util.strip_path( tool_config ) tool_section_dicts = generate_tool_section_dicts( tool_config=file_name, tool_sections=tool_sections ) tool_panel_dict[ guid ] = tool_section_dicts return tool_panel_dict @@ -412,11 +413,11 @@ This method is restricted to tool_config files rather than any file since it is likely that, with the exception of tool config files, multiple files will have the same name in various directories within the repository. """ - stripped_filename = suc.strip_path( filename ) - for changeset in suc.reversed_upper_bounded_changelog( repo, changeset_revision ): + stripped_filename = basic_util.strip_path( filename ) + for changeset in hg_util.reversed_upper_bounded_changelog( repo, changeset_revision ): manifest_ctx = repo.changectx( changeset ) for ctx_file in manifest_ctx.files(): - ctx_file_name = suc.strip_path( ctx_file ) + ctx_file_name = basic_util.strip_path( ctx_file ) if ctx_file_name == stripped_filename: try: fctx = manifest_ctx[ ctx_file ] @@ -442,14 +443,14 @@ """ deleted_sample_files = [] sample_files = [] - for changeset in suc.reversed_upper_bounded_changelog( repo, ctx ): + for changeset in hg_util.reversed_upper_bounded_changelog( repo, ctx ): changeset_ctx = repo.changectx( changeset ) for ctx_file in changeset_ctx.files(): - ctx_file_name = suc.strip_path( ctx_file ) + ctx_file_name = basic_util.strip_path( ctx_file ) # If we decide in the future that files deleted later in the changelog should not be used, we can use the following if statement. # if ctx_file_name.endswith( '.sample' ) and ctx_file_name not in sample_files and ctx_file_name not in deleted_sample_files: if ctx_file_name.endswith( '.sample' ) and ctx_file_name not in sample_files: - fctx = suc.get_file_context_from_ctx( changeset_ctx, ctx_file ) + fctx = hg_util.get_file_context_from_ctx( changeset_ctx, ctx_file ) if fctx in [ 'DELETED' ]: # Since the possibly future used if statement above is commented out, the same file that was initially added will be # discovered in an earlier changeset in the change log and fall through to the else block below. In other words, if @@ -536,7 +537,7 @@ version_lineage = [ guid ] # Get all ancestor guids of the received guid. current_child_guid = guid - for changeset in suc.reversed_upper_bounded_changelog( repo, repository_metadata.changeset_revision ): + for changeset in hg_util.reversed_upper_bounded_changelog( repo, repository_metadata.changeset_revision ): ctx = repo.changectx( changeset ) rm = suc.get_repository_metadata_by_changeset_revision( trans.app, repository_id, str( ctx ) ) if rm: @@ -546,9 +547,9 @@ current_child_guid = parent_guid # Get all descendant guids of the received guid. current_parent_guid = guid - for changeset in suc.reversed_lower_upper_bounded_changelog( repo, - repository_metadata.changeset_revision, - repository.tip( trans.app ) ): + for changeset in hg_util.reversed_lower_upper_bounded_changelog( repo, + repository_metadata.changeset_revision, + repository.tip( trans.app ) ): ctx = repo.changectx( changeset ) rm = suc.get_repository_metadata_by_changeset_revision( trans.app, repository_id, str( ctx ) ) if rm: @@ -574,7 +575,7 @@ break if missing_data_table_entry: # The repository must contain a tool_data_table_conf.xml.sample file that includes all required entries for all tools in the repository. - sample_tool_data_table_conf = suc.get_config_from_disk( 'tool_data_table_conf.xml.sample', relative_install_dir ) + sample_tool_data_table_conf = hg_util.get_config_from_disk( 'tool_data_table_conf.xml.sample', relative_install_dir ) if sample_tool_data_table_conf: # Add entries to the ToolDataTableManager's in-memory data_tables dictionary. error, message = handle_sample_tool_data_table_conf_file( app, sample_tool_data_table_conf, persist=True ) @@ -598,11 +599,11 @@ params_with_missing_index_file = repository_tool.params_with_missing_index_file for param in params_with_missing_index_file: options = param.options - missing_file_name = suc.strip_path( options.missing_index_file ) + missing_file_name = basic_util.strip_path( options.missing_index_file ) if missing_file_name not in sample_files_copied: # The repository must contain the required xxx.loc.sample file. for sample_file in sample_files: - sample_file_name = suc.strip_path( sample_file ) + sample_file_name = basic_util.strip_path( sample_file ) if sample_file_name == '%s.sample' % missing_file_name: copy_sample_file( app, sample_file ) if options.tool_data_table and options.tool_data_table.missing_index_file: @@ -643,7 +644,7 @@ error, message = handle_sample_tool_data_table_conf_file( trans.app, tool_data_table_config ) if error: log.debug( message ) - manifest_ctx, ctx_file = suc.get_ctx_file_path_from_manifest( tool_config_filename, repo, changeset_revision ) + manifest_ctx, ctx_file = hg_util.get_ctx_file_path_from_manifest( tool_config_filename, repo, changeset_revision ) if manifest_ctx and ctx_file: tool, message2 = load_tool_from_tmp_config( trans, repo, repository_id, manifest_ctx, ctx_file, work_dir ) message = concat_messages( message, message2 ) @@ -885,7 +886,7 @@ message = concat_messages( message, message2 ) else: tool, message, sample_files = handle_sample_files_and_load_tool_from_tmp_config( trans, repo, repository_id, changeset_revision, tool_config_filename, work_dir ) - suc.remove_dir( work_dir ) + basic_util.remove_dir( work_dir ) trans.app.config.tool_data_path = original_tool_data_path # Reset the tool_data_tables by loading the empty tool_data_table_conf.xml file. reset_tool_data_tables( trans.app ) @@ -911,7 +912,7 @@ def load_tool_from_tmp_config( trans, repo, repository_id, ctx, ctx_file, work_dir ): tool = None message = '' - tmp_tool_config = suc.get_named_tmpfile_from_ctx( ctx, ctx_file, work_dir ) + tmp_tool_config = hg_util.get_named_tmpfile_from_ctx( ctx, ctx_file, work_dir ) if tmp_tool_config: element_tree, error_message = xml_util.parse_xml( tmp_tool_config ) if element_tree is None: @@ -921,7 +922,7 @@ tmp_code_files = [] for code_elem in element_tree_root.findall( 'code' ): code_file_name = code_elem.get( 'file' ) - tmp_code_file_name = suc.copy_file_from_manifest( repo, ctx, code_file_name, work_dir ) + tmp_code_file_name = hg_util.copy_file_from_manifest( repo, ctx, code_file_name, work_dir ) if tmp_code_file_name: tmp_code_files.append( tmp_code_file_name ) tool, valid, message = load_tool_from_config( trans.app, repository_id, tmp_tool_config ) diff -r f6aa2d17d38430e92beb9da80f53c9989dd504ee -r dbc3d5c3506ed833dc453faa4f00797d95b8a8bb templates/admin/review_tool_migration_stages.mako --- a/templates/admin/review_tool_migration_stages.mako +++ b/templates/admin/review_tool_migration_stages.mako @@ -35,7 +35,7 @@ </p></div><table class="grid"> - <% from tool_shed.util.shed_util_common import to_html_string %> + <% from tool_shed.util.basic_util import to_html_string %> %for stage in migration_stages_dict.keys(): <% migration_command = 'sh ./scripts/migrate_tools/%04d_tools.sh' % stage diff -r f6aa2d17d38430e92beb9da80f53c9989dd504ee -r dbc3d5c3506ed833dc453faa4f00797d95b8a8bb templates/admin/tool_shed_repository/manage_repository_tool_dependencies.mako --- a/templates/admin/tool_shed_repository/manage_repository_tool_dependencies.mako +++ b/templates/admin/tool_shed_repository/manage_repository_tool_dependencies.mako @@ -31,7 +31,7 @@ %for tool_dependency in repository.tool_dependencies: <% if tool_dependency.error_message: - from tool_shed.util.shed_util_common import to_html_string + from tool_shed.util.basic_util import to_html_string error_message = to_html_string( tool_dependency.error_message ) else: error_message = '' This diff is so big that we needed to truncate the remainder. Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: 2 new changesets
by commits-noreply＠bitbucket.org 28 May '14

28 May '14

2 new commits in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/eed45027e219/ Changeset: eed45027e219 Branch: next-stable User: carlfeberhard Date: 2014-05-28 15:24:01 Summary: Bug fix: allow tags to be added in display_base.mako by swapping the load order of autocomplete plugins (autocomplete still not working) Affected #: 2 files diff -r 26952d141e42bd936eab9465c2761325204f9531 -r eed45027e219df9c4fc87fcfc788380ecc500b1a templates/display_base.mako --- a/templates/display_base.mako +++ b/templates/display_base.mako @@ -32,10 +32,19 @@ <%def name="javascripts()"> ${parent.javascripts()} - ${h.js( "libs/jquery/jstorage", "libs/jquery/jquery.autocomplete", "libs/jquery/jquery.rating", - "galaxy.autocom_tagging" )} - ${h.js( "galaxy.panels", "libs/jquery/jstorage", "libs/jquery/jquery.event.drag", "libs/jquery/jquery.event.hover", - "libs/jquery/jquery.mousewheel", "libs/jquery/jquery-ui", "libs/require", "libs/farbtastic" )} + ${h.js( + "libs/jquery/jstorage", + "libs/jquery/jquery.rating", + "galaxy.panels", + "libs/jquery/jquery.event.drag", + "libs/jquery/jquery.event.hover", + "libs/jquery/jquery.mousewheel", + "libs/jquery/jquery-ui", + "libs/require", + "libs/farbtastic", + "libs/jquery/jquery.autocomplete", + "galaxy.autocom_tagging" + )} <script type="text/javascript"> diff -r 26952d141e42bd936eab9465c2761325204f9531 -r eed45027e219df9c4fc87fcfc788380ecc500b1a templates/tagging_common.mako --- a/templates/tagging_common.mako +++ b/templates/tagging_common.mako @@ -211,7 +211,7 @@ use_toggle_link: ${iff( use_toggle_link, 'true', 'false' )} }; - $('#${elt_id}').find( 'input' ).autocomplete_tagging(options); + $('#${elt_id}').autocomplete_tagging(options); </script> ## Use style to hide/display the tag area. https://bitbucket.org/galaxy/galaxy-central/commits/f6aa2d17d384/ Changeset: f6aa2d17d384 User: carlfeberhard Date: 2014-05-28 15:25:23 Summary: merge Affected #: 2 files diff -r 950e68808ba693d8eeec87b4be9ce78e4b864456 -r f6aa2d17d38430e92beb9da80f53c9989dd504ee templates/display_base.mako --- a/templates/display_base.mako +++ b/templates/display_base.mako @@ -32,10 +32,19 @@ <%def name="javascripts()"> ${parent.javascripts()} - ${h.js( "libs/jquery/jstorage", "libs/jquery/jquery.autocomplete", "libs/jquery/jquery.rating", - "galaxy.autocom_tagging" )} - ${h.js( "galaxy.panels", "libs/jquery/jstorage", "libs/jquery/jquery.event.drag", "libs/jquery/jquery.event.hover", - "libs/jquery/jquery.mousewheel", "libs/jquery/jquery-ui", "libs/require", "libs/farbtastic" )} + ${h.js( + "libs/jquery/jstorage", + "libs/jquery/jquery.rating", + "galaxy.panels", + "libs/jquery/jquery.event.drag", + "libs/jquery/jquery.event.hover", + "libs/jquery/jquery.mousewheel", + "libs/jquery/jquery-ui", + "libs/require", + "libs/farbtastic", + "libs/jquery/jquery.autocomplete", + "galaxy.autocom_tagging" + )} <script type="text/javascript"> diff -r 950e68808ba693d8eeec87b4be9ce78e4b864456 -r f6aa2d17d38430e92beb9da80f53c9989dd504ee templates/tagging_common.mako --- a/templates/tagging_common.mako +++ b/templates/tagging_common.mako @@ -211,7 +211,7 @@ use_toggle_link: ${iff( use_toggle_link, 'true', 'false' )} }; - $('#${elt_id}').find( 'input' ).autocomplete_tagging(options); + $('#${elt_id}').autocomplete_tagging(options); </script> ## Use style to hide/display the tag area. Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: jgoecks: Remove jStorage from Trackster requirements because it is not used.
by commits-noreply＠bitbucket.org 27 May '14

27 May '14

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/950e68808ba6/ Changeset: 950e68808ba6 User: jgoecks Date: 2014-05-27 23:42:10 Summary: Remove jStorage from Trackster requirements because it is not used. Affected #: 1 file diff -r be58520000c71329d6e63830fc5776604c7b8382 -r 950e68808ba693d8eeec87b4be9ce78e4b864456 static/scripts/viz/trackster.js --- a/static/scripts/viz/trackster.js +++ b/static/scripts/viz/trackster.js @@ -8,7 +8,6 @@ [ // load js libraries 'utils/utils', - 'libs/jquery/jstorage', 'libs/jquery/jquery.event.drag', 'libs/jquery/jquery.event.hover', 'libs/jquery/jquery.mousewheel', Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0