[galaxy-commits] commit/galaxy-central: greg: Eliminate the Tool Shed's td_common_util.py module by moving the functions to appropriate classes.

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/b1b1e5cefca5/ Changeset: b1b1e5cefca5 User: greg Date: 2014-05-28 21:49:02 Summary: Eliminate the Tool Shed's td_common_util.py module by moving the functions to appropriate classes. Affected #: 15 files diff -r 31740dffaf9ae4e345fd2d4de4ab44ed133dd567 -r b1b1e5cefca5b0193c8623a9aee7a39cd1dda55c lib/galaxy/webapps/tool_shed/controllers/repository.py --- a/lib/galaxy/webapps/tool_shed/controllers/repository.py +++ b/lib/galaxy/webapps/tool_shed/controllers/repository.py @@ -3120,12 +3120,12 @@ options_dict = hg_util.get_mercurial_default_options_dict( 'diff' ) # Not quite sure if the following settings make any difference, but with a combination of them and the size check on each # diff, we don't run out of memory when viewing the changelog of the cisortho2 repository on the test tool shed. - options_dict[ 'maxfile' ] = suc.MAXDIFFSIZE - options_dict[ 'maxtotal' ] = suc.MAXDIFFSIZE + options_dict[ 'maxfile' ] = basic_util.MAXDIFFSIZE + options_dict[ 'maxtotal' ] = basic_util.MAXDIFFSIZE diffopts = mdiff.diffopts( **options_dict ) for diff in patch.diff( repo, node1=ctx_parent.node(), node2=ctx.node(), opts=diffopts ): - if len( diff ) > suc.MAXDIFFSIZE: - diff = util.shrink_string_by_size( diff, suc.MAXDIFFSIZE ) + if len( diff ) > basic_util.MAXDIFFSIZE: + diff = util.shrink_string_by_size( diff, basic_util.MAXDIFFSIZE ) diffs.append( basic_util.to_html_string( diff ) ) modified, added, removed, deleted, unknown, ignored, clean = repo.status( node1=ctx_parent.node(), node2=ctx.node() ) anchors = modified + added + removed + deleted + unknown + ignored + clean diff -r 31740dffaf9ae4e345fd2d4de4ab44ed133dd567 -r b1b1e5cefca5b0193c8623a9aee7a39cd1dda55c lib/tool_shed/galaxy_install/install_manager.py --- a/lib/tool_shed/galaxy_install/install_manager.py +++ b/lib/tool_shed/galaxy_install/install_manager.py @@ -1,5 +1,7 @@ import logging import os +import sys +import traceback from galaxy import eggs @@ -11,7 +13,6 @@ from tool_shed.util import tool_dependency_util -from tool_shed.galaxy_install.tool_dependencies import td_common_util from tool_shed.galaxy_install.tool_dependencies.recipe.env_file_builder import EnvFileBuilder from tool_shed.galaxy_install.tool_dependencies.recipe.install_environment import InstallEnvironment from tool_shed.galaxy_install.tool_dependencies.recipe.recipe_manager import StepManager @@ -25,6 +26,10 @@ class InstallManager( object ): + def format_traceback( self ): + ex_type, ex, tb = sys.exc_info() + return ''.join( traceback.format_tb( tb ) ) + def get_tool_shed_repository_install_dir( self, app, tool_shed_repository ): return os.path.abspath( tool_shed_repository.repo_files_directory( app ) ) @@ -112,7 +117,7 @@ log.exception( 'Error installing tool dependency %s version %s.', str( tool_dependency.name ), str( tool_dependency.version ) ) # Since there was an installation error, update the tool dependency status to Error. The remove_installation_path option must # be left False here. - error_message = '%s\n%s' % ( td_common_util.format_traceback(), str( e ) ) + error_message = '%s\n%s' % ( self.format_traceback(), str( e ) ) tool_dependency = tool_dependency_util.handle_tool_dependency_installation_error( app, tool_dependency, error_message, diff -r 31740dffaf9ae4e345fd2d4de4ab44ed133dd567 -r b1b1e5cefca5b0193c8623a9aee7a39cd1dda55c lib/tool_shed/galaxy_install/tool_dependencies/env_manager.py --- /dev/null +++ b/lib/tool_shed/galaxy_install/tool_dependencies/env_manager.py @@ -0,0 +1,147 @@ +import logging +import os +import sys +from tool_shed.util import common_util +import tool_shed.util.shed_util_common as suc + +log = logging.getLogger( __name__ ) + + +class EnvManager( object ): + + def __init__( self, app ): + self.app = app + + def create_env_var_dict( self, elem, install_environment ): + env_var_name = elem.get( 'name', 'PATH' ) + env_var_action = elem.get( 'action', 'prepend_to' ) + env_var_text = None + tool_dependency_install_dir = install_environment.install_dir + tool_shed_repository_install_dir = install_environment.tool_shed_repository_install_dir + if elem.text and elem.text.find( 'REPOSITORY_INSTALL_DIR' ) >= 0: + if tool_shed_repository_install_dir and elem.text.find( '$REPOSITORY_INSTALL_DIR' ) != -1: + env_var_text = elem.text.replace( '$REPOSITORY_INSTALL_DIR', tool_shed_repository_install_dir ) + return dict( name=env_var_name, action=env_var_action, value=env_var_text ) + else: + env_var_text = elem.text.replace( '$REPOSITORY_INSTALL_DIR', tool_dependency_install_dir ) + return dict( name=env_var_name, action=env_var_action, value=env_var_text ) + if elem.text and elem.text.find( 'INSTALL_DIR' ) >= 0: + if tool_dependency_install_dir: + env_var_text = elem.text.replace( '$INSTALL_DIR', tool_dependency_install_dir ) + return dict( name=env_var_name, action=env_var_action, value=env_var_text ) + else: + env_var_text = elem.text.replace( '$INSTALL_DIR', tool_shed_repository_install_dir ) + return dict( name=env_var_name, action=env_var_action, value=env_var_text ) + if elem.text: + # Allow for environment variables that contain neither REPOSITORY_INSTALL_DIR nor INSTALL_DIR + # since there may be command line parameters that are tuned for a Galaxy instance. Allowing them + # to be set in one location rather than being hard coded into each tool config is the best approach. + # For example: + # <environment_variable name="GATK2_SITE_OPTIONS" action="set_to"> + # "--num_threads 4 --num_cpu_threads_per_data_thread 3 --phone_home STANDARD" + # </environment_variable> + return dict( name=env_var_name, action=env_var_action, value=elem.text) + return None + + def get_env_shell_file_path( self, installation_directory ): + env_shell_file_name = 'env.sh' + default_location = os.path.abspath( os.path.join( installation_directory, env_shell_file_name ) ) + if os.path.exists( default_location ): + return default_location + for root, dirs, files in os.walk( installation_directory ): + for name in files: + if name == env_shell_file_name: + return os.path.abspath( os.path.join( root, name ) ) + return None + + def get_env_shell_file_paths( self, elem ): + # Currently only the following tag set is supported. + # <repository toolshed="http://localhost:9009/" name="package_numpy_1_7" owner="test" changeset_revision="c84c6a8be056"> + # <package name="numpy" version="1.7.1" /> + # </repository> + env_shell_file_paths = [] + toolshed = elem.get( 'toolshed', None ) + repository_name = elem.get( 'name', None ) + repository_owner = elem.get( 'owner', None ) + changeset_revision = elem.get( 'changeset_revision', None ) + if toolshed and repository_name and repository_owner and changeset_revision: + # The protocol is not stored, but the port is if it exists. + toolshed = common_util.remove_protocol_from_tool_shed_url( toolshed ) + repository = suc.get_repository_for_dependency_relationship( self.app, + toolshed, + repository_name, + repository_owner, + changeset_revision ) + if repository: + for sub_elem in elem: + tool_dependency_type = sub_elem.tag + tool_dependency_name = sub_elem.get( 'name' ) + tool_dependency_version = sub_elem.get( 'version' ) + if tool_dependency_type and tool_dependency_name and tool_dependency_version: + # Get the tool_dependency so we can get its installation directory. + tool_dependency = None + for tool_dependency in repository.tool_dependencies: + if tool_dependency.type == tool_dependency_type and \ + tool_dependency.name == tool_dependency_name and \ + tool_dependency.version == tool_dependency_version: + break + if tool_dependency: + tool_dependency_key = '%s/%s' % ( tool_dependency_name, tool_dependency_version ) + installation_directory = tool_dependency.installation_directory( self.app ) + env_shell_file_path = self.get_env_shell_file_path( installation_directory ) + if env_shell_file_path: + env_shell_file_paths.append( env_shell_file_path ) + else: + error_message = "Skipping tool dependency definition because unable to locate env.sh file for tool dependency " + error_message += "type %s, name %s, version %s for repository %s" % \ + ( str( tool_dependency_type ), str( tool_dependency_name ), str( tool_dependency_version ), str( repository.name ) ) + log.debug( error_message ) + continue + else: + error_message = "Skipping tool dependency definition because unable to locate tool dependency " + error_message += "type %s, name %s, version %s for repository %s" % \ + ( str( tool_dependency_type ), str( tool_dependency_name ), str( tool_dependency_version ), str( repository.name ) ) + log.debug( error_message ) + continue + else: + error_message = "Skipping invalid tool dependency definition: type %s, name %s, version %s." % \ + ( str( tool_dependency_type ), str( tool_dependency_name ), str( tool_dependency_version ) ) + log.debug( error_message ) + continue + else: + error_message = "Skipping set_environment_for_install definition because unable to locate required installed tool shed repository: " + error_message += "toolshed %s, name %s, owner %s, changeset_revision %s." % \ + ( str( toolshed ), str( repository_name ), str( repository_owner ), str( changeset_revision ) ) + log.debug( error_message ) + else: + error_message = "Skipping invalid set_environment_for_install definition: toolshed %s, name %s, owner %s, changeset_revision %s." % \ + ( str( toolshed ), str( repository_name ), str( repository_owner ), str( changeset_revision ) ) + log.debug( error_message ) + return env_shell_file_paths + + def get_env_shell_file_paths_from_setup_environment_elem( self, all_env_shell_file_paths, elem, action_dict ): + """ + Parse an XML tag set to discover all child repository dependency tags and define the path to an env.sh file associated + with the repository (this requires the repository dependency to be in an installed state). The received action_dict + will be updated with these discovered paths and returned to the caller. This method handles tool dependency definition + tag sets <setup_r_environment>, <setup_ruby_environment> and <setup_perl_environment>. + """ + # An example elem is: + # <action type="setup_perl_environment"> + # <repository name="package_perl_5_18" owner="iuc"> + # <package name="perl" version="5.18.1" /> + # </repository> + # <repository name="package_expat_2_1" owner="iuc" prior_installation_required="True"> + # <package name="expat" version="2.1.0" /> + # </repository> + # <package>http://search.cpan.org/CPAN/authors/id/T/TO/TODDR/XML-Parser-2.41.tar.gz</package> + # <package>http://search.cpan.org/CPAN/authors/id/L/LD/LDS/CGI.pm-3.43.tar.gz</package> + # </action> + for action_elem in elem: + if action_elem.tag == 'repository': + env_shell_file_paths = self.get_env_shell_file_paths( action_elem ) + all_env_shell_file_paths.extend( env_shell_file_paths ) + if all_env_shell_file_paths: + action_dict[ 'env_shell_file_paths' ] = all_env_shell_file_paths + action_dict[ 'action_shell_file_paths' ] = env_shell_file_paths + return action_dict diff -r 31740dffaf9ae4e345fd2d4de4ab44ed133dd567 -r b1b1e5cefca5b0193c8623a9aee7a39cd1dda55c lib/tool_shed/galaxy_install/tool_dependencies/recipe/install_environment.py --- a/lib/tool_shed/galaxy_install/tool_dependencies/recipe/install_environment.py +++ b/lib/tool_shed/galaxy_install/tool_dependencies/recipe/install_environment.py @@ -24,11 +24,13 @@ from galaxy.util import shrink_string_by_size from galaxy.util import unicodify -from tool_shed.galaxy_install.tool_dependencies import td_common_util from tool_shed.galaxy_install.tool_dependencies.recipe import asynchronous_reader +from tool_shed.util import basic_util + log = logging.getLogger( __name__ ) + class InstallEnvironment( object ): """Object describing the environment built up as part of the process of building and installing a package.""" @@ -44,7 +46,7 @@ self.tool_shed_repository_install_dir = tool_shed_repository_install_dir def __call__( self ): - with settings( warn_only=True, **td_common_util.get_env_var_values( self ) ): + with settings( warn_only=True, **basic_util.get_env_var_values( self ) ): with prefix( self.__setup_environment() ): yield @@ -125,7 +127,7 @@ context = app.install_model.context command = str( cmd ) output = self.handle_complex_command( command ) - self.log_results( cmd, output, os.path.join( self.install_dir, td_common_util.INSTALLATION_LOG ) ) + self.log_results( cmd, output, os.path.join( self.install_dir, basic_util.INSTALLATION_LOG ) ) stdout = output.stdout stderr = output.stderr if len( stdout ) > DATABASE_MAX_STRING_SIZE: @@ -214,18 +216,18 @@ # Sleep a bit before asking the readers again. time.sleep( .1 ) current_wait_time = time.time() - start_timer - if stdout_queue.empty() and stderr_queue.empty() and current_wait_time > td_common_util.NO_OUTPUT_TIMEOUT: + if stdout_queue.empty() and stderr_queue.empty() and current_wait_time > basic_util.NO_OUTPUT_TIMEOUT: err_msg = "\nShutting down process id %s because it generated no output for the defined timeout period of %.1f seconds.\n" % \ - ( pid, td_common_util.NO_OUTPUT_TIMEOUT ) + ( pid, basic_util.NO_OUTPUT_TIMEOUT ) stderr_reader.lines.append( err_msg ) process_handle.kill() break thread_lock.release() # Wait until each of the threads we've started terminate. The following calls will block each thread # until it terminates either normally, through an unhandled exception, or until the timeout occurs. - stdio_thread.join( td_common_util.NO_OUTPUT_TIMEOUT ) - stdout_reader.join( td_common_util.NO_OUTPUT_TIMEOUT ) - stderr_reader.join( td_common_util.NO_OUTPUT_TIMEOUT ) + stdio_thread.join( basic_util.NO_OUTPUT_TIMEOUT ) + stdout_reader.join( basic_util.NO_OUTPUT_TIMEOUT ) + stderr_reader.join( basic_util.NO_OUTPUT_TIMEOUT ) # Close subprocess' file descriptors. error = self.close_file_descriptor( process_handle.stdout ) error = self.close_file_descriptor( process_handle.stderr ) diff -r 31740dffaf9ae4e345fd2d4de4ab44ed133dd567 -r b1b1e5cefca5b0193c8623a9aee7a39cd1dda55c lib/tool_shed/galaxy_install/tool_dependencies/recipe/step_handler.py --- a/lib/tool_shed/galaxy_install/tool_dependencies/recipe/step_handler.py +++ b/lib/tool_shed/galaxy_install/tool_dependencies/recipe/step_handler.py @@ -4,12 +4,17 @@ import stat from string import Template import sys +import tarfile +import time +import urllib2 +import zipfile from galaxy.util import asbool from galaxy.util.template import fill_template +from tool_shed.util import basic_util from tool_shed.util import tool_dependency_util -from tool_shed.galaxy_install.tool_dependencies import td_common_util +from tool_shed.galaxy_install.tool_dependencies.env_manager import EnvManager # TODO: eliminate the use of fabric here. from galaxy import eggs @@ -26,6 +31,152 @@ VIRTUALENV_URL = 'https://pypi.python.org/packages/source/v/virtualenv/virtualenv-1.9.1.tar.gz' +class CompressedFile( object ): + + def __init__( self, file_path, mode='r' ): + if tarfile.is_tarfile( file_path ): + self.file_type = 'tar' + elif zipfile.is_zipfile( file_path ) and not file_path.endswith( '.jar' ): + self.file_type = 'zip' + self.file_name = os.path.splitext( os.path.basename( file_path ) )[ 0 ] + if self.file_name.endswith( '.tar' ): + self.file_name = os.path.splitext( self.file_name )[ 0 ] + self.type = self.file_type + method = 'open_%s' % self.file_type + if hasattr( self, method ): + self.archive = getattr( self, method )( file_path, mode ) + else: + raise NameError( 'File type %s specified, no open method found.' % self.file_type ) + + def extract( self, path ): + '''Determine the path to which the archive should be extracted.''' + contents = self.getmembers() + extraction_path = path + if len( contents ) == 1: + # The archive contains a single file, return the extraction path. + if self.isfile( contents[ 0 ] ): + extraction_path = os.path.join( path, self.file_name ) + if not os.path.exists( extraction_path ): + os.makedirs( extraction_path ) + self.archive.extractall( extraction_path ) + else: + # Get the common prefix for all the files in the archive. If the common prefix ends with a slash, + # or self.isdir() returns True, the archive contains a single directory with the desired contents. + # Otherwise, it contains multiple files and/or directories at the root of the archive. + common_prefix = os.path.commonprefix( [ self.getname( item ) for item in contents ] ) + if len( common_prefix ) >= 1 and not common_prefix.endswith( os.sep ) and self.isdir( self.getmember( common_prefix ) ): + common_prefix += os.sep + if common_prefix.endswith( os.sep ): + self.archive.extractall( os.path.join( path ) ) + extraction_path = os.path.join( path, common_prefix ) + else: + extraction_path = os.path.join( path, self.file_name ) + if not os.path.exists( extraction_path ): + os.makedirs( extraction_path ) + self.archive.extractall( os.path.join( extraction_path ) ) + return os.path.abspath( extraction_path ) + + def getmembers_tar( self ): + return self.archive.getmembers() + + def getmembers_zip( self ): + return self.archive.infolist() + + def getname_tar( self, item ): + return item.name + + def getname_zip( self, item ): + return item.filename + + def getmember( self, name ): + for member in self.getmembers(): + if self.getname( member ) == name: + return member + + def getmembers( self ): + return getattr( self, 'getmembers_%s' % self.type )() + + def getname( self, member ): + return getattr( self, 'getname_%s' % self.type )( member ) + + def isdir( self, member ): + return getattr( self, 'isdir_%s' % self.type )( member ) + + def isdir_tar( self, member ): + return member.isdir() + + def isdir_zip( self, member ): + if member.filename.endswith( os.sep ): + return True + return False + + def isfile( self, member ): + if not self.isdir( member ): + return True + return False + + def open_tar( self, filepath, mode ): + return tarfile.open( filepath, mode, errorlevel=0 ) + + def open_zip( self, filepath, mode ): + return zipfile.ZipFile( filepath, mode ) + + def zipfile_ok( self, path_to_archive ): + """ + This function is a bit pedantic and not functionally necessary. It checks whether there is + no file pointing outside of the extraction, because ZipFile.extractall() has some potential + security holes. See python zipfile documentation for more details. + """ + basename = os.path.realpath( os.path.dirname( path_to_archive ) ) + zip_archive = zipfile.ZipFile( path_to_archive ) + for member in zip_archive.namelist(): + member_path = os.path.realpath( os.path.join( basename, member ) ) + if not member_path.startswith( basename ): + return False + return True + + +class Download( object ): + + def url_download( self, install_dir, downloaded_file_name, download_url, extract=True ): + file_path = os.path.join( install_dir, downloaded_file_name ) + src = None + dst = None + # Set a timer so we don't sit here forever. + start_time = time.time() + try: + src = urllib2.urlopen( download_url ) + dst = open( file_path, 'wb' ) + while True: + chunk = src.read( basic_util.CHUNK_SIZE ) + if chunk: + dst.write( chunk ) + else: + break + time_taken = time.time() - start_time + if time_taken > basic_util.NO_OUTPUT_TIMEOUT: + err_msg = 'Downloading from URL %s took longer than the defined timeout period of %.1f seconds.' % \ + ( str( download_url ), basic_util.NO_OUTPUT_TIMEOUT ) + raise Exception( err_msg ) + except Exception, e: + err_msg = err_msg = 'Error downloading from URL\n%s:\n%s' % ( str( download_url ), str( e ) ) + raise Exception( err_msg ) + finally: + if src: + src.close() + if dst: + dst.close() + if extract: + if tarfile.is_tarfile( file_path ) or ( zipfile.is_zipfile( file_path ) and not file_path.endswith( '.jar' ) ): + archive = CompressedFile( file_path ) + extraction_path = archive.extract( install_dir ) + else: + extraction_path = os.path.abspath( install_dir ) + else: + extraction_path = os.path.abspath( install_dir ) + return extraction_path + + class RecipeStep( object ): """Abstract class that defines a standard format for handling recipe steps when installing packages.""" @@ -42,6 +193,22 @@ def __init__( self ): self.type = 'assert_directory_executable' + def assert_directory_executable( self, full_path ): + """ + Return True if a symbolic link or directory exists and is executable, but if + full_path is a file, return False. + """ + if full_path is None: + return False + if os.path.isfile( full_path ): + return False + if os.path.isdir( full_path ): + # Make sure the owner has execute permission on the directory. + # See http://docs.python.org/2/library/stat.html + if stat.S_IXUSR & os.stat( full_path )[ stat.ST_MODE ] == 64: + return True + return False + def execute_step( self, app, tool_dependency, package_name, actions, action_dict, filtered_actions, env_file_builder, install_environment, work_dir, current_dir=None, initial_download=False ): """ @@ -53,7 +220,7 @@ full_path = action_dict[ 'full_path' ] else: full_path = os.path.join( current_dir, action_dict[ 'full_path' ] ) - if not td_common_util.assert_directory_executable( full_path=full_path ): + if not self.assert_directory_executable( full_path=full_path ): status = app.install_model.ToolDependency.installation_status.ERROR error_message = 'The path %s is not a directory or is not executable by the owner.' % str( full_path ) tool_dependency = tool_dependency_util.set_tool_dependency_attributes( app, @@ -66,7 +233,7 @@ def prepare_step( self, app, tool_dependency, action_elem, action_dict, install_environment, is_binary_download ): # <action type="assert_executable">$INSTALL_DIR/mira/my_file</action> if action_elem.text: - action_dict[ 'full_path' ] = td_common_util.evaluate_template( action_elem.text, install_environment ) + action_dict[ 'full_path' ] = basic_util.evaluate_template( action_elem.text, install_environment ) return action_dict @@ -75,6 +242,18 @@ def __init__( self ): self.type = 'assert_directory_exists' + def assert_directory_exists( self, full_path ): + """ + Return True if a symbolic link or directory exists, but if full_path is a file, + return False. """ + if full_path is None: + return False + if os.path.isfile( full_path ): + return False + if os.path.isdir( full_path ): + return True + return False + def execute_step( self, app, tool_dependency, package_name, actions, action_dict, filtered_actions, env_file_builder, install_environment, work_dir, current_dir=None, initial_download=False ): """ @@ -86,7 +265,7 @@ full_path = action_dict[ 'full_path' ] else: full_path = os.path.join( current_dir, action_dict[ 'full_path' ] ) - if not td_common_util.assert_directory_exists( full_path=full_path ): + if not self.assert_directory_exists( full_path=full_path ): status = app.install_model.ToolDependency.installation_status.ERROR error_message = 'The path %s is not a directory or does not exist.' % str( full_path ) tool_dependency = tool_dependency_util.set_tool_dependency_attributes( app, @@ -99,7 +278,7 @@ def prepare_step( self, app, tool_dependency, action_elem, action_dict, install_environment, is_binary_download ): # <action type="make_directory">$INSTALL_DIR/mira</action> if action_elem.text: - action_dict[ 'full_path' ] = td_common_util.evaluate_template( action_elem.text, install_environment ) + action_dict[ 'full_path' ] = basic_util.evaluate_template( action_elem.text, install_environment ) return action_dict @@ -108,6 +287,22 @@ def __init__( self ): self.type = 'assert_file_executable' + def assert_file_executable( self, full_path ): + """ + Return True if a symbolic link or file exists and is executable, but if full_path + is a directory, return False. + """ + if full_path is None: + return False + if os.path.isdir( full_path ): + return False + if os.path.exists( full_path ): + # Make sure the owner has execute permission on the file. + # See http://docs.python.org/2/library/stat.html + if stat.S_IXUSR & os.stat( full_path )[ stat.ST_MODE ] == 64: + return True + return False + def execute_step( self, app, tool_dependency, package_name, actions, action_dict, filtered_actions, env_file_builder, install_environment, work_dir, current_dir=None, initial_download=False ): """ @@ -119,7 +314,7 @@ full_path = action_dict[ 'full_path' ] else: full_path = os.path.join( current_dir, action_dict[ 'full_path' ] ) - if not td_common_util.assert_file_executable( full_path=full_path ): + if not self.assert_file_executable( full_path=full_path ): status = app.install_model.ToolDependency.installation_status.ERROR error_message = 'The path %s is not a file or is not executable by the owner.' % str( full_path ) tool_dependency = tool_dependency_util.set_tool_dependency_attributes( app, @@ -132,7 +327,7 @@ def prepare_step( self, app, tool_dependency, action_elem, action_dict, install_environment, is_binary_download ): # <action type="assert_executable">$INSTALL_DIR/mira/my_file</action> if action_elem.text: - action_dict[ 'full_path' ] = td_common_util.evaluate_template( action_elem.text, install_environment ) + action_dict[ 'full_path' ] = basic_util.evaluate_template( action_elem.text, install_environment ) return action_dict @@ -141,6 +336,19 @@ def __init__( self ): self.type = 'assert_file_exists' + def assert_file_exists( self, full_path ): + """ + Return True if a symbolic link or file exists, but if full_path is a directory, + return False. + """ + if full_path is None: + return False + if os.path.isdir( full_path ): + return False + if os.path.exists( full_path ): + return True + return False + def execute_step( self, app, tool_dependency, package_name, actions, action_dict, filtered_actions, env_file_builder, install_environment, work_dir, current_dir=None, initial_download=False ): """ @@ -152,7 +360,7 @@ full_path = action_dict[ 'full_path' ] else: full_path = os.path.join( current_dir, action_dict[ 'full_path' ] ) - if not td_common_util.assert_file_exists( full_path=full_path ): + if not self.assert_file_exists( full_path=full_path ): status = app.install_model.ToolDependency.installation_status.ERROR error_message = 'The path %s is not a file or does not exist.' % str( full_path ) tool_dependency = tool_dependency_util.set_tool_dependency_attributes( app, @@ -165,7 +373,7 @@ def prepare_step( self, app, tool_dependency, action_elem, action_dict, install_environment, is_binary_download ): # <action type="assert_on_path">$INSTALL_DIR/mira/my_file</action> if action_elem.text: - action_dict[ 'full_path' ] = td_common_util.evaluate_template( action_elem.text, install_environment ) + action_dict[ 'full_path' ] = basic_util.evaluate_template( action_elem.text, install_environment ) return action_dict @@ -187,7 +395,7 @@ pre_cmd = './configure %s && make && make install' % configure_opts else: pre_cmd = './configure --prefix=$INSTALL_DIR %s && make && make install' % configure_opts - cmd = install_environment.build_command( td_common_util.evaluate_template( pre_cmd, install_environment ) ) + cmd = install_environment.build_command( basic_util.evaluate_template( pre_cmd, install_environment ) ) return_code = install_environment.handle_command( app=app, tool_dependency=tool_dependency, cmd=cmd, @@ -199,7 +407,7 @@ def prepare_step( self, app, tool_dependency, action_elem, action_dict, install_environment, is_binary_download ): # Handle configure, make and make install allow providing configuration options if action_elem.text: - configure_opts = td_common_util.evaluate_template( action_elem.text, install_environment ) + configure_opts = basic_util.evaluate_template( action_elem.text, install_environment ) action_dict[ 'configure_opts' ] = configure_opts return action_dict @@ -274,7 +482,7 @@ received_mode = int( file_elem.get( 'mode', 600 ), base=8 ) # For added security, ensure that the setuid and setgid bits are not set. mode = received_mode & ~( stat.S_ISUID | stat.S_ISGID ) - file = td_common_util.evaluate_template( file_elem.text, install_environment ) + file = basic_util.evaluate_template( file_elem.text, install_environment ) chmod_tuple = ( file, mode ) chmod_actions.append( chmod_tuple ) if chmod_actions: @@ -282,11 +490,17 @@ return action_dict -class DownloadBinary( RecipeStep ): +class DownloadBinary( Download, RecipeStep ): def __init__( self ): self.type = 'download_binary' + def download_binary( self, url, work_dir ): + """Download a pre-compiled binary from the specified URL.""" + downloaded_filename = os.path.split( url )[ -1 ] + dir = self.url_download( work_dir, downloaded_filename, url, extract=False ) + return downloaded_filename + def filter_actions_after_binary_installation( self, actions ): '''Filter out actions that should not be processed if a binary download succeeded.''' filtered_actions = [] @@ -311,7 +525,7 @@ log.debug( 'Attempting to download from %s to %s', url, str( target_directory ) ) downloaded_filename = None try: - downloaded_filename = td_common_util.download_binary( url, work_dir ) + downloaded_filename = self.download_binary( url, work_dir ) if initial_download: # Filter out any actions that are not download_binary, chmod, or set_environment. filtered_actions = self.filter_actions_after_binary_installation( actions[ 1: ] ) @@ -338,9 +552,9 @@ full_path_to_dir = os.path.abspath( install_environment.install_dir ) else: full_path_to_dir = os.path.abspath( install_environment.install_dir ) - td_common_util.move_file( current_dir=work_dir, - source=downloaded_filename, - destination=full_path_to_dir ) + basic_util.move_file( current_dir=work_dir, + source=downloaded_filename, + destination=full_path_to_dir ) # Not sure why dir is ignored in this method, need to investigate... dir = None if initial_download: @@ -368,7 +582,7 @@ return action_dict -class DownloadByUrl( RecipeStep ): +class DownloadByUrl( Download, RecipeStep ): def __init__( self ): self.type = 'download_by_url' @@ -394,9 +608,9 @@ downloaded_filename = action_dict[ 'target_filename' ] else: downloaded_filename = os.path.split( url )[ -1 ] - dir = td_common_util.url_download( work_dir, downloaded_filename, url, extract=True ) + dir = self.url_download( work_dir, downloaded_filename, url, extract=True ) if is_binary: - log_file = os.path.join( install_environment.install_dir, td_common_util.INSTALLATION_LOG ) + log_file = os.path.join( install_environment.install_dir, basic_util.INSTALLATION_LOG ) if os.path.exists( log_file ): logfile = open( log_file, 'ab' ) else: @@ -422,7 +636,7 @@ return action_dict -class DownloadFile( RecipeStep ): +class DownloadFile( Download, RecipeStep ): def __init__( self ): self.type = 'download_file' @@ -447,7 +661,7 @@ filename = action_dict[ 'target_filename' ] else: filename = url.split( '/' )[ -1 ] - td_common_util.url_download( work_dir, filename, url ) + self.url_download( work_dir, filename, url ) if initial_download: dir = os.path.curdir return tool_dependency, filtered_actions, dir @@ -479,13 +693,17 @@ full_path = action_dict[ 'full_path' ] else: full_path = os.path.join( current_dir, action_dict[ 'full_path' ] ) - td_common_util.make_directory( full_path=full_path ) + self.make_directory( full_path=full_path ) return tool_dependency, None, None + def make_directory( self, full_path ): + if not os.path.exists( full_path ): + os.makedirs( full_path ) + def prepare_step( self, app, tool_dependency, action_elem, action_dict, install_environment, is_binary_download ): # <action type="make_directory">$INSTALL_DIR/lib/python</action> if action_elem.text: - action_dict[ 'full_path' ] = td_common_util.evaluate_template( action_elem.text, install_environment ) + action_dict[ 'full_path' ] = basic_util.evaluate_template( action_elem.text, install_environment ) return action_dict @@ -515,7 +733,7 @@ def prepare_step( self, app, tool_dependency, action_elem, action_dict, install_environment, is_binary_download ): # make; make install; allow providing make options if action_elem.text: - make_opts = td_common_util.evaluate_template( action_elem.text, install_environment ) + make_opts = basic_util.evaluate_template( action_elem.text, install_environment ) action_dict[ 'make_opts' ] = make_opts return action_dict @@ -531,18 +749,38 @@ Move a directory of files. Since this class is not used in the initial download stage, no recipe step filtering is performed here, and None values are always returned for filtered_actions and dir. """ - td_common_util.move_directory_files( current_dir=current_dir, - source_dir=os.path.join( action_dict[ 'source_directory' ] ), - destination_dir=os.path.join( action_dict[ 'destination_directory' ] ) ) + self.move_directory_files( current_dir=current_dir, + source_dir=os.path.join( action_dict[ 'source_directory' ] ), + destination_dir=os.path.join( action_dict[ 'destination_directory' ] ) ) return tool_dependency, None, None + def move_directory_files( self, current_dir, source_dir, destination_dir ): + source_directory = os.path.abspath( os.path.join( current_dir, source_dir ) ) + destination_directory = os.path.join( destination_dir ) + if not os.path.isdir( destination_directory ): + os.makedirs( destination_directory ) + symlinks = [] + regular_files = [] + for file_name in os.listdir( source_directory ): + source_file = os.path.join( source_directory, file_name ) + destination_file = os.path.join( destination_directory, file_name ) + files_tuple = ( source_file, destination_file ) + if os.path.islink( source_file ): + symlinks.append( files_tuple ) + else: + regular_files.append( files_tuple ) + for source_file, destination_file in symlinks: + shutil.move( source_file, destination_file ) + for source_file, destination_file in regular_files: + shutil.move( source_file, destination_file ) + def prepare_step( self, app, tool_dependency, action_elem, action_dict, install_environment, is_binary_download ): # <action type="move_directory_files"> # <source_directory>bin</source_directory> # <destination_directory>$INSTALL_DIR/bin</destination_directory> # </action> for move_elem in action_elem: - move_elem_text = td_common_util.evaluate_template( move_elem.text, install_environment ) + move_elem_text = basic_util.evaluate_template( move_elem.text, install_environment ) if move_elem_text: action_dict[ move_elem.tag ] = move_elem_text return action_dict @@ -559,10 +797,10 @@ Move a file on disk. Since this class is not used in the initial download stage, no recipe step filtering is performed here, and None values are always returned for filtered_actions and dir. """ - td_common_util.move_file( current_dir=current_dir, - source=os.path.join( action_dict[ 'source' ] ), - destination=os.path.join( action_dict[ 'destination' ] ), - rename_to=action_dict[ 'rename_to' ] ) + basic_util.move_file( current_dir=current_dir, + source=os.path.join( action_dict[ 'source' ] ), + destination=os.path.join( action_dict[ 'destination' ] ), + rename_to=action_dict[ 'rename_to' ] ) return tool_dependency, None, None def prepare_step( self, app, tool_dependency, action_elem, action_dict, install_environment, is_binary_download ): @@ -570,8 +808,8 @@ # <source>misc/some_file</source> # <destination>$INSTALL_DIR/bin</destination> # </action> - action_dict[ 'source' ] = td_common_util.evaluate_template( action_elem.find( 'source' ).text, install_environment ) - action_dict[ 'destination' ] = td_common_util.evaluate_template( action_elem.find( 'destination' ).text, install_environment ) + action_dict[ 'source' ] = basic_util.evaluate_template( action_elem.find( 'source' ).text, install_environment ) + action_dict[ 'destination' ] = basic_util.evaluate_template( action_elem.find( 'destination' ).text, install_environment ) action_dict[ 'rename_to' ] = action_elem.get( 'rename_to' ) return action_dict @@ -717,12 +955,12 @@ # <action type="set_environment"> # <environment_variable name="PATH" action="prepend_to">$INSTALL_DIR</environment_variable> # </action> + env_manager = EnvManager( app ) env_var_dicts = [] for env_elem in action_elem: if env_elem.tag == 'environment_variable': - env_var_dict = \ - td_common_util.create_env_var_dict( elem=env_elem, - install_environment=install_environment ) + env_var_dict = env_manager.create_env_var_dict( elem=env_elem, + install_environment=install_environment ) if env_var_dict: env_var_dicts.append( env_var_dict ) if env_var_dicts: @@ -764,16 +1002,17 @@ # the current tool dependency package. See the package_matplotlib_1_2 repository in the test tool # shed for a real-world example. all_env_shell_file_paths = [] + env_manager = EnvManager( app ) for env_elem in action_elem: if env_elem.tag == 'repository': - env_shell_file_paths = td_common_util.get_env_shell_file_paths( app, env_elem ) + env_shell_file_paths = env_manager.get_env_shell_file_paths( env_elem ) if env_shell_file_paths: all_env_shell_file_paths.extend( env_shell_file_paths ) action_dict[ 'env_shell_file_paths' ] = all_env_shell_file_paths return action_dict -class SetupPerlEnvironment( RecipeStep ): +class SetupPerlEnvironment( Download, RecipeStep ): def __init__( self ): self.type = 'setup_purl_environment' @@ -822,7 +1061,7 @@ # We assume a URL to a gem file. url = perl_package perl_package_name = url.split( '/' )[ -1 ] - dir = td_common_util.url_download( work_dir, perl_package_name, url, extract=True ) + dir = self.url_download( work_dir, perl_package_name, url, extract=True ) # Search for Build.PL or Makefile.PL (ExtUtils::MakeMaker vs. Module::Build). tmp_work_dir = os.path.join( work_dir, dir ) if os.path.exists( os.path.join( tmp_work_dir, 'Makefile.PL' ) ): @@ -836,7 +1075,7 @@ return tool_dependency, filtered_actions, dir return tool_dependency, None, None with lcd( tmp_work_dir ): - cmd = install_environment.build_command( td_common_util.evaluate_template( cmd, install_environment ) ) + cmd = install_environment.build_command( basic_util.evaluate_template( cmd, install_environment ) ) return_code = install_environment.handle_command( app=app, tool_dependency=tool_dependency, cmd=cmd, @@ -849,7 +1088,7 @@ # perl package from CPAN without version number. # cpanm should be installed with the parent perl distribution, otherwise this will not work. cmd += '''cpanm --local-lib=$INSTALL_DIR %s''' % ( perl_package ) - cmd = install_environment.build_command( td_common_util.evaluate_template( cmd, install_environment ) ) + cmd = install_environment.build_command( basic_util.evaluate_template( cmd, install_environment ) ) return_code = install_environment.handle_command( app=app, tool_dependency=tool_dependency, cmd=cmd, @@ -890,10 +1129,11 @@ # with each repository. This will potentially update the value of the 'env_shell_file_paths' entry # in action_dict. all_env_shell_file_paths = [] - action_dict = td_common_util.get_env_shell_file_paths_from_setup_environment_elem( app, - all_env_shell_file_paths, - action_elem, - action_dict ) + env_manager = EnvManager( app ) + action_dict = env_manager.get_env_shell_file_paths_from_setup_environment_elem( app, + all_env_shell_file_paths, + action_elem, + action_dict ) perl_packages = [] for env_elem in action_elem: if env_elem.tag == 'package': @@ -908,7 +1148,7 @@ return action_dict -class SetupREnvironment( RecipeStep ): +class SetupREnvironment( Download, RecipeStep ): def __init__( self ): self.type = 'setup_r_environment' @@ -947,7 +1187,7 @@ for url in action_dict[ 'r_packages' ]: filename = url.split( '/' )[ -1 ] tarball_names.append( filename ) - td_common_util.url_download( work_dir, filename, url, extract=False ) + self.url_download( work_dir, filename, url, extract=False ) dir = os.path.curdir current_dir = os.path.abspath( os.path.join( work_dir, dir ) ) with lcd( current_dir ): @@ -958,7 +1198,7 @@ cmd = r'''PATH=$PATH:$R_HOME/bin; export PATH; R_LIBS=$INSTALL_DIR; export R_LIBS; Rscript -e "install.packages(c('%s'),lib='$INSTALL_DIR', repos=NULL, dependencies=FALSE)"''' % \ ( str( tarball_name ) ) - cmd = install_environment.build_command( td_common_util.evaluate_template( cmd, install_environment ) ) + cmd = install_environment.build_command( basic_util.evaluate_template( cmd, install_environment ) ) return_code = install_environment.handle_command( app=app, tool_dependency=tool_dependency, cmd=cmd, @@ -993,10 +1233,11 @@ # associated with each repository. This will potentially update the value of the # 'env_shell_file_paths' entry in action_dict. all_env_shell_file_paths = [] - action_dict = td_common_util.get_env_shell_file_paths_from_setup_environment_elem( app, - all_env_shell_file_paths, - action_elem, - action_dict ) + env_manager = EnvManager( app ) + action_dict = env_manager.get_env_shell_file_paths_from_setup_environment_elem( app, + all_env_shell_file_paths, + action_elem, + action_dict ) r_packages = list() for env_elem in action_elem: if env_elem.tag == 'package': @@ -1006,7 +1247,7 @@ return action_dict -class SetupRubyEnvironment( RecipeStep ): +class SetupRubyEnvironment( Download, RecipeStep ): def __init__( self ): self.type = 'setup_ruby_environment' @@ -1058,7 +1299,7 @@ # We assume a URL to a gem file. url = gem gem_name = url.split( '/' )[ -1 ] - td_common_util.url_download( work_dir, gem_name, url, extract=False ) + self.url_download( work_dir, gem_name, url, extract=False ) cmd = '''PATH=$PATH:$RUBY_HOME/bin; export PATH; GEM_HOME=$INSTALL_DIR; export GEM_HOME; gem install --local %s ''' % ( gem_name ) else: @@ -1073,7 +1314,7 @@ # no version number given cmd = '''PATH=$PATH:$RUBY_HOME/bin; export PATH; GEM_HOME=$INSTALL_DIR; export GEM_HOME; gem install %s''' % ( gem ) - cmd = install_environment.build_command( td_common_util.evaluate_template( cmd, install_environment ) ) + cmd = install_environment.build_command( basic_util.evaluate_template( cmd, install_environment ) ) return_code = install_environment.handle_command( app=app, tool_dependency=tool_dependency, cmd=cmd, @@ -1114,10 +1355,11 @@ # associated with each repository. This will potentially update the value of the # 'env_shell_file_paths' entry in action_dict. all_env_shell_file_paths = [] - action_dict = td_common_util.get_env_shell_file_paths_from_setup_environment_elem( app, - all_env_shell_file_paths, - action_elem, - action_dict ) + env_manager = EnvManager( app ) + action_dict = env_manager.get_env_shell_file_paths_from_setup_environment_elem( app, + all_env_shell_file_paths, + action_elem, + action_dict ) ruby_package_tups = [] for env_elem in action_elem: if env_elem.tag == 'package': @@ -1140,7 +1382,7 @@ return action_dict -class SetupVirtualEnv( RecipeStep ): +class SetupVirtualEnv( Download, RecipeStep ): def __init__( self ): self.type = 'setup_virtualenv' @@ -1228,9 +1470,10 @@ with install_environment.make_tmp_dir() as work_dir: downloaded_filename = VIRTUALENV_URL.rsplit('/', 1)[-1] try: - dir = td_common_util.url_download( work_dir, downloaded_filename, VIRTUALENV_URL ) + dir = self.url_download( work_dir, downloaded_filename, VIRTUALENV_URL ) except: - log.error( "Failed to download virtualenv: td_common_util.url_download( '%s', '%s', '%s' ) threw an exception", work_dir, downloaded_filename, VIRTUALENV_URL ) + log.error( "Failed to download virtualenv: url_download( '%s', '%s', '%s' ) threw an exception", + work_dir, downloaded_filename, VIRTUALENV_URL ) return False full_path_to_dir = os.path.abspath( os.path.join( work_dir, dir ) ) shutil.move( full_path_to_dir, venv_dir ) @@ -1245,7 +1488,7 @@ # lxml==2.3.0</action> ## Manually specify contents of requirements.txt file to create dynamically. action_dict[ 'use_requirements_file' ] = asbool( action_elem.get( 'use_requirements_file', True ) ) - action_dict[ 'requirements' ] = td_common_util.evaluate_template( action_elem.text or 'requirements.txt', install_environment ) + action_dict[ 'requirements' ] = basic_util.evaluate_template( action_elem.text or 'requirements.txt', install_environment ) action_dict[ 'python' ] = action_elem.get( 'python', 'python' ) return action_dict @@ -1316,7 +1559,7 @@ def prepare_step( self, app, tool_dependency, action_elem, action_dict, install_environment, is_binary_download ): # <action type="shell_command">make</action> - action_elem_text = td_common_util.evaluate_template( action_elem.text, install_environment ) + action_elem_text = basic_util.evaluate_template( action_elem.text, install_environment ) if action_elem_text: action_dict[ 'command' ] = action_elem_text return action_dict @@ -1338,7 +1581,7 @@ env_vars = dict() env_vars = install_environment.environment_dict() tool_shed_repository = tool_dependency.tool_shed_repository - env_vars.update( td_common_util.get_env_var_values( install_environment ) ) + env_vars.update( basic_util.get_env_var_values( install_environment ) ) language = action_dict[ 'language' ] with settings( warn_only=True, **env_vars ): if language == 'cheetah': diff -r 31740dffaf9ae4e345fd2d4de4ab44ed133dd567 -r b1b1e5cefca5b0193c8623a9aee7a39cd1dda55c lib/tool_shed/galaxy_install/tool_dependencies/recipe/tag_handler.py --- a/lib/tool_shed/galaxy_install/tool_dependencies/recipe/tag_handler.py +++ b/lib/tool_shed/galaxy_install/tool_dependencies/recipe/tag_handler.py @@ -10,7 +10,7 @@ from tool_shed.util import tool_dependency_util from tool_shed.util import xml_util -from tool_shed.galaxy_install.tool_dependencies import td_common_util +from tool_shed.galaxy_install.tool_dependencies.env_manager import EnvManager from tool_shed.galaxy_install.tool_dependencies.recipe.env_file_builder import EnvFileBuilder from tool_shed.galaxy_install.tool_dependencies.recipe.install_environment import InstallEnvironment @@ -85,9 +85,9 @@ platform_info_dict = tool_dependency_util.get_platform_info_dict() if package_install_version == '1.0': # Handle tool dependency installation using a fabric method included in the Galaxy framework. - actions_elem_tuples = td_common_util.parse_package_elem( package_elem, - platform_info_dict=platform_info_dict, - include_after_install_actions=True ) + actions_elem_tuples = tool_dependency_util.parse_package_elem( package_elem, + platform_info_dict=platform_info_dict, + include_after_install_actions=True ) if not actions_elem_tuples: proceed_with_install = False error_message = 'Version %s of the %s package cannot be installed because ' % ( str( package_version ), str( package_name ) ) @@ -491,6 +491,7 @@ # <set_environment version="1.0"> # <repository toolshed="<tool shed>" name="<repository name>" owner="<repository owner>" changeset_revision="<changeset revision>" /> # </set_environment> + env_manager = EnvManager( app ) tool_dependencies = [] env_var_version = elem.get( 'version', '1.0' ) tool_shed_repository_install_dir = os.path.abspath( tool_shed_repository.repo_files_directory( app ) ) @@ -514,8 +515,8 @@ tool_dependency_version=None ) install_environment = InstallEnvironment( tool_shed_repository_install_dir=tool_shed_repository_install_dir, install_dir=install_dir ) - env_var_dict = td_common_util.create_env_var_dict( elem=env_var_elem, - install_environment=install_environment ) + env_var_dict = env_manager.create_env_var_dict( elem=env_var_elem, + install_environment=install_environment ) if env_var_dict: if not os.path.exists( install_dir ): os.makedirs( install_dir ) diff -r 31740dffaf9ae4e345fd2d4de4ab44ed133dd567 -r b1b1e5cefca5b0193c8623a9aee7a39cd1dda55c lib/tool_shed/galaxy_install/tool_dependencies/td_common_util.py --- a/lib/tool_shed/galaxy_install/tool_dependencies/td_common_util.py +++ /dev/null @@ -1,578 +0,0 @@ -import logging -import os -import re -import shutil -import stat -import sys -import tarfile -import time -import traceback -import urllib2 -import zipfile -from string import Template -from tool_shed.util import common_util -import tool_shed.util.shed_util_common as suc -from galaxy.datatypes import checkers - -log = logging.getLogger( __name__ ) - -# Set no activity timeout to 20 minutes. -NO_OUTPUT_TIMEOUT = 1200.0 -INSTALLATION_LOG = 'INSTALLATION.log' - - -class CompressedFile( object ): - - def __init__( self, file_path, mode='r' ): - if istar( file_path ): - self.file_type = 'tar' - elif iszip( file_path ) and not isjar( file_path ): - self.file_type = 'zip' - self.file_name = os.path.splitext( os.path.basename( file_path ) )[ 0 ] - if self.file_name.endswith( '.tar' ): - self.file_name = os.path.splitext( self.file_name )[ 0 ] - self.type = self.file_type - method = 'open_%s' % self.file_type - if hasattr( self, method ): - self.archive = getattr( self, method )( file_path, mode ) - else: - raise NameError( 'File type %s specified, no open method found.' % self.file_type ) - - def extract( self, path ): - '''Determine the path to which the archive should be extracted.''' - contents = self.getmembers() - extraction_path = path - if len( contents ) == 1: - # The archive contains a single file, return the extraction path. - if self.isfile( contents[ 0 ] ): - extraction_path = os.path.join( path, self.file_name ) - if not os.path.exists( extraction_path ): - os.makedirs( extraction_path ) - self.archive.extractall( extraction_path ) - else: - # Get the common prefix for all the files in the archive. If the common prefix ends with a slash, - # or self.isdir() returns True, the archive contains a single directory with the desired contents. - # Otherwise, it contains multiple files and/or directories at the root of the archive. - common_prefix = os.path.commonprefix( [ self.getname( item ) for item in contents ] ) - if len( common_prefix ) >= 1 and not common_prefix.endswith( os.sep ) and self.isdir( self.getmember( common_prefix ) ): - common_prefix += os.sep - if common_prefix.endswith( os.sep ): - self.archive.extractall( os.path.join( path ) ) - extraction_path = os.path.join( path, common_prefix ) - else: - extraction_path = os.path.join( path, self.file_name ) - if not os.path.exists( extraction_path ): - os.makedirs( extraction_path ) - self.archive.extractall( os.path.join( extraction_path ) ) - return os.path.abspath( extraction_path ) - - def getmembers_tar( self ): - return self.archive.getmembers() - - def getmembers_zip( self ): - return self.archive.infolist() - - def getname_tar( self, item ): - return item.name - - def getname_zip( self, item ): - return item.filename - - def getmember( self, name ): - for member in self.getmembers(): - if self.getname( member ) == name: - return member - - def getmembers( self ): - return getattr( self, 'getmembers_%s' % self.type )() - - def getname( self, member ): - return getattr( self, 'getname_%s' % self.type )( member ) - - def isdir( self, member ): - return getattr( self, 'isdir_%s' % self.type )( member ) - - def isdir_tar( self, member ): - return member.isdir() - - def isdir_zip( self, member ): - if member.filename.endswith( os.sep ): - return True - return False - - def isfile( self, member ): - if not self.isdir( member ): - return True - return False - - def open_tar( self, filepath, mode ): - return tarfile.open( filepath, mode, errorlevel=0 ) - - def open_zip( self, filepath, mode ): - return zipfile.ZipFile( filepath, mode ) - -def assert_directory_executable( full_path ): - """ - Return True if a symbolic link or directory exists and is executable, but if - full_path is a file, return False. - """ - if full_path is None: - return False - if os.path.isfile( full_path ): - return False - if os.path.isdir( full_path ): - # Make sure the owner has execute permission on the directory. - # See http://docs.python.org/2/library/stat.html - if stat.S_IXUSR & os.stat( full_path )[ stat.ST_MODE ] == 64: - return True - return False - -def assert_directory_exists( full_path ): - """ - Return True if a symbolic link or directory exists, but if full_path is a file, - return False. """ - if full_path is None: - return False - if os.path.isfile( full_path ): - return False - if os.path.isdir( full_path ): - return True - return False - -def assert_file_executable( full_path ): - """ - Return True if a symbolic link or file exists and is executable, but if full_path - is a directory, return False. - """ - if full_path is None: - return False - if os.path.isdir( full_path ): - return False - if os.path.exists( full_path ): - # Make sure the owner has execute permission on the file. - # See http://docs.python.org/2/library/stat.html - if stat.S_IXUSR & os.stat( full_path )[ stat.ST_MODE ] == 64: - return True - return False - -def assert_file_exists( full_path ): - """ - Return True if a symbolic link or file exists, but if full_path is a directory, - return False. - """ - if full_path is None: - return False - if os.path.isdir( full_path ): - return False - if os.path.exists( full_path ): - return True - return False - -def create_env_var_dict( elem, install_environment ): - env_var_name = elem.get( 'name', 'PATH' ) - env_var_action = elem.get( 'action', 'prepend_to' ) - env_var_text = None - tool_dependency_install_dir = install_environment.install_dir - tool_shed_repository_install_dir = install_environment.tool_shed_repository_install_dir - if elem.text and elem.text.find( 'REPOSITORY_INSTALL_DIR' ) >= 0: - if tool_shed_repository_install_dir and elem.text.find( '$REPOSITORY_INSTALL_DIR' ) != -1: - env_var_text = elem.text.replace( '$REPOSITORY_INSTALL_DIR', tool_shed_repository_install_dir ) - return dict( name=env_var_name, action=env_var_action, value=env_var_text ) - else: - env_var_text = elem.text.replace( '$REPOSITORY_INSTALL_DIR', tool_dependency_install_dir ) - return dict( name=env_var_name, action=env_var_action, value=env_var_text ) - if elem.text and elem.text.find( 'INSTALL_DIR' ) >= 0: - if tool_dependency_install_dir: - env_var_text = elem.text.replace( '$INSTALL_DIR', tool_dependency_install_dir ) - return dict( name=env_var_name, action=env_var_action, value=env_var_text ) - else: - env_var_text = elem.text.replace( '$INSTALL_DIR', tool_shed_repository_install_dir ) - return dict( name=env_var_name, action=env_var_action, value=env_var_text ) - if elem.text: - # Allow for environment variables that contain neither REPOSITORY_INSTALL_DIR nor INSTALL_DIR - # since there may be command line parameters that are tuned for a Galaxy instance. Allowing them - # to be set in one location rather than being hard coded into each tool config is the best approach. - # For example: - # <environment_variable name="GATK2_SITE_OPTIONS" action="set_to"> - # "--num_threads 4 --num_cpu_threads_per_data_thread 3 --phone_home STANDARD" - # </environment_variable> - return dict( name=env_var_name, action=env_var_action, value=elem.text) - return None - -def download_binary( url, work_dir ): - """Download a pre-compiled binary from the specified URL.""" - downloaded_filename = os.path.split( url )[ -1 ] - dir = url_download( work_dir, downloaded_filename, url, extract=False ) - return downloaded_filename - -def egrep_escape( text ): - """Escape ``text`` to allow literal matching using egrep.""" - regex = re.escape( text ) - # Seems like double escaping is needed for \ - regex = regex.replace( '\\\\', '\\\\\\' ) - # Triple-escaping seems to be required for $ signs - regex = regex.replace( r'\$', r'\\\$' ) - # Whereas single quotes should not be escaped - regex = regex.replace( r"\'", "'" ) - return regex - -def evaluate_template( text, install_environment ): - """ - Substitute variables defined in XML blocks from dependencies file. The value of the received - repository_install_dir is the root installation directory of the repository that contains the - tool dependency. The value of the received install_dir is the root installation directory of - the tool_dependency. - """ - return Template( text ).safe_substitute( get_env_var_values( install_environment ) ) - -def format_traceback(): - ex_type, ex, tb = sys.exc_info() - return ''.join( traceback.format_tb( tb ) ) - -def get_env_shell_file_path( installation_directory ): - env_shell_file_name = 'env.sh' - default_location = os.path.abspath( os.path.join( installation_directory, env_shell_file_name ) ) - if os.path.exists( default_location ): - return default_location - for root, dirs, files in os.walk( installation_directory ): - for name in files: - if name == env_shell_file_name: - return os.path.abspath( os.path.join( root, name ) ) - return None - -def get_env_shell_file_paths( app, elem ): - # Currently only the following tag set is supported. - # <repository toolshed="http://localhost:9009/" name="package_numpy_1_7" owner="test" changeset_revision="c84c6a8be056"> - # <package name="numpy" version="1.7.1" /> - # </repository> - env_shell_file_paths = [] - toolshed = elem.get( 'toolshed', None ) - repository_name = elem.get( 'name', None ) - repository_owner = elem.get( 'owner', None ) - changeset_revision = elem.get( 'changeset_revision', None ) - if toolshed and repository_name and repository_owner and changeset_revision: - # The protocol is not stored, but the port is if it exists. - toolshed = common_util.remove_protocol_from_tool_shed_url( toolshed ) - repository = suc.get_repository_for_dependency_relationship( app, toolshed, repository_name, repository_owner, changeset_revision ) - if repository: - for sub_elem in elem: - tool_dependency_type = sub_elem.tag - tool_dependency_name = sub_elem.get( 'name' ) - tool_dependency_version = sub_elem.get( 'version' ) - if tool_dependency_type and tool_dependency_name and tool_dependency_version: - # Get the tool_dependency so we can get its installation directory. - tool_dependency = None - for tool_dependency in repository.tool_dependencies: - if tool_dependency.type == tool_dependency_type and \ - tool_dependency.name == tool_dependency_name and \ - tool_dependency.version == tool_dependency_version: - break - if tool_dependency: - tool_dependency_key = '%s/%s' % ( tool_dependency_name, tool_dependency_version ) - installation_directory = tool_dependency.installation_directory( app ) - env_shell_file_path = get_env_shell_file_path( installation_directory ) - if env_shell_file_path: - env_shell_file_paths.append( env_shell_file_path ) - else: - error_message = "Skipping tool dependency definition because unable to locate env.sh file for tool dependency " - error_message += "type %s, name %s, version %s for repository %s" % \ - ( str( tool_dependency_type ), str( tool_dependency_name ), str( tool_dependency_version ), str( repository.name ) ) - log.debug( error_message ) - continue - else: - error_message = "Skipping tool dependency definition because unable to locate tool dependency " - error_message += "type %s, name %s, version %s for repository %s" % \ - ( str( tool_dependency_type ), str( tool_dependency_name ), str( tool_dependency_version ), str( repository.name ) ) - log.debug( error_message ) - continue - else: - error_message = "Skipping invalid tool dependency definition: type %s, name %s, version %s." % \ - ( str( tool_dependency_type ), str( tool_dependency_name ), str( tool_dependency_version ) ) - log.debug( error_message ) - continue - else: - error_message = "Skipping set_environment_for_install definition because unable to locate required installed tool shed repository: " - error_message += "toolshed %s, name %s, owner %s, changeset_revision %s." % \ - ( str( toolshed ), str( repository_name ), str( repository_owner ), str( changeset_revision ) ) - log.debug( error_message ) - else: - error_message = "Skipping invalid set_environment_for_install definition: toolshed %s, name %s, owner %s, changeset_revision %s." % \ - ( str( toolshed ), str( repository_name ), str( repository_owner ), str( changeset_revision ) ) - log.debug( error_message ) - return env_shell_file_paths - -def get_env_shell_file_paths_from_setup_environment_elem( app, all_env_shell_file_paths, elem, action_dict ): - """ - Parse an XML tag set to discover all child repository dependency tags and define the path to an env.sh file associated - with the repository (this requires the repository dependency to be in an installed state). The received action_dict - will be updated with these discovered paths and returned to the caller. This method handles tool dependency definition - tag sets <setup_r_environment>, <setup_ruby_environment> and <setup_perl_environment>. - """ - # An example elem is: - # <action type="setup_perl_environment"> - # <repository name="package_perl_5_18" owner="iuc"> - # <package name="perl" version="5.18.1" /> - # </repository> - # <repository name="package_expat_2_1" owner="iuc" prior_installation_required="True"> - # <package name="expat" version="2.1.0" /> - # </repository> - # <package>http://search.cpan.org/CPAN/authors/id/T/TO/TODDR/XML-Parser-2.41.tar.gz</package> - # <package>http://search.cpan.org/CPAN/authors/id/L/LD/LDS/CGI.pm-3.43.tar.gz</package> - # </action> - for action_elem in elem: - if action_elem.tag == 'repository': - env_shell_file_paths = get_env_shell_file_paths( app, action_elem ) - all_env_shell_file_paths.extend( env_shell_file_paths ) - if all_env_shell_file_paths: - action_dict[ 'env_shell_file_paths' ] = all_env_shell_file_paths - action_dict[ 'action_shell_file_paths' ] = env_shell_file_paths - return action_dict - -def get_env_var_values( install_environment ): - """ - Return a dictionary of values, some of which enable substitution of reserved words for the values. - The received install_enviroment object has 2 important attributes for reserved word substitution: - install_environment.tool_shed_repository_install_dir is the root installation directory of the repository - that contains the tool dependency being installed, and install_environment.install_dir is the root - installation directory of the tool dependency. - """ - env_var_dict = {} - env_var_dict[ 'REPOSITORY_INSTALL_DIR' ] = install_environment.tool_shed_repository_install_dir - env_var_dict[ 'INSTALL_DIR' ] = install_environment.install_dir - env_var_dict[ 'system_install' ] = install_environment.install_dir - # If the Python interpreter is 64bit then we can safely assume that the underlying system is also 64bit. - env_var_dict[ '__is64bit__' ] = sys.maxsize > 2**32 - return env_var_dict - -def isbz2( file_path ): - return checkers.is_bz2( file_path ) - -def isgzip( file_path ): - return checkers.is_gzip( file_path ) - -def isjar( file_path ): - return iszip( file_path ) and file_path.endswith( '.jar' ) - -def istar( file_path ): - return tarfile.is_tarfile( file_path ) - -def iszip( file_path ): - return checkers.check_zip( file_path ) - -def is_compressed( file_path ): - if isjar( file_path ): - return False - else: - return iszip( file_path ) or isgzip( file_path ) or istar( file_path ) or isbz2( file_path ) - -def make_directory( full_path ): - if not os.path.exists( full_path ): - os.makedirs( full_path ) - -def move_directory_files( current_dir, source_dir, destination_dir ): - source_directory = os.path.abspath( os.path.join( current_dir, source_dir ) ) - destination_directory = os.path.join( destination_dir ) - if not os.path.isdir( destination_directory ): - os.makedirs( destination_directory ) - symlinks = [] - regular_files = [] - for file_name in os.listdir( source_directory ): - source_file = os.path.join( source_directory, file_name ) - destination_file = os.path.join( destination_directory, file_name ) - files_tuple = ( source_file, destination_file ) - if os.path.islink( source_file ): - symlinks.append( files_tuple ) - else: - regular_files.append( files_tuple ) - for source_file, destination_file in symlinks: - shutil.move( source_file, destination_file ) - for source_file, destination_file in regular_files: - shutil.move( source_file, destination_file ) - -def move_file( current_dir, source, destination, rename_to=None ): - source_path = os.path.abspath( os.path.join( current_dir, source ) ) - source_file = os.path.basename( source_path ) - if rename_to is not None: - destination_file = rename_to - destination_directory = os.path.join( destination ) - destination_path = os.path.join( destination_directory, destination_file ) - else: - destination_directory = os.path.join( destination ) - destination_path = os.path.join( destination_directory, source_file ) - if not os.path.exists( destination_directory ): - os.makedirs( destination_directory ) - shutil.move( source_path, destination_path ) - -def parse_package_elem( package_elem, platform_info_dict=None, include_after_install_actions=True ): - """ - Parse a <package> element within a tool dependency definition and return a list of action tuples. - This method is called when setting metadata on a repository that includes a tool_dependencies.xml - file or when installing a repository that includes a tool_dependencies.xml file. If installing, - platform_info_dict must be a valid dictionary and include_after_install_actions must be True. - """ - # The actions_elem_tuples list contains <actions> tag sets (possibly inside of an <actions_group> - # tag set) to be processed in the order they are defined in the tool_dependencies.xml file. - actions_elem_tuples = [] - # The tag sets that will go into the actions_elem_list are those that install a compiled binary if - # the architecture and operating system match its defined attributes. If compiled binary is not - # installed, the first <actions> tag set [following those that have the os and architecture attributes] - # that does not have os or architecture attributes will be processed. This tag set must contain the - # recipe for downloading and compiling source. - actions_elem_list = [] - for elem in package_elem: - if elem.tag == 'actions': - # We have an <actions> tag that should not be matched against a specific combination of - # architecture and operating system. - in_actions_group = False - actions_elem_tuples.append( ( in_actions_group, elem ) ) - elif elem.tag == 'actions_group': - # We have an actions_group element, and its child <actions> elements should therefore be compared - # with the current operating system - # and processor architecture. - in_actions_group = True - # Record the number of <actions> elements so we can filter out any <action> elements that precede - # <actions> elements. - actions_elem_count = len( elem.findall( 'actions' ) ) - # Record the number of <actions> elements that have both architecture and os specified, in order - # to filter out any platform-independent <actions> elements that come before platform-specific - # <actions> elements. - platform_actions_elements = [] - for actions_elem in elem.findall( 'actions' ): - if actions_elem.get( 'architecture' ) is not None and actions_elem.get( 'os' ) is not None: - platform_actions_elements.append( actions_elem ) - platform_actions_element_count = len( platform_actions_elements ) - platform_actions_elements_processed = 0 - actions_elems_processed = 0 - # The tag sets that will go into the after_install_actions list are <action> tags instead of <actions> - # tags. These will be processed only if they are at the very end of the <actions_group> tag set (after - # all <actions> tag sets). See below for details. - after_install_actions = [] - # Inspect the <actions_group> element and build the actions_elem_list and the after_install_actions list. - for child_element in elem: - if child_element.tag == 'actions': - actions_elems_processed += 1 - system = child_element.get( 'os' ) - architecture = child_element.get( 'architecture' ) - # Skip <actions> tags that have only one of architecture or os specified, in order for the - # count in platform_actions_elements_processed to remain accurate. - if ( system and not architecture ) or ( architecture and not system ): - log.debug( 'Error: Both architecture and os attributes must be specified in an <actions> tag.' ) - continue - # Since we are inside an <actions_group> tag set, compare it with our current platform information - # and filter the <actions> tag sets that don't match. Require both the os and architecture attributes - # to be defined in order to find a match. - if system and architecture: - platform_actions_elements_processed += 1 - # If either the os or architecture do not match the platform, this <actions> tag will not be - # considered a match. Skip it and proceed with checking the next one. - if platform_info_dict: - if platform_info_dict[ 'os' ] != system or platform_info_dict[ 'architecture' ] != architecture: - continue - else: - # We must not be installing a repository into Galaxy, so determining if we can install a - # binary is not necessary. - continue - else: - # <actions> tags without both os and architecture attributes are only allowed to be specified - # after platform-specific <actions> tags. If we find a platform-independent <actions> tag before - # all platform-specific <actions> tags have been processed. - if platform_actions_elements_processed < platform_actions_element_count: - debug_msg = 'Error: <actions> tags without os and architecture attributes are only allowed ' - debug_msg += 'after all <actions> tags with os and architecture attributes have been defined. ' - debug_msg += 'Skipping the <actions> tag set with no os or architecture attributes that has ' - debug_msg += 'been defined between two <actions> tag sets that have these attributes defined. ' - log.debug( debug_msg ) - continue - # If we reach this point, it means one of two things: 1) The system and architecture attributes are - # not defined in this <actions> tag, or 2) The system and architecture attributes are defined, and - # they are an exact match for the current platform. Append the child element to the list of elements - # to process. - actions_elem_list.append( child_element ) - elif child_element.tag == 'action': - # Any <action> tags within an <actions_group> tag set must come after all <actions> tags. - if actions_elems_processed == actions_elem_count: - # If all <actions> elements have been processed, then this <action> element can be appended to the - # list of actions to execute within this group. - after_install_actions.append( child_element ) - else: - # If any <actions> elements remain to be processed, then log a message stating that <action> - # elements are not allowed to precede any <actions> elements within an <actions_group> tag set. - debug_msg = 'Error: <action> tags are only allowed at the end of an <actions_group> tag set after ' - debug_msg += 'all <actions> tags. Skipping <%s> element with type %s.' % \ - ( child_element.tag, child_element.get( 'type', 'unknown' ) ) - log.debug( debug_msg ) - continue - if platform_info_dict is None and not include_after_install_actions: - # We must be setting metadata on a repository. - if len( actions_elem_list ) >= 1: - actions_elem_tuples.append( ( in_actions_group, actions_elem_list[ 0 ] ) ) - else: - # We are processing a recipe that contains only an <actions_group> tag set for installing a binary, - # but does not include an additional recipe for installing and compiling from source. - actions_elem_tuples.append( ( in_actions_group, [] ) ) - elif platform_info_dict is not None and include_after_install_actions: - # We must be installing a repository. - if after_install_actions: - actions_elem_list.extend( after_install_actions ) - actions_elem_tuples.append( ( in_actions_group, actions_elem_list ) ) - else: - # Skip any element that is not <actions> or <actions_group> - this will skip comments, <repository> tags - # and <readme> tags. - in_actions_group = False - continue - return actions_elem_tuples - -def __shellquote( s ): - """Quote and escape the supplied string for use in shell expressions.""" - return "'" + s.replace( "'", "'\\''" ) + "'" - -def url_download( install_dir, downloaded_file_name, download_url, extract=True ): - file_path = os.path.join( install_dir, downloaded_file_name ) - src = None - dst = None - # Set a timer so we don't sit here forever. - start_time = time.time() - try: - src = urllib2.urlopen( download_url ) - dst = open( file_path, 'wb' ) - while True: - chunk = src.read( suc.CHUNK_SIZE ) - if chunk: - dst.write( chunk ) - else: - break - time_taken = time.time() - start_time - if time_taken > NO_OUTPUT_TIMEOUT: - err_msg = 'Downloading from URL %s took longer than the defined timeout period of %.1f seconds.' % \ - ( str( download_url ), NO_OUTPUT_TIMEOUT ) - raise Exception( err_msg ) - except Exception, e: - err_msg = err_msg = 'Error downloading from URL\n%s:\n%s' % ( str( download_url ), str( e ) ) - raise Exception( err_msg ) - finally: - if src: - src.close() - if dst: - dst.close() - if extract: - if istar( file_path ) or ( iszip( file_path ) and not isjar( file_path ) ): - archive = CompressedFile( file_path ) - extraction_path = archive.extract( install_dir ) - else: - extraction_path = os.path.abspath( install_dir ) - else: - extraction_path = os.path.abspath( install_dir ) - return extraction_path - -def zipfile_ok( path_to_archive ): - """ - This function is a bit pedantic and not functionally necessary. It checks whether there is no file pointing outside of the extraction, - because ZipFile.extractall() has some potential security holes. See python zipfile documentation for more details. - """ - basename = os.path.realpath( os.path.dirname( path_to_archive ) ) - zip_archive = zipfile.ZipFile( path_to_archive ) - for member in zip_archive.namelist(): - member_path = os.path.realpath( os.path.join( basename, member ) ) - if not member_path.startswith( basename ): - return False - return True diff -r 31740dffaf9ae4e345fd2d4de4ab44ed133dd567 -r b1b1e5cefca5b0193c8623a9aee7a39cd1dda55c lib/tool_shed/scripts/check_filesystem_for_empty_tool_dependency_installation_paths.py --- a/lib/tool_shed/scripts/check_filesystem_for_empty_tool_dependency_installation_paths.py +++ b/lib/tool_shed/scripts/check_filesystem_for_empty_tool_dependency_installation_paths.py @@ -7,7 +7,7 @@ new_path.extend( sys.path[1:] ) sys.path = new_path -from tool_shed.galaxy_install.tool_dependencies import td_common_util +from tool_shed.util.basic_util import INSTALLATION_LOG def main( args ): empty_installation_paths = [] @@ -31,13 +31,13 @@ no_files = False if len( dirs ) == 0: no_dirs = True - if len( files ) == 0 or len( files ) == 1 and td_common_util.INSTALLATION_LOG in files: + if len( files ) == 0 or len( files ) == 1 and INSTALLATION_LOG in files: no_files = True if no_files and no_dirs and root not in empty_installation_paths: empty_installation_paths.append( root ) if len( empty_installation_paths ) > 0: print 'The following %d tool dependency installation directories were found to be empty or contain only the file %s.' % \ - ( len( empty_installation_paths ), td_common_util.INSTALLATION_LOG ) + ( len( empty_installation_paths ), INSTALLATION_LOG ) if args.delete: for path in empty_installation_paths: if os.path.exists( path ): diff -r 31740dffaf9ae4e345fd2d4de4ab44ed133dd567 -r b1b1e5cefca5b0193c8623a9aee7a39cd1dda55c lib/tool_shed/scripts/check_s3_for_empty_tool_dependency_installation_paths.py --- a/lib/tool_shed/scripts/check_s3_for_empty_tool_dependency_installation_paths.py +++ b/lib/tool_shed/scripts/check_s3_for_empty_tool_dependency_installation_paths.py @@ -12,8 +12,7 @@ import boto -from tool_shed.galaxy_install.tool_dependencies import td_common_util - +from tool_shed.util.basic_util import INSTALLATION_LOG class BucketList( object ): @@ -77,7 +76,7 @@ # This would not be the case in a Galaxy instance, since the Galaxy admin will need to verify the contents of # the installation path in order to determine which action should be taken. elif len( tool_dependency_path_contents ) == 2 and \ - tool_dependency_path_contents[1].name.endswith( td_common_util.INSTALLATION_LOG ): + tool_dependency_path_contents[1].name.endswith( INSTALLATION_LOG ): empty_directories.append( tool_dependency_path_contents[ 0 ] ) return [ item.name for item in empty_directories ] @@ -106,7 +105,7 @@ print 'No empty installation paths found, exiting.' return 0 print 'The following %d tool dependency installation paths were found to be empty or contain only the file %s.' % \ - ( len( dependency_cleaner.empty_installation_paths ), td_common_util.INSTALLATION_LOG ) + ( len( dependency_cleaner.empty_installation_paths ), INSTALLATION_LOG ) if asbool( args.delete ): dependency_cleaner.delete_empty_installation_paths() else: diff -r 31740dffaf9ae4e345fd2d4de4ab44ed133dd567 -r b1b1e5cefca5b0193c8623a9aee7a39cd1dda55c lib/tool_shed/util/basic_util.py --- a/lib/tool_shed/util/basic_util.py +++ b/lib/tool_shed/util/basic_util.py @@ -1,5 +1,8 @@ import logging import os +import shutil +import sys +from string import Template from galaxy.util import unicodify @@ -10,8 +13,52 @@ log = logging.getLogger( __name__ ) +CHUNK_SIZE = 2**20 # 1Mb +INSTALLATION_LOG = 'INSTALLATION.log' +# Set no activity timeout to 20 minutes. +NO_OUTPUT_TIMEOUT = 1200.0 +MAXDIFFSIZE = 8000 MAX_DISPLAY_SIZE = 32768 +def evaluate_template( text, install_environment ): + """ + Substitute variables defined in XML blocks from dependencies file. The value of the received + repository_install_dir is the root installation directory of the repository that contains the + tool dependency. The value of the received install_dir is the root installation directory of + the tool_dependency. + """ + return Template( text ).safe_substitute( get_env_var_values( install_environment ) ) + +def get_env_var_values( install_environment ): + """ + Return a dictionary of values, some of which enable substitution of reserved words for the values. + The received install_enviroment object has 2 important attributes for reserved word substitution: + install_environment.tool_shed_repository_install_dir is the root installation directory of the repository + that contains the tool dependency being installed, and install_environment.install_dir is the root + installation directory of the tool dependency. + """ + env_var_dict = {} + env_var_dict[ 'REPOSITORY_INSTALL_DIR' ] = install_environment.tool_shed_repository_install_dir + env_var_dict[ 'INSTALL_DIR' ] = install_environment.install_dir + env_var_dict[ 'system_install' ] = install_environment.install_dir + # If the Python interpreter is 64bit then we can safely assume that the underlying system is also 64bit. + env_var_dict[ '__is64bit__' ] = sys.maxsize > 2**32 + return env_var_dict + +def move_file( current_dir, source, destination, rename_to=None ): + source_path = os.path.abspath( os.path.join( current_dir, source ) ) + source_file = os.path.basename( source_path ) + if rename_to is not None: + destination_file = rename_to + destination_directory = os.path.join( destination ) + destination_path = os.path.join( destination_directory, destination_file ) + else: + destination_directory = os.path.join( destination ) + destination_path = os.path.join( destination_directory, source_file ) + if not os.path.exists( destination_directory ): + os.makedirs( destination_directory ) + shutil.move( source_path, destination_path ) + def remove_dir( dir ): """Attempt to remove a directory from disk.""" if dir: diff -r 31740dffaf9ae4e345fd2d4de4ab44ed133dd567 -r b1b1e5cefca5b0193c8623a9aee7a39cd1dda55c lib/tool_shed/util/commit_util.py --- a/lib/tool_shed/util/commit_util.py +++ b/lib/tool_shed/util/commit_util.py @@ -12,6 +12,7 @@ from galaxy.util.odict import odict from galaxy.web import url_for import tool_shed.util.shed_util_common as suc +from tool_shed.util import basic_util from tool_shed.util import hg_util from tool_shed.util import tool_util from tool_shed.util import xml_util @@ -126,7 +127,7 @@ bzipped_file = bz2.BZ2File( uploaded_file_name, 'rb' ) while 1: try: - chunk = bzipped_file.read( suc.CHUNK_SIZE ) + chunk = bzipped_file.read( basic_util.CHUNK_SIZE ) except IOError: os.close( fd ) os.remove( uncompressed ) @@ -239,7 +240,7 @@ gzipped_file = gzip.GzipFile( uploaded_file_name, 'rb' ) while 1: try: - chunk = gzipped_file.read( suc.CHUNK_SIZE ) + chunk = gzipped_file.read( basic_util.CHUNK_SIZE ) except IOError, e: os.close( fd ) os.remove( uncompressed ) diff -r 31740dffaf9ae4e345fd2d4de4ab44ed133dd567 -r b1b1e5cefca5b0193c8623a9aee7a39cd1dda55c lib/tool_shed/util/metadata_util.py --- a/lib/tool_shed/util/metadata_util.py +++ b/lib/tool_shed/util/metadata_util.py @@ -22,11 +22,12 @@ from tool_shed.util import tool_dependency_util from tool_shed.util import tool_util from tool_shed.util import xml_util -from tool_shed.galaxy_install.tool_dependencies import td_common_util import tool_shed.repository_types.util as rt_util log = logging.getLogger( __name__ ) +REPOSITORY_DATA_MANAGER_CONFIG_FILENAME = 'data_manager_conf.xml' + # Repository metadata comparisons for changeset revisions. EQUAL = 'equal' NO_METADATA = 'no metadata' @@ -37,7 +38,7 @@ NOT_TOOL_CONFIGS = [ suc.DATATYPES_CONFIG_FILENAME, rt_util.REPOSITORY_DEPENDENCY_DEFINITION_FILENAME, rt_util.TOOL_DEPENDENCY_DEFINITION_FILENAME, - suc.REPOSITORY_DATA_MANAGER_CONFIG_FILENAME ] + REPOSITORY_DATA_MANAGER_CONFIG_FILENAME ] def add_tool_versions( trans, id, repository_metadata, changeset_revisions ): # Build a dictionary of { 'tool id' : 'parent tool id' } pairs for each tool in repository_metadata. @@ -750,7 +751,7 @@ metadata_dict = generate_data_manager_metadata( app, repository, files_dir, - hg_util.get_config_from_disk( suc.REPOSITORY_DATA_MANAGER_CONFIG_FILENAME, files_dir ), + hg_util.get_config_from_disk( REPOSITORY_DATA_MANAGER_CONFIG_FILENAME, files_dir ), metadata_dict, shed_config_dict=shed_config_dict ) @@ -809,10 +810,10 @@ if package_install_version == '1.0': # Complex repository dependencies can be defined within the last <actions> tag set contained in an # <actions_group> tag set. Comments, <repository> tag sets and <readme> tag sets will be skipped - # in td_common_util.parse_package_elem(). - actions_elem_tuples = td_common_util.parse_package_elem( sub_elem, - platform_info_dict=None, - include_after_install_actions=False ) + # in tool_dependency_util.parse_package_elem(). + actions_elem_tuples = tool_dependency_util.parse_package_elem( sub_elem, + platform_info_dict=None, + include_after_install_actions=False ) if actions_elem_tuples: # We now have a list of a single tuple that looks something like: # [(True, <Element 'actions' at 0x104017850>)] diff -r 31740dffaf9ae4e345fd2d4de4ab44ed133dd567 -r b1b1e5cefca5b0193c8623a9aee7a39cd1dda55c lib/tool_shed/util/shed_util_common.py --- a/lib/tool_shed/util/shed_util_common.py +++ b/lib/tool_shed/util/shed_util_common.py @@ -24,11 +24,8 @@ log = logging.getLogger( __name__ ) -CHUNK_SIZE = 2**20 # 1Mb MAX_CONTENT_SIZE = 1048576 -MAXDIFFSIZE = 8000 DATATYPES_CONFIG_FILENAME = 'datatypes_conf.xml' -REPOSITORY_DATA_MANAGER_CONFIG_FILENAME = 'data_manager_conf.xml' new_repo_email_alert_template = """ Sharable link: ${sharable_link} This diff is so big that we needed to truncate the remainder. Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.