1 new commit in galaxy-central:
https://bitbucket.org/galaxy/galaxy-central/commits/b1b1e5cefca5/
Changeset: b1b1e5cefca5
User: greg
Date: 2014-05-28 21:49:02
Summary: Eliminate the Tool Shed's td_common_util.py module by moving the
functions to appropriate classes.
Affected #: 15 files
diff -r 31740dffaf9ae4e345fd2d4de4ab44ed133dd567 -r
b1b1e5cefca5b0193c8623a9aee7a39cd1dda55c
lib/galaxy/webapps/tool_shed/controllers/repository.py
--- a/lib/galaxy/webapps/tool_shed/controllers/repository.py
+++ b/lib/galaxy/webapps/tool_shed/controllers/repository.py
@@ -3120,12 +3120,12 @@
options_dict = hg_util.get_mercurial_default_options_dict( 'diff' )
# Not quite sure if the following settings make any difference, but with a
combination of them and the size check on each
# diff, we don't run out of memory when viewing the changelog of the
cisortho2 repository on the test tool shed.
- options_dict[ 'maxfile' ] = suc.MAXDIFFSIZE
- options_dict[ 'maxtotal' ] = suc.MAXDIFFSIZE
+ options_dict[ 'maxfile' ] = basic_util.MAXDIFFSIZE
+ options_dict[ 'maxtotal' ] = basic_util.MAXDIFFSIZE
diffopts = mdiff.diffopts( **options_dict )
for diff in patch.diff( repo, node1=ctx_parent.node(), node2=ctx.node(),
opts=diffopts ):
- if len( diff ) > suc.MAXDIFFSIZE:
- diff = util.shrink_string_by_size( diff, suc.MAXDIFFSIZE )
+ if len( diff ) > basic_util.MAXDIFFSIZE:
+ diff = util.shrink_string_by_size( diff, basic_util.MAXDIFFSIZE )
diffs.append( basic_util.to_html_string( diff ) )
modified, added, removed, deleted, unknown, ignored, clean = repo.status(
node1=ctx_parent.node(), node2=ctx.node() )
anchors = modified + added + removed + deleted + unknown + ignored + clean
diff -r 31740dffaf9ae4e345fd2d4de4ab44ed133dd567 -r
b1b1e5cefca5b0193c8623a9aee7a39cd1dda55c lib/tool_shed/galaxy_install/install_manager.py
--- a/lib/tool_shed/galaxy_install/install_manager.py
+++ b/lib/tool_shed/galaxy_install/install_manager.py
@@ -1,5 +1,7 @@
import logging
import os
+import sys
+import traceback
from galaxy import eggs
@@ -11,7 +13,6 @@
from tool_shed.util import tool_dependency_util
-from tool_shed.galaxy_install.tool_dependencies import td_common_util
from tool_shed.galaxy_install.tool_dependencies.recipe.env_file_builder import
EnvFileBuilder
from tool_shed.galaxy_install.tool_dependencies.recipe.install_environment import
InstallEnvironment
from tool_shed.galaxy_install.tool_dependencies.recipe.recipe_manager import StepManager
@@ -25,6 +26,10 @@
class InstallManager( object ):
+ def format_traceback( self ):
+ ex_type, ex, tb = sys.exc_info()
+ return ''.join( traceback.format_tb( tb ) )
+
def get_tool_shed_repository_install_dir( self, app, tool_shed_repository ):
return os.path.abspath( tool_shed_repository.repo_files_directory( app ) )
@@ -112,7 +117,7 @@
log.exception( 'Error installing tool dependency %s version %s.',
str( tool_dependency.name ), str( tool_dependency.version ) )
# Since there was an installation error, update the tool dependency status to
Error. The remove_installation_path option must
# be left False here.
- error_message = '%s\n%s' % ( td_common_util.format_traceback(), str(
e ) )
+ error_message = '%s\n%s' % ( self.format_traceback(), str( e ) )
tool_dependency =
tool_dependency_util.handle_tool_dependency_installation_error( app,
tool_dependency,
error_message,
diff -r 31740dffaf9ae4e345fd2d4de4ab44ed133dd567 -r
b1b1e5cefca5b0193c8623a9aee7a39cd1dda55c
lib/tool_shed/galaxy_install/tool_dependencies/env_manager.py
--- /dev/null
+++ b/lib/tool_shed/galaxy_install/tool_dependencies/env_manager.py
@@ -0,0 +1,147 @@
+import logging
+import os
+import sys
+from tool_shed.util import common_util
+import tool_shed.util.shed_util_common as suc
+
+log = logging.getLogger( __name__ )
+
+
+class EnvManager( object ):
+
+ def __init__( self, app ):
+ self.app = app
+
+ def create_env_var_dict( self, elem, install_environment ):
+ env_var_name = elem.get( 'name', 'PATH' )
+ env_var_action = elem.get( 'action', 'prepend_to' )
+ env_var_text = None
+ tool_dependency_install_dir = install_environment.install_dir
+ tool_shed_repository_install_dir =
install_environment.tool_shed_repository_install_dir
+ if elem.text and elem.text.find( 'REPOSITORY_INSTALL_DIR' ) >= 0:
+ if tool_shed_repository_install_dir and elem.text.find(
'$REPOSITORY_INSTALL_DIR' ) != -1:
+ env_var_text = elem.text.replace( '$REPOSITORY_INSTALL_DIR',
tool_shed_repository_install_dir )
+ return dict( name=env_var_name, action=env_var_action, value=env_var_text
)
+ else:
+ env_var_text = elem.text.replace( '$REPOSITORY_INSTALL_DIR',
tool_dependency_install_dir )
+ return dict( name=env_var_name, action=env_var_action, value=env_var_text
)
+ if elem.text and elem.text.find( 'INSTALL_DIR' ) >= 0:
+ if tool_dependency_install_dir:
+ env_var_text = elem.text.replace( '$INSTALL_DIR',
tool_dependency_install_dir )
+ return dict( name=env_var_name, action=env_var_action, value=env_var_text
)
+ else:
+ env_var_text = elem.text.replace( '$INSTALL_DIR',
tool_shed_repository_install_dir )
+ return dict( name=env_var_name, action=env_var_action, value=env_var_text
)
+ if elem.text:
+ # Allow for environment variables that contain neither REPOSITORY_INSTALL_DIR
nor INSTALL_DIR
+ # since there may be command line parameters that are tuned for a Galaxy
instance. Allowing them
+ # to be set in one location rather than being hard coded into each tool
config is the best approach.
+ # For example:
+ # <environment_variable name="GATK2_SITE_OPTIONS"
action="set_to">
+ # "--num_threads 4 --num_cpu_threads_per_data_thread 3 --phone_home
STANDARD"
+ # </environment_variable>
+ return dict( name=env_var_name, action=env_var_action, value=elem.text)
+ return None
+
+ def get_env_shell_file_path( self, installation_directory ):
+ env_shell_file_name = 'env.sh'
+ default_location = os.path.abspath( os.path.join( installation_directory,
env_shell_file_name ) )
+ if os.path.exists( default_location ):
+ return default_location
+ for root, dirs, files in os.walk( installation_directory ):
+ for name in files:
+ if name == env_shell_file_name:
+ return os.path.abspath( os.path.join( root, name ) )
+ return None
+
+ def get_env_shell_file_paths( self, elem ):
+ # Currently only the following tag set is supported.
+ # <repository toolshed="http://localhost:9009/"
name="package_numpy_1_7" owner="test"
changeset_revision="c84c6a8be056">
+ # <package name="numpy" version="1.7.1" />
+ # </repository>
+ env_shell_file_paths = []
+ toolshed = elem.get( 'toolshed', None )
+ repository_name = elem.get( 'name', None )
+ repository_owner = elem.get( 'owner', None )
+ changeset_revision = elem.get( 'changeset_revision', None )
+ if toolshed and repository_name and repository_owner and changeset_revision:
+ # The protocol is not stored, but the port is if it exists.
+ toolshed = common_util.remove_protocol_from_tool_shed_url( toolshed )
+ repository = suc.get_repository_for_dependency_relationship( self.app,
+ toolshed,
+
repository_name,
+
repository_owner,
+
changeset_revision )
+ if repository:
+ for sub_elem in elem:
+ tool_dependency_type = sub_elem.tag
+ tool_dependency_name = sub_elem.get( 'name' )
+ tool_dependency_version = sub_elem.get( 'version' )
+ if tool_dependency_type and tool_dependency_name and
tool_dependency_version:
+ # Get the tool_dependency so we can get its installation
directory.
+ tool_dependency = None
+ for tool_dependency in repository.tool_dependencies:
+ if tool_dependency.type == tool_dependency_type and \
+ tool_dependency.name == tool_dependency_name and \
+ tool_dependency.version == tool_dependency_version:
+ break
+ if tool_dependency:
+ tool_dependency_key = '%s/%s' % (
tool_dependency_name, tool_dependency_version )
+ installation_directory =
tool_dependency.installation_directory( self.app )
+ env_shell_file_path = self.get_env_shell_file_path(
installation_directory )
+ if env_shell_file_path:
+ env_shell_file_paths.append( env_shell_file_path )
+ else:
+ error_message = "Skipping tool dependency definition
because unable to locate env.sh file for tool dependency "
+ error_message += "type %s, name %s, version %s for
repository %s" % \
+ ( str( tool_dependency_type ), str(
tool_dependency_name ), str( tool_dependency_version ), str( repository.name ) )
+ log.debug( error_message )
+ continue
+ else:
+ error_message = "Skipping tool dependency definition
because unable to locate tool dependency "
+ error_message += "type %s, name %s, version %s for
repository %s" % \
+ ( str( tool_dependency_type ), str( tool_dependency_name
), str( tool_dependency_version ), str( repository.name ) )
+ log.debug( error_message )
+ continue
+ else:
+ error_message = "Skipping invalid tool dependency
definition: type %s, name %s, version %s." % \
+ ( str( tool_dependency_type ), str( tool_dependency_name ),
str( tool_dependency_version ) )
+ log.debug( error_message )
+ continue
+ else:
+ error_message = "Skipping set_environment_for_install definition
because unable to locate required installed tool shed repository: "
+ error_message += "toolshed %s, name %s, owner %s, changeset_revision
%s." % \
+ ( str( toolshed ), str( repository_name ), str( repository_owner ),
str( changeset_revision ) )
+ log.debug( error_message )
+ else:
+ error_message = "Skipping invalid set_environment_for_install
definition: toolshed %s, name %s, owner %s, changeset_revision %s." % \
+ ( str( toolshed ), str( repository_name ), str( repository_owner ), str(
changeset_revision ) )
+ log.debug( error_message )
+ return env_shell_file_paths
+
+ def get_env_shell_file_paths_from_setup_environment_elem( self,
all_env_shell_file_paths, elem, action_dict ):
+ """
+ Parse an XML tag set to discover all child repository dependency tags and define
the path to an env.sh file associated
+ with the repository (this requires the repository dependency to be in an
installed state). The received action_dict
+ will be updated with these discovered paths and returned to the caller. This
method handles tool dependency definition
+ tag sets <setup_r_environment>, <setup_ruby_environment> and
<setup_perl_environment>.
+ """
+ # An example elem is:
+ # <action type="setup_perl_environment">
+ # <repository name="package_perl_5_18"
owner="iuc">
+ # <package name="perl" version="5.18.1" />
+ # </repository>
+ # <repository name="package_expat_2_1" owner="iuc"
prior_installation_required="True">
+ # <package name="expat" version="2.1.0" />
+ # </repository>
+ #
<
package>http://search.cpan.org/CPAN/authors/id/T/TO/TODDR/XML-Parser-2...
+ #
<
package>http://search.cpan.org/CPAN/authors/id/L/LD/LDS/CGI.pm-3.43.ta...
+ # </action>
+ for action_elem in elem:
+ if action_elem.tag == 'repository':
+ env_shell_file_paths = self.get_env_shell_file_paths( action_elem )
+ all_env_shell_file_paths.extend( env_shell_file_paths )
+ if all_env_shell_file_paths:
+ action_dict[ 'env_shell_file_paths' ] = all_env_shell_file_paths
+ action_dict[ 'action_shell_file_paths' ] = env_shell_file_paths
+ return action_dict
diff -r 31740dffaf9ae4e345fd2d4de4ab44ed133dd567 -r
b1b1e5cefca5b0193c8623a9aee7a39cd1dda55c
lib/tool_shed/galaxy_install/tool_dependencies/recipe/install_environment.py
--- a/lib/tool_shed/galaxy_install/tool_dependencies/recipe/install_environment.py
+++ b/lib/tool_shed/galaxy_install/tool_dependencies/recipe/install_environment.py
@@ -24,11 +24,13 @@
from galaxy.util import shrink_string_by_size
from galaxy.util import unicodify
-from tool_shed.galaxy_install.tool_dependencies import td_common_util
from tool_shed.galaxy_install.tool_dependencies.recipe import asynchronous_reader
+from tool_shed.util import basic_util
+
log = logging.getLogger( __name__ )
+
class InstallEnvironment( object ):
"""Object describing the environment built up as part of the process
of building and installing a package."""
@@ -44,7 +46,7 @@
self.tool_shed_repository_install_dir = tool_shed_repository_install_dir
def __call__( self ):
- with settings( warn_only=True, **td_common_util.get_env_var_values( self ) ):
+ with settings( warn_only=True, **basic_util.get_env_var_values( self ) ):
with prefix( self.__setup_environment() ):
yield
@@ -125,7 +127,7 @@
context = app.install_model.context
command = str( cmd )
output = self.handle_complex_command( command )
- self.log_results( cmd, output, os.path.join( self.install_dir,
td_common_util.INSTALLATION_LOG ) )
+ self.log_results( cmd, output, os.path.join( self.install_dir,
basic_util.INSTALLATION_LOG ) )
stdout = output.stdout
stderr = output.stderr
if len( stdout ) > DATABASE_MAX_STRING_SIZE:
@@ -214,18 +216,18 @@
# Sleep a bit before asking the readers again.
time.sleep( .1 )
current_wait_time = time.time() - start_timer
- if stdout_queue.empty() and stderr_queue.empty() and current_wait_time >
td_common_util.NO_OUTPUT_TIMEOUT:
+ if stdout_queue.empty() and stderr_queue.empty() and current_wait_time >
basic_util.NO_OUTPUT_TIMEOUT:
err_msg = "\nShutting down process id %s because it generated no
output for the defined timeout period of %.1f seconds.\n" % \
- ( pid, td_common_util.NO_OUTPUT_TIMEOUT )
+ ( pid, basic_util.NO_OUTPUT_TIMEOUT )
stderr_reader.lines.append( err_msg )
process_handle.kill()
break
thread_lock.release()
# Wait until each of the threads we've started terminate. The following
calls will block each thread
# until it terminates either normally, through an unhandled exception, or until
the timeout occurs.
- stdio_thread.join( td_common_util.NO_OUTPUT_TIMEOUT )
- stdout_reader.join( td_common_util.NO_OUTPUT_TIMEOUT )
- stderr_reader.join( td_common_util.NO_OUTPUT_TIMEOUT )
+ stdio_thread.join( basic_util.NO_OUTPUT_TIMEOUT )
+ stdout_reader.join( basic_util.NO_OUTPUT_TIMEOUT )
+ stderr_reader.join( basic_util.NO_OUTPUT_TIMEOUT )
# Close subprocess' file descriptors.
error = self.close_file_descriptor( process_handle.stdout )
error = self.close_file_descriptor( process_handle.stderr )
diff -r 31740dffaf9ae4e345fd2d4de4ab44ed133dd567 -r
b1b1e5cefca5b0193c8623a9aee7a39cd1dda55c
lib/tool_shed/galaxy_install/tool_dependencies/recipe/step_handler.py
--- a/lib/tool_shed/galaxy_install/tool_dependencies/recipe/step_handler.py
+++ b/lib/tool_shed/galaxy_install/tool_dependencies/recipe/step_handler.py
@@ -4,12 +4,17 @@
import stat
from string import Template
import sys
+import tarfile
+import time
+import urllib2
+import zipfile
from galaxy.util import asbool
from galaxy.util.template import fill_template
+from tool_shed.util import basic_util
from tool_shed.util import tool_dependency_util
-from tool_shed.galaxy_install.tool_dependencies import td_common_util
+from tool_shed.galaxy_install.tool_dependencies.env_manager import EnvManager
# TODO: eliminate the use of fabric here.
from galaxy import eggs
@@ -26,6 +31,152 @@
VIRTUALENV_URL =
'https://pypi.python.org/packages/source/v/virtualenv/virtualenv-1.9.1.tar.gz'
+class CompressedFile( object ):
+
+ def __init__( self, file_path, mode='r' ):
+ if tarfile.is_tarfile( file_path ):
+ self.file_type = 'tar'
+ elif zipfile.is_zipfile( file_path ) and not file_path.endswith( '.jar'
):
+ self.file_type = 'zip'
+ self.file_name = os.path.splitext( os.path.basename( file_path ) )[ 0 ]
+ if self.file_name.endswith( '.tar' ):
+ self.file_name = os.path.splitext( self.file_name )[ 0 ]
+ self.type = self.file_type
+ method = 'open_%s' % self.file_type
+ if hasattr( self, method ):
+ self.archive = getattr( self, method )( file_path, mode )
+ else:
+ raise NameError( 'File type %s specified, no open method found.' %
self.file_type )
+
+ def extract( self, path ):
+ '''Determine the path to which the archive should be
extracted.'''
+ contents = self.getmembers()
+ extraction_path = path
+ if len( contents ) == 1:
+ # The archive contains a single file, return the extraction path.
+ if self.isfile( contents[ 0 ] ):
+ extraction_path = os.path.join( path, self.file_name )
+ if not os.path.exists( extraction_path ):
+ os.makedirs( extraction_path )
+ self.archive.extractall( extraction_path )
+ else:
+ # Get the common prefix for all the files in the archive. If the common
prefix ends with a slash,
+ # or self.isdir() returns True, the archive contains a single directory with
the desired contents.
+ # Otherwise, it contains multiple files and/or directories at the root of the
archive.
+ common_prefix = os.path.commonprefix( [ self.getname( item ) for item in
contents ] )
+ if len( common_prefix ) >= 1 and not common_prefix.endswith( os.sep ) and
self.isdir( self.getmember( common_prefix ) ):
+ common_prefix += os.sep
+ if common_prefix.endswith( os.sep ):
+ self.archive.extractall( os.path.join( path ) )
+ extraction_path = os.path.join( path, common_prefix )
+ else:
+ extraction_path = os.path.join( path, self.file_name )
+ if not os.path.exists( extraction_path ):
+ os.makedirs( extraction_path )
+ self.archive.extractall( os.path.join( extraction_path ) )
+ return os.path.abspath( extraction_path )
+
+ def getmembers_tar( self ):
+ return self.archive.getmembers()
+
+ def getmembers_zip( self ):
+ return self.archive.infolist()
+
+ def getname_tar( self, item ):
+ return item.name
+
+ def getname_zip( self, item ):
+ return item.filename
+
+ def getmember( self, name ):
+ for member in self.getmembers():
+ if self.getname( member ) == name:
+ return member
+
+ def getmembers( self ):
+ return getattr( self, 'getmembers_%s' % self.type )()
+
+ def getname( self, member ):
+ return getattr( self, 'getname_%s' % self.type )( member )
+
+ def isdir( self, member ):
+ return getattr( self, 'isdir_%s' % self.type )( member )
+
+ def isdir_tar( self, member ):
+ return member.isdir()
+
+ def isdir_zip( self, member ):
+ if member.filename.endswith( os.sep ):
+ return True
+ return False
+
+ def isfile( self, member ):
+ if not self.isdir( member ):
+ return True
+ return False
+
+ def open_tar( self, filepath, mode ):
+ return tarfile.open( filepath, mode, errorlevel=0 )
+
+ def open_zip( self, filepath, mode ):
+ return zipfile.ZipFile( filepath, mode )
+
+ def zipfile_ok( self, path_to_archive ):
+ """
+ This function is a bit pedantic and not functionally necessary. It checks
whether there is
+ no file pointing outside of the extraction, because ZipFile.extractall() has some
potential
+ security holes. See python zipfile documentation for more details.
+ """
+ basename = os.path.realpath( os.path.dirname( path_to_archive ) )
+ zip_archive = zipfile.ZipFile( path_to_archive )
+ for member in zip_archive.namelist():
+ member_path = os.path.realpath( os.path.join( basename, member ) )
+ if not member_path.startswith( basename ):
+ return False
+ return True
+
+
+class Download( object ):
+
+ def url_download( self, install_dir, downloaded_file_name, download_url, extract=True
):
+ file_path = os.path.join( install_dir, downloaded_file_name )
+ src = None
+ dst = None
+ # Set a timer so we don't sit here forever.
+ start_time = time.time()
+ try:
+ src = urllib2.urlopen( download_url )
+ dst = open( file_path, 'wb' )
+ while True:
+ chunk = src.read( basic_util.CHUNK_SIZE )
+ if chunk:
+ dst.write( chunk )
+ else:
+ break
+ time_taken = time.time() - start_time
+ if time_taken > basic_util.NO_OUTPUT_TIMEOUT:
+ err_msg = 'Downloading from URL %s took longer than the defined
timeout period of %.1f seconds.' % \
+ ( str( download_url ), basic_util.NO_OUTPUT_TIMEOUT )
+ raise Exception( err_msg )
+ except Exception, e:
+ err_msg = err_msg = 'Error downloading from URL\n%s:\n%s' % ( str(
download_url ), str( e ) )
+ raise Exception( err_msg )
+ finally:
+ if src:
+ src.close()
+ if dst:
+ dst.close()
+ if extract:
+ if tarfile.is_tarfile( file_path ) or ( zipfile.is_zipfile( file_path ) and
not file_path.endswith( '.jar' ) ):
+ archive = CompressedFile( file_path )
+ extraction_path = archive.extract( install_dir )
+ else:
+ extraction_path = os.path.abspath( install_dir )
+ else:
+ extraction_path = os.path.abspath( install_dir )
+ return extraction_path
+
+
class RecipeStep( object ):
"""Abstract class that defines a standard format for handling recipe
steps when installing packages."""
@@ -42,6 +193,22 @@
def __init__( self ):
self.type = 'assert_directory_executable'
+ def assert_directory_executable( self, full_path ):
+ """
+ Return True if a symbolic link or directory exists and is executable, but if
+ full_path is a file, return False.
+ """
+ if full_path is None:
+ return False
+ if os.path.isfile( full_path ):
+ return False
+ if os.path.isdir( full_path ):
+ # Make sure the owner has execute permission on the directory.
+ # See
http://docs.python.org/2/library/stat.html
+ if stat.S_IXUSR & os.stat( full_path )[ stat.ST_MODE ] == 64:
+ return True
+ return False
+
def execute_step( self, app, tool_dependency, package_name, actions, action_dict,
filtered_actions, env_file_builder,
install_environment, work_dir, current_dir=None,
initial_download=False ):
"""
@@ -53,7 +220,7 @@
full_path = action_dict[ 'full_path' ]
else:
full_path = os.path.join( current_dir, action_dict[ 'full_path' ] )
- if not td_common_util.assert_directory_executable( full_path=full_path ):
+ if not self.assert_directory_executable( full_path=full_path ):
status = app.install_model.ToolDependency.installation_status.ERROR
error_message = 'The path %s is not a directory or is not executable by
the owner.' % str( full_path )
tool_dependency = tool_dependency_util.set_tool_dependency_attributes( app,
@@ -66,7 +233,7 @@
def prepare_step( self, app, tool_dependency, action_elem, action_dict,
install_environment, is_binary_download ):
# <action
type="assert_executable">$INSTALL_DIR/mira/my_file</action>
if action_elem.text:
- action_dict[ 'full_path' ] = td_common_util.evaluate_template(
action_elem.text, install_environment )
+ action_dict[ 'full_path' ] = basic_util.evaluate_template(
action_elem.text, install_environment )
return action_dict
@@ -75,6 +242,18 @@
def __init__( self ):
self.type = 'assert_directory_exists'
+ def assert_directory_exists( self, full_path ):
+ """
+ Return True if a symbolic link or directory exists, but if full_path is a file,
+ return False. """
+ if full_path is None:
+ return False
+ if os.path.isfile( full_path ):
+ return False
+ if os.path.isdir( full_path ):
+ return True
+ return False
+
def execute_step( self, app, tool_dependency, package_name, actions, action_dict,
filtered_actions, env_file_builder,
install_environment, work_dir, current_dir=None,
initial_download=False ):
"""
@@ -86,7 +265,7 @@
full_path = action_dict[ 'full_path' ]
else:
full_path = os.path.join( current_dir, action_dict[ 'full_path' ] )
- if not td_common_util.assert_directory_exists( full_path=full_path ):
+ if not self.assert_directory_exists( full_path=full_path ):
status = app.install_model.ToolDependency.installation_status.ERROR
error_message = 'The path %s is not a directory or does not exist.' %
str( full_path )
tool_dependency = tool_dependency_util.set_tool_dependency_attributes( app,
@@ -99,7 +278,7 @@
def prepare_step( self, app, tool_dependency, action_elem, action_dict,
install_environment, is_binary_download ):
# <action type="make_directory">$INSTALL_DIR/mira</action>
if action_elem.text:
- action_dict[ 'full_path' ] = td_common_util.evaluate_template(
action_elem.text, install_environment )
+ action_dict[ 'full_path' ] = basic_util.evaluate_template(
action_elem.text, install_environment )
return action_dict
@@ -108,6 +287,22 @@
def __init__( self ):
self.type = 'assert_file_executable'
+ def assert_file_executable( self, full_path ):
+ """
+ Return True if a symbolic link or file exists and is executable, but if
full_path
+ is a directory, return False.
+ """
+ if full_path is None:
+ return False
+ if os.path.isdir( full_path ):
+ return False
+ if os.path.exists( full_path ):
+ # Make sure the owner has execute permission on the file.
+ # See
http://docs.python.org/2/library/stat.html
+ if stat.S_IXUSR & os.stat( full_path )[ stat.ST_MODE ] == 64:
+ return True
+ return False
+
def execute_step( self, app, tool_dependency, package_name, actions, action_dict,
filtered_actions, env_file_builder,
install_environment, work_dir, current_dir=None,
initial_download=False ):
"""
@@ -119,7 +314,7 @@
full_path = action_dict[ 'full_path' ]
else:
full_path = os.path.join( current_dir, action_dict[ 'full_path' ] )
- if not td_common_util.assert_file_executable( full_path=full_path ):
+ if not self.assert_file_executable( full_path=full_path ):
status = app.install_model.ToolDependency.installation_status.ERROR
error_message = 'The path %s is not a file or is not executable by the
owner.' % str( full_path )
tool_dependency = tool_dependency_util.set_tool_dependency_attributes( app,
@@ -132,7 +327,7 @@
def prepare_step( self, app, tool_dependency, action_elem, action_dict,
install_environment, is_binary_download ):
# <action
type="assert_executable">$INSTALL_DIR/mira/my_file</action>
if action_elem.text:
- action_dict[ 'full_path' ] = td_common_util.evaluate_template(
action_elem.text, install_environment )
+ action_dict[ 'full_path' ] = basic_util.evaluate_template(
action_elem.text, install_environment )
return action_dict
@@ -141,6 +336,19 @@
def __init__( self ):
self.type = 'assert_file_exists'
+ def assert_file_exists( self, full_path ):
+ """
+ Return True if a symbolic link or file exists, but if full_path is a directory,
+ return False.
+ """
+ if full_path is None:
+ return False
+ if os.path.isdir( full_path ):
+ return False
+ if os.path.exists( full_path ):
+ return True
+ return False
+
def execute_step( self, app, tool_dependency, package_name, actions, action_dict,
filtered_actions, env_file_builder,
install_environment, work_dir, current_dir=None,
initial_download=False ):
"""
@@ -152,7 +360,7 @@
full_path = action_dict[ 'full_path' ]
else:
full_path = os.path.join( current_dir, action_dict[ 'full_path' ] )
- if not td_common_util.assert_file_exists( full_path=full_path ):
+ if not self.assert_file_exists( full_path=full_path ):
status = app.install_model.ToolDependency.installation_status.ERROR
error_message = 'The path %s is not a file or does not exist.' % str(
full_path )
tool_dependency = tool_dependency_util.set_tool_dependency_attributes( app,
@@ -165,7 +373,7 @@
def prepare_step( self, app, tool_dependency, action_elem, action_dict,
install_environment, is_binary_download ):
# <action
type="assert_on_path">$INSTALL_DIR/mira/my_file</action>
if action_elem.text:
- action_dict[ 'full_path' ] = td_common_util.evaluate_template(
action_elem.text, install_environment )
+ action_dict[ 'full_path' ] = basic_util.evaluate_template(
action_elem.text, install_environment )
return action_dict
@@ -187,7 +395,7 @@
pre_cmd = './configure %s && make && make
install' % configure_opts
else:
pre_cmd = './configure --prefix=$INSTALL_DIR %s && make
&& make install' % configure_opts
- cmd = install_environment.build_command( td_common_util.evaluate_template(
pre_cmd, install_environment ) )
+ cmd = install_environment.build_command( basic_util.evaluate_template(
pre_cmd, install_environment ) )
return_code = install_environment.handle_command( app=app,
tool_dependency=tool_dependency,
cmd=cmd,
@@ -199,7 +407,7 @@
def prepare_step( self, app, tool_dependency, action_elem, action_dict,
install_environment, is_binary_download ):
# Handle configure, make and make install allow providing configuration options
if action_elem.text:
- configure_opts = td_common_util.evaluate_template( action_elem.text,
install_environment )
+ configure_opts = basic_util.evaluate_template( action_elem.text,
install_environment )
action_dict[ 'configure_opts' ] = configure_opts
return action_dict
@@ -274,7 +482,7 @@
received_mode = int( file_elem.get( 'mode', 600 ), base=8 )
# For added security, ensure that the setuid and setgid bits are not set.
mode = received_mode & ~( stat.S_ISUID | stat.S_ISGID )
- file = td_common_util.evaluate_template( file_elem.text, install_environment
)
+ file = basic_util.evaluate_template( file_elem.text, install_environment )
chmod_tuple = ( file, mode )
chmod_actions.append( chmod_tuple )
if chmod_actions:
@@ -282,11 +490,17 @@
return action_dict
-class DownloadBinary( RecipeStep ):
+class DownloadBinary( Download, RecipeStep ):
def __init__( self ):
self.type = 'download_binary'
+ def download_binary( self, url, work_dir ):
+ """Download a pre-compiled binary from the specified
URL."""
+ downloaded_filename = os.path.split( url )[ -1 ]
+ dir = self.url_download( work_dir, downloaded_filename, url, extract=False )
+ return downloaded_filename
+
def filter_actions_after_binary_installation( self, actions ):
'''Filter out actions that should not be processed if a binary
download succeeded.'''
filtered_actions = []
@@ -311,7 +525,7 @@
log.debug( 'Attempting to download from %s to %s', url, str(
target_directory ) )
downloaded_filename = None
try:
- downloaded_filename = td_common_util.download_binary( url, work_dir )
+ downloaded_filename = self.download_binary( url, work_dir )
if initial_download:
# Filter out any actions that are not download_binary, chmod, or
set_environment.
filtered_actions = self.filter_actions_after_binary_installation(
actions[ 1: ] )
@@ -338,9 +552,9 @@
full_path_to_dir = os.path.abspath( install_environment.install_dir
)
else:
full_path_to_dir = os.path.abspath( install_environment.install_dir )
- td_common_util.move_file( current_dir=work_dir,
- source=downloaded_filename,
- destination=full_path_to_dir )
+ basic_util.move_file( current_dir=work_dir,
+ source=downloaded_filename,
+ destination=full_path_to_dir )
# Not sure why dir is ignored in this method, need to investigate...
dir = None
if initial_download:
@@ -368,7 +582,7 @@
return action_dict
-class DownloadByUrl( RecipeStep ):
+class DownloadByUrl( Download, RecipeStep ):
def __init__( self ):
self.type = 'download_by_url'
@@ -394,9 +608,9 @@
downloaded_filename = action_dict[ 'target_filename' ]
else:
downloaded_filename = os.path.split( url )[ -1 ]
- dir = td_common_util.url_download( work_dir, downloaded_filename, url,
extract=True )
+ dir = self.url_download( work_dir, downloaded_filename, url, extract=True )
if is_binary:
- log_file = os.path.join( install_environment.install_dir,
td_common_util.INSTALLATION_LOG )
+ log_file = os.path.join( install_environment.install_dir,
basic_util.INSTALLATION_LOG )
if os.path.exists( log_file ):
logfile = open( log_file, 'ab' )
else:
@@ -422,7 +636,7 @@
return action_dict
-class DownloadFile( RecipeStep ):
+class DownloadFile( Download, RecipeStep ):
def __init__( self ):
self.type = 'download_file'
@@ -447,7 +661,7 @@
filename = action_dict[ 'target_filename' ]
else:
filename = url.split( '/' )[ -1 ]
- td_common_util.url_download( work_dir, filename, url )
+ self.url_download( work_dir, filename, url )
if initial_download:
dir = os.path.curdir
return tool_dependency, filtered_actions, dir
@@ -479,13 +693,17 @@
full_path = action_dict[ 'full_path' ]
else:
full_path = os.path.join( current_dir, action_dict[ 'full_path' ] )
- td_common_util.make_directory( full_path=full_path )
+ self.make_directory( full_path=full_path )
return tool_dependency, None, None
+ def make_directory( self, full_path ):
+ if not os.path.exists( full_path ):
+ os.makedirs( full_path )
+
def prepare_step( self, app, tool_dependency, action_elem, action_dict,
install_environment, is_binary_download ):
# <action
type="make_directory">$INSTALL_DIR/lib/python</action>
if action_elem.text:
- action_dict[ 'full_path' ] = td_common_util.evaluate_template(
action_elem.text, install_environment )
+ action_dict[ 'full_path' ] = basic_util.evaluate_template(
action_elem.text, install_environment )
return action_dict
@@ -515,7 +733,7 @@
def prepare_step( self, app, tool_dependency, action_elem, action_dict,
install_environment, is_binary_download ):
# make; make install; allow providing make options
if action_elem.text:
- make_opts = td_common_util.evaluate_template( action_elem.text,
install_environment )
+ make_opts = basic_util.evaluate_template( action_elem.text,
install_environment )
action_dict[ 'make_opts' ] = make_opts
return action_dict
@@ -531,18 +749,38 @@
Move a directory of files. Since this class is not used in the initial download
stage, no recipe step
filtering is performed here, and None values are always returned for
filtered_actions and dir.
"""
- td_common_util.move_directory_files( current_dir=current_dir,
- source_dir=os.path.join( action_dict[
'source_directory' ] ),
- destination_dir=os.path.join( action_dict[
'destination_directory' ] ) )
+ self.move_directory_files( current_dir=current_dir,
+ source_dir=os.path.join( action_dict[
'source_directory' ] ),
+ destination_dir=os.path.join( action_dict[
'destination_directory' ] ) )
return tool_dependency, None, None
+ def move_directory_files( self, current_dir, source_dir, destination_dir ):
+ source_directory = os.path.abspath( os.path.join( current_dir, source_dir ) )
+ destination_directory = os.path.join( destination_dir )
+ if not os.path.isdir( destination_directory ):
+ os.makedirs( destination_directory )
+ symlinks = []
+ regular_files = []
+ for file_name in os.listdir( source_directory ):
+ source_file = os.path.join( source_directory, file_name )
+ destination_file = os.path.join( destination_directory, file_name )
+ files_tuple = ( source_file, destination_file )
+ if os.path.islink( source_file ):
+ symlinks.append( files_tuple )
+ else:
+ regular_files.append( files_tuple )
+ for source_file, destination_file in symlinks:
+ shutil.move( source_file, destination_file )
+ for source_file, destination_file in regular_files:
+ shutil.move( source_file, destination_file )
+
def prepare_step( self, app, tool_dependency, action_elem, action_dict,
install_environment, is_binary_download ):
# <action type="move_directory_files">
# <source_directory>bin</source_directory>
#
<destination_directory>$INSTALL_DIR/bin</destination_directory>
# </action>
for move_elem in action_elem:
- move_elem_text = td_common_util.evaluate_template( move_elem.text,
install_environment )
+ move_elem_text = basic_util.evaluate_template( move_elem.text,
install_environment )
if move_elem_text:
action_dict[ move_elem.tag ] = move_elem_text
return action_dict
@@ -559,10 +797,10 @@
Move a file on disk. Since this class is not used in the initial download stage,
no recipe step
filtering is performed here, and None values are always returned for
filtered_actions and dir.
"""
- td_common_util.move_file( current_dir=current_dir,
- source=os.path.join( action_dict[ 'source' ]
),
- destination=os.path.join( action_dict[
'destination' ] ),
- rename_to=action_dict[ 'rename_to' ] )
+ basic_util.move_file( current_dir=current_dir,
+ source=os.path.join( action_dict[ 'source' ] ),
+ destination=os.path.join( action_dict[
'destination' ] ),
+ rename_to=action_dict[ 'rename_to' ] )
return tool_dependency, None, None
def prepare_step( self, app, tool_dependency, action_elem, action_dict,
install_environment, is_binary_download ):
@@ -570,8 +808,8 @@
# <source>misc/some_file</source>
# <destination>$INSTALL_DIR/bin</destination>
# </action>
- action_dict[ 'source' ] = td_common_util.evaluate_template(
action_elem.find( 'source' ).text, install_environment )
- action_dict[ 'destination' ] = td_common_util.evaluate_template(
action_elem.find( 'destination' ).text, install_environment )
+ action_dict[ 'source' ] = basic_util.evaluate_template( action_elem.find(
'source' ).text, install_environment )
+ action_dict[ 'destination' ] = basic_util.evaluate_template(
action_elem.find( 'destination' ).text, install_environment )
action_dict[ 'rename_to' ] = action_elem.get( 'rename_to' )
return action_dict
@@ -717,12 +955,12 @@
# <action type="set_environment">
# <environment_variable name="PATH"
action="prepend_to">$INSTALL_DIR</environment_variable>
# </action>
+ env_manager = EnvManager( app )
env_var_dicts = []
for env_elem in action_elem:
if env_elem.tag == 'environment_variable':
- env_var_dict = \
- td_common_util.create_env_var_dict( elem=env_elem,
-
install_environment=install_environment )
+ env_var_dict = env_manager.create_env_var_dict( elem=env_elem,
+
install_environment=install_environment )
if env_var_dict:
env_var_dicts.append( env_var_dict )
if env_var_dicts:
@@ -764,16 +1002,17 @@
# the current tool dependency package. See the package_matplotlib_1_2 repository
in the test tool
# shed for a real-world example.
all_env_shell_file_paths = []
+ env_manager = EnvManager( app )
for env_elem in action_elem:
if env_elem.tag == 'repository':
- env_shell_file_paths = td_common_util.get_env_shell_file_paths( app,
env_elem )
+ env_shell_file_paths = env_manager.get_env_shell_file_paths( env_elem )
if env_shell_file_paths:
all_env_shell_file_paths.extend( env_shell_file_paths )
action_dict[ 'env_shell_file_paths' ] = all_env_shell_file_paths
return action_dict
-class SetupPerlEnvironment( RecipeStep ):
+class SetupPerlEnvironment( Download, RecipeStep ):
def __init__( self ):
self.type = 'setup_purl_environment'
@@ -822,7 +1061,7 @@
# We assume a URL to a gem file.
url = perl_package
perl_package_name = url.split( '/' )[ -1 ]
- dir = td_common_util.url_download( work_dir, perl_package_name,
url, extract=True )
+ dir = self.url_download( work_dir, perl_package_name, url,
extract=True )
# Search for Build.PL or Makefile.PL (ExtUtils::MakeMaker vs.
Module::Build).
tmp_work_dir = os.path.join( work_dir, dir )
if os.path.exists( os.path.join( tmp_work_dir,
'Makefile.PL' ) ):
@@ -836,7 +1075,7 @@
return tool_dependency, filtered_actions, dir
return tool_dependency, None, None
with lcd( tmp_work_dir ):
- cmd = install_environment.build_command(
td_common_util.evaluate_template( cmd, install_environment ) )
+ cmd = install_environment.build_command(
basic_util.evaluate_template( cmd, install_environment ) )
return_code = install_environment.handle_command( app=app,
tool_dependency=tool_dependency,
cmd=cmd,
@@ -849,7 +1088,7 @@
# perl package from CPAN without version number.
# cpanm should be installed with the parent perl distribution,
otherwise this will not work.
cmd += '''cpanm --local-lib=$INSTALL_DIR
%s''' % ( perl_package )
- cmd = install_environment.build_command(
td_common_util.evaluate_template( cmd, install_environment ) )
+ cmd = install_environment.build_command(
basic_util.evaluate_template( cmd, install_environment ) )
return_code = install_environment.handle_command( app=app,
tool_dependency=tool_dependency,
cmd=cmd,
@@ -890,10 +1129,11 @@
# with each repository. This will potentially update the value of the
'env_shell_file_paths' entry
# in action_dict.
all_env_shell_file_paths = []
- action_dict =
td_common_util.get_env_shell_file_paths_from_setup_environment_elem( app,
-
all_env_shell_file_paths,
-
action_elem,
-
action_dict )
+ env_manager = EnvManager( app )
+ action_dict = env_manager.get_env_shell_file_paths_from_setup_environment_elem(
app,
+
all_env_shell_file_paths,
+
action_elem,
+
action_dict )
perl_packages = []
for env_elem in action_elem:
if env_elem.tag == 'package':
@@ -908,7 +1148,7 @@
return action_dict
-class SetupREnvironment( RecipeStep ):
+class SetupREnvironment( Download, RecipeStep ):
def __init__( self ):
self.type = 'setup_r_environment'
@@ -947,7 +1187,7 @@
for url in action_dict[ 'r_packages' ]:
filename = url.split( '/' )[ -1 ]
tarball_names.append( filename )
- td_common_util.url_download( work_dir, filename, url, extract=False )
+ self.url_download( work_dir, filename, url, extract=False )
dir = os.path.curdir
current_dir = os.path.abspath( os.path.join( work_dir, dir ) )
with lcd( current_dir ):
@@ -958,7 +1198,7 @@
cmd = r'''PATH=$PATH:$R_HOME/bin; export PATH;
R_LIBS=$INSTALL_DIR; export R_LIBS;
Rscript -e
"install.packages(c('%s'),lib='$INSTALL_DIR', repos=NULL,
dependencies=FALSE)"''' % \
( str( tarball_name ) )
- cmd = install_environment.build_command(
td_common_util.evaluate_template( cmd, install_environment ) )
+ cmd = install_environment.build_command(
basic_util.evaluate_template( cmd, install_environment ) )
return_code = install_environment.handle_command( app=app,
tool_dependency=tool_dependency,
cmd=cmd,
@@ -993,10 +1233,11 @@
# associated with each repository. This will potentially update the value of
the
# 'env_shell_file_paths' entry in action_dict.
all_env_shell_file_paths = []
- action_dict =
td_common_util.get_env_shell_file_paths_from_setup_environment_elem( app,
-
all_env_shell_file_paths,
-
action_elem,
-
action_dict )
+ env_manager = EnvManager( app )
+ action_dict = env_manager.get_env_shell_file_paths_from_setup_environment_elem(
app,
+
all_env_shell_file_paths,
+
action_elem,
+
action_dict )
r_packages = list()
for env_elem in action_elem:
if env_elem.tag == 'package':
@@ -1006,7 +1247,7 @@
return action_dict
-class SetupRubyEnvironment( RecipeStep ):
+class SetupRubyEnvironment( Download, RecipeStep ):
def __init__( self ):
self.type = 'setup_ruby_environment'
@@ -1058,7 +1299,7 @@
# We assume a URL to a gem file.
url = gem
gem_name = url.split( '/' )[ -1 ]
- td_common_util.url_download( work_dir, gem_name, url,
extract=False )
+ self.url_download( work_dir, gem_name, url, extract=False )
cmd = '''PATH=$PATH:$RUBY_HOME/bin; export PATH;
GEM_HOME=$INSTALL_DIR; export GEM_HOME;
gem install --local %s ''' % ( gem_name )
else:
@@ -1073,7 +1314,7 @@
# no version number given
cmd = '''PATH=$PATH:$RUBY_HOME/bin; export PATH;
GEM_HOME=$INSTALL_DIR; export GEM_HOME;
gem install %s''' % ( gem )
- cmd = install_environment.build_command(
td_common_util.evaluate_template( cmd, install_environment ) )
+ cmd = install_environment.build_command(
basic_util.evaluate_template( cmd, install_environment ) )
return_code = install_environment.handle_command( app=app,
tool_dependency=tool_dependency,
cmd=cmd,
@@ -1114,10 +1355,11 @@
# associated with each repository. This will potentially update the value of
the
# 'env_shell_file_paths' entry in action_dict.
all_env_shell_file_paths = []
- action_dict =
td_common_util.get_env_shell_file_paths_from_setup_environment_elem( app,
-
all_env_shell_file_paths,
-
action_elem,
-
action_dict )
+ env_manager = EnvManager( app )
+ action_dict = env_manager.get_env_shell_file_paths_from_setup_environment_elem(
app,
+
all_env_shell_file_paths,
+
action_elem,
+
action_dict )
ruby_package_tups = []
for env_elem in action_elem:
if env_elem.tag == 'package':
@@ -1140,7 +1382,7 @@
return action_dict
-class SetupVirtualEnv( RecipeStep ):
+class SetupVirtualEnv( Download, RecipeStep ):
def __init__( self ):
self.type = 'setup_virtualenv'
@@ -1228,9 +1470,10 @@
with install_environment.make_tmp_dir() as work_dir:
downloaded_filename = VIRTUALENV_URL.rsplit('/', 1)[-1]
try:
- dir = td_common_util.url_download( work_dir, downloaded_filename,
VIRTUALENV_URL )
+ dir = self.url_download( work_dir, downloaded_filename,
VIRTUALENV_URL )
except:
- log.error( "Failed to download virtualenv:
td_common_util.url_download( '%s', '%s', '%s' ) threw an
exception", work_dir, downloaded_filename, VIRTUALENV_URL )
+ log.error( "Failed to download virtualenv: url_download(
'%s', '%s', '%s' ) threw an exception",
+ work_dir, downloaded_filename, VIRTUALENV_URL )
return False
full_path_to_dir = os.path.abspath( os.path.join( work_dir, dir ) )
shutil.move( full_path_to_dir, venv_dir )
@@ -1245,7 +1488,7 @@
# lxml==2.3.0</action>
## Manually specify contents of requirements.txt file to create dynamically.
action_dict[ 'use_requirements_file' ] = asbool( action_elem.get(
'use_requirements_file', True ) )
- action_dict[ 'requirements' ] = td_common_util.evaluate_template(
action_elem.text or 'requirements.txt', install_environment )
+ action_dict[ 'requirements' ] = basic_util.evaluate_template(
action_elem.text or 'requirements.txt', install_environment )
action_dict[ 'python' ] = action_elem.get( 'python',
'python' )
return action_dict
@@ -1316,7 +1559,7 @@
def prepare_step( self, app, tool_dependency, action_elem, action_dict,
install_environment, is_binary_download ):
# <action type="shell_command">make</action>
- action_elem_text = td_common_util.evaluate_template( action_elem.text,
install_environment )
+ action_elem_text = basic_util.evaluate_template( action_elem.text,
install_environment )
if action_elem_text:
action_dict[ 'command' ] = action_elem_text
return action_dict
@@ -1338,7 +1581,7 @@
env_vars = dict()
env_vars = install_environment.environment_dict()
tool_shed_repository = tool_dependency.tool_shed_repository
- env_vars.update( td_common_util.get_env_var_values( install_environment ) )
+ env_vars.update( basic_util.get_env_var_values( install_environment ) )
language = action_dict[ 'language' ]
with settings( warn_only=True, **env_vars ):
if language == 'cheetah':
diff -r 31740dffaf9ae4e345fd2d4de4ab44ed133dd567 -r
b1b1e5cefca5b0193c8623a9aee7a39cd1dda55c
lib/tool_shed/galaxy_install/tool_dependencies/recipe/tag_handler.py
--- a/lib/tool_shed/galaxy_install/tool_dependencies/recipe/tag_handler.py
+++ b/lib/tool_shed/galaxy_install/tool_dependencies/recipe/tag_handler.py
@@ -10,7 +10,7 @@
from tool_shed.util import tool_dependency_util
from tool_shed.util import xml_util
-from tool_shed.galaxy_install.tool_dependencies import td_common_util
+from tool_shed.galaxy_install.tool_dependencies.env_manager import EnvManager
from tool_shed.galaxy_install.tool_dependencies.recipe.env_file_builder import
EnvFileBuilder
from tool_shed.galaxy_install.tool_dependencies.recipe.install_environment import
InstallEnvironment
@@ -85,9 +85,9 @@
platform_info_dict = tool_dependency_util.get_platform_info_dict()
if package_install_version == '1.0':
# Handle tool dependency installation using a fabric method included in
the Galaxy framework.
- actions_elem_tuples = td_common_util.parse_package_elem( package_elem,
-
platform_info_dict=platform_info_dict,
-
include_after_install_actions=True )
+ actions_elem_tuples = tool_dependency_util.parse_package_elem(
package_elem,
+
platform_info_dict=platform_info_dict,
+
include_after_install_actions=True )
if not actions_elem_tuples:
proceed_with_install = False
error_message = 'Version %s of the %s package cannot be installed
because ' % ( str( package_version ), str( package_name ) )
@@ -491,6 +491,7 @@
# <set_environment version="1.0">
# <repository toolshed="<tool shed>"
name="<repository name>" owner="<repository owner>"
changeset_revision="<changeset revision>" />
# </set_environment>
+ env_manager = EnvManager( app )
tool_dependencies = []
env_var_version = elem.get( 'version', '1.0' )
tool_shed_repository_install_dir = os.path.abspath(
tool_shed_repository.repo_files_directory( app ) )
@@ -514,8 +515,8 @@
tool_dependency_version=None )
install_environment = InstallEnvironment(
tool_shed_repository_install_dir=tool_shed_repository_install_dir,
install_dir=install_dir )
- env_var_dict = td_common_util.create_env_var_dict(
elem=env_var_elem,
-
install_environment=install_environment )
+ env_var_dict = env_manager.create_env_var_dict( elem=env_var_elem,
+
install_environment=install_environment )
if env_var_dict:
if not os.path.exists( install_dir ):
os.makedirs( install_dir )
diff -r 31740dffaf9ae4e345fd2d4de4ab44ed133dd567 -r
b1b1e5cefca5b0193c8623a9aee7a39cd1dda55c
lib/tool_shed/galaxy_install/tool_dependencies/td_common_util.py
--- a/lib/tool_shed/galaxy_install/tool_dependencies/td_common_util.py
+++ /dev/null
@@ -1,578 +0,0 @@
-import logging
-import os
-import re
-import shutil
-import stat
-import sys
-import tarfile
-import time
-import traceback
-import urllib2
-import zipfile
-from string import Template
-from tool_shed.util import common_util
-import tool_shed.util.shed_util_common as suc
-from galaxy.datatypes import checkers
-
-log = logging.getLogger( __name__ )
-
-# Set no activity timeout to 20 minutes.
-NO_OUTPUT_TIMEOUT = 1200.0
-INSTALLATION_LOG = 'INSTALLATION.log'
-
-
-class CompressedFile( object ):
-
- def __init__( self, file_path, mode='r' ):
- if istar( file_path ):
- self.file_type = 'tar'
- elif iszip( file_path ) and not isjar( file_path ):
- self.file_type = 'zip'
- self.file_name = os.path.splitext( os.path.basename( file_path ) )[ 0 ]
- if self.file_name.endswith( '.tar' ):
- self.file_name = os.path.splitext( self.file_name )[ 0 ]
- self.type = self.file_type
- method = 'open_%s' % self.file_type
- if hasattr( self, method ):
- self.archive = getattr( self, method )( file_path, mode )
- else:
- raise NameError( 'File type %s specified, no open method found.' %
self.file_type )
-
- def extract( self, path ):
- '''Determine the path to which the archive should be
extracted.'''
- contents = self.getmembers()
- extraction_path = path
- if len( contents ) == 1:
- # The archive contains a single file, return the extraction path.
- if self.isfile( contents[ 0 ] ):
- extraction_path = os.path.join( path, self.file_name )
- if not os.path.exists( extraction_path ):
- os.makedirs( extraction_path )
- self.archive.extractall( extraction_path )
- else:
- # Get the common prefix for all the files in the archive. If the common
prefix ends with a slash,
- # or self.isdir() returns True, the archive contains a single directory with
the desired contents.
- # Otherwise, it contains multiple files and/or directories at the root of the
archive.
- common_prefix = os.path.commonprefix( [ self.getname( item ) for item in
contents ] )
- if len( common_prefix ) >= 1 and not common_prefix.endswith( os.sep ) and
self.isdir( self.getmember( common_prefix ) ):
- common_prefix += os.sep
- if common_prefix.endswith( os.sep ):
- self.archive.extractall( os.path.join( path ) )
- extraction_path = os.path.join( path, common_prefix )
- else:
- extraction_path = os.path.join( path, self.file_name )
- if not os.path.exists( extraction_path ):
- os.makedirs( extraction_path )
- self.archive.extractall( os.path.join( extraction_path ) )
- return os.path.abspath( extraction_path )
-
- def getmembers_tar( self ):
- return self.archive.getmembers()
-
- def getmembers_zip( self ):
- return self.archive.infolist()
-
- def getname_tar( self, item ):
- return item.name
-
- def getname_zip( self, item ):
- return item.filename
-
- def getmember( self, name ):
- for member in self.getmembers():
- if self.getname( member ) == name:
- return member
-
- def getmembers( self ):
- return getattr( self, 'getmembers_%s' % self.type )()
-
- def getname( self, member ):
- return getattr( self, 'getname_%s' % self.type )( member )
-
- def isdir( self, member ):
- return getattr( self, 'isdir_%s' % self.type )( member )
-
- def isdir_tar( self, member ):
- return member.isdir()
-
- def isdir_zip( self, member ):
- if member.filename.endswith( os.sep ):
- return True
- return False
-
- def isfile( self, member ):
- if not self.isdir( member ):
- return True
- return False
-
- def open_tar( self, filepath, mode ):
- return tarfile.open( filepath, mode, errorlevel=0 )
-
- def open_zip( self, filepath, mode ):
- return zipfile.ZipFile( filepath, mode )
-
-def assert_directory_executable( full_path ):
- """
- Return True if a symbolic link or directory exists and is executable, but if
- full_path is a file, return False.
- """
- if full_path is None:
- return False
- if os.path.isfile( full_path ):
- return False
- if os.path.isdir( full_path ):
- # Make sure the owner has execute permission on the directory.
- # See
http://docs.python.org/2/library/stat.html
- if stat.S_IXUSR & os.stat( full_path )[ stat.ST_MODE ] == 64:
- return True
- return False
-
-def assert_directory_exists( full_path ):
- """
- Return True if a symbolic link or directory exists, but if full_path is a file,
- return False. """
- if full_path is None:
- return False
- if os.path.isfile( full_path ):
- return False
- if os.path.isdir( full_path ):
- return True
- return False
-
-def assert_file_executable( full_path ):
- """
- Return True if a symbolic link or file exists and is executable, but if full_path
- is a directory, return False.
- """
- if full_path is None:
- return False
- if os.path.isdir( full_path ):
- return False
- if os.path.exists( full_path ):
- # Make sure the owner has execute permission on the file.
- # See
http://docs.python.org/2/library/stat.html
- if stat.S_IXUSR & os.stat( full_path )[ stat.ST_MODE ] == 64:
- return True
- return False
-
-def assert_file_exists( full_path ):
- """
- Return True if a symbolic link or file exists, but if full_path is a directory,
- return False.
- """
- if full_path is None:
- return False
- if os.path.isdir( full_path ):
- return False
- if os.path.exists( full_path ):
- return True
- return False
-
-def create_env_var_dict( elem, install_environment ):
- env_var_name = elem.get( 'name', 'PATH' )
- env_var_action = elem.get( 'action', 'prepend_to' )
- env_var_text = None
- tool_dependency_install_dir = install_environment.install_dir
- tool_shed_repository_install_dir =
install_environment.tool_shed_repository_install_dir
- if elem.text and elem.text.find( 'REPOSITORY_INSTALL_DIR' ) >= 0:
- if tool_shed_repository_install_dir and elem.text.find(
'$REPOSITORY_INSTALL_DIR' ) != -1:
- env_var_text = elem.text.replace( '$REPOSITORY_INSTALL_DIR',
tool_shed_repository_install_dir )
- return dict( name=env_var_name, action=env_var_action, value=env_var_text )
- else:
- env_var_text = elem.text.replace( '$REPOSITORY_INSTALL_DIR',
tool_dependency_install_dir )
- return dict( name=env_var_name, action=env_var_action, value=env_var_text )
- if elem.text and elem.text.find( 'INSTALL_DIR' ) >= 0:
- if tool_dependency_install_dir:
- env_var_text = elem.text.replace( '$INSTALL_DIR',
tool_dependency_install_dir )
- return dict( name=env_var_name, action=env_var_action, value=env_var_text )
- else:
- env_var_text = elem.text.replace( '$INSTALL_DIR',
tool_shed_repository_install_dir )
- return dict( name=env_var_name, action=env_var_action, value=env_var_text )
- if elem.text:
- # Allow for environment variables that contain neither REPOSITORY_INSTALL_DIR nor
INSTALL_DIR
- # since there may be command line parameters that are tuned for a Galaxy
instance. Allowing them
- # to be set in one location rather than being hard coded into each tool config is
the best approach.
- # For example:
- # <environment_variable name="GATK2_SITE_OPTIONS"
action="set_to">
- # "--num_threads 4 --num_cpu_threads_per_data_thread 3 --phone_home
STANDARD"
- # </environment_variable>
- return dict( name=env_var_name, action=env_var_action, value=elem.text)
- return None
-
-def download_binary( url, work_dir ):
- """Download a pre-compiled binary from the specified
URL."""
- downloaded_filename = os.path.split( url )[ -1 ]
- dir = url_download( work_dir, downloaded_filename, url, extract=False )
- return downloaded_filename
-
-def egrep_escape( text ):
- """Escape ``text`` to allow literal matching using
egrep."""
- regex = re.escape( text )
- # Seems like double escaping is needed for \
- regex = regex.replace( '\\\\', '\\\\\\' )
- # Triple-escaping seems to be required for $ signs
- regex = regex.replace( r'\$', r'\\\$' )
- # Whereas single quotes should not be escaped
- regex = regex.replace( r"\'", "'" )
- return regex
-
-def evaluate_template( text, install_environment ):
- """
- Substitute variables defined in XML blocks from dependencies file. The value of the
received
- repository_install_dir is the root installation directory of the repository that
contains the
- tool dependency. The value of the received install_dir is the root installation
directory of
- the tool_dependency.
- """
- return Template( text ).safe_substitute( get_env_var_values( install_environment ) )
-
-def format_traceback():
- ex_type, ex, tb = sys.exc_info()
- return ''.join( traceback.format_tb( tb ) )
-
-def get_env_shell_file_path( installation_directory ):
- env_shell_file_name = 'env.sh'
- default_location = os.path.abspath( os.path.join( installation_directory,
env_shell_file_name ) )
- if os.path.exists( default_location ):
- return default_location
- for root, dirs, files in os.walk( installation_directory ):
- for name in files:
- if name == env_shell_file_name:
- return os.path.abspath( os.path.join( root, name ) )
- return None
-
-def get_env_shell_file_paths( app, elem ):
- # Currently only the following tag set is supported.
- # <repository toolshed="http://localhost:9009/"
name="package_numpy_1_7" owner="test"
changeset_revision="c84c6a8be056">
- # <package name="numpy" version="1.7.1" />
- # </repository>
- env_shell_file_paths = []
- toolshed = elem.get( 'toolshed', None )
- repository_name = elem.get( 'name', None )
- repository_owner = elem.get( 'owner', None )
- changeset_revision = elem.get( 'changeset_revision', None )
- if toolshed and repository_name and repository_owner and changeset_revision:
- # The protocol is not stored, but the port is if it exists.
- toolshed = common_util.remove_protocol_from_tool_shed_url( toolshed )
- repository = suc.get_repository_for_dependency_relationship( app, toolshed,
repository_name, repository_owner, changeset_revision )
- if repository:
- for sub_elem in elem:
- tool_dependency_type = sub_elem.tag
- tool_dependency_name = sub_elem.get( 'name' )
- tool_dependency_version = sub_elem.get( 'version' )
- if tool_dependency_type and tool_dependency_name and
tool_dependency_version:
- # Get the tool_dependency so we can get its installation directory.
- tool_dependency = None
- for tool_dependency in repository.tool_dependencies:
- if tool_dependency.type == tool_dependency_type and \
- tool_dependency.name == tool_dependency_name and \
- tool_dependency.version == tool_dependency_version:
- break
- if tool_dependency:
- tool_dependency_key = '%s/%s' % ( tool_dependency_name,
tool_dependency_version )
- installation_directory = tool_dependency.installation_directory(
app )
- env_shell_file_path = get_env_shell_file_path(
installation_directory )
- if env_shell_file_path:
- env_shell_file_paths.append( env_shell_file_path )
- else:
- error_message = "Skipping tool dependency definition
because unable to locate env.sh file for tool dependency "
- error_message += "type %s, name %s, version %s for
repository %s" % \
- ( str( tool_dependency_type ), str( tool_dependency_name
), str( tool_dependency_version ), str( repository.name ) )
- log.debug( error_message )
- continue
- else:
- error_message = "Skipping tool dependency definition because
unable to locate tool dependency "
- error_message += "type %s, name %s, version %s for
repository %s" % \
- ( str( tool_dependency_type ), str( tool_dependency_name ),
str( tool_dependency_version ), str( repository.name ) )
- log.debug( error_message )
- continue
- else:
- error_message = "Skipping invalid tool dependency definition:
type %s, name %s, version %s." % \
- ( str( tool_dependency_type ), str( tool_dependency_name ), str(
tool_dependency_version ) )
- log.debug( error_message )
- continue
- else:
- error_message = "Skipping set_environment_for_install definition because
unable to locate required installed tool shed repository: "
- error_message += "toolshed %s, name %s, owner %s, changeset_revision
%s." % \
- ( str( toolshed ), str( repository_name ), str( repository_owner ), str(
changeset_revision ) )
- log.debug( error_message )
- else:
- error_message = "Skipping invalid set_environment_for_install definition:
toolshed %s, name %s, owner %s, changeset_revision %s." % \
- ( str( toolshed ), str( repository_name ), str( repository_owner ), str(
changeset_revision ) )
- log.debug( error_message )
- return env_shell_file_paths
-
-def get_env_shell_file_paths_from_setup_environment_elem( app, all_env_shell_file_paths,
elem, action_dict ):
- """
- Parse an XML tag set to discover all child repository dependency tags and define the
path to an env.sh file associated
- with the repository (this requires the repository dependency to be in an installed
state). The received action_dict
- will be updated with these discovered paths and returned to the caller. This method
handles tool dependency definition
- tag sets <setup_r_environment>, <setup_ruby_environment> and
<setup_perl_environment>.
- """
- # An example elem is:
- # <action type="setup_perl_environment">
- # <repository name="package_perl_5_18" owner="iuc">
- # <package name="perl" version="5.18.1" />
- # </repository>
- # <repository name="package_expat_2_1" owner="iuc"
prior_installation_required="True">
- # <package name="expat" version="2.1.0" />
- # </repository>
- #
<
package>http://search.cpan.org/CPAN/authors/id/T/TO/TODDR/XML-Parser-2...
- #
<
package>http://search.cpan.org/CPAN/authors/id/L/LD/LDS/CGI.pm-3.43.ta...
- # </action>
- for action_elem in elem:
- if action_elem.tag == 'repository':
- env_shell_file_paths = get_env_shell_file_paths( app, action_elem )
- all_env_shell_file_paths.extend( env_shell_file_paths )
- if all_env_shell_file_paths:
- action_dict[ 'env_shell_file_paths' ] = all_env_shell_file_paths
- action_dict[ 'action_shell_file_paths' ] = env_shell_file_paths
- return action_dict
-
-def get_env_var_values( install_environment ):
- """
- Return a dictionary of values, some of which enable substitution of reserved words
for the values.
- The received install_enviroment object has 2 important attributes for reserved word
substitution:
- install_environment.tool_shed_repository_install_dir is the root installation
directory of the repository
- that contains the tool dependency being installed, and
install_environment.install_dir is the root
- installation directory of the tool dependency.
- """
- env_var_dict = {}
- env_var_dict[ 'REPOSITORY_INSTALL_DIR' ] =
install_environment.tool_shed_repository_install_dir
- env_var_dict[ 'INSTALL_DIR' ] = install_environment.install_dir
- env_var_dict[ 'system_install' ] = install_environment.install_dir
- # If the Python interpreter is 64bit then we can safely assume that the underlying
system is also 64bit.
- env_var_dict[ '__is64bit__' ] = sys.maxsize > 2**32
- return env_var_dict
-
-def isbz2( file_path ):
- return checkers.is_bz2( file_path )
-
-def isgzip( file_path ):
- return checkers.is_gzip( file_path )
-
-def isjar( file_path ):
- return iszip( file_path ) and file_path.endswith( '.jar' )
-
-def istar( file_path ):
- return tarfile.is_tarfile( file_path )
-
-def iszip( file_path ):
- return checkers.check_zip( file_path )
-
-def is_compressed( file_path ):
- if isjar( file_path ):
- return False
- else:
- return iszip( file_path ) or isgzip( file_path ) or istar( file_path ) or isbz2(
file_path )
-
-def make_directory( full_path ):
- if not os.path.exists( full_path ):
- os.makedirs( full_path )
-
-def move_directory_files( current_dir, source_dir, destination_dir ):
- source_directory = os.path.abspath( os.path.join( current_dir, source_dir ) )
- destination_directory = os.path.join( destination_dir )
- if not os.path.isdir( destination_directory ):
- os.makedirs( destination_directory )
- symlinks = []
- regular_files = []
- for file_name in os.listdir( source_directory ):
- source_file = os.path.join( source_directory, file_name )
- destination_file = os.path.join( destination_directory, file_name )
- files_tuple = ( source_file, destination_file )
- if os.path.islink( source_file ):
- symlinks.append( files_tuple )
- else:
- regular_files.append( files_tuple )
- for source_file, destination_file in symlinks:
- shutil.move( source_file, destination_file )
- for source_file, destination_file in regular_files:
- shutil.move( source_file, destination_file )
-
-def move_file( current_dir, source, destination, rename_to=None ):
- source_path = os.path.abspath( os.path.join( current_dir, source ) )
- source_file = os.path.basename( source_path )
- if rename_to is not None:
- destination_file = rename_to
- destination_directory = os.path.join( destination )
- destination_path = os.path.join( destination_directory, destination_file )
- else:
- destination_directory = os.path.join( destination )
- destination_path = os.path.join( destination_directory, source_file )
- if not os.path.exists( destination_directory ):
- os.makedirs( destination_directory )
- shutil.move( source_path, destination_path )
-
-def parse_package_elem( package_elem, platform_info_dict=None,
include_after_install_actions=True ):
- """
- Parse a <package> element within a tool dependency definition and return a list
of action tuples.
- This method is called when setting metadata on a repository that includes a
tool_dependencies.xml
- file or when installing a repository that includes a tool_dependencies.xml file. If
installing,
- platform_info_dict must be a valid dictionary and include_after_install_actions must
be True.
- """
- # The actions_elem_tuples list contains <actions> tag sets (possibly inside of
an <actions_group>
- # tag set) to be processed in the order they are defined in the tool_dependencies.xml
file.
- actions_elem_tuples = []
- # The tag sets that will go into the actions_elem_list are those that install a
compiled binary if
- # the architecture and operating system match its defined attributes. If compiled
binary is not
- # installed, the first <actions> tag set [following those that have the os and
architecture attributes]
- # that does not have os or architecture attributes will be processed. This tag set
must contain the
- # recipe for downloading and compiling source.
- actions_elem_list = []
- for elem in package_elem:
- if elem.tag == 'actions':
- # We have an <actions> tag that should not be matched against a
specific combination of
- # architecture and operating system.
- in_actions_group = False
- actions_elem_tuples.append( ( in_actions_group, elem ) )
- elif elem.tag == 'actions_group':
- # We have an actions_group element, and its child <actions> elements
should therefore be compared
- # with the current operating system
- # and processor architecture.
- in_actions_group = True
- # Record the number of <actions> elements so we can filter out any
<action> elements that precede
- # <actions> elements.
- actions_elem_count = len( elem.findall( 'actions' ) )
- # Record the number of <actions> elements that have both architecture
and os specified, in order
- # to filter out any platform-independent <actions> elements that come
before platform-specific
- # <actions> elements.
- platform_actions_elements = []
- for actions_elem in elem.findall( 'actions' ):
- if actions_elem.get( 'architecture' ) is not None and
actions_elem.get( 'os' ) is not None:
- platform_actions_elements.append( actions_elem )
- platform_actions_element_count = len( platform_actions_elements )
- platform_actions_elements_processed = 0
- actions_elems_processed = 0
- # The tag sets that will go into the after_install_actions list are
<action> tags instead of <actions>
- # tags. These will be processed only if they are at the very end of the
<actions_group> tag set (after
- # all <actions> tag sets). See below for details.
- after_install_actions = []
- # Inspect the <actions_group> element and build the actions_elem_list
and the after_install_actions list.
- for child_element in elem:
- if child_element.tag == 'actions':
- actions_elems_processed += 1
- system = child_element.get( 'os' )
- architecture = child_element.get( 'architecture' )
- # Skip <actions> tags that have only one of architecture or os
specified, in order for the
- # count in platform_actions_elements_processed to remain accurate.
- if ( system and not architecture ) or ( architecture and not system
):
- log.debug( 'Error: Both architecture and os attributes must
be specified in an <actions> tag.' )
- continue
- # Since we are inside an <actions_group> tag set, compare it
with our current platform information
- # and filter the <actions> tag sets that don't match.
Require both the os and architecture attributes
- # to be defined in order to find a match.
- if system and architecture:
- platform_actions_elements_processed += 1
- # If either the os or architecture do not match the platform,
this <actions> tag will not be
- # considered a match. Skip it and proceed with checking the next
one.
- if platform_info_dict:
- if platform_info_dict[ 'os' ] != system or
platform_info_dict[ 'architecture' ] != architecture:
- continue
- else:
- # We must not be installing a repository into Galaxy, so
determining if we can install a
- # binary is not necessary.
- continue
- else:
- # <actions> tags without both os and architecture
attributes are only allowed to be specified
- # after platform-specific <actions> tags. If we find a
platform-independent <actions> tag before
- # all platform-specific <actions> tags have been
processed.
- if platform_actions_elements_processed <
platform_actions_element_count:
- debug_msg = 'Error: <actions> tags without os and
architecture attributes are only allowed '
- debug_msg += 'after all <actions> tags with os and
architecture attributes have been defined. '
- debug_msg += 'Skipping the <actions> tag set with
no os or architecture attributes that has '
- debug_msg += 'been defined between two <actions>
tag sets that have these attributes defined. '
- log.debug( debug_msg )
- continue
- # If we reach this point, it means one of two things: 1) The system
and architecture attributes are
- # not defined in this <actions> tag, or 2) The system and
architecture attributes are defined, and
- # they are an exact match for the current platform. Append the child
element to the list of elements
- # to process.
- actions_elem_list.append( child_element )
- elif child_element.tag == 'action':
- # Any <action> tags within an <actions_group> tag set
must come after all <actions> tags.
- if actions_elems_processed == actions_elem_count:
- # If all <actions> elements have been processed, then this
<action> element can be appended to the
- # list of actions to execute within this group.
- after_install_actions.append( child_element )
- else:
- # If any <actions> elements remain to be processed, then
log a message stating that <action>
- # elements are not allowed to precede any <actions>
elements within an <actions_group> tag set.
- debug_msg = 'Error: <action> tags are only allowed at
the end of an <actions_group> tag set after '
- debug_msg += 'all <actions> tags. Skipping <%s>
element with type %s.' % \
- ( child_element.tag, child_element.get( 'type',
'unknown' ) )
- log.debug( debug_msg )
- continue
- if platform_info_dict is None and not include_after_install_actions:
- # We must be setting metadata on a repository.
- if len( actions_elem_list ) >= 1:
- actions_elem_tuples.append( ( in_actions_group, actions_elem_list[ 0
] ) )
- else:
- # We are processing a recipe that contains only an
<actions_group> tag set for installing a binary,
- # but does not include an additional recipe for installing and
compiling from source.
- actions_elem_tuples.append( ( in_actions_group, [] ) )
- elif platform_info_dict is not None and include_after_install_actions:
- # We must be installing a repository.
- if after_install_actions:
- actions_elem_list.extend( after_install_actions )
- actions_elem_tuples.append( ( in_actions_group, actions_elem_list ) )
- else:
- # Skip any element that is not <actions> or <actions_group> -
this will skip comments, <repository> tags
- # and <readme> tags.
- in_actions_group = False
- continue
- return actions_elem_tuples
-
-def __shellquote( s ):
- """Quote and escape the supplied string for use in shell
expressions."""
- return "'" + s.replace( "'",
"'\\''" ) + "'"
-
-def url_download( install_dir, downloaded_file_name, download_url, extract=True ):
- file_path = os.path.join( install_dir, downloaded_file_name )
- src = None
- dst = None
- # Set a timer so we don't sit here forever.
- start_time = time.time()
- try:
- src = urllib2.urlopen( download_url )
- dst = open( file_path, 'wb' )
- while True:
- chunk = src.read( suc.CHUNK_SIZE )
- if chunk:
- dst.write( chunk )
- else:
- break
- time_taken = time.time() - start_time
- if time_taken > NO_OUTPUT_TIMEOUT:
- err_msg = 'Downloading from URL %s took longer than the defined
timeout period of %.1f seconds.' % \
- ( str( download_url ), NO_OUTPUT_TIMEOUT )
- raise Exception( err_msg )
- except Exception, e:
- err_msg = err_msg = 'Error downloading from URL\n%s:\n%s' % ( str(
download_url ), str( e ) )
- raise Exception( err_msg )
- finally:
- if src:
- src.close()
- if dst:
- dst.close()
- if extract:
- if istar( file_path ) or ( iszip( file_path ) and not isjar( file_path ) ):
- archive = CompressedFile( file_path )
- extraction_path = archive.extract( install_dir )
- else:
- extraction_path = os.path.abspath( install_dir )
- else:
- extraction_path = os.path.abspath( install_dir )
- return extraction_path
-
-def zipfile_ok( path_to_archive ):
- """
- This function is a bit pedantic and not functionally necessary. It checks whether
there is no file pointing outside of the extraction,
- because ZipFile.extractall() has some potential security holes. See python zipfile
documentation for more details.
- """
- basename = os.path.realpath( os.path.dirname( path_to_archive ) )
- zip_archive = zipfile.ZipFile( path_to_archive )
- for member in zip_archive.namelist():
- member_path = os.path.realpath( os.path.join( basename, member ) )
- if not member_path.startswith( basename ):
- return False
- return True
diff -r 31740dffaf9ae4e345fd2d4de4ab44ed133dd567 -r
b1b1e5cefca5b0193c8623a9aee7a39cd1dda55c
lib/tool_shed/scripts/check_filesystem_for_empty_tool_dependency_installation_paths.py
---
a/lib/tool_shed/scripts/check_filesystem_for_empty_tool_dependency_installation_paths.py
+++
b/lib/tool_shed/scripts/check_filesystem_for_empty_tool_dependency_installation_paths.py
@@ -7,7 +7,7 @@
new_path.extend( sys.path[1:] )
sys.path = new_path
-from tool_shed.galaxy_install.tool_dependencies import td_common_util
+from tool_shed.util.basic_util import INSTALLATION_LOG
def main( args ):
empty_installation_paths = []
@@ -31,13 +31,13 @@
no_files = False
if len( dirs ) == 0:
no_dirs = True
- if len( files ) == 0 or len( files ) == 1 and td_common_util.INSTALLATION_LOG in
files:
+ if len( files ) == 0 or len( files ) == 1 and INSTALLATION_LOG in files:
no_files = True
if no_files and no_dirs and root not in empty_installation_paths:
empty_installation_paths.append( root )
if len( empty_installation_paths ) > 0:
print 'The following %d tool dependency installation directories were found
to be empty or contain only the file %s.' % \
- ( len( empty_installation_paths ), td_common_util.INSTALLATION_LOG )
+ ( len( empty_installation_paths ), INSTALLATION_LOG )
if args.delete:
for path in empty_installation_paths:
if os.path.exists( path ):
diff -r 31740dffaf9ae4e345fd2d4de4ab44ed133dd567 -r
b1b1e5cefca5b0193c8623a9aee7a39cd1dda55c
lib/tool_shed/scripts/check_s3_for_empty_tool_dependency_installation_paths.py
--- a/lib/tool_shed/scripts/check_s3_for_empty_tool_dependency_installation_paths.py
+++ b/lib/tool_shed/scripts/check_s3_for_empty_tool_dependency_installation_paths.py
@@ -12,8 +12,7 @@
import boto
-from tool_shed.galaxy_install.tool_dependencies import td_common_util
-
+from tool_shed.util.basic_util import INSTALLATION_LOG
class BucketList( object ):
@@ -77,7 +76,7 @@
# This would not be the case in a Galaxy instance, since the Galaxy admin
will need to verify the contents of
# the installation path in order to determine which action should be taken.
elif len( tool_dependency_path_contents ) == 2 and \
- tool_dependency_path_contents[1].name.endswith(
td_common_util.INSTALLATION_LOG ):
+ tool_dependency_path_contents[1].name.endswith( INSTALLATION_LOG ):
empty_directories.append( tool_dependency_path_contents[ 0 ] )
return [ item.name for item in empty_directories ]
@@ -106,7 +105,7 @@
print 'No empty installation paths found, exiting.'
return 0
print 'The following %d tool dependency installation paths were found to be empty
or contain only the file %s.' % \
- ( len( dependency_cleaner.empty_installation_paths ),
td_common_util.INSTALLATION_LOG )
+ ( len( dependency_cleaner.empty_installation_paths ), INSTALLATION_LOG )
if asbool( args.delete ):
dependency_cleaner.delete_empty_installation_paths()
else:
diff -r 31740dffaf9ae4e345fd2d4de4ab44ed133dd567 -r
b1b1e5cefca5b0193c8623a9aee7a39cd1dda55c lib/tool_shed/util/basic_util.py
--- a/lib/tool_shed/util/basic_util.py
+++ b/lib/tool_shed/util/basic_util.py
@@ -1,5 +1,8 @@
import logging
import os
+import shutil
+import sys
+from string import Template
from galaxy.util import unicodify
@@ -10,8 +13,52 @@
log = logging.getLogger( __name__ )
+CHUNK_SIZE = 2**20 # 1Mb
+INSTALLATION_LOG = 'INSTALLATION.log'
+# Set no activity timeout to 20 minutes.
+NO_OUTPUT_TIMEOUT = 1200.0
+MAXDIFFSIZE = 8000
MAX_DISPLAY_SIZE = 32768
+def evaluate_template( text, install_environment ):
+ """
+ Substitute variables defined in XML blocks from dependencies file. The value of the
received
+ repository_install_dir is the root installation directory of the repository that
contains the
+ tool dependency. The value of the received install_dir is the root installation
directory of
+ the tool_dependency.
+ """
+ return Template( text ).safe_substitute( get_env_var_values( install_environment ) )
+
+def get_env_var_values( install_environment ):
+ """
+ Return a dictionary of values, some of which enable substitution of reserved words
for the values.
+ The received install_enviroment object has 2 important attributes for reserved word
substitution:
+ install_environment.tool_shed_repository_install_dir is the root installation
directory of the repository
+ that contains the tool dependency being installed, and
install_environment.install_dir is the root
+ installation directory of the tool dependency.
+ """
+ env_var_dict = {}
+ env_var_dict[ 'REPOSITORY_INSTALL_DIR' ] =
install_environment.tool_shed_repository_install_dir
+ env_var_dict[ 'INSTALL_DIR' ] = install_environment.install_dir
+ env_var_dict[ 'system_install' ] = install_environment.install_dir
+ # If the Python interpreter is 64bit then we can safely assume that the underlying
system is also 64bit.
+ env_var_dict[ '__is64bit__' ] = sys.maxsize > 2**32
+ return env_var_dict
+
+def move_file( current_dir, source, destination, rename_to=None ):
+ source_path = os.path.abspath( os.path.join( current_dir, source ) )
+ source_file = os.path.basename( source_path )
+ if rename_to is not None:
+ destination_file = rename_to
+ destination_directory = os.path.join( destination )
+ destination_path = os.path.join( destination_directory, destination_file )
+ else:
+ destination_directory = os.path.join( destination )
+ destination_path = os.path.join( destination_directory, source_file )
+ if not os.path.exists( destination_directory ):
+ os.makedirs( destination_directory )
+ shutil.move( source_path, destination_path )
+
def remove_dir( dir ):
"""Attempt to remove a directory from disk."""
if dir:
diff -r 31740dffaf9ae4e345fd2d4de4ab44ed133dd567 -r
b1b1e5cefca5b0193c8623a9aee7a39cd1dda55c lib/tool_shed/util/commit_util.py
--- a/lib/tool_shed/util/commit_util.py
+++ b/lib/tool_shed/util/commit_util.py
@@ -12,6 +12,7 @@
from galaxy.util.odict import odict
from galaxy.web import url_for
import tool_shed.util.shed_util_common as suc
+from tool_shed.util import basic_util
from tool_shed.util import hg_util
from tool_shed.util import tool_util
from tool_shed.util import xml_util
@@ -126,7 +127,7 @@
bzipped_file = bz2.BZ2File( uploaded_file_name, 'rb' )
while 1:
try:
- chunk = bzipped_file.read( suc.CHUNK_SIZE )
+ chunk = bzipped_file.read( basic_util.CHUNK_SIZE )
except IOError:
os.close( fd )
os.remove( uncompressed )
@@ -239,7 +240,7 @@
gzipped_file = gzip.GzipFile( uploaded_file_name, 'rb' )
while 1:
try:
- chunk = gzipped_file.read( suc.CHUNK_SIZE )
+ chunk = gzipped_file.read( basic_util.CHUNK_SIZE )
except IOError, e:
os.close( fd )
os.remove( uncompressed )
diff -r 31740dffaf9ae4e345fd2d4de4ab44ed133dd567 -r
b1b1e5cefca5b0193c8623a9aee7a39cd1dda55c lib/tool_shed/util/metadata_util.py
--- a/lib/tool_shed/util/metadata_util.py
+++ b/lib/tool_shed/util/metadata_util.py
@@ -22,11 +22,12 @@
from tool_shed.util import tool_dependency_util
from tool_shed.util import tool_util
from tool_shed.util import xml_util
-from tool_shed.galaxy_install.tool_dependencies import td_common_util
import tool_shed.repository_types.util as rt_util
log = logging.getLogger( __name__ )
+REPOSITORY_DATA_MANAGER_CONFIG_FILENAME = 'data_manager_conf.xml'
+
# Repository metadata comparisons for changeset revisions.
EQUAL = 'equal'
NO_METADATA = 'no metadata'
@@ -37,7 +38,7 @@
NOT_TOOL_CONFIGS = [ suc.DATATYPES_CONFIG_FILENAME,
rt_util.REPOSITORY_DEPENDENCY_DEFINITION_FILENAME,
rt_util.TOOL_DEPENDENCY_DEFINITION_FILENAME,
- suc.REPOSITORY_DATA_MANAGER_CONFIG_FILENAME ]
+ REPOSITORY_DATA_MANAGER_CONFIG_FILENAME ]
def add_tool_versions( trans, id, repository_metadata, changeset_revisions ):
# Build a dictionary of { 'tool id' : 'parent tool id' } pairs for
each tool in repository_metadata.
@@ -750,7 +751,7 @@
metadata_dict = generate_data_manager_metadata( app,
repository,
files_dir,
- hg_util.get_config_from_disk(
suc.REPOSITORY_DATA_MANAGER_CONFIG_FILENAME, files_dir ),
+ hg_util.get_config_from_disk(
REPOSITORY_DATA_MANAGER_CONFIG_FILENAME, files_dir ),
metadata_dict,
shed_config_dict=shed_config_dict )
@@ -809,10 +810,10 @@
if package_install_version == '1.0':
# Complex repository dependencies can be defined within the last
<actions> tag set contained in an
# <actions_group> tag set. Comments, <repository> tag
sets and <readme> tag sets will be skipped
- # in td_common_util.parse_package_elem().
- actions_elem_tuples = td_common_util.parse_package_elem( sub_elem,
-
platform_info_dict=None,
-
include_after_install_actions=False )
+ # in tool_dependency_util.parse_package_elem().
+ actions_elem_tuples = tool_dependency_util.parse_package_elem(
sub_elem,
+
platform_info_dict=None,
+
include_after_install_actions=False )
if actions_elem_tuples:
# We now have a list of a single tuple that looks something
like:
# [(True, <Element 'actions' at 0x104017850>)]
diff -r 31740dffaf9ae4e345fd2d4de4ab44ed133dd567 -r
b1b1e5cefca5b0193c8623a9aee7a39cd1dda55c lib/tool_shed/util/shed_util_common.py
--- a/lib/tool_shed/util/shed_util_common.py
+++ b/lib/tool_shed/util/shed_util_common.py
@@ -24,11 +24,8 @@
log = logging.getLogger( __name__ )
-CHUNK_SIZE = 2**20 # 1Mb
MAX_CONTENT_SIZE = 1048576
-MAXDIFFSIZE = 8000
DATATYPES_CONFIG_FILENAME = 'datatypes_conf.xml'
-REPOSITORY_DATA_MANAGER_CONFIG_FILENAME = 'data_manager_conf.xml'
new_repo_email_alert_template = """
Sharable link: ${sharable_link}
This diff is so big that we needed to truncate the remainder.
Repository URL:
https://bitbucket.org/galaxy/galaxy-central/
--
This is a commit notification from
bitbucket.org. You are receiving
this because you have the service enabled, addressing the recipient of
this email.