1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/1a915a754396/ Changeset: 1a915a754396 User: greg Date: 2013-04-24 20:46:54 Summary: Add support for a new <action type="set_environmnet_for_install"> tag in tool_dependencies.xml files included in tool shed repositories. This tag currently can include any number of <repository> tags, each of which will contain any number of tool dependency tags (i.e., <package> or <set_environmnet> tags). The settings in the env.sh file for each of the tool dependency tags will be injected into the environment for all following shell commands defined in the tool_dependencies.xml file, ensuring that the required tool dependencies are sued when compiling the current dependency. I'm sure this will make no sense to anyone reading this commit message. Affected #: 7 files diff -r 333dfc42a8688ec81627ae1164e3be9ba2aa864c -r 1a915a754396da8ac986dd410ee99bed12668235 lib/galaxy/model/__init__.py --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -3443,7 +3443,7 @@ """Return the repository's tool dependencies that are currently installed.""" installed_dependencies = [] for tool_dependency in self.tool_dependencies: - if tool_dependency.status == ToolDependency.installation_status.INSTALLED: + if tool_dependency.status in [ ToolDependency.installation_status.INSTALLED, ToolDependency.installation_status.ERROR ]: installed_dependencies.append( tool_dependency ) return installed_dependencies @property diff -r 333dfc42a8688ec81627ae1164e3be9ba2aa864c -r 1a915a754396da8ac986dd410ee99bed12668235 lib/tool_shed/galaxy_install/repository_util.py --- a/lib/tool_shed/galaxy_install/repository_util.py +++ b/lib/tool_shed/galaxy_install/repository_util.py @@ -478,7 +478,7 @@ tool_shed_repository, trans.model.ToolShedRepository.installation_status.INSTALLING_TOOL_DEPENDENCIES ) # Get the tool_dependencies.xml file from the repository. - tool_dependencies_config = suc.get_config_from_disk( 'tool_dependencies.xml', install_dir )#relative_install_dir ) + tool_dependencies_config = suc.get_config_from_disk( 'tool_dependencies.xml', install_dir ) installed_tool_dependencies = common_install_util.handle_tool_dependencies( app=trans.app, tool_shed_repository=tool_shed_repository, tool_dependencies_config=tool_dependencies_config, diff -r 333dfc42a8688ec81627ae1164e3be9ba2aa864c -r 1a915a754396da8ac986dd410ee99bed12668235 lib/tool_shed/galaxy_install/tool_dependencies/common_util.py --- a/lib/tool_shed/galaxy_install/tool_dependencies/common_util.py +++ b/lib/tool_shed/galaxy_install/tool_dependencies/common_util.py @@ -1,6 +1,14 @@ -import os, shutil, tarfile, urllib2, zipfile +import logging +import os +import shutil +import tarfile +import urllib2 +import zipfile +import tool_shed.util.shed_util_common as suc from galaxy.datatypes import checkers +log = logging.getLogger( __name__ ) + def create_env_var_dict( elem, tool_dependency_install_dir=None, tool_shed_repository_install_dir=None ): env_var_name = elem.get( 'name', 'PATH' ) env_var_action = elem.get( 'action', 'prepend_to' ) @@ -76,6 +84,67 @@ zip_archive.close() return True +def get_env_shell_file_path( installation_directory ): + env_shell_file_name = 'env.sh' + default_location = os.path.abspath( os.path.join( installation_directory, env_shell_file_name ) ) + if os.path.exists( default_location ): + return default_location + for root, dirs, files in os.walk( installation_directory ): + for name in files: + if name == env_shell_file_name: + return os.path.abspath( os.path.join( root, name ) ) + return None + +def get_env_shell_file_paths( app, elem ): + # Currently only the following tag set is supported. + # <repository toolshed="http://localhost:9009/" name="package_numpy_1_7" owner="test" changeset_revision="c84c6a8be056"> + # <package name="numpy" version="1.7.1" /> + # </repository> + env_shell_file_paths = [] + toolshed = elem.get( 'toolshed', None ) + repository_name = elem.get( 'name', None ) + repository_owner = elem.get( 'owner', None ) + changeset_revision = elem.get( 'changeset_revision', None ) + if toolshed and repository_name and repository_owner and changeset_revision: + repository = suc.get_repository_for_dependency_relationship( app, toolshed, repository_name, repository_owner, changeset_revision ) + if repository: + for sub_elem in elem: + tool_dependency_type = sub_elem.tag + tool_dependency_name = sub_elem.get( 'name' ) + tool_dependency_version = sub_elem.get( 'version' ) + if tool_dependency_type and tool_dependency_name and tool_dependency_version: + # Get the tool_dependency so we can get it's installation directory. + tool_dependency = None + for tool_dependency in repository.tool_dependencies: + if tool_dependency.type == tool_dependency_type and tool_dependency.name == tool_dependency_name and tool_dependency.version == tool_dependency_version: + break + if tool_dependency: + tool_dependency_key = '%s/%s' % ( tool_dependency_name, tool_dependency_version ) + installation_directory = tool_dependency.installation_directory( app ) + env_shell_file_path = get_env_shell_file_path( installation_directory ) + env_shell_file_paths.append( env_shell_file_path ) + else: + error_message = "Skipping tool dependency definition because unable to locate tool dependency " + error_message += "type %s, name %s, version %s for repository %s" % \ + ( str( tool_dependency_type ), str( tool_dependency_name ), str( tool_dependency_version ), str( repository.name ) ) + log.debug( error_message ) + continue + else: + error_message = "Skipping invalid tool dependency definition: type %s, name %s, version %s." % \ + ( str( tool_dependency_type ), str( tool_dependency_name ), str( tool_dependency_version ) ) + log.debug( error_message ) + continue + else: + error_message = "Skipping set_environment_for_install definition because unable to locate required installed tool shed repository: " + error_message += "toolshed %s, name %s, owner %s, changeset_revision %s." % \ + ( str( toolshed ), str( repository_name ), str( repository_owner ), str( changeset_revision ) ) + log.debug( error_message ) + else: + error_message = "Skipping invalid set_environment_for_install definition: toolshed %s, name %s, owner %s, changeset_revision %s." % \ + ( str( toolshed ), str( repository_name ), str( repository_owner ), str( changeset_revision ) ) + log.debug( error_message ) + return env_shell_file_paths + def isbz2( file_path ): return checkers.is_bz2( file_path ) diff -r 333dfc42a8688ec81627ae1164e3be9ba2aa864c -r 1a915a754396da8ac986dd410ee99bed12668235 lib/tool_shed/galaxy_install/tool_dependencies/fabric_util.py --- a/lib/tool_shed/galaxy_install/tool_dependencies/fabric_util.py +++ b/lib/tool_shed/galaxy_install/tool_dependencies/fabric_util.py @@ -28,30 +28,18 @@ if int( version.split( "." )[ 0 ] ) < 1: raise NotImplementedError( "Install Fabric version 1.0 or later." ) -def set_galaxy_environment( galaxy_user, tool_dependency_dir, host='localhost', shell='/bin/bash -l -c' ): - """General Galaxy environment configuration""" - env.user = galaxy_user - env.install_dir = tool_dependency_dir - env.host_string = host - env.shell = shell - env.use_sudo = False - env.safe_cmd = local - return env - -@contextmanager -def make_tmp_dir(): - work_dir = tempfile.mkdtemp() - yield work_dir - if os.path.exists( work_dir ): - local( 'rm -rf %s' % work_dir ) - def handle_command( app, tool_dependency, install_dir, cmd ): sa_session = app.model.context.current output = local( cmd, capture=True ) log_results( cmd, output, os.path.join( install_dir, INSTALLATION_LOG ) ) if output.return_code: tool_dependency.status = app.model.ToolDependency.installation_status.ERROR - tool_dependency.error_message = str( output.stderr ) + if output.stderr: + tool_dependency.error_message = str( output.stderr )[ :32768 ] + elif output.stdout: + tool_dependency.error_message = str( output.stdout )[ :32768 ] + else: + tool_dependency.error_message = "Unknown error occurred executing shell command %s, return_code: %s" % ( str( cmd ), str( output.return_code ) ) sa_session.add( tool_dependency ) sa_session.flush() return output.return_code @@ -63,6 +51,7 @@ package_name = actions_dict[ 'package_name' ] actions = actions_dict.get( 'actions', None ) filtered_actions = [] + env_shell_file_paths = [] if actions: with make_tmp_dir() as work_dir: with lcd( work_dir ): @@ -133,9 +122,19 @@ return_code = handle_command( app, tool_dependency, install_dir, cmd ) if return_code: return + elif action_type == 'set_environment_for_install': + # Currently the only action supported in this category is a list of paths to one or more tool dependency env.sh files, + # the environment setting in each of which will be injected into the environment for all <action type="shell_command"> + # tags that follow this <action type="set_environment_for_install"> tag set in the tool_dependencies.xml file. + env_shell_file_paths = action_dict[ 'env_shell_file_paths' ] elif action_type == 'shell_command': with settings( warn_only=True ): - return_code = handle_command( app, tool_dependency, install_dir, action_dict[ 'command' ] ) + cmd = '' + for env_shell_file_path in env_shell_file_paths: + for i, env_setting in enumerate( open( env_shell_file_path ) ): + cmd += '%s\n' % env_setting + cmd += action_dict[ 'command' ] + return_code = handle_command( app, tool_dependency, install_dir, cmd ) if return_code: return @@ -157,3 +156,20 @@ logfile.write( str( fabric_AttributeString.stderr ) ) logfile.write( "\n#############################################\n" ) logfile.close() + +@contextmanager +def make_tmp_dir(): + work_dir = tempfile.mkdtemp() + yield work_dir + if os.path.exists( work_dir ): + local( 'rm -rf %s' % work_dir ) + +def set_galaxy_environment( galaxy_user, tool_dependency_dir, host='localhost', shell='/bin/bash -l -c' ): + """General Galaxy environment configuration. This method is not currently used.""" + env.user = galaxy_user + env.install_dir = tool_dependency_dir + env.host_string = host + env.shell = shell + env.use_sudo = False + env.safe_cmd = local + return env diff -r 333dfc42a8688ec81627ae1164e3be9ba2aa864c -r 1a915a754396da8ac986dd410ee99bed12668235 lib/tool_shed/galaxy_install/tool_dependencies/install_util.py --- a/lib/tool_shed/galaxy_install/tool_dependencies/install_util.py +++ b/lib/tool_shed/galaxy_install/tool_dependencies/install_util.py @@ -282,9 +282,6 @@ tool_dependency.status = app.model.ToolDependency.installation_status.INSTALLED sa_session.add( tool_dependency ) sa_session.flush() - - - else: package_install_version = package_elem.get( 'version', '1.0' ) tool_dependency = tool_dependency_util.create_or_update_tool_dependency( app=app, @@ -378,7 +375,28 @@ action_dict[ env_elem.tag ] = env_var_dicts else: continue + elif action_type == 'set_environment_for_install': + # <action type="set_environment_for_install"> + # <repository toolshed="http://localhost:9009/" name="package_numpy_1_7" owner="test" changeset_revision="c84c6a8be056"> + # <package name="numpy" version="1.7.1" /> + # </repository> + # </action> + # This action type allows for defining an environment that will properly compile a tool dependency. Currently, tag set definitions like + # that above are supported, but in the future other approaches to setting environment variables or other environment attributes can be + # supported. The above tag set will result in the installed and compiled numpy version 1.7.1 binary to be used when compiling the current + # tool dependency package. See the package_matplotlib_1_2 repository in the test tool shed for a real-world example. + all_env_shell_file_paths = [] + for env_elem in action_elem: + if env_elem.tag == 'repository': + env_shell_file_paths = common_util.get_env_shell_file_paths( app, env_elem ) + if env_shell_file_paths: + all_env_shell_file_paths.extend( env_shell_file_paths ) + if all_env_shell_file_paths: + action_dict[ 'env_shell_file_paths' ] = all_env_shell_file_paths + else: + continue else: + log.debug( "Skipping unsupported action type '%s'." % str( action_type ) ) continue actions.append( ( action_type, action_dict ) ) if actions: diff -r 333dfc42a8688ec81627ae1164e3be9ba2aa864c -r 1a915a754396da8ac986dd410ee99bed12668235 lib/tool_shed/util/common_install_util.py --- a/lib/tool_shed/util/common_install_util.py +++ b/lib/tool_shed/util/common_install_util.py @@ -307,6 +307,7 @@ will be installed in: ~/<app.config.tool_dependency_dir>/<package_name>/<package_version>/<repo_owner>/<repo_name>/<repo_installed_changeset_revision> """ + sa_session = app.model.context.current installed_tool_dependencies = [] # Parse the tool_dependencies.xml config. try: @@ -327,12 +328,30 @@ if tool_dependency.name==package_name and tool_dependency.version==package_version: break if tool_dependency.can_install: - tool_dependency = install_package( app, elem, tool_shed_repository, tool_dependencies=tool_dependencies ) + try: + tool_dependency = install_package( app, elem, tool_shed_repository, tool_dependencies=tool_dependencies ) + except Exception, e: + error_message = "Error installing tool dependency %s version %s: %s" % ( str( package_name ), str( package_version ), str( e ) ) + log.debug( error_message ) + if tool_dependency: + tool_dependency.status = app.model.ToolDependency.installation_status.ERROR + tool_dependency.error_message = error_message + sa_session.add( tool_dependency ) + sa_session.flush() if tool_dependency and tool_dependency.status in [ app.model.ToolDependency.installation_status.INSTALLED, app.model.ToolDependency.installation_status.ERROR ]: installed_tool_dependencies.append( tool_dependency ) elif elem.tag == 'set_environment': - tool_dependency = set_environment( app, elem, tool_shed_repository ) + try: + tool_dependency = set_environment( app, elem, tool_shed_repository ) + except Exception, e: + error_message = "Error setting environment for tool dependency: %s" % str( e ) + log.debug( error_message ) + if tool_dependency: + tool_dependency.status = app.model.ToolDependency.installation_status.ERROR + tool_dependency.error_message = error_message + sa_session.add( tool_dependency ) + sa_session.flush() if tool_dependency and tool_dependency.status in [ app.model.ToolDependency.installation_status.INSTALLED, app.model.ToolDependency.installation_status.ERROR ]: installed_tool_dependencies.append( tool_dependency ) diff -r 333dfc42a8688ec81627ae1164e3be9ba2aa864c -r 1a915a754396da8ac986dd410ee99bed12668235 lib/tool_shed/util/shed_util_common.py --- a/lib/tool_shed/util/shed_util_common.py +++ b/lib/tool_shed/util/shed_util_common.py @@ -32,7 +32,8 @@ log = logging.getLogger( __name__ ) INITIAL_CHANGELOG_HASH = '000000000000' -MAX_CONTENT_SIZE = 32768 +MAX_CONTENT_SIZE = 1048576 +MAX_DISPLAY_SIZE = 32768 VALID_CHARS = set( string.letters + string.digits + "'\"-=_.()/+*^,:?!#[]%\\$@;{}&<>" ) new_repo_email_alert_template = """ @@ -667,10 +668,17 @@ safe_str = '' for i, line in enumerate( open( file_path ) ): safe_str = '%s%s' % ( safe_str, to_safe_string( line ) ) + # Stop reading after string is larger than MAX_CONTENT_SIZE. if len( safe_str ) > MAX_CONTENT_SIZE: - large_str = '\nFile contents truncated because file size is larger than maximum viewing size of %s\n' % util.nice_size( MAX_CONTENT_SIZE ) - safe_str = '%s%s' % ( safe_str, to_safe_string( large_str ) ) + large_str = \ + to_safe_string( '\nFile contents truncated because file size is larger than maximum viewing size of %s\n' % util.nice_size( MAX_CONTENT_SIZE ) ) + safe_str = '%s%s' % ( safe_str, large_str ) break + if len( safe_str ) > MAX_DISPLAY_SIZE: + # Eliminate the middle of the file to display a file no larger than MAX_DISPLAY_SIZE. This may not be ideal if the file is larger than MAX_CONTENT_SIZE. + join_by_str = \ + to_safe_string( "\n\n...some text eliminated here because file size is larger than maximum viewing size of %s...\n\n" % util.nice_size( MAX_DISPLAY_SIZE ) ) + safe_str = util.shrink_string_by_size( safe_str, MAX_DISPLAY_SIZE, join_by=join_by_str, left_larger=True, beginning_on_size_error=True ) return safe_str def get_repository_files( trans, folder_path ): @@ -787,9 +795,8 @@ This method assumes all repository tools are defined in a single shed-related tool panel config. """ tool_shed = clean_tool_shed_url( repository.tool_shed ) - partial_install_dir = '%s/repos/%s/%s/%s' % ( tool_shed, repository.owner, repository.name, repository.installed_changeset_revision ) + relative_install_dir = '%s/repos/%s/%s/%s' % ( tool_shed, repository.owner, repository.name, repository.installed_changeset_revision ) # Get the relative tool installation paths from each of the shed tool configs. - relative_install_dir = None shed_config_dict = repository.get_shed_config_dict( app ) if not shed_config_dict: # Just pick a semi-random shed config. @@ -799,7 +806,6 @@ break shed_tool_conf = shed_config_dict[ 'config_filename' ] tool_path = shed_config_dict[ 'tool_path' ] - relative_install_dir = partial_install_dir return shed_tool_conf, tool_path, relative_install_dir def get_tool_path_by_shed_tool_conf_filename( trans, shed_tool_conf ): @@ -1188,13 +1194,13 @@ return toolshed_base_url.rstrip( '/' ) == str( url_for( '/', qualified=True ) ).rstrip( '/' ) def translate_string( raw_text, to_html=True ): - """Return a subset of a string (up to MAX_CONTENT_SIZE) translated to a safe string for display in a browser.""" + """Return a subset of a string (up to MAX_DISPLAY_SIZE) translated to a safe string for display in a browser.""" if raw_text: - if len( raw_text ) <= MAX_CONTENT_SIZE: + if len( raw_text ) <= MAX_DISPLAY_SIZE: translated_string = to_safe_string( raw_text, to_html=to_html ) else: - large_str = '\nFile contents truncated because file size is larger than maximum viewing size of %s\n' % util.nice_size( MAX_CONTENT_SIZE ) - translated_string = to_safe_string( '%s%s' % ( raw_text[ 0:MAX_CONTENT_SIZE ], large_str ), to_html=to_html ) + large_str = '\nFile contents truncated because file size is larger than maximum viewing size of %s\n' % util.nice_size( MAX_DISPLAY_SIZE ) + translated_string = to_safe_string( '%s%s' % ( raw_text[ 0:MAX_DISPLAY_SIZE ], large_str ), to_html=to_html ) else: translated_string = '' return translated_string Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.