2 new commits in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/886a722b0aef/ Changeset: 886a722b0aef Branch: next-stable User: Dave Bouvier Date: 2013-05-29 19:26:38 Summary: Fix for set_environment appearing in multiple action sections. Fix for interpolating multiple environment variables in set_environment actions. Affected #: 2 files diff -r 8c69611137c19dd59d9e6f7cc339484213a88b18 -r 886a722b0aef6560320c1ddf7382e6207731f49f lib/tool_shed/galaxy_install/tool_dependencies/fabric_util.py --- a/lib/tool_shed/galaxy_install/tool_dependencies/fabric_util.py +++ b/lib/tool_shed/galaxy_install/tool_dependencies/fabric_util.py @@ -22,7 +22,6 @@ log = logging.getLogger( __name__ ) -CMD_SEPARATOR = '__CMD_SEP__' INSTALLATION_LOG = 'INSTALLATION.log' VIRTUALENV_URL = 'https://pypi.python.org/packages/source/v/virtualenv/virtualenv-1.9.1.tar.gz' @@ -51,21 +50,90 @@ return output.return_code def handle_environment_variables( app, tool_dependency, install_dir, env_var_dict, set_prior_environment_commands ): + """ + This method works with with a combination of three tool dependency definition tag sets, which are defined in the tool_dependencies.xml file in the + order discussed here. The example for this discussion is the tool_dependencies.xml file contained in the osra repository, which is available at: + + http://testtoolshed.g2.bx.psu.edu/view/bgruening/osra + + The first tag set defines a complex repository dependency like this. This tag set ensures that changeset revision XXX of the repository named + package_graphicsmagick_1_3 owned by YYY in the tool shed ZZZ has been previously installed. + + <tool_dependency> + <package name="graphicsmagick" version="1.3.18"> + <repository changeset_revision="XXX" name="package_graphicsmagick_1_3" owner="YYY" prior_installation_required="True" toolshed="ZZZ" /> + </package> + ... + + * By the way, there is an env.sh file associated with version 1.3.18 of the graphicsmagick package which looks something like this (we'll reference + this file later in this discussion. + ---- + GRAPHICSMAGICK_ROOT_DIR=/<my configured tool dependency path>/graphicsmagick/1.3.18/YYY/package_graphicsmagick_1_3/XXX/gmagick; + export GRAPHICSMAGICK_ROOT_DIR + ---- + + The second tag set defines a specific package dependency that has been previously installed (guaranteed by the tag set discussed above) and compiled, + where the compiled dependency is needed by the tool dependency currently being installed (osra version 2.0.0 in this case) and complied in order for + it's installation and compilation to succeed. This tag set is contained within the <package name="osra" version="2.0.0"> tag set, which implies that + version 2.0.0 of the osra package requires version 1.3.18 of the graphicsmagick package in order to successfully compile. When this tag set is handled, + one of the effects is that the env.sh file associated with graphicsmagick version 1.3.18 is "sourced", which undoubtedly sets or alters certain environment + variables (e.g. PATH, PYTHONPATH, etc). + + <!-- populate the environment variables from the dependent repositories --> + <action type="set_environment_for_install"> + <repository changeset_revision="XXX" name="package_graphicsmagick_1_3" owner="YYY" toolshed="ZZZ"> + <package name="graphicsmagick" version="1.3.18" /> + </repository> + </action> + + The third tag set enables discovery of the same required package dependency discussed above for correctly compiling the osra version 2.0.0 package, but + in this case the package can be discovered at tool execution time. Using the $ENV[] option as shown in this example, the value of the environment + variable named GRAPHICSMAGICK_ROOT_DIR (which was set in the environment using the second tag set described above) will be used to automatically alter + the env.sh file associated with the osra version 2.0.0 tool dependency when it is installed into Galaxy. * Refer to where we discussed the env.sh file + for version 1.3.18 of the graphicsmagick package above. + + <action type="set_environment"> + <environment_variable action="prepend_to" name="LD_LIBRARY_PATH">$ENV[GRAPHICSMAGICK_ROOT_DIR]/lib/</environment_variable> + <environment_variable action="prepend_to" name="LD_LIBRARY_PATH">$INSTALL_DIR/potrace/build/lib/</environment_variable> + <environment_variable action="prepend_to" name="PATH">$INSTALL_DIR/bin</environment_variable> + <!-- OSRA_DATA_FILES is only used by the galaxy wrapper and is not part of OSRA --> + <environment_variable action="set_to" name="OSRA_DATA_FILES">$INSTALL_DIR/share</environment_variable> + </action> + + The above tag will produce an env.sh file for version 2.0.0 of the osra package when it it installed into Galaxy that looks something like this. Notice + that the path to the gmagick binary is included here since it expands the defined $ENV[GRAPHICSMAGICK_ROOT_DIR] value in the above tag set. + + ---- + LD_LIBRARY_PATH=/<my configured tool dependency path>/graphicsmagick/1.3.18/YYY/package_graphicsmagick_1_3/XXX/gmagick/lib/:$LD_LIBRARY_PATH; + export LD_LIBRARY_PATH + LD_LIBRARY_PATH=/<my configured tool dependency path>/osra/1.4.0/YYY/depends_on/XXX/potrace/build/lib/:$LD_LIBRARY_PATH; + export LD_LIBRARY_PATH + PATH=/<my configured tool dependency path>/osra/1.4.0/YYY/depends_on/XXX/bin:$PATH; + export PATH + OSRA_DATA_FILES=/<my configured tool dependency path>/osra/1.4.0/YYY/depends_on/XXX/share; + export OSRA_DATA_FILES + ---- + """ env_var_value = env_var_dict[ 'value' ] + # env_var_value is the text of an environment variable tag like this: <environment_variable action="prepend_to" name="LD_LIBRARY_PATH"> + # Here is an example of what env_var_value could look like: $ENV[GRAPHICSMAGICK_ROOT_DIR]/lib/ if '$ENV[' in env_var_value and ']' in env_var_value: # Pull out the name of the environment variable to populate. inherited_env_var_name = env_var_value.split( '[' )[1].split( ']' )[0] to_replace = '$ENV[%s]' % inherited_env_var_name - # Build a command line that outputs CMD_SEPARATOR + environment variable value + CMD_SEPARATOR. - set_prior_environment_commands.extend( [ "echo '%s'" % CMD_SEPARATOR, 'echo $%s' % inherited_env_var_name, "echo '%s'" % CMD_SEPARATOR ] ) + # Build a command line that outputs VARIABLE_NAME: <the value of the variable>. + set_prior_environment_commands.append( 'echo "%s: $%s"' % ( inherited_env_var_name, inherited_env_var_name ) ) command = ' ; '.join( set_prior_environment_commands ) # Run the command and capture the output. command_return = handle_command( app, tool_dependency, install_dir, command, return_output=True ) - # And extract anything between the two instances of CMD_SEPARATOR. - environment_variable_value = command_return.split( CMD_SEPARATOR )[1].split( CMD_SEPARATOR )[0].strip( '\n' ) - if environment_variable_value: - log.info( 'Replacing %s with %s in env.sh for this repository.', to_replace, environment_variable_value ) - env_var_value = env_var_value.replace( to_replace, environment_variable_value ) + # And extract anything labeled with the name of the environment variable we're populating here. + if '%s: ' % inherited_env_var_name in command_return: + environment_variable_value = command_return.split( '\n' ) + for line in environment_variable_value: + if line.startswith( inherited_env_var_name ): + inherited_env_var_value = line.replace( '%s: ' % inherited_env_var_name, '' ) + log.info( 'Replacing %s with %s in env.sh for this repository.', to_replace, inherited_env_var_value ) + env_var_value = env_var_value.replace( to_replace, inherited_env_var_value ) else: # If the return is empty, replace the original $ENV[] with nothing, to avoid any shell misparsings later on. log.error( 'Environment variable %s not found, removing from set_environment.', inherited_env_var_name ) diff -r 8c69611137c19dd59d9e6f7cc339484213a88b18 -r 886a722b0aef6560320c1ddf7382e6207731f49f lib/tool_shed/galaxy_install/tool_dependencies/install_util.py --- a/lib/tool_shed/galaxy_install/tool_dependencies/install_util.py +++ b/lib/tool_shed/galaxy_install/tool_dependencies/install_util.py @@ -339,6 +339,7 @@ actions_dict[ 'package_name' ] = package_name actions = [] all_env_shell_file_paths = [] + env_var_dicts = [] # Make sure to skip all comments, since they are now included in the XML tree. for action_elem in actions_elem.findall( 'action' ): action_dict = {} @@ -389,7 +390,6 @@ # <environment_variable name="PYTHONPATH" action="append_to">$INSTALL_DIR/lib/python</environment_variable> # <environment_variable name="PATH" action="prepend_to">$INSTALL_DIR/bin</environment_variable> # </action> - env_var_dicts = [] for env_elem in action_elem: if env_elem.tag == 'environment_variable': env_var_dict = common_util.create_env_var_dict( env_elem, tool_dependency_install_dir=install_dir ) https://bitbucket.org/galaxy/galaxy-central/commits/35ac562e7e81/ Changeset: 35ac562e7e81 User: Dave Bouvier Date: 2013-05-29 19:27:48 Summary: Merge with next-stable. Affected #: 2 files diff -r 964071e63ee3173e35f245d193e3cb8252fabc7c -r 35ac562e7e81f232b99c2c377d415b6aeab538cb lib/tool_shed/galaxy_install/tool_dependencies/fabric_util.py --- a/lib/tool_shed/galaxy_install/tool_dependencies/fabric_util.py +++ b/lib/tool_shed/galaxy_install/tool_dependencies/fabric_util.py @@ -22,7 +22,6 @@ log = logging.getLogger( __name__ ) -CMD_SEPARATOR = '__CMD_SEP__' INSTALLATION_LOG = 'INSTALLATION.log' VIRTUALENV_URL = 'https://pypi.python.org/packages/source/v/virtualenv/virtualenv-1.9.1.tar.gz' @@ -94,7 +93,7 @@ for version 1.3.18 of the graphicsmagick package above. <action type="set_environment"> - <environment_variable action="prepend_to" name="LD_LIBRARY_PATH">$ENV[$GRAPHICSMAGICK_ROOT_DIR]/lib/</environment_variable> + <environment_variable action="prepend_to" name="LD_LIBRARY_PATH">$ENV[GRAPHICSMAGICK_ROOT_DIR]/lib/</environment_variable><environment_variable action="prepend_to" name="LD_LIBRARY_PATH">$INSTALL_DIR/potrace/build/lib/</environment_variable><environment_variable action="prepend_to" name="PATH">$INSTALL_DIR/bin</environment_variable><!-- OSRA_DATA_FILES is only used by the galaxy wrapper and is not part of OSRA --> @@ -102,7 +101,7 @@ </action> The above tag will produce an env.sh file for version 2.0.0 of the osra package when it it installed into Galaxy that looks something like this. Notice - that the path to the gmagick binary is included here since it expands the defined $ENV[$GRAPHICSMAGICK_ROOT_DIR] value in the above tag set. + that the path to the gmagick binary is included here since it expands the defined $ENV[GRAPHICSMAGICK_ROOT_DIR] value in the above tag set. ---- LD_LIBRARY_PATH=/<my configured tool dependency path>/graphicsmagick/1.3.18/YYY/package_graphicsmagick_1_3/XXX/gmagick/lib/:$LD_LIBRARY_PATH; @@ -116,20 +115,25 @@ ---- """ env_var_value = env_var_dict[ 'value' ] + # env_var_value is the text of an environment variable tag like this: <environment_variable action="prepend_to" name="LD_LIBRARY_PATH"> + # Here is an example of what env_var_value could look like: $ENV[GRAPHICSMAGICK_ROOT_DIR]/lib/ if '$ENV[' in env_var_value and ']' in env_var_value: # Pull out the name of the environment variable to populate. inherited_env_var_name = env_var_value.split( '[' )[1].split( ']' )[0] to_replace = '$ENV[%s]' % inherited_env_var_name - # Build a command line that outputs CMD_SEPARATOR + environment variable value + CMD_SEPARATOR. - set_prior_environment_commands.extend( [ "echo '%s'" % CMD_SEPARATOR, 'echo $%s' % inherited_env_var_name, "echo '%s'" % CMD_SEPARATOR ] ) + # Build a command line that outputs VARIABLE_NAME: <the value of the variable>. + set_prior_environment_commands.append( 'echo "%s: $%s"' % ( inherited_env_var_name, inherited_env_var_name ) ) command = ' ; '.join( set_prior_environment_commands ) # Run the command and capture the output. command_return = handle_command( app, tool_dependency, install_dir, command, return_output=True ) - # And extract anything between the two instances of CMD_SEPARATOR. - environment_variable_value = command_return.split( CMD_SEPARATOR )[1].split( CMD_SEPARATOR )[0].strip( '\n' ) - if environment_variable_value: - log.info( 'Replacing %s with %s in env.sh for this repository.', to_replace, environment_variable_value ) - env_var_value = env_var_value.replace( to_replace, environment_variable_value ) + # And extract anything labeled with the name of the environment variable we're populating here. + if '%s: ' % inherited_env_var_name in command_return: + environment_variable_value = command_return.split( '\n' ) + for line in environment_variable_value: + if line.startswith( inherited_env_var_name ): + inherited_env_var_value = line.replace( '%s: ' % inherited_env_var_name, '' ) + log.info( 'Replacing %s with %s in env.sh for this repository.', to_replace, inherited_env_var_value ) + env_var_value = env_var_value.replace( to_replace, inherited_env_var_value ) else: # If the return is empty, replace the original $ENV[] with nothing, to avoid any shell misparsings later on. log.error( 'Environment variable %s not found, removing from set_environment.', inherited_env_var_name ) diff -r 964071e63ee3173e35f245d193e3cb8252fabc7c -r 35ac562e7e81f232b99c2c377d415b6aeab538cb lib/tool_shed/galaxy_install/tool_dependencies/install_util.py --- a/lib/tool_shed/galaxy_install/tool_dependencies/install_util.py +++ b/lib/tool_shed/galaxy_install/tool_dependencies/install_util.py @@ -339,6 +339,7 @@ actions_dict[ 'package_name' ] = package_name actions = [] all_env_shell_file_paths = [] + env_var_dicts = [] # Make sure to skip all comments, since they are now included in the XML tree. for action_elem in actions_elem.findall( 'action' ): action_dict = {} @@ -389,7 +390,6 @@ # <environment_variable name="PYTHONPATH" action="append_to">$INSTALL_DIR/lib/python</environment_variable> # <environment_variable name="PATH" action="prepend_to">$INSTALL_DIR/bin</environment_variable> # </action> - env_var_dicts = [] for env_elem in action_elem: if env_elem.tag == 'environment_variable': env_var_dict = common_util.create_env_var_dict( env_elem, tool_dependency_install_dir=install_dir ) Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.