3 new commits in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/e90ee102ac65/ Changeset: e90ee102ac65 Branch: next-stable User: jmchilton Date: 2014-08-04 18:18:00 Summary: Restructure tool external file logic in load_tool_from_tmp_config in ToolValidator... ... so internals of tool XML description are only utilized in the galaxy.tools module (and submodules). Add some unit tests for this new method for finding externally referenced files. Affected #: 4 files diff -r 2438770e2cfbf804acc0d5c7b2f3d6e491f30695 -r e90ee102ac654cc88bc3cf156aa1319008412846 lib/galaxy/tools/__init__.py --- a/lib/galaxy/tools/__init__.py +++ b/lib/galaxy/tools/__init__.py @@ -69,7 +69,7 @@ from galaxy.model import Workflow from tool_shed.util import common_util from tool_shed.util import shed_util_common as suc -from .loader import load_tool, template_macro_params +from .loader import load_tool, template_macro_params, raw_tool_xml_tree from .execute import execute as execute_job from .wrappers import ( ToolParameterValueWrapper, @@ -2989,6 +2989,21 @@ def get_default_history_by_trans( self, trans, create=False ): return trans.get_history( create=create ) + @classmethod + def get_externally_referenced_paths( self, path ): + """ Return relative paths to externally referenced files by the tool + described by file at `path`. External components should not assume things + about the structure of tool xml files (this is the tool's responsibility). + """ + tree = raw_tool_xml_tree(path) + root = tree.getroot() + external_paths = [] + for code_elem in root.findall( 'code' ): + external_path = code_elem.get( 'file' ) + if external_path: + external_paths.append( external_path ) + return external_paths + class OutputParameterJSONTool( Tool ): """ diff -r 2438770e2cfbf804acc0d5c7b2f3d6e491f30695 -r e90ee102ac654cc88bc3cf156aa1319008412846 lib/galaxy/tools/loader.py --- a/lib/galaxy/tools/loader.py +++ b/lib/galaxy/tools/loader.py @@ -10,7 +10,7 @@ """ Loads tool from file system and preprocesses tool macros. """ - tree = parse_xml(path) + tree = raw_tool_xml_tree(path) root = tree.getroot() _import_macros(root, path) @@ -38,6 +38,14 @@ return param_dict +def raw_tool_xml_tree(path): + """ Load raw (no macro expansion) tree representation of tool represented + at the specified path. + """ + tree = parse_xml(path) + return tree + + def _import_macros(root, path): tool_dir = os.path.dirname(path) macros_el = root.find('macros') diff -r 2438770e2cfbf804acc0d5c7b2f3d6e491f30695 -r e90ee102ac654cc88bc3cf156aa1319008412846 lib/tool_shed/tools/tool_validator.py --- a/lib/tool_shed/tools/tool_validator.py +++ b/lib/tool_shed/tools/tool_validator.py @@ -3,6 +3,7 @@ import os import tempfile +from galaxy.tools import Tool from galaxy.tools import parameters from galaxy.tools.parameters import dynamic_options @@ -311,15 +312,14 @@ message = '' tmp_tool_config = hg_util.get_named_tmpfile_from_ctx( ctx, ctx_file, work_dir ) if tmp_tool_config: - element_tree, error_message = xml_util.parse_xml( tmp_tool_config ) - if element_tree is None: + tool_element, error_message = xml_util.parse_xml( tmp_tool_config ) + if tool_element is None: return tool, message - element_tree_root = element_tree.getroot() - # Look for code files required by the tool config. + # Look for external files required by the tool config. tmp_code_files = [] - for code_elem in element_tree_root.findall( 'code' ): - code_file_name = code_elem.get( 'file' ) - tmp_code_file_name = hg_util.copy_file_from_manifest( repo, ctx, code_file_name, work_dir ) + external_paths = Tool.get_externally_referenced_paths( tmp_tool_config ) + for path in external_paths: + tmp_code_file_name = hg_util.copy_file_from_manifest( repo, ctx, path, work_dir ) if tmp_code_file_name: tmp_code_files.append( tmp_code_file_name ) tool, valid, message = self.load_tool_from_config( repository_id, tmp_tool_config ) diff -r 2438770e2cfbf804acc0d5c7b2f3d6e491f30695 -r e90ee102ac654cc88bc3cf156aa1319008412846 test/unit/tools/test_tool_external_files.py --- /dev/null +++ b/test/unit/tools/test_tool_external_files.py @@ -0,0 +1,26 @@ +""" Unit test logic related to finding externally referenced files in tool +descriptions. +""" +import tempfile +import os +import shutil +from galaxy.tools import Tool + + +def test_finds_external_code_file(): + assert __external_files("""<tool><code file="foo.py" /></tool>""") == ["foo.py"] + + +def test_finds_skips_empty_code_file_attribute(): + assert __external_files("""<tool><code /></tool>""") == [] + + +def __external_files(contents): + base_path = tempfile.mkdtemp() + try: + tool_path = os.path.join(base_path, "tool.xml") + with open(tool_path, "w") as f: + f.write(contents) + return Tool.get_externally_referenced_paths(tool_path) + finally: + shutil.rmtree(base_path) https://bitbucket.org/galaxy/galaxy-central/commits/d026577e04ac/ Changeset: d026577e04ac Branch: next-stable User: jmchilton Date: 2014-08-04 18:18:00 Summary: Fix new Tool.get_externally_referenced_paths to find imported macro files. Should fix this bug https://trello.com/c/TeguDYSq discovered by Bjoern... on stage at the GCC. Affected #: 3 files diff -r e90ee102ac654cc88bc3cf156aa1319008412846 -r d026577e04ac9802249aeecb64d14292bf59e20d lib/galaxy/tools/__init__.py --- a/lib/galaxy/tools/__init__.py +++ b/lib/galaxy/tools/__init__.py @@ -69,7 +69,7 @@ from galaxy.model import Workflow from tool_shed.util import common_util from tool_shed.util import shed_util_common as suc -from .loader import load_tool, template_macro_params, raw_tool_xml_tree +from .loader import load_tool, template_macro_params, raw_tool_xml_tree, imported_macro_paths from .execute import execute as execute_job from .wrappers import ( ToolParameterValueWrapper, @@ -3002,6 +3002,8 @@ external_path = code_elem.get( 'file' ) if external_path: external_paths.append( external_path ) + external_paths.extend( imported_macro_paths( root ) ) + # May also need to load external citation files as well at some point. return external_paths diff -r e90ee102ac654cc88bc3cf156aa1319008412846 -r d026577e04ac9802249aeecb64d14292bf59e20d lib/galaxy/tools/loader.py --- a/lib/galaxy/tools/loader.py +++ b/lib/galaxy/tools/loader.py @@ -46,14 +46,23 @@ return tree +def imported_macro_paths(root): + macros_el = _macros_el(root) + return _imported_macro_paths_from_el(macros_el) + + def _import_macros(root, path): tool_dir = os.path.dirname(path) - macros_el = root.find('macros') + macros_el = _macros_el(root) if macros_el: macro_els = _load_macros(macros_el, tool_dir) _xml_set_children(macros_el, macro_els) +def _macros_el(root): + return root.find('macros') + + def _macros_of_type(root, type, el_func): macros_el = root.find('macros') macro_dict = {} @@ -167,6 +176,17 @@ def _load_imported_macros(macros_el, tool_dir): macros = [] + for tool_relative_import_path in _imported_macro_paths_from_el(macros_el): + import_path = \ + os.path.join(tool_dir, tool_relative_import_path) + file_macros = _load_macro_file(import_path, tool_dir) + macros.extend(file_macros) + + return macros + + +def _imported_macro_paths_from_el(macros_el): + imported_macro_paths = [] macro_import_els = [] if macros_el: macro_import_els = macros_el.findall("import") @@ -174,12 +194,8 @@ raw_import_path = macro_import_el.text tool_relative_import_path = \ os.path.basename(raw_import_path) # Sanitize this - import_path = \ - os.path.join(tool_dir, tool_relative_import_path) - file_macros = _load_macro_file(import_path, tool_dir) - macros.extend(file_macros) - - return macros + imported_macro_paths.append( tool_relative_import_path ) + return imported_macro_paths def _load_macro_file(path, tool_dir): diff -r e90ee102ac654cc88bc3cf156aa1319008412846 -r d026577e04ac9802249aeecb64d14292bf59e20d test/unit/tools/test_tool_external_files.py --- a/test/unit/tools/test_tool_external_files.py +++ b/test/unit/tools/test_tool_external_files.py @@ -15,6 +15,10 @@ assert __external_files("""<tool><code /></tool>""") == [] +def test_finds_external_macro_file(): + assert __external_files("""<tool><macros><import>cool_macros.xml</import></macros></tool>""") == ["cool_macros.xml"] + + def __external_files(contents): base_path = tempfile.mkdtemp() try: https://bitbucket.org/galaxy/galaxy-central/commits/7cacfb33db90/ Changeset: 7cacfb33db90 Branch: next-stable User: jmchilton Date: 2014-08-04 20:57:31 Summary: Merged in jmchilton/galaxy-central-fork-1/next-stable (pull request #449) Fix tool shed's logic related to finding tool externally referenced files. Affected #: 4 files diff -r 7ac5a36c0ce7a6eb280d339de2107740552825ce -r 7cacfb33db90dee8c4e9ce78a3a735cd6cef8490 lib/galaxy/tools/__init__.py --- a/lib/galaxy/tools/__init__.py +++ b/lib/galaxy/tools/__init__.py @@ -69,7 +69,7 @@ from galaxy.model import Workflow from tool_shed.util import common_util from tool_shed.util import shed_util_common as suc -from .loader import load_tool, template_macro_params +from .loader import load_tool, template_macro_params, raw_tool_xml_tree, imported_macro_paths from .execute import execute as execute_job from .wrappers import ( ToolParameterValueWrapper, @@ -2989,6 +2989,23 @@ def get_default_history_by_trans( self, trans, create=False ): return trans.get_history( create=create ) + @classmethod + def get_externally_referenced_paths( self, path ): + """ Return relative paths to externally referenced files by the tool + described by file at `path`. External components should not assume things + about the structure of tool xml files (this is the tool's responsibility). + """ + tree = raw_tool_xml_tree(path) + root = tree.getroot() + external_paths = [] + for code_elem in root.findall( 'code' ): + external_path = code_elem.get( 'file' ) + if external_path: + external_paths.append( external_path ) + external_paths.extend( imported_macro_paths( root ) ) + # May also need to load external citation files as well at some point. + return external_paths + class OutputParameterJSONTool( Tool ): """ diff -r 7ac5a36c0ce7a6eb280d339de2107740552825ce -r 7cacfb33db90dee8c4e9ce78a3a735cd6cef8490 lib/galaxy/tools/loader.py --- a/lib/galaxy/tools/loader.py +++ b/lib/galaxy/tools/loader.py @@ -10,7 +10,7 @@ """ Loads tool from file system and preprocesses tool macros. """ - tree = parse_xml(path) + tree = raw_tool_xml_tree(path) root = tree.getroot() _import_macros(root, path) @@ -38,14 +38,31 @@ return param_dict +def raw_tool_xml_tree(path): + """ Load raw (no macro expansion) tree representation of tool represented + at the specified path. + """ + tree = parse_xml(path) + return tree + + +def imported_macro_paths(root): + macros_el = _macros_el(root) + return _imported_macro_paths_from_el(macros_el) + + def _import_macros(root, path): tool_dir = os.path.dirname(path) - macros_el = root.find('macros') + macros_el = _macros_el(root) if macros_el: macro_els = _load_macros(macros_el, tool_dir) _xml_set_children(macros_el, macro_els) +def _macros_el(root): + return root.find('macros') + + def _macros_of_type(root, type, el_func): macros_el = root.find('macros') macro_dict = {} @@ -159,6 +176,17 @@ def _load_imported_macros(macros_el, tool_dir): macros = [] + for tool_relative_import_path in _imported_macro_paths_from_el(macros_el): + import_path = \ + os.path.join(tool_dir, tool_relative_import_path) + file_macros = _load_macro_file(import_path, tool_dir) + macros.extend(file_macros) + + return macros + + +def _imported_macro_paths_from_el(macros_el): + imported_macro_paths = [] macro_import_els = [] if macros_el: macro_import_els = macros_el.findall("import") @@ -166,12 +194,8 @@ raw_import_path = macro_import_el.text tool_relative_import_path = \ os.path.basename(raw_import_path) # Sanitize this - import_path = \ - os.path.join(tool_dir, tool_relative_import_path) - file_macros = _load_macro_file(import_path, tool_dir) - macros.extend(file_macros) - - return macros + imported_macro_paths.append( tool_relative_import_path ) + return imported_macro_paths def _load_macro_file(path, tool_dir): diff -r 7ac5a36c0ce7a6eb280d339de2107740552825ce -r 7cacfb33db90dee8c4e9ce78a3a735cd6cef8490 lib/tool_shed/tools/tool_validator.py --- a/lib/tool_shed/tools/tool_validator.py +++ b/lib/tool_shed/tools/tool_validator.py @@ -3,6 +3,7 @@ import os import tempfile +from galaxy.tools import Tool from galaxy.tools import parameters from galaxy.tools.parameters import dynamic_options @@ -311,15 +312,14 @@ message = '' tmp_tool_config = hg_util.get_named_tmpfile_from_ctx( ctx, ctx_file, work_dir ) if tmp_tool_config: - element_tree, error_message = xml_util.parse_xml( tmp_tool_config ) - if element_tree is None: + tool_element, error_message = xml_util.parse_xml( tmp_tool_config ) + if tool_element is None: return tool, message - element_tree_root = element_tree.getroot() - # Look for code files required by the tool config. + # Look for external files required by the tool config. tmp_code_files = [] - for code_elem in element_tree_root.findall( 'code' ): - code_file_name = code_elem.get( 'file' ) - tmp_code_file_name = hg_util.copy_file_from_manifest( repo, ctx, code_file_name, work_dir ) + external_paths = Tool.get_externally_referenced_paths( tmp_tool_config ) + for path in external_paths: + tmp_code_file_name = hg_util.copy_file_from_manifest( repo, ctx, path, work_dir ) if tmp_code_file_name: tmp_code_files.append( tmp_code_file_name ) tool, valid, message = self.load_tool_from_config( repository_id, tmp_tool_config ) diff -r 7ac5a36c0ce7a6eb280d339de2107740552825ce -r 7cacfb33db90dee8c4e9ce78a3a735cd6cef8490 test/unit/tools/test_tool_external_files.py --- /dev/null +++ b/test/unit/tools/test_tool_external_files.py @@ -0,0 +1,30 @@ +""" Unit test logic related to finding externally referenced files in tool +descriptions. +""" +import tempfile +import os +import shutil +from galaxy.tools import Tool + + +def test_finds_external_code_file(): + assert __external_files("""<tool><code file="foo.py" /></tool>""") == ["foo.py"] + + +def test_finds_skips_empty_code_file_attribute(): + assert __external_files("""<tool><code /></tool>""") == [] + + +def test_finds_external_macro_file(): + assert __external_files("""<tool><macros><import>cool_macros.xml</import></macros></tool>""") == ["cool_macros.xml"] + + +def __external_files(contents): + base_path = tempfile.mkdtemp() + try: + tool_path = os.path.join(base_path, "tool.xml") + with open(tool_path, "w") as f: + f.write(contents) + return Tool.get_externally_referenced_paths(tool_path) + finally: + shutil.rmtree(base_path) Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.