9 new commits in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/08f8850853d0/ Changeset: 08f8850853d0 User: jmchilton Date: 2014-12-31 23:21:10+00:00 Summary: Allow multiple tools with the same id in ToolBox. How to use: 1.) Place multiple tools with different IDs in your tool conf. 2.) ... ummm ... no step 2 - just use the tools. Implementation: The Tool Shed allows tool lineages by assigning each tool version a GUID and tracking versions in a database. This implementation works by simply allowing the ToolBox to contain multiple tools with the same ID and orders them by the version specified by the tool author. To track enable this a second tool lineage has been introduced that just uses tool versions instead of a database (non-toolshed installed tools are not longer placed into the Tool Shed install database). The ToolBox has been updated to allow multiple versions per tool id (defaulting to the 'latest' version for all operations which do not specify a version). Both jobs and workflow steps would track tool versions but did not use that version when fetching tools from the Toolbox - these components have been updated to try to use the tool version. Unit tests working through most of the ToolBox and tool panel have been added, as well as functional tests exercising the tools API and to ensure workflows now at least attempt to respect tool versions (still kind of silently switches versions in some cases). Manual tests against the new tool form seem to demonstrate the tool switching and tool re-running work with only minor changes to the tools API and the job handler. Affected #: 17 files diff -r 02efb1123d469986a051e7a8015c9d56635efd8a -r 08f8850853d004bf8a456a147436a6694d0a1a11 lib/galaxy/jobs/__init__.py --- a/lib/galaxy/jobs/__init__.py +++ b/lib/galaxy/jobs/__init__.py @@ -711,7 +711,7 @@ self.job_id = job.id self.session_id = job.session_id self.user_id = job.user_id - self.tool = queue.app.toolbox.get_tool( job.tool_id, exact=True ) + self.tool = queue.app.toolbox.get_tool( job.tool_id, job.tool_version, exact=True ) self.queue = queue self.app = queue.app self.sa_session = self.app.model.context diff -r 02efb1123d469986a051e7a8015c9d56635efd8a -r 08f8850853d004bf8a456a147436a6694d0a1a11 lib/galaxy/jobs/handler.py --- a/lib/galaxy/jobs/handler.py +++ b/lib/galaxy/jobs/handler.py @@ -119,7 +119,7 @@ & ( model.Job.handler == self.app.config.server_name ) ).all() for job in jobs_at_startup: - if not self.app.toolbox.has_tool( job.tool_id, exact=True ): + if not self.app.toolbox.has_tool( job.tool_id, job.tool_version, exact=True ): log.warning( "(%s) Tool '%s' removed from tool config, unable to recover job" % ( job.id, job.tool_id ) ) self.job_wrapper( job ).fail( 'This tool was disabled before the job completed. Please contact your Galaxy administrator.' ) elif job.job_runner_name is not None and job.job_runner_external_id is None: diff -r 02efb1123d469986a051e7a8015c9d56635efd8a -r 08f8850853d004bf8a456a147436a6694d0a1a11 lib/galaxy/tools/__init__.py --- a/lib/galaxy/tools/__init__.py +++ b/lib/galaxy/tools/__init__.py @@ -126,6 +126,10 @@ # shed_tool_conf.xml file. self._dynamic_tool_confs = [] self._tools_by_id = {} + # Tool lineages can contain chains of related tools with different ids + # so each will be present once in the above dictionary. The following + # dictionary can instead hold multiple tools with different versions. + self._tool_versions_by_id = {} self._workflows_by_id = {} # In-memory dictionary that defines the layout of the tool panel. self._tool_panel = odict() @@ -352,7 +356,8 @@ inserted = True if not inserted: # Check the tool's installed versions. - for lineage_id in tool.lineage_ids: + for tool_lineage_version in tool.lineage.get_versions(): + lineage_id = tool_lineage_version.id lineage_id_key = 'tool_%s' % lineage_id for index, integrated_panel_key in enumerate( self._integrated_tool_panel.keys() ): if lineage_id_key == integrated_panel_key: @@ -503,20 +508,26 @@ def get_tool( self, tool_id, tool_version=None, get_all_versions=False, exact=False ): """Attempt to locate a tool in the tool box.""" + if tool_version: + tool_version = str( tool_version ) + if get_all_versions and exact: raise AssertionError("Cannot specify get_tool with both get_all_versions and exact as True") if tool_id in self._tools_by_id and not get_all_versions: + if tool_version and tool_version in self._tool_versions_by_id[ tool_id ]: + return self._tool_versions_by_id[ tool_id ][ tool_version ] #tool_id exactly matches an available tool by id (which is 'old' tool_id or guid) return self._tools_by_id[ tool_id ] #exact tool id match not found, or all versions requested, search for other options, e.g. migrated tools or different versions rval = [] tool_lineage = self._lineage_map.get( tool_id ) if tool_lineage: - tool_version_ids = tool_lineage.get_version_ids( ) - for tool_version_id in tool_version_ids: - if tool_version_id in self._tools_by_id: - rval.append( self._tools_by_id[ tool_version_id ] ) + lineage_tool_versions = tool_lineage.get_versions( ) + for lineage_tool_version in lineage_tool_versions: + lineage_tool = self._tool_from_lineage_version( lineage_tool_version ) + if lineage_tool: + rval.append( lineage_tool ) if not rval: #still no tool, do a deeper search and try to match by old ids for tool in self._tools_by_id.itervalues(): @@ -561,11 +572,12 @@ """Get all loaded tools associated by lineage to the tool whose id is tool_id.""" tool_lineage = self._lineage_map.get( tool_id ) if tool_lineage: - tool_version_ids = tool_lineage.get_version_ids( ) + lineage_tool_versions = tool_lineage.get_versions( ) available_tool_versions = [] - for tool_version_id in tool_version_ids: - if tool_version_id in self._tools_by_id: - available_tool_versions.append( self._tools_by_id[ tool_version_id ] ) + for lineage_tool_version in lineage_tool_versions: + tool = self._tool_from_lineage_version( lineage_tool_version ) + if tool: + available_tool_versions.append( tool ) return available_tool_versions else: if tool_id in self._tools_by_id: @@ -746,7 +758,6 @@ tool_lineage = self._lineage_map.register( tool, tool_shed_repository=tool_shed_repository ) # Load the tool's lineage ids. tool.lineage = tool_lineage - tool.lineage_ids = tool_lineage.get_version_ids( ) self._tool_tag_manager.handle_tags( tool.id, elem ) self.__add_tool( tool, load_panel_dict, panel_dict ) # Always load the tool into the integrated_panel_dict, or it will not be included in the integrated_tool_panel.xml file. @@ -925,7 +936,20 @@ return tool def register_tool( self, tool ): - self._tools_by_id[ tool.id ] = tool + tool_id = tool.id + version = tool.version or None + if tool_id not in self._tool_versions_by_id: + self._tool_versions_by_id[ tool_id ] = { version: tool } + else: + self._tool_versions_by_id[ tool_id ][ version ] = tool + if tool_id in self._tools_by_id: + related_tool = self._tools_by_id[ tool_id ] + # This one becomes the default un-versioned tool + # if newer. + if self._newer_tool( tool, related_tool ): + self._tools_by_id[ tool_id ] = tool + else: + self._tools_by_id[ tool_id ] = tool def package_tool( self, trans, tool_id ): """ @@ -1202,9 +1226,11 @@ if not hasattr( tool, "lineage" ): return None tool_lineage = tool.lineage - lineage_ids = tool_lineage.get_version_ids( reverse=True ) - for lineage_id in lineage_ids: - if lineage_id in self._tools_by_id: + lineage_tool_versions = tool_lineage.get_versions( reverse=True ) + for lineage_tool_version in lineage_tool_versions: + lineage_tool = self._tool_from_lineage_version( lineage_tool_version ) + if lineage_tool: + lineage_id = lineage_tool.id loaded_version_key = 'tool_%s' % lineage_id if loaded_version_key in panel_dict: return panel_dict[ loaded_version_key ] @@ -1214,7 +1240,22 @@ """ Return True if tool1 is considered "newer" given its own lineage description. """ - return tool1.lineage_ids.index( tool1.id ) > tool1.lineage_ids.index( tool2.id ) + if not hasattr( tool1, "lineage" ): + return True + lineage_tool_versions = tool1.lineage.get_versions() + for lineage_tool_version in lineage_tool_versions: + lineage_tool = self._tool_from_lineage_version( lineage_tool_version ) + if lineage_tool is tool1: + return False + if lineage_tool is tool2: + return True + return True + + def _tool_from_lineage_version( self, lineage_tool_version ): + if lineage_tool_version.id_based: + return self._tools_by_id.get( lineage_tool_version.id, None ) + else: + return self._tool_versions_by_id.get( lineage_tool_version.id, {} ).get( lineage_tool_version.version, None ) def _filter_for_panel( item, filters, context ): diff -r 02efb1123d469986a051e7a8015c9d56635efd8a -r 08f8850853d004bf8a456a147436a6694d0a1a11 lib/galaxy/tools/toolbox/lineages/factory.py --- a/lib/galaxy/tools/toolbox/lineages/factory.py +++ b/lib/galaxy/tools/toolbox/lineages/factory.py @@ -1,4 +1,5 @@ from .tool_shed import ToolShedLineage +from .stock import StockLineage class LineageMap(object): @@ -13,7 +14,10 @@ tool_id = tool.id if tool_id not in self.lineage_map: tool_shed_repository = kwds.get("tool_shed_repository", None) - lineage = ToolShedLineage.from_tool(self.app, tool, tool_shed_repository) + if tool_shed_repository: + lineage = ToolShedLineage.from_tool(self.app, tool, tool_shed_repository) + else: + lineage = StockLineage.from_tool( tool ) self.lineage_map[tool_id] = lineage return self.lineage_map[tool_id] diff -r 02efb1123d469986a051e7a8015c9d56635efd8a -r 08f8850853d004bf8a456a147436a6694d0a1a11 lib/galaxy/tools/toolbox/lineages/interface.py --- a/lib/galaxy/tools/toolbox/lineages/interface.py +++ b/lib/galaxy/tools/toolbox/lineages/interface.py @@ -8,7 +8,35 @@ __metaclass__ = ABCMeta @abstractmethod - def get_version_ids( self, reverse=False ): - """ Return an ordered list of lineages in this chain, from - oldest to newest. + def get_versions( self, reverse=False ): + """ Return an ordered list of lineages (ToolLineageVersion) in this + chain, from oldest to newest. """ + + +class ToolLineageVersion(object): + """ Represents a single tool in a lineage. If lineage is based + around GUIDs that somehow encode the version (either using GUID + or a simple tool id and a version). """ + + def __init__(self, id, version): + self.id = id + self.version = version + + @staticmethod + def from_id_and_verion( id, version ): + assert version is not None + return ToolLineageVersion( id, version ) + + @staticmethod + def from_guid( guid ): + return ToolLineageVersion( guid, None ) + + @property + def id_based( self ): + """ Return True if the lineage is defined by GUIDs (in this + case the indexer of the tools (i.e. the ToolBox) should ignore + the tool_version (because it is encoded in the GUID and managed + externally). + """ + return self.version is None diff -r 02efb1123d469986a051e7a8015c9d56635efd8a -r 08f8850853d004bf8a456a147436a6694d0a1a11 lib/galaxy/tools/toolbox/lineages/stock.py --- /dev/null +++ b/lib/galaxy/tools/toolbox/lineages/stock.py @@ -0,0 +1,44 @@ +import threading + +from distutils.version import LooseVersion + +from .interface import ToolLineage +from .interface import ToolLineageVersion + + +class StockLineage(ToolLineage): + """ Simple tool's loaded directly from file system with lineage + determined solely by distutil's LooseVersion naming scheme. + """ + lineages_by_id = {} + lock = threading.Lock() + + def __init__(self, tool_id, **kwds): + self.tool_id = tool_id + self.tool_versions = set() + + @staticmethod + def from_tool( tool ): + tool_id = tool.id + lineages_by_id = StockLineage.lineages_by_id + with StockLineage.lock: + if tool_id not in lineages_by_id: + lineages_by_id[ tool_id ] = StockLineage( tool_id ) + lineage = lineages_by_id[ tool_id ] + lineage.register_version( tool.version ) + return lineage + + def register_version( self, tool_version ): + assert tool_version is not None + self.tool_versions.add( tool_version ) + + def get_versions( self, reverse=False ): + versions = [ ToolLineageVersion( self.tool_id, v ) for v in self.tool_versions ] + # Sort using LooseVersion which defines an appropriate __cmp__ + # method for comparing tool versions. + return sorted( versions, key=_to_loose_version ) + + +def _to_loose_version( tool_lineage_version ): + version = str( tool_lineage_version.version ) + return LooseVersion( version ) diff -r 02efb1123d469986a051e7a8015c9d56635efd8a -r 08f8850853d004bf8a456a147436a6694d0a1a11 lib/galaxy/tools/toolbox/lineages/tool_shed.py --- a/lib/galaxy/tools/toolbox/lineages/tool_shed.py +++ b/lib/galaxy/tools/toolbox/lineages/tool_shed.py @@ -1,4 +1,5 @@ from .interface import ToolLineage +from .interface import ToolLineageVersion from galaxy.model.tool_shed_install import ToolVersion @@ -32,6 +33,9 @@ tool_version = self.app.install_model.context.query( ToolVersion ).get( self.tool_version_id ) return tool_version.get_version_ids( self.app, reverse=reverse ) + def get_versions( self, reverse=False ): + return map( ToolLineageVersion.from_guid, self.get_version_ids( reverse=reverse ) ) + def get_install_tool_version( app, tool_id ): return app.install_model.context.query( diff -r 02efb1123d469986a051e7a8015c9d56635efd8a -r 08f8850853d004bf8a456a147436a6694d0a1a11 lib/galaxy/workflow/modules.py --- a/lib/galaxy/workflow/modules.py +++ b/lib/galaxy/workflow/modules.py @@ -467,10 +467,10 @@ type = "tool" - def __init__( self, trans, tool_id ): + def __init__( self, trans, tool_id, tool_version=None ): self.trans = trans self.tool_id = tool_id - self.tool = trans.app.toolbox.get_tool( tool_id ) + self.tool = trans.app.toolbox.get_tool( tool_id, tool_version=tool_version ) self.post_job_actions = {} self.workflow_outputs = [] self.state = None @@ -493,11 +493,14 @@ @classmethod def from_dict( Class, trans, d, secure=True ): tool_id = d[ 'tool_id' ] - module = Class( trans, tool_id ) + tool_version = str( d.get( 'tool_version', None ) ) + module = Class( trans, tool_id, tool_version=tool_version ) module.state = galaxy.tools.DefaultToolState() if module.tool is not None: if d.get('tool_version', 'Unspecified') != module.get_tool_version(): - module.version_changes.append( "%s: using version '%s' instead of version '%s' indicated in this workflow." % ( tool_id, d.get( 'tool_version', 'Unspecified' ), module.get_tool_version() ) ) + message = "%s: using version '%s' instead of version '%s' indicated in this workflow." % ( tool_id, d.get( 'tool_version', 'Unspecified' ), module.get_tool_version() ) + log.debug(message) + module.version_changes.append(message) module.state.decode( d[ "tool_state" ], module.tool, module.trans.app, secure=secure ) module.errors = d.get( "tool_errors", None ) module.post_job_actions = d.get( "post_job_actions", {} ) @@ -519,9 +522,12 @@ # This step has its state saved in the config field due to the # tool being previously unavailable. return module_factory.from_dict(trans, loads(step.config), secure=False) - module = Class( trans, tool_id ) + tool_version = step.tool_version + module = Class( trans, tool_id, tool_version=tool_version ) if step.tool_version and (step.tool_version != module.tool.version): - module.version_changes.append("%s: using version '%s' instead of version '%s' indicated in this workflow." % (tool_id, module.tool.version, step.tool_version)) + message = "%s: using version '%s' instead of version '%s' indicated in this workflow." % (tool_id, module.tool.version, step.tool_version) + log.debug(message) + module.version_changes.append(message) module.recover_state( step.tool_inputs ) module.errors = step.tool_errors module.workflow_outputs = step.workflow_outputs @@ -723,7 +729,7 @@ return state, step_errors def execute( self, trans, progress, invocation, step ): - tool = trans.app.toolbox.get_tool( step.tool_id ) + tool = trans.app.toolbox.get_tool( step.tool_id, tool_version=step.tool_version ) tool_state = step.state collections_to_match = self._find_collections_to_match( tool, progress, step ) diff -r 02efb1123d469986a051e7a8015c9d56635efd8a -r 08f8850853d004bf8a456a147436a6694d0a1a11 test/api/test_tools.py --- a/test/api/test_tools.py +++ b/test/api/test_tools.py @@ -165,6 +165,18 @@ output1_content = self.dataset_populator.get_history_dataset_content( history_id, dataset=output1 ) self.assertEqual( output1_content.strip(), "Cat1Testlistified" ) + @skip_without_tool( "multiple_versions" ) + def test_run_by_versions( self ): + for version in ["0.1", "0.2"]: + # Run simple non-upload tool with an input data parameter. + history_id = self.dataset_populator.new_history() + inputs = dict() + outputs = self._run_and_get_outputs( tool_id="multiple_versions", history_id=history_id, inputs=inputs, tool_version=version ) + self.assertEquals( len( outputs ), 1 ) + output1 = outputs[ 0 ] + output1_content = self.dataset_populator.get_history_dataset_content( history_id, dataset=output1 ) + self.assertEqual( output1_content.strip(), "Version " + version ) + @skip_without_tool( "cat1" ) def test_run_cat1_single_meta_wrapper( self ): # Wrap input in a no-op meta parameter wrapper like Sam is planning to @@ -650,8 +662,8 @@ def _cat1_outputs( self, history_id, inputs ): return self._run_outputs( self._run_cat1( history_id, inputs ) ) - def _run_and_get_outputs( self, tool_id, history_id, inputs ): - return self._run_outputs( self._run( tool_id, history_id, inputs ) ) + def _run_and_get_outputs( self, tool_id, history_id, inputs, tool_version=None ): + return self._run_outputs( self._run( tool_id, history_id, inputs, tool_version=tool_version ) ) def _run_outputs( self, create_response ): self._assert_status_code_is( create_response, 200 ) @@ -660,12 +672,14 @@ def _run_cat1( self, history_id, inputs, assert_ok=False ): return self._run( 'cat1', history_id, inputs, assert_ok=assert_ok ) - def _run( self, tool_id, history_id, inputs, assert_ok=False ): + def _run( self, tool_id, history_id, inputs, assert_ok=False, tool_version=None ): payload = self.dataset_populator.run_tool_payload( tool_id=tool_id, inputs=inputs, history_id=history_id, ) + if tool_version is not None: + payload[ "tool_version" ] = tool_version create_response = self._post( "tools", data=payload ) if assert_ok: self._assert_status_code_is( create_response, 200 ) diff -r 02efb1123d469986a051e7a8015c9d56635efd8a -r 08f8850853d004bf8a456a147436a6694d0a1a11 test/api/test_workflows.py --- a/test/api/test_workflows.py +++ b/test/api/test_workflows.py @@ -444,6 +444,28 @@ def test_run_workflow( self ): self.__run_cat_workflow( inputs_by='step_id' ) + @skip_without_tool( "multiple_versions" ) + def test_run_versioned_tools( self ): + history_01_id = self.dataset_populator.new_history() + workflow_version_01 = self._upload_yaml_workflow( """ +- tool_id: multiple_versions + tool_version: "0.1" + state: + inttest: 0 +""" ) + self.__invoke_workflow( history_01_id, workflow_version_01 ) + self.dataset_populator.wait_for_history( history_01_id, assert_ok=True ) + + history_02_id = self.dataset_populator.new_history() + workflow_version_02 = self._upload_yaml_workflow( """ +- tool_id: multiple_versions + tool_version: "0.2" + state: + inttest: 1 +""" ) + self.__invoke_workflow( history_02_id, workflow_version_02 ) + self.dataset_populator.wait_for_history( history_02_id, assert_ok=True ) + def __run_cat_workflow( self, inputs_by ): workflow = self.workflow_populator.load_workflow( name="test_for_run" ) workflow["steps"]["0"]["uuid"] = str(uuid4()) @@ -910,7 +932,7 @@ self._assert_status_code_is( hda_info_response, 200 ) self.assertEquals( hda_info_response.json()[ "metadata_data_lines" ], lines ) - def __invoke_workflow( self, history_id, workflow_id, inputs, assert_ok=True ): + def __invoke_workflow( self, history_id, workflow_id, inputs={}, assert_ok=True ): workflow_request = dict( history="hist_id=%s" % history_id, ) diff -r 02efb1123d469986a051e7a8015c9d56635efd8a -r 08f8850853d004bf8a456a147436a6694d0a1a11 test/functional/tools/multiple_versions_v01.xml --- /dev/null +++ b/test/functional/tools/multiple_versions_v01.xml @@ -0,0 +1,11 @@ +<tool id="multiple_versions" name="multiple_versions" version="0.1"> + <command> + echo "Version 0.1" > $out_file1 + </command> + <inputs> + <param name="inttest" value="1" type="integer" /> + </inputs> + <outputs> + <data name="out_file1" format="txt" /> + </outputs> +</tool> diff -r 02efb1123d469986a051e7a8015c9d56635efd8a -r 08f8850853d004bf8a456a147436a6694d0a1a11 test/functional/tools/multiple_versions_v02.xml --- /dev/null +++ b/test/functional/tools/multiple_versions_v02.xml @@ -0,0 +1,11 @@ +<tool id="multiple_versions" name="multiple_versions" version="0.2"> + <command> + echo "Version 0.2" > $out_file1 + </command> + <inputs> + <param name="inttest" value="1" type="integer" /> + </inputs> + <outputs> + <data name="out_file1" format="txt" /> + </outputs> +</tool> diff -r 02efb1123d469986a051e7a8015c9d56635efd8a -r 08f8850853d004bf8a456a147436a6694d0a1a11 test/functional/tools/samples_tool_conf.xml --- a/test/functional/tools/samples_tool_conf.xml +++ b/test/functional/tools/samples_tool_conf.xml @@ -36,6 +36,9 @@ <tool file="collection_two_paired.xml" /><tool file="collection_optional_param.xml" /> + <tool file="multiple_versions_v01.xml" /> + <tool file="multiple_versions_v02.xml" /> + <!-- Tools interesting only for building up test workflows. --><!-- Next three tools demonstrate concatenating multiple datasets diff -r 02efb1123d469986a051e7a8015c9d56635efd8a -r 08f8850853d004bf8a456a147436a6694d0a1a11 test/unit/jobs/test_job_wrapper.py --- a/test/unit/jobs/test_job_wrapper.py +++ b/test/unit/jobs/test_job_wrapper.py @@ -194,7 +194,7 @@ assert tool_id == TEST_TOOL_ID return self.test_tool - def get_tool( self, tool_id, exact=False ): + def get_tool( self, tool_id, tool_version, exact=False ): tool = self.get(tool_id) return tool diff -r 02efb1123d469986a051e7a8015c9d56635efd8a -r 08f8850853d004bf8a456a147436a6694d0a1a11 test/unit/tools/test_toolbox.py --- a/test/unit/tools/test_toolbox.py +++ b/test/unit/tools/test_toolbox.py @@ -286,7 +286,7 @@ # Verify lineage_ids on both tools is correctly ordered. for version in ["0.1", "0.2"]: guid = "github.com/galaxyproject/example/test_tool/" + version - lineage_ids = self.toolbox.get_tool( guid ).lineage_ids + lineage_ids = self.toolbox.get_tool( guid ).lineage.get_version_ids() assert lineage_ids[ 0 ] == "github.com/galaxyproject/example/test_tool/0.1" assert lineage_ids[ 1 ] == "github.com/galaxyproject/example/test_tool/0.2" @@ -294,6 +294,53 @@ assert self.toolbox.get_tool( "test_tool", tool_version="0.1" ).guid == "github.com/galaxyproject/example/test_tool/0.1" assert self.toolbox.get_tool( "test_tool", tool_version="0.2" ).guid == "github.com/galaxyproject/example/test_tool/0.2" + def test_default_lineage( self ): + self.__init_versioned_tools() + self._add_config( """<toolbox><tool file="tool_v01.xml" /><tool file="tool_v02.xml" /></toolbox>""" ) + self.__verify_get_tool_for_default_lineage() + + def test_default_lineage_reversed( self ): + # Run same test as above but with entries in tool_conf reversed to + # ensure versioning is at work and not order effects. + self.__init_versioned_tools() + self._add_config( """<toolbox><tool file="tool_v02.xml" /><tool file="tool_v01.xml" /></toolbox>""" ) + self.__verify_get_tool_for_default_lineage() + + def test_grouping_with_default_lineage( self ): + self.__init_versioned_tools() + self._add_config( """<toolbox><tool file="tool_v01.xml" /><tool file="tool_v02.xml" /></toolbox>""" ) + self.__verify_tool_panel_for_default_lineage() + + def test_grouping_with_default_lineage_reversed( self ): + # Run same test as above but with entries in tool_conf reversed to + # ensure versioning is at work and not order effects. + self.__init_versioned_tools() + self._add_config( """<toolbox><tool file="tool_v02.xml" /><tool file="tool_v02.xml" /></toolbox>""" ) + self.__verify_tool_panel_for_default_lineage() + + def __init_versioned_tools( self ): + self._init_tool( filename="tool_v01.xml", version="0.1" ) + self._init_tool( filename="tool_v02.xml", version="0.2" ) + + def __verify_tool_panel_for_default_lineage( self ): + assert len( self.toolbox._tool_panel ) == 1 + tool = self.toolbox._tool_panel["tool_test_tool"] + assert tool.version == "0.2", tool.version + assert tool.id == "test_tool" + + def __verify_get_tool_for_default_lineage( self ): + tool_v01 = self.toolbox.get_tool( "test_tool", tool_version="0.1" ) + tool_v02 = self.toolbox.get_tool( "test_tool", tool_version="0.2" ) + assert tool_v02.id == "test_tool" + assert tool_v02.version == "0.2", tool_v02.version + assert tool_v01.id == "test_tool" + assert tool_v01.version == "0.1" + + # Newer variant gets to be default for that id. + default_tool = self.toolbox.get_tool( "test_tool" ) + assert default_tool.id == "test_tool" + assert default_tool.version == "0.2" + def __remove_itp( self ): os.remove( os.path) diff -r 02efb1123d469986a051e7a8015c9d56635efd8a -r 08f8850853d004bf8a456a147436a6694d0a1a11 test/unit/tools_support.py --- a/test/unit/tools_support.py +++ b/test/unit/tools_support.py @@ -7,6 +7,7 @@ import os.path import tempfile import shutil +import string from galaxy.util.bunch import Bunch from galaxy.web.security import SecurityHelper @@ -30,7 +31,7 @@ # Simple tool with just one text parameter and output. -SIMPLE_TOOL_CONTENTS = '''<tool id="test_tool" name="Test Tool"> +SIMPLE_TOOL_CONTENTS = '''<tool id="test_tool" name="Test Tool" version="$version"><command>echo "$param1" < $out1</command><inputs><param type="text" name="param1" value="" /> @@ -43,7 +44,7 @@ # A tool with data parameters (kind of like cat1) my favorite test tool :) -SIMPLE_CAT_TOOL_CONTENTS = '''<tool id="test_tool" name="Test Tool"> +SIMPLE_CAT_TOOL_CONTENTS = '''<tool id="test_tool" name="Test Tool" version="$version"><command>cat "$param1" #for $r in $repeat# "$r.param2" #end for# < $out1</command><inputs><param type="data" format="tabular" name="param1" value="" /> @@ -60,14 +61,24 @@ class UsesTools( object ): - def _init_tool( self, tool_contents=SIMPLE_TOOL_CONTENTS ): - self.tool_file = os.path.join( self.test_directory, "tool.xml" ) + def _init_tool( + self, + tool_contents=SIMPLE_TOOL_CONTENTS, + filename="tool.xml", + version="1.0" + ): + self._init_app_for_tools() + self.tool_file = os.path.join( self.test_directory, filename ) + contents_template = string.Template( tool_contents ) + tool_contents = contents_template.safe_substitute( dict( version=version ) ) + self.__write_tool( tool_contents ) + self.__setup_tool( ) + + def _init_app_for_tools( self ): self.app.config.drmaa_external_runjob_script = "" self.app.config.tool_secret = "testsecret" self.app.config.track_jobs_in_database = False self.app.job_config["get_job_tool_configurations"] = lambda ids: [Bunch(handler=Bunch())] - self.__write_tool( tool_contents ) - self.__setup_tool( ) def __setup_tool( self ): tool_source = get_tool_source( self.tool_file ) diff -r 02efb1123d469986a051e7a8015c9d56635efd8a -r 08f8850853d004bf8a456a147436a6694d0a1a11 test/unit/workflows/workflow_support.py --- a/test/unit/workflows/workflow_support.py +++ b/test/unit/workflows/workflow_support.py @@ -37,7 +37,7 @@ def __init__( self ): self.tools = {} - def get_tool( self, tool_id ): + def get_tool( self, tool_id, tool_version=None ): # Real tool box returns None of missing tool also return self.tools.get( tool_id, None ) https://bitbucket.org/galaxy/galaxy-central/commits/5c4f0045565b/ Changeset: 5c4f0045565b User: jmchilton Date: 2014-12-31 23:21:10+00:00 Summary: Introduce ToolPanel abtraction. There are some repeated operations on an odict in Toolbox that could be cleaned up (fewer lines of code, more readable) with new abstraction in place. Starting with __add_tool_to_tool_panel but will work its way to other methods in Toolbox eventually. Abstraction reduces cyclomatic complexity of Toolbox.__add_tool_to_tool_panel from 23 down to 18. Affected #: 3 files diff -r 08f8850853d004bf8a456a147436a6694d0a1a11 -r 5c4f0045565b46df32c4bd78c710275de68a4f65 lib/galaxy/tools/__init__.py --- a/lib/galaxy/tools/__init__.py +++ b/lib/galaxy/tools/__init__.py @@ -52,6 +52,7 @@ from galaxy.tools.test import parse_tests from galaxy.tools.parser import get_tool_source from galaxy.tools.parser.xml import XmlPageSource +from galaxy.tools.toolbox import ToolPanelElements from galaxy.tools.toolbox import tool_tag_manager from galaxy.tools.toolbox.lineages import LineageMap from galaxy.util import listify, parse_xml, rst_to_html, string_as_bool, string_to_object @@ -132,7 +133,7 @@ self._tool_versions_by_id = {} self._workflows_by_id = {} # In-memory dictionary that defines the layout of the tool panel. - self._tool_panel = odict() + self._tool_panel = ToolPanelElements() self._index = 0 self.data_manager_tools = odict() self._lineage_map = LineageMap( app ) @@ -141,7 +142,7 @@ # (in a way similar to the single tool_conf.xml file in the past) to alter the layout of the tool panel. self._integrated_tool_panel_config = app.config.integrated_tool_panel_config # In-memory dictionary that defines the layout of the tool_panel.xml file on disk. - self._integrated_tool_panel = odict() + self._integrated_tool_panel = ToolPanelElements() self._integrated_tool_panel_config_has_contents = os.path.exists( self._integrated_tool_panel_config ) and os.stat( self._integrated_tool_panel_config ).st_size > 0 if self._integrated_tool_panel_config_has_contents: self._load_integrated_tool_panel_keys() @@ -337,32 +338,30 @@ panel_dict = panel_component.elems else: panel_dict = panel_component + related_tool = self._lineage_in_panel( panel_dict, tool=tool ) if related_tool: if self._newer_tool( tool, related_tool ): - lineage_id = "tool_%s" % related_tool.id - index = panel_dict.keys().index( lineage_id ) - del panel_dict[ lineage_id ] - key = 'tool_%s' % tool.id - panel_dict.insert( index, key, tool ) + panel_dict.replace_tool( + previous_tool_id=related_tool.id, + new_tool_id=tool_id, + tool=tool, + ) log.debug( "Loaded tool id: %s, version: %s into tool panel." % ( tool.id, tool.version ) ) else: inserted = False - key = 'tool_%s' % tool.id - # The value of panel_component is the in-memory tool panel dictionary. - for index, integrated_panel_key in enumerate( self._integrated_tool_panel.keys() ): - if key == integrated_panel_key: - panel_dict.insert( index, key, tool ) - inserted = True + index = self._integrated_tool_panel.index_of_tool_id( tool_id ) + if index: + panel_dict.insert_tool( index, tool ) + inserted = True if not inserted: # Check the tool's installed versions. for tool_lineage_version in tool.lineage.get_versions(): lineage_id = tool_lineage_version.id - lineage_id_key = 'tool_%s' % lineage_id - for index, integrated_panel_key in enumerate( self._integrated_tool_panel.keys() ): - if lineage_id_key == integrated_panel_key: - panel_dict.insert( index, key, tool ) - inserted = True + index = self._integrated_tool_panel.index_of_tool_id( lineage_id ) + if index: + panel_dict.insert_tool( index, tool ) + inserted = True if not inserted: if ( tool.guid is None or @@ -375,7 +374,7 @@ # Shed, but is also not yet defined in # integrated_tool_panel.xml, so append it to the tool # panel. - panel_dict[ key ] = tool + panel_dict.append_tool( tool ) log.debug( "Loaded tool id: %s, version: %s into tool panel.." % ( tool.id, tool.version ) ) else: # We are in the process of installing the tool. @@ -384,7 +383,7 @@ already_loaded = self._lineage_in_panel( panel_dict, tool_lineage=tool_lineage ) is not None if not already_loaded: # If the tool is not defined in integrated_tool_panel.xml, append it to the tool panel. - panel_dict[ key ] = tool + panel_dict.append_tool( tool ) log.debug( "Loaded tool id: %s, version: %s into tool panel...." % ( tool.id, tool.version ) ) def _load_tool_panel( self ): @@ -1231,9 +1230,8 @@ lineage_tool = self._tool_from_lineage_version( lineage_tool_version ) if lineage_tool: lineage_id = lineage_tool.id - loaded_version_key = 'tool_%s' % lineage_id - if loaded_version_key in panel_dict: - return panel_dict[ loaded_version_key ] + if panel_dict.has_tool_with_id( lineage_id ): + return panel_dict.get_tool_with_id( lineage_id ) return None def _newer_tool( self, tool1, tool2 ): @@ -1327,7 +1325,7 @@ self.name = f( elem, 'name' ) self.id = f( elem, 'id' ) self.version = f( elem, 'version' ) - self.elems = odict() + self.elems = ToolPanelElements() def copy( self ): copy = ToolSection() diff -r 08f8850853d004bf8a456a147436a6694d0a1a11 -r 5c4f0045565b46df32c4bd78c710275de68a4f65 lib/galaxy/tools/toolbox/__init__.py --- a/lib/galaxy/tools/toolbox/__init__.py +++ b/lib/galaxy/tools/toolbox/__init__.py @@ -2,6 +2,7 @@ """ from .tags import tool_tag_manager +from .panel import ToolPanelElements -__all__ = ["tool_tag_manager"] +__all__ = ["ToolPanelElements", "tool_tag_manager"] diff -r 08f8850853d004bf8a456a147436a6694d0a1a11 -r 5c4f0045565b46df32c4bd78c710275de68a4f65 lib/galaxy/tools/toolbox/panel.py --- /dev/null +++ b/lib/galaxy/tools/toolbox/panel.py @@ -0,0 +1,37 @@ +from galaxy.util.odict import odict + + +class ToolPanelElements( odict ): + """ Represents an ordered dictionary of tool entries - abstraction + used both by tool panel itself (normal and integrated) and its sections. + """ + + def has_tool_with_id( self, tool_id ): + key = 'tool_%s' % tool_id + return key in self + + def replace_tool( self, previous_tool_id, new_tool_id, tool ): + previous_key = 'tool_%s' % previous_tool_id + new_key = 'tool_%s' % new_tool_id + index = self.keys().index( previous_key ) + del self[ previous_key ] + self.insert( index, new_key, tool ) + + def index_of_tool_id( self, tool_id ): + query_key = 'tool_%s' % tool_id + for index, target_key in enumerate( self.keys() ): + if query_key == target_key: + return index + return None + + def insert_tool( self, index, tool ): + key = "tool_%s" % tool.id + self.insert( index, key, tool ) + + def get_tool_with_id( self, tool_id ): + key = "tool_%s" % tool_id + return self[ key ] + + def append_tool( self, tool ): + key = "tool_%s" % tool.id + self[ key ] = tool https://bitbucket.org/galaxy/galaxy-central/commits/67ae665c8e1f/ Changeset: 67ae665c8e1f User: jmchilton Date: 2014-12-31 23:21:10+00:00 Summary: Move more logic into ToolSectionElements, raising level abstraction in Toolbox. Affected #: 2 files diff -r 5c4f0045565b46df32c4bd78c710275de68a4f65 -r 67ae665c8e1feed7f2d9cabce44a55ec0344be58 lib/galaxy/tools/__init__.py --- a/lib/galaxy/tools/__init__.py +++ b/lib/galaxy/tools/__init__.py @@ -435,29 +435,24 @@ tree = parse_xml( self._integrated_tool_panel_config ) root = tree.getroot() for elem in root: + key = elem.get( 'id' ) if elem.tag == 'tool': - key = 'tool_%s' % elem.get( 'id' ) - self._integrated_tool_panel[ key ] = None + self._integrated_tool_panel.stub_tool( key ) elif elem.tag == 'workflow': - key = 'workflow_%s' % elem.get( 'id' ) - self._integrated_tool_panel[ key ] = None + self._integrated_tool_panel.stub_workflow( key ) elif elem.tag == 'section': section = ToolSection( elem ) for section_elem in elem: + section_id = section_elem.get( 'id' ) if section_elem.tag == 'tool': - key = 'tool_%s' % section_elem.get( 'id' ) - section.elems[ key ] = None + section.elems.stub_tool( section_id ) elif section_elem.tag == 'workflow': - key = 'workflow_%s' % section_elem.get( 'id' ) - section.elems[ key ] = None + section.elems.stub_workflow( section_id ) elif section_elem.tag == 'label': - key = 'label_%s' % section_elem.get( 'id' ) - section.elems[ key ] = None - key = elem.get( 'id' ) - self._integrated_tool_panel[ key ] = section + section.elems.stub_label( section_id ) + self._integrated_tool_panel.append_section( key, section ) elif elem.tag == 'label': - key = 'label_%s' % elem.get( 'id' ) - self._integrated_tool_panel[ key ] = None + section.stub_label( key ) def _write_integrated_tool_panel_config_file( self ): """ diff -r 5c4f0045565b46df32c4bd78c710275de68a4f65 -r 67ae665c8e1feed7f2d9cabce44a55ec0344be58 lib/galaxy/tools/toolbox/panel.py --- a/lib/galaxy/tools/toolbox/panel.py +++ b/lib/galaxy/tools/toolbox/panel.py @@ -35,3 +35,18 @@ def append_tool( self, tool ): key = "tool_%s" % tool.id self[ key ] = tool + + def stub_tool( self, key ): + key = "tool_%s" % key + self[ key ] = None + + def stub_workflow( self, key ): + key = 'workflow_%s' % key + self[ key ] = None + + def stub_label( self, key ): + key = 'label_%s' % key + self[ key ] = None + + def append_section( self, key, section_elems ): + self[ key ] = section_elems https://bitbucket.org/galaxy/galaxy-central/commits/afdd95f45c4e/ Changeset: afdd95f45c4e User: jmchilton Date: 2014-12-31 23:21:10+00:00 Summary: Replace another repeated idiom in Toolbox with higher level method in toolbox.panel. Affected #: 2 files diff -r 67ae665c8e1feed7f2d9cabce44a55ec0344be58 -r afdd95f45c4e46f2dc18ce0e5b3e7b28d413993b lib/galaxy/tools/__init__.py --- a/lib/galaxy/tools/__init__.py +++ b/lib/galaxy/tools/__init__.py @@ -755,10 +755,7 @@ self._tool_tag_manager.handle_tags( tool.id, elem ) self.__add_tool( tool, load_panel_dict, panel_dict ) # Always load the tool into the integrated_panel_dict, or it will not be included in the integrated_tool_panel.xml file. - if key in integrated_panel_dict or index is None: - integrated_panel_dict[ key ] = tool - else: - integrated_panel_dict.insert( index, key, tool ) + integrated_panel_dict.update_or_append( index, key, tool ) except IOError: log.error( "Error reading tool configuration file from path: %s." % path ) except Exception: @@ -787,10 +784,7 @@ if load_panel_dict: panel_dict[ key ] = workflow # Always load workflows into the integrated_panel_dict. - if key in integrated_panel_dict or index is None: - integrated_panel_dict[ key ] = workflow - else: - integrated_panel_dict.insert( index, key, workflow ) + integrated_panel_dict.update_or_append( index, key, workflow ) except: log.exception( "Error loading workflow: %s" % workflow_id ) @@ -799,10 +793,7 @@ key = 'label_' + label.id if load_panel_dict: panel_dict[ key ] = label - if key in integrated_panel_dict or index is None: - integrated_panel_dict[ key ] = label - else: - integrated_panel_dict.insert( index, key, label ) + integrated_panel_dict.update_or_append( index, key, label ) def _load_section_tag_set( self, elem, tool_path, load_panel_dict, index=None ): key = elem.get( "id" ) @@ -832,10 +823,7 @@ if load_panel_dict: self._tool_panel[ key ] = section # Always load sections into the integrated_tool_panel. - if key in self._integrated_tool_panel or index is None: - self._integrated_tool_panel[ key ] = integrated_section - else: - self._integrated_tool_panel.insert( index, key, integrated_section ) + self._integrated_tool_panel.update_or_append( index, key, integrated_section ) def _load_tooldir_tag_set(self, sub_elem, elems, tool_path, integrated_elems, load_panel_dict): directory = os.path.join( tool_path, sub_elem.attrib.get("dir") ) diff -r 67ae665c8e1feed7f2d9cabce44a55ec0344be58 -r afdd95f45c4e46f2dc18ce0e5b3e7b28d413993b lib/galaxy/tools/toolbox/panel.py --- a/lib/galaxy/tools/toolbox/panel.py +++ b/lib/galaxy/tools/toolbox/panel.py @@ -6,6 +6,12 @@ used both by tool panel itself (normal and integrated) and its sections. """ + def update_or_append( self, index, key, value ): + if key in self or index is None: + self[ key ] = value + else: + self.insert( index, key, value ) + def has_tool_with_id( self, tool_id ): key = 'tool_%s' % tool_id return key in self https://bitbucket.org/galaxy/galaxy-central/commits/8527f5a00f19/ Changeset: 8527f5a00f19 User: jmchilton Date: 2014-12-31 23:21:10+00:00 Summary: Move ToolSection and ToolSectionLabel out of galaxy.tools directly. Would like to move the whole ToolBox out of galaxy.tools and into galaxy.tools.toolbox but need to separate all dependencies on galaxy.tools first. Affected #: 8 files diff -r afdd95f45c4e46f2dc18ce0e5b3e7b28d413993b -r 8527f5a00f196ac8b641c57808d26edf305dc16a lib/galaxy/tools/__init__.py --- a/lib/galaxy/tools/__init__.py +++ b/lib/galaxy/tools/__init__.py @@ -53,6 +53,8 @@ from galaxy.tools.parser import get_tool_source from galaxy.tools.parser.xml import XmlPageSource from galaxy.tools.toolbox import ToolPanelElements +from galaxy.tools.toolbox import ToolSection +from galaxy.tools.toolbox import ToolSectionLabel from galaxy.tools.toolbox import tool_tag_manager from galaxy.tools.toolbox.lineages import LineageMap from galaxy.util import listify, parse_xml, rst_to_html, string_as_bool, string_to_object @@ -1293,67 +1295,6 @@ return None -class ToolSection( object, Dictifiable ): - """ - A group of tools with similar type/purpose that will be displayed as a - group in the user interface. - """ - - dict_collection_visible_keys = ( 'id', 'name', 'version' ) - - def __init__( self, elem=None ): - """ Build a ToolSection from an ElementTree element or a dictionary. - """ - f = lambda elem, val: elem is not None and elem.get( val ) or '' - self.name = f( elem, 'name' ) - self.id = f( elem, 'id' ) - self.version = f( elem, 'version' ) - self.elems = ToolPanelElements() - - def copy( self ): - copy = ToolSection() - copy.name = self.name - copy.id = self.id - copy.version = self.version - copy.elems = self.elems.copy() - return copy - - def to_dict( self, trans, link_details=False ): - """ Return a dict that includes section's attributes. """ - - section_dict = super( ToolSection, self ).to_dict() - section_elts = [] - kwargs = dict( - trans=trans, - link_details=link_details - ) - for elt in self.elems.values(): - section_elts.append( elt.to_dict( **kwargs ) ) - section_dict[ 'elems' ] = section_elts - - return section_dict - - -class ToolSectionLabel( object, Dictifiable ): - """ - A label for a set of tools that can be displayed above groups of tools - and sections in the user interface - """ - - dict_collection_visible_keys = ( 'id', 'text', 'version' ) - - def __init__( self, elem ): - """ Build a ToolSectionLabel from an ElementTree element or a - dictionary. - """ - self.text = elem.get( "text" ) - self.id = elem.get( "id" ) - self.version = elem.get( "version" ) or '' - - def to_dict( self, **kwds ): - return super( ToolSectionLabel, self ).to_dict() - - class DefaultToolState( object ): """ Keeps track of the state of a users interaction with a tool between diff -r afdd95f45c4e46f2dc18ce0e5b3e7b28d413993b -r 8527f5a00f196ac8b641c57808d26edf305dc16a lib/galaxy/tools/toolbox/__init__.py --- a/lib/galaxy/tools/toolbox/__init__.py +++ b/lib/galaxy/tools/toolbox/__init__.py @@ -3,6 +3,8 @@ from .tags import tool_tag_manager from .panel import ToolPanelElements +from .panel import ToolSection +from .panel import ToolSectionLabel -__all__ = ["ToolPanelElements", "tool_tag_manager"] +__all__ = ["ToolPanelElements", "tool_tag_manager", "ToolSection", "ToolSectionLabel"] diff -r afdd95f45c4e46f2dc18ce0e5b3e7b28d413993b -r 8527f5a00f196ac8b641c57808d26edf305dc16a lib/galaxy/tools/toolbox/panel.py --- a/lib/galaxy/tools/toolbox/panel.py +++ b/lib/galaxy/tools/toolbox/panel.py @@ -1,4 +1,66 @@ from galaxy.util.odict import odict +from galaxy.model.item_attrs import Dictifiable + + +class ToolSection( object, Dictifiable ): + """ + A group of tools with similar type/purpose that will be displayed as a + group in the user interface. + """ + + dict_collection_visible_keys = ( 'id', 'name', 'version' ) + + def __init__( self, elem=None ): + """ Build a ToolSection from an ElementTree element or a dictionary. + """ + f = lambda elem, val: elem is not None and elem.get( val ) or '' + self.name = f( elem, 'name' ) + self.id = f( elem, 'id' ) + self.version = f( elem, 'version' ) + self.elems = ToolPanelElements() + + def copy( self ): + copy = ToolSection() + copy.name = self.name + copy.id = self.id + copy.version = self.version + copy.elems = self.elems.copy() + return copy + + def to_dict( self, trans, link_details=False ): + """ Return a dict that includes section's attributes. """ + + section_dict = super( ToolSection, self ).to_dict() + section_elts = [] + kwargs = dict( + trans=trans, + link_details=link_details + ) + for elt in self.elems.values(): + section_elts.append( elt.to_dict( **kwargs ) ) + section_dict[ 'elems' ] = section_elts + + return section_dict + + +class ToolSectionLabel( object, Dictifiable ): + """ + A label for a set of tools that can be displayed above groups of tools + and sections in the user interface + """ + + dict_collection_visible_keys = ( 'id', 'text', 'version' ) + + def __init__( self, elem ): + """ Build a ToolSectionLabel from an ElementTree element or a + dictionary. + """ + self.text = elem.get( "text" ) + self.id = elem.get( "id" ) + self.version = elem.get( "version" ) or '' + + def to_dict( self, **kwds ): + return super( ToolSectionLabel, self ).to_dict() class ToolPanelElements( odict ): diff -r afdd95f45c4e46f2dc18ce0e5b3e7b28d413993b -r 8527f5a00f196ac8b641c57808d26edf305dc16a lib/tool_shed/galaxy_install/tool_migration_manager.py --- a/lib/tool_shed/galaxy_install/tool_migration_manager.py +++ b/lib/tool_shed/galaxy_install/tool_migration_manager.py @@ -10,7 +10,7 @@ import logging from galaxy import util -from galaxy.tools import ToolSection +from galaxy.tools.toolbox import ToolSection from galaxy.util.odict import odict from tool_shed.galaxy_install import install_manager diff -r afdd95f45c4e46f2dc18ce0e5b3e7b28d413993b -r 8527f5a00f196ac8b641c57808d26edf305dc16a templates/admin/package_tool.mako --- a/templates/admin/package_tool.mako +++ b/templates/admin/package_tool.mako @@ -1,6 +1,9 @@ <%inherit file="/base.mako"/><%namespace file="/message.mako" import="render_msg" /> -<% from galaxy.tools import Tool, ToolSection %> +<% + from galaxy.tools import Tool + from galaxy.tools.toolbox import ToolSection +%><script type="text/javascript"> $().ready(function() { diff -r afdd95f45c4e46f2dc18ce0e5b3e7b28d413993b -r 8527f5a00f196ac8b641c57808d26edf305dc16a templates/admin/reload_tool.mako --- a/templates/admin/reload_tool.mako +++ b/templates/admin/reload_tool.mako @@ -1,6 +1,9 @@ <%inherit file="/base.mako"/><%namespace file="/message.mako" import="render_msg" /> -<% from galaxy.tools import Tool, ToolSection %> +<% + from galaxy.tools import Tool + from galaxy.tools.toolbox import ToolSection +%><script type="text/javascript"> $().ready(function() { diff -r afdd95f45c4e46f2dc18ce0e5b3e7b28d413993b -r 8527f5a00f196ac8b641c57808d26edf305dc16a templates/webapps/galaxy/workflow/editor.mako --- a/templates/webapps/galaxy/workflow/editor.mako +++ b/templates/webapps/galaxy/workflow/editor.mako @@ -269,7 +269,10 @@ </%def><%def name="left_panel()"> - <% from galaxy.tools import Tool, ToolSection, ToolSectionLabel %> + <% + from galaxy.tools import Tool + from galaxy.tools.toolbox import ToolSection, ToolSectionLabel + %><div class="unified-panel-header" unselectable="on"><div class='unified-panel-header-inner'> diff -r afdd95f45c4e46f2dc18ce0e5b3e7b28d413993b -r 8527f5a00f196ac8b641c57808d26edf305dc16a test/unit/tools/test_tool_panel.py --- a/test/unit/tools/test_tool_panel.py +++ b/test/unit/tools/test_tool_panel.py @@ -1,5 +1,6 @@ from xml.etree import ElementTree as ET -from galaxy.tools import ToolSection + +from galaxy.tools.toolbox import ToolSection def test_tool_section( ): https://bitbucket.org/galaxy/galaxy-central/commits/24aa2e802a35/ Changeset: 24aa2e802a35 User: jmchilton Date: 2014-12-31 23:21:10+00:00 Summary: Introduce HasPanelItems abstraction with several benefits. - Abstraction allows treating tool panels and sections more uniformly. - Using this abstraction for iterating over items eliminates isinstance checks (considered harmful - not ducky enough - http://www.voidspace.org.uk/python/articles/duck_typing.shtml). - Eliminates several direct references to the Tool and Workflow classes in ToolBox - (useful toward decoupling ToolBox from galaxy.tools). Affected #: 3 files diff -r 8527f5a00f196ac8b641c57808d26edf305dc16a -r 24aa2e802a351882695cb98d772001cc2a1316d8 lib/galaxy/tools/__init__.py --- a/lib/galaxy/tools/__init__.py +++ b/lib/galaxy/tools/__init__.py @@ -55,6 +55,7 @@ from galaxy.tools.toolbox import ToolPanelElements from galaxy.tools.toolbox import ToolSection from galaxy.tools.toolbox import ToolSectionLabel +from galaxy.tools.toolbox import panel_item_types from galaxy.tools.toolbox import tool_tag_manager from galaxy.tools.toolbox.lineages import LineageMap from galaxy.util import listify, parse_xml, rst_to_html, string_as_bool, string_to_object @@ -304,16 +305,16 @@ def get_integrated_section_for_tool( self, tool ): tool_id = tool.id - for key, item in self._integrated_tool_panel.items(): + for key, item_type, item in self._integrated_tool_panel.panel_items_iter(): if item: - if isinstance( item, Tool ): + if item_type == panel_item_types.TOOL: if item.id == tool_id: return '', '' - if isinstance( item, ToolSection ): + if item_type == panel_item_types.SECTION: section_id = item.id or '' section_name = item.name or '' - for section_key, section_item in item.elems.items(): - if isinstance( section_item, Tool ): + for section_key, section_item_type, section_item in item.panel_items_iter(): + if section_item_type == panel_item_types.TOOL: if section_item: if section_item.id == tool_id: return section_id, section_name @@ -389,20 +390,20 @@ log.debug( "Loaded tool id: %s, version: %s into tool panel...." % ( tool.id, tool.version ) ) def _load_tool_panel( self ): - for key, val in self._integrated_tool_panel.items(): - if isinstance( val, Tool ): + for key, item_type, val in self._integrated_tool_panel.panel_items_iter(): + if item_type == panel_item_types.TOOL: tool_id = key.replace( 'tool_', '', 1 ) if tool_id in self._tools_by_id: self.__add_tool_to_tool_panel( val, self._tool_panel, section=False ) - elif isinstance( val, Workflow ): + elif item_type == panel_item_types.WORKFLOW: workflow_id = key.replace( 'workflow_', '', 1 ) if workflow_id in self._workflows_by_id: workflow = self._workflows_by_id[ workflow_id ] self._tool_panel[ key ] = workflow log.debug( "Loaded workflow: %s %s" % ( workflow_id, workflow.name ) ) - elif isinstance( val, ToolSectionLabel ): + elif item_type == panel_item_types.LABEL: self._tool_panel[ key ] = val - elif isinstance( val, ToolSection ): + elif item_type == panel_item_types.SECTION: section_dict = { 'id': val.id or '', 'name': val.name or '', @@ -410,18 +411,18 @@ } section = ToolSection( section_dict ) log.debug( "Loading section: %s" % section_dict.get( 'name' ) ) - for section_key, section_val in val.elems.items(): - if isinstance( section_val, Tool ): + for section_key, section_item_type, section_val in val.panel_items_iter(): + if section_item_type == panel_item_types.TOOL: tool_id = section_key.replace( 'tool_', '', 1 ) if tool_id in self._tools_by_id: self.__add_tool_to_tool_panel( section_val, section, section=True ) - elif isinstance( section_val, Workflow ): + elif section_item_type == panel_item_types.WORKFLOW: workflow_id = section_key.replace( 'workflow_', '', 1 ) if workflow_id in self._workflows_by_id: workflow = self._workflows_by_id[ workflow_id ] section.elems[ section_key ] = workflow log.debug( "Loaded workflow: %s %s" % ( workflow_id, workflow.name ) ) - elif isinstance( section_val, ToolSectionLabel ): + elif section_item_type == panel_item_types.LABEL: if section_val: section.elems[ section_key ] = section_val log.debug( "Loaded label: %s" % ( section_val.text ) ) @@ -467,30 +468,30 @@ os.write( fd, ' <!--\n ') os.write( fd, '\n '.join( [ l for l in INTEGRATED_TOOL_PANEL_DESCRIPTION.split("\n") if l ] ) ) os.write( fd, '\n -->\n') - for key, item in self._integrated_tool_panel.items(): + for key, item_type, item in self._integrated_tool_panel.panel_items_iter(): if item: - if isinstance( item, Tool ): + if item_type == panel_item_types.TOOL: os.write( fd, ' <tool id="%s" />\n' % item.id ) - elif isinstance( item, Workflow ): + elif item_type == panel_item_types.WORKFLOW: os.write( fd, ' <workflow id="%s" />\n' % item.id ) - elif isinstance( item, ToolSectionLabel ): + elif item_type == panel_item_types.LABEL: label_id = item.id or '' label_text = item.text or '' label_version = item.version or '' os.write( fd, ' <label id="%s" text="%s" version="%s" />\n' % ( label_id, label_text, label_version ) ) - elif isinstance( item, ToolSection ): + elif item_type == panel_item_types.SECTION: section_id = item.id or '' section_name = item.name or '' section_version = item.version or '' os.write( fd, ' <section id="%s" name="%s" version="%s">\n' % ( section_id, section_name, section_version ) ) - for section_key, section_item in item.elems.items(): - if isinstance( section_item, Tool ): + for section_key, section_item_type, section_item in item.panel_items_iter(): + if section_item_type == panel_item_types.TOOL: if section_item: os.write( fd, ' <tool id="%s" />\n' % section_item.id ) - elif isinstance( section_item, Workflow ): + elif section_item_type == panel_item_types.WORKFLOW: if section_item: os.write( fd, ' <workflow id="%s" />\n' % section_item.id ) - elif isinstance( section_item, ToolSectionLabel ): + elif section_item_type == panel_item_types.LABEL: if section_item: label_id = section_item.id or '' label_text = section_item.text or '' @@ -709,10 +710,9 @@ else: del has_elems[ tool_key ] if remove_from_config: - if hasattr( integrated_has_elems, "elems" ): - integrated_has_elems = integrated_has_elems.elems - if tool_key in integrated_has_elems: - del integrated_has_elems[ tool_key ] + itegrated_items = integrated_has_elems.panel_items() + if tool_key in itegrated_items: + del itegrated_items[ tool_key ] if section_key: _, tool_section = self.get_section( section_key ) @@ -1174,8 +1174,8 @@ """ context = Bunch( toolbox=self, trans=trans ) filters = self._filter_factory.build_filters( trans ) - for elt in self._tool_panel.itervalues(): - elt = _filter_for_panel( elt, filters, context ) + for _, item_type, elt in self._tool_panel.panel_items_iter(): + elt = _filter_for_panel( elt, item_type, filters, context ) if elt: yield elt @@ -1241,7 +1241,7 @@ return self._tool_versions_by_id.get( lineage_tool_version.id, {} ).get( lineage_tool_version.version, None ) -def _filter_for_panel( item, filters, context ): +def _filter_for_panel( item, item_type, filters, context ): """ Filters tool panel elements so that only those that are compatible with provided filters are kept. @@ -1251,13 +1251,13 @@ if not filter_method( context, filter_item ): return False return True - if isinstance( item, Tool ): + if item_type == panel_item_types.TOOL: if _apply_filter( item, filters[ 'tool' ] ): return item - elif isinstance( item, ToolSectionLabel ): + elif item_type == panel_item_types.LABEL: if _apply_filter( item, filters[ 'label' ] ): return item - elif isinstance( item, ToolSection ): + elif item_type == panel_item_types.SECTION: # Filter section item-by-item. Only show a label if there are # non-filtered tools below it. @@ -1265,14 +1265,14 @@ cur_label_key = None tools_under_label = False filtered_elems = item.elems.copy() - for key, section_item in item.elems.items(): - if isinstance( section_item, Tool ): + for key, section_item_type, section_item in item.panel_items_iter(): + if section_item_type == panel_item_types.TOOL: # Filter tool. if _apply_filter( section_item, filters[ 'tool' ] ): tools_under_label = True else: del filtered_elems[ key ] - elif isinstance( section_item, ToolSectionLabel ): + elif section_item_type == panel_item_types.LABEL: # If there is a label and it does not have tools, # remove it. if ( cur_label_key and not tools_under_label ) or not _apply_filter( section_item, filters[ 'label' ] ): diff -r 8527f5a00f196ac8b641c57808d26edf305dc16a -r 24aa2e802a351882695cb98d772001cc2a1316d8 lib/galaxy/tools/toolbox/__init__.py --- a/lib/galaxy/tools/toolbox/__init__.py +++ b/lib/galaxy/tools/toolbox/__init__.py @@ -5,6 +5,12 @@ from .panel import ToolPanelElements from .panel import ToolSection from .panel import ToolSectionLabel +from .panel import panel_item_types - -__all__ = ["ToolPanelElements", "tool_tag_manager", "ToolSection", "ToolSectionLabel"] +__all__ = [ + "ToolPanelElements", + "tool_tag_manager", + "ToolSection", + "ToolSectionLabel", + "panel_item_types" +] diff -r 8527f5a00f196ac8b641c57808d26edf305dc16a -r 24aa2e802a351882695cb98d772001cc2a1316d8 lib/galaxy/tools/toolbox/panel.py --- a/lib/galaxy/tools/toolbox/panel.py +++ b/lib/galaxy/tools/toolbox/panel.py @@ -1,8 +1,46 @@ +from abc import abstractmethod + from galaxy.util.odict import odict +from galaxy.util import bunch from galaxy.model.item_attrs import Dictifiable -class ToolSection( object, Dictifiable ): +panel_item_types = bunch.Bunch( + TOOL="TOOL", + WORKFLOW="WORKFLOW", + SECTION="SECTION", + LABEL="LABEL", +) + + +class HasPanelItems: + """ + """ + + @abstractmethod + def panel_items( self ): + """ Return an ordered dictionary-like object describing tool panel + items (such as workflows, tools, labels, and sections). + """ + + def panel_items_iter( self ): + """ Iterate through panel items each represented as a tuple of + (panel_key, panel_type, panel_content). + """ + for panel_key, panel_value in self.panel_items().iteritems(): + if panel_value is None: + continue + panel_type = panel_item_types.SECTION + if panel_key.startswith("tool_"): + panel_type = panel_item_types.TOOL + elif panel_key.startswith("label_"): + panel_type = panel_item_types.LABEL + elif panel_key.startswith("workflow_"): + panel_type = panel_item_types.WORKFLOW + yield (panel_key, panel_type, panel_value) + + +class ToolSection( object, Dictifiable, HasPanelItems ): """ A group of tools with similar type/purpose that will be displayed as a group in the user interface. @@ -42,6 +80,9 @@ return section_dict + def panel_items( self ): + return self.elems + class ToolSectionLabel( object, Dictifiable ): """ @@ -63,7 +104,7 @@ return super( ToolSectionLabel, self ).to_dict() -class ToolPanelElements( odict ): +class ToolPanelElements( odict, HasPanelItems ): """ Represents an ordered dictionary of tool entries - abstraction used both by tool panel itself (normal and integrated) and its sections. """ @@ -118,3 +159,6 @@ def append_section( self, key, section_elems ): self[ key ] = section_elems + + def panel_items( self ): + return self https://bitbucket.org/galaxy/galaxy-central/commits/edacabfca615/ Changeset: edacabfca615 User: jmchilton Date: 2014-12-31 23:21:10+00:00 Summary: Move 95% of ToolBox into its own module. Including logic related to panels and sections, the integrated tool panels, labels, etc.... Leave a variant of ToolBox in galaxy.tools that needs to know about Tool runtime stuff (Tool creation, actions, and dependency management). Affected #: 3 files diff -r 24aa2e802a351882695cb98d772001cc2a1316d8 -r edacabfca615542ca435a97fc1afcc99daa470d5 lib/galaxy/tools/__init__.py --- a/lib/galaxy/tools/__init__.py +++ b/lib/galaxy/tools/__init__.py @@ -9,9 +9,6 @@ import os import re import shutil -import string -import tarfile -import tempfile import threading import types import urllib @@ -27,12 +24,10 @@ from elementtree import ElementTree from mako.template import Template from paste import httpexceptions -from sqlalchemy import and_ from galaxy import jobs, model from galaxy.datatypes.metadata import JobExternalOutputMetadataWrapper from galaxy import exceptions -from galaxy.tools.toolbox import watcher from galaxy.tools.actions import DefaultToolAction from galaxy.tools.actions.upload import UploadToolAction from galaxy.tools.actions.data_source import DataSourceToolAction @@ -48,17 +43,11 @@ from galaxy.tools.parameters.input_translation import ToolInputTranslator from galaxy.tools.parameters.output import ToolOutputActionGroup from galaxy.tools.parameters.validation import LateValidationError -from galaxy.tools.toolbox.filters import FilterFactory from galaxy.tools.test import parse_tests from galaxy.tools.parser import get_tool_source from galaxy.tools.parser.xml import XmlPageSource -from galaxy.tools.toolbox import ToolPanelElements -from galaxy.tools.toolbox import ToolSection -from galaxy.tools.toolbox import ToolSectionLabel -from galaxy.tools.toolbox import panel_item_types -from galaxy.tools.toolbox import tool_tag_manager -from galaxy.tools.toolbox.lineages import LineageMap -from galaxy.util import listify, parse_xml, rst_to_html, string_as_bool, string_to_object +from galaxy.tools.toolbox import AbstractToolBox +from galaxy.util import rst_to_html, string_as_bool, string_to_object from galaxy.tools.parameters.meta import expand_meta_parameters from galaxy.util.bunch import Bunch from galaxy.util.expressions import ExpressionContext @@ -66,11 +55,7 @@ from galaxy.util.odict import odict from galaxy.util.template import fill_template from galaxy.web import url_for -from galaxy.web.form_builder import SelectField -from galaxy.web.framework.helpers import escape from galaxy.model.item_attrs import Dictifiable -from galaxy.model import Workflow -from tool_shed.util import common_util from tool_shed.util import shed_util_common as suc from .loader import template_macro_params, raw_tool_xml_tree, imported_macro_paths from .execute import execute as execute_job @@ -102,774 +87,30 @@ HELP_UNINITIALIZED = threading.Lock() -INTEGRATED_TOOL_PANEL_DESCRIPTION = """ -This is Galaxy's integrated tool panel and probably should not be modified -directly. It will be regenerated each time Galaxy starts up. To modify locally -managed tools (e.g. from tool_conf.xml) modify that file directly and restart -Galaxy. Whenever possible Tool Shed managed tools (e.g. from shed_tool_conf.xml) -should be managed from within the Galaxy interface of via is UI - but if changes -are nessecary (such as to hide a tool or re-assign its section) modify that file -and restart Galaxy. -""" - class ToolNotFoundException( Exception ): pass -class ToolBox( object, Dictifiable ): - """Container for a collection of tools""" +class ToolBox( AbstractToolBox ): + """ A derivative of AbstractToolBox with knowledge about Tool internals - + how to construct them, action types, dependency management, etc.... + """ def __init__( self, config_filenames, tool_root_dir, app ): - """ - Create a toolbox from the config files named by `config_filenames`, using - `tool_root_dir` as the base directory for finding individual tool config files. - """ - # The _dynamic_tool_confs list contains dictionaries storing - # information about the tools defined in each shed-related - # shed_tool_conf.xml file. - self._dynamic_tool_confs = [] - self._tools_by_id = {} - # Tool lineages can contain chains of related tools with different ids - # so each will be present once in the above dictionary. The following - # dictionary can instead hold multiple tools with different versions. - self._tool_versions_by_id = {} - self._workflows_by_id = {} - # In-memory dictionary that defines the layout of the tool panel. - self._tool_panel = ToolPanelElements() - self._index = 0 - self.data_manager_tools = odict() - self._lineage_map = LineageMap( app ) - # File that contains the XML section and tool tags from all tool panel config files integrated into a - # single file that defines the tool panel layout. This file can be changed by the Galaxy administrator - # (in a way similar to the single tool_conf.xml file in the past) to alter the layout of the tool panel. - self._integrated_tool_panel_config = app.config.integrated_tool_panel_config - # In-memory dictionary that defines the layout of the tool_panel.xml file on disk. - self._integrated_tool_panel = ToolPanelElements() - self._integrated_tool_panel_config_has_contents = os.path.exists( self._integrated_tool_panel_config ) and os.stat( self._integrated_tool_panel_config ).st_size > 0 - if self._integrated_tool_panel_config_has_contents: - self._load_integrated_tool_panel_keys() - # The following refers to the tool_path config setting for backward compatibility. The shed-related - # (e.g., shed_tool_conf.xml) files include the tool_path attribute within the <toolbox> tag. - self._tool_root_dir = tool_root_dir - self.app = app - self._tool_watcher = watcher.get_watcher( self, app.config ) - self._filter_factory = FilterFactory( self ) + super( ToolBox, self ).__init__( + config_filenames=config_filenames, + tool_root_dir=tool_root_dir, + app=app, + ) self._init_dependency_manager() - self._tool_tag_manager = tool_tag_manager( app ) - self._init_tools_from_configs( config_filenames ) - if self.app.name == 'galaxy' and self._integrated_tool_panel_config_has_contents: - # Load self._tool_panel based on the order in self._integrated_tool_panel. - self._load_tool_panel() - if app.config.update_integrated_tool_panel: - # Write the current in-memory integrated_tool_panel to the integrated_tool_panel.xml file. - # This will cover cases where the Galaxy administrator manually edited one or more of the tool panel - # config files, adding or removing locally developed tools or workflows. The value of integrated_tool_panel - # will be False when things like functional tests are the caller. - self._write_integrated_tool_panel_config_file() - - def _init_tools_from_configs( self, config_filenames ): - """ Read through all tool config files and initialize tools in each - with init_tools_from_config below. - """ - self._tool_tag_manager.reset_tags() - config_filenames = listify( config_filenames ) - for config_filename in config_filenames: - if os.path.isdir( config_filename ): - directory_contents = sorted( os.listdir( config_filename ) ) - directory_config_files = [ config_file for config_file in directory_contents if config_file.endswith( ".xml" ) ] - config_filenames.remove( config_filename ) - config_filenames.extend( directory_config_files ) - for config_filename in config_filenames: - try: - self._init_tools_from_config( config_filename ) - except: - log.exception( "Error loading tools defined in config %s", config_filename ) - - def _init_tools_from_config( self, config_filename ): - """ - Read the configuration file and load each tool. The following tags are currently supported: - - .. raw:: xml - - <toolbox> - <tool file="data_source/upload.xml"/> # tools outside sections - <label text="Basic Tools" id="basic_tools" /> # labels outside sections - <workflow id="529fd61ab1c6cc36" /> # workflows outside sections - <section name="Get Data" id="getext"> # sections - <tool file="data_source/biomart.xml" /> # tools inside sections - <label text="In Section" id="in_section" /> # labels inside sections - <workflow id="adb5f5c93f827949" /> # workflows inside sections - </section> - </toolbox> - - """ - log.info( "Parsing the tool configuration %s" % config_filename ) - tree = parse_xml( config_filename ) - root = tree.getroot() - tool_path = root.get( 'tool_path' ) - if tool_path: - # We're parsing a shed_tool_conf file since we have a tool_path attribute. - parsing_shed_tool_conf = True - # Keep an in-memory list of xml elements to enable persistence of the changing tool config. - config_elems = [] - else: - parsing_shed_tool_conf = False - tool_path = self.__resolve_tool_path(tool_path, config_filename) - # Only load the panel_dict under certain conditions. - load_panel_dict = not self._integrated_tool_panel_config_has_contents - for _, elem in enumerate( root ): - index = self._index - self._index += 1 - if parsing_shed_tool_conf: - config_elems.append( elem ) - self.load_item( - elem, - tool_path=tool_path, - load_panel_dict=load_panel_dict, - guid=elem.get( 'guid' ), - index=index, - internal=True - ) - - if parsing_shed_tool_conf: - shed_tool_conf_dict = dict( config_filename=config_filename, - tool_path=tool_path, - config_elems=config_elems ) - self._dynamic_tool_confs.append( shed_tool_conf_dict ) - - def load_item( self, elem, tool_path, panel_dict=None, integrated_panel_dict=None, load_panel_dict=True, guid=None, index=None, internal=False ): - item_type = elem.tag - if item_type not in ['tool', 'section'] and not internal: - # External calls from tool shed code cannot load labels or tool - # directories. - return - - if panel_dict is None: - panel_dict = self._tool_panel - if integrated_panel_dict is None: - integrated_panel_dict = self._integrated_tool_panel - if item_type == 'tool': - self._load_tool_tag_set( elem, panel_dict=panel_dict, integrated_panel_dict=integrated_panel_dict, tool_path=tool_path, load_panel_dict=load_panel_dict, guid=guid, index=index ) - elif item_type == 'workflow': - self._load_workflow_tag_set( elem, panel_dict=panel_dict, integrated_panel_dict=integrated_panel_dict, load_panel_dict=load_panel_dict, index=index ) - elif item_type == 'section': - self._load_section_tag_set( elem, tool_path=tool_path, load_panel_dict=load_panel_dict, index=index ) - elif item_type == 'label': - self._load_label_tag_set( elem, panel_dict=panel_dict, integrated_panel_dict=integrated_panel_dict, load_panel_dict=load_panel_dict, index=index ) - elif item_type == 'tool_dir': - self._load_tooldir_tag_set( elem, panel_dict, tool_path, integrated_panel_dict, load_panel_dict=load_panel_dict ) - - def get_shed_config_dict_by_filename( self, filename, default=None ): - for shed_config_dict in self._dynamic_tool_confs: - if shed_config_dict[ 'config_filename' ] == filename: - return shed_config_dict - return default - - def update_shed_config( self, shed_conf, integrated_panel_changes=True ): - """ Update the in-memory descriptions of tools and write out the changes - to integrated tool panel unless we are just deactivating a tool (since - that doesn't affect that file). - """ - app = self.app - for index, my_shed_tool_conf in enumerate( self._dynamic_tool_confs ): - if shed_conf['config_filename'] == my_shed_tool_conf['config_filename']: - self._dynamic_tool_confs[ index ] = shed_conf - if integrated_panel_changes and app.config.update_integrated_tool_panel: - # Write the current in-memory version of the integrated_tool_panel.xml file to disk. - self._write_integrated_tool_panel_config_file() - app.reindex_tool_search() - - def get_section( self, section_id, new_label=None, create_if_needed=False ): - tool_panel_section_key = str( section_id ) - if tool_panel_section_key in self._tool_panel: - # Appending a tool to an existing section in toolbox._tool_panel - tool_section = self._tool_panel[ tool_panel_section_key ] - log.debug( "Appending to tool panel section: %s" % str( tool_section.name ) ) - elif create_if_needed: - # Appending a new section to toolbox._tool_panel - if new_label is None: - # This might add an ugly section label to the tool panel, but, oh well... - new_label = section_id - section_dict = { - 'name': new_label, - 'id': section_id, - 'version': '', - } - tool_section = ToolSection( section_dict ) - self._tool_panel[ tool_panel_section_key ] = tool_section - log.debug( "Loading new tool panel section: %s" % str( tool_section.name ) ) - else: - tool_section = None - return tool_panel_section_key, tool_section - - def get_integrated_section_for_tool( self, tool ): - tool_id = tool.id - for key, item_type, item in self._integrated_tool_panel.panel_items_iter(): - if item: - if item_type == panel_item_types.TOOL: - if item.id == tool_id: - return '', '' - if item_type == panel_item_types.SECTION: - section_id = item.id or '' - section_name = item.name or '' - for section_key, section_item_type, section_item in item.panel_items_iter(): - if section_item_type == panel_item_types.TOOL: - if section_item: - if section_item.id == tool_id: - return section_id, section_name - return None, None - - def __resolve_tool_path(self, tool_path, config_filename): - if not tool_path: - # Default to backward compatible config setting. - tool_path = self._tool_root_dir - else: - # Allow use of __tool_conf_dir__ in toolbox config files. - tool_conf_dir = os.path.dirname(config_filename) - tool_path_vars = {"tool_conf_dir": tool_conf_dir} - tool_path = string.Template(tool_path).safe_substitute(tool_path_vars) - return tool_path - - def __add_tool_to_tool_panel( self, tool, panel_component, section=False ): - # See if a version of this tool is already loaded into the tool panel. - # The value of panel_component will be a ToolSection (if the value of - # section=True) or self._tool_panel (if section=False). - tool_id = str( tool.id ) - tool = self._tools_by_id[ tool_id ] - if section: - panel_dict = panel_component.elems - else: - panel_dict = panel_component - - related_tool = self._lineage_in_panel( panel_dict, tool=tool ) - if related_tool: - if self._newer_tool( tool, related_tool ): - panel_dict.replace_tool( - previous_tool_id=related_tool.id, - new_tool_id=tool_id, - tool=tool, - ) - log.debug( "Loaded tool id: %s, version: %s into tool panel." % ( tool.id, tool.version ) ) - else: - inserted = False - index = self._integrated_tool_panel.index_of_tool_id( tool_id ) - if index: - panel_dict.insert_tool( index, tool ) - inserted = True - if not inserted: - # Check the tool's installed versions. - for tool_lineage_version in tool.lineage.get_versions(): - lineage_id = tool_lineage_version.id - index = self._integrated_tool_panel.index_of_tool_id( lineage_id ) - if index: - panel_dict.insert_tool( index, tool ) - inserted = True - if not inserted: - if ( - tool.guid is None or - tool.tool_shed is None or - tool.repository_name is None or - tool.repository_owner is None or - tool.installed_changeset_revision is None - ): - # We have a tool that was not installed from the Tool - # Shed, but is also not yet defined in - # integrated_tool_panel.xml, so append it to the tool - # panel. - panel_dict.append_tool( tool ) - log.debug( "Loaded tool id: %s, version: %s into tool panel.." % ( tool.id, tool.version ) ) - else: - # We are in the process of installing the tool. - - tool_lineage = self._lineage_map.get( tool_id ) - already_loaded = self._lineage_in_panel( panel_dict, tool_lineage=tool_lineage ) is not None - if not already_loaded: - # If the tool is not defined in integrated_tool_panel.xml, append it to the tool panel. - panel_dict.append_tool( tool ) - log.debug( "Loaded tool id: %s, version: %s into tool panel...." % ( tool.id, tool.version ) ) - - def _load_tool_panel( self ): - for key, item_type, val in self._integrated_tool_panel.panel_items_iter(): - if item_type == panel_item_types.TOOL: - tool_id = key.replace( 'tool_', '', 1 ) - if tool_id in self._tools_by_id: - self.__add_tool_to_tool_panel( val, self._tool_panel, section=False ) - elif item_type == panel_item_types.WORKFLOW: - workflow_id = key.replace( 'workflow_', '', 1 ) - if workflow_id in self._workflows_by_id: - workflow = self._workflows_by_id[ workflow_id ] - self._tool_panel[ key ] = workflow - log.debug( "Loaded workflow: %s %s" % ( workflow_id, workflow.name ) ) - elif item_type == panel_item_types.LABEL: - self._tool_panel[ key ] = val - elif item_type == panel_item_types.SECTION: - section_dict = { - 'id': val.id or '', - 'name': val.name or '', - 'version': val.version or '', - } - section = ToolSection( section_dict ) - log.debug( "Loading section: %s" % section_dict.get( 'name' ) ) - for section_key, section_item_type, section_val in val.panel_items_iter(): - if section_item_type == panel_item_types.TOOL: - tool_id = section_key.replace( 'tool_', '', 1 ) - if tool_id in self._tools_by_id: - self.__add_tool_to_tool_panel( section_val, section, section=True ) - elif section_item_type == panel_item_types.WORKFLOW: - workflow_id = section_key.replace( 'workflow_', '', 1 ) - if workflow_id in self._workflows_by_id: - workflow = self._workflows_by_id[ workflow_id ] - section.elems[ section_key ] = workflow - log.debug( "Loaded workflow: %s %s" % ( workflow_id, workflow.name ) ) - elif section_item_type == panel_item_types.LABEL: - if section_val: - section.elems[ section_key ] = section_val - log.debug( "Loaded label: %s" % ( section_val.text ) ) - self._tool_panel[ key ] = section - - def _load_integrated_tool_panel_keys( self ): - """ - Load the integrated tool panel keys, setting values for tools and - workflows to None. The values will be reset when the various tool - panel config files are parsed, at which time the tools and workflows - are loaded. - """ - tree = parse_xml( self._integrated_tool_panel_config ) - root = tree.getroot() - for elem in root: - key = elem.get( 'id' ) - if elem.tag == 'tool': - self._integrated_tool_panel.stub_tool( key ) - elif elem.tag == 'workflow': - self._integrated_tool_panel.stub_workflow( key ) - elif elem.tag == 'section': - section = ToolSection( elem ) - for section_elem in elem: - section_id = section_elem.get( 'id' ) - if section_elem.tag == 'tool': - section.elems.stub_tool( section_id ) - elif section_elem.tag == 'workflow': - section.elems.stub_workflow( section_id ) - elif section_elem.tag == 'label': - section.elems.stub_label( section_id ) - self._integrated_tool_panel.append_section( key, section ) - elif elem.tag == 'label': - section.stub_label( key ) - - def _write_integrated_tool_panel_config_file( self ): - """ - Write the current in-memory version of the integrated_tool_panel.xml file to disk. Since Galaxy administrators - use this file to manage the tool panel, we'll not use xml_to_string() since it doesn't write XML quite right. - """ - fd, filename = tempfile.mkstemp() - os.write( fd, '<?xml version="1.0"?>\n' ) - os.write( fd, '<toolbox>\n' ) - os.write( fd, ' <!--\n ') - os.write( fd, '\n '.join( [ l for l in INTEGRATED_TOOL_PANEL_DESCRIPTION.split("\n") if l ] ) ) - os.write( fd, '\n -->\n') - for key, item_type, item in self._integrated_tool_panel.panel_items_iter(): - if item: - if item_type == panel_item_types.TOOL: - os.write( fd, ' <tool id="%s" />\n' % item.id ) - elif item_type == panel_item_types.WORKFLOW: - os.write( fd, ' <workflow id="%s" />\n' % item.id ) - elif item_type == panel_item_types.LABEL: - label_id = item.id or '' - label_text = item.text or '' - label_version = item.version or '' - os.write( fd, ' <label id="%s" text="%s" version="%s" />\n' % ( label_id, label_text, label_version ) ) - elif item_type == panel_item_types.SECTION: - section_id = item.id or '' - section_name = item.name or '' - section_version = item.version or '' - os.write( fd, ' <section id="%s" name="%s" version="%s">\n' % ( section_id, section_name, section_version ) ) - for section_key, section_item_type, section_item in item.panel_items_iter(): - if section_item_type == panel_item_types.TOOL: - if section_item: - os.write( fd, ' <tool id="%s" />\n' % section_item.id ) - elif section_item_type == panel_item_types.WORKFLOW: - if section_item: - os.write( fd, ' <workflow id="%s" />\n' % section_item.id ) - elif section_item_type == panel_item_types.LABEL: - if section_item: - label_id = section_item.id or '' - label_text = section_item.text or '' - label_version = section_item.version or '' - os.write( fd, ' <label id="%s" text="%s" version="%s" />\n' % ( label_id, label_text, label_version ) ) - os.write( fd, ' </section>\n' ) - os.write( fd, '</toolbox>\n' ) - os.close( fd ) - shutil.move( filename, os.path.abspath( self._integrated_tool_panel_config ) ) - os.chmod( self._integrated_tool_panel_config, 0644 ) - - def get_tool( self, tool_id, tool_version=None, get_all_versions=False, exact=False ): - """Attempt to locate a tool in the tool box.""" - if tool_version: - tool_version = str( tool_version ) - - if get_all_versions and exact: - raise AssertionError("Cannot specify get_tool with both get_all_versions and exact as True") - - if tool_id in self._tools_by_id and not get_all_versions: - if tool_version and tool_version in self._tool_versions_by_id[ tool_id ]: - return self._tool_versions_by_id[ tool_id ][ tool_version ] - #tool_id exactly matches an available tool by id (which is 'old' tool_id or guid) - return self._tools_by_id[ tool_id ] - #exact tool id match not found, or all versions requested, search for other options, e.g. migrated tools or different versions - rval = [] - tool_lineage = self._lineage_map.get( tool_id ) - if tool_lineage: - lineage_tool_versions = tool_lineage.get_versions( ) - for lineage_tool_version in lineage_tool_versions: - lineage_tool = self._tool_from_lineage_version( lineage_tool_version ) - if lineage_tool: - rval.append( lineage_tool ) - if not rval: - #still no tool, do a deeper search and try to match by old ids - for tool in self._tools_by_id.itervalues(): - if tool.old_id == tool_id: - rval.append( tool ) - if rval: - if get_all_versions: - return rval - else: - if tool_version: - #return first tool with matching version - for tool in rval: - if tool.version == tool_version: - return tool - #No tool matches by version, simply return the first available tool found - return rval[0] - #We now likely have a Toolshed guid passed in, but no supporting database entries - #If the tool exists by exact id and is loaded then provide exact match within a list - if tool_id in self._tools_by_id: - return[ self._tools_by_id[ tool_id ] ] - return None - - def has_tool( self, tool_id, exact=False ): - return self.get_tool( tool_id, exact=exact ) is not None - - def get_tool_id( self, tool_id ): - """ Take a tool id (potentially from a different Galaxy instance or that - is no longer loaded - and find the closest match to the currently loaded - tools (using get_tool for inexact matches which currently returns the oldest - tool shed installed tool with the same short id). - """ - if tool_id not in self._tools_by_id: - tool = self.get_tool( tool_id ) - if tool: - tool_id = tool.id - else: - tool_id = None - # else exact match - leave unmodified. - return tool_id - - def get_loaded_tools_by_lineage( self, tool_id ): - """Get all loaded tools associated by lineage to the tool whose id is tool_id.""" - tool_lineage = self._lineage_map.get( tool_id ) - if tool_lineage: - lineage_tool_versions = tool_lineage.get_versions( ) - available_tool_versions = [] - for lineage_tool_version in lineage_tool_versions: - tool = self._tool_from_lineage_version( lineage_tool_version ) - if tool: - available_tool_versions.append( tool ) - return available_tool_versions - else: - if tool_id in self._tools_by_id: - tool = self._tools_by_id[ tool_id ] - return [ tool ] - return [] @property def tools_by_id( self ): # Deprecated method, TODO - eliminate calls to this in test/. return self._tools_by_id - def tools( self ): - return self._tools_by_id.iteritems() - - def __elem_to_tool_shed_repository( self, elem ): - # The tool is contained in an installed tool shed repository, so load - # the tool only if the repository has not been marked deleted. - tool_shed = elem.find( "tool_shed" ).text - repository_name = elem.find( "repository_name" ).text - repository_owner = elem.find( "repository_owner" ).text - installed_changeset_revision_elem = elem.find( "installed_changeset_revision" ) - if installed_changeset_revision_elem is None: - # Backward compatibility issue - the tag used to be named 'changeset_revision'. - installed_changeset_revision_elem = elem.find( "changeset_revision" ) - installed_changeset_revision = installed_changeset_revision_elem.text - tool_shed_repository = self.__get_tool_shed_repository( tool_shed, - repository_name, - repository_owner, - installed_changeset_revision ) - return tool_shed_repository - - def __get_tool_shed_repository( self, tool_shed, name, owner, installed_changeset_revision ): - # We store only the port, if one exists, in the database. - tool_shed = common_util.remove_protocol_from_tool_shed_url( tool_shed ) - return self.app.install_model.context.query( self.app.install_model.ToolShedRepository ) \ - .filter( and_( self.app.install_model.ToolShedRepository.table.c.tool_shed == tool_shed, - self.app.install_model.ToolShedRepository.table.c.name == name, - self.app.install_model.ToolShedRepository.table.c.owner == owner, - self.app.install_model.ToolShedRepository.table.c.installed_changeset_revision == installed_changeset_revision ) ) \ - .first() - - def get_tool_components( self, tool_id, tool_version=None, get_loaded_tools_by_lineage=False, set_selected=False ): - """ - Retrieve all loaded versions of a tool from the toolbox and return a select list enabling - selection of a different version, the list of the tool's loaded versions, and the specified tool. - """ - toolbox = self - tool_version_select_field = None - tools = [] - tool = None - # Backwards compatibility for datasource tools that have default tool_id configured, but which - # are now using only GALAXY_URL. - tool_ids = listify( tool_id ) - for tool_id in tool_ids: - if get_loaded_tools_by_lineage: - tools = toolbox.get_loaded_tools_by_lineage( tool_id ) - else: - tools = toolbox.get_tool( tool_id, tool_version=tool_version, get_all_versions=True ) - if tools: - tool = toolbox.get_tool( tool_id, tool_version=tool_version, get_all_versions=False ) - if len( tools ) > 1: - tool_version_select_field = self.build_tool_version_select_field( tools, tool.id, set_selected ) - break - return tool_version_select_field, tools, tool - - def dynamic_confs( self, include_migrated_tool_conf=False ): - confs = [] - for dynamic_tool_conf_dict in self._dynamic_tool_confs: - dynamic_tool_conf_filename = dynamic_tool_conf_dict[ 'config_filename' ] - if include_migrated_tool_conf or (dynamic_tool_conf_filename != self.app.config.migrated_tools_config): - confs.append( dynamic_tool_conf_dict ) - return confs - - def dynamic_conf_filenames( self, include_migrated_tool_conf=False ): - """ Return list of dynamic tool configuration filenames (shed_tools). - These must be used with various dynamic tool configuration update - operations (e.g. with update_shed_config). - """ - for dynamic_tool_conf_dict in self.dynamic_confs( include_migrated_tool_conf=include_migrated_tool_conf ): - yield dynamic_tool_conf_dict[ 'config_filename' ] - - def build_tool_version_select_field( self, tools, tool_id, set_selected ): - """Build a SelectField whose options are the ids for the received list of tools.""" - options = [] - refresh_on_change_values = [] - for tool in tools: - options.insert( 0, ( tool.version, tool.id ) ) - refresh_on_change_values.append( tool.id ) - select_field = SelectField( name='tool_id', refresh_on_change=True, refresh_on_change_values=refresh_on_change_values ) - for option_tup in options: - selected = set_selected and option_tup[ 1 ] == tool_id - if selected: - select_field.add_option( 'version %s' % option_tup[ 0 ], option_tup[ 1 ], selected=True ) - else: - select_field.add_option( 'version %s' % option_tup[ 0 ], option_tup[ 1 ] ) - return select_field - - def remove_from_panel( self, tool_id, section_key='', remove_from_config=True ): - - def remove_from_dict( has_elems, integrated_has_elems ): - tool_key = 'tool_%s' % str( tool_id ) - available_tool_versions = self.get_loaded_tools_by_lineage( tool_id ) - if tool_key in has_elems: - if available_tool_versions: - available_tool_versions.reverse() - replacement_tool_key = None - replacement_tool_version = None - # Since we are going to remove the tool from the section, replace it with - # the newest loaded version of the tool. - for available_tool_version in available_tool_versions: - available_tool_section_id, available_tool_section_name = available_tool_version.get_panel_section() - # I suspect "available_tool_version.id in has_elems.keys()" doesn't - # belong in the following line or at least I don't understand what - # purpose it might serve. -John - if available_tool_version.id in has_elems.keys() or (available_tool_section_id == section_key): - replacement_tool_key = 'tool_%s' % str( available_tool_version.id ) - replacement_tool_version = available_tool_version - break - if replacement_tool_key and replacement_tool_version: - # Get the index of the tool_key in the tool_section. - for tool_panel_index, key in enumerate( has_elems.keys() ): - if key == tool_key: - break - # Remove the tool from the tool panel. - del has_elems[ tool_key ] - # Add the replacement tool at the same location in the tool panel. - has_elems.insert( tool_panel_index, - replacement_tool_key, - replacement_tool_version ) - else: - del has_elems[ tool_key ] - else: - del has_elems[ tool_key ] - if remove_from_config: - itegrated_items = integrated_has_elems.panel_items() - if tool_key in itegrated_items: - del itegrated_items[ tool_key ] - - if section_key: - _, tool_section = self.get_section( section_key ) - if tool_section: - remove_from_dict( tool_section.elems, self._integrated_tool_panel.get( section_key, {} ) ) - else: - remove_from_dict( self._tool_panel, self._integrated_tool_panel ) - - def _load_tool_tag_set( self, elem, panel_dict, integrated_panel_dict, tool_path, load_panel_dict, guid=None, index=None ): - try: - path = elem.get( "file" ) - repository_id = None - - tool_shed_repository = None - can_load_into_panel_dict = True - if guid is not None: - # The tool is contained in an installed tool shed repository, so load - # the tool only if the repository has not been marked deleted. - tool_shed_repository = self.__elem_to_tool_shed_repository( elem ) - if tool_shed_repository: - # Only load tools if the repository is not deactivated or uninstalled. - can_load_into_panel_dict = not tool_shed_repository.deleted - repository_id = self.app.security.encode_id( tool_shed_repository.id ) - # Else there is not yet a tool_shed_repository record, we're in the process of installing - # a new repository, so any included tools can be loaded into the tool panel. - tool = self.load_tool( os.path.join( tool_path, path ), guid=guid, repository_id=repository_id ) - if string_as_bool(elem.get( 'hidden', False )): - tool.hidden = True - key = 'tool_%s' % str( tool.id ) - if can_load_into_panel_dict: - if guid is not None: - tool.tool_shed = tool_shed_repository.tool_shed - tool.repository_name = tool_shed_repository.name - tool.repository_owner = tool_shed_repository.owner - tool.installed_changeset_revision = tool_shed_repository.installed_changeset_revision - tool.guid = guid - tool.version = elem.find( "version" ).text - # Make sure the tool has a tool_version. - tool_lineage = self._lineage_map.register( tool, tool_shed_repository=tool_shed_repository ) - # Load the tool's lineage ids. - tool.lineage = tool_lineage - self._tool_tag_manager.handle_tags( tool.id, elem ) - self.__add_tool( tool, load_panel_dict, panel_dict ) - # Always load the tool into the integrated_panel_dict, or it will not be included in the integrated_tool_panel.xml file. - integrated_panel_dict.update_or_append( index, key, tool ) - except IOError: - log.error( "Error reading tool configuration file from path: %s." % path ) - except Exception: - log.exception( "Error reading tool from path: %s" % path ) - - def __add_tool( self, tool, load_panel_dict, panel_dict ): - # Allow for the same tool to be loaded into multiple places in the - # tool panel. We have to handle the case where the tool is contained - # in a repository installed from the tool shed, and the Galaxy - # administrator has retrieved updates to the installed repository. In - # this case, the tool may have been updated, but the version was not - # changed, so the tool should always be reloaded here. We used to - # only load the tool if it was not found in self._tools_by_id, but - # performing that check did not enable this scenario. - self.register_tool( tool ) - if load_panel_dict: - self.__add_tool_to_tool_panel( tool, panel_dict, section=isinstance( panel_dict, ToolSection ) ) - - def _load_workflow_tag_set( self, elem, panel_dict, integrated_panel_dict, load_panel_dict, index=None ): - try: - # TODO: should id be encoded? - workflow_id = elem.get( 'id' ) - workflow = self._load_workflow( workflow_id ) - self._workflows_by_id[ workflow_id ] = workflow - key = 'workflow_' + workflow_id - if load_panel_dict: - panel_dict[ key ] = workflow - # Always load workflows into the integrated_panel_dict. - integrated_panel_dict.update_or_append( index, key, workflow ) - except: - log.exception( "Error loading workflow: %s" % workflow_id ) - - def _load_label_tag_set( self, elem, panel_dict, integrated_panel_dict, load_panel_dict, index=None ): - label = ToolSectionLabel( elem ) - key = 'label_' + label.id - if load_panel_dict: - panel_dict[ key ] = label - integrated_panel_dict.update_or_append( index, key, label ) - - def _load_section_tag_set( self, elem, tool_path, load_panel_dict, index=None ): - key = elem.get( "id" ) - if key in self._tool_panel: - section = self._tool_panel[ key ] - elems = section.elems - else: - section = ToolSection( elem ) - elems = section.elems - if key in self._integrated_tool_panel: - integrated_section = self._integrated_tool_panel[ key ] - integrated_elems = integrated_section.elems - else: - integrated_section = ToolSection( elem ) - integrated_elems = integrated_section.elems - for sub_index, sub_elem in enumerate( elem ): - self.load_item( - sub_elem, - tool_path=tool_path, - panel_dict=elems, - integrated_panel_dict=integrated_elems, - load_panel_dict=load_panel_dict, - guid=sub_elem.get( 'guid' ), - index=sub_index, - internal=True, - ) - if load_panel_dict: - self._tool_panel[ key ] = section - # Always load sections into the integrated_tool_panel. - self._integrated_tool_panel.update_or_append( index, key, integrated_section ) - - def _load_tooldir_tag_set(self, sub_elem, elems, tool_path, integrated_elems, load_panel_dict): - directory = os.path.join( tool_path, sub_elem.attrib.get("dir") ) - recursive = string_as_bool( sub_elem.attrib.get("recursive", True) ) - self.__watch_directory( directory, elems, integrated_elems, load_panel_dict, recursive ) - - def __watch_directory( self, directory, elems, integrated_elems, load_panel_dict, recursive): - - def quick_load( tool_file, async=True ): - try: - tool = self.load_tool( tool_file ) - self.__add_tool( tool, load_panel_dict, elems ) - # Always load the tool into the integrated_panel_dict, or it will not be included in the integrated_tool_panel.xml file. - key = 'tool_%s' % str( tool.id ) - integrated_elems[ key ] = tool - - if async: - self._load_tool_panel() - - if self.app.config.update_integrated_tool_panel: - # Write the current in-memory integrated_tool_panel to the integrated_tool_panel.xml file. - # This will cover cases where the Galaxy administrator manually edited one or more of the tool panel - # config files, adding or removing locally developed tools or workflows. The value of integrated_tool_panel - # will be False when things like functional tests are the caller. - self._write_integrated_tool_panel_config_file() - return tool.id - except Exception: - log.exception("Failed to load potential tool %s." % tool_file) - return None - - tool_loaded = False - for name in os.listdir( directory ): - child_path = os.path.join(directory, name) - if os.path.isdir(child_path) and recursive: - self.__watch_directory(child_path, elems, integrated_elems, load_panel_dict, recursive) - elif name.endswith( ".xml" ): - quick_load( child_path, async=False ) - tool_loaded = True - if tool_loaded: - self._tool_watcher.watch_directory( directory, quick_load ) - - def load_tool( self, config_file, guid=None, repository_id=None, **kwds ): - """Load a single tool from the file named by `config_file` and return an instance of `Tool`.""" - # Parse XML configuration file and get the root element + def create_tool( self, config_file, repository_id=None, guid=None, **kwds ): tool_source = get_tool_source( config_file, getattr( self.app.config, "enable_beta_tool_formats", False ) ) # Allow specifying a different tool subclass to instantiate tool_module = tool_source.parse_tool_module() @@ -904,254 +145,8 @@ ToolClass = Tool tool = ToolClass( config_file, tool_source, self.app, guid=guid, repository_id=repository_id, **kwds ) - tool_id = tool.id - if not tool_id.startswith("__"): - # do not monitor special tools written to tmp directory - no reason - # to monitor such a large directory. - self._tool_watcher.watch_file( config_file, tool.id ) return tool - def load_hidden_tool( self, config_file, **kwds ): - """ Load a hidden tool (in this context meaning one that does not - appear in the tool panel) and register it in _tools_by_id. - """ - tool = self.load_tool( config_file, **kwds ) - self.register_tool( tool ) - return tool - - def register_tool( self, tool ): - tool_id = tool.id - version = tool.version or None - if tool_id not in self._tool_versions_by_id: - self._tool_versions_by_id[ tool_id ] = { version: tool } - else: - self._tool_versions_by_id[ tool_id ][ version ] = tool - if tool_id in self._tools_by_id: - related_tool = self._tools_by_id[ tool_id ] - # This one becomes the default un-versioned tool - # if newer. - if self._newer_tool( tool, related_tool ): - self._tools_by_id[ tool_id ] = tool - else: - self._tools_by_id[ tool_id ] = tool - - def package_tool( self, trans, tool_id ): - """ - Create a tarball with the tool's xml, help images, and test data. - :param trans: the web transaction - :param tool_id: the tool ID from app.toolbox - :returns: tuple of tarball filename, success True/False, message/None - """ - # Make sure the tool is actually loaded. - if tool_id not in self._tools_by_id: - return None, False, "No tool with id %s" % escape( tool_id ) - else: - tool = self._tools_by_id[ tool_id ] - tarball_files = [] - temp_files = [] - tool_xml = file( os.path.abspath( tool.config_file ), 'r' ).read() - # Retrieve tool help images and rewrite the tool's xml into a temporary file with the path - # modified to be relative to the repository root. - image_found = False - if tool.help is not None: - tool_help = tool.help._source - # Check each line of the rendered tool help for an image tag that points to a location under static/ - for help_line in tool_help.split( '\n' ): - image_regex = re.compile( 'img alt="[^"]+" src="\${static_path}/([^"]+)"' ) - matches = re.search( image_regex, help_line ) - if matches is not None: - tool_help_image = matches.group(1) - tarball_path = tool_help_image - filesystem_path = os.path.abspath( os.path.join( trans.app.config.root, 'static', tool_help_image ) ) - if os.path.exists( filesystem_path ): - tarball_files.append( ( filesystem_path, tarball_path ) ) - image_found = True - tool_xml = tool_xml.replace( '${static_path}/%s' % tarball_path, tarball_path ) - # If one or more tool help images were found, add the modified tool XML to the tarball instead of the original. - if image_found: - fd, new_tool_config = tempfile.mkstemp( suffix='.xml' ) - os.close( fd ) - file( new_tool_config, 'w' ).write( tool_xml ) - tool_tup = ( os.path.abspath( new_tool_config ), os.path.split( tool.config_file )[-1] ) - temp_files.append( os.path.abspath( new_tool_config ) ) - else: - tool_tup = ( os.path.abspath( tool.config_file ), os.path.split( tool.config_file )[-1] ) - tarball_files.append( tool_tup ) - # TODO: This feels hacky. - tool_command = tool.command.strip().split()[0] - tool_path = os.path.dirname( os.path.abspath( tool.config_file ) ) - # Add the tool XML to the tuple that will be used to populate the tarball. - if os.path.exists( os.path.join( tool_path, tool_command ) ): - tarball_files.append( ( os.path.join( tool_path, tool_command ), tool_command ) ) - # Find and add macros and code files. - for external_file in tool.get_externally_referenced_paths( os.path.abspath( tool.config_file ) ): - external_file_abspath = os.path.abspath( os.path.join( tool_path, external_file ) ) - tarball_files.append( ( external_file_abspath, external_file ) ) - if os.path.exists( os.path.join( tool_path, "Dockerfile" ) ): - tarball_files.append( ( os.path.join( tool_path, "Dockerfile" ), "Dockerfile" ) ) - # Find tests, and check them for test data. - tests = tool.tests - if tests is not None: - for test in tests: - # Add input file tuples to the list. - for input in test.inputs: - for input_value in test.inputs[ input ]: - input_path = os.path.abspath( os.path.join( 'test-data', input_value ) ) - if os.path.exists( input_path ): - td_tup = ( input_path, os.path.join( 'test-data', input_value ) ) - tarball_files.append( td_tup ) - # And add output file tuples to the list. - for label, filename, _ in test.outputs: - output_filepath = os.path.abspath( os.path.join( 'test-data', filename ) ) - if os.path.exists( output_filepath ): - td_tup = ( output_filepath, os.path.join( 'test-data', filename ) ) - tarball_files.append( td_tup ) - for param in tool.input_params: - # Check for tool data table definitions. - if hasattr( param, 'options' ): - if hasattr( param.options, 'tool_data_table' ): - data_table = param.options.tool_data_table - if hasattr( data_table, 'filenames' ): - data_table_definitions = [] - for data_table_filename in data_table.filenames: - # FIXME: from_shed_config seems to always be False. - if not data_table.filenames[ data_table_filename ][ 'from_shed_config' ]: - tar_file = data_table.filenames[ data_table_filename ][ 'filename' ] + '.sample' - sample_file = os.path.join( data_table.filenames[ data_table_filename ][ 'tool_data_path' ], - tar_file ) - # Use the .sample file, if one exists. If not, skip this data table. - if os.path.exists( sample_file ): - tarfile_path, tarfile_name = os.path.split( tar_file ) - tarfile_path = os.path.join( 'tool-data', tarfile_name ) - tarball_files.append( ( sample_file, tarfile_path ) ) - data_table_definitions.append( data_table.xml_string ) - if len( data_table_definitions ) > 0: - # Put the data table definition XML in a temporary file. - table_definition = '<?xml version="1.0" encoding="utf-8"?>\n<tables>\n %s</tables>' - table_definition = table_definition % '\n'.join( data_table_definitions ) - fd, table_conf = tempfile.mkstemp() - os.close( fd ) - file( table_conf, 'w' ).write( table_definition ) - tarball_files.append( ( table_conf, os.path.join( 'tool-data', 'tool_data_table_conf.xml.sample' ) ) ) - temp_files.append( table_conf ) - # Create the tarball. - fd, tarball_archive = tempfile.mkstemp( suffix='.tgz' ) - os.close( fd ) - tarball = tarfile.open( name=tarball_archive, mode='w:gz' ) - # Add the files from the previously generated list. - for fspath, tarpath in tarball_files: - tarball.add( fspath, arcname=tarpath ) - tarball.close() - # Delete any temporary files that were generated. - for temp_file in temp_files: - os.remove( temp_file ) - return tarball_archive, True, None - return None, False, "An unknown error occurred." - - def reload_tool_by_id( self, tool_id ): - """ - Attempt to reload the tool identified by 'tool_id', if successful - replace the old tool. - """ - if tool_id not in self._tools_by_id: - message = "No tool with id %s" % escape( tool_id ) - status = 'error' - else: - old_tool = self._tools_by_id[ tool_id ] - new_tool = self.load_tool( old_tool.config_file ) - # The tool may have been installed from a tool shed, so set the tool shed attributes. - # Since the tool version may have changed, we don't override it here. - new_tool.id = old_tool.id - new_tool.guid = old_tool.guid - new_tool.tool_shed = old_tool.tool_shed - new_tool.repository_name = old_tool.repository_name - new_tool.repository_owner = old_tool.repository_owner - new_tool.installed_changeset_revision = old_tool.installed_changeset_revision - new_tool.old_id = old_tool.old_id - # Replace old_tool with new_tool in self._tool_panel - tool_key = 'tool_' + tool_id - for key, val in self._tool_panel.items(): - if key == tool_key: - self._tool_panel[ key ] = new_tool - break - elif key.startswith( 'section' ): - if tool_key in val.elems: - self._tool_panel[ key ].elems[ tool_key ] = new_tool - break - self._tools_by_id[ tool_id ] = new_tool - message = "Reloaded the tool:<br/>" - message += "<b>name:</b> %s<br/>" % old_tool.name - message += "<b>id:</b> %s<br/>" % old_tool.id - message += "<b>version:</b> %s" % old_tool.version - status = 'done' - return message, status - - def remove_tool_by_id( self, tool_id, remove_from_panel=True ): - """ - Attempt to remove the tool identified by 'tool_id'. Ignores - tool lineage - so to remove a tool with potentially multiple - versions send remove_from_panel=False and handle the logic of - promoting the next newest version of the tool into the panel - if needed. - """ - if tool_id not in self._tools_by_id: - message = "No tool with id %s" % escape( tool_id ) - status = 'error' - else: - tool = self._tools_by_id[ tool_id ] - del self._tools_by_id[ tool_id ] - if remove_from_panel: - tool_key = 'tool_' + tool_id - for key, val in self._tool_panel.items(): - if key == tool_key: - del self._tool_panel[ key ] - break - elif key.startswith( 'section' ): - if tool_key in val.elems: - del self._tool_panel[ key ].elems[ tool_key ] - break - if tool_id in self.data_manager_tools: - del self.data_manager_tools[ tool_id ] - #TODO: do we need to manually remove from the integrated panel here? - message = "Removed the tool:<br/>" - message += "<b>name:</b> %s<br/>" % tool.name - message += "<b>id:</b> %s<br/>" % tool.id - message += "<b>version:</b> %s" % tool.version - status = 'done' - return message, status - - def get_sections( self ): - for k, v in self._tool_panel.items(): - if isinstance( v, ToolSection ): - yield (v.id, v.name) - - def find_section_id( self, tool_panel_section_id ): - """ - Find the section ID referenced by the key or return '' indicating - no such section id. - """ - if not tool_panel_section_id: - tool_panel_section_id = '' - else: - if tool_panel_section_id not in self._tool_panel: - # Hack introduced without comment in a29d54619813d5da992b897557162a360b8d610c- - # not sure why it is needed. - fixed_tool_panel_section_id = 'section_%s' % tool_panel_section_id - if fixed_tool_panel_section_id in self._tool_panel: - tool_panel_section_id = fixed_tool_panel_section_id - else: - tool_panel_section_id = '' - return tool_panel_section_id - - def _load_workflow( self, workflow_id ): - """ - Return an instance of 'Workflow' identified by `id`, - which is encoded in the tool panel. - """ - id = self.app.security.decode_id( workflow_id ) - stored = self.app.model.context.query( self.app.model.StoredWorkflow ).get( id ) - return stored.latest_workflow - def _init_dependency_manager( self ): self.dependency_manager = build_dependency_manager( self.app.config ) @@ -1162,138 +157,6 @@ if isinstance( tool.tool_action, UploadToolAction ): self.reload_tool_by_id( tool_id ) - @property - def sa_session( self ): - """ - Returns a SQLAlchemy session - """ - return self.app.model.context - - def tool_panel_contents( self, trans, **kwds ): - """ Filter tool_panel contents for displaying for user. - """ - context = Bunch( toolbox=self, trans=trans ) - filters = self._filter_factory.build_filters( trans ) - for _, item_type, elt in self._tool_panel.panel_items_iter(): - elt = _filter_for_panel( elt, item_type, filters, context ) - if elt: - yield elt - - def to_dict( self, trans, in_panel=True, **kwds ): - """ - to_dict toolbox. - """ - if in_panel: - panel_elts = list( self.tool_panel_contents( trans, **kwds ) ) - # Produce panel. - rval = [] - kwargs = dict( - trans=trans, - link_details=True - ) - for elt in panel_elts: - rval.append( elt.to_dict( **kwargs ) ) - else: - tools = [] - for id, tool in self._tools_by_id.items(): - tools.append( tool.to_dict( trans, link_details=True ) ) - rval = tools - - return rval - - def _lineage_in_panel( self, panel_dict, tool=None, tool_lineage=None ): - """ If tool with same lineage already in panel (or section) - find - and return it. Otherwise return None. - """ - if tool_lineage is None: - assert tool is not None - if not hasattr( tool, "lineage" ): - return None - tool_lineage = tool.lineage - lineage_tool_versions = tool_lineage.get_versions( reverse=True ) - for lineage_tool_version in lineage_tool_versions: - lineage_tool = self._tool_from_lineage_version( lineage_tool_version ) - if lineage_tool: - lineage_id = lineage_tool.id - if panel_dict.has_tool_with_id( lineage_id ): - return panel_dict.get_tool_with_id( lineage_id ) - return None - - def _newer_tool( self, tool1, tool2 ): - """ Return True if tool1 is considered "newer" given its own lineage - description. - """ - if not hasattr( tool1, "lineage" ): - return True - lineage_tool_versions = tool1.lineage.get_versions() - for lineage_tool_version in lineage_tool_versions: - lineage_tool = self._tool_from_lineage_version( lineage_tool_version ) - if lineage_tool is tool1: - return False - if lineage_tool is tool2: - return True - return True - - def _tool_from_lineage_version( self, lineage_tool_version ): - if lineage_tool_version.id_based: - return self._tools_by_id.get( lineage_tool_version.id, None ) - else: - return self._tool_versions_by_id.get( lineage_tool_version.id, {} ).get( lineage_tool_version.version, None ) - - -def _filter_for_panel( item, item_type, filters, context ): - """ - Filters tool panel elements so that only those that are compatible - with provided filters are kept. - """ - def _apply_filter( filter_item, filter_list ): - for filter_method in filter_list: - if not filter_method( context, filter_item ): - return False - return True - if item_type == panel_item_types.TOOL: - if _apply_filter( item, filters[ 'tool' ] ): - return item - elif item_type == panel_item_types.LABEL: - if _apply_filter( item, filters[ 'label' ] ): - return item - elif item_type == panel_item_types.SECTION: - # Filter section item-by-item. Only show a label if there are - # non-filtered tools below it. - - if _apply_filter( item, filters[ 'section' ] ): - cur_label_key = None - tools_under_label = False - filtered_elems = item.elems.copy() - for key, section_item_type, section_item in item.panel_items_iter(): - if section_item_type == panel_item_types.TOOL: - # Filter tool. - if _apply_filter( section_item, filters[ 'tool' ] ): - tools_under_label = True - else: - del filtered_elems[ key ] - elif section_item_type == panel_item_types.LABEL: - # If there is a label and it does not have tools, - # remove it. - if ( cur_label_key and not tools_under_label ) or not _apply_filter( section_item, filters[ 'label' ] ): - del filtered_elems[ cur_label_key ] - - # Reset attributes for new label. - cur_label_key = key - tools_under_label = False - - # Handle last label. - if cur_label_key and not tools_under_label: - del filtered_elems[ cur_label_key ] - - # Only return section if there are elements. - if len( filtered_elems ) != 0: - copy = item.copy() - copy.elems = filtered_elems - return copy - - return None - class DefaultToolState( object ): """ diff -r 24aa2e802a351882695cb98d772001cc2a1316d8 -r edacabfca615542ca435a97fc1afcc99daa470d5 lib/galaxy/tools/toolbox/__init__.py --- a/lib/galaxy/tools/toolbox/__init__.py +++ b/lib/galaxy/tools/toolbox/__init__.py @@ -1,16 +1,15 @@ """ API for this module containing functionality related to the tool box. """ -from .tags import tool_tag_manager -from .panel import ToolPanelElements +from .panel import panel_item_types from .panel import ToolSection from .panel import ToolSectionLabel -from .panel import panel_item_types + +from .base import AbstractToolBox __all__ = [ - "ToolPanelElements", - "tool_tag_manager", "ToolSection", "ToolSectionLabel", - "panel_item_types" + "panel_item_types", + "AbstractToolBox", ] This diff is so big that we needed to truncate the remainder. https://bitbucket.org/galaxy/galaxy-central/commits/cb673f24cd75/ Changeset: cb673f24cd75 User: jmchilton Date: 2014-12-31 23:21:10+00:00 Summary: Rework galaxy.tools' galaxy.jobs import. This version of the import I believe is less likely to cause circular import problems (http://effbot.org/zone/import-confusion.htm). Affected #: 1 file diff -r edacabfca615542ca435a97fc1afcc99daa470d5 -r cb673f24cd75ef491aa850373704535691b0a9dd lib/galaxy/tools/__init__.py --- a/lib/galaxy/tools/__init__.py +++ b/lib/galaxy/tools/__init__.py @@ -25,7 +25,7 @@ from mako.template import Template from paste import httpexceptions -from galaxy import jobs, model +from galaxy import model from galaxy.datatypes.metadata import JobExternalOutputMetadataWrapper from galaxy import exceptions from galaxy.tools.actions import DefaultToolAction @@ -69,6 +69,7 @@ DatasetListWrapper, DatasetCollectionWrapper, ) +import galaxy.jobs log = logging.getLogger( __name__ ) @@ -2127,7 +2128,7 @@ json_params = {} json_params[ 'param_dict' ] = self._prepare_json_param_dict( param_dict ) # it would probably be better to store the original incoming parameters here, instead of the Galaxy modified ones? json_params[ 'output_data' ] = [] - json_params[ 'job_config' ] = dict( GALAXY_DATATYPES_CONF_FILE=param_dict.get( 'GALAXY_DATATYPES_CONF_FILE' ), GALAXY_ROOT_DIR=param_dict.get( 'GALAXY_ROOT_DIR' ), TOOL_PROVIDED_JOB_METADATA_FILE=jobs.TOOL_PROVIDED_JOB_METADATA_FILE ) + json_params[ 'job_config' ] = dict( GALAXY_DATATYPES_CONF_FILE=param_dict.get( 'GALAXY_DATATYPES_CONF_FILE' ), GALAXY_ROOT_DIR=param_dict.get( 'GALAXY_ROOT_DIR' ), TOOL_PROVIDED_JOB_METADATA_FILE=galaxy.jobs.TOOL_PROVIDED_JOB_METADATA_FILE ) json_filename = None for i, ( out_name, data ) in enumerate( out_data.iteritems() ): #use wrapped dataset to access certain values @@ -2177,7 +2178,7 @@ json_params = {} json_params[ 'param_dict' ] = self._prepare_json_param_dict( param_dict ) # it would probably be better to store the original incoming parameters here, instead of the Galaxy modified ones? json_params[ 'output_data' ] = [] - json_params[ 'job_config' ] = dict( GALAXY_DATATYPES_CONF_FILE=param_dict.get( 'GALAXY_DATATYPES_CONF_FILE' ), GALAXY_ROOT_DIR=param_dict.get( 'GALAXY_ROOT_DIR' ), TOOL_PROVIDED_JOB_METADATA_FILE=jobs.TOOL_PROVIDED_JOB_METADATA_FILE ) + json_params[ 'job_config' ] = dict( GALAXY_DATATYPES_CONF_FILE=param_dict.get( 'GALAXY_DATATYPES_CONF_FILE' ), GALAXY_ROOT_DIR=param_dict.get( 'GALAXY_ROOT_DIR' ), TOOL_PROVIDED_JOB_METADATA_FILE=galaxy.jobs.TOOL_PROVIDED_JOB_METADATA_FILE ) json_filename = None for i, ( out_name, data ) in enumerate( out_data.iteritems() ): #use wrapped dataset to access certain values https://bitbucket.org/galaxy/galaxy-central/commits/bb548455022c/ Changeset: bb548455022c User: jmchilton Date: 2014-12-31 23:21:10+00:00 Summary: Mostly back out of 54b7789. It changes the behavior in a subtle way under error conditions. Previously if the on-database config file became out of sync with the tool shed database - Galaxy would loose lineage information (tools would be duplicated in side panel) but all tools from the config file would still be loaded up. This change preventing all tools from loading at all in those scenarios so I am undoing it. Affected #: 1 file diff -r cb673f24cd75ef491aa850373704535691b0a9dd -r bb548455022c7ad21c74ba61b4fe37b606737db7 lib/galaxy/tools/toolbox/base.py --- a/lib/galaxy/tools/toolbox/base.py +++ b/lib/galaxy/tools/toolbox/base.py @@ -521,23 +521,6 @@ def tools( self ): return self._tools_by_id.iteritems() - def __elem_to_tool_shed_repository( self, elem ): - # The tool is contained in an installed tool shed repository, so load - # the tool only if the repository has not been marked deleted. - tool_shed = elem.find( "tool_shed" ).text - repository_name = elem.find( "repository_name" ).text - repository_owner = elem.find( "repository_owner" ).text - installed_changeset_revision_elem = elem.find( "installed_changeset_revision" ) - if installed_changeset_revision_elem is None: - # Backward compatibility issue - the tag used to be named 'changeset_revision'. - installed_changeset_revision_elem = elem.find( "changeset_revision" ) - installed_changeset_revision = installed_changeset_revision_elem.text - tool_shed_repository = self.__get_tool_shed_repository( tool_shed, - repository_name, - repository_owner, - installed_changeset_revision ) - return tool_shed_repository - def __get_tool_shed_repository( self, tool_shed, name, owner, installed_changeset_revision ): # We store only the port, if one exists, in the database. tool_shed = common_util.remove_protocol_from_tool_shed_url( tool_shed ) @@ -662,7 +645,19 @@ if guid is not None: # The tool is contained in an installed tool shed repository, so load # the tool only if the repository has not been marked deleted. - tool_shed_repository = self.__elem_to_tool_shed_repository( elem ) + tool_shed = elem.find( "tool_shed" ).text + repository_name = elem.find( "repository_name" ).text + repository_owner = elem.find( "repository_owner" ).text + installed_changeset_revision_elem = elem.find( "installed_changeset_revision" ) + if installed_changeset_revision_elem is None: + # Backward compatibility issue - the tag used to be named 'changeset_revision'. + installed_changeset_revision_elem = elem.find( "changeset_revision" ) + installed_changeset_revision = installed_changeset_revision_elem.text + tool_shed_repository = self.__get_tool_shed_repository( tool_shed, + repository_name, + repository_owner, + installed_changeset_revision ) + if tool_shed_repository: # Only load tools if the repository is not deactivated or uninstalled. can_load_into_panel_dict = not tool_shed_repository.deleted @@ -675,10 +670,10 @@ key = 'tool_%s' % str( tool.id ) if can_load_into_panel_dict: if guid is not None: - tool.tool_shed = tool_shed_repository.tool_shed - tool.repository_name = tool_shed_repository.name - tool.repository_owner = tool_shed_repository.owner - tool.installed_changeset_revision = tool_shed_repository.installed_changeset_revision + tool.tool_shed = tool_shed + tool.repository_name = repository_name + tool.repository_owner = repository_owner + tool.installed_changeset_revision = installed_changeset_revision tool.guid = guid tool.version = elem.find( "version" ).text # Make sure the tool has a tool_version. Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.