December 2014 - galaxy-commits - lists.galaxyproject.org

commit/galaxy-central: 9 new changesets
by commits-noreply＠bitbucket.org 31 Dec '14

31 Dec '14

9 new commits in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/08f8850853d0/ Changeset: 08f8850853d0 User: jmchilton Date: 2014-12-31 23:21:10+00:00 Summary: Allow multiple tools with the same id in ToolBox. How to use: 1.) Place multiple tools with different IDs in your tool conf. 2.) ... ummm ... no step 2 - just use the tools. Implementation: The Tool Shed allows tool lineages by assigning each tool version a GUID and tracking versions in a database. This implementation works by simply allowing the ToolBox to contain multiple tools with the same ID and orders them by the version specified by the tool author. To track enable this a second tool lineage has been introduced that just uses tool versions instead of a database (non-toolshed installed tools are not longer placed into the Tool Shed install database). The ToolBox has been updated to allow multiple versions per tool id (defaulting to the 'latest' version for all operations which do not specify a version). Both jobs and workflow steps would track tool versions but did not use that version when fetching tools from the Toolbox - these components have been updated to try to use the tool version. Unit tests working through most of the ToolBox and tool panel have been added, as well as functional tests exercising the tools API and to ensure workflows now at least attempt to respect tool versions (still kind of silently switches versions in some cases). Manual tests against the new tool form seem to demonstrate the tool switching and tool re-running work with only minor changes to the tools API and the job handler. Affected #: 17 files diff -r 02efb1123d469986a051e7a8015c9d56635efd8a -r 08f8850853d004bf8a456a147436a6694d0a1a11 lib/galaxy/jobs/__init__.py --- a/lib/galaxy/jobs/__init__.py +++ b/lib/galaxy/jobs/__init__.py @@ -711,7 +711,7 @@ self.job_id = job.id self.session_id = job.session_id self.user_id = job.user_id - self.tool = queue.app.toolbox.get_tool( job.tool_id, exact=True ) + self.tool = queue.app.toolbox.get_tool( job.tool_id, job.tool_version, exact=True ) self.queue = queue self.app = queue.app self.sa_session = self.app.model.context diff -r 02efb1123d469986a051e7a8015c9d56635efd8a -r 08f8850853d004bf8a456a147436a6694d0a1a11 lib/galaxy/jobs/handler.py --- a/lib/galaxy/jobs/handler.py +++ b/lib/galaxy/jobs/handler.py @@ -119,7 +119,7 @@ & ( model.Job.handler == self.app.config.server_name ) ).all() for job in jobs_at_startup: - if not self.app.toolbox.has_tool( job.tool_id, exact=True ): + if not self.app.toolbox.has_tool( job.tool_id, job.tool_version, exact=True ): log.warning( "(%s) Tool '%s' removed from tool config, unable to recover job" % ( job.id, job.tool_id ) ) self.job_wrapper( job ).fail( 'This tool was disabled before the job completed. Please contact your Galaxy administrator.' ) elif job.job_runner_name is not None and job.job_runner_external_id is None: diff -r 02efb1123d469986a051e7a8015c9d56635efd8a -r 08f8850853d004bf8a456a147436a6694d0a1a11 lib/galaxy/tools/__init__.py --- a/lib/galaxy/tools/__init__.py +++ b/lib/galaxy/tools/__init__.py @@ -126,6 +126,10 @@ # shed_tool_conf.xml file. self._dynamic_tool_confs = [] self._tools_by_id = {} + # Tool lineages can contain chains of related tools with different ids + # so each will be present once in the above dictionary. The following + # dictionary can instead hold multiple tools with different versions. + self._tool_versions_by_id = {} self._workflows_by_id = {} # In-memory dictionary that defines the layout of the tool panel. self._tool_panel = odict() @@ -352,7 +356,8 @@ inserted = True if not inserted: # Check the tool's installed versions. - for lineage_id in tool.lineage_ids: + for tool_lineage_version in tool.lineage.get_versions(): + lineage_id = tool_lineage_version.id lineage_id_key = 'tool_%s' % lineage_id for index, integrated_panel_key in enumerate( self._integrated_tool_panel.keys() ): if lineage_id_key == integrated_panel_key: @@ -503,20 +508,26 @@ def get_tool( self, tool_id, tool_version=None, get_all_versions=False, exact=False ): """Attempt to locate a tool in the tool box.""" + if tool_version: + tool_version = str( tool_version ) + if get_all_versions and exact: raise AssertionError("Cannot specify get_tool with both get_all_versions and exact as True") if tool_id in self._tools_by_id and not get_all_versions: + if tool_version and tool_version in self._tool_versions_by_id[ tool_id ]: + return self._tool_versions_by_id[ tool_id ][ tool_version ] #tool_id exactly matches an available tool by id (which is 'old' tool_id or guid) return self._tools_by_id[ tool_id ] #exact tool id match not found, or all versions requested, search for other options, e.g. migrated tools or different versions rval = [] tool_lineage = self._lineage_map.get( tool_id ) if tool_lineage: - tool_version_ids = tool_lineage.get_version_ids( ) - for tool_version_id in tool_version_ids: - if tool_version_id in self._tools_by_id: - rval.append( self._tools_by_id[ tool_version_id ] ) + lineage_tool_versions = tool_lineage.get_versions( ) + for lineage_tool_version in lineage_tool_versions: + lineage_tool = self._tool_from_lineage_version( lineage_tool_version ) + if lineage_tool: + rval.append( lineage_tool ) if not rval: #still no tool, do a deeper search and try to match by old ids for tool in self._tools_by_id.itervalues(): @@ -561,11 +572,12 @@ """Get all loaded tools associated by lineage to the tool whose id is tool_id.""" tool_lineage = self._lineage_map.get( tool_id ) if tool_lineage: - tool_version_ids = tool_lineage.get_version_ids( ) + lineage_tool_versions = tool_lineage.get_versions( ) available_tool_versions = [] - for tool_version_id in tool_version_ids: - if tool_version_id in self._tools_by_id: - available_tool_versions.append( self._tools_by_id[ tool_version_id ] ) + for lineage_tool_version in lineage_tool_versions: + tool = self._tool_from_lineage_version( lineage_tool_version ) + if tool: + available_tool_versions.append( tool ) return available_tool_versions else: if tool_id in self._tools_by_id: @@ -746,7 +758,6 @@ tool_lineage = self._lineage_map.register( tool, tool_shed_repository=tool_shed_repository ) # Load the tool's lineage ids. tool.lineage = tool_lineage - tool.lineage_ids = tool_lineage.get_version_ids( ) self._tool_tag_manager.handle_tags( tool.id, elem ) self.__add_tool( tool, load_panel_dict, panel_dict ) # Always load the tool into the integrated_panel_dict, or it will not be included in the integrated_tool_panel.xml file. @@ -925,7 +936,20 @@ return tool def register_tool( self, tool ): - self._tools_by_id[ tool.id ] = tool + tool_id = tool.id + version = tool.version or None + if tool_id not in self._tool_versions_by_id: + self._tool_versions_by_id[ tool_id ] = { version: tool } + else: + self._tool_versions_by_id[ tool_id ][ version ] = tool + if tool_id in self._tools_by_id: + related_tool = self._tools_by_id[ tool_id ] + # This one becomes the default un-versioned tool + # if newer. + if self._newer_tool( tool, related_tool ): + self._tools_by_id[ tool_id ] = tool + else: + self._tools_by_id[ tool_id ] = tool def package_tool( self, trans, tool_id ): """ @@ -1202,9 +1226,11 @@ if not hasattr( tool, "lineage" ): return None tool_lineage = tool.lineage - lineage_ids = tool_lineage.get_version_ids( reverse=True ) - for lineage_id in lineage_ids: - if lineage_id in self._tools_by_id: + lineage_tool_versions = tool_lineage.get_versions( reverse=True ) + for lineage_tool_version in lineage_tool_versions: + lineage_tool = self._tool_from_lineage_version( lineage_tool_version ) + if lineage_tool: + lineage_id = lineage_tool.id loaded_version_key = 'tool_%s' % lineage_id if loaded_version_key in panel_dict: return panel_dict[ loaded_version_key ] @@ -1214,7 +1240,22 @@ """ Return True if tool1 is considered "newer" given its own lineage description. """ - return tool1.lineage_ids.index( tool1.id ) > tool1.lineage_ids.index( tool2.id ) + if not hasattr( tool1, "lineage" ): + return True + lineage_tool_versions = tool1.lineage.get_versions() + for lineage_tool_version in lineage_tool_versions: + lineage_tool = self._tool_from_lineage_version( lineage_tool_version ) + if lineage_tool is tool1: + return False + if lineage_tool is tool2: + return True + return True + + def _tool_from_lineage_version( self, lineage_tool_version ): + if lineage_tool_version.id_based: + return self._tools_by_id.get( lineage_tool_version.id, None ) + else: + return self._tool_versions_by_id.get( lineage_tool_version.id, {} ).get( lineage_tool_version.version, None ) def _filter_for_panel( item, filters, context ): diff -r 02efb1123d469986a051e7a8015c9d56635efd8a -r 08f8850853d004bf8a456a147436a6694d0a1a11 lib/galaxy/tools/toolbox/lineages/factory.py --- a/lib/galaxy/tools/toolbox/lineages/factory.py +++ b/lib/galaxy/tools/toolbox/lineages/factory.py @@ -1,4 +1,5 @@ from .tool_shed import ToolShedLineage +from .stock import StockLineage class LineageMap(object): @@ -13,7 +14,10 @@ tool_id = tool.id if tool_id not in self.lineage_map: tool_shed_repository = kwds.get("tool_shed_repository", None) - lineage = ToolShedLineage.from_tool(self.app, tool, tool_shed_repository) + if tool_shed_repository: + lineage = ToolShedLineage.from_tool(self.app, tool, tool_shed_repository) + else: + lineage = StockLineage.from_tool( tool ) self.lineage_map[tool_id] = lineage return self.lineage_map[tool_id] diff -r 02efb1123d469986a051e7a8015c9d56635efd8a -r 08f8850853d004bf8a456a147436a6694d0a1a11 lib/galaxy/tools/toolbox/lineages/interface.py --- a/lib/galaxy/tools/toolbox/lineages/interface.py +++ b/lib/galaxy/tools/toolbox/lineages/interface.py @@ -8,7 +8,35 @@ __metaclass__ = ABCMeta @abstractmethod - def get_version_ids( self, reverse=False ): - """ Return an ordered list of lineages in this chain, from - oldest to newest. + def get_versions( self, reverse=False ): + """ Return an ordered list of lineages (ToolLineageVersion) in this + chain, from oldest to newest. """ + + +class ToolLineageVersion(object): + """ Represents a single tool in a lineage. If lineage is based + around GUIDs that somehow encode the version (either using GUID + or a simple tool id and a version). """ + + def __init__(self, id, version): + self.id = id + self.version = version + + @staticmethod + def from_id_and_verion( id, version ): + assert version is not None + return ToolLineageVersion( id, version ) + + @staticmethod + def from_guid( guid ): + return ToolLineageVersion( guid, None ) + + @property + def id_based( self ): + """ Return True if the lineage is defined by GUIDs (in this + case the indexer of the tools (i.e. the ToolBox) should ignore + the tool_version (because it is encoded in the GUID and managed + externally). + """ + return self.version is None diff -r 02efb1123d469986a051e7a8015c9d56635efd8a -r 08f8850853d004bf8a456a147436a6694d0a1a11 lib/galaxy/tools/toolbox/lineages/stock.py --- /dev/null +++ b/lib/galaxy/tools/toolbox/lineages/stock.py @@ -0,0 +1,44 @@ +import threading + +from distutils.version import LooseVersion + +from .interface import ToolLineage +from .interface import ToolLineageVersion + + +class StockLineage(ToolLineage): + """ Simple tool's loaded directly from file system with lineage + determined solely by distutil's LooseVersion naming scheme. + """ + lineages_by_id = {} + lock = threading.Lock() + + def __init__(self, tool_id, **kwds): + self.tool_id = tool_id + self.tool_versions = set() + + @staticmethod + def from_tool( tool ): + tool_id = tool.id + lineages_by_id = StockLineage.lineages_by_id + with StockLineage.lock: + if tool_id not in lineages_by_id: + lineages_by_id[ tool_id ] = StockLineage( tool_id ) + lineage = lineages_by_id[ tool_id ] + lineage.register_version( tool.version ) + return lineage + + def register_version( self, tool_version ): + assert tool_version is not None + self.tool_versions.add( tool_version ) + + def get_versions( self, reverse=False ): + versions = [ ToolLineageVersion( self.tool_id, v ) for v in self.tool_versions ] + # Sort using LooseVersion which defines an appropriate __cmp__ + # method for comparing tool versions. + return sorted( versions, key=_to_loose_version ) + + +def _to_loose_version( tool_lineage_version ): + version = str( tool_lineage_version.version ) + return LooseVersion( version ) diff -r 02efb1123d469986a051e7a8015c9d56635efd8a -r 08f8850853d004bf8a456a147436a6694d0a1a11 lib/galaxy/tools/toolbox/lineages/tool_shed.py --- a/lib/galaxy/tools/toolbox/lineages/tool_shed.py +++ b/lib/galaxy/tools/toolbox/lineages/tool_shed.py @@ -1,4 +1,5 @@ from .interface import ToolLineage +from .interface import ToolLineageVersion from galaxy.model.tool_shed_install import ToolVersion @@ -32,6 +33,9 @@ tool_version = self.app.install_model.context.query( ToolVersion ).get( self.tool_version_id ) return tool_version.get_version_ids( self.app, reverse=reverse ) + def get_versions( self, reverse=False ): + return map( ToolLineageVersion.from_guid, self.get_version_ids( reverse=reverse ) ) + def get_install_tool_version( app, tool_id ): return app.install_model.context.query( diff -r 02efb1123d469986a051e7a8015c9d56635efd8a -r 08f8850853d004bf8a456a147436a6694d0a1a11 lib/galaxy/workflow/modules.py --- a/lib/galaxy/workflow/modules.py +++ b/lib/galaxy/workflow/modules.py @@ -467,10 +467,10 @@ type = "tool" - def __init__( self, trans, tool_id ): + def __init__( self, trans, tool_id, tool_version=None ): self.trans = trans self.tool_id = tool_id - self.tool = trans.app.toolbox.get_tool( tool_id ) + self.tool = trans.app.toolbox.get_tool( tool_id, tool_version=tool_version ) self.post_job_actions = {} self.workflow_outputs = [] self.state = None @@ -493,11 +493,14 @@ @classmethod def from_dict( Class, trans, d, secure=True ): tool_id = d[ 'tool_id' ] - module = Class( trans, tool_id ) + tool_version = str( d.get( 'tool_version', None ) ) + module = Class( trans, tool_id, tool_version=tool_version ) module.state = galaxy.tools.DefaultToolState() if module.tool is not None: if d.get('tool_version', 'Unspecified') != module.get_tool_version(): - module.version_changes.append( "%s: using version '%s' instead of version '%s' indicated in this workflow." % ( tool_id, d.get( 'tool_version', 'Unspecified' ), module.get_tool_version() ) ) + message = "%s: using version '%s' instead of version '%s' indicated in this workflow." % ( tool_id, d.get( 'tool_version', 'Unspecified' ), module.get_tool_version() ) + log.debug(message) + module.version_changes.append(message) module.state.decode( d[ "tool_state" ], module.tool, module.trans.app, secure=secure ) module.errors = d.get( "tool_errors", None ) module.post_job_actions = d.get( "post_job_actions", {} ) @@ -519,9 +522,12 @@ # This step has its state saved in the config field due to the # tool being previously unavailable. return module_factory.from_dict(trans, loads(step.config), secure=False) - module = Class( trans, tool_id ) + tool_version = step.tool_version + module = Class( trans, tool_id, tool_version=tool_version ) if step.tool_version and (step.tool_version != module.tool.version): - module.version_changes.append("%s: using version '%s' instead of version '%s' indicated in this workflow." % (tool_id, module.tool.version, step.tool_version)) + message = "%s: using version '%s' instead of version '%s' indicated in this workflow." % (tool_id, module.tool.version, step.tool_version) + log.debug(message) + module.version_changes.append(message) module.recover_state( step.tool_inputs ) module.errors = step.tool_errors module.workflow_outputs = step.workflow_outputs @@ -723,7 +729,7 @@ return state, step_errors def execute( self, trans, progress, invocation, step ): - tool = trans.app.toolbox.get_tool( step.tool_id ) + tool = trans.app.toolbox.get_tool( step.tool_id, tool_version=step.tool_version ) tool_state = step.state collections_to_match = self._find_collections_to_match( tool, progress, step ) diff -r 02efb1123d469986a051e7a8015c9d56635efd8a -r 08f8850853d004bf8a456a147436a6694d0a1a11 test/api/test_tools.py --- a/test/api/test_tools.py +++ b/test/api/test_tools.py @@ -165,6 +165,18 @@ output1_content = self.dataset_populator.get_history_dataset_content( history_id, dataset=output1 ) self.assertEqual( output1_content.strip(), "Cat1Testlistified" ) + @skip_without_tool( "multiple_versions" ) + def test_run_by_versions( self ): + for version in ["0.1", "0.2"]: + # Run simple non-upload tool with an input data parameter. + history_id = self.dataset_populator.new_history() + inputs = dict() + outputs = self._run_and_get_outputs( tool_id="multiple_versions", history_id=history_id, inputs=inputs, tool_version=version ) + self.assertEquals( len( outputs ), 1 ) + output1 = outputs[ 0 ] + output1_content = self.dataset_populator.get_history_dataset_content( history_id, dataset=output1 ) + self.assertEqual( output1_content.strip(), "Version " + version ) + @skip_without_tool( "cat1" ) def test_run_cat1_single_meta_wrapper( self ): # Wrap input in a no-op meta parameter wrapper like Sam is planning to @@ -650,8 +662,8 @@ def _cat1_outputs( self, history_id, inputs ): return self._run_outputs( self._run_cat1( history_id, inputs ) ) - def _run_and_get_outputs( self, tool_id, history_id, inputs ): - return self._run_outputs( self._run( tool_id, history_id, inputs ) ) + def _run_and_get_outputs( self, tool_id, history_id, inputs, tool_version=None ): + return self._run_outputs( self._run( tool_id, history_id, inputs, tool_version=tool_version ) ) def _run_outputs( self, create_response ): self._assert_status_code_is( create_response, 200 ) @@ -660,12 +672,14 @@ def _run_cat1( self, history_id, inputs, assert_ok=False ): return self._run( 'cat1', history_id, inputs, assert_ok=assert_ok ) - def _run( self, tool_id, history_id, inputs, assert_ok=False ): + def _run( self, tool_id, history_id, inputs, assert_ok=False, tool_version=None ): payload = self.dataset_populator.run_tool_payload( tool_id=tool_id, inputs=inputs, history_id=history_id, ) + if tool_version is not None: + payload[ "tool_version" ] = tool_version create_response = self._post( "tools", data=payload ) if assert_ok: self._assert_status_code_is( create_response, 200 ) diff -r 02efb1123d469986a051e7a8015c9d56635efd8a -r 08f8850853d004bf8a456a147436a6694d0a1a11 test/api/test_workflows.py --- a/test/api/test_workflows.py +++ b/test/api/test_workflows.py @@ -444,6 +444,28 @@ def test_run_workflow( self ): self.__run_cat_workflow( inputs_by='step_id' ) + @skip_without_tool( "multiple_versions" ) + def test_run_versioned_tools( self ): + history_01_id = self.dataset_populator.new_history() + workflow_version_01 = self._upload_yaml_workflow( """ +- tool_id: multiple_versions + tool_version: "0.1" + state: + inttest: 0 +""" ) + self.__invoke_workflow( history_01_id, workflow_version_01 ) + self.dataset_populator.wait_for_history( history_01_id, assert_ok=True ) + + history_02_id = self.dataset_populator.new_history() + workflow_version_02 = self._upload_yaml_workflow( """ +- tool_id: multiple_versions + tool_version: "0.2" + state: + inttest: 1 +""" ) + self.__invoke_workflow( history_02_id, workflow_version_02 ) + self.dataset_populator.wait_for_history( history_02_id, assert_ok=True ) + def __run_cat_workflow( self, inputs_by ): workflow = self.workflow_populator.load_workflow( name="test_for_run" ) workflow["steps"]["0"]["uuid"] = str(uuid4()) @@ -910,7 +932,7 @@ self._assert_status_code_is( hda_info_response, 200 ) self.assertEquals( hda_info_response.json()[ "metadata_data_lines" ], lines ) - def __invoke_workflow( self, history_id, workflow_id, inputs, assert_ok=True ): + def __invoke_workflow( self, history_id, workflow_id, inputs={}, assert_ok=True ): workflow_request = dict( history="hist_id=%s" % history_id, ) diff -r 02efb1123d469986a051e7a8015c9d56635efd8a -r 08f8850853d004bf8a456a147436a6694d0a1a11 test/functional/tools/multiple_versions_v01.xml --- /dev/null +++ b/test/functional/tools/multiple_versions_v01.xml @@ -0,0 +1,11 @@ +<tool id="multiple_versions" name="multiple_versions" version="0.1"> + <command> + echo "Version 0.1" > $out_file1 + </command> + <inputs> + <param name="inttest" value="1" type="integer" /> + </inputs> + <outputs> + <data name="out_file1" format="txt" /> + </outputs> +</tool> diff -r 02efb1123d469986a051e7a8015c9d56635efd8a -r 08f8850853d004bf8a456a147436a6694d0a1a11 test/functional/tools/multiple_versions_v02.xml --- /dev/null +++ b/test/functional/tools/multiple_versions_v02.xml @@ -0,0 +1,11 @@ +<tool id="multiple_versions" name="multiple_versions" version="0.2"> + <command> + echo "Version 0.2" > $out_file1 + </command> + <inputs> + <param name="inttest" value="1" type="integer" /> + </inputs> + <outputs> + <data name="out_file1" format="txt" /> + </outputs> +</tool> diff -r 02efb1123d469986a051e7a8015c9d56635efd8a -r 08f8850853d004bf8a456a147436a6694d0a1a11 test/functional/tools/samples_tool_conf.xml --- a/test/functional/tools/samples_tool_conf.xml +++ b/test/functional/tools/samples_tool_conf.xml @@ -36,6 +36,9 @@ <tool file="collection_two_paired.xml" /><tool file="collection_optional_param.xml" /> + <tool file="multiple_versions_v01.xml" /> + <tool file="multiple_versions_v02.xml" /> + \n') - for key, item in self._integrated_tool_panel.items(): + for key, item_type, item in self._integrated_tool_panel.panel_items_iter(): if item: - if isinstance( item, Tool ): + if item_type == panel_item_types.TOOL: os.write( fd, ' <tool id="%s" />\n' % item.id ) - elif isinstance( item, Workflow ): + elif item_type == panel_item_types.WORKFLOW: os.write( fd, ' <workflow id="%s" />\n' % item.id ) - elif isinstance( item, ToolSectionLabel ): + elif item_type == panel_item_types.LABEL: label_id = item.id or '' label_text = item.text or '' label_version = item.version or '' os.write( fd, ' <label id="%s" text="%s" version="%s" />\n' % ( label_id, label_text, label_version ) ) - elif isinstance( item, ToolSection ): + elif item_type == panel_item_types.SECTION: section_id = item.id or '' section_name = item.name or '' section_version = item.version or '' os.write( fd, ' <section id="%s" name="%s" version="%s">\n' % ( section_id, section_name, section_version ) ) - for section_key, section_item in item.elems.items(): - if isinstance( section_item, Tool ): + for section_key, section_item_type, section_item in item.panel_items_iter(): + if section_item_type == panel_item_types.TOOL: if section_item: os.write( fd, ' <tool id="%s" />\n' % section_item.id ) - elif isinstance( section_item, Workflow ): + elif section_item_type == panel_item_types.WORKFLOW: if section_item: os.write( fd, ' <workflow id="%s" />\n' % section_item.id ) - elif isinstance( section_item, ToolSectionLabel ): + elif section_item_type == panel_item_types.LABEL: if section_item: label_id = section_item.id or '' label_text = section_item.text or '' @@ -709,10 +710,9 @@ else: del has_elems[ tool_key ] if remove_from_config: - if hasattr( integrated_has_elems, "elems" ): - integrated_has_elems = integrated_has_elems.elems - if tool_key in integrated_has_elems: - del integrated_has_elems[ tool_key ] + itegrated_items = integrated_has_elems.panel_items() + if tool_key in itegrated_items: + del itegrated_items[ tool_key ] if section_key: _, tool_section = self.get_section( section_key ) @@ -1174,8 +1174,8 @@ """ context = Bunch( toolbox=self, trans=trans ) filters = self._filter_factory.build_filters( trans ) - for elt in self._tool_panel.itervalues(): - elt = _filter_for_panel( elt, filters, context ) + for _, item_type, elt in self._tool_panel.panel_items_iter(): + elt = _filter_for_panel( elt, item_type, filters, context ) if elt: yield elt @@ -1241,7 +1241,7 @@ return self._tool_versions_by_id.get( lineage_tool_version.id, {} ).get( lineage_tool_version.version, None ) -def _filter_for_panel( item, filters, context ): +def _filter_for_panel( item, item_type, filters, context ): """ Filters tool panel elements so that only those that are compatible with provided filters are kept. @@ -1251,13 +1251,13 @@ if not filter_method( context, filter_item ): return False return True - if isinstance( item, Tool ): + if item_type == panel_item_types.TOOL: if _apply_filter( item, filters[ 'tool' ] ): return item - elif isinstance( item, ToolSectionLabel ): + elif item_type == panel_item_types.LABEL: if _apply_filter( item, filters[ 'label' ] ): return item - elif isinstance( item, ToolSection ): + elif item_type == panel_item_types.SECTION: # Filter section item-by-item. Only show a label if there are # non-filtered tools below it. @@ -1265,14 +1265,14 @@ cur_label_key = None tools_under_label = False filtered_elems = item.elems.copy() - for key, section_item in item.elems.items(): - if isinstance( section_item, Tool ): + for key, section_item_type, section_item in item.panel_items_iter(): + if section_item_type == panel_item_types.TOOL: # Filter tool. if _apply_filter( section_item, filters[ 'tool' ] ): tools_under_label = True else: del filtered_elems[ key ] - elif isinstance( section_item, ToolSectionLabel ): + elif section_item_type == panel_item_types.LABEL: # If there is a label and it does not have tools, # remove it. if ( cur_label_key and not tools_under_label ) or not _apply_filter( section_item, filters[ 'label' ] ): diff -r 8527f5a00f196ac8b641c57808d26edf305dc16a -r 24aa2e802a351882695cb98d772001cc2a1316d8 lib/galaxy/tools/toolbox/__init__.py --- a/lib/galaxy/tools/toolbox/__init__.py +++ b/lib/galaxy/tools/toolbox/__init__.py @@ -5,6 +5,12 @@ from .panel import ToolPanelElements from .panel import ToolSection from .panel import ToolSectionLabel +from .panel import panel_item_types - -__all__ = ["ToolPanelElements", "tool_tag_manager", "ToolSection", "ToolSectionLabel"] +__all__ = [ + "ToolPanelElements", + "tool_tag_manager", + "ToolSection", + "ToolSectionLabel", + "panel_item_types" +] diff -r 8527f5a00f196ac8b641c57808d26edf305dc16a -r 24aa2e802a351882695cb98d772001cc2a1316d8 lib/galaxy/tools/toolbox/panel.py --- a/lib/galaxy/tools/toolbox/panel.py +++ b/lib/galaxy/tools/toolbox/panel.py @@ -1,8 +1,46 @@ +from abc import abstractmethod + from galaxy.util.odict import odict +from galaxy.util import bunch from galaxy.model.item_attrs import Dictifiable -class ToolSection( object, Dictifiable ): +panel_item_types = bunch.Bunch( + TOOL="TOOL", + WORKFLOW="WORKFLOW", + SECTION="SECTION", + LABEL="LABEL", +) + + +class HasPanelItems: + """ + """ + + @abstractmethod + def panel_items( self ): + """ Return an ordered dictionary-like object describing tool panel + items (such as workflows, tools, labels, and sections). + """ + + def panel_items_iter( self ): + """ Iterate through panel items each represented as a tuple of + (panel_key, panel_type, panel_content). + """ + for panel_key, panel_value in self.panel_items().iteritems(): + if panel_value is None: + continue + panel_type = panel_item_types.SECTION + if panel_key.startswith("tool_"): + panel_type = panel_item_types.TOOL + elif panel_key.startswith("label_"): + panel_type = panel_item_types.LABEL + elif panel_key.startswith("workflow_"): + panel_type = panel_item_types.WORKFLOW + yield (panel_key, panel_type, panel_value) + + +class ToolSection( object, Dictifiable, HasPanelItems ): """ A group of tools with similar type/purpose that will be displayed as a group in the user interface. @@ -42,6 +80,9 @@ return section_dict + def panel_items( self ): + return self.elems + class ToolSectionLabel( object, Dictifiable ): """ @@ -63,7 +104,7 @@ return super( ToolSectionLabel, self ).to_dict() -class ToolPanelElements( odict ): +class ToolPanelElements( odict, HasPanelItems ): """ Represents an ordered dictionary of tool entries - abstraction used both by tool panel itself (normal and integrated) and its sections. """ @@ -118,3 +159,6 @@ def append_section( self, key, section_elems ): self[ key ] = section_elems + + def panel_items( self ): + return self https://bitbucket.org/galaxy/galaxy-central/commits/edacabfca615/ Changeset: edacabfca615 User: jmchilton Date: 2014-12-31 23:21:10+00:00 Summary: Move 95% of ToolBox into its own module. Including logic related to panels and sections, the integrated tool panels, labels, etc.... Leave a variant of ToolBox in galaxy.tools that needs to know about Tool runtime stuff (Tool creation, actions, and dependency management). Affected #: 3 files diff -r 24aa2e802a351882695cb98d772001cc2a1316d8 -r edacabfca615542ca435a97fc1afcc99daa470d5 lib/galaxy/tools/__init__.py --- a/lib/galaxy/tools/__init__.py +++ b/lib/galaxy/tools/__init__.py @@ -9,9 +9,6 @@ import os import re import shutil -import string -import tarfile -import tempfile import threading import types import urllib @@ -27,12 +24,10 @@ from elementtree import ElementTree from mako.template import Template from paste import httpexceptions -from sqlalchemy import and_ from galaxy import jobs, model from galaxy.datatypes.metadata import JobExternalOutputMetadataWrapper from galaxy import exceptions -from galaxy.tools.toolbox import watcher from galaxy.tools.actions import DefaultToolAction from galaxy.tools.actions.upload import UploadToolAction from galaxy.tools.actions.data_source import DataSourceToolAction @@ -48,17 +43,11 @@ from galaxy.tools.parameters.input_translation import ToolInputTranslator from galaxy.tools.parameters.output import ToolOutputActionGroup from galaxy.tools.parameters.validation import LateValidationError -from galaxy.tools.toolbox.filters import FilterFactory from galaxy.tools.test import parse_tests from galaxy.tools.parser import get_tool_source from galaxy.tools.parser.xml import XmlPageSource -from galaxy.tools.toolbox import ToolPanelElements -from galaxy.tools.toolbox import ToolSection -from galaxy.tools.toolbox import ToolSectionLabel -from galaxy.tools.toolbox import panel_item_types -from galaxy.tools.toolbox import tool_tag_manager -from galaxy.tools.toolbox.lineages import LineageMap -from galaxy.util import listify, parse_xml, rst_to_html, string_as_bool, string_to_object +from galaxy.tools.toolbox import AbstractToolBox +from galaxy.util import rst_to_html, string_as_bool, string_to_object from galaxy.tools.parameters.meta import expand_meta_parameters from galaxy.util.bunch import Bunch from galaxy.util.expressions import ExpressionContext @@ -66,11 +55,7 @@ from galaxy.util.odict import odict from galaxy.util.template import fill_template from galaxy.web import url_for -from galaxy.web.form_builder import SelectField -from galaxy.web.framework.helpers import escape from galaxy.model.item_attrs import Dictifiable -from galaxy.model import Workflow -from tool_shed.util import common_util from tool_shed.util import shed_util_common as suc from .loader import template_macro_params, raw_tool_xml_tree, imported_macro_paths from .execute import execute as execute_job @@ -102,774 +87,30 @@ HELP_UNINITIALIZED = threading.Lock() -INTEGRATED_TOOL_PANEL_DESCRIPTION = """ -This is Galaxy's integrated tool panel and probably should not be modified -directly. It will be regenerated each time Galaxy starts up. To modify locally -managed tools (e.g. from tool_conf.xml) modify that file directly and restart -Galaxy. Whenever possible Tool Shed managed tools (e.g. from shed_tool_conf.xml) -should be managed from within the Galaxy interface of via is UI - but if changes -are nessecary (such as to hide a tool or re-assign its section) modify that file -and restart Galaxy. -""" - class ToolNotFoundException( Exception ): pass -class ToolBox( object, Dictifiable ): - """Container for a collection of tools""" +class ToolBox( AbstractToolBox ): + """ A derivative of AbstractToolBox with knowledge about Tool internals - + how to construct them, action types, dependency management, etc.... + """ def __init__( self, config_filenames, tool_root_dir, app ): - """ - Create a toolbox from the config files named by `config_filenames`, using - `tool_root_dir` as the base directory for finding individual tool config files. - """ - # The _dynamic_tool_confs list contains dictionaries storing - # information about the tools defined in each shed-related - # shed_tool_conf.xml file. - self._dynamic_tool_confs = [] - self._tools_by_id = {} - # Tool lineages can contain chains of related tools with different ids - # so each will be present once in the above dictionary. The following - # dictionary can instead hold multiple tools with different versions. - self._tool_versions_by_id = {} - self._workflows_by_id = {} - # In-memory dictionary that defines the layout of the tool panel. - self._tool_panel = ToolPanelElements() - self._index = 0 - self.data_manager_tools = odict() - self._lineage_map = LineageMap( app ) - # File that contains the XML section and tool tags from all tool panel config files integrated into a - # single file that defines the tool panel layout. This file can be changed by the Galaxy administrator - # (in a way similar to the single tool_conf.xml file in the past) to alter the layout of the tool panel. - self._integrated_tool_panel_config = app.config.integrated_tool_panel_config - # In-memory dictionary that defines the layout of the tool_panel.xml file on disk. - self._integrated_tool_panel = ToolPanelElements() - self._integrated_tool_panel_config_has_contents = os.path.exists( self._integrated_tool_panel_config ) and os.stat( self._integrated_tool_panel_config ).st_size > 0 - if self._integrated_tool_panel_config_has_contents: - self._load_integrated_tool_panel_keys() - # The following refers to the tool_path config setting for backward compatibility. The shed-related - # (e.g., shed_tool_conf.xml) files include the tool_path attribute within the <toolbox> tag. - self._tool_root_dir = tool_root_dir - self.app = app - self._tool_watcher = watcher.get_watcher( self, app.config ) - self._filter_factory = FilterFactory( self ) + super( ToolBox, self ).__init__( + config_filenames=config_filenames, + tool_root_dir=tool_root_dir, + app=app, + ) self._init_dependency_manager() - self._tool_tag_manager = tool_tag_manager( app ) - self._init_tools_from_configs( config_filenames ) - if self.app.name == 'galaxy' and self._integrated_tool_panel_config_has_contents: - # Load self._tool_panel based on the order in self._integrated_tool_panel. - self._load_tool_panel() - if app.config.update_integrated_tool_panel: - # Write the current in-memory integrated_tool_panel to the integrated_tool_panel.xml file. - # This will cover cases where the Galaxy administrator manually edited one or more of the tool panel - # config files, adding or removing locally developed tools or workflows. The value of integrated_tool_panel - # will be False when things like functional tests are the caller. - self._write_integrated_tool_panel_config_file() - - def _init_tools_from_configs( self, config_filenames ): - """ Read through all tool config files and initialize tools in each - with init_tools_from_config below. - """ - self._tool_tag_manager.reset_tags() - config_filenames = listify( config_filenames ) - for config_filename in config_filenames: - if os.path.isdir( config_filename ): - directory_contents = sorted( os.listdir( config_filename ) ) - directory_config_files = [ config_file for config_file in directory_contents if config_file.endswith( ".xml" ) ] - config_filenames.remove( config_filename ) - config_filenames.extend( directory_config_files ) - for config_filename in config_filenames: - try: - self._init_tools_from_config( config_filename ) - except: - log.exception( "Error loading tools defined in config %s", config_filename ) - - def _init_tools_from_config( self, config_filename ): - """ - Read the configuration file and load each tool. The following tags are currently supported: - - .. raw:: xml - - <toolbox> - <tool file="data_source/upload.xml"/> # tools outside sections - <label text="Basic Tools" id="basic_tools" /> # labels outside sections - <workflow id="529fd61ab1c6cc36" /> # workflows outside sections - <section name="Get Data" id="getext"> # sections - <tool file="data_source/biomart.xml" /> # tools inside sections - <label text="In Section" id="in_section" /> # labels inside sections - <workflow id="adb5f5c93f827949" /> # workflows inside sections - </section> - </toolbox> - - """ - log.info( "Parsing the tool configuration %s" % config_filename ) - tree = parse_xml( config_filename ) - root = tree.getroot() - tool_path = root.get( 'tool_path' ) - if tool_path: - # We're parsing a shed_tool_conf file since we have a tool_path attribute. - parsing_shed_tool_conf = True - # Keep an in-memory list of xml elements to enable persistence of the changing tool config. - config_elems = [] - else: - parsing_shed_tool_conf = False - tool_path = self.__resolve_tool_path(tool_path, config_filename) - # Only load the panel_dict under certain conditions. - load_panel_dict = not self._integrated_tool_panel_config_has_contents - for _, elem in enumerate( root ): - index = self._index - self._index += 1 - if parsing_shed_tool_conf: - config_elems.append( elem ) - self.load_item( - elem, - tool_path=tool_path, - load_panel_dict=load_panel_dict, - guid=elem.get( 'guid' ), - index=index, - internal=True - ) - - if parsing_shed_tool_conf: - shed_tool_conf_dict = dict( config_filename=config_filename, - tool_path=tool_path, - config_elems=config_elems ) - self._dynamic_tool_confs.append( shed_tool_conf_dict ) - - def load_item( self, elem, tool_path, panel_dict=None, integrated_panel_dict=None, load_panel_dict=True, guid=None, index=None, internal=False ): - item_type = elem.tag - if item_type not in ['tool', 'section'] and not internal: - # External calls from tool shed code cannot load labels or tool - # directories. - return - - if panel_dict is None: - panel_dict = self._tool_panel - if integrated_panel_dict is None: - integrated_panel_dict = self._integrated_tool_panel - if item_type == 'tool': - self._load_tool_tag_set( elem, panel_dict=panel_dict, integrated_panel_dict=integrated_panel_dict, tool_path=tool_path, load_panel_dict=load_panel_dict, guid=guid, index=index ) - elif item_type == 'workflow': - self._load_workflow_tag_set( elem, panel_dict=panel_dict, integrated_panel_dict=integrated_panel_dict, load_panel_dict=load_panel_dict, index=index ) - elif item_type == 'section': - self._load_section_tag_set( elem, tool_path=tool_path, load_panel_dict=load_panel_dict, index=index ) - elif item_type == 'label': - self._load_label_tag_set( elem, panel_dict=panel_dict, integrated_panel_dict=integrated_panel_dict, load_panel_dict=load_panel_dict, index=index ) - elif item_type == 'tool_dir': - self._load_tooldir_tag_set( elem, panel_dict, tool_path, integrated_panel_dict, load_panel_dict=load_panel_dict ) - - def get_shed_config_dict_by_filename( self, filename, default=None ): - for shed_config_dict in self._dynamic_tool_confs: - if shed_config_dict[ 'config_filename' ] == filename: - return shed_config_dict - return default - - def update_shed_config( self, shed_conf, integrated_panel_changes=True ): - """ Update the in-memory descriptions of tools and write out the changes - to integrated tool panel unless we are just deactivating a tool (since - that doesn't affect that file). - """ - app = self.app - for index, my_shed_tool_conf in enumerate( self._dynamic_tool_confs ): - if shed_conf['config_filename'] == my_shed_tool_conf['config_filename']: - self._dynamic_tool_confs[ index ] = shed_conf - if integrated_panel_changes and app.config.update_integrated_tool_panel: - # Write the current in-memory version of the integrated_tool_panel.xml file to disk. - self._write_integrated_tool_panel_config_file() - app.reindex_tool_search() - - def get_section( self, section_id, new_label=None, create_if_needed=False ): - tool_panel_section_key = str( section_id ) - if tool_panel_section_key in self._tool_panel: - # Appending a tool to an existing section in toolbox._tool_panel - tool_section = self._tool_panel[ tool_panel_section_key ] - log.debug( "Appending to tool panel section: %s" % str( tool_section.name ) ) - elif create_if_needed: - # Appending a new section to toolbox._tool_panel - if new_label is None: - # This might add an ugly section label to the tool panel, but, oh well... - new_label = section_id - section_dict = { - 'name': new_label, - 'id': section_id, - 'version': '', - } - tool_section = ToolSection( section_dict ) - self._tool_panel[ tool_panel_section_key ] = tool_section - log.debug( "Loading new tool panel section: %s" % str( tool_section.name ) ) - else: - tool_section = None - return tool_panel_section_key, tool_section - - def get_integrated_section_for_tool( self, tool ): - tool_id = tool.id - for key, item_type, item in self._integrated_tool_panel.panel_items_iter(): - if item: - if item_type == panel_item_types.TOOL: - if item.id == tool_id: - return '', '' - if item_type == panel_item_types.SECTION: - section_id = item.id or '' - section_name = item.name or '' - for section_key, section_item_type, section_item in item.panel_items_iter(): - if section_item_type == panel_item_types.TOOL: - if section_item: - if section_item.id == tool_id: - return section_id, section_name - return None, None - - def __resolve_tool_path(self, tool_path, config_filename): - if not tool_path: - # Default to backward compatible config setting. - tool_path = self._tool_root_dir - else: - # Allow use of __tool_conf_dir__ in toolbox config files. - tool_conf_dir = os.path.dirname(config_filename) - tool_path_vars = {"tool_conf_dir": tool_conf_dir} - tool_path = string.Template(tool_path).safe_substitute(tool_path_vars) - return tool_path - - def __add_tool_to_tool_panel( self, tool, panel_component, section=False ): - # See if a version of this tool is already loaded into the tool panel. - # The value of panel_component will be a ToolSection (if the value of - # section=True) or self._tool_panel (if section=False). - tool_id = str( tool.id ) - tool = self._tools_by_id[ tool_id ] - if section: - panel_dict = panel_component.elems - else: - panel_dict = panel_component - - related_tool = self._lineage_in_panel( panel_dict, tool=tool ) - if related_tool: - if self._newer_tool( tool, related_tool ): - panel_dict.replace_tool( - previous_tool_id=related_tool.id, - new_tool_id=tool_id, - tool=tool, - ) - log.debug( "Loaded tool id: %s, version: %s into tool panel." % ( tool.id, tool.version ) ) - else: - inserted = False - index = self._integrated_tool_panel.index_of_tool_id( tool_id ) - if index: - panel_dict.insert_tool( index, tool ) - inserted = True - if not inserted: - # Check the tool's installed versions. - for tool_lineage_version in tool.lineage.get_versions(): - lineage_id = tool_lineage_version.id - index = self._integrated_tool_panel.index_of_tool_id( lineage_id ) - if index: - panel_dict.insert_tool( index, tool ) - inserted = True - if not inserted: - if ( - tool.guid is None or - tool.tool_shed is None or - tool.repository_name is None or - tool.repository_owner is None or - tool.installed_changeset_revision is None - ): - # We have a tool that was not installed from the Tool - # Shed, but is also not yet defined in - # integrated_tool_panel.xml, so append it to the tool - # panel. - panel_dict.append_tool( tool ) - log.debug( "Loaded tool id: %s, version: %s into tool panel.." % ( tool.id, tool.version ) ) - else: - # We are in the process of installing the tool. - - tool_lineage = self._lineage_map.get( tool_id ) - already_loaded = self._lineage_in_panel( panel_dict, tool_lineage=tool_lineage ) is not None - if not already_loaded: - # If the tool is not defined in integrated_tool_panel.xml, append it to the tool panel. - panel_dict.append_tool( tool ) - log.debug( "Loaded tool id: %s, version: %s into tool panel...." % ( tool.id, tool.version ) ) - - def _load_tool_panel( self ): - for key, item_type, val in self._integrated_tool_panel.panel_items_iter(): - if item_type == panel_item_types.TOOL: - tool_id = key.replace( 'tool_', '', 1 ) - if tool_id in self._tools_by_id: - self.__add_tool_to_tool_panel( val, self._tool_panel, section=False ) - elif item_type == panel_item_types.WORKFLOW: - workflow_id = key.replace( 'workflow_', '', 1 ) - if workflow_id in self._workflows_by_id: - workflow = self._workflows_by_id[ workflow_id ] - self._tool_panel[ key ] = workflow - log.debug( "Loaded workflow: %s %s" % ( workflow_id, workflow.name ) ) - elif item_type == panel_item_types.LABEL: - self._tool_panel[ key ] = val - elif item_type == panel_item_types.SECTION: - section_dict = { - 'id': val.id or '', - 'name': val.name or '', - 'version': val.version or '', - } - section = ToolSection( section_dict ) - log.debug( "Loading section: %s" % section_dict.get( 'name' ) ) - for section_key, section_item_type, section_val in val.panel_items_iter(): - if section_item_type == panel_item_types.TOOL: - tool_id = section_key.replace( 'tool_', '', 1 ) - if tool_id in self._tools_by_id: - self.__add_tool_to_tool_panel( section_val, section, section=True ) - elif section_item_type == panel_item_types.WORKFLOW: - workflow_id = section_key.replace( 'workflow_', '', 1 ) - if workflow_id in self._workflows_by_id: - workflow = self._workflows_by_id[ workflow_id ] - section.elems[ section_key ] = workflow - log.debug( "Loaded workflow: %s %s" % ( workflow_id, workflow.name ) ) - elif section_item_type == panel_item_types.LABEL: - if section_val: - section.elems[ section_key ] = section_val - log.debug( "Loaded label: %s" % ( section_val.text ) ) - self._tool_panel[ key ] = section - - def _load_integrated_tool_panel_keys( self ): - """ - Load the integrated tool panel keys, setting values for tools and - workflows to None. The values will be reset when the various tool - panel config files are parsed, at which time the tools and workflows - are loaded. - """ - tree = parse_xml( self._integrated_tool_panel_config ) - root = tree.getroot() - for elem in root: - key = elem.get( 'id' ) - if elem.tag == 'tool': - self._integrated_tool_panel.stub_tool( key ) - elif elem.tag == 'workflow': - self._integrated_tool_panel.stub_workflow( key ) - elif elem.tag == 'section': - section = ToolSection( elem ) - for section_elem in elem: - section_id = section_elem.get( 'id' ) - if section_elem.tag == 'tool': - section.elems.stub_tool( section_id ) - elif section_elem.tag == 'workflow': - section.elems.stub_workflow( section_id ) - elif section_elem.tag == 'label': - section.elems.stub_label( section_id ) - self._integrated_tool_panel.append_section( key, section ) - elif elem.tag == 'label': - section.stub_label( key ) - - def _write_integrated_tool_panel_config_file( self ): - """ - Write the current in-memory version of the integrated_tool_panel.xml file to disk. Since Galaxy administrators - use this file to manage the tool panel, we'll not use xml_to_string() since it doesn't write XML quite right. - """ - fd, filename = tempfile.mkstemp() - os.write( fd, '<?xml version="1.0"?>\n' ) - os.write( fd, '<toolbox>\n' ) - os.write( fd, ' \n') - for key, item_type, item in self._integrated_tool_panel.panel_items_iter(): - if item: - if item_type == panel_item_types.TOOL: - os.write( fd, ' <tool id="%s" />\n' % item.id ) - elif item_type == panel_item_types.WORKFLOW: - os.write( fd, ' <workflow id="%s" />\n' % item.id ) - elif item_type == panel_item_types.LABEL: - label_id = item.id or '' - label_text = item.text or '' - label_version = item.version or '' - os.write( fd, ' <label id="%s" text="%s" version="%s" />\n' % ( label_id, label_text, label_version ) ) - elif item_type == panel_item_types.SECTION: - section_id = item.id or '' - section_name = item.name or '' - section_version = item.version or '' - os.write( fd, ' <section id="%s" name="%s" version="%s">\n' % ( section_id, section_name, section_version ) ) - for section_key, section_item_type, section_item in item.panel_items_iter(): - if section_item_type == panel_item_types.TOOL: - if section_item: - os.write( fd, ' <tool id="%s" />\n' % section_item.id ) - elif section_item_type == panel_item_types.WORKFLOW: - if section_item: - os.write( fd, ' <workflow id="%s" />\n' % section_item.id ) - elif section_item_type == panel_item_types.LABEL: - if section_item: - label_id = section_item.id or '' - label_text = section_item.text or '' - label_version = section_item.version or '' - os.write( fd, ' <label id="%s" text="%s" version="%s" />\n' % ( label_id, label_text, label_version ) ) - os.write( fd, ' </section>\n' ) - os.write( fd, '</toolbox>\n' ) - os.close( fd ) - shutil.move( filename, os.path.abspath( self._integrated_tool_panel_config ) ) - os.chmod( self._integrated_tool_panel_config, 0644 ) - - def get_tool( self, tool_id, tool_version=None, get_all_versions=False, exact=False ): - """Attempt to locate a tool in the tool box.""" - if tool_version: - tool_version = str( tool_version ) - - if get_all_versions and exact: - raise AssertionError("Cannot specify get_tool with both get_all_versions and exact as True") - - if tool_id in self._tools_by_id and not get_all_versions: - if tool_version and tool_version in self._tool_versions_by_id[ tool_id ]: - return self._tool_versions_by_id[ tool_id ][ tool_version ] - #tool_id exactly matches an available tool by id (which is 'old' tool_id or guid) - return self._tools_by_id[ tool_id ] - #exact tool id match not found, or all versions requested, search for other options, e.g. migrated tools or different versions - rval = [] - tool_lineage = self._lineage_map.get( tool_id ) - if tool_lineage: - lineage_tool_versions = tool_lineage.get_versions( ) - for lineage_tool_version in lineage_tool_versions: - lineage_tool = self._tool_from_lineage_version( lineage_tool_version ) - if lineage_tool: - rval.append( lineage_tool ) - if not rval: - #still no tool, do a deeper search and try to match by old ids - for tool in self._tools_by_id.itervalues(): - if tool.old_id == tool_id: - rval.append( tool ) - if rval: - if get_all_versions: - return rval - else: - if tool_version: - #return first tool with matching version - for tool in rval: - if tool.version == tool_version: - return tool - #No tool matches by version, simply return the first available tool found - return rval[0] - #We now likely have a Toolshed guid passed in, but no supporting database entries - #If the tool exists by exact id and is loaded then provide exact match within a list - if tool_id in self._tools_by_id: - return[ self._tools_by_id[ tool_id ] ] - return None - - def has_tool( self, tool_id, exact=False ): - return self.get_tool( tool_id, exact=exact ) is not None - - def get_tool_id( self, tool_id ): - """ Take a tool id (potentially from a different Galaxy instance or that - is no longer loaded - and find the closest match to the currently loaded - tools (using get_tool for inexact matches which currently returns the oldest - tool shed installed tool with the same short id). - """ - if tool_id not in self._tools_by_id: - tool = self.get_tool( tool_id ) - if tool: - tool_id = tool.id - else: - tool_id = None - # else exact match - leave unmodified. - return tool_id - - def get_loaded_tools_by_lineage( self, tool_id ): - """Get all loaded tools associated by lineage to the tool whose id is tool_id.""" - tool_lineage = self._lineage_map.get( tool_id ) - if tool_lineage: - lineage_tool_versions = tool_lineage.get_versions( ) - available_tool_versions = [] - for lineage_tool_version in lineage_tool_versions: - tool = self._tool_from_lineage_version( lineage_tool_version ) - if tool: - available_tool_versions.append( tool ) - return available_tool_versions - else: - if tool_id in self._tools_by_id: - tool = self._tools_by_id[ tool_id ] - return [ tool ] - return [] @property def tools_by_id( self ): # Deprecated method, TODO - eliminate calls to this in test/. return self._tools_by_id - def tools( self ): - return self._tools_by_id.iteritems() - - def __elem_to_tool_shed_repository( self, elem ): - # The tool is contained in an installed tool shed repository, so load - # the tool only if the repository has not been marked deleted. - tool_shed = elem.find( "tool_shed" ).text - repository_name = elem.find( "repository_name" ).text - repository_owner = elem.find( "repository_owner" ).text - installed_changeset_revision_elem = elem.find( "installed_changeset_revision" ) - if installed_changeset_revision_elem is None: - # Backward compatibility issue - the tag used to be named 'changeset_revision'. - installed_changeset_revision_elem = elem.find( "changeset_revision" ) - installed_changeset_revision = installed_changeset_revision_elem.text - tool_shed_repository = self.__get_tool_shed_repository( tool_shed, - repository_name, - repository_owner, - installed_changeset_revision ) - return tool_shed_repository - - def __get_tool_shed_repository( self, tool_shed, name, owner, installed_changeset_revision ): - # We store only the port, if one exists, in the database. - tool_shed = common_util.remove_protocol_from_tool_shed_url( tool_shed ) - return self.app.install_model.context.query( self.app.install_model.ToolShedRepository ) \ - .filter( and_( self.app.install_model.ToolShedRepository.table.c.tool_shed == tool_shed, - self.app.install_model.ToolShedRepository.table.c.name == name, - self.app.install_model.ToolShedRepository.table.c.owner == owner, - self.app.install_model.ToolShedRepository.table.c.installed_changeset_revision == installed_changeset_revision ) ) \ - .first() - - def get_tool_components( self, tool_id, tool_version=None, get_loaded_tools_by_lineage=False, set_selected=False ): - """ - Retrieve all loaded versions of a tool from the toolbox and return a select list enabling - selection of a different version, the list of the tool's loaded versions, and the specified tool. - """ - toolbox = self - tool_version_select_field = None - tools = [] - tool = None - # Backwards compatibility for datasource tools that have default tool_id configured, but which - # are now using only GALAXY_URL. - tool_ids = listify( tool_id ) - for tool_id in tool_ids: - if get_loaded_tools_by_lineage: - tools = toolbox.get_loaded_tools_by_lineage( tool_id ) - else: - tools = toolbox.get_tool( tool_id, tool_version=tool_version, get_all_versions=True ) - if tools: - tool = toolbox.get_tool( tool_id, tool_version=tool_version, get_all_versions=False ) - if len( tools ) > 1: - tool_version_select_field = self.build_tool_version_select_field( tools, tool.id, set_selected ) - break - return tool_version_select_field, tools, tool - - def dynamic_confs( self, include_migrated_tool_conf=False ): - confs = [] - for dynamic_tool_conf_dict in self._dynamic_tool_confs: - dynamic_tool_conf_filename = dynamic_tool_conf_dict[ 'config_filename' ] - if include_migrated_tool_conf or (dynamic_tool_conf_filename != self.app.config.migrated_tools_config): - confs.append( dynamic_tool_conf_dict ) - return confs - - def dynamic_conf_filenames( self, include_migrated_tool_conf=False ): - """ Return list of dynamic tool configuration filenames (shed_tools). - These must be used with various dynamic tool configuration update - operations (e.g. with update_shed_config). - """ - for dynamic_tool_conf_dict in self.dynamic_confs( include_migrated_tool_conf=include_migrated_tool_conf ): - yield dynamic_tool_conf_dict[ 'config_filename' ] - - def build_tool_version_select_field( self, tools, tool_id, set_selected ): - """Build a SelectField whose options are the ids for the received list of tools.""" - options = [] - refresh_on_change_values = [] - for tool in tools: - options.insert( 0, ( tool.version, tool.id ) ) - refresh_on_change_values.append( tool.id ) - select_field = SelectField( name='tool_id', refresh_on_change=True, refresh_on_change_values=refresh_on_change_values ) - for option_tup in options: - selected = set_selected and option_tup[ 1 ] == tool_id - if selected: - select_field.add_option( 'version %s' % option_tup[ 0 ], option_tup[ 1 ], selected=True ) - else: - select_field.add_option( 'version %s' % option_tup[ 0 ], option_tup[ 1 ] ) - return select_field - - def remove_from_panel( self, tool_id, section_key='', remove_from_config=True ): - - def remove_from_dict( has_elems, integrated_has_elems ): - tool_key = 'tool_%s' % str( tool_id ) - available_tool_versions = self.get_loaded_tools_by_lineage( tool_id ) - if tool_key in has_elems: - if available_tool_versions: - available_tool_versions.reverse() - replacement_tool_key = None - replacement_tool_version = None - # Since we are going to remove the tool from the section, replace it with - # the newest loaded version of the tool. - for available_tool_version in available_tool_versions: - available_tool_section_id, available_tool_section_name = available_tool_version.get_panel_section() - # I suspect "available_tool_version.id in has_elems.keys()" doesn't - # belong in the following line or at least I don't understand what - # purpose it might serve. -John - if available_tool_version.id in has_elems.keys() or (available_tool_section_id == section_key): - replacement_tool_key = 'tool_%s' % str( available_tool_version.id ) - replacement_tool_version = available_tool_version - break - if replacement_tool_key and replacement_tool_version: - # Get the index of the tool_key in the tool_section. - for tool_panel_index, key in enumerate( has_elems.keys() ): - if key == tool_key: - break - # Remove the tool from the tool panel. - del has_elems[ tool_key ] - # Add the replacement tool at the same location in the tool panel. - has_elems.insert( tool_panel_index, - replacement_tool_key, - replacement_tool_version ) - else: - del has_elems[ tool_key ] - else: - del has_elems[ tool_key ] - if remove_from_config: - itegrated_items = integrated_has_elems.panel_items() - if tool_key in itegrated_items: - del itegrated_items[ tool_key ] - - if section_key: - _, tool_section = self.get_section( section_key ) - if tool_section: - remove_from_dict( tool_section.elems, self._integrated_tool_panel.get( section_key, {} ) ) - else: - remove_from_dict( self._tool_panel, self._integrated_tool_panel ) - - def _load_tool_tag_set( self, elem, panel_dict, integrated_panel_dict, tool_path, load_panel_dict, guid=None, index=None ): - try: - path = elem.get( "file" ) - repository_id = None - - tool_shed_repository = None - can_load_into_panel_dict = True - if guid is not None: - # The tool is contained in an installed tool shed repository, so load - # the tool only if the repository has not been marked deleted. - tool_shed_repository = self.__elem_to_tool_shed_repository( elem ) - if tool_shed_repository: - # Only load tools if the repository is not deactivated or uninstalled. - can_load_into_panel_dict = not tool_shed_repository.deleted - repository_id = self.app.security.encode_id( tool_shed_repository.id ) - # Else there is not yet a tool_shed_repository record, we're in the process of installing - # a new repository, so any included tools can be loaded into the tool panel. - tool = self.load_tool( os.path.join( tool_path, path ), guid=guid, repository_id=repository_id ) - if string_as_bool(elem.get( 'hidden', False )): - tool.hidden = True - key = 'tool_%s' % str( tool.id ) - if can_load_into_panel_dict: - if guid is not None: - tool.tool_shed = tool_shed_repository.tool_shed - tool.repository_name = tool_shed_repository.name - tool.repository_owner = tool_shed_repository.owner - tool.installed_changeset_revision = tool_shed_repository.installed_changeset_revision - tool.guid = guid - tool.version = elem.find( "version" ).text - # Make sure the tool has a tool_version. - tool_lineage = self._lineage_map.register( tool, tool_shed_repository=tool_shed_repository ) - # Load the tool's lineage ids. - tool.lineage = tool_lineage - self._tool_tag_manager.handle_tags( tool.id, elem ) - self.__add_tool( tool, load_panel_dict, panel_dict ) - # Always load the tool into the integrated_panel_dict, or it will not be included in the integrated_tool_panel.xml file. - integrated_panel_dict.update_or_append( index, key, tool ) - except IOError: - log.error( "Error reading tool configuration file from path: %s." % path ) - except Exception: - log.exception( "Error reading tool from path: %s" % path ) - - def __add_tool( self, tool, load_panel_dict, panel_dict ): - # Allow for the same tool to be loaded into multiple places in the - # tool panel. We have to handle the case where the tool is contained - # in a repository installed from the tool shed, and the Galaxy - # administrator has retrieved updates to the installed repository. In - # this case, the tool may have been updated, but the version was not - # changed, so the tool should always be reloaded here. We used to - # only load the tool if it was not found in self._tools_by_id, but - # performing that check did not enable this scenario. - self.register_tool( tool ) - if load_panel_dict: - self.__add_tool_to_tool_panel( tool, panel_dict, section=isinstance( panel_dict, ToolSection ) ) - - def _load_workflow_tag_set( self, elem, panel_dict, integrated_panel_dict, load_panel_dict, index=None ): - try: - # TODO: should id be encoded? - workflow_id = elem.get( 'id' ) - workflow = self._load_workflow( workflow_id ) - self._workflows_by_id[ workflow_id ] = workflow - key = 'workflow_' + workflow_id - if load_panel_dict: - panel_dict[ key ] = workflow - # Always load workflows into the integrated_panel_dict. - integrated_panel_dict.update_or_append( index, key, workflow ) - except: - log.exception( "Error loading workflow: %s" % workflow_id ) - - def _load_label_tag_set( self, elem, panel_dict, integrated_panel_dict, load_panel_dict, index=None ): - label = ToolSectionLabel( elem ) - key = 'label_' + label.id - if load_panel_dict: - panel_dict[ key ] = label - integrated_panel_dict.update_or_append( index, key, label ) - - def _load_section_tag_set( self, elem, tool_path, load_panel_dict, index=None ): - key = elem.get( "id" ) - if key in self._tool_panel: - section = self._tool_panel[ key ] - elems = section.elems - else: - section = ToolSection( elem ) - elems = section.elems - if key in self._integrated_tool_panel: - integrated_section = self._integrated_tool_panel[ key ] - integrated_elems = integrated_section.elems - else: - integrated_section = ToolSection( elem ) - integrated_elems = integrated_section.elems - for sub_index, sub_elem in enumerate( elem ): - self.load_item( - sub_elem, - tool_path=tool_path, - panel_dict=elems, - integrated_panel_dict=integrated_elems, - load_panel_dict=load_panel_dict, - guid=sub_elem.get( 'guid' ), - index=sub_index, - internal=True, - ) - if load_panel_dict: - self._tool_panel[ key ] = section - # Always load sections into the integrated_tool_panel. - self._integrated_tool_panel.update_or_append( index, key, integrated_section ) - - def _load_tooldir_tag_set(self, sub_elem, elems, tool_path, integrated_elems, load_panel_dict): - directory = os.path.join( tool_path, sub_elem.attrib.get("dir") ) - recursive = string_as_bool( sub_elem.attrib.get("recursive", True) ) - self.__watch_directory( directory, elems, integrated_elems, load_panel_dict, recursive ) - - def __watch_directory( self, directory, elems, integrated_elems, load_panel_dict, recursive): - - def quick_load( tool_file, async=True ): - try: - tool = self.load_tool( tool_file ) - self.__add_tool( tool, load_panel_dict, elems ) - # Always load the tool into the integrated_panel_dict, or it will not be included in the integrated_tool_panel.xml file. - key = 'tool_%s' % str( tool.id ) - integrated_elems[ key ] = tool - - if async: - self._load_tool_panel() - - if self.app.config.update_integrated_tool_panel: - # Write the current in-memory integrated_tool_panel to the integrated_tool_panel.xml file. - # This will cover cases where the Galaxy administrator manually edited one or more of the tool panel - # config files, adding or removing locally developed tools or workflows. The value of integrated_tool_panel - # will be False when things like functional tests are the caller. - self._write_integrated_tool_panel_config_file() - return tool.id - except Exception: - log.exception("Failed to load potential tool %s." % tool_file) - return None - - tool_loaded = False - for name in os.listdir( directory ): - child_path = os.path.join(directory, name) - if os.path.isdir(child_path) and recursive: - self.__watch_directory(child_path, elems, integrated_elems, load_panel_dict, recursive) - elif name.endswith( ".xml" ): - quick_load( child_path, async=False ) - tool_loaded = True - if tool_loaded: - self._tool_watcher.watch_directory( directory, quick_load ) - - def load_tool( self, config_file, guid=None, repository_id=None, **kwds ): - """Load a single tool from the file named by `config_file` and return an instance of `Tool`.""" - # Parse XML configuration file and get the root element + def create_tool( self, config_file, repository_id=None, guid=None, **kwds ): tool_source = get_tool_source( config_file, getattr( self.app.config, "enable_beta_tool_formats", False ) ) # Allow specifying a different tool subclass to instantiate tool_module = tool_source.parse_tool_module() @@ -904,254 +145,8 @@ ToolClass = Tool tool = ToolClass( config_file, tool_source, self.app, guid=guid, repository_id=repository_id, **kwds ) - tool_id = tool.id - if not tool_id.startswith("__"): - # do not monitor special tools written to tmp directory - no reason - # to monitor such a large directory. - self._tool_watcher.watch_file( config_file, tool.id ) return tool - def load_hidden_tool( self, config_file, **kwds ): - """ Load a hidden tool (in this context meaning one that does not - appear in the tool panel) and register it in _tools_by_id. - """ - tool = self.load_tool( config_file, **kwds ) - self.register_tool( tool ) - return tool - - def register_tool( self, tool ): - tool_id = tool.id - version = tool.version or None - if tool_id not in self._tool_versions_by_id: - self._tool_versions_by_id[ tool_id ] = { version: tool } - else: - self._tool_versions_by_id[ tool_id ][ version ] = tool - if tool_id in self._tools_by_id: - related_tool = self._tools_by_id[ tool_id ] - # This one becomes the default un-versioned tool - # if newer. - if self._newer_tool( tool, related_tool ): - self._tools_by_id[ tool_id ] = tool - else: - self._tools_by_id[ tool_id ] = tool - - def package_tool( self, trans, tool_id ): - """ - Create a tarball with the tool's xml, help images, and test data. - :param trans: the web transaction - :param tool_id: the tool ID from app.toolbox - :returns: tuple of tarball filename, success True/False, message/None - """ - # Make sure the tool is actually loaded. - if tool_id not in self._tools_by_id: - return None, False, "No tool with id %s" % escape( tool_id ) - else: - tool = self._tools_by_id[ tool_id ] - tarball_files = [] - temp_files = [] - tool_xml = file( os.path.abspath( tool.config_file ), 'r' ).read() - # Retrieve tool help images and rewrite the tool's xml into a temporary file with the path - # modified to be relative to the repository root. - image_found = False - if tool.help is not None: - tool_help = tool.help._source - # Check each line of the rendered tool help for an image tag that points to a location under static/ - for help_line in tool_help.split( '\n' ): - image_regex = re.compile( 'img alt="[^"]+" src="\${static_path}/([^"]+)"' ) - matches = re.search( image_regex, help_line ) - if matches is not None: - tool_help_image = matches.group(1) - tarball_path = tool_help_image - filesystem_path = os.path.abspath( os.path.join( trans.app.config.root, 'static', tool_help_image ) ) - if os.path.exists( filesystem_path ): - tarball_files.append( ( filesystem_path, tarball_path ) ) - image_found = True - tool_xml = tool_xml.replace( '${static_path}/%s' % tarball_path, tarball_path ) - # If one or more tool help images were found, add the modified tool XML to the tarball instead of the original. - if image_found: - fd, new_tool_config = tempfile.mkstemp( suffix='.xml' ) - os.close( fd ) - file( new_tool_config, 'w' ).write( tool_xml ) - tool_tup = ( os.path.abspath( new_tool_config ), os.path.split( tool.config_file )[-1] ) - temp_files.append( os.path.abspath( new_tool_config ) ) - else: - tool_tup = ( os.path.abspath( tool.config_file ), os.path.split( tool.config_file )[-1] ) - tarball_files.append( tool_tup ) - # TODO: This feels hacky. - tool_command = tool.command.strip().split()[0] - tool_path = os.path.dirname( os.path.abspath( tool.config_file ) ) - # Add the tool XML to the tuple that will be used to populate the tarball. - if os.path.exists( os.path.join( tool_path, tool_command ) ): - tarball_files.append( ( os.path.join( tool_path, tool_command ), tool_command ) ) - # Find and add macros and code files. - for external_file in tool.get_externally_referenced_paths( os.path.abspath( tool.config_file ) ): - external_file_abspath = os.path.abspath( os.path.join( tool_path, external_file ) ) - tarball_files.append( ( external_file_abspath, external_file ) ) - if os.path.exists( os.path.join( tool_path, "Dockerfile" ) ): - tarball_files.append( ( os.path.join( tool_path, "Dockerfile" ), "Dockerfile" ) ) - # Find tests, and check them for test data. - tests = tool.tests - if tests is not None: - for test in tests: - # Add input file tuples to the list. - for input in test.inputs: - for input_value in test.inputs[ input ]: - input_path = os.path.abspath( os.path.join( 'test-data', input_value ) ) - if os.path.exists( input_path ): - td_tup = ( input_path, os.path.join( 'test-data', input_value ) ) - tarball_files.append( td_tup ) - # And add output file tuples to the list. - for label, filename, _ in test.outputs: - output_filepath = os.path.abspath( os.path.join( 'test-data', filename ) ) - if os.path.exists( output_filepath ): - td_tup = ( output_filepath, os.path.join( 'test-data', filename ) ) - tarball_files.append( td_tup ) - for param in tool.input_params: - # Check for tool data table definitions. - if hasattr( param, 'options' ): - if hasattr( param.options, 'tool_data_table' ): - data_table = param.options.tool_data_table - if hasattr( data_table, 'filenames' ): - data_table_definitions = [] - for data_table_filename in data_table.filenames: - # FIXME: from_shed_config seems to always be False. - if not data_table.filenames[ data_table_filename ][ 'from_shed_config' ]: - tar_file = data_table.filenames[ data_table_filename ][ 'filename' ] + '.sample' - sample_file = os.path.join( data_table.filenames[ data_table_filename ][ 'tool_data_path' ], - tar_file ) - # Use the .sample file, if one exists. If not, skip this data table. - if os.path.exists( sample_file ): - tarfile_path, tarfile_name = os.path.split( tar_file ) - tarfile_path = os.path.join( 'tool-data', tarfile_name ) - tarball_files.append( ( sample_file, tarfile_path ) ) - data_table_definitions.append( data_table.xml_string ) - if len( data_table_definitions ) > 0: - # Put the data table definition XML in a temporary file. - table_definition = '<?xml version="1.0" encoding="utf-8"?>\n<tables>\n %s</tables>' - table_definition = table_definition % '\n'.join( data_table_definitions ) - fd, table_conf = tempfile.mkstemp() - os.close( fd ) - file( table_conf, 'w' ).write( table_definition ) - tarball_files.append( ( table_conf, os.path.join( 'tool-data', 'tool_data_table_conf.xml.sample' ) ) ) - temp_files.append( table_conf ) - # Create the tarball. - fd, tarball_archive = tempfile.mkstemp( suffix='.tgz' ) - os.close( fd ) - tarball = tarfile.open( name=tarball_archive, mode='w:gz' ) - # Add the files from the previously generated list. - for fspath, tarpath in tarball_files: - tarball.add( fspath, arcname=tarpath ) - tarball.close() - # Delete any temporary files that were generated. - for temp_file in temp_files: - os.remove( temp_file ) - return tarball_archive, True, None - return None, False, "An unknown error occurred." - - def reload_tool_by_id( self, tool_id ): - """ - Attempt to reload the tool identified by 'tool_id', if successful - replace the old tool. - """ - if tool_id not in self._tools_by_id: - message = "No tool with id %s" % escape( tool_id ) - status = 'error' - else: - old_tool = self._tools_by_id[ tool_id ] - new_tool = self.load_tool( old_tool.config_file ) - # The tool may have been installed from a tool shed, so set the tool shed attributes. - # Since the tool version may have changed, we don't override it here. - new_tool.id = old_tool.id - new_tool.guid = old_tool.guid - new_tool.tool_shed = old_tool.tool_shed - new_tool.repository_name = old_tool.repository_name - new_tool.repository_owner = old_tool.repository_owner - new_tool.installed_changeset_revision = old_tool.installed_changeset_revision - new_tool.old_id = old_tool.old_id - # Replace old_tool with new_tool in self._tool_panel - tool_key = 'tool_' + tool_id - for key, val in self._tool_panel.items(): - if key == tool_key: - self._tool_panel[ key ] = new_tool - break - elif key.startswith( 'section' ): - if tool_key in val.elems: - self._tool_panel[ key ].elems[ tool_key ] = new_tool - break - self._tools_by_id[ tool_id ] = new_tool - message = "Reloaded the tool:<br/>" - message += "<b>name:</b> %s<br/>" % old_tool.name - message += "<b>id:</b> %s<br/>" % old_tool.id - message += "<b>version:</b> %s" % old_tool.version - status = 'done' - return message, status - - def remove_tool_by_id( self, tool_id, remove_from_panel=True ): - """ - Attempt to remove the tool identified by 'tool_id'. Ignores - tool lineage - so to remove a tool with potentially multiple - versions send remove_from_panel=False and handle the logic of - promoting the next newest version of the tool into the panel - if needed. - """ - if tool_id not in self._tools_by_id: - message = "No tool with id %s" % escape( tool_id ) - status = 'error' - else: - tool = self._tools_by_id[ tool_id ] - del self._tools_by_id[ tool_id ] - if remove_from_panel: - tool_key = 'tool_' + tool_id - for key, val in self._tool_panel.items(): - if key == tool_key: - del self._tool_panel[ key ] - break - elif key.startswith( 'section' ): - if tool_key in val.elems: - del self._tool_panel[ key ].elems[ tool_key ] - break - if tool_id in self.data_manager_tools: - del self.data_manager_tools[ tool_id ] - #TODO: do we need to manually remove from the integrated panel here? - message = "Removed the tool:<br/>" - message += "<b>name:</b> %s<br/>" % tool.name - message += "<b>id:</b> %s<br/>" % tool.id - message += "<b>version:</b> %s" % tool.version - status = 'done' - return message, status - - def get_sections( self ): - for k, v in self._tool_panel.items(): - if isinstance( v, ToolSection ): - yield (v.id, v.name) - - def find_section_id( self, tool_panel_section_id ): - """ - Find the section ID referenced by the key or return '' indicating - no such section id. - """ - if not tool_panel_section_id: - tool_panel_section_id = '' - else: - if tool_panel_section_id not in self._tool_panel: - # Hack introduced without comment in a29d54619813d5da992b897557162a360b8d610c- - # not sure why it is needed. - fixed_tool_panel_section_id = 'section_%s' % tool_panel_section_id - if fixed_tool_panel_section_id in self._tool_panel: - tool_panel_section_id = fixed_tool_panel_section_id - else: - tool_panel_section_id = '' - return tool_panel_section_id - - def _load_workflow( self, workflow_id ): - """ - Return an instance of 'Workflow' identified by `id`, - which is encoded in the tool panel. - """ - id = self.app.security.decode_id( workflow_id ) - stored = self.app.model.context.query( self.app.model.StoredWorkflow ).get( id ) - return stored.latest_workflow - def _init_dependency_manager( self ): self.dependency_manager = build_dependency_manager( self.app.config ) @@ -1162,138 +157,6 @@ if isinstance( tool.tool_action, UploadToolAction ): self.reload_tool_by_id( tool_id ) - @property - def sa_session( self ): - """ - Returns a SQLAlchemy session - """ - return self.app.model.context - - def tool_panel_contents( self, trans, **kwds ): - """ Filter tool_panel contents for displaying for user. - """ - context = Bunch( toolbox=self, trans=trans ) - filters = self._filter_factory.build_filters( trans ) - for _, item_type, elt in self._tool_panel.panel_items_iter(): - elt = _filter_for_panel( elt, item_type, filters, context ) - if elt: - yield elt - - def to_dict( self, trans, in_panel=True, **kwds ): - """ - to_dict toolbox. - """ - if in_panel: - panel_elts = list( self.tool_panel_contents( trans, **kwds ) ) - # Produce panel. - rval = [] - kwargs = dict( - trans=trans, - link_details=True - ) - for elt in panel_elts: - rval.append( elt.to_dict( **kwargs ) ) - else: - tools = [] - for id, tool in self._tools_by_id.items(): - tools.append( tool.to_dict( trans, link_details=True ) ) - rval = tools - - return rval - - def _lineage_in_panel( self, panel_dict, tool=None, tool_lineage=None ): - """ If tool with same lineage already in panel (or section) - find - and return it. Otherwise return None. - """ - if tool_lineage is None: - assert tool is not None - if not hasattr( tool, "lineage" ): - return None - tool_lineage = tool.lineage - lineage_tool_versions = tool_lineage.get_versions( reverse=True ) - for lineage_tool_version in lineage_tool_versions: - lineage_tool = self._tool_from_lineage_version( lineage_tool_version ) - if lineage_tool: - lineage_id = lineage_tool.id - if panel_dict.has_tool_with_id( lineage_id ): - return panel_dict.get_tool_with_id( lineage_id ) - return None - - def _newer_tool( self, tool1, tool2 ): - """ Return True if tool1 is considered "newer" given its own lineage - description. - """ - if not hasattr( tool1, "lineage" ): - return True - lineage_tool_versions = tool1.lineage.get_versions() - for lineage_tool_version in lineage_tool_versions: - lineage_tool = self._tool_from_lineage_version( lineage_tool_version ) - if lineage_tool is tool1: - return False - if lineage_tool is tool2: - return True - return True - - def _tool_from_lineage_version( self, lineage_tool_version ): - if lineage_tool_version.id_based: - return self._tools_by_id.get( lineage_tool_version.id, None ) - else: - return self._tool_versions_by_id.get( lineage_tool_version.id, {} ).get( lineage_tool_version.version, None ) - - -def _filter_for_panel( item, item_type, filters, context ): - """ - Filters tool panel elements so that only those that are compatible - with provided filters are kept. - """ - def _apply_filter( filter_item, filter_list ): - for filter_method in filter_list: - if not filter_method( context, filter_item ): - return False - return True - if item_type == panel_item_types.TOOL: - if _apply_filter( item, filters[ 'tool' ] ): - return item - elif item_type == panel_item_types.LABEL: - if _apply_filter( item, filters[ 'label' ] ): - return item - elif item_type == panel_item_types.SECTION: - # Filter section item-by-item. Only show a label if there are - # non-filtered tools below it. - - if _apply_filter( item, filters[ 'section' ] ): - cur_label_key = None - tools_under_label = False - filtered_elems = item.elems.copy() - for key, section_item_type, section_item in item.panel_items_iter(): - if section_item_type == panel_item_types.TOOL: - # Filter tool. - if _apply_filter( section_item, filters[ 'tool' ] ): - tools_under_label = True - else: - del filtered_elems[ key ] - elif section_item_type == panel_item_types.LABEL: - # If there is a label and it does not have tools, - # remove it. - if ( cur_label_key and not tools_under_label ) or not _apply_filter( section_item, filters[ 'label' ] ): - del filtered_elems[ cur_label_key ] - - # Reset attributes for new label. - cur_label_key = key - tools_under_label = False - - # Handle last label. - if cur_label_key and not tools_under_label: - del filtered_elems[ cur_label_key ] - - # Only return section if there are elements. - if len( filtered_elems ) != 0: - copy = item.copy() - copy.elems = filtered_elems - return copy - - return None - class DefaultToolState( object ): """ diff -r 24aa2e802a351882695cb98d772001cc2a1316d8 -r edacabfca615542ca435a97fc1afcc99daa470d5 lib/galaxy/tools/toolbox/__init__.py --- a/lib/galaxy/tools/toolbox/__init__.py +++ b/lib/galaxy/tools/toolbox/__init__.py @@ -1,16 +1,15 @@ """ API for this module containing functionality related to the tool box. """ -from .tags import tool_tag_manager -from .panel import ToolPanelElements +from .panel import panel_item_types from .panel import ToolSection from .panel import ToolSectionLabel -from .panel import panel_item_types + +from .base import AbstractToolBox __all__ = [ - "ToolPanelElements", - "tool_tag_manager", "ToolSection", "ToolSectionLabel", - "panel_item_types" + "panel_item_types", + "AbstractToolBox", ] This diff is so big that we needed to truncate the remainder. https://bitbucket.org/galaxy/galaxy-central/commits/cb673f24cd75/ Changeset: cb673f24cd75 User: jmchilton Date: 2014-12-31 23:21:10+00:00 Summary: Rework galaxy.tools' galaxy.jobs import. This version of the import I believe is less likely to cause circular import problems (http://effbot.org/zone/import-confusion.htm) Affected #: 1 file diff -r edacabfca615542ca435a97fc1afcc99daa470d5 -r cb673f24cd75ef491aa850373704535691b0a9dd lib/galaxy/tools/__init__.py --- a/lib/galaxy/tools/__init__.py +++ b/lib/galaxy/tools/__init__.py @@ -25,7 +25,7 @@ from mako.template import Template from paste import httpexceptions -from galaxy import jobs, model +from galaxy import model from galaxy.datatypes.metadata import JobExternalOutputMetadataWrapper from galaxy import exceptions from galaxy.tools.actions import DefaultToolAction @@ -69,6 +69,7 @@ DatasetListWrapper, DatasetCollectionWrapper, ) +import galaxy.jobs log = logging.getLogger( __name__ ) @@ -2127,7 +2128,7 @@ json_params = {} json_params[ 'param_dict' ] = self._prepare_json_param_dict( param_dict ) # it would probably be better to store the original incoming parameters here, instead of the Galaxy modified ones? json_params[ 'output_data' ] = [] - json_params[ 'job_config' ] = dict( GALAXY_DATATYPES_CONF_FILE=param_dict.get( 'GALAXY_DATATYPES_CONF_FILE' ), GALAXY_ROOT_DIR=param_dict.get( 'GALAXY_ROOT_DIR' ), TOOL_PROVIDED_JOB_METADATA_FILE=jobs.TOOL_PROVIDED_JOB_METADATA_FILE ) + json_params[ 'job_config' ] = dict( GALAXY_DATATYPES_CONF_FILE=param_dict.get( 'GALAXY_DATATYPES_CONF_FILE' ), GALAXY_ROOT_DIR=param_dict.get( 'GALAXY_ROOT_DIR' ), TOOL_PROVIDED_JOB_METADATA_FILE=galaxy.jobs.TOOL_PROVIDED_JOB_METADATA_FILE ) json_filename = None for i, ( out_name, data ) in enumerate( out_data.iteritems() ): #use wrapped dataset to access certain values @@ -2177,7 +2178,7 @@ json_params = {} json_params[ 'param_dict' ] = self._prepare_json_param_dict( param_dict ) # it would probably be better to store the original incoming parameters here, instead of the Galaxy modified ones? json_params[ 'output_data' ] = [] - json_params[ 'job_config' ] = dict( GALAXY_DATATYPES_CONF_FILE=param_dict.get( 'GALAXY_DATATYPES_CONF_FILE' ), GALAXY_ROOT_DIR=param_dict.get( 'GALAXY_ROOT_DIR' ), TOOL_PROVIDED_JOB_METADATA_FILE=jobs.TOOL_PROVIDED_JOB_METADATA_FILE ) + json_params[ 'job_config' ] = dict( GALAXY_DATATYPES_CONF_FILE=param_dict.get( 'GALAXY_DATATYPES_CONF_FILE' ), GALAXY_ROOT_DIR=param_dict.get( 'GALAXY_ROOT_DIR' ), TOOL_PROVIDED_JOB_METADATA_FILE=galaxy.jobs.TOOL_PROVIDED_JOB_METADATA_FILE ) json_filename = None for i, ( out_name, data ) in enumerate( out_data.iteritems() ): #use wrapped dataset to access certain values https://bitbucket.org/galaxy/galaxy-central/commits/bb548455022c/ Changeset: bb548455022c User: jmchilton Date: 2014-12-31 23:21:10+00:00 Summary: Mostly back out of 54b7789. It changes the behavior in a subtle way under error conditions. Previously if the on-database config file became out of sync with the tool shed database - Galaxy would loose lineage information (tools would be duplicated in side panel) but all tools from the config file would still be loaded up. This change preventing all tools from loading at all in those scenarios so I am undoing it. Affected #: 1 file diff -r cb673f24cd75ef491aa850373704535691b0a9dd -r bb548455022c7ad21c74ba61b4fe37b606737db7 lib/galaxy/tools/toolbox/base.py --- a/lib/galaxy/tools/toolbox/base.py +++ b/lib/galaxy/tools/toolbox/base.py @@ -521,23 +521,6 @@ def tools( self ): return self._tools_by_id.iteritems() - def __elem_to_tool_shed_repository( self, elem ): - # The tool is contained in an installed tool shed repository, so load - # the tool only if the repository has not been marked deleted. - tool_shed = elem.find( "tool_shed" ).text - repository_name = elem.find( "repository_name" ).text - repository_owner = elem.find( "repository_owner" ).text - installed_changeset_revision_elem = elem.find( "installed_changeset_revision" ) - if installed_changeset_revision_elem is None: - # Backward compatibility issue - the tag used to be named 'changeset_revision'. - installed_changeset_revision_elem = elem.find( "changeset_revision" ) - installed_changeset_revision = installed_changeset_revision_elem.text - tool_shed_repository = self.__get_tool_shed_repository( tool_shed, - repository_name, - repository_owner, - installed_changeset_revision ) - return tool_shed_repository - def __get_tool_shed_repository( self, tool_shed, name, owner, installed_changeset_revision ): # We store only the port, if one exists, in the database. tool_shed = common_util.remove_protocol_from_tool_shed_url( tool_shed ) @@ -662,7 +645,19 @@ if guid is not None: # The tool is contained in an installed tool shed repository, so load # the tool only if the repository has not been marked deleted. - tool_shed_repository = self.__elem_to_tool_shed_repository( elem ) + tool_shed = elem.find( "tool_shed" ).text + repository_name = elem.find( "repository_name" ).text + repository_owner = elem.find( "repository_owner" ).text + installed_changeset_revision_elem = elem.find( "installed_changeset_revision" ) + if installed_changeset_revision_elem is None: + # Backward compatibility issue - the tag used to be named 'changeset_revision'. + installed_changeset_revision_elem = elem.find( "changeset_revision" ) + installed_changeset_revision = installed_changeset_revision_elem.text + tool_shed_repository = self.__get_tool_shed_repository( tool_shed, + repository_name, + repository_owner, + installed_changeset_revision ) + if tool_shed_repository: # Only load tools if the repository is not deactivated or uninstalled. can_load_into_panel_dict = not tool_shed_repository.deleted @@ -675,10 +670,10 @@ key = 'tool_%s' % str( tool.id ) if can_load_into_panel_dict: if guid is not None: - tool.tool_shed = tool_shed_repository.tool_shed - tool.repository_name = tool_shed_repository.name - tool.repository_owner = tool_shed_repository.owner - tool.installed_changeset_revision = tool_shed_repository.installed_changeset_revision + tool.tool_shed = tool_shed + tool.repository_name = repository_name + tool.repository_owner = repository_owner + tool.installed_changeset_revision = installed_changeset_revision tool.guid = guid tool.version = elem.find( "version" ).text # Make sure the tool has a tool_version. Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: jmchilton: Bugfix for functional tests broken with 68daf6d.
by commits-noreply＠bitbucket.org 30 Dec '14

30 Dec '14

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/02efb1123d46/ Changeset: 02efb1123d46 User: jmchilton Date: 2014-12-30 14:43:28+00:00 Summary: Bugfix for functional tests broken with 68daf6d. Affected #: 1 file diff -r 87825915116972a38b2f0d18943ef2959d674f97 -r 02efb1123d469986a051e7a8015c9d56635efd8a lib/galaxy/tools/__init__.py --- a/lib/galaxy/tools/__init__.py +++ b/lib/galaxy/tools/__init__.py @@ -573,6 +573,11 @@ return [ tool ] return [] + @property + def tools_by_id( self ): + # Deprecated method, TODO - eliminate calls to this in test/. + return self._tools_by_id + def tools( self ): return self._tools_by_id.iteritems() Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: 5 new changesets
by commits-noreply＠bitbucket.org 29 Dec '14

29 Dec '14

5 new commits in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/d62272c55cf4/ Changeset: d62272c55cf4 User: jmchilton Date: 2014-12-30 04:07:01+00:00 Summary: Override ToolSectionLabel.to_dict so don't need to_dict_helper. I feel this is cleaner and it makes ToolSection and ToolSectionLabel easier to move into a new module. Affected #: 1 file diff -r fa404b077f67ff492db5d8c3405ea57cdb2d1420 -r d62272c55cf4d99fcb5f22111de7a4b5e104190e lib/galaxy/tools/__init__.py --- a/lib/galaxy/tools/__init__.py +++ b/lib/galaxy/tools/__init__.py @@ -113,16 +113,6 @@ pass -def to_dict_helper( obj, kwargs ): - """ Helper function that provides the appropriate kwargs to to_dict an object. """ - - # Label.to_dict cannot have kwargs. - if isinstance( obj, ToolSectionLabel ): - kwargs = {} - - return obj.to_dict( **kwargs ) - - class ToolBox( object, Dictifiable ): """Container for a collection of tools""" @@ -1189,7 +1179,7 @@ link_details=True ) for elt in panel_elts: - rval.append( to_dict_helper( elt, kwargs ) ) + rval.append( elt.to_dict( **kwargs ) ) else: tools = [] for id, tool in self._tools_by_id.items(): @@ -1311,7 +1301,7 @@ link_details=link_details ) for elt in self.elems.values(): - section_elts.append( to_dict_helper( elt, kwargs ) ) + section_elts.append( elt.to_dict( **kwargs ) ) section_dict[ 'elems' ] = section_elts return section_dict @@ -1333,6 +1323,9 @@ self.id = elem.get( "id" ) self.version = elem.get( "version" ) or '' + def to_dict( self, **kwds ): + return super( ToolSectionLabel, self ).to_dict() + class DefaultToolState( object ): """ https://bitbucket.org/galaxy/galaxy-central/commits/7b64cb79e2b0/ Changeset: 7b64cb79e2b0 User: jmchilton Date: 2014-12-30 04:07:01+00:00 Summary: Various ToolBox filtering improvements and tests. - Allow overriding the base module location for ToolBox filters (needed for OS packages, etc...). - Allow scanning multiple base modules for filters. - Unit tests for module loading functionality, custom tool, label, and section filters, default hidden and require_login filters. Affected #: 5 files diff -r d62272c55cf4d99fcb5f22111de7a4b5e104190e -r 7b64cb79e2b0fc3a7bd2366ca8af6f140ed78a08 lib/galaxy/config.py --- a/lib/galaxy/config.py +++ b/lib/galaxy/config.py @@ -83,6 +83,7 @@ # The value of migrated_tools_config is the file reserved for containing only those tools that have been eliminated from the distribution # and moved to the tool shed. self.integrated_tool_panel_config = resolve_path( kwargs.get( 'integrated_tool_panel_config', 'integrated_tool_panel.xml' ), self.root ) + self.toolbox_filter_base_modules = listify( kwargs.get( "toolbox_filter_base_modules", "galaxy.tools.filters" ) ) self.tool_filters = listify( kwargs.get( "tool_filters", [] ), do_strip=True ) self.tool_label_filters = listify( kwargs.get( "tool_label_filters", [] ), do_strip=True ) self.tool_section_filters = listify( kwargs.get( "tool_section_filters", [] ), do_strip=True ) diff -r d62272c55cf4d99fcb5f22111de7a4b5e104190e -r 7b64cb79e2b0fc3a7bd2366ca8af6f140ed78a08 lib/galaxy/tools/filters/__init__.py --- a/lib/galaxy/tools/filters/__init__.py +++ b/lib/galaxy/tools/filters/__init__.py @@ -1,6 +1,8 @@ +from copy import deepcopy +import sys import logging + from galaxy.util import listify -from copy import deepcopy log = logging.getLogger( __name__ ) @@ -20,6 +22,7 @@ self.default_filters = dict( tool=[ _not_hidden, _handle_requires_login ], section=[], label=[] ) # Add dynamic filters to these default filters. config = toolbox.app.config + self.__base_modules = listify( getattr( config, "toolbox_filter_base_modules", "galaxy.tools.filters" ) ) self.__init_filters( "tool", getattr( config, "tool_filters", "" ), self.default_filters ) self.__init_filters( "section", getattr( config, "tool_section_filters", "" ), self.default_filters ) self.__init_filters( "label", getattr( config, "tool_label_filters", "" ), self.default_filters ) @@ -65,14 +68,27 @@ if ":" in filter_name: # Should be a submodule of filters (e.g. examples:restrict_development_tools) (module_name, function_name) = filter_name.rsplit(":", 1) - module = __import__( module_name.strip(), globals() ) - function = getattr( module, function_name.strip() ) + function = self._import_filter( module_name, function_name ) else: # No module found, just load a function from this file or # one that has be explicitly imported. function = getattr( globals(), filter_name.strip() ) return function + def _import_filter( self, module_name, function_name ): + function_name = function_name.strip() + for base_module in self.__base_modules: + full_module_name = "%s.%s" % ( base_module, module_name.strip() ) + try: + __import__( full_module_name ) + except ImportError: + # log.debug("Failed to load module.", exc_info=True) + continue + module = sys.modules[ full_module_name ] + if hasattr( module, function_name ): + return getattr( module, function_name ) + raise Exception("Failed to find filter %s.%s" % (module_name, function_name)) + ## Stock Filter Functions def _not_hidden( context, tool ): diff -r d62272c55cf4d99fcb5f22111de7a4b5e104190e -r 7b64cb79e2b0fc3a7bd2366ca8af6f140ed78a08 test/unit/tools/filter_modules/filtermod.py --- /dev/null +++ b/test/unit/tools/filter_modules/filtermod.py @@ -0,0 +1,22 @@ +""" Test filters used by test_toolbox_filters.py. +""" + + +def filter_tool( context, tool ): + """Test Filter Tool""" + return False + + +def filter_section( context, section ): + """Test Filter Section""" + return False + + +def filter_label_1( context, label ): + """Test Filter Label 1""" + return False + + +def filter_label_2( context, label ): + """Test Filter Label 2""" + return False diff -r d62272c55cf4d99fcb5f22111de7a4b5e104190e -r 7b64cb79e2b0fc3a7bd2366ca8af6f140ed78a08 test/unit/tools/test_toolbox_filters.py --- /dev/null +++ b/test/unit/tools/test_toolbox_filters.py @@ -0,0 +1,79 @@ +from galaxy.util.bunch import Bunch + +from galaxy.tools.filters import FilterFactory + + +def test_stock_filtering_requires_login_tools( ): + anonymous_user_trans = mock_trans( has_user=False ) + filters = filter_factory( {} ).build_filters( mock_trans() )[ 'tool' ] + assert not is_filtered( filters, anonymous_user_trans, mock_tool( require_login=False ) ) + assert is_filtered( filters, anonymous_user_trans, mock_tool( require_login=True ) ) + + logged_in_trans = mock_trans( has_user=True ) + filters = filter_factory( {} ).build_filters( logged_in_trans )[ 'tool' ] + assert not is_filtered( filters, logged_in_trans, mock_tool( require_login=True ) ) + + +def test_stock_filtering_hidden_tools( ): + filters = filter_factory( {} ).build_filters( mock_trans() )[ 'tool' ] + assert not is_filtered( filters, mock_trans(), mock_tool( hidden=False ) ) + assert is_filtered( filters, mock_trans(), mock_tool( hidden=True ) ) + + +def test_trackster_filtering( ): + filters = filter_factory( {} ).build_filters( mock_trans(), trackster=True )[ 'tool' ] + # Ekkk... is trackster_conf broken? Why is it showing up. + # assert is_filtered( filters, mock_trans(), mock_tool( trackster_conf=False ) ) + assert not is_filtered( filters, mock_trans(), mock_tool( trackster_conf=True ) ) + + +def test_custom_filters(): + filters = filter_factory().build_filters( mock_trans() ) + tool_filters = filters[ "tool" ] + # TODO: the fact that -1 is the custom filter is an implementation + # detail that should not be tested here. + assert tool_filters[ -1 ].__doc__ == "Test Filter Tool" + + section_filters = filters[ "section" ] + assert section_filters[ 0 ].__doc__ == "Test Filter Section" + + label_filters = filters[ "label" ] + assert label_filters[ 0 ].__doc__ == "Test Filter Label 1" + assert label_filters[ 1 ].__doc__ == "Test Filter Label 2" + + +def filter_factory(config_dict=None): + if config_dict is None: + config_dict = dict( + tool_filters=["filtermod:filter_tool"], + tool_section_filters=["filtermod:filter_section"], + tool_label_filters=["filtermod:filter_label_1", "filtermod:filter_label_2"], + ) + config = Bunch( **config_dict ) + config.toolbox_filter_base_modules = "galaxy.tools.filters,tools.filter_modules" + app = Bunch(config=config) + toolbox = Bunch(app=app) + return FilterFactory(toolbox) + + +def is_filtered( filters, trans, tool ): + context = Bunch( trans=trans ) + return not all( map( lambda filter: filter( context, tool ), filters ) ) + + +def mock_tool( require_login=False, hidden=False, trackster_conf=False ): + tool = Bunch( + require_login=require_login, + hidden=hidden, + trackster_conf=trackster_conf, + ) + return tool + + +def mock_trans( has_user=True ): + trans = Bunch( ) + if has_user: + trans.user = Bunch(preferences={}) + else: + trans.user = None + return trans https://bitbucket.org/galaxy/galaxy-central/commits/4a96516a503c/ Changeset: 4a96516a503c User: jmchilton Date: 2014-12-30 04:07:01+00:00 Summary: Move galaxy.tools.watcher into galaxy.tools.toolbox.watcher. Affected #: 4 files diff -r 7b64cb79e2b0fc3a7bd2366ca8af6f140ed78a08 -r 4a96516a503cc3bfae4d26b431f8ad8c67496074 lib/galaxy/tools/__init__.py --- a/lib/galaxy/tools/__init__.py +++ b/lib/galaxy/tools/__init__.py @@ -32,7 +32,7 @@ from galaxy import jobs, model from galaxy.datatypes.metadata import JobExternalOutputMetadataWrapper from galaxy import exceptions -from galaxy.tools import watcher +from galaxy.tools.toolbox import watcher from galaxy.tools.actions import DefaultToolAction from galaxy.tools.actions.upload import UploadToolAction from galaxy.tools.actions.data_source import DataSourceToolAction diff -r 7b64cb79e2b0fc3a7bd2366ca8af6f140ed78a08 -r 4a96516a503cc3bfae4d26b431f8ad8c67496074 lib/galaxy/tools/toolbox/watcher.py --- /dev/null +++ b/lib/galaxy/tools/toolbox/watcher.py @@ -0,0 +1,99 @@ +import os.path +try: + from watchdog.events import FileSystemEventHandler + from watchdog.observers import Observer + can_watch = True +except ImportError: + FileSystemEventHandler = object + can_watch = False + +import logging +log = logging.getLogger( __name__ ) + + +def get_watcher(toolbox, config): + watch_tools = getattr(config, "watch_tools", False) + if watch_tools: + return ToolWatcher(toolbox) + else: + return NullWatcher() + + +class ToolWatcher(object): + + def __init__(self, toolbox): + if not can_watch: + raise Exception("Watchdog library unavailble, cannot watch tools.") + self.toolbox = toolbox + self.tool_file_ids = {} + self.tool_dir_callbacks = {} + self.monitored_dirs = {} + self.observer = Observer() + self.event_handler = ToolFileEventHandler(self) + self.start() + + def start(self): + self.observer.start() + + def monitor(self, dir): + self.observer.schedule(self.event_handler, dir, recursive=False) + + def watch_file(self, tool_file, tool_id): + tool_file = os.path.abspath( tool_file ) + self.tool_file_ids[tool_file] = tool_id + tool_dir = os.path.dirname( tool_file ) + if tool_dir not in self.monitored_dirs: + self.monitored_dirs[ tool_dir ] = tool_dir + self.monitor( tool_dir ) + + def watch_directory(self, tool_dir, callback): + tool_dir = os.path.abspath( tool_dir ) + self.tool_dir_callbacks[tool_dir] = callback + if tool_dir not in self.monitored_dirs: + self.monitored_dirs[ tool_dir ] = tool_dir + self.monitor( tool_dir ) + + +class ToolFileEventHandler(FileSystemEventHandler): + + def __init__(self, tool_watcher): + self.tool_watcher = tool_watcher + + def on_any_event(self, event): + self._handle(event) + + def _handle(self, event): + # modified events will only have src path, move events will + # have dest_path and src_path but we only care about dest. So + # look at dest if it exists else use src. + path = getattr( event, 'dest_path', None ) or event.src_path + path = os.path.abspath( path ) + tool_id = self.tool_watcher.tool_file_ids.get( path, None ) + if tool_id: + try: + self.tool_watcher.toolbox.reload_tool_by_id(tool_id) + except Exception: + pass + elif path.endswith(".xml"): + directory = os.path.dirname( path ) + dir_callback = self.tool_watcher.tool_dir_callbacks.get( directory, None ) + if dir_callback: + tool_file = event.src_path + tool_id = dir_callback( tool_file ) + if tool_id: + self.tool_watcher.tool_file_ids[ tool_file ] = tool_id + + +class NullWatcher(object): + + def start(self): + pass + + def shutdown(self): + pass + + def watch_file(self, tool_file, tool_id): + pass + + def watch_directory(self, tool_dir, callback): + pass diff -r 7b64cb79e2b0fc3a7bd2366ca8af6f140ed78a08 -r 4a96516a503cc3bfae4d26b431f8ad8c67496074 lib/galaxy/tools/watcher.py --- a/lib/galaxy/tools/watcher.py +++ /dev/null @@ -1,99 +0,0 @@ -import os.path -try: - from watchdog.events import FileSystemEventHandler - from watchdog.observers import Observer - can_watch = True -except ImportError: - FileSystemEventHandler = object - can_watch = False - -import logging -log = logging.getLogger( __name__ ) - - -def get_watcher(toolbox, config): - watch_tools = getattr(config, "watch_tools", False) - if watch_tools: - return ToolWatcher(toolbox) - else: - return NullWatcher() - - -class ToolWatcher(object): - - def __init__(self, toolbox): - if not can_watch: - raise Exception("Watchdog library unavailble, cannot watch tools.") - self.toolbox = toolbox - self.tool_file_ids = {} - self.tool_dir_callbacks = {} - self.monitored_dirs = {} - self.observer = Observer() - self.event_handler = ToolFileEventHandler(self) - self.start() - - def start(self): - self.observer.start() - - def monitor(self, dir): - self.observer.schedule(self.event_handler, dir, recursive=False) - - def watch_file(self, tool_file, tool_id): - tool_file = os.path.abspath( tool_file ) - self.tool_file_ids[tool_file] = tool_id - tool_dir = os.path.dirname( tool_file ) - if tool_dir not in self.monitored_dirs: - self.monitored_dirs[ tool_dir ] = tool_dir - self.monitor( tool_dir ) - - def watch_directory(self, tool_dir, callback): - tool_dir = os.path.abspath( tool_dir ) - self.tool_dir_callbacks[tool_dir] = callback - if tool_dir not in self.monitored_dirs: - self.monitored_dirs[ tool_dir ] = tool_dir - self.monitor( tool_dir ) - - -class ToolFileEventHandler(FileSystemEventHandler): - - def __init__(self, tool_watcher): - self.tool_watcher = tool_watcher - - def on_any_event(self, event): - self._handle(event) - - def _handle(self, event): - # modified events will only have src path, move events will - # have dest_path and src_path but we only care about dest. So - # look at dest if it exists else use src. - path = getattr( event, 'dest_path', None ) or event.src_path - path = os.path.abspath( path ) - tool_id = self.tool_watcher.tool_file_ids.get( path, None ) - if tool_id: - try: - self.tool_watcher.toolbox.reload_tool_by_id(tool_id) - except Exception: - pass - elif path.endswith(".xml"): - directory = os.path.dirname( path ) - dir_callback = self.tool_watcher.tool_dir_callbacks.get( directory, None ) - if dir_callback: - tool_file = event.src_path - tool_id = dir_callback( tool_file ) - if tool_id: - self.tool_watcher.tool_file_ids[ tool_file ] = tool_id - - -class NullWatcher(object): - - def start(self): - pass - - def shutdown(self): - pass - - def watch_file(self, tool_file, tool_id): - pass - - def watch_directory(self, tool_dir, callback): - pass diff -r 7b64cb79e2b0fc3a7bd2366ca8af6f140ed78a08 -r 4a96516a503cc3bfae4d26b431f8ad8c67496074 test/unit/tools/test_watcher.py --- a/test/unit/tools/test_watcher.py +++ b/test/unit/tools/test_watcher.py @@ -4,7 +4,7 @@ import tempfile import time -from galaxy.tools import watcher +from galaxy.tools.toolbox import watcher from galaxy.util import bunch https://bitbucket.org/galaxy/galaxy-central/commits/507483359a67/ Changeset: 507483359a67 User: jmchilton Date: 2014-12-30 04:07:01+00:00 Summary: Remove lib/galaxy/tools/filters/examples.py likely incorrectly added in d816d66. Affected #: 1 file diff -r 4a96516a503cc3bfae4d26b431f8ad8c67496074 -r 507483359a67da800ce7a56536d31b7911211474 lib/galaxy/tools/filters/examples.py --- a/lib/galaxy/tools/filters/examples.py +++ /dev/null @@ -1,115 +0,0 @@ -import logging -log = logging.getLogger( __name__ ) - - -def restrict_encode( content, tool ): - """ - Disable the random interval ENCODE tool - - This tool filter will disable all the ENCODE tool when enabled. - """ - if tool.id == 'random_intervals1': - return False - return True - - -def restrict_text( content, section ): - """ - Disable Text sections - - This tool filter will disable all Tools groups under a 'Text' section when enabled. - """ - if section.name.find('Text') != -1: - return False - return True - - -def restrict_upload_to_admins( context, tool ): - """ - Disable Upload tool for all non-admin users. - - This tool filter will hide the upload tool from all users except admin - users. - - This can be enabled by renaming this file to examples.py and adding - the following to the ``app:main`` section of ``galaxy.ini``: - - tool_filters = examples:restrict_upload_to_admins - """ - if tool.name == "Upload File": - return context.trans.user_is_admin() - return True - - -def disable_gatk( context, tool ): - """ - This tool filter will disable all gatk tools when enabled. - - This can be enabled by renaming this file to examples.py and adding the following to the - ``app:main`` section of ``galaxy.ini``: - - tool_filters = examples:disable_gatk - """ - return not any( [ requirement.name == "gatk" for requirement in tool.requirements ] ) - - -def explicit_user_mapping( context, section ): - """ - This tool section filter uses an explicit mapping to describe what users can view - which tool sections. Anonymous users will only be able to view the "Get Data" - tool section (with id getext). This can be enabled by renaming this file to - examples.py and adding the following to the ``app:main`` section of - ``galaxy.ini``: - - tool_section_filters = examples:explicit_user_mapping - """ - users_sections = { - None: [ "getext" ], - "bob(a)example.com": [ "getext", "textutil", "filter" ], - "mary(a)example.com": [ "getext", "textutil", "filter", "ngs" ], - } - user = context.trans.user - email = user and user.email - valid_sections = users_sections.get( email, [] ) - return section.id in valid_sections - - -DEVELOPERS = [ "mary(a)example.com" ] - - -def restrict_development_tools( context, tool ): - """ - This tool filter will disable all tools with the string alpha appearing in - the version for all users except those explicitly appearing in the DEVELOPERS list - defined above. This can be enabled by renaming this file to examples.py and - adding the following to the ``app:main`` section of ``galaxy.ini``: - - tool_filters = examples:restrict_development_tools - """ - version = tool.version - user = context.trans.user - email = user and user.email - return "alpha" not in version or email in DEVELOPERS - - -def per_host_tool_sections( context, section ): - """ - This tool section filter results in different sections being display based on - the URL the user is making the request to. This could allow a single Galaxy instance - to seem like several different instances hosting different tools based on the URL used - to access the Galxy. This can be enabled by renaming this file to examples.py and adding - the following to the ``app:main`` section of ``galaxy.ini``: - - tool_section_filters = examples:per_host_tool_sections - """ - host = context.trans.request.host - # Core tools used by all virtual hosts. - valid_sections = [ "getext", "textutil", "filter" ] - if "ngs.galaxy.example.com" in host: - valid_sections += [ "ngs" ] - elif "microarray.galaxy.example.com" in host: - valid_sections += [ "microarray" ] - elif "proteomics.galaxy.example.com" in host: - valid_sections += [ "proteomics" ] - return section.id in valid_sections - https://bitbucket.org/galaxy/galaxy-central/commits/878259151169/ Changeset: 878259151169 User: jmchilton Date: 2014-12-30 04:07:01+00:00 Summary: Move galaxy.tools.filters to galaxy.tools.toolbox.filters. Leave empty module in galaxy.tools.filters and modify config to ensure backward compatibility (filters in this old directory will continue to work for now). Update config/galaxy.ini.sample with more discussion of ToolBox filter. Affected #: 8 files diff -r 507483359a67da800ce7a56536d31b7911211474 -r 87825915116972a38b2f0d18943ef2959d674f97 config/galaxy.ini.sample --- a/config/galaxy.ini.sample +++ b/config/galaxy.ini.sample @@ -910,17 +910,27 @@ #cache_user_job_count = False # ToolBox filtering -# Modules from lib/galaxy/tools/filters/ can be specified in the following lines. -# tool_* filters will be applied for all users and can not be changed by them. -# user_tool_* filters will be shown under user preferences and can be toogled -# on and off by runtime -#tool_filters -#tool_label_filters -#tool_section_filters + +# Modules from lib/galaxy/tools/toolbox/filters/ can be specified in +# the following lines. tool_* filters will be applied for all users +# and can not be changed by them. user_tool_* filters will be shown +# under user preferences and can be toogled on and off by +# runtime. Example shown below are not real defaults (no custom +# filters applied by defualt) but can be enabled with by renaming the +# example.py.sample in the filters directory to example.py. + +#tool_filters = +#tool_label_filters = +#tool_section_filters = #user_tool_filters = examples:restrict_upload_to_admins, examples:restrict_encode #user_tool_section_filters = examples:restrict_text #user_tool_label_filters = examples:restrict_upload_to_admins, examples:restrict_encode +# The base modules that are searched for modules as described above +# can be modified and modules external to Galaxy can be searched by +# modifying the following option. +#toolbox_filter_base_modules = galaxy.tools.toolbox.filters,galaxy.tools.filters + # Galaxy Application Internal Message Queue # Galaxy uses AMQP internally TODO more documentation on what for. diff -r 507483359a67da800ce7a56536d31b7911211474 -r 87825915116972a38b2f0d18943ef2959d674f97 lib/galaxy/config.py --- a/lib/galaxy/config.py +++ b/lib/galaxy/config.py @@ -83,7 +83,7 @@ # The value of migrated_tools_config is the file reserved for containing only those tools that have been eliminated from the distribution # and moved to the tool shed. self.integrated_tool_panel_config = resolve_path( kwargs.get( 'integrated_tool_panel_config', 'integrated_tool_panel.xml' ), self.root ) - self.toolbox_filter_base_modules = listify( kwargs.get( "toolbox_filter_base_modules", "galaxy.tools.filters" ) ) + self.toolbox_filter_base_modules = listify( kwargs.get( "toolbox_filter_base_modules", "galaxy.tools.filters,galaxy.tools.toolbox.filters" ) ) self.tool_filters = listify( kwargs.get( "tool_filters", [] ), do_strip=True ) self.tool_label_filters = listify( kwargs.get( "tool_label_filters", [] ), do_strip=True ) self.tool_section_filters = listify( kwargs.get( "tool_section_filters", [] ), do_strip=True ) diff -r 507483359a67da800ce7a56536d31b7911211474 -r 87825915116972a38b2f0d18943ef2959d674f97 lib/galaxy/tools/__init__.py --- a/lib/galaxy/tools/__init__.py +++ b/lib/galaxy/tools/__init__.py @@ -48,7 +48,7 @@ from galaxy.tools.parameters.input_translation import ToolInputTranslator from galaxy.tools.parameters.output import ToolOutputActionGroup from galaxy.tools.parameters.validation import LateValidationError -from galaxy.tools.filters import FilterFactory +from galaxy.tools.toolbox.filters import FilterFactory from galaxy.tools.test import parse_tests from galaxy.tools.parser import get_tool_source from galaxy.tools.parser.xml import XmlPageSource diff -r 507483359a67da800ce7a56536d31b7911211474 -r 87825915116972a38b2f0d18943ef2959d674f97 lib/galaxy/tools/filters/__init__.py --- a/lib/galaxy/tools/filters/__init__.py +++ b/lib/galaxy/tools/filters/__init__.py @@ -1,103 +1,5 @@ -from copy import deepcopy -import sys -import logging +"""Deprecated module for toolbox filters. -from galaxy.util import listify - -log = logging.getLogger( __name__ ) - - -class FilterFactory( object ): - """ - An instance of this class is responsible for filtering the list - of tools presented to a given user in a given context. - """ - - def __init__( self, toolbox ): - self.toolbox = toolbox - - # Prepopulate dict containing filters that are always checked, - # other filters that get checked depending on context (e.g. coming from - # trackster or no user found are added in build filters). - self.default_filters = dict( tool=[ _not_hidden, _handle_requires_login ], section=[], label=[] ) - # Add dynamic filters to these default filters. - config = toolbox.app.config - self.__base_modules = listify( getattr( config, "toolbox_filter_base_modules", "galaxy.tools.filters" ) ) - self.__init_filters( "tool", getattr( config, "tool_filters", "" ), self.default_filters ) - self.__init_filters( "section", getattr( config, "tool_section_filters", "" ), self.default_filters ) - self.__init_filters( "label", getattr( config, "tool_label_filters", "" ), self.default_filters ) - - def build_filters( self, trans, **kwds ): - """ - Build list of filters to check tools against given current context. - """ - filters = deepcopy( self.default_filters ) - if trans.user: - for name, value in trans.user.preferences.items(): - if value.strip(): - user_filters = listify( value, do_strip=True ) - category = '' - if name == 'toolbox_tool_filters': - category = "tool" - elif name == 'toolbox_section_filters': - category = "section" - elif name == 'toolbox_label_filters': - category = "label" - if category: - validate = getattr( trans.app.config, 'user_%s_filters' % category, [] ) - self.__init_filters( category, user_filters, filters, validate=validate ) - else: - if kwds.get( "trackster", False ): - filters[ "tool" ].append( _has_trackster_conf ) - - return filters - - def __init_filters( self, key, filters, toolbox_filters, validate=None ): - for filter in filters: - if validate is None or filter in validate or filter in self.default_filters: - filter_function = self.__build_filter_function( filter ) - toolbox_filters[ key ].append( filter_function ) - else: - log.warning( "Refusing to load %s filter '%s' which is not defined in config", key, filter ) - return toolbox_filters - - def __build_filter_function( self, filter_name ): - """Obtain python function (importing a submodule if needed) - corresponding to filter_name. - """ - if ":" in filter_name: - # Should be a submodule of filters (e.g. examples:restrict_development_tools) - (module_name, function_name) = filter_name.rsplit(":", 1) - function = self._import_filter( module_name, function_name ) - else: - # No module found, just load a function from this file or - # one that has be explicitly imported. - function = getattr( globals(), filter_name.strip() ) - return function - - def _import_filter( self, module_name, function_name ): - function_name = function_name.strip() - for base_module in self.__base_modules: - full_module_name = "%s.%s" % ( base_module, module_name.strip() ) - try: - __import__( full_module_name ) - except ImportError: - # log.debug("Failed to load module.", exc_info=True) - continue - module = sys.modules[ full_module_name ] - if hasattr( module, function_name ): - return getattr( module, function_name ) - raise Exception("Failed to find filter %s.%s" % (module_name, function_name)) - - -## Stock Filter Functions -def _not_hidden( context, tool ): - return not tool.hidden - - -def _handle_requires_login( context, tool ): - return not tool.require_login or context.trans.user - - -def _has_trackster_conf( context, tool ): - return tool.trackster_conf +Filters placed in this module will still work - but filters should be +moved to lib/galaxy/tools/toolbox/filters. +""" diff -r 507483359a67da800ce7a56536d31b7911211474 -r 87825915116972a38b2f0d18943ef2959d674f97 lib/galaxy/tools/filters/examples.py.sample --- a/lib/galaxy/tools/filters/examples.py.sample +++ /dev/null @@ -1,87 +0,0 @@ -import logging -log = logging.getLogger( __name__ ) - - -def restrict_upload_to_admins( context, tool ): - """ - This tool filter will hide the upload tool from all users except admin - users. This can be enabled by renaming this file to examples.py and adding - the following to the ``app:main`` section of ``galaxy.ini``: - - tool_filters = examples:restrict_upload_to_admins - """ - if tool.name == "Upload File": - return context.trans.user_is_admin() - return True - - -def disable_gatk( context, tool ): - """ - This tool filter will disable all gatk tools when enabled. This can be - enabled by renaming this file to examples.py and adding the following to the - ``app:main`` section of ``galaxy.ini``: - - tool_filters = examples:disable_gatk - """ - return not any( [ requirement.name == "gatk" for requirement in tool.requirements ] ) - - -def explicit_user_mapping( context, section ): - """ - This tool section filter uses an explicit mapping to describe what users can view - which tool sections. Anonymous users will only be able to view the "Get Data" - tool section (with id getext). This can be enabled by renaming this file to - examples.py and adding the following to the ``app:main`` section of - ``galaxy.ini``: - - tool_section_filters = examples:explicit_user_mapping - """ - users_sections = { - None: [ "getext" ], - "bob(a)example.com": [ "getext", "textutil", "filter" ], - "mary(a)example.com": [ "getext", "textutil", "filter", "ngs" ], - } - user = context.trans.user - email = user and user.email - valid_sections = users_sections.get( email, [] ) - return section.id in valid_sections - - -DEVELOPERS = [ "mary(a)example.com" ] - - -def restrict_development_tools( context, tool ): - """ - This tool filter will disable all tools with the string alpha appearing in - the version for all users except those explicitly appearing in the DEVELOPERS list - defined above. This can be enabled by renaming this file to examples.py and - adding the following to the ``app:main`` section of ``galaxy.ini``: - - tool_filters = examples:restrict_development_tools - """ - version = tool.version - user = context.trans.user - email = user and user.email - return "alpha" not in version or email in DEVELOPERS - - -def per_host_tool_sections( context, section ): - """ - This tool section filter results in different sections being display based on - the URL the user is making the request to. This could allow a single Galaxy instance - to seem like several different instances hosting different tools based on the URL used - to access the Galxy. This can be enabled by renaming this file to examples.py and adding - the following to the ``app:main`` section of ``galaxy.ini``: - - tool_section_filters = examples:per_host_tool_sections - """ - host = context.trans.request.host - # Core tools used by all virtual hosts. - valid_sections = [ "getext", "textutil", "filter" ] - if "ngs.galaxy.example.com" in host: - valid_sections += [ "ngs" ] - elif "microarray.galaxy.example.com" in host: - valid_sections += [ "microarray" ] - elif "proteomics.galaxy.example.com" in host: - valid_sections += [ "proteomics" ] - return section.id in valid_sections diff -r 507483359a67da800ce7a56536d31b7911211474 -r 87825915116972a38b2f0d18943ef2959d674f97 lib/galaxy/tools/toolbox/filters/__init__.py --- /dev/null +++ b/lib/galaxy/tools/toolbox/filters/__init__.py @@ -0,0 +1,103 @@ +from copy import deepcopy +import sys +import logging + +from galaxy.util import listify + +log = logging.getLogger( __name__ ) + + +class FilterFactory( object ): + """ + An instance of this class is responsible for filtering the list + of tools presented to a given user in a given context. + """ + + def __init__( self, toolbox ): + self.toolbox = toolbox + + # Prepopulate dict containing filters that are always checked, + # other filters that get checked depending on context (e.g. coming from + # trackster or no user found are added in build filters). + self.default_filters = dict( tool=[ _not_hidden, _handle_requires_login ], section=[], label=[] ) + # Add dynamic filters to these default filters. + config = toolbox.app.config + self.__base_modules = listify( getattr( config, "toolbox_filter_base_modules", "galaxy.tools.filters" ) ) + self.__init_filters( "tool", getattr( config, "tool_filters", "" ), self.default_filters ) + self.__init_filters( "section", getattr( config, "tool_section_filters", "" ), self.default_filters ) + self.__init_filters( "label", getattr( config, "tool_label_filters", "" ), self.default_filters ) + + def build_filters( self, trans, **kwds ): + """ + Build list of filters to check tools against given current context. + """ + filters = deepcopy( self.default_filters ) + if trans.user: + for name, value in trans.user.preferences.items(): + if value.strip(): + user_filters = listify( value, do_strip=True ) + category = '' + if name == 'toolbox_tool_filters': + category = "tool" + elif name == 'toolbox_section_filters': + category = "section" + elif name == 'toolbox_label_filters': + category = "label" + if category: + validate = getattr( trans.app.config, 'user_%s_filters' % category, [] ) + self.__init_filters( category, user_filters, filters, validate=validate ) + else: + if kwds.get( "trackster", False ): + filters[ "tool" ].append( _has_trackster_conf ) + + return filters + + def __init_filters( self, key, filters, toolbox_filters, validate=None ): + for filter in filters: + if validate is None or filter in validate or filter in self.default_filters: + filter_function = self.__build_filter_function( filter ) + toolbox_filters[ key ].append( filter_function ) + else: + log.warning( "Refusing to load %s filter '%s' which is not defined in config", key, filter ) + return toolbox_filters + + def __build_filter_function( self, filter_name ): + """Obtain python function (importing a submodule if needed) + corresponding to filter_name. + """ + if ":" in filter_name: + # Should be a submodule of filters (e.g. examples:restrict_development_tools) + (module_name, function_name) = filter_name.rsplit(":", 1) + function = self._import_filter( module_name, function_name ) + else: + # No module found, just load a function from this file or + # one that has be explicitly imported. + function = getattr( globals(), filter_name.strip() ) + return function + + def _import_filter( self, module_name, function_name ): + function_name = function_name.strip() + for base_module in self.__base_modules: + full_module_name = "%s.%s" % ( base_module, module_name.strip() ) + try: + __import__( full_module_name ) + except ImportError: + # log.debug("Failed to load module.", exc_info=True) + continue + module = sys.modules[ full_module_name ] + if hasattr( module, function_name ): + return getattr( module, function_name ) + raise Exception("Failed to find filter %s.%s" % (module_name, function_name)) + + +## Stock Filter Functions +def _not_hidden( context, tool ): + return not tool.hidden + + +def _handle_requires_login( context, tool ): + return not tool.require_login or context.trans.user + + +def _has_trackster_conf( context, tool ): + return tool.trackster_conf diff -r 507483359a67da800ce7a56536d31b7911211474 -r 87825915116972a38b2f0d18943ef2959d674f97 lib/galaxy/tools/toolbox/filters/examples.py.sample --- /dev/null +++ b/lib/galaxy/tools/toolbox/filters/examples.py.sample @@ -0,0 +1,87 @@ +import logging +log = logging.getLogger( __name__ ) + + +def restrict_upload_to_admins( context, tool ): + """ + This tool filter will hide the upload tool from all users except admin + users. This can be enabled by renaming this file to examples.py and adding + the following to the ``app:main`` section of ``galaxy.ini``: + + tool_filters = examples:restrict_upload_to_admins + """ + if tool.name == "Upload File": + return context.trans.user_is_admin() + return True + + +def disable_gatk( context, tool ): + """ + This tool filter will disable all gatk tools when enabled. This can be + enabled by renaming this file to examples.py and adding the following to the + ``app:main`` section of ``galaxy.ini``: + + tool_filters = examples:disable_gatk + """ + return not any( [ requirement.name == "gatk" for requirement in tool.requirements ] ) + + +def explicit_user_mapping( context, section ): + """ + This tool section filter uses an explicit mapping to describe what users can view + which tool sections. Anonymous users will only be able to view the "Get Data" + tool section (with id getext). This can be enabled by renaming this file to + examples.py and adding the following to the ``app:main`` section of + ``galaxy.ini``: + + tool_section_filters = examples:explicit_user_mapping + """ + users_sections = { + None: [ "getext" ], + "bob(a)example.com": [ "getext", "textutil", "filter" ], + "mary(a)example.com": [ "getext", "textutil", "filter", "ngs" ], + } + user = context.trans.user + email = user and user.email + valid_sections = users_sections.get( email, [] ) + return section.id in valid_sections + + +DEVELOPERS = [ "mary(a)example.com" ] + + +def restrict_development_tools( context, tool ): + """ + This tool filter will disable all tools with the string alpha appearing in + the version for all users except those explicitly appearing in the DEVELOPERS list + defined above. This can be enabled by renaming this file to examples.py and + adding the following to the ``app:main`` section of ``galaxy.ini``: + + tool_filters = examples:restrict_development_tools + """ + version = tool.version + user = context.trans.user + email = user and user.email + return "alpha" not in version or email in DEVELOPERS + + +def per_host_tool_sections( context, section ): + """ + This tool section filter results in different sections being display based on + the URL the user is making the request to. This could allow a single Galaxy instance + to seem like several different instances hosting different tools based on the URL used + to access the Galxy. This can be enabled by renaming this file to examples.py and adding + the following to the ``app:main`` section of ``galaxy.ini``: + + tool_section_filters = examples:per_host_tool_sections + """ + host = context.trans.request.host + # Core tools used by all virtual hosts. + valid_sections = [ "getext", "textutil", "filter" ] + if "ngs.galaxy.example.com" in host: + valid_sections += [ "ngs" ] + elif "microarray.galaxy.example.com" in host: + valid_sections += [ "microarray" ] + elif "proteomics.galaxy.example.com" in host: + valid_sections += [ "proteomics" ] + return section.id in valid_sections diff -r 507483359a67da800ce7a56536d31b7911211474 -r 87825915116972a38b2f0d18943ef2959d674f97 test/unit/tools/test_toolbox_filters.py --- a/test/unit/tools/test_toolbox_filters.py +++ b/test/unit/tools/test_toolbox_filters.py @@ -1,6 +1,6 @@ from galaxy.util.bunch import Bunch -from galaxy.tools.filters import FilterFactory +from galaxy.tools.toolbox.filters import FilterFactory def test_stock_filtering_requires_login_tools( ): Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: jmchilton: Bugfix for 267fa4b when tool tags are enabled.
by commits-noreply＠bitbucket.org 29 Dec '14

29 Dec '14

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/fa404b077f67/ Changeset: fa404b077f67 User: jmchilton Date: 2014-12-29 19:25:12+00:00 Summary: Bugfix for 267fa4b when tool tags are enabled. Thanks to JJ for the bug report - apparently at least one person has tool tags enabled :). Affected #: 1 file diff -r e84f0ccd8533d76d3ee3f5b7fa739f096e9aeb41 -r fa404b077f67ff492db5d8c3405ea57cdb2d1420 lib/galaxy/tools/toolbox/tags.py --- a/lib/galaxy/tools/toolbox/tags.py +++ b/lib/galaxy/tools/toolbox/tags.py @@ -44,7 +44,7 @@ def __init__( self, app ): self.app = app - self.sa_session = app.sa_session + self.sa_session = app.model.context def reset_tags( self ): log.info("removing all tool tag associations (" + str( self.sa_session.query( self.app.model.ToolTagAssociation ).count() ) + ")" ) Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: guerler: Add classic hidden_data field to tool form
by commits-noreply＠bitbucket.org 29 Dec '14

29 Dec '14

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/e84f0ccd8533/ Changeset: e84f0ccd8533 User: guerler Date: 2014-12-29 19:08:35+00:00 Summary: Add classic hidden_data field to tool form Affected #: 3 files diff -r dc14223818bd8b8a908e056f361f2be1e8972d91 -r e84f0ccd8533d76d3ee3f5b7fa739f096e9aeb41 client/galaxy/scripts/mvc/tools/tools-section.js --- a/client/galaxy/scripts/mvc/tools/tools-section.js +++ b/client/galaxy/scripts/mvc/tools/tools-section.js @@ -316,6 +316,11 @@ field = this._fieldHidden(input_def); break; + // hidden field + case 'hidden_data': + field = this._fieldHidden(input_def); + break; + // integer field case 'integer': field = this._fieldSlider(input_def); diff -r dc14223818bd8b8a908e056f361f2be1e8972d91 -r e84f0ccd8533d76d3ee3f5b7fa739f096e9aeb41 static/scripts/mvc/tools/tools-section.js --- a/static/scripts/mvc/tools/tools-section.js +++ b/static/scripts/mvc/tools/tools-section.js @@ -316,6 +316,11 @@ field = this._fieldHidden(input_def); break; + // hidden field + case 'hidden_data': + field = this._fieldHidden(input_def); + break; + // integer field case 'integer': field = this._fieldSlider(input_def); diff -r dc14223818bd8b8a908e056f361f2be1e8972d91 -r e84f0ccd8533d76d3ee3f5b7fa739f096e9aeb41 static/scripts/packed/mvc/tools/tools-section.js --- a/static/scripts/packed/mvc/tools/tools-section.js +++ b/static/scripts/packed/mvc/tools/tools-section.js @@ -1,1 +1,1 @@ -define(["utils/utils","mvc/ui/ui-table","mvc/ui/ui-misc","mvc/tools/tools-repeat","mvc/tools/tools-select-content","mvc/tools/tools-input"],function(d,b,g,c,a,e){var f=Backbone.View.extend({initialize:function(i,h){this.app=i;this.inputs=h.inputs;h.cls_tr="section-row";this.table=new b.View(h);this.setElement(this.table.$el);this.render()},render:function(){this.table.delAll();for(var h in this.inputs){this._add(this.inputs[h])}},_add:function(j){var i=this;var h=jQuery.extend(true,{},j);h.id=j.id=d.uuid();this.app.input_list[h.id]=h;var k=h.type;switch(k){case"conditional":this._addConditional(h);break;case"repeat":this._addRepeat(h);break;default:this._addRow(h)}},_addConditional:function(h){var j=this;h.test_param.id=h.id;var m=this._addRow(h.test_param);m.options.onchange=function(t){var p=j.app.tree.matchCase(h,t);for(var r in h.cases){var w=h.cases[r];var u=h.id+"-section-"+r;var o=j.table.get(u);var v=false;for(var q in w.inputs){var s=w.inputs[q].type;if(s&&s!=="hidden"){v=true;break}}if(r==p&&v){o.fadeIn("fast")}else{o.hide()}}j.app.trigger("refresh")};for(var l in h.cases){var k=h.id+"-section-"+l;var n=new f(this.app,{inputs:h.cases[l].inputs,cls:"ui-table-plain"});n.$el.addClass("ui-table-form-section");this.table.add(n.$el);this.table.append(k)}m.trigger("change")},_addRepeat:function(o){var r=this;var p=0;function m(i,t){var s=o.id+"-section-"+(p++);var u=null;if(t){u=function(){k.del(s);k.retitle(o.title);r.app.trigger("refresh")}}var v=new f(r.app,{inputs:i,cls:"ui-table-plain"});k.add({id:s,title:o.title,$el:v.$el,ondel:u});k.retitle(o.title)}var k=new c.View({title_new:o.title,max:o.max,onnew:function(){m(o.inputs,true);r.app.trigger("refresh")}});var h=o.min;var q=_.size(o.cache);for(var l=0;l<Math.max(q,h);l++){var n=null;if(l<q){n=o.cache[l]}else{n=o.inputs}m(n,l>=h)}var j=new e(this.app,{label:o.title,help:o.help,field:k});j.$el.addClass("ui-table-form-section");this.table.add(j.$el);this.table.append(o.id)},_addRow:function(h){var k=h.id;var i=this._createField(h);this.app.field_list[k]=i;var j=new e(this.app,{label:h.label,optional:h.optional,help:h.help,field:i});this.app.element_list[k]=j;this.table.add(j.$el);this.table.append(k);return i},_createField:function(h){var i=null;switch(h.type){case"text":i=this._fieldText(h);break;case"select":i=this._fieldSelect(h);break;case"data":i=this._fieldData(h);break;case"data_collection":i=this._fieldData(h);break;case"data_column":h.error_text="Missing columns in referenced dataset.";i=this._fieldSelect(h);break;case"hidden":i=this._fieldHidden(h);break;case"integer":i=this._fieldSlider(h);break;case"float":i=this._fieldSlider(h);break;case"boolean":i=this._fieldBoolean(h);break;case"genomebuild":h.searchable=true;i=this._fieldSelect(h);break;case"drill_down":i=this._fieldDrilldown(h);break;case"baseurl":i=this._fieldHidden(h);break;default:this.app.incompatible=true;if(h.options){i=this._fieldSelect(h)}else{i=this._fieldText(h)}console.debug("tools-form::_addRow() : Auto matched field type ("+h.type+").")}if(h.value!==undefined){i.value(h.value)}return i},_fieldData:function(h){var i=this;return new a.View(this.app,{id:"field-"+h.id,extensions:h.extensions,multiple:h.multiple,type:h.type,data:h.options,onchange:function(){i.app.trigger("refresh")}})},_fieldSelect:function(h){var k=[];for(var l in h.options){var m=h.options[l];k.push({label:m[0],value:m[1]})}var n=g.Select;switch(h.display){case"checkboxes":n=g.Checkbox;break;case"radio":n=g.Radio;break}var j=this;return new n.View({id:"field-"+h.id,data:k,error_text:h.error_text||"No options available",multiple:h.multiple,searchable:h.searchable,onchange:function(){j.app.trigger("refresh")}})},_fieldDrilldown:function(h){var i=this;return new g.Drilldown.View({id:"field-"+h.id,data:h.options,display:h.display,onchange:function(){i.app.trigger("refresh")}})},_fieldText:function(h){var i=this;return new g.Input({id:"field-"+h.id,area:h.area,onchange:function(){i.app.trigger("refresh")}})},_fieldSlider:function(h){return new g.Slider.View({id:"field-"+h.id,precise:h.type=="float",min:h.min,max:h.max})},_fieldHidden:function(h){return new g.Hidden({id:"field-"+h.id})},_fieldBoolean:function(h){return new g.RadioButton.View({id:"field-"+h.id,data:[{label:"Yes",value:"true"},{label:"No",value:"false"}]})}});return{View:f}}); \ No newline at end of file +define(["utils/utils","mvc/ui/ui-table","mvc/ui/ui-misc","mvc/tools/tools-repeat","mvc/tools/tools-select-content","mvc/tools/tools-input"],function(d,b,g,c,a,e){var f=Backbone.View.extend({initialize:function(i,h){this.app=i;this.inputs=h.inputs;h.cls_tr="section-row";this.table=new b.View(h);this.setElement(this.table.$el);this.render()},render:function(){this.table.delAll();for(var h in this.inputs){this._add(this.inputs[h])}},_add:function(j){var i=this;var h=jQuery.extend(true,{},j);h.id=j.id=d.uuid();this.app.input_list[h.id]=h;var k=h.type;switch(k){case"conditional":this._addConditional(h);break;case"repeat":this._addRepeat(h);break;default:this._addRow(h)}},_addConditional:function(h){var j=this;h.test_param.id=h.id;var m=this._addRow(h.test_param);m.options.onchange=function(t){var p=j.app.tree.matchCase(h,t);for(var r in h.cases){var w=h.cases[r];var u=h.id+"-section-"+r;var o=j.table.get(u);var v=false;for(var q in w.inputs){var s=w.inputs[q].type;if(s&&s!=="hidden"){v=true;break}}if(r==p&&v){o.fadeIn("fast")}else{o.hide()}}j.app.trigger("refresh")};for(var l in h.cases){var k=h.id+"-section-"+l;var n=new f(this.app,{inputs:h.cases[l].inputs,cls:"ui-table-plain"});n.$el.addClass("ui-table-form-section");this.table.add(n.$el);this.table.append(k)}m.trigger("change")},_addRepeat:function(o){var r=this;var p=0;function m(i,t){var s=o.id+"-section-"+(p++);var u=null;if(t){u=function(){k.del(s);k.retitle(o.title);r.app.trigger("refresh")}}var v=new f(r.app,{inputs:i,cls:"ui-table-plain"});k.add({id:s,title:o.title,$el:v.$el,ondel:u});k.retitle(o.title)}var k=new c.View({title_new:o.title,max:o.max,onnew:function(){m(o.inputs,true);r.app.trigger("refresh")}});var h=o.min;var q=_.size(o.cache);for(var l=0;l<Math.max(q,h);l++){var n=null;if(l<q){n=o.cache[l]}else{n=o.inputs}m(n,l>=h)}var j=new e(this.app,{label:o.title,help:o.help,field:k});j.$el.addClass("ui-table-form-section");this.table.add(j.$el);this.table.append(o.id)},_addRow:function(h){var k=h.id;var i=this._createField(h);this.app.field_list[k]=i;var j=new e(this.app,{label:h.label,optional:h.optional,help:h.help,field:i});this.app.element_list[k]=j;this.table.add(j.$el);this.table.append(k);return i},_createField:function(h){var i=null;switch(h.type){case"text":i=this._fieldText(h);break;case"select":i=this._fieldSelect(h);break;case"data":i=this._fieldData(h);break;case"data_collection":i=this._fieldData(h);break;case"data_column":h.error_text="Missing columns in referenced dataset.";i=this._fieldSelect(h);break;case"hidden":i=this._fieldHidden(h);break;case"hidden_data":i=this._fieldHidden(h);break;case"integer":i=this._fieldSlider(h);break;case"float":i=this._fieldSlider(h);break;case"boolean":i=this._fieldBoolean(h);break;case"genomebuild":h.searchable=true;i=this._fieldSelect(h);break;case"drill_down":i=this._fieldDrilldown(h);break;case"baseurl":i=this._fieldHidden(h);break;default:this.app.incompatible=true;if(h.options){i=this._fieldSelect(h)}else{i=this._fieldText(h)}console.debug("tools-form::_addRow() : Auto matched field type ("+h.type+").")}if(h.value!==undefined){i.value(h.value)}return i},_fieldData:function(h){var i=this;return new a.View(this.app,{id:"field-"+h.id,extensions:h.extensions,multiple:h.multiple,type:h.type,data:h.options,onchange:function(){i.app.trigger("refresh")}})},_fieldSelect:function(h){var k=[];for(var l in h.options){var m=h.options[l];k.push({label:m[0],value:m[1]})}var n=g.Select;switch(h.display){case"checkboxes":n=g.Checkbox;break;case"radio":n=g.Radio;break}var j=this;return new n.View({id:"field-"+h.id,data:k,error_text:h.error_text||"No options available",multiple:h.multiple,searchable:h.searchable,onchange:function(){j.app.trigger("refresh")}})},_fieldDrilldown:function(h){var i=this;return new g.Drilldown.View({id:"field-"+h.id,data:h.options,display:h.display,onchange:function(){i.app.trigger("refresh")}})},_fieldText:function(h){var i=this;return new g.Input({id:"field-"+h.id,area:h.area,onchange:function(){i.app.trigger("refresh")}})},_fieldSlider:function(h){return new g.Slider.View({id:"field-"+h.id,precise:h.type=="float",min:h.min,max:h.max})},_fieldHidden:function(h){return new g.Hidden({id:"field-"+h.id})},_fieldBoolean:function(h){return new g.RadioButton.View({id:"field-"+h.id,data:[{label:"Yes",value:"true"},{label:"No",value:"false"}]})}});return{View:f}}); \ No newline at end of file Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: 2 new changesets
by commits-noreply＠bitbucket.org 29 Dec '14

29 Dec '14

2 new commits in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/68daf6dfc8b4/ Changeset: 68daf6dfc8b4 User: jmchilton Date: 2014-12-29 17:23:55+00:00 Summary: Update attribute and method names in ToolBox to reflect encapsulation efforts. Using ToolBox._xxx instead of ToolBox.__xxx because realistically ToolBox is still much to large to grok and so I imagine it will need to be broken up even more (base class focused on just the panel details perhaps - or mixins - etc...). Affected #: 3 files diff -r 6268351b242feb5c2167b35996e264e8a51c11a9 -r 68daf6dfc8b405acbdfc64fd21d520178aff1ac4 lib/galaxy/tools/__init__.py --- a/lib/galaxy/tools/__init__.py +++ b/lib/galaxy/tools/__init__.py @@ -131,49 +131,50 @@ Create a toolbox from the config files named by `config_filenames`, using `tool_root_dir` as the base directory for finding individual tool config files. """ - # The shed_tool_confs list contains dictionaries storing information about the tools defined in each - # shed-related shed_tool_conf.xml file. - self.shed_tool_confs = [] - self.tools_by_id = {} - self.workflows_by_id = {} + # The _dynamic_tool_confs list contains dictionaries storing + # information about the tools defined in each shed-related + # shed_tool_conf.xml file. + self._dynamic_tool_confs = [] + self._tools_by_id = {} + self._workflows_by_id = {} # In-memory dictionary that defines the layout of the tool panel. - self.tool_panel = odict() - self.index = 0 + self._tool_panel = odict() + self._index = 0 self.data_manager_tools = odict() - self.lineage_map = LineageMap( app ) + self._lineage_map = LineageMap( app ) # File that contains the XML section and tool tags from all tool panel config files integrated into a # single file that defines the tool panel layout. This file can be changed by the Galaxy administrator # (in a way similar to the single tool_conf.xml file in the past) to alter the layout of the tool panel. - self.integrated_tool_panel_config = app.config.integrated_tool_panel_config + self._integrated_tool_panel_config = app.config.integrated_tool_panel_config # In-memory dictionary that defines the layout of the tool_panel.xml file on disk. - self.integrated_tool_panel = odict() - self.integrated_tool_panel_config_has_contents = os.path.exists( self.integrated_tool_panel_config ) and os.stat( self.integrated_tool_panel_config ).st_size > 0 - if self.integrated_tool_panel_config_has_contents: - self.load_integrated_tool_panel_keys() + self._integrated_tool_panel = odict() + self._integrated_tool_panel_config_has_contents = os.path.exists( self._integrated_tool_panel_config ) and os.stat( self._integrated_tool_panel_config ).st_size > 0 + if self._integrated_tool_panel_config_has_contents: + self._load_integrated_tool_panel_keys() # The following refers to the tool_path config setting for backward compatibility. The shed-related # (e.g., shed_tool_conf.xml) files include the tool_path attribute within the <toolbox> tag. - self.tool_root_dir = tool_root_dir + self._tool_root_dir = tool_root_dir self.app = app - self.tool_watcher = watcher.get_watcher( self, app.config ) - self.filter_factory = FilterFactory( self ) - self.init_dependency_manager() - self.tool_tag_manager = tool_tag_manager( app ) - self.init_tools_from_configs( config_filenames ) - if self.app.name == 'galaxy' and self.integrated_tool_panel_config_has_contents: - # Load self.tool_panel based on the order in self.integrated_tool_panel. - self.load_tool_panel() + self._tool_watcher = watcher.get_watcher( self, app.config ) + self._filter_factory = FilterFactory( self ) + self._init_dependency_manager() + self._tool_tag_manager = tool_tag_manager( app ) + self._init_tools_from_configs( config_filenames ) + if self.app.name == 'galaxy' and self._integrated_tool_panel_config_has_contents: + # Load self._tool_panel based on the order in self._integrated_tool_panel. + self._load_tool_panel() if app.config.update_integrated_tool_panel: # Write the current in-memory integrated_tool_panel to the integrated_tool_panel.xml file. # This will cover cases where the Galaxy administrator manually edited one or more of the tool panel # config files, adding or removing locally developed tools or workflows. The value of integrated_tool_panel # will be False when things like functional tests are the caller. - self.write_integrated_tool_panel_config_file() - - def init_tools_from_configs( self, config_filenames ): + self._write_integrated_tool_panel_config_file() + + def _init_tools_from_configs( self, config_filenames ): """ Read through all tool config files and initialize tools in each with init_tools_from_config below. """ - self.tool_tag_manager.reset_tags() + self._tool_tag_manager.reset_tags() config_filenames = listify( config_filenames ) for config_filename in config_filenames: if os.path.isdir( config_filename ): @@ -183,11 +184,11 @@ config_filenames.extend( directory_config_files ) for config_filename in config_filenames: try: - self.init_tools_from_config( config_filename ) + self._init_tools_from_config( config_filename ) except: log.exception( "Error loading tools defined in config %s", config_filename ) - def init_tools_from_config( self, config_filename ): + def _init_tools_from_config( self, config_filename ): """ Read the configuration file and load each tool. The following tags are currently supported: @@ -218,10 +219,10 @@ parsing_shed_tool_conf = False tool_path = self.__resolve_tool_path(tool_path, config_filename) # Only load the panel_dict under certain conditions. - load_panel_dict = not self.integrated_tool_panel_config_has_contents + load_panel_dict = not self._integrated_tool_panel_config_has_contents for _, elem in enumerate( root ): - index = self.index - self.index += 1 + index = self._index + self._index += 1 if parsing_shed_tool_conf: config_elems.append( elem ) self.load_item( @@ -237,7 +238,7 @@ shed_tool_conf_dict = dict( config_filename=config_filename, tool_path=tool_path, config_elems=config_elems ) - self.shed_tool_confs.append( shed_tool_conf_dict ) + self._dynamic_tool_confs.append( shed_tool_conf_dict ) def load_item( self, elem, tool_path, panel_dict=None, integrated_panel_dict=None, load_panel_dict=True, guid=None, index=None, internal=False ): item_type = elem.tag @@ -247,22 +248,22 @@ return if panel_dict is None: - panel_dict = self.tool_panel + panel_dict = self._tool_panel if integrated_panel_dict is None: - integrated_panel_dict = self.integrated_tool_panel + integrated_panel_dict = self._integrated_tool_panel if item_type == 'tool': - self.load_tool_tag_set( elem, panel_dict=panel_dict, integrated_panel_dict=integrated_panel_dict, tool_path=tool_path, load_panel_dict=load_panel_dict, guid=guid, index=index ) + self._load_tool_tag_set( elem, panel_dict=panel_dict, integrated_panel_dict=integrated_panel_dict, tool_path=tool_path, load_panel_dict=load_panel_dict, guid=guid, index=index ) elif item_type == 'workflow': - self.load_workflow_tag_set( elem, panel_dict=panel_dict, integrated_panel_dict=integrated_panel_dict, load_panel_dict=load_panel_dict, index=index ) + self._load_workflow_tag_set( elem, panel_dict=panel_dict, integrated_panel_dict=integrated_panel_dict, load_panel_dict=load_panel_dict, index=index ) elif item_type == 'section': - self.load_section_tag_set( elem, tool_path=tool_path, load_panel_dict=load_panel_dict, index=index ) + self._load_section_tag_set( elem, tool_path=tool_path, load_panel_dict=load_panel_dict, index=index ) elif item_type == 'label': - self.load_label_tag_set( elem, panel_dict=panel_dict, integrated_panel_dict=integrated_panel_dict, load_panel_dict=load_panel_dict, index=index ) + self._load_label_tag_set( elem, panel_dict=panel_dict, integrated_panel_dict=integrated_panel_dict, load_panel_dict=load_panel_dict, index=index ) elif item_type == 'tool_dir': - self.load_tooldir_tag_set( elem, panel_dict, tool_path, integrated_panel_dict, load_panel_dict=load_panel_dict ) + self._load_tooldir_tag_set( elem, panel_dict, tool_path, integrated_panel_dict, load_panel_dict=load_panel_dict ) def get_shed_config_dict_by_filename( self, filename, default=None ): - for shed_config_dict in self.shed_tool_confs: + for shed_config_dict in self._dynamic_tool_confs: if shed_config_dict[ 'config_filename' ] == filename: return shed_config_dict return default @@ -273,22 +274,22 @@ that doesn't affect that file). """ app = self.app - for index, my_shed_tool_conf in enumerate( self.shed_tool_confs ): + for index, my_shed_tool_conf in enumerate( self._dynamic_tool_confs ): if shed_conf['config_filename'] == my_shed_tool_conf['config_filename']: - self.shed_tool_confs[ index ] = shed_conf + self._dynamic_tool_confs[ index ] = shed_conf if integrated_panel_changes and app.config.update_integrated_tool_panel: # Write the current in-memory version of the integrated_tool_panel.xml file to disk. - self.write_integrated_tool_panel_config_file() + self._write_integrated_tool_panel_config_file() app.reindex_tool_search() def get_section( self, section_id, new_label=None, create_if_needed=False ): tool_panel_section_key = str( section_id ) - if tool_panel_section_key in self.tool_panel: - # Appending a tool to an existing section in toolbox.tool_panel - tool_section = self.tool_panel[ tool_panel_section_key ] + if tool_panel_section_key in self._tool_panel: + # Appending a tool to an existing section in toolbox._tool_panel + tool_section = self._tool_panel[ tool_panel_section_key ] log.debug( "Appending to tool panel section: %s" % str( tool_section.name ) ) elif create_if_needed: - # Appending a new section to toolbox.tool_panel + # Appending a new section to toolbox._tool_panel if new_label is None: # This might add an ugly section label to the tool panel, but, oh well... new_label = section_id @@ -298,7 +299,7 @@ 'version': '', } tool_section = ToolSection( section_dict ) - self.tool_panel[ tool_panel_section_key ] = tool_section + self._tool_panel[ tool_panel_section_key ] = tool_section log.debug( "Loading new tool panel section: %s" % str( tool_section.name ) ) else: tool_section = None @@ -306,7 +307,7 @@ def get_integrated_section_for_tool( self, tool ): tool_id = tool.id - for key, item in self.integrated_tool_panel.items(): + for key, item in self._integrated_tool_panel.items(): if item: if isinstance( item, Tool ): if item.id == tool_id: @@ -324,7 +325,7 @@ def __resolve_tool_path(self, tool_path, config_filename): if not tool_path: # Default to backward compatible config setting. - tool_path = self.tool_root_dir + tool_path = self._tool_root_dir else: # Allow use of __tool_conf_dir__ in toolbox config files. tool_conf_dir = os.path.dirname(config_filename) @@ -335,9 +336,9 @@ def __add_tool_to_tool_panel( self, tool, panel_component, section=False ): # See if a version of this tool is already loaded into the tool panel. # The value of panel_component will be a ToolSection (if the value of - # section=True) or self.tool_panel (if section=False). + # section=True) or self._tool_panel (if section=False). tool_id = str( tool.id ) - tool = self.tools_by_id[ tool_id ] + tool = self._tools_by_id[ tool_id ] if section: panel_dict = panel_component.elems else: @@ -355,7 +356,7 @@ inserted = False key = 'tool_%s' % tool.id # The value of panel_component is the in-memory tool panel dictionary. - for index, integrated_panel_key in enumerate( self.integrated_tool_panel.keys() ): + for index, integrated_panel_key in enumerate( self._integrated_tool_panel.keys() ): if key == integrated_panel_key: panel_dict.insert( index, key, tool ) inserted = True @@ -363,7 +364,7 @@ # Check the tool's installed versions. for lineage_id in tool.lineage_ids: lineage_id_key = 'tool_%s' % lineage_id - for index, integrated_panel_key in enumerate( self.integrated_tool_panel.keys() ): + for index, integrated_panel_key in enumerate( self._integrated_tool_panel.keys() ): if lineage_id_key == integrated_panel_key: panel_dict.insert( index, key, tool ) inserted = True @@ -383,27 +384,28 @@ log.debug( "Loaded tool id: %s, version: %s into tool panel.." % ( tool.id, tool.version ) ) else: # We are in the process of installing the tool. - tool_lineage = self.lineage_map.get( tool_id ) + + tool_lineage = self._lineage_map.get( tool_id ) already_loaded = self._lineage_in_panel( panel_dict, tool_lineage=tool_lineage ) is not None if not already_loaded: # If the tool is not defined in integrated_tool_panel.xml, append it to the tool panel. panel_dict[ key ] = tool log.debug( "Loaded tool id: %s, version: %s into tool panel...." % ( tool.id, tool.version ) ) - def load_tool_panel( self ): - for key, val in self.integrated_tool_panel.items(): + def _load_tool_panel( self ): + for key, val in self._integrated_tool_panel.items(): if isinstance( val, Tool ): tool_id = key.replace( 'tool_', '', 1 ) - if tool_id in self.tools_by_id: - self.__add_tool_to_tool_panel( val, self.tool_panel, section=False ) + if tool_id in self._tools_by_id: + self.__add_tool_to_tool_panel( val, self._tool_panel, section=False ) elif isinstance( val, Workflow ): workflow_id = key.replace( 'workflow_', '', 1 ) - if workflow_id in self.workflows_by_id: - workflow = self.workflows_by_id[ workflow_id ] - self.tool_panel[ key ] = workflow + if workflow_id in self._workflows_by_id: + workflow = self._workflows_by_id[ workflow_id ] + self._tool_panel[ key ] = workflow log.debug( "Loaded workflow: %s %s" % ( workflow_id, workflow.name ) ) elif isinstance( val, ToolSectionLabel ): - self.tool_panel[ key ] = val + self._tool_panel[ key ] = val elif isinstance( val, ToolSection ): section_dict = { 'id': val.id or '', @@ -415,36 +417,36 @@ for section_key, section_val in val.elems.items(): if isinstance( section_val, Tool ): tool_id = section_key.replace( 'tool_', '', 1 ) - if tool_id in self.tools_by_id: + if tool_id in self._tools_by_id: self.__add_tool_to_tool_panel( section_val, section, section=True ) elif isinstance( section_val, Workflow ): workflow_id = section_key.replace( 'workflow_', '', 1 ) - if workflow_id in self.workflows_by_id: - workflow = self.workflows_by_id[ workflow_id ] + if workflow_id in self._workflows_by_id: + workflow = self._workflows_by_id[ workflow_id ] section.elems[ section_key ] = workflow log.debug( "Loaded workflow: %s %s" % ( workflow_id, workflow.name ) ) elif isinstance( section_val, ToolSectionLabel ): if section_val: section.elems[ section_key ] = section_val log.debug( "Loaded label: %s" % ( section_val.text ) ) - self.tool_panel[ key ] = section - - def load_integrated_tool_panel_keys( self ): + self._tool_panel[ key ] = section + + def _load_integrated_tool_panel_keys( self ): """ Load the integrated tool panel keys, setting values for tools and workflows to None. The values will be reset when the various tool panel config files are parsed, at which time the tools and workflows are loaded. """ - tree = parse_xml( self.integrated_tool_panel_config ) + tree = parse_xml( self._integrated_tool_panel_config ) root = tree.getroot() for elem in root: if elem.tag == 'tool': key = 'tool_%s' % elem.get( 'id' ) - self.integrated_tool_panel[ key ] = None + self._integrated_tool_panel[ key ] = None elif elem.tag == 'workflow': key = 'workflow_%s' % elem.get( 'id' ) - self.integrated_tool_panel[ key ] = None + self._integrated_tool_panel[ key ] = None elif elem.tag == 'section': section = ToolSection( elem ) for section_elem in elem: @@ -458,12 +460,12 @@ key = 'label_%s' % section_elem.get( 'id' ) section.elems[ key ] = None key = elem.get( 'id' ) - self.integrated_tool_panel[ key ] = section + self._integrated_tool_panel[ key ] = section elif elem.tag == 'label': key = 'label_%s' % elem.get( 'id' ) - self.integrated_tool_panel[ key ] = None - - def write_integrated_tool_panel_config_file( self ): + self._integrated_tool_panel[ key ] = None + + def _write_integrated_tool_panel_config_file( self ): """ Write the current in-memory version of the integrated_tool_panel.xml file to disk. Since Galaxy administrators use this file to manage the tool panel, we'll not use xml_to_string() since it doesn't write XML quite right. @@ -474,7 +476,7 @@ os.write( fd, ' \n') - for key, item in self.integrated_tool_panel.items(): + for key, item in self._integrated_tool_panel.items(): if item: if isinstance( item, Tool ): os.write( fd, ' <tool id="%s" />\n' % item.id ) @@ -506,28 +508,28 @@ os.write( fd, ' </section>\n' ) os.write( fd, '</toolbox>\n' ) os.close( fd ) - shutil.move( filename, os.path.abspath( self.integrated_tool_panel_config ) ) - os.chmod( self.integrated_tool_panel_config, 0644 ) + shutil.move( filename, os.path.abspath( self._integrated_tool_panel_config ) ) + os.chmod( self._integrated_tool_panel_config, 0644 ) def get_tool( self, tool_id, tool_version=None, get_all_versions=False, exact=False ): """Attempt to locate a tool in the tool box.""" if get_all_versions and exact: raise AssertionError("Cannot specify get_tool with both get_all_versions and exact as True") - if tool_id in self.tools_by_id and not get_all_versions: + if tool_id in self._tools_by_id and not get_all_versions: #tool_id exactly matches an available tool by id (which is 'old' tool_id or guid) - return self.tools_by_id[ tool_id ] + return self._tools_by_id[ tool_id ] #exact tool id match not found, or all versions requested, search for other options, e.g. migrated tools or different versions rval = [] - tool_lineage = self.lineage_map.get( tool_id ) + tool_lineage = self._lineage_map.get( tool_id ) if tool_lineage: tool_version_ids = tool_lineage.get_version_ids( ) for tool_version_id in tool_version_ids: - if tool_version_id in self.tools_by_id: - rval.append( self.tools_by_id[ tool_version_id ] ) + if tool_version_id in self._tools_by_id: + rval.append( self._tools_by_id[ tool_version_id ] ) if not rval: #still no tool, do a deeper search and try to match by old ids - for tool in self.tools_by_id.itervalues(): + for tool in self._tools_by_id.itervalues(): if tool.old_id == tool_id: rval.append( tool ) if rval: @@ -543,8 +545,8 @@ return rval[0] #We now likely have a Toolshed guid passed in, but no supporting database entries #If the tool exists by exact id and is loaded then provide exact match within a list - if tool_id in self.tools_by_id: - return[ self.tools_by_id[ tool_id ] ] + if tool_id in self._tools_by_id: + return[ self._tools_by_id[ tool_id ] ] return None def has_tool( self, tool_id, exact=False ): @@ -556,7 +558,7 @@ tools (using get_tool for inexact matches which currently returns the oldest tool shed installed tool with the same short id). """ - if tool_id not in self.tools_by_id: + if tool_id not in self._tools_by_id: tool = self.get_tool( tool_id ) if tool: tool_id = tool.id @@ -567,22 +569,22 @@ def get_loaded_tools_by_lineage( self, tool_id ): """Get all loaded tools associated by lineage to the tool whose id is tool_id.""" - tool_lineage = self.lineage_map.get( tool_id ) + tool_lineage = self._lineage_map.get( tool_id ) if tool_lineage: tool_version_ids = tool_lineage.get_version_ids( ) available_tool_versions = [] for tool_version_id in tool_version_ids: - if tool_version_id in self.tools_by_id: - available_tool_versions.append( self.tools_by_id[ tool_version_id ] ) + if tool_version_id in self._tools_by_id: + available_tool_versions.append( self._tools_by_id[ tool_version_id ] ) return available_tool_versions else: - if tool_id in self.tools_by_id: - tool = self.tools_by_id[ tool_id ] + if tool_id in self._tools_by_id: + tool = self._tools_by_id[ tool_id ] return [ tool ] return [] def tools( self ): - return self.tools_by_id.iteritems() + return self._tools_by_id.iteritems() def __elem_to_tool_shed_repository( self, elem ): # The tool is contained in an installed tool shed repository, so load @@ -637,7 +639,7 @@ def dynamic_confs( self, include_migrated_tool_conf=False ): confs = [] - for dynamic_tool_conf_dict in self.shed_tool_confs: + for dynamic_tool_conf_dict in self._dynamic_tool_confs: dynamic_tool_conf_filename = dynamic_tool_conf_dict[ 'config_filename' ] if include_migrated_tool_conf or (dynamic_tool_conf_filename != self.app.config.migrated_tools_config): confs.append( dynamic_tool_conf_dict ) @@ -712,11 +714,11 @@ if section_key: _, tool_section = self.get_section( section_key ) if tool_section: - remove_from_dict( tool_section.elems, self.integrated_tool_panel.get( section_key, {} ) ) + remove_from_dict( tool_section.elems, self._integrated_tool_panel.get( section_key, {} ) ) else: - remove_from_dict( self.tool_panel, self.integrated_tool_panel ) - - def load_tool_tag_set( self, elem, panel_dict, integrated_panel_dict, tool_path, load_panel_dict, guid=None, index=None ): + remove_from_dict( self._tool_panel, self._integrated_tool_panel ) + + def _load_tool_tag_set( self, elem, panel_dict, integrated_panel_dict, tool_path, load_panel_dict, guid=None, index=None ): try: path = elem.get( "file" ) repository_id = None @@ -746,11 +748,11 @@ tool.guid = guid tool.version = elem.find( "version" ).text # Make sure the tool has a tool_version. - tool_lineage = self.lineage_map.register( tool, tool_shed_repository=tool_shed_repository ) + tool_lineage = self._lineage_map.register( tool, tool_shed_repository=tool_shed_repository ) # Load the tool's lineage ids. tool.lineage = tool_lineage tool.lineage_ids = tool_lineage.get_version_ids( ) - self.tool_tag_manager.handle_tags( tool.id, elem ) + self._tool_tag_manager.handle_tags( tool.id, elem ) self.__add_tool( tool, load_panel_dict, panel_dict ) # Always load the tool into the integrated_panel_dict, or it will not be included in the integrated_tool_panel.xml file. if key in integrated_panel_dict or index is None: @@ -769,18 +771,18 @@ # administrator has retrieved updates to the installed repository. In # this case, the tool may have been updated, but the version was not # changed, so the tool should always be reloaded here. We used to - # only load the tool if it was not found in self.tools_by_id, but + # only load the tool if it was not found in self._tools_by_id, but # performing that check did not enable this scenario. self.register_tool( tool ) if load_panel_dict: self.__add_tool_to_tool_panel( tool, panel_dict, section=isinstance( panel_dict, ToolSection ) ) - def load_workflow_tag_set( self, elem, panel_dict, integrated_panel_dict, load_panel_dict, index=None ): + def _load_workflow_tag_set( self, elem, panel_dict, integrated_panel_dict, load_panel_dict, index=None ): try: # TODO: should id be encoded? workflow_id = elem.get( 'id' ) - workflow = self.load_workflow( workflow_id ) - self.workflows_by_id[ workflow_id ] = workflow + workflow = self._load_workflow( workflow_id ) + self._workflows_by_id[ workflow_id ] = workflow key = 'workflow_' + workflow_id if load_panel_dict: panel_dict[ key ] = workflow @@ -792,7 +794,7 @@ except: log.exception( "Error loading workflow: %s" % workflow_id ) - def load_label_tag_set( self, elem, panel_dict, integrated_panel_dict, load_panel_dict, index=None ): + def _load_label_tag_set( self, elem, panel_dict, integrated_panel_dict, load_panel_dict, index=None ): label = ToolSectionLabel( elem ) key = 'label_' + label.id if load_panel_dict: @@ -802,16 +804,16 @@ else: integrated_panel_dict.insert( index, key, label ) - def load_section_tag_set( self, elem, tool_path, load_panel_dict, index=None ): + def _load_section_tag_set( self, elem, tool_path, load_panel_dict, index=None ): key = elem.get( "id" ) - if key in self.tool_panel: - section = self.tool_panel[ key ] + if key in self._tool_panel: + section = self._tool_panel[ key ] elems = section.elems else: section = ToolSection( elem ) elems = section.elems - if key in self.integrated_tool_panel: - integrated_section = self.integrated_tool_panel[ key ] + if key in self._integrated_tool_panel: + integrated_section = self._integrated_tool_panel[ key ] integrated_elems = integrated_section.elems else: integrated_section = ToolSection( elem ) @@ -828,14 +830,14 @@ internal=True, ) if load_panel_dict: - self.tool_panel[ key ] = section + self._tool_panel[ key ] = section # Always load sections into the integrated_tool_panel. - if key in self.integrated_tool_panel or index is None: - self.integrated_tool_panel[ key ] = integrated_section + if key in self._integrated_tool_panel or index is None: + self._integrated_tool_panel[ key ] = integrated_section else: - self.integrated_tool_panel.insert( index, key, integrated_section ) - - def load_tooldir_tag_set(self, sub_elem, elems, tool_path, integrated_elems, load_panel_dict): + self._integrated_tool_panel.insert( index, key, integrated_section ) + + def _load_tooldir_tag_set(self, sub_elem, elems, tool_path, integrated_elems, load_panel_dict): directory = os.path.join( tool_path, sub_elem.attrib.get("dir") ) recursive = string_as_bool( sub_elem.attrib.get("recursive", True) ) self.__watch_directory( directory, elems, integrated_elems, load_panel_dict, recursive ) @@ -851,14 +853,14 @@ integrated_elems[ key ] = tool if async: - self.load_tool_panel() + self._load_tool_panel() if self.app.config.update_integrated_tool_panel: # Write the current in-memory integrated_tool_panel to the integrated_tool_panel.xml file. # This will cover cases where the Galaxy administrator manually edited one or more of the tool panel # config files, adding or removing locally developed tools or workflows. The value of integrated_tool_panel # will be False when things like functional tests are the caller. - self.write_integrated_tool_panel_config_file() + self._write_integrated_tool_panel_config_file() return tool.id except Exception: log.exception("Failed to load potential tool %s." % tool_file) @@ -873,7 +875,7 @@ quick_load( child_path, async=False ) tool_loaded = True if tool_loaded: - self.tool_watcher.watch_directory( directory, quick_load ) + self._tool_watcher.watch_directory( directory, quick_load ) def load_tool( self, config_file, guid=None, repository_id=None, **kwds ): """Load a single tool from the file named by `config_file` and return an instance of `Tool`.""" @@ -916,19 +918,19 @@ if not tool_id.startswith("__"): # do not monitor special tools written to tmp directory - no reason # to monitor such a large directory. - self.tool_watcher.watch_file( config_file, tool.id ) + self._tool_watcher.watch_file( config_file, tool.id ) return tool def load_hidden_tool( self, config_file, **kwds ): """ Load a hidden tool (in this context meaning one that does not - appear in the tool panel) and register it in tools_by_id. + appear in the tool panel) and register it in _tools_by_id. """ tool = self.load_tool( config_file, **kwds ) self.register_tool( tool ) return tool def register_tool( self, tool ): - self.tools_by_id[ tool.id ] = tool + self._tools_by_id[ tool.id ] = tool def package_tool( self, trans, tool_id ): """ @@ -938,10 +940,10 @@ :returns: tuple of tarball filename, success True/False, message/None """ # Make sure the tool is actually loaded. - if tool_id not in self.tools_by_id: + if tool_id not in self._tools_by_id: return None, False, "No tool with id %s" % escape( tool_id ) else: - tool = self.tools_by_id[ tool_id ] + tool = self._tools_by_id[ tool_id ] tarball_files = [] temp_files = [] tool_xml = file( os.path.abspath( tool.config_file ), 'r' ).read() @@ -1048,11 +1050,11 @@ Attempt to reload the tool identified by 'tool_id', if successful replace the old tool. """ - if tool_id not in self.tools_by_id: + if tool_id not in self._tools_by_id: message = "No tool with id %s" % escape( tool_id ) status = 'error' else: - old_tool = self.tools_by_id[ tool_id ] + old_tool = self._tools_by_id[ tool_id ] new_tool = self.load_tool( old_tool.config_file ) # The tool may have been installed from a tool shed, so set the tool shed attributes. # Since the tool version may have changed, we don't override it here. @@ -1063,17 +1065,17 @@ new_tool.repository_owner = old_tool.repository_owner new_tool.installed_changeset_revision = old_tool.installed_changeset_revision new_tool.old_id = old_tool.old_id - # Replace old_tool with new_tool in self.tool_panel + # Replace old_tool with new_tool in self._tool_panel tool_key = 'tool_' + tool_id - for key, val in self.tool_panel.items(): + for key, val in self._tool_panel.items(): if key == tool_key: - self.tool_panel[ key ] = new_tool + self._tool_panel[ key ] = new_tool break elif key.startswith( 'section' ): if tool_key in val.elems: - self.tool_panel[ key ].elems[ tool_key ] = new_tool + self._tool_panel[ key ].elems[ tool_key ] = new_tool break - self.tools_by_id[ tool_id ] = new_tool + self._tools_by_id[ tool_id ] = new_tool message = "Reloaded the tool:<br/>" message += "<b>name:</b> %s<br/>" % old_tool.name message += "<b>id:</b> %s<br/>" % old_tool.id @@ -1089,21 +1091,21 @@ promoting the next newest version of the tool into the panel if needed. """ - if tool_id not in self.tools_by_id: + if tool_id not in self._tools_by_id: message = "No tool with id %s" % escape( tool_id ) status = 'error' else: - tool = self.tools_by_id[ tool_id ] - del self.tools_by_id[ tool_id ] + tool = self._tools_by_id[ tool_id ] + del self._tools_by_id[ tool_id ] if remove_from_panel: tool_key = 'tool_' + tool_id - for key, val in self.tool_panel.items(): + for key, val in self._tool_panel.items(): if key == tool_key: - del self.tool_panel[ key ] + del self._tool_panel[ key ] break elif key.startswith( 'section' ): if tool_key in val.elems: - del self.tool_panel[ key ].elems[ tool_key ] + del self._tool_panel[ key ].elems[ tool_key ] break if tool_id in self.data_manager_tools: del self.data_manager_tools[ tool_id ] @@ -1116,7 +1118,7 @@ return message, status def get_sections( self ): - for k, v in self.tool_panel.items(): + for k, v in self._tool_panel.items(): if isinstance( v, ToolSection ): yield (v.id, v.name) @@ -1128,17 +1130,17 @@ if not tool_panel_section_id: tool_panel_section_id = '' else: - if tool_panel_section_id not in self.tool_panel: + if tool_panel_section_id not in self._tool_panel: # Hack introduced without comment in a29d54619813d5da992b897557162a360b8d610c- # not sure why it is needed. fixed_tool_panel_section_id = 'section_%s' % tool_panel_section_id - if fixed_tool_panel_section_id in self.tool_panel: + if fixed_tool_panel_section_id in self._tool_panel: tool_panel_section_id = fixed_tool_panel_section_id else: tool_panel_section_id = '' return tool_panel_section_id - def load_workflow( self, workflow_id ): + def _load_workflow( self, workflow_id ): """ Return an instance of 'Workflow' identified by `id`, which is encoded in the tool panel. @@ -1147,13 +1149,13 @@ stored = self.app.model.context.query( self.app.model.StoredWorkflow ).get( id ) return stored.latest_workflow - def init_dependency_manager( self ): + def _init_dependency_manager( self ): self.dependency_manager = build_dependency_manager( self.app.config ) def handle_datatypes_changed( self ): """ Refresh upload tools when new datatypes are added. """ - for tool_id in self.tools_by_id: - tool = self.tools_by_id[ tool_id ] + for tool_id in self._tools_by_id: + tool = self._tools_by_id[ tool_id ] if isinstance( tool.tool_action, UploadToolAction ): self.reload_tool_by_id( tool_id ) @@ -1168,8 +1170,8 @@ """ Filter tool_panel contents for displaying for user. """ context = Bunch( toolbox=self, trans=trans ) - filters = self.filter_factory.build_filters( trans ) - for elt in self.tool_panel.itervalues(): + filters = self._filter_factory.build_filters( trans ) + for elt in self._tool_panel.itervalues(): elt = _filter_for_panel( elt, filters, context ) if elt: yield elt @@ -1190,7 +1192,7 @@ rval.append( to_dict_helper( elt, kwargs ) ) else: tools = [] - for id, tool in self.tools_by_id.items(): + for id, tool in self._tools_by_id.items(): tools.append( tool.to_dict( trans, link_details=True ) ) rval = tools @@ -1205,7 +1207,7 @@ tool_lineage = tool.lineage lineage_ids = tool_lineage.get_version_ids( reverse=True ) for lineage_id in lineage_ids: - if lineage_id in self.tools_by_id: + if lineage_id in self._tools_by_id: loaded_version_key = 'tool_%s' % lineage_id if loaded_version_key in panel_dict: return panel_dict[ loaded_version_key ] diff -r 6268351b242feb5c2167b35996e264e8a51c11a9 -r 68daf6dfc8b405acbdfc64fd21d520178aff1ac4 test/unit/tool_shed_unit_tests/test_tool_panel_manager.py --- a/test/unit/tool_shed_unit_tests/test_tool_panel_manager.py +++ b/test/unit/tool_shed_unit_tests/test_tool_panel_manager.py @@ -18,20 +18,20 @@ assert section_id == "tid" assert len( section.elems ) == 1 # tool.xml assert section.id == "tid" - assert len( toolbox.tool_panel ) == 1 + assert len( toolbox._tool_panel ) == 1 section_id, section = tpm.handle_tool_panel_section( toolbox, new_tool_panel_section_label="tid2" ) assert section_id == "tid2" assert len( section.elems ) == 0 # new section assert section.id == "tid2" - assert len( toolbox.tool_panel ) == 2 + assert len( toolbox._tool_panel ) == 2 # Test re-fetch new section by same id. section_id, section = tpm.handle_tool_panel_section( toolbox, new_tool_panel_section_label="tid2" ) assert section_id == "tid2" assert len( section.elems ) == 0 # new section assert section.id == "tid2" - assert len( toolbox.tool_panel ) == 2 + assert len( toolbox._tool_panel ) == 2 def test_add_tool_to_panel( self ): self._init_tool() @@ -77,7 +77,7 @@ assert not all_versions # Check that tool panel has reverted to old value... - section = self.toolbox.tool_panel["tid"] + section = self.toolbox._tool_panel["tid"] assert len(section.elems) == 0 def test_uninstall_in_section( self ): @@ -85,7 +85,7 @@ self._verify_version_2_removed_from_panel( ) # Not in tool conf because it was uninstalled. assert "github.com/galaxyproject/example/test_tool/0.2" not in open(os.path.join(self.test_directory, "tool_conf.xml"), "r").read() - assert "tool_github.com/galaxyproject/example/test_tool/0.2" not in self.toolbox.integrated_tool_panel["tid"].elems + assert "tool_github.com/galaxyproject/example/test_tool/0.2" not in self.toolbox._integrated_tool_panel["tid"].elems self._verify_tool_confs() def test_deactivate_outside_section( self ): @@ -109,7 +109,7 @@ assert not all_versions # Check that tool panel has reverted to old value... - section = self.toolbox.tool_panel["tid"] + section = self.toolbox._tool_panel["tid"] assert len(section.elems) == 0 def _setup_two_versions_remove_one( self, section, uninstall ): @@ -126,14 +126,14 @@ # Check that tool panel has reverted to old value... if section: - section = self.toolbox.tool_panel["tid"] + section = self.toolbox._tool_panel["tid"] assert len(section.elems) == 1 assert section.elems.values()[0].id == "github.com/galaxyproject/example/test_tool/0.1" - assert "github.com/galaxyproject/example/test_tool/0.2" not in self.toolbox.integrated_tool_panel["tid"].elems + assert "github.com/galaxyproject/example/test_tool/0.2" not in self.toolbox._integrated_tool_panel["tid"].elems else: - self.toolbox.tool_panel.values()[0].id == "github.com/galaxyproject/example/test_tool/0.1" - assert "github.com/galaxyproject/example/test_tool/0.2" not in self.toolbox.integrated_tool_panel + self.toolbox._tool_panel.values()[0].id == "github.com/galaxyproject/example/test_tool/0.1" + assert "github.com/galaxyproject/example/test_tool/0.2" not in self.toolbox._integrated_tool_panel def _remove_guids( self, guids, uninstall, shed_tool_conf="tool_conf.xml" ): self.tpm.remove_guids( diff -r 6268351b242feb5c2167b35996e264e8a51c11a9 -r 68daf6dfc8b405acbdfc64fd21d520178aff1ac4 test/unit/tools/test_toolbox.py --- a/test/unit/tools/test_toolbox.py +++ b/test/unit/tools/test_toolbox.py @@ -182,7 +182,7 @@ self.__verify_two_test_tools( ) # Assert only newer version of the tool loaded into the panel. - section = self.toolbox.tool_panel["tid"] + section = self.toolbox._tool_panel["tid"] assert len(section.elems) == 1 assert section.elems.values()[0].id == "github.com/galaxyproject/example/test_tool/0.2" @@ -193,7 +193,7 @@ self.__verify_two_test_tools( ) # Assert tools merged in tool panel. - assert len( self.toolbox.tool_panel ) == 1 + assert len( self.toolbox._tool_panel ) == 1 def test_update_shed_conf(self): self.__setup_shed_tool_conf() @@ -228,8 +228,8 @@ stored_workflow = self.__test_workflow() encoded_id = self.app.security.encode_id( stored_workflow.id ) self._add_config( """<toolbox><workflow id="%s" /></toolbox>""" % encoded_id ) - assert len( self.toolbox.tool_panel ) == 1 - panel_workflow = self.toolbox.tool_panel.values()[ 0 ] + assert len( self.toolbox._tool_panel ) == 1 + panel_workflow = self.toolbox._tool_panel.values()[ 0 ] assert panel_workflow == stored_workflow.latest_workflow # TODO: test to_dict with workflows @@ -237,21 +237,21 @@ stored_workflow = self.__test_workflow() encoded_id = self.app.security.encode_id( stored_workflow.id ) self._add_config( """<toolbox><section id="tid" name="TID"><workflow id="%s" /></section></toolbox>""" % encoded_id ) - assert len( self.toolbox.tool_panel ) == 1 - section = self.toolbox.tool_panel[ 'tid' ] + assert len( self.toolbox._tool_panel ) == 1 + section = self.toolbox._tool_panel[ 'tid' ] assert len( section.elems ) == 1 panel_workflow = section.elems.values()[ 0 ] assert panel_workflow == stored_workflow.latest_workflow def test_label_in_panel( self ): self._add_config( """<toolbox><label id="lab1" text="Label 1" /><label id="lab2" text="Label 2" /></toolbox>""" ) - assert len( self.toolbox.tool_panel ) == 2 - self.__check_test_labels( self.toolbox.tool_panel ) + assert len( self.toolbox._tool_panel ) == 2 + self.__check_test_labels( self.toolbox._tool_panel ) def test_label_in_section( self ): self._add_config( """<toolbox><section id="tid" name="TID"><label id="lab1" text="Label 1" /><label id="lab2" text="Label 2" /></section></toolbox>""" ) - assert len( self.toolbox.tool_panel ) == 1 - section = self.toolbox.tool_panel[ 'tid' ] + assert len( self.toolbox._tool_panel ) == 1 + section = self.toolbox._tool_panel[ 'tid' ] self.__check_test_labels( section.elems ) def __check_test_labels( self, panel_dict ): https://bitbucket.org/galaxy/galaxy-central/commits/dc14223818bd/ Changeset: dc14223818bd User: jmchilton Date: 2014-12-29 17:50:27+00:00 Summary: Hack to fix bug introduced in 806202d. I may be trying to juggle to many competing branches working on same component. Affected #: 1 file diff -r 68daf6dfc8b405acbdfc64fd21d520178aff1ac4 -r dc14223818bd8b8a908e056f361f2be1e8972d91 lib/galaxy/tools/__init__.py --- a/lib/galaxy/tools/__init__.py +++ b/lib/galaxy/tools/__init__.py @@ -1204,6 +1204,8 @@ """ if tool_lineage is None: assert tool is not None + if not hasattr( tool, "lineage" ): + return None tool_lineage = tool.lineage lineage_ids = tool_lineage.get_version_ids( reverse=True ) for lineage_id in lineage_ids: Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: jmchilton: Shield ToolBox details from Tool.
by commits-noreply＠bitbucket.org 29 Dec '14

29 Dec '14

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/6268351b242f/ Changeset: 6268351b242f User: jmchilton Date: 2014-12-29 17:14:41+00:00 Summary: Shield ToolBox details from Tool. Better seperation of concerns. Affected #: 1 file diff -r 069b1c6c5f67e74c88d8109ba89cb52ed28deee4 -r 6268351b242feb5c2167b35996e264e8a51c11a9 lib/galaxy/tools/__init__.py --- a/lib/galaxy/tools/__init__.py +++ b/lib/galaxy/tools/__init__.py @@ -304,6 +304,23 @@ tool_section = None return tool_panel_section_key, tool_section + def get_integrated_section_for_tool( self, tool ): + tool_id = tool.id + for key, item in self.integrated_tool_panel.items(): + if item: + if isinstance( item, Tool ): + if item.id == tool_id: + return '', '' + if isinstance( item, ToolSection ): + section_id = item.id or '' + section_name = item.name or '' + for section_key, section_item in item.elems.items(): + if isinstance( section_item, Tool ): + if section_item: + if section_item.id == tool_id: + return section_id, section_name + return None, None + def __resolve_tool_path(self, tool_path, config_filename): if not tool_path: # Default to backward compatible config setting. @@ -1553,20 +1570,7 @@ return self.app.job_config.get_destination(self.__get_job_tool_configuration(job_params=job_params).destination) def get_panel_section( self ): - for key, item in self.app.toolbox.integrated_tool_panel.items(): - if item: - if isinstance( item, Tool ): - if item.id == self.id: - return '', '' - if isinstance( item, ToolSection ): - section_id = item.id or '' - section_name = item.name or '' - for section_key, section_item in item.elems.items(): - if isinstance( section_item, Tool ): - if section_item: - if section_item.id == self.id: - return section_id, section_name - return None, None + return self.app.toolbox.get_integrated_section_for_tool( self ) def parse( self, tool_source, guid=None ): """ Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: jmchilton: Bugfix: Apply tool panel filtering in workflow editor.
by commits-noreply＠bitbucket.org 29 Dec '14

29 Dec '14

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/069b1c6c5f67/ Changeset: 069b1c6c5f67 User: jmchilton Date: 2014-12-29 16:48:38+00:00 Summary: Bugfix: Apply tool panel filtering in workflow editor. Bug originally reported by Eric here - http://dev.list.galaxyproject.org/Bug-Toolbox-filters-not-applied-in-workfl… (though this only fixes the superficial bug of them showing up in the workflow editor - and does not yet implement "strong" toolbox filters as requested). This is actually a broader change that centralizes logic for listing out tool panel contents when displaying components to the user and the added filtering comes about from that. It also eliminates the last places Galaxy was accessing toolbox.tool_panel directly (in those mako templates). Affected #: 4 files diff -r ad5094376c1042a3c5aeab793e326cebcc9818b9 -r 069b1c6c5f67e74c88d8109ba89cb52ed28deee4 lib/galaxy/tools/__init__.py --- a/lib/galaxy/tools/__init__.py +++ b/lib/galaxy/tools/__init__.py @@ -1147,24 +1147,22 @@ """ return self.app.model.context + def tool_panel_contents( self, trans, **kwds ): + """ Filter tool_panel contents for displaying for user. + """ + context = Bunch( toolbox=self, trans=trans ) + filters = self.filter_factory.build_filters( trans ) + for elt in self.tool_panel.itervalues(): + elt = _filter_for_panel( elt, filters, context ) + if elt: + yield elt + def to_dict( self, trans, in_panel=True, **kwds ): """ to_dict toolbox. """ - - context = Bunch( toolbox=self, trans=trans, **kwds ) if in_panel: - panel_elts = [ val for val in self.tool_panel.itervalues() ] - - filters = self.filter_factory.build_filters( trans, **kwds ) - - filtered_panel_elts = [] - for index, elt in enumerate( panel_elts ): - elt = _filter_for_panel( elt, filters, context ) - if elt: - filtered_panel_elts.append( elt ) - panel_elts = filtered_panel_elts - + panel_elts = list( self.tool_panel_contents( trans, **kwds ) ) # Produce panel. rval = [] kwargs = dict( diff -r ad5094376c1042a3c5aeab793e326cebcc9818b9 -r 069b1c6c5f67e74c88d8109ba89cb52ed28deee4 templates/admin/package_tool.mako --- a/templates/admin/package_tool.mako +++ b/templates/admin/package_tool.mako @@ -26,7 +26,7 @@ Tool to bundle: </label><select name="tool_id"> - %for key, val in toolbox.tool_panel.items(): + %for val in toolbox.tool_panel_contents( trans ): %if isinstance( val, Tool ): <option value="${val.id|h}">${val.name|h}</option> %elif isinstance( val, ToolSection ): diff -r ad5094376c1042a3c5aeab793e326cebcc9818b9 -r 069b1c6c5f67e74c88d8109ba89cb52ed28deee4 templates/admin/reload_tool.mako --- a/templates/admin/reload_tool.mako +++ b/templates/admin/reload_tool.mako @@ -26,7 +26,7 @@ Tool to reload: </label><select name="tool_id"> - %for key, val in toolbox.tool_panel.items(): + %for val in toolbox.tool_panel_contents( trans ): %if isinstance( val, Tool ): <option value="${val.id}">${val.name|h}</option> %elif isinstance( val, ToolSection ): diff -r ad5094376c1042a3c5aeab793e326cebcc9818b9 -r 069b1c6c5f67e74c88d8109ba89cb52ed28deee4 templates/webapps/galaxy/workflow/editor.mako --- a/templates/webapps/galaxy/workflow/editor.mako +++ b/templates/webapps/galaxy/workflow/editor.mako @@ -284,7 +284,7 @@ <img src="${h.url_for('/static/images/loading_small_white_bg.gif')}" id="search-spinner" class="search-spinner" /></div><div class="toolSectionList"> - %for key, val in app.toolbox.tool_panel.items(): + %for val in app.toolbox.tool_panel_contents( trans ): <div class="toolSectionWrapper"> %if isinstance( val, Tool ): ${render_tool( val, False )} Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: jmchilton: Fix unit tests for 7cee92f.
by commits-noreply＠bitbucket.org 29 Dec '14

29 Dec '14

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/ad5094376c10/ Changeset: ad5094376c10 User: jmchilton Date: 2014-12-29 16:10:41+00:00 Summary: Fix unit tests for 7cee92f. Affected #: 1 file diff -r 806202db57a7502c2aa5476f2db46f7aae75f439 -r ad5094376c1042a3c5aeab793e326cebcc9818b9 test/unit/tools/test_toolbox.py --- a/test/unit/tools/test_toolbox.py +++ b/test/unit/tools/test_toolbox.py @@ -197,13 +197,13 @@ def test_update_shed_conf(self): self.__setup_shed_tool_conf() - self.toolbox.update_shed_config( 0, {} ) + self.toolbox.update_shed_config( { "config_filename": "tool_conf.xml" } ) assert self.reindexed self.assert_integerated_tool_panel(exists=True) def test_update_shed_conf_deactivate_only(self): self.__setup_shed_tool_conf() - self.toolbox.update_shed_config( 0, {}, integrated_panel_changes=False ) + self.toolbox.update_shed_config( { "config_filename": "tool_conf.xml" }, integrated_panel_changes=False ) assert self.reindexed # No changes, should be regenerated self.assert_integerated_tool_panel(exists=False) Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: 2 new changesets
by commits-noreply＠bitbucket.org 29 Dec '14

29 Dec '14

2 new commits in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/b26dc879b09c/ Changeset: b26dc879b09c User: jmchilton Date: 2014-12-29 15:28:37+00:00 Summary: Abstract tool lineage handling out of ToolBox. Start work on infrastructure to support tool lineages not defined within the tool shed install model. Affected #: 5 files diff -r 68255ebe3f54500aa26db814c41f67f382731462 -r b26dc879b09c639b47c76cb97b067073a3c956e6 lib/galaxy/tools/__init__.py --- a/lib/galaxy/tools/__init__.py +++ b/lib/galaxy/tools/__init__.py @@ -53,6 +53,7 @@ from galaxy.tools.parser import get_tool_source from galaxy.tools.parser.xml import XmlPageSource from galaxy.tools.toolbox import tool_tag_manager +from galaxy.tools.toolbox.lineages import LineageMap from galaxy.util import listify, parse_xml, rst_to_html, string_as_bool, string_to_object from galaxy.tools.parameters.meta import expand_meta_parameters from galaxy.util.bunch import Bunch @@ -139,6 +140,7 @@ self.tool_panel = odict() self.index = 0 self.data_manager_tools = odict() + self.lineage_map = LineageMap( app ) # File that contains the XML section and tool tags from all tool panel config files integrated into a # single file that defines the tool panel layout. This file can be changed by the Galaxy administrator # (in a way similar to the single tool_conf.xml file in the past) to alter the layout of the tool panel. @@ -371,8 +373,8 @@ log.debug( "Loaded tool id: %s, version: %s into tool panel.." % ( tool.id, tool.version ) ) else: # We are in the process of installing the tool. - tool_version = self.__get_tool_version( tool_id ) - tool_lineage_ids = tool_version.get_version_ids( self.app, reverse=True ) + tool_lineage = self.lineage_map.get( tool_id ) + tool_lineage_ids = tool_lineage.get_version_ids( reverse=True ) for lineage_id in tool_lineage_ids: if lineage_id in self.tools_by_id: loaded_version_key = 'tool_%s' % lineage_id @@ -512,9 +514,9 @@ return self.tools_by_id[ tool_id ] #exact tool id match not found, or all versions requested, search for other options, e.g. migrated tools or different versions rval = [] - tv = self.__get_tool_version( tool_id ) - if tv: - tool_version_ids = tv.get_version_ids( self.app ) + tool_lineage = self.lineage_map.get( tool_id ) + if tool_lineage: + tool_version_ids = tool_lineage.get_version_ids( ) for tool_version_id in tool_version_ids: if tool_version_id in self.tools_by_id: rval.append( self.tools_by_id[ tool_version_id ] ) @@ -560,9 +562,9 @@ def get_loaded_tools_by_lineage( self, tool_id ): """Get all loaded tools associated by lineage to the tool whose id is tool_id.""" - tv = self.__get_tool_version( tool_id ) - if tv: - tool_version_ids = tv.get_version_ids( self.app ) + tool_lineage = self.lineage_map.get( tool_id ) + if tool_lineage: + tool_version_ids = tool_lineage.get_version_ids( ) available_tool_versions = [] for tool_version_id in tool_version_ids: if tool_version_id in self.tools_by_id: @@ -574,12 +576,6 @@ return [ tool ] return [] - def __get_tool_version( self, tool_id ): - """Return a ToolVersion if one exists for the tool_id""" - return self.app.install_model.context.query( self.app.install_model.ToolVersion ) \ - .filter( self.app.install_model.ToolVersion.table.c.tool_id == tool_id ) \ - .first() - def tools( self ): return self.tools_by_id.iteritems() @@ -745,12 +741,10 @@ tool.guid = guid tool.version = elem.find( "version" ).text # Make sure the tool has a tool_version. - if not self.__get_tool_version( tool.id ): - tool_version = self.app.install_model.ToolVersion( tool_id=tool.id, tool_shed_repository=tool_shed_repository ) - self.app.install_model.context.add( tool_version ) - self.app.install_model.context.flush() + tool_lineage = self.lineage_map.register( tool, tool_shed_repository=tool_shed_repository ) # Load the tool's lineage ids. - tool.lineage_ids = tool.tool_version.get_version_ids( self.app ) + tool.lineage = tool_lineage + tool.lineage_ids = tool_lineage.get_version_ids( ) self.tool_tag_manager.handle_tags( tool.id, elem ) self.__add_tool( tool, load_panel_dict, panel_dict ) # Always load the tool into the integrated_panel_dict, or it will not be included in the integrated_tool_panel.xml file. @@ -1493,14 +1487,6 @@ return [] @property - def tool_version_ids( self ): - # If we have versions, return a list of their tool_ids. - tool_version = self.tool_version - if tool_version: - return tool_version.get_version_ids( self.app ) - return [] - - @property def tool_shed_repository( self ): # If this tool is included in an installed tool shed repository, return it. if self.tool_shed: diff -r 68255ebe3f54500aa26db814c41f67f382731462 -r b26dc879b09c639b47c76cb97b067073a3c956e6 lib/galaxy/tools/toolbox/lineages/__init__.py --- /dev/null +++ b/lib/galaxy/tools/toolbox/lineages/__init__.py @@ -0,0 +1,5 @@ +from .factory import LineageMap +from .interface import ToolLineage + + +__all__ = ["LineageMap", "ToolLineage"] diff -r 68255ebe3f54500aa26db814c41f67f382731462 -r b26dc879b09c639b47c76cb97b067073a3c956e6 lib/galaxy/tools/toolbox/lineages/factory.py --- /dev/null +++ b/lib/galaxy/tools/toolbox/lineages/factory.py @@ -0,0 +1,28 @@ +from .tool_shed import ToolShedLineage + + +class LineageMap(object): + """ Map each unique tool id to a lineage object. + """ + + def __init__(self, app): + self.lineage_map = {} + self.app = app + + def register(self, tool, **kwds): + tool_id = tool.id + if tool_id not in self.lineage_map: + tool_shed_repository = kwds.get("tool_shed_repository", None) + lineage = ToolShedLineage.from_tool(self.app, tool, tool_shed_repository) + self.lineage_map[tool_id] = lineage + return self.lineage_map[tool_id] + + def get(self, tool_id): + if tool_id not in self.lineage_map: + lineage = ToolShedLineage.from_tool_id( self.app, tool_id ) + if lineage: + self.lineage_map[tool_id] = lineage + + return self.lineage_map.get(tool_id, None) + +__all__ = ["LineageMap"] diff -r 68255ebe3f54500aa26db814c41f67f382731462 -r b26dc879b09c639b47c76cb97b067073a3c956e6 lib/galaxy/tools/toolbox/lineages/interface.py --- /dev/null +++ b/lib/galaxy/tools/toolbox/lineages/interface.py @@ -0,0 +1,14 @@ +from abc import ABCMeta +from abc import abstractmethod + + +class ToolLineage(object): + """ + """ + __metaclass__ = ABCMeta + + @abstractmethod + def get_version_ids( self, reverse=False ): + """ Return an ordered list of lineages in this chain, from + oldest to newest. + """ diff -r 68255ebe3f54500aa26db814c41f67f382731462 -r b26dc879b09c639b47c76cb97b067073a3c956e6 lib/galaxy/tools/toolbox/lineages/tool_shed.py --- /dev/null +++ b/lib/galaxy/tools/toolbox/lineages/tool_shed.py @@ -0,0 +1,43 @@ +from .interface import ToolLineage + +from galaxy.model.tool_shed_install import ToolVersion + + +class ToolShedLineage(ToolLineage): + """ Representation of tool lineage derived from tool shed repository + installations. """ + + def __init__(self, app, tool_version): + self.app = app + self.tool_version_id = tool_version.id + + @staticmethod + def from_tool( app, tool, tool_shed_repository ): + # Make sure the tool has a tool_version. + if not get_install_tool_version( app, tool.id ): + tool_version = ToolVersion( tool_id=tool.id, tool_shed_repository=tool_shed_repository ) + app.install_model.context.add( tool_version ) + app.install_model.context.flush() + return ToolShedLineage( app, tool.tool_version ) + + @staticmethod + def from_tool_id( app, tool_id ): + tool_version = get_install_tool_version( app, tool_id ) + if tool_version: + return ToolShedLineage( app, tool_version ) + else: + return None + + def get_version_ids( self, reverse=False ): + tool_version = self.app.install_model.context.query( ToolVersion ).get( self.tool_version_id ) + return tool_version.get_version_ids( self.app, reverse=reverse ) + + +def get_install_tool_version( app, tool_id ): + return app.install_model.context.query( + app.install_model.ToolVersion + ).filter( + app.install_model.ToolVersion.table.c.tool_id == tool_id + ).first() + +__all__ = [ "ToolShedLineage" ] https://bitbucket.org/galaxy/galaxy-central/commits/806202db57a7/ Changeset: 806202db57a7 User: jmchilton Date: 2014-12-29 15:28:37+00:00 Summary: Abstract out higher-level helpers in ToolBox for lineage reasoning. Reduces code duplication slightly and simplifies __add_tool_to_tool_panel (cyclomatic complexity reduced from 20 to 14). Affected #: 1 file diff -r b26dc879b09c639b47c76cb97b067073a3c956e6 -r 806202db57a7502c2aa5476f2db46f7aae75f439 lib/galaxy/tools/__init__.py --- a/lib/galaxy/tools/__init__.py +++ b/lib/galaxy/tools/__init__.py @@ -325,20 +325,13 @@ panel_dict = panel_component.elems else: panel_dict = panel_component - already_loaded = False - loaded_version_key = None - lineage_id = None - for lineage_id in tool.lineage_ids: - if lineage_id in self.tools_by_id: - loaded_version_key = 'tool_%s' % lineage_id - if loaded_version_key in panel_dict: - already_loaded = True - break - if already_loaded: - if tool.lineage_ids.index( tool_id ) > tool.lineage_ids.index( lineage_id ): + related_tool = self._lineage_in_panel( panel_dict, tool=tool ) + if related_tool: + if self._newer_tool( tool, related_tool ): + lineage_id = "tool_%s" % related_tool.id + index = panel_dict.keys().index( lineage_id ) + del panel_dict[ lineage_id ] key = 'tool_%s' % tool.id - index = panel_dict.keys().index( loaded_version_key ) - del panel_dict[ loaded_version_key ] panel_dict.insert( index, key, tool ) log.debug( "Loaded tool id: %s, version: %s into tool panel." % ( tool.id, tool.version ) ) else: @@ -374,12 +367,7 @@ else: # We are in the process of installing the tool. tool_lineage = self.lineage_map.get( tool_id ) - tool_lineage_ids = tool_lineage.get_version_ids( reverse=True ) - for lineage_id in tool_lineage_ids: - if lineage_id in self.tools_by_id: - loaded_version_key = 'tool_%s' % lineage_id - if loaded_version_key in panel_dict: - already_loaded = True + already_loaded = self._lineage_in_panel( panel_dict, tool_lineage=tool_lineage ) is not None if not already_loaded: # If the tool is not defined in integrated_tool_panel.xml, append it to the tool panel. panel_dict[ key ] = tool @@ -1193,6 +1181,27 @@ return rval + def _lineage_in_panel( self, panel_dict, tool=None, tool_lineage=None ): + """ If tool with same lineage already in panel (or section) - find + and return it. Otherwise return None. + """ + if tool_lineage is None: + assert tool is not None + tool_lineage = tool.lineage + lineage_ids = tool_lineage.get_version_ids( reverse=True ) + for lineage_id in lineage_ids: + if lineage_id in self.tools_by_id: + loaded_version_key = 'tool_%s' % lineage_id + if loaded_version_key in panel_dict: + return panel_dict[ loaded_version_key ] + return None + + def _newer_tool( self, tool1, tool2 ): + """ Return True if tool1 is considered "newer" given its own lineage + description. + """ + return tool1.lineage_ids.index( tool1.id ) > tool1.lineage_ids.index( tool2.id ) + def _filter_for_panel( item, filters, context ): """ Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0