- galaxy-commits - lists.galaxyproject.org

commit/galaxy-central: 2 new changesets
by commits-noreply＠bitbucket.org 05 Jan '15

05 Jan '15

2 new commits in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/ddcb4c7e2957/ Changeset: ddcb4c7e2957 User: jmchilton Date: 2014-12-19 14:15:27+00:00 Summary: Replace preferred workflow usage terminology in the API (s/usage/invocation) `GET /api/workflows/<workflow_id>/usage` made sense for read-only operations - but now for instance `POST /api/workflows/<workflow_id>/usage` is the preferred method for invoking workflows and `PUT /api/workflows/<workflow_id>/usage/<usage_id>/steps/<step_id>` can be used to interact with steps while the workflow is running. I think the term is a little awkward in that sense and is incongruous with the rest of Galaxy which uses the terminology "invocations". In a backward compatible way this replaces the usage terminology with invocation terminology (and uses the plural in the route which is more consistent with the rest of the API - `POST /api/histories` for instance). Affected #: 2 files diff -r 815d38c48a5639d47eed92806ef4c196406f58f1 -r ddcb4c7e29574f8605e6a9cd99b039b25300eee9 lib/galaxy/webapps/galaxy/api/workflows.py --- a/lib/galaxy/webapps/galaxy/api/workflows.py +++ b/lib/galaxy/webapps/galaxy/api/workflows.py @@ -372,9 +372,9 @@ return item @expose_api - def workflow_request( self, trans, workflow_id, payload, **kwd ): + def invoke( self, trans, workflow_id, payload, **kwd ): """ - POST /api/workflows/{encoded_workflow_id}/usage + POST /api/workflows/{encoded_workflow_id}/invocations Schedule the workflow specified by `workflow_id` to run. """ @@ -398,10 +398,11 @@ return self.encode_all_ids( trans, workflow_invocation.to_dict(), recursive=True ) @expose_api - def workflow_usage(self, trans, workflow_id, **kwd): + def index_invocations(self, trans, workflow_id, **kwd): """ - GET /api/workflows/{workflow_id}/usage - Get the list of the workflow usage + GET /api/workflows/{workflow_id}/invocations + + Get the list of the workflow invocations :param workflow_id: the workflow id (required) :type workflow_id: str @@ -416,53 +417,53 @@ return out @expose_api - def workflow_usage_contents(self, trans, workflow_id, usage_id, **kwd): + def show_invocation(self, trans, workflow_id, invocation_id, **kwd): """ - GET /api/workflows/{workflow_id}/usage/{usage_id} - Get detailed description of workflow usage + GET /api/workflows/{workflow_id}/invocation/{invocation_id} + Get detailed description of workflow invocation - :param workflow_id: the workflow id (required) - :type workflow_id: str + :param workflow_id: the workflow id (required) + :type workflow_id: str - :param usage_id: the usage id (required) - :type usage_id: str + :param invocation_id: the invocation id (required) + :type invocation_id: str :raises: exceptions.MessageException, exceptions.ObjectNotFound """ - decoded_workflow_invocation_id = self.__decode_id( trans, usage_id ) + decoded_workflow_invocation_id = self.__decode_id( trans, invocation_id ) workflow_invocation = self.workflow_manager.get_invocation( trans, decoded_workflow_invocation_id ) if workflow_invocation: return self.__encode_invocation( trans, workflow_invocation ) return None @expose_api - def cancel_workflow_invocation(self, trans, workflow_id, usage_id, **kwd): + def cancel_invocation(self, trans, workflow_id, invocation_id, **kwd): """ - DELETE /api/workflows/{workflow_id}/usage/{usage_id} + DELETE /api/workflows/{workflow_id}/invocation/{invocation_id} Cancel the specified workflow invocation. :param workflow_id: the workflow id (required) :type workflow_id: str - :param usage_id: the usage id (required) - :type usage_id: str + :param invocation_id: the usage id (required) + :type invocation_id: str :raises: exceptions.MessageException, exceptions.ObjectNotFound """ - decoded_workflow_invocation_id = self.__decode_id( trans, usage_id ) + decoded_workflow_invocation_id = self.__decode_id( trans, invocation_id ) workflow_invocation = self.workflow_manager.cancel_invocation( trans, decoded_workflow_invocation_id ) return self.__encode_invocation( trans, workflow_invocation ) @expose_api - def workflow_invocation_step(self, trans, workflow_id, usage_id, step_id, **kwd): + def invocation_step(self, trans, workflow_id, invocation_id, step_id, **kwd): """ - GET /api/workflows/{workflow_id}/usage/{usage_id}/steps/{step_id} + GET /api/workflows/{workflow_id}/invocation/{invocation_id}/steps/{step_id} - :param workflow_id: the workflow id (required) - :type workflow_id: str + :param workflow_id: the workflow id (required) + :type workflow_id: str - :param usage_id: the usage id (required) - :type usage_id: str + :param invocation_id: the invocation id (required) + :type invocation_id: str :param step_id: encoded id of the WorkflowInvocationStep (required) :type step_id: str @@ -480,9 +481,9 @@ return self.__encode_invocation_step( trans, invocation_step ) @expose_api - def workflow_invocation_step_update(self, trans, workflow_id, usage_id, step_id, payload, **kwd): + def update_invocation_step(self, trans, workflow_id, invocation_id, step_id, payload, **kwd): """ - PUT /api/workflows/{workflow_id}/usage/{usage_id}/steps/{step_id} + PUT /api/workflows/{workflow_id}/invocation/{invocation_id}/steps/{step_id} Update state of running workflow step invocation - still very nebulous but this would be for stuff like confirming paused steps can proceed etc.... @@ -491,8 +492,8 @@ :param workflow_id: the workflow id (required) :type workflow_id: str - :param usage_id: the usage id (required) - :type usage_id: str + :param invocation_id: the usage id (required) + :type invocation_id: str :param step_id: encoded id of the WorkflowInvocationStep (required) :type step_id: str diff -r 815d38c48a5639d47eed92806ef4c196406f58f1 -r ddcb4c7e29574f8605e6a9cd99b039b25300eee9 lib/galaxy/webapps/galaxy/buildapp.py --- a/lib/galaxy/webapps/galaxy/buildapp.py +++ b/lib/galaxy/webapps/galaxy/buildapp.py @@ -227,14 +227,62 @@ webapp.mapper.connect( 'workflow_dict', '/api/workflows/download/{workflow_id}', controller='workflows', action='workflow_dict', conditions=dict( method=['GET'] ) ) # Deprecated in favor of POST /api/workflows with shared_workflow_id in payload. webapp.mapper.connect( 'import_shared_workflow_deprecated', '/api/workflows/import', controller='workflows', action='import_shared_workflow_deprecated', conditions=dict( method=['POST'] ) ) - webapp.mapper.connect( 'workflow_usage', '/api/workflows/{workflow_id}/usage', controller='workflows', action='workflow_usage', conditions=dict(method=['GET'])) - webapp.mapper.connect( 'workflow_usage_contents', '/api/workflows/{workflow_id}/usage/{usage_id}', controller='workflows', action='workflow_usage_contents', conditions=dict(method=['GET'])) - webapp.mapper.connect( 'cancel_workflow_invocation', '/api/workflows/{workflow_id}/usage/{usage_id}', controller='workflows', action='cancel_workflow_invocation', conditions=dict(method=['DELETE'])) - webapp.mapper.connect( 'workflow_invocation_step', '/api/workflows/{workflow_id}/usage/{usage_id}/steps/{step_id}', controller='workflows', action='workflow_invocation_step', conditions=dict(method=['GET'])) - webapp.mapper.connect( 'workflow_invocation_step_update', '/api/workflows/{workflow_id}/usage/{usage_id}/steps/{step_id}', controller='workflows', action='workflow_invocation_step_update', conditions=dict(method=['PUT'])) + # API refers to usages and invocations - these mean the same thing but the + # usage routes should be considered deprecated. + invoke_names = { + "invocations": "", + "usage": "_deprecated", + } + for noun, suffix in invoke_names.iteritems(): + name = "%s%s" % (noun, suffix) + webapp.mapper.connect( + 'list_workflow_%s' % name, + '/api/workflows/{workflow_id}/%s' % noun, + controller='workflows', + action='index_invocations', + conditions=dict(method=['GET']) + ) - webapp.mapper.connect( 'workflow_request', '/api/workflows/{workflow_id}/usage', controller='workflows', action='workflow_request', conditions=dict( method=['POST'] ) ) + webapp.mapper.connect( + 'workflow_%s_contents' % name, + '/api/workflows/{workflow_id}/%s/{invocation_id}' % noun, + controller='workflows', + action='show_invocation', + conditions=dict(method=['GET']) + ) + + webapp.mapper.connect( + 'cancel_workflow_%s' % name, + '/api/workflows/{workflow_id}/%s/{invocation_id}' % noun, + controller='workflows', + action='cancel_invocation', + conditions=dict(method=['DELETE']) + ) + + webapp.mapper.connect( + 'workflow_%s_step' % name, + '/api/workflows/{workflow_id}/%s/{invocation_id}/steps/{step_id}' % noun, + controller='workflows', + action='invocation_step', + conditions=dict(method=['GET']) + ) + + webapp.mapper.connect( + 'workflow_%s_step_update' % name, + '/api/workflows/{workflow_id}/%s/{invocation_id}/steps/{step_id}' % noun, + controller='workflows', + action='update_invocation_step', + conditions=dict(method=['PUT']) + ) + + webapp.mapper.connect( + 'workflow_%s' % name, + '/api/workflows/{workflow_id}/%s' % noun, + controller='workflows', + action='invoke', + conditions=dict( method=['POST'] ) + ) # ============================ # ===== AUTHENTICATE API ===== # ============================ https://bitbucket.org/galaxy/galaxy-central/commits/2aa2f508c508/ Changeset: 2aa2f508c508 User: jmchilton Date: 2015-01-05 20:59:02+00:00 Summary: Merged in jmchilton/galaxy-central-fork-1 (pull request #621) Workflow API: Replace preferred usage terminology in the API (s/usage/invocation) Affected #: 2 files diff -r 03955af117563248e950ec97d151c4a4615eaa58 -r 2aa2f508c50864b3d309d384137e53f5750aed2d lib/galaxy/webapps/galaxy/api/workflows.py --- a/lib/galaxy/webapps/galaxy/api/workflows.py +++ b/lib/galaxy/webapps/galaxy/api/workflows.py @@ -372,9 +372,9 @@ return item @expose_api - def workflow_request( self, trans, workflow_id, payload, **kwd ): + def invoke( self, trans, workflow_id, payload, **kwd ): """ - POST /api/workflows/{encoded_workflow_id}/usage + POST /api/workflows/{encoded_workflow_id}/invocations Schedule the workflow specified by `workflow_id` to run. """ @@ -398,10 +398,11 @@ return self.encode_all_ids( trans, workflow_invocation.to_dict(), recursive=True ) @expose_api - def workflow_usage(self, trans, workflow_id, **kwd): + def index_invocations(self, trans, workflow_id, **kwd): """ - GET /api/workflows/{workflow_id}/usage - Get the list of the workflow usage + GET /api/workflows/{workflow_id}/invocations + + Get the list of the workflow invocations :param workflow_id: the workflow id (required) :type workflow_id: str @@ -416,53 +417,53 @@ return out @expose_api - def workflow_usage_contents(self, trans, workflow_id, usage_id, **kwd): + def show_invocation(self, trans, workflow_id, invocation_id, **kwd): """ - GET /api/workflows/{workflow_id}/usage/{usage_id} - Get detailed description of workflow usage + GET /api/workflows/{workflow_id}/invocation/{invocation_id} + Get detailed description of workflow invocation - :param workflow_id: the workflow id (required) - :type workflow_id: str + :param workflow_id: the workflow id (required) + :type workflow_id: str - :param usage_id: the usage id (required) - :type usage_id: str + :param invocation_id: the invocation id (required) + :type invocation_id: str :raises: exceptions.MessageException, exceptions.ObjectNotFound """ - decoded_workflow_invocation_id = self.__decode_id( trans, usage_id ) + decoded_workflow_invocation_id = self.__decode_id( trans, invocation_id ) workflow_invocation = self.workflow_manager.get_invocation( trans, decoded_workflow_invocation_id ) if workflow_invocation: return self.__encode_invocation( trans, workflow_invocation ) return None @expose_api - def cancel_workflow_invocation(self, trans, workflow_id, usage_id, **kwd): + def cancel_invocation(self, trans, workflow_id, invocation_id, **kwd): """ - DELETE /api/workflows/{workflow_id}/usage/{usage_id} + DELETE /api/workflows/{workflow_id}/invocation/{invocation_id} Cancel the specified workflow invocation. :param workflow_id: the workflow id (required) :type workflow_id: str - :param usage_id: the usage id (required) - :type usage_id: str + :param invocation_id: the usage id (required) + :type invocation_id: str :raises: exceptions.MessageException, exceptions.ObjectNotFound """ - decoded_workflow_invocation_id = self.__decode_id( trans, usage_id ) + decoded_workflow_invocation_id = self.__decode_id( trans, invocation_id ) workflow_invocation = self.workflow_manager.cancel_invocation( trans, decoded_workflow_invocation_id ) return self.__encode_invocation( trans, workflow_invocation ) @expose_api - def workflow_invocation_step(self, trans, workflow_id, usage_id, step_id, **kwd): + def invocation_step(self, trans, workflow_id, invocation_id, step_id, **kwd): """ - GET /api/workflows/{workflow_id}/usage/{usage_id}/steps/{step_id} + GET /api/workflows/{workflow_id}/invocation/{invocation_id}/steps/{step_id} - :param workflow_id: the workflow id (required) - :type workflow_id: str + :param workflow_id: the workflow id (required) + :type workflow_id: str - :param usage_id: the usage id (required) - :type usage_id: str + :param invocation_id: the invocation id (required) + :type invocation_id: str :param step_id: encoded id of the WorkflowInvocationStep (required) :type step_id: str @@ -480,9 +481,9 @@ return self.__encode_invocation_step( trans, invocation_step ) @expose_api - def workflow_invocation_step_update(self, trans, workflow_id, usage_id, step_id, payload, **kwd): + def update_invocation_step(self, trans, workflow_id, invocation_id, step_id, payload, **kwd): """ - PUT /api/workflows/{workflow_id}/usage/{usage_id}/steps/{step_id} + PUT /api/workflows/{workflow_id}/invocation/{invocation_id}/steps/{step_id} Update state of running workflow step invocation - still very nebulous but this would be for stuff like confirming paused steps can proceed etc.... @@ -491,8 +492,8 @@ :param workflow_id: the workflow id (required) :type workflow_id: str - :param usage_id: the usage id (required) - :type usage_id: str + :param invocation_id: the usage id (required) + :type invocation_id: str :param step_id: encoded id of the WorkflowInvocationStep (required) :type step_id: str diff -r 03955af117563248e950ec97d151c4a4615eaa58 -r 2aa2f508c50864b3d309d384137e53f5750aed2d lib/galaxy/webapps/galaxy/buildapp.py --- a/lib/galaxy/webapps/galaxy/buildapp.py +++ b/lib/galaxy/webapps/galaxy/buildapp.py @@ -227,14 +227,62 @@ webapp.mapper.connect( 'workflow_dict', '/api/workflows/download/{workflow_id}', controller='workflows', action='workflow_dict', conditions=dict( method=['GET'] ) ) # Deprecated in favor of POST /api/workflows with shared_workflow_id in payload. webapp.mapper.connect( 'import_shared_workflow_deprecated', '/api/workflows/import', controller='workflows', action='import_shared_workflow_deprecated', conditions=dict( method=['POST'] ) ) - webapp.mapper.connect( 'workflow_usage', '/api/workflows/{workflow_id}/usage', controller='workflows', action='workflow_usage', conditions=dict(method=['GET'])) - webapp.mapper.connect( 'workflow_usage_contents', '/api/workflows/{workflow_id}/usage/{usage_id}', controller='workflows', action='workflow_usage_contents', conditions=dict(method=['GET'])) - webapp.mapper.connect( 'cancel_workflow_invocation', '/api/workflows/{workflow_id}/usage/{usage_id}', controller='workflows', action='cancel_workflow_invocation', conditions=dict(method=['DELETE'])) - webapp.mapper.connect( 'workflow_invocation_step', '/api/workflows/{workflow_id}/usage/{usage_id}/steps/{step_id}', controller='workflows', action='workflow_invocation_step', conditions=dict(method=['GET'])) - webapp.mapper.connect( 'workflow_invocation_step_update', '/api/workflows/{workflow_id}/usage/{usage_id}/steps/{step_id}', controller='workflows', action='workflow_invocation_step_update', conditions=dict(method=['PUT'])) + # API refers to usages and invocations - these mean the same thing but the + # usage routes should be considered deprecated. + invoke_names = { + "invocations": "", + "usage": "_deprecated", + } + for noun, suffix in invoke_names.iteritems(): + name = "%s%s" % (noun, suffix) + webapp.mapper.connect( + 'list_workflow_%s' % name, + '/api/workflows/{workflow_id}/%s' % noun, + controller='workflows', + action='index_invocations', + conditions=dict(method=['GET']) + ) - webapp.mapper.connect( 'workflow_request', '/api/workflows/{workflow_id}/usage', controller='workflows', action='workflow_request', conditions=dict( method=['POST'] ) ) + webapp.mapper.connect( + 'workflow_%s_contents' % name, + '/api/workflows/{workflow_id}/%s/{invocation_id}' % noun, + controller='workflows', + action='show_invocation', + conditions=dict(method=['GET']) + ) + + webapp.mapper.connect( + 'cancel_workflow_%s' % name, + '/api/workflows/{workflow_id}/%s/{invocation_id}' % noun, + controller='workflows', + action='cancel_invocation', + conditions=dict(method=['DELETE']) + ) + + webapp.mapper.connect( + 'workflow_%s_step' % name, + '/api/workflows/{workflow_id}/%s/{invocation_id}/steps/{step_id}' % noun, + controller='workflows', + action='invocation_step', + conditions=dict(method=['GET']) + ) + + webapp.mapper.connect( + 'workflow_%s_step_update' % name, + '/api/workflows/{workflow_id}/%s/{invocation_id}/steps/{step_id}' % noun, + controller='workflows', + action='update_invocation_step', + conditions=dict(method=['PUT']) + ) + + webapp.mapper.connect( + 'workflow_%s' % name, + '/api/workflows/{workflow_id}/%s' % noun, + controller='workflows', + action='invoke', + conditions=dict( method=['POST'] ) + ) # ============================ # ===== AUTHENTICATE API ===== # ============================ Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: jmchilton: Allow specification of post job actions at workflow invocation.
by commits-noreply＠bitbucket.org 05 Jan '15

05 Jan '15

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/03955af11756/ Changeset: 03955af11756 User: jmchilton Date: 2015-01-05 20:38:27+00:00 Summary: Allow specification of post job actions at workflow invocation. Feature requested by Kyle. Implemented only in the API at this point - not sure it is a feature valuable to UI consumers. Pass in PJAs along with step parameters map but keyed on __POST_JOB_ACTIONS__. JSON definition same as when defining PJA in the workflow definition JSON. Includes test cases for normal use and for use after delayed workflow steps have been evaluated by the new workflow scheduling stuff. Affected #: 3 files diff -r 3fc8ce62285bc3df5b934439e624f2d56cb062cf -r 03955af117563248e950ec97d151c4a4615eaa58 lib/galaxy/workflow/modules.py --- a/lib/galaxy/workflow/modules.py +++ b/lib/galaxy/workflow/modules.py @@ -29,6 +29,13 @@ log = logging.getLogger( __name__ ) +# Key into Tool state to describe invocation-specific runtime properties. +RUNTIME_STEP_META_STATE_KEY = "__STEP_META_STATE__" +# Key into step runtime state dict describing invocation-specific post job +# actions (i.e. PJA specified at runtime on top of the workflow-wide defined +# ones. +RUNTIME_POST_JOB_ACTIONS_KEY = "__POST_JOB_ACTIONS__" + class WorkflowModule( object ): @@ -472,6 +479,7 @@ self.tool_id = tool_id self.tool = trans.app.toolbox.get_tool( tool_id, tool_version=tool_version ) self.post_job_actions = {} + self.runtime_post_job_actions = {} self.workflow_outputs = [] self.state = None self.version_changes = [] @@ -556,6 +564,9 @@ state = galaxy.tools.DefaultToolState() app = self.trans.app state.decode( runtime_state, self.tool, app, secure=False ) + state_dict = loads( runtime_state ) + if RUNTIME_STEP_META_STATE_KEY in state_dict: + self.__restore_step_meta_runtime_state( loads( state_dict[ RUNTIME_STEP_META_STATE_KEY ] ) ) return state def normalize_runtime_state( self, runtime_state ): @@ -572,16 +583,19 @@ step.tool_inputs = None step.tool_errors = self.errors for k, v in self.post_job_actions.iteritems(): - # Must have action_type, step. output and a_args are optional. - if 'output_name' in v: - output_name = v['output_name'] - else: - output_name = None - if 'action_arguments' in v: - action_arguments = v['action_arguments'] - else: - action_arguments = None - self.trans.sa_session.add(PostJobAction(v['action_type'], step, output_name, action_arguments)) + pja = self.__to_pja( k, v, step ) + self.trans.sa_session.add( pja ) + + def __to_pja( self, key, value, step ): + if 'output_name' in value: + output_name = value['output_name'] + else: + output_name = None + if 'action_arguments' in value: + action_arguments = value['action_arguments'] + else: + action_arguments = None + return PostJobAction(value['action_type'], step, output_name, action_arguments) def get_name( self ): if self.tool: @@ -677,7 +691,8 @@ tool=self.tool, values=self.state.inputs, errors=( self.errors or {} ) ) def encode_runtime_state( self, trans, state ): - return state.encode( self.tool, self.trans.app ) + encoded = state.encode( self.tool, self.trans.app ) + return encoded def update_state( self, incoming ): # Build a callback that handles setting an input to be required at @@ -712,26 +727,46 @@ self.errors = errors or None def check_and_update_state( self ): - return self.tool.check_and_update_param_values( self.state.inputs, self.trans, allow_workflow_parameters=True ) + inputs = self.state.inputs + return self.tool.check_and_update_param_values( inputs, self.trans, allow_workflow_parameters=True ) def compute_runtime_state( self, trans, step_updates=None, source="html" ): # Warning: This method destructively modifies existing step state. step_errors = None state = self.state + self.runtime_post_job_actions = {} if step_updates: # Get the tool tool = self.tool # Get old errors old_errors = state.inputs.pop( "__errors__", {} ) # Update the state + self.runtime_post_job_actions = step_updates.get(RUNTIME_POST_JOB_ACTIONS_KEY, {}) step_errors = tool.update_state( trans, tool.inputs, state.inputs, step_updates, update_only=True, old_errors=old_errors, source=source ) + step_metadata_runtime_state = self.__step_meta_runtime_state() + if step_metadata_runtime_state: + state.inputs[ RUNTIME_STEP_META_STATE_KEY ] = step_metadata_runtime_state return state, step_errors + def __step_meta_runtime_state( self ): + """ Build a dictionary a of meta-step runtime state (state about how + the workflow step - not the tool state) to be serialized with the Tool + state. + """ + return { RUNTIME_POST_JOB_ACTIONS_KEY: self.runtime_post_job_actions } + + def __restore_step_meta_runtime_state( self, step_runtime_state ): + if RUNTIME_POST_JOB_ACTIONS_KEY in step_runtime_state: + self.runtime_post_job_actions = step_runtime_state[ RUNTIME_POST_JOB_ACTIONS_KEY ] + def execute( self, trans, progress, invocation, step ): tool = trans.app.toolbox.get_tool( step.tool_id, tool_version=step.tool_version ) tool_state = step.state - + # Not strictly needed - but keep Tool state clean by stripping runtime + # metadata parameters from it. + if RUNTIME_STEP_META_STATE_KEY in tool_state.inputs: + del tool_state.inputs[ RUNTIME_STEP_META_STATE_KEY ] collections_to_match = self._find_collections_to_match( tool, progress, step ) # Have implicit collections... if collections_to_match.has_collections(): @@ -815,7 +850,10 @@ # Create new PJA associations with the created job, to be run on completion. # PJA Parameter Replacement (only applies to immediate actions-- rename specifically, for now) # Pass along replacement dict with the execution of the PJA so we don't have to modify the object. - for pja in step.post_job_actions: + post_job_actions = step.post_job_actions + for key, value in self.runtime_post_job_actions.iteritems(): + post_job_actions.append( self.__to_pja( key, value, step ) ) + for pja in post_job_actions: if pja.action_type in ActionBox.immediate_actions: ActionBox.execute( self.trans.app, self.trans.sa_session, pja, job, replacement_dict ) else: diff -r 3fc8ce62285bc3df5b934439e624f2d56cb062cf -r 03955af117563248e950ec97d151c4a4615eaa58 lib/galaxy/workflow/run_request.py --- a/lib/galaxy/workflow/run_request.py +++ b/lib/galaxy/workflow/run_request.py @@ -151,7 +151,7 @@ else: effective_key = key value = param_dict[key] - if isinstance(value, dict) and not ('src' in value and 'id' in value): + if isinstance(value, dict) and (not ('src' in value and 'id' in value) and key != "__POST_JOB_ACTIONS__"): new_params.update(_flatten_step_params( value, effective_key) ) else: new_params[effective_key] = value diff -r 3fc8ce62285bc3df5b934439e624f2d56cb062cf -r 03955af117563248e950ec97d151c4a4615eaa58 test/api/test_workflows.py --- a/test/api/test_workflows.py +++ b/test/api/test_workflows.py @@ -163,8 +163,7 @@ # Wait for workflow to become fully scheduled and then for all jobs # complete. if wait: - self.wait_for_invocation( workflow_id, invocation_id ) - self.dataset_populator.wait_for_history( history_id, assert_ok=True ) + self._wait_for_workflow( workflow_id, invocation_id, history_id ) jobs = self._history_jobs( history_id ) return RunJobsSummary( history_id=history_id, @@ -180,6 +179,12 @@ def _history_jobs( self, history_id ): return self._get("jobs", { "history_id": history_id, "order_by": "create_time" } ).json() + def _wait_for_workflow( self, workflow_id, invocation_id, history_id, assert_ok=True ): + """ Wait for a workflow invocation to completely schedule and then history + to be complete. """ + self.wait_for_invocation( workflow_id, invocation_id ) + self.dataset_populator.wait_for_history( history_id, assert_ok=True ) + # Workflow API TODO: # - Allow history_id as param to workflow run action. (hist_id) @@ -743,6 +748,87 @@ content = self.dataset_populator.get_history_dataset_details( history_id, wait=True, assert_ok=True ) assert content[ "name" ] == "foo was replaced" + @skip_without_tool( "cat1" ) + def test_run_with_runtime_pja( self ): + workflow = self.workflow_populator.load_workflow( name="test_for_pja_runtime" ) + uuid0, uuid1, uuid2 = str(uuid4()), str(uuid4()), str(uuid4()) + workflow["steps"]["0"]["uuid"] = uuid0 + workflow["steps"]["1"]["uuid"] = uuid1 + workflow["steps"]["2"]["uuid"] = uuid2 + workflow_request, history_id = self._setup_workflow_run( workflow, inputs_by='step_index' ) + workflow_request[ "replacement_params" ] = dumps( dict( replaceme="was replaced" ) ) + + pja_map = { + "RenameDatasetActionout_file1": dict( + action_type="RenameDatasetAction", + output_name="out_file1", + action_arguments=dict( newname="foo ${replaceme}" ), + ) + } + workflow_request[ "parameters" ] = dumps({ + uuid2: { "__POST_JOB_ACTIONS__": pja_map } + }) + + run_workflow_response = self._post( "workflows", data=workflow_request ) + self._assert_status_code_is( run_workflow_response, 200 ) + content = self.dataset_populator.get_history_dataset_details( history_id, wait=True, assert_ok=True ) + assert content[ "name" ] == "foo was replaced", content[ "name" ] + + @skip_without_tool( "cat1" ) + def test_run_with_delayed_runtime_pja( self ): + workflow_id = self._upload_yaml_workflow(""" +- label: test_input + type: input +- label: first_cat + tool_id: cat1 + state: + input1: + $link: test_input +- label: the_pause + type: pause + connect: + input: + - first_cat#out_file1 +- label: second_cat + tool_id: cat1 + state: + input1: + $link: the_pause +""") + downloaded_workflow = self._download_workflow( workflow_id ) + print downloaded_workflow + uuid_dict = dict( map( lambda (index, step): ( int( index ), step["uuid"] ), downloaded_workflow["steps"].iteritems() ) ) + history_id = self.dataset_populator.new_history() + hda = self.dataset_populator.new_dataset( history_id, content="1 2 3" ) + self.dataset_populator.wait_for_history( history_id ) + inputs = { + '0': self._ds_entry( hda ), + } + print inputs + uuid2 = uuid_dict[ 3 ] + workflow_request = {} + workflow_request[ "replacement_params" ] = dumps( dict( replaceme="was replaced" ) ) + pja_map = { + "RenameDatasetActionout_file1": dict( + action_type="RenameDatasetAction", + output_name="out_file1", + action_arguments=dict( newname="foo ${replaceme}" ), + ) + } + workflow_request[ "parameters" ] = dumps({ + uuid2: { "__POST_JOB_ACTIONS__": pja_map } + }) + invocation_id = self.__invoke_workflow( history_id, workflow_id, inputs=inputs, request=workflow_request ) + + time.sleep( 2 ) + self.dataset_populator.wait_for_history( history_id ) + self.__review_paused_steps( workflow_id, invocation_id, order_index=2, action=True ) + + self._wait_for_workflow( workflow_id, invocation_id, history_id ) + time.sleep( 1 ) + content = self.dataset_populator.get_history_dataset_details( history_id ) + assert content[ "name" ] == "foo was replaced", content[ "name" ] + @skip_without_tool( "random_lines1" ) def test_run_replace_params_by_tool( self ): workflow_request, history_id = self._setup_random_x2_workflow( "test_for_replace_tool_params" ) @@ -932,15 +1018,13 @@ self._assert_status_code_is( hda_info_response, 200 ) self.assertEquals( hda_info_response.json()[ "metadata_data_lines" ], lines ) - def __invoke_workflow( self, history_id, workflow_id, inputs={}, assert_ok=True ): - workflow_request = dict( - history="hist_id=%s" % history_id, - ) - workflow_request[ "inputs" ] = dumps( inputs ) - workflow_request[ "inputs_by" ] = 'step_index' + def __invoke_workflow( self, history_id, workflow_id, inputs={}, request={}, assert_ok=True ): + request["history"] = "hist_id=%s" % history_id, + if inputs: + request[ "inputs" ] = dumps( inputs ) + request[ "inputs_by" ] = 'step_index' url = "workflows/%s/usage" % ( workflow_id ) - - invocation_response = self._post( url, data=workflow_request ) + invocation_response = self._post( url, data=request ) if assert_ok: self._assert_status_code_is( invocation_response, 200 ) invocation_id = invocation_response.json()[ "id" ] Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: jmchilton: Another hack to fix 08f8850.
by commits-noreply＠bitbucket.org 05 Jan '15

05 Jan '15

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/3fc8ce62285b/ Changeset: 3fc8ce62285b User: jmchilton Date: 2015-01-05 17:35:38+00:00 Summary: Another hack to fix 08f8850. Need to figure out when tools are being loaded but don't have lineages defined. Affected #: 1 file diff -r 95189e44b9bfe5adac1841b72d621a99bff6c927 -r 3fc8ce62285bc3df5b934439e624f2d56cb062cf lib/galaxy/tools/toolbox/base.py --- a/lib/galaxy/tools/toolbox/base.py +++ b/lib/galaxy/tools/toolbox/base.py @@ -295,7 +295,10 @@ inserted = True if not inserted: # Check the tool's installed versions. - for tool_lineage_version in tool.lineage.get_versions(): + versions = [] + if hasattr( tool, 'lineage' ): + versions = tool.lineage.get_versions() + for tool_lineage_version in versions: lineage_id = tool_lineage_version.id index = self._integrated_tool_panel.index_of_tool_id( lineage_id ) if index: Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: carlfeberhard: Browser tests: remove console statements interferring with test json parsing
by commits-noreply＠bitbucket.org 05 Jan '15

05 Jan '15

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/95189e44b9bf/ Changeset: 95189e44b9bf User: carlfeberhard Date: 2015-01-05 15:34:05+00:00 Summary: Browser tests: remove console statements interferring with test json parsing Affected #: 2 files diff -r e34bf92384b6a6722657d83833fb52268865126d -r 95189e44b9bfe5adac1841b72d621a99bff6c927 test/casperjs/modules/api.js --- a/test/casperjs/modules/api.js +++ b/test/casperjs/modules/api.js @@ -355,7 +355,6 @@ // have to attach like GET param - due to body loss in jq url = utils.format( this.urlTpls.update, historyId, id ); if( purge ){ -console.debug( 'adding purge:' + purge ); url += '?purge=True'; } return this.api._ajax( url, { diff -r e34bf92384b6a6722657d83833fb52268865126d -r 95189e44b9bfe5adac1841b72d621a99bff6c927 test/casperjs/modules/historypanel.js --- a/test/casperjs/modules/historypanel.js +++ b/test/casperjs/modules/historypanel.js @@ -139,9 +139,6 @@ var hdaOpacity = this.evaluate( function( name ){ // locate the hda by name and return its opacity - console.debug( name ); - console.debug( $( '.name:contains("' + name + '")' ) ); - console.debug( $( '.name:contains("' + name + '")' ).parents( '.hda' ) ); return $( '.name:contains("' + name + '")' ).parents( '.dataset' ).css( 'opacity' ); }, hdaName ); //this.debug( 'fading in: ' + hdaOpacity ); Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

[galaxyproject/usegalaxy-playbook] b5acab: Increase memory for fastq_paired_end_joiner
by GitHub 05 Jan '15

05 Jan '15

Branch: refs/heads/master Home: https://github.com/galaxyproject/usegalaxy-playbook Commit: b5acab82552361587405c1b3140c82fbb709b610 https://github.com/galaxyproject/usegalaxy-playbook/commit/b5acab8255236158… Author: Nate Coraor <nate(a)bx.psu.edu> Date: 2015-01-05 (Mon, 05 Jan 2015) Changed paths: M templates/galaxy/usegalaxy.org/config/job_conf.xml.j2 Log Message: ----------- Increase memory for fastq_paired_end_joiner

1 0

commit/galaxy-central: jmchilton: Attempt to fix bug introduced in 08f8850853d004bf8a456a147436a669.
by commits-noreply＠bitbucket.org 05 Jan '15

05 Jan '15

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/e34bf92384b6/ Changeset: e34bf92384b6 User: jmchilton Date: 2015-01-05 15:05:46+00:00 Summary: Attempt to fix bug introduced in 08f8850853d004bf8a456a147436a669. Thanks to Nicola for the bug report (https://bitbucket.org/galaxy/galaxy-central/commits/08f8850853d004bf8a456a1…) Affected #: 1 file diff -r 4e60046a38c4bf79c77e5232856208b7e52dc162 -r e34bf92384b6a6722657d83833fb52268865126d lib/galaxy/tools/toolbox/base.py --- a/lib/galaxy/tools/toolbox/base.py +++ b/lib/galaxy/tools/toolbox/base.py @@ -483,8 +483,8 @@ return[ self._tools_by_id[ tool_id ] ] return None - def has_tool( self, tool_id, exact=False ): - return self.get_tool( tool_id, exact=exact ) is not None + def has_tool( self, tool_id, tool_version=None, exact=False ): + return self.get_tool( tool_id, tool_version=tool_version, exact=exact ) is not None def get_tool_id( self, tool_id ): """ Take a tool id (potentially from a different Galaxy instance or that Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: dannon: Slight modification to prevent unintentional (safe and unused, but still) reassignment of parameter 'value' in loop.
by commits-noreply＠bitbucket.org 05 Jan '15

05 Jan '15

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/4e60046a38c4/ Changeset: 4e60046a38c4 User: dannon Date: 2015-01-05 14:06:01+00:00 Summary: Slight modification to prevent unintentional (safe and unused, but still) reassignment of parameter 'value' in loop. Affected #: 1 file diff -r f55708c39129ab2b293edb7ec54faa85506d2800 -r 4e60046a38c4bf79c77e5232856208b7e52dc162 lib/tool_shed/util/web_util.py --- a/lib/tool_shed/util/web_util.py +++ b/lib/tool_shed/util/web_util.py @@ -4,7 +4,7 @@ ALLOWED_MAP = dict(map(lambda x: (x, raw_escape(x)), ALLOWED_ELEMENTS)) -def escape( value ): +def escape( string ): """ A tool shed variant of markupsafe.escape that allows a select few HTML elements that are repeatedly used in messages created deep in the toolshed components. Ideally abstract things would be produced @@ -14,7 +14,7 @@ >>> escape("A <b>repo</b>") u'A <b>repo</b>' """ - escaped = str( raw_escape( value ) ) + escaped = str( raw_escape( string ) ) # Unescape few selected tags. for key, value in ALLOWED_MAP.iteritems(): escaped = escaped.replace(value, key) Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: natefoo: Update tag latest_2014.10.06 for changeset 793d9cd5f9de
by commits-noreply＠bitbucket.org 05 Jan '15

05 Jan '15

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/dff4071c13f9/ Changeset: dff4071c13f9 Branch: stable User: natefoo Date: 2015-01-05 14:00:18+00:00 Summary: Update tag latest_2014.10.06 for changeset 793d9cd5f9de Affected #: 1 file diff -r 793d9cd5f9dec66ceafb36bc38f53453457056e6 -r dff4071c13f9b2908556583227766c65fcfb8a29 .hgtags --- a/.hgtags +++ b/.hgtags @@ -20,4 +20,4 @@ ca45b78adb4152fc6e7395514d46eba6b7d0b838 release_2014.08.11 548ab24667d6206780237bd807f7d857a484c461 latest_2014.08.11 2092948937ac30ef82f71463a235c66d34987088 release_2014.10.06 -5834b1066462dd219f67c1c3cbd77c78e7cf3a6c latest_2014.10.06 +793d9cd5f9dec66ceafb36bc38f53453457056e6 latest_2014.10.06 Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: 2 new changesets
by commits-noreply＠bitbucket.org 05 Jan '15

05 Jan '15

2 new commits in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/13f767186d5e/ Changeset: 13f767186d5e Branch: next-stable User: dannon Date: 2015-01-05 13:56:20+00:00 Summary: Merge stable. Affected #: 3 files diff -r 93a868eb314d257c21bb19380b360bcf0d8dd82c -r 13f767186d5e281a9d647f8e497d6aca48d03417 .hgtags --- a/.hgtags +++ b/.hgtags @@ -20,4 +20,4 @@ ca45b78adb4152fc6e7395514d46eba6b7d0b838 release_2014.08.11 548ab24667d6206780237bd807f7d857a484c461 latest_2014.08.11 2092948937ac30ef82f71463a235c66d34987088 release_2014.10.06 -7086b87d83a9092cbece0fec6f3e3ed03266be0c latest_2014.10.06 +5834b1066462dd219f67c1c3cbd77c78e7cf3a6c latest_2014.10.06 https://bitbucket.org/galaxy/galaxy-central/commits/f55708c39129/ Changeset: f55708c39129 User: dannon Date: 2015-01-05 13:56:37+00:00 Summary: Merge next-stable. Affected #: 3 files diff -r ac1859e27eb0217ae5dabd7495a3b9314f65a1f7 -r f55708c39129ab2b293edb7ec54faa85506d2800 .hgtags --- a/.hgtags +++ b/.hgtags @@ -20,4 +20,4 @@ ca45b78adb4152fc6e7395514d46eba6b7d0b838 release_2014.08.11 548ab24667d6206780237bd807f7d857a484c461 latest_2014.08.11 2092948937ac30ef82f71463a235c66d34987088 release_2014.10.06 -7086b87d83a9092cbece0fec6f3e3ed03266be0c latest_2014.10.06 +5834b1066462dd219f67c1c3cbd77c78e7cf3a6c latest_2014.10.06 Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: jmchilton: Fixes for over escaping in c2bed0a.
by commits-noreply＠bitbucket.org 05 Jan '15

05 Jan '15

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/793d9cd5f9de/ Changeset: 793d9cd5f9de Branch: stable User: jmchilton Date: 2014-12-27 22:30:59+00:00 Summary: Fixes for over escaping in c2bed0a. Fixes dozens of tool functional tests. Affected #: 9 files diff -r 7197e949d3ab990ceb76c193ebc940d90aa04cad -r 793d9cd5f9dec66ceafb36bc38f53453457056e6 lib/galaxy/web/base/controllers/admin.py --- a/lib/galaxy/web/base/controllers/admin.py +++ b/lib/galaxy/web/base/controllers/admin.py @@ -7,7 +7,7 @@ from galaxy.web.form_builder import CheckboxField from string import punctuation as PUNCTUATION import galaxy.queue_worker -from markupsafe import escape +from tool_shed.util.web_util import escape from tool_shed.util import shed_util_common as suc diff -r 7197e949d3ab990ceb76c193ebc940d90aa04cad -r 793d9cd5f9dec66ceafb36bc38f53453457056e6 lib/galaxy/webapps/galaxy/controllers/admin.py --- a/lib/galaxy/webapps/galaxy/controllers/admin.py +++ b/lib/galaxy/webapps/galaxy/controllers/admin.py @@ -17,7 +17,7 @@ from galaxy.web.params import QuotaParamParser from tool_shed.util import common_util from tool_shed.util import encoding_util -from markupsafe import escape +from tool_shed.util.web_util import escape log = logging.getLogger( __name__ ) diff -r 7197e949d3ab990ceb76c193ebc940d90aa04cad -r 793d9cd5f9dec66ceafb36bc38f53453457056e6 lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py --- a/lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py +++ b/lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py @@ -8,9 +8,9 @@ from galaxy.web.form_builder import CheckboxField from galaxy.util import json from galaxy.model.orm import or_ -from markupsafe import escape import tool_shed.repository_types.util as rt_util +from tool_shed.util.web_util import escape from tool_shed.util import common_util from tool_shed.util import encoding_util diff -r 7197e949d3ab990ceb76c193ebc940d90aa04cad -r 793d9cd5f9dec66ceafb36bc38f53453457056e6 lib/galaxy/webapps/tool_shed/controllers/admin.py --- a/lib/galaxy/webapps/tool_shed/controllers/admin.py +++ b/lib/galaxy/webapps/tool_shed/controllers/admin.py @@ -3,7 +3,7 @@ from galaxy import util from galaxy.util import inflector from galaxy import web -from markupsafe import escape +from tool_shed.util.web_util import escape from galaxy.web.base.controller import BaseUIController from galaxy.web.base.controllers.admin import Admin diff -r 7197e949d3ab990ceb76c193ebc940d90aa04cad -r 793d9cd5f9dec66ceafb36bc38f53453457056e6 lib/galaxy/webapps/tool_shed/controllers/repository.py --- a/lib/galaxy/webapps/tool_shed/controllers/repository.py +++ b/lib/galaxy/webapps/tool_shed/controllers/repository.py @@ -6,7 +6,6 @@ from time import strftime from datetime import date from datetime import datetime -from markupsafe import escape from galaxy import util from galaxy import web @@ -19,6 +18,7 @@ from tool_shed.capsule import capsule_manager from tool_shed.dependencies.repository import relation_builder +from tool_shed.util.web_util import escape from tool_shed.galaxy_install import dependency_display from tool_shed.metadata import repository_metadata_manager from tool_shed.utility_containers import ToolShedUtilityContainerManager diff -r 7197e949d3ab990ceb76c193ebc940d90aa04cad -r 793d9cd5f9dec66ceafb36bc38f53453457056e6 lib/galaxy/webapps/tool_shed/controllers/repository_review.py --- a/lib/galaxy/webapps/tool_shed/controllers/repository_review.py +++ b/lib/galaxy/webapps/tool_shed/controllers/repository_review.py @@ -2,7 +2,7 @@ import os from sqlalchemy.sql.expression import func -from markupsafe import escape +from tool_shed.util.web_util import escape from galaxy import util from galaxy import web diff -r 7197e949d3ab990ceb76c193ebc940d90aa04cad -r 793d9cd5f9dec66ceafb36bc38f53453457056e6 lib/galaxy/webapps/tool_shed/controllers/upload.py --- a/lib/galaxy/webapps/tool_shed/controllers/upload.py +++ b/lib/galaxy/webapps/tool_shed/controllers/upload.py @@ -9,7 +9,7 @@ from galaxy import web from galaxy.datatypes import checkers from galaxy.web.base.controller import BaseUIController -from markupsafe import escape +from tool_shed.util.web_util import escape from tool_shed.dependencies import attribute_handlers from tool_shed.galaxy_install import dependency_display diff -r 7197e949d3ab990ceb76c193ebc940d90aa04cad -r 793d9cd5f9dec66ceafb36bc38f53453457056e6 lib/tool_shed/util/repository_util.py --- a/lib/tool_shed/util/repository_util.py +++ b/lib/tool_shed/util/repository_util.py @@ -7,7 +7,7 @@ from galaxy import web from galaxy.web.form_builder import build_select_field from galaxy.webapps.tool_shed.model import directory_hash_id -from markupsafe import escape +from tool_shed.util.web_util import escape from tool_shed.dependencies.repository import relation_builder diff -r 7197e949d3ab990ceb76c193ebc940d90aa04cad -r 793d9cd5f9dec66ceafb36bc38f53453457056e6 lib/tool_shed/util/web_util.py --- /dev/null +++ b/lib/tool_shed/util/web_util.py @@ -0,0 +1,21 @@ +from markupsafe import escape as raw_escape + +ALLOWED_ELEMENTS = ["<b>", "</b>", "<br/>"] +ALLOWED_MAP = dict(map(lambda x: (x, raw_escape(x)), ALLOWED_ELEMENTS)) + + +def escape( value ): + """ A tool shed variant of markupsafe.escape that allows a select few + HTML elements that are repeatedly used in messages created deep + in the toolshed components. Ideally abstract things would be produced + in these components and messages in the views or client side - this is + what should be worked toward - but for now - we have this hack. + + >>> escape("A <b>repo</b>") + u'A <b>repo</b>' + """ + escaped = str( raw_escape( value ) ) + # Unescape few selected tags. + for key, value in ALLOWED_MAP.iteritems(): + escaped = escaped.replace(value, key) + return escaped Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: jmchilton: Merged in erasche2/galaxy-central2 (pull request #619)
by commits-noreply＠bitbucket.org 01 Jan '15

01 Jan '15

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/ac1859e27eb0/ Changeset: ac1859e27eb0 User: jmchilton Date: 2015-01-01 19:16:50+00:00 Summary: Merged in erasche2/galaxy-central2 (pull request #619) Allow for shared secret between Galaxy and upstream proxies Affected #: 9 files diff -r 84d5a72790735ae4da03f27735c52182bb925d9b -r ac1859e27eb0217ae5dabd7495a3b9314f65a1f7 config/galaxy.ini.sample --- a/config/galaxy.ini.sample +++ b/config/galaxy.ini.sample @@ -713,6 +713,13 @@ # *not* include 'HTTP_' at the beginning of the header name. #remote_user_header = HTTP_REMOTE_USER +# If use_remote_user is enabled, anyone who can log in to the Galaxy host may +# impersonate any other user by simply sending the appropriate header. Thus a +# secret shared between the upstream proxy server, and Galaxy is required. +# If anyone other than the Galaxy user is using the server, then apache/nginx should +# pass a value in the header 'GX_SECRET' that is identical the one below +#remote_user_secret = USING THE DEFAULT IS NOT SECURE! + # If use_remote_user is enabled, you can set this to a URL that will log your # users out. #remote_user_logout_href = None diff -r 84d5a72790735ae4da03f27735c52182bb925d9b -r ac1859e27eb0217ae5dabd7495a3b9314f65a1f7 config/tool_shed.ini.sample --- a/config/tool_shed.ini.sample +++ b/config/tool_shed.ini.sample @@ -67,6 +67,13 @@ # https://wiki.galaxyproject.org/Admin/Config/ApacheProxy #use_remote_user = False +# If use_remote_user is enabled, anyone who can log in to the Galaxy host may +# impersonate any other user by simply sending the appropriate header. Thus a +# secret shared between the upstream proxy server, and Galaxy is required. +# If anyone other than the Galaxy user is using the server, then apache/nginx should +# pass a value in the header 'GX_SECRET' that is identical the one below +#remote_user_secret = changethisinproductiontoo + # Configuration for debugging middleware debug = true use_lint = false diff -r 84d5a72790735ae4da03f27735c52182bb925d9b -r ac1859e27eb0217ae5dabd7495a3b9314f65a1f7 lib/galaxy/config.py --- a/lib/galaxy/config.py +++ b/lib/galaxy/config.py @@ -133,6 +133,7 @@ self.remote_user_maildomain = kwargs.get( "remote_user_maildomain", None ) self.remote_user_header = kwargs.get( "remote_user_header", 'HTTP_REMOTE_USER' ) self.remote_user_logout_href = kwargs.get( "remote_user_logout_href", None ) + self.remote_user_secret = kwargs.get( "remote_user_secret", None ) self.require_login = string_as_bool( kwargs.get( "require_login", "False" ) ) self.allow_user_creation = string_as_bool( kwargs.get( "allow_user_creation", "True" ) ) self.allow_user_deletion = string_as_bool( kwargs.get( "allow_user_deletion", "False" ) ) diff -r 84d5a72790735ae4da03f27735c52182bb925d9b -r ac1859e27eb0217ae5dabd7495a3b9314f65a1f7 lib/galaxy/util/__init__.py --- a/lib/galaxy/util/__init__.py +++ b/lib/galaxy/util/__init__.py @@ -1119,6 +1119,8 @@ def safe_str_cmp(a, b): + """safely compare two strings in a timing-attack-resistant manner + """ if len(a) != len(b): return False rv = 0 diff -r 84d5a72790735ae4da03f27735c52182bb925d9b -r ac1859e27eb0217ae5dabd7495a3b9314f65a1f7 lib/galaxy/web/framework/middleware/remoteuser.py --- a/lib/galaxy/web/framework/middleware/remoteuser.py +++ b/lib/galaxy/web/framework/middleware/remoteuser.py @@ -3,6 +3,7 @@ """ import socket +from galaxy.util import safe_str_cmp errorpage = """ <!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN" "http://www.w3.org/TR/html4/loose.dtd"> @@ -37,12 +38,13 @@ class RemoteUser( object ): - def __init__( self, app, maildomain=None, display_servers=None, admin_users=None, remote_user_header=None ): + def __init__( self, app, maildomain=None, display_servers=None, admin_users=None, remote_user_header=None, remote_user_secret_header=None ): self.app = app self.maildomain = maildomain self.display_servers = display_servers or [] self.admin_users = admin_users or [] self.remote_user_header = remote_user_header or 'HTTP_REMOTE_USER' + self.config_secret_header = remote_user_secret_header def __call__( self, environ, start_response ): # Allow display servers @@ -59,6 +61,34 @@ # Rewrite* method for passing REMOTE_USER and a user is # un-authenticated. Any other possible values need to go here as well. path_info = environ.get('PATH_INFO', '') + + # If the secret header is enabled, we expect upstream to send along some key + # in HTTP_GX_SECRET, so we'll need to compare that here to the correct value + # + # This is not an ideal location for this function. The reason being + # that because this check is done BEFORE the REMOTE_USER check, it is + # possible to attack the GX_SECRET key without having correct + # credentials. However, that's why it's not "ideal", but it is "good + # enough". The only users able to exploit this are ones with access to + # the local system (unless Galaxy is listening on 0.0.0.0....). It + # seems improbable that an attacker with access to the server hosting + # Galaxy would not have access to Galaxy itself, and be attempting to + # attack the system + if self.config_secret_header is not None: + if not safe_str_cmp(environ.get('HTTP_GX_SECRET'), self.config_secret_header): + title = "Access to Galaxy is denied" + message = """ + Galaxy is configured to authenticate users via an external + method (such as HTTP authentication in Apache), but an + incorrect shared secret key was provided by the + upstream (proxy) server.</p> + <p>Please contact your local Galaxy administrator. The + variable <code>remote_user_secret</code> and + <code>GX_SECRET</code> header must be set before you may + access Galaxy. + """ + return self.error( start_response, title, message ) + if not environ.get(self.remote_user_header, '(null)').startswith('(null)'): if not environ[ self.remote_user_header ].count( '@' ): if self.maildomain is not None: diff -r 84d5a72790735ae4da03f27735c52182bb925d9b -r ac1859e27eb0217ae5dabd7495a3b9314f65a1f7 lib/galaxy/webapps/galaxy/buildapp.py --- a/lib/galaxy/webapps/galaxy/buildapp.py +++ b/lib/galaxy/webapps/galaxy/buildapp.py @@ -503,8 +503,8 @@ app = RemoteUser( app, maildomain = conf.get( 'remote_user_maildomain', None ), display_servers = util.listify( conf.get( 'display_servers', '' ) ), admin_users = conf.get( 'admin_users', '' ).split( ',' ), - remote_user_header = conf.get( 'remote_user_header', 'HTTP_REMOTE_USER' ) ) - log.debug( "Enabling 'remote user' middleware" ) + remote_user_header = conf.get( 'remote_user_header', 'HTTP_REMOTE_USER' ), + remote_user_secret_header = conf.get('remote_user_secret', None) ) # The recursive middleware allows for including requests in other # requests or forwarding of requests, all on the server side. if asbool(conf.get('use_recursive', True)): diff -r 84d5a72790735ae4da03f27735c52182bb925d9b -r ac1859e27eb0217ae5dabd7495a3b9314f65a1f7 lib/galaxy/webapps/tool_shed/buildapp.py --- a/lib/galaxy/webapps/tool_shed/buildapp.py +++ b/lib/galaxy/webapps/tool_shed/buildapp.py @@ -157,7 +157,8 @@ from galaxy.webapps.tool_shed.framework.middleware.remoteuser import RemoteUser app = RemoteUser( app, maildomain = conf.get( 'remote_user_maildomain', None ), display_servers = util.listify( conf.get( 'display_servers', '' ) ), - admin_users = conf.get( 'admin_users', '' ).split( ',' ) ) + admin_users = conf.get( 'admin_users', '' ).split( ',' ), + remote_user_secret_header = conf.get('remote_user_secret', None) ) log.debug( "Enabling 'remote user' middleware" ) # The recursive middleware allows for including requests in other # requests or forwarding of requests, all on the server side. diff -r 84d5a72790735ae4da03f27735c52182bb925d9b -r ac1859e27eb0217ae5dabd7495a3b9314f65a1f7 lib/galaxy/webapps/tool_shed/config.py --- a/lib/galaxy/webapps/tool_shed/config.py +++ b/lib/galaxy/webapps/tool_shed/config.py @@ -73,6 +73,7 @@ self.remote_user_maildomain = kwargs.get( "remote_user_maildomain", None ) self.remote_user_header = kwargs.get( "remote_user_header", 'HTTP_REMOTE_USER' ) self.remote_user_logout_href = kwargs.get( "remote_user_logout_href", None ) + self.remote_user_secret = kwargs.get( "remote_user_secret", None ) self.require_login = string_as_bool( kwargs.get( "require_login", "False" ) ) self.allow_user_creation = string_as_bool( kwargs.get( "allow_user_creation", "True" ) ) self.allow_user_deletion = string_as_bool( kwargs.get( "allow_user_deletion", "False" ) ) diff -r 84d5a72790735ae4da03f27735c52182bb925d9b -r ac1859e27eb0217ae5dabd7495a3b9314f65a1f7 lib/galaxy/webapps/tool_shed/framework/middleware/remoteuser.py --- a/lib/galaxy/webapps/tool_shed/framework/middleware/remoteuser.py +++ b/lib/galaxy/webapps/tool_shed/framework/middleware/remoteuser.py @@ -3,6 +3,7 @@ """ import socket +from galaxy.util import safe_str_cmp errorpage = """ <!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN" "http://www.w3.org/TR/html4/loose.dtd"> @@ -36,11 +37,13 @@ """ class RemoteUser( object ): - def __init__( self, app, maildomain=None, display_servers=None, admin_users=None ): + def __init__( self, app, maildomain=None, display_servers=None, admin_users=None, remote_user_secret_header=None ): self.app = app self.maildomain = maildomain self.display_servers = display_servers or [] self.admin_users = admin_users or [] + self.config_secret_header = remote_user_secret_header + def __call__( self, environ, start_response ): environ[ 'webapp' ] = 'tool_shed' # Allow display servers @@ -53,6 +56,34 @@ if host in self.display_servers: environ[ 'HTTP_REMOTE_USER' ] = 'remote_display_server@%s' % ( self.maildomain or 'example.org' ) return self.app( environ, start_response ) + + # If the secret header is enabled, we expect upstream to send along some key + # in HTTP_GX_SECRET, so we'll need to compare that here to the correct value + # + # This is not an ideal location for this function. The reason being + # that because this check is done BEFORE the REMOTE_USER check, it is + # possible to attack the GX_SECRET key without having correct + # credentials. However, that's why it's not "ideal", but it is "good + # enough". The only users able to exploit this are ones with access to + # the local system (unless Galaxy is listening on 0.0.0.0....). It + # seems improbable that an attacker with access to the server hosting + # Galaxy would not have access to Galaxy itself, and be attempting to + # attack the system + if self.config_secret_header is not None: + if not safe_str_cmp(environ.get('HTTP_GX_SECRET'), self.config_secret_header): + title = "Access to Galaxy is denied" + message = """ + Galaxy is configured to authenticate users via an external + method (such as HTTP authentication in Apache), but an + incorrect shared secret key was provided by the + upstream (proxy) server.</p> + <p>Please contact your local Galaxy administrator. The + variable <code>remote_user_secret</code> and + <code>GX_SECRET</code> header must be set before you may + access Galaxy. + """ + return self.error( start_response, title, message ) + # Apache sets REMOTE_USER to the string '(null)' when using the Rewrite* method for passing REMOTE_USER and a user is # un-authenticated. Any other possible values need to go here as well. path_info = environ.get('PATH_INFO', '') @@ -88,6 +119,7 @@ <p>Contact your local Galaxy tool shed administrator. """ return self.error( start_response, title, message ) + def error( self, start_response, title="Access denied", message="Contact your local Galaxy tool shed administrator." ): start_response( '403 Forbidden', [('Content-type', 'text/html')] ) return [errorpage % (title, message)] Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: 3 new changesets
by commits-noreply＠bitbucket.org 01 Jan '15

01 Jan '15

3 new commits in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/7a889340aba7/ Changeset: 7a889340aba7 User: erasche2 Date: 2014-12-17 21:25:07+00:00 Summary: Allow for shared secret between Galaxy and upstream proxies This prevents an impersonation attack available on Galaxies with command line users and REMOTE_USER authentication. It is not uncommon to run Galaxy on a "lab server" (or similar), where there are command line users in addition to the Galaxy service. With REMOTE_USER authentication, Galaxy blindly (and correctly) trusts the upstream value of REMOTE_USER, enabling such attacks as: curl -H "REMOTE_USER: admin.user(a)domain.edu" http://localhost:8000/galaxy/ This changest adds a shared secret between Galaxy and the upstream proxy, thereby preventing that attack. Only requests from the upstream proxy with the correct secret key are recognised, everything else is tossed out with a 403 Forbidden. This was implemented as part of the REMOTE_USER stack as the situation only applies if that middleware is in use. Affected #: 9 files diff -r 5e5c0cf930bb96bf8af0bd19daad1a999b40c374 -r 7a889340aba7e078d31963aa99139c7ddaea07c2 config/galaxy.ini.sample --- a/config/galaxy.ini.sample +++ b/config/galaxy.ini.sample @@ -713,6 +713,13 @@ # *not* include 'HTTP_' at the beginning of the header name. #remote_user_header = HTTP_REMOTE_USER +# If use_remote_user is enabled, anyone who can log in to the Galaxy host may +# impersonate any other user by simply sending the appropriate header. Thus a +# secret shared between the upstream proxy server, and Galaxy is required. +# If anyone other than the Galaxy user is using the server, then apache/nginx should +# pass a value in the header 'GX_SECRET' that is identical the one below +#remote_user_secret = USING THE DEFAULT IS NOT SECURE! + # If use_remote_user is enabled, you can set this to a URL that will log your # users out. #remote_user_logout_href = None diff -r 5e5c0cf930bb96bf8af0bd19daad1a999b40c374 -r 7a889340aba7e078d31963aa99139c7ddaea07c2 config/tool_shed.ini.sample --- a/config/tool_shed.ini.sample +++ b/config/tool_shed.ini.sample @@ -67,6 +67,13 @@ # https://wiki.galaxyproject.org/Admin/Config/ApacheProxy #use_remote_user = False +# If use_remote_user is enabled, anyone who can log in to the Galaxy host may +# impersonate any other user by simply sending the appropriate header. Thus a +# secret shared between the upstream proxy server, and Galaxy is required. +# If anyone other than the Galaxy user is using the server, then apache/nginx should +# pass a value in the header 'GX_SECRET' that is identical the one below +#remote_user_secret = changethisinproductiontoo + # Configuration for debugging middleware debug = true use_lint = false diff -r 5e5c0cf930bb96bf8af0bd19daad1a999b40c374 -r 7a889340aba7e078d31963aa99139c7ddaea07c2 lib/galaxy/config.py --- a/lib/galaxy/config.py +++ b/lib/galaxy/config.py @@ -132,6 +132,7 @@ self.remote_user_maildomain = kwargs.get( "remote_user_maildomain", None ) self.remote_user_header = kwargs.get( "remote_user_header", 'HTTP_REMOTE_USER' ) self.remote_user_logout_href = kwargs.get( "remote_user_logout_href", None ) + self.remote_user_secret = kwargs.get( "remote_user_secret", None ) self.require_login = string_as_bool( kwargs.get( "require_login", "False" ) ) self.allow_user_creation = string_as_bool( kwargs.get( "allow_user_creation", "True" ) ) self.allow_user_deletion = string_as_bool( kwargs.get( "allow_user_deletion", "False" ) ) diff -r 5e5c0cf930bb96bf8af0bd19daad1a999b40c374 -r 7a889340aba7e078d31963aa99139c7ddaea07c2 lib/galaxy/util/__init__.py --- a/lib/galaxy/util/__init__.py +++ b/lib/galaxy/util/__init__.py @@ -1119,6 +1119,8 @@ def safe_str_cmp(a, b): + """safely compare two strings in a timing-attack-resistant manner + """ if len(a) != len(b): return False rv = 0 diff -r 5e5c0cf930bb96bf8af0bd19daad1a999b40c374 -r 7a889340aba7e078d31963aa99139c7ddaea07c2 lib/galaxy/web/framework/middleware/remoteuser.py --- a/lib/galaxy/web/framework/middleware/remoteuser.py +++ b/lib/galaxy/web/framework/middleware/remoteuser.py @@ -3,6 +3,7 @@ """ import socket +from galaxy.util import safe_str_cmp errorpage = """ <!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN" "http://www.w3.org/TR/html4/loose.dtd"> @@ -37,12 +38,13 @@ class RemoteUser( object ): - def __init__( self, app, maildomain=None, display_servers=None, admin_users=None, remote_user_header=None ): + def __init__( self, app, maildomain=None, display_servers=None, admin_users=None, remote_user_header=None, remote_user_secret_header=None ): self.app = app self.maildomain = maildomain self.display_servers = display_servers or [] self.admin_users = admin_users or [] self.remote_user_header = remote_user_header or 'HTTP_REMOTE_USER' + self.config_secret_header = remote_user_secret_header def __call__( self, environ, start_response ): # Allow display servers @@ -59,6 +61,34 @@ # Rewrite* method for passing REMOTE_USER and a user is # un-authenticated. Any other possible values need to go here as well. path_info = environ.get('PATH_INFO', '') + + # If the secret header is enabled, we expect upstream to send along some key + # in HTTP_GX_SECRET, so we'll need to compare that here to the correct value + # + # This is not an ideal location for this function. The reason being + # that because this check is done BEFORE the REMOTE_USER check, it is + # possible to attack the GX_SECRET key without having correct + # credentials. However, that's why it's not "ideal", but it is "good + # enough". The only users able to exploit this are ones with access to + # the local system (unless Galaxy is listening on 0.0.0.0....). It + # seems improbable that an attacker with access to the server hosting + # Galaxy would not have acess to Galaxy itself, and be attempting to + # attack the system + if self.config_secret_header is not None: + if not safe_str_cmp(environ.get('HTTP_GX_SECRET'), self.config_secret_header): + title = "Access to Galaxy is denied" + message = """ + Galaxy is configured to authenticate users via an external + method (such as HTTP authentication in Apache), but an + incorrect shared secret key was provided by the + upstream (proxy) server.</p> + <p>Please contact your local Galaxy administrator. The + variable <code>remote_user_secret</code> and + <code>GX_SECRET</code> header must be set before you may + access Galaxy. + """ + return self.error( start_response, title, message ) + if not environ.get(self.remote_user_header, '(null)').startswith('(null)'): if not environ[ self.remote_user_header ].count( '@' ): if self.maildomain is not None: diff -r 5e5c0cf930bb96bf8af0bd19daad1a999b40c374 -r 7a889340aba7e078d31963aa99139c7ddaea07c2 lib/galaxy/webapps/galaxy/buildapp.py --- a/lib/galaxy/webapps/galaxy/buildapp.py +++ b/lib/galaxy/webapps/galaxy/buildapp.py @@ -503,8 +503,8 @@ app = RemoteUser( app, maildomain = conf.get( 'remote_user_maildomain', None ), display_servers = util.listify( conf.get( 'display_servers', '' ) ), admin_users = conf.get( 'admin_users', '' ).split( ',' ), - remote_user_header = conf.get( 'remote_user_header', 'HTTP_REMOTE_USER' ) ) - log.debug( "Enabling 'remote user' middleware" ) + remote_user_header = conf.get( 'remote_user_header', 'HTTP_REMOTE_USER' ), + remote_user_secret_header = conf.get('remote_user_secret', None) ) # The recursive middleware allows for including requests in other # requests or forwarding of requests, all on the server side. if asbool(conf.get('use_recursive', True)): diff -r 5e5c0cf930bb96bf8af0bd19daad1a999b40c374 -r 7a889340aba7e078d31963aa99139c7ddaea07c2 lib/galaxy/webapps/tool_shed/buildapp.py --- a/lib/galaxy/webapps/tool_shed/buildapp.py +++ b/lib/galaxy/webapps/tool_shed/buildapp.py @@ -157,7 +157,8 @@ from galaxy.webapps.tool_shed.framework.middleware.remoteuser import RemoteUser app = RemoteUser( app, maildomain = conf.get( 'remote_user_maildomain', None ), display_servers = util.listify( conf.get( 'display_servers', '' ) ), - admin_users = conf.get( 'admin_users', '' ).split( ',' ) ) + admin_users = conf.get( 'admin_users', '' ).split( ',' ), + remote_user_secret_header = conf.get('remote_user_secret', None) ) log.debug( "Enabling 'remote user' middleware" ) # The recursive middleware allows for including requests in other # requests or forwarding of requests, all on the server side. diff -r 5e5c0cf930bb96bf8af0bd19daad1a999b40c374 -r 7a889340aba7e078d31963aa99139c7ddaea07c2 lib/galaxy/webapps/tool_shed/config.py --- a/lib/galaxy/webapps/tool_shed/config.py +++ b/lib/galaxy/webapps/tool_shed/config.py @@ -83,6 +83,7 @@ self.remote_user_maildomain = kwargs.get( "remote_user_maildomain", None ) self.remote_user_header = kwargs.get( "remote_user_header", 'HTTP_REMOTE_USER' ) self.remote_user_logout_href = kwargs.get( "remote_user_logout_href", None ) + self.remote_user_secret = kwargs.get( "remote_user_secret", None ) self.require_login = string_as_bool( kwargs.get( "require_login", "False" ) ) self.allow_user_creation = string_as_bool( kwargs.get( "allow_user_creation", "True" ) ) self.allow_user_deletion = string_as_bool( kwargs.get( "allow_user_deletion", "False" ) ) diff -r 5e5c0cf930bb96bf8af0bd19daad1a999b40c374 -r 7a889340aba7e078d31963aa99139c7ddaea07c2 lib/galaxy/webapps/tool_shed/framework/middleware/remoteuser.py --- a/lib/galaxy/webapps/tool_shed/framework/middleware/remoteuser.py +++ b/lib/galaxy/webapps/tool_shed/framework/middleware/remoteuser.py @@ -3,6 +3,7 @@ """ import socket +from galaxy.util import safe_str_cmp errorpage = """ <!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN" "http://www.w3.org/TR/html4/loose.dtd"> @@ -36,11 +37,13 @@ """ class RemoteUser( object ): - def __init__( self, app, maildomain=None, display_servers=None, admin_users=None ): + def __init__( self, app, maildomain=None, display_servers=None, admin_users=None, remote_user_secret_header=None ): self.app = app self.maildomain = maildomain self.display_servers = display_servers or [] self.admin_users = admin_users or [] + self.config_secret_header = remote_user_secret_header + def __call__( self, environ, start_response ): environ[ 'webapp' ] = 'tool_shed' # Allow display servers @@ -53,6 +56,34 @@ if host in self.display_servers: environ[ 'HTTP_REMOTE_USER' ] = 'remote_display_server@%s' % ( self.maildomain or 'example.org' ) return self.app( environ, start_response ) + + # If the secret header is enabled, we expect upstream to send along some key + # in HTTP_GX_SECRET, so we'll need to compare that here to the correct value + # + # This is not an ideal location for this function. The reason being + # that because this check is done BEFORE the REMOTE_USER check, it is + # possible to attack the GX_SECRET key without having correct + # credentials. However, that's why it's not "ideal", but it is "good + # enough". The only users able to exploit this are ones with access to + # the local system (unless Galaxy is listening on 0.0.0.0....). It + # seems improbable that an attacker with access to the server hosting + # Galaxy would not have acess to Galaxy itself, and be attempting to + # attack the system + if self.config_secret_header is not None: + if not safe_str_cmp(environ.get('HTTP_GX_SECRET'), self.config_secret_header): + title = "Access to Galaxy is denied" + message = """ + Galaxy is configured to authenticate users via an external + method (such as HTTP authentication in Apache), but an + incorrect shared secret key was provided by the + upstream (proxy) server.</p> + <p>Please contact your local Galaxy administrator. The + variable <code>remote_user_secret</code> and + <code>GX_SECRET</code> header must be set before you may + access Galaxy. + """ + return self.error( start_response, title, message ) + # Apache sets REMOTE_USER to the string '(null)' when using the Rewrite* method for passing REMOTE_USER and a user is # un-authenticated. Any other possible values need to go here as well. path_info = environ.get('PATH_INFO', '') @@ -88,6 +119,7 @@ <p>Contact your local Galaxy tool shed administrator. """ return self.error( start_response, title, message ) + def error( self, start_response, title="Access denied", message="Contact your local Galaxy tool shed administrator." ): start_response( '403 Forbidden', [('Content-type', 'text/html')] ) return [errorpage % (title, message)] https://bitbucket.org/galaxy/galaxy-central/commits/b7deaa9fb8b7/ Changeset: b7deaa9fb8b7 User: erasche2 Date: 2014-12-18 19:43:00+00:00 Summary: Fixed typos in documentation Affected #: 2 files diff -r 7a889340aba7e078d31963aa99139c7ddaea07c2 -r b7deaa9fb8b7cadd141fd07b2c50654716c88720 lib/galaxy/web/framework/middleware/remoteuser.py --- a/lib/galaxy/web/framework/middleware/remoteuser.py +++ b/lib/galaxy/web/framework/middleware/remoteuser.py @@ -72,7 +72,7 @@ # enough". The only users able to exploit this are ones with access to # the local system (unless Galaxy is listening on 0.0.0.0....). It # seems improbable that an attacker with access to the server hosting - # Galaxy would not have acess to Galaxy itself, and be attempting to + # Galaxy would not have access to Galaxy itself, and be attempting to # attack the system if self.config_secret_header is not None: if not safe_str_cmp(environ.get('HTTP_GX_SECRET'), self.config_secret_header): diff -r 7a889340aba7e078d31963aa99139c7ddaea07c2 -r b7deaa9fb8b7cadd141fd07b2c50654716c88720 lib/galaxy/webapps/tool_shed/framework/middleware/remoteuser.py --- a/lib/galaxy/webapps/tool_shed/framework/middleware/remoteuser.py +++ b/lib/galaxy/webapps/tool_shed/framework/middleware/remoteuser.py @@ -67,7 +67,7 @@ # enough". The only users able to exploit this are ones with access to # the local system (unless Galaxy is listening on 0.0.0.0....). It # seems improbable that an attacker with access to the server hosting - # Galaxy would not have acess to Galaxy itself, and be attempting to + # Galaxy would not have access to Galaxy itself, and be attempting to # attack the system if self.config_secret_header is not None: if not safe_str_cmp(environ.get('HTTP_GX_SECRET'), self.config_secret_header): https://bitbucket.org/galaxy/galaxy-central/commits/ac1859e27eb0/ Changeset: ac1859e27eb0 User: jmchilton Date: 2015-01-01 19:16:50+00:00 Summary: Merged in erasche2/galaxy-central2 (pull request #619) Allow for shared secret between Galaxy and upstream proxies Affected #: 9 files diff -r 84d5a72790735ae4da03f27735c52182bb925d9b -r ac1859e27eb0217ae5dabd7495a3b9314f65a1f7 config/galaxy.ini.sample --- a/config/galaxy.ini.sample +++ b/config/galaxy.ini.sample @@ -713,6 +713,13 @@ # *not* include 'HTTP_' at the beginning of the header name. #remote_user_header = HTTP_REMOTE_USER +# If use_remote_user is enabled, anyone who can log in to the Galaxy host may +# impersonate any other user by simply sending the appropriate header. Thus a +# secret shared between the upstream proxy server, and Galaxy is required. +# If anyone other than the Galaxy user is using the server, then apache/nginx should +# pass a value in the header 'GX_SECRET' that is identical the one below +#remote_user_secret = USING THE DEFAULT IS NOT SECURE! + # If use_remote_user is enabled, you can set this to a URL that will log your # users out. #remote_user_logout_href = None diff -r 84d5a72790735ae4da03f27735c52182bb925d9b -r ac1859e27eb0217ae5dabd7495a3b9314f65a1f7 config/tool_shed.ini.sample --- a/config/tool_shed.ini.sample +++ b/config/tool_shed.ini.sample @@ -67,6 +67,13 @@ # https://wiki.galaxyproject.org/Admin/Config/ApacheProxy #use_remote_user = False +# If use_remote_user is enabled, anyone who can log in to the Galaxy host may +# impersonate any other user by simply sending the appropriate header. Thus a +# secret shared between the upstream proxy server, and Galaxy is required. +# If anyone other than the Galaxy user is using the server, then apache/nginx should +# pass a value in the header 'GX_SECRET' that is identical the one below +#remote_user_secret = changethisinproductiontoo + # Configuration for debugging middleware debug = true use_lint = false diff -r 84d5a72790735ae4da03f27735c52182bb925d9b -r ac1859e27eb0217ae5dabd7495a3b9314f65a1f7 lib/galaxy/config.py --- a/lib/galaxy/config.py +++ b/lib/galaxy/config.py @@ -133,6 +133,7 @@ self.remote_user_maildomain = kwargs.get( "remote_user_maildomain", None ) self.remote_user_header = kwargs.get( "remote_user_header", 'HTTP_REMOTE_USER' ) self.remote_user_logout_href = kwargs.get( "remote_user_logout_href", None ) + self.remote_user_secret = kwargs.get( "remote_user_secret", None ) self.require_login = string_as_bool( kwargs.get( "require_login", "False" ) ) self.allow_user_creation = string_as_bool( kwargs.get( "allow_user_creation", "True" ) ) self.allow_user_deletion = string_as_bool( kwargs.get( "allow_user_deletion", "False" ) ) diff -r 84d5a72790735ae4da03f27735c52182bb925d9b -r ac1859e27eb0217ae5dabd7495a3b9314f65a1f7 lib/galaxy/util/__init__.py --- a/lib/galaxy/util/__init__.py +++ b/lib/galaxy/util/__init__.py @@ -1119,6 +1119,8 @@ def safe_str_cmp(a, b): + """safely compare two strings in a timing-attack-resistant manner + """ if len(a) != len(b): return False rv = 0 diff -r 84d5a72790735ae4da03f27735c52182bb925d9b -r ac1859e27eb0217ae5dabd7495a3b9314f65a1f7 lib/galaxy/web/framework/middleware/remoteuser.py --- a/lib/galaxy/web/framework/middleware/remoteuser.py +++ b/lib/galaxy/web/framework/middleware/remoteuser.py @@ -3,6 +3,7 @@ """ import socket +from galaxy.util import safe_str_cmp errorpage = """ <!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN" "http://www.w3.org/TR/html4/loose.dtd"> @@ -37,12 +38,13 @@ class RemoteUser( object ): - def __init__( self, app, maildomain=None, display_servers=None, admin_users=None, remote_user_header=None ): + def __init__( self, app, maildomain=None, display_servers=None, admin_users=None, remote_user_header=None, remote_user_secret_header=None ): self.app = app self.maildomain = maildomain self.display_servers = display_servers or [] self.admin_users = admin_users or [] self.remote_user_header = remote_user_header or 'HTTP_REMOTE_USER' + self.config_secret_header = remote_user_secret_header def __call__( self, environ, start_response ): # Allow display servers @@ -59,6 +61,34 @@ # Rewrite* method for passing REMOTE_USER and a user is # un-authenticated. Any other possible values need to go here as well. path_info = environ.get('PATH_INFO', '') + + # If the secret header is enabled, we expect upstream to send along some key + # in HTTP_GX_SECRET, so we'll need to compare that here to the correct value + # + # This is not an ideal location for this function. The reason being + # that because this check is done BEFORE the REMOTE_USER check, it is + # possible to attack the GX_SECRET key without having correct + # credentials. However, that's why it's not "ideal", but it is "good + # enough". The only users able to exploit this are ones with access to + # the local system (unless Galaxy is listening on 0.0.0.0....). It + # seems improbable that an attacker with access to the server hosting + # Galaxy would not have access to Galaxy itself, and be attempting to + # attack the system + if self.config_secret_header is not None: + if not safe_str_cmp(environ.get('HTTP_GX_SECRET'), self.config_secret_header): + title = "Access to Galaxy is denied" + message = """ + Galaxy is configured to authenticate users via an external + method (such as HTTP authentication in Apache), but an + incorrect shared secret key was provided by the + upstream (proxy) server.</p> + <p>Please contact your local Galaxy administrator. The + variable <code>remote_user_secret</code> and + <code>GX_SECRET</code> header must be set before you may + access Galaxy. + """ + return self.error( start_response, title, message ) + if not environ.get(self.remote_user_header, '(null)').startswith('(null)'): if not environ[ self.remote_user_header ].count( '@' ): if self.maildomain is not None: diff -r 84d5a72790735ae4da03f27735c52182bb925d9b -r ac1859e27eb0217ae5dabd7495a3b9314f65a1f7 lib/galaxy/webapps/galaxy/buildapp.py --- a/lib/galaxy/webapps/galaxy/buildapp.py +++ b/lib/galaxy/webapps/galaxy/buildapp.py @@ -503,8 +503,8 @@ app = RemoteUser( app, maildomain = conf.get( 'remote_user_maildomain', None ), display_servers = util.listify( conf.get( 'display_servers', '' ) ), admin_users = conf.get( 'admin_users', '' ).split( ',' ), - remote_user_header = conf.get( 'remote_user_header', 'HTTP_REMOTE_USER' ) ) - log.debug( "Enabling 'remote user' middleware" ) + remote_user_header = conf.get( 'remote_user_header', 'HTTP_REMOTE_USER' ), + remote_user_secret_header = conf.get('remote_user_secret', None) ) # The recursive middleware allows for including requests in other # requests or forwarding of requests, all on the server side. if asbool(conf.get('use_recursive', True)): diff -r 84d5a72790735ae4da03f27735c52182bb925d9b -r ac1859e27eb0217ae5dabd7495a3b9314f65a1f7 lib/galaxy/webapps/tool_shed/buildapp.py --- a/lib/galaxy/webapps/tool_shed/buildapp.py +++ b/lib/galaxy/webapps/tool_shed/buildapp.py @@ -157,7 +157,8 @@ from galaxy.webapps.tool_shed.framework.middleware.remoteuser import RemoteUser app = RemoteUser( app, maildomain = conf.get( 'remote_user_maildomain', None ), display_servers = util.listify( conf.get( 'display_servers', '' ) ), - admin_users = conf.get( 'admin_users', '' ).split( ',' ) ) + admin_users = conf.get( 'admin_users', '' ).split( ',' ), + remote_user_secret_header = conf.get('remote_user_secret', None) ) log.debug( "Enabling 'remote user' middleware" ) # The recursive middleware allows for including requests in other # requests or forwarding of requests, all on the server side. diff -r 84d5a72790735ae4da03f27735c52182bb925d9b -r ac1859e27eb0217ae5dabd7495a3b9314f65a1f7 lib/galaxy/webapps/tool_shed/config.py --- a/lib/galaxy/webapps/tool_shed/config.py +++ b/lib/galaxy/webapps/tool_shed/config.py @@ -73,6 +73,7 @@ self.remote_user_maildomain = kwargs.get( "remote_user_maildomain", None ) self.remote_user_header = kwargs.get( "remote_user_header", 'HTTP_REMOTE_USER' ) self.remote_user_logout_href = kwargs.get( "remote_user_logout_href", None ) + self.remote_user_secret = kwargs.get( "remote_user_secret", None ) self.require_login = string_as_bool( kwargs.get( "require_login", "False" ) ) self.allow_user_creation = string_as_bool( kwargs.get( "allow_user_creation", "True" ) ) self.allow_user_deletion = string_as_bool( kwargs.get( "allow_user_deletion", "False" ) ) diff -r 84d5a72790735ae4da03f27735c52182bb925d9b -r ac1859e27eb0217ae5dabd7495a3b9314f65a1f7 lib/galaxy/webapps/tool_shed/framework/middleware/remoteuser.py --- a/lib/galaxy/webapps/tool_shed/framework/middleware/remoteuser.py +++ b/lib/galaxy/webapps/tool_shed/framework/middleware/remoteuser.py @@ -3,6 +3,7 @@ """ import socket +from galaxy.util import safe_str_cmp errorpage = """ <!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN" "http://www.w3.org/TR/html4/loose.dtd"> @@ -36,11 +37,13 @@ """ class RemoteUser( object ): - def __init__( self, app, maildomain=None, display_servers=None, admin_users=None ): + def __init__( self, app, maildomain=None, display_servers=None, admin_users=None, remote_user_secret_header=None ): self.app = app self.maildomain = maildomain self.display_servers = display_servers or [] self.admin_users = admin_users or [] + self.config_secret_header = remote_user_secret_header + def __call__( self, environ, start_response ): environ[ 'webapp' ] = 'tool_shed' # Allow display servers @@ -53,6 +56,34 @@ if host in self.display_servers: environ[ 'HTTP_REMOTE_USER' ] = 'remote_display_server@%s' % ( self.maildomain or 'example.org' ) return self.app( environ, start_response ) + + # If the secret header is enabled, we expect upstream to send along some key + # in HTTP_GX_SECRET, so we'll need to compare that here to the correct value + # + # This is not an ideal location for this function. The reason being + # that because this check is done BEFORE the REMOTE_USER check, it is + # possible to attack the GX_SECRET key without having correct + # credentials. However, that's why it's not "ideal", but it is "good + # enough". The only users able to exploit this are ones with access to + # the local system (unless Galaxy is listening on 0.0.0.0....). It + # seems improbable that an attacker with access to the server hosting + # Galaxy would not have access to Galaxy itself, and be attempting to + # attack the system + if self.config_secret_header is not None: + if not safe_str_cmp(environ.get('HTTP_GX_SECRET'), self.config_secret_header): + title = "Access to Galaxy is denied" + message = """ + Galaxy is configured to authenticate users via an external + method (such as HTTP authentication in Apache), but an + incorrect shared secret key was provided by the + upstream (proxy) server.</p> + <p>Please contact your local Galaxy administrator. The + variable <code>remote_user_secret</code> and + <code>GX_SECRET</code> header must be set before you may + access Galaxy. + """ + return self.error( start_response, title, message ) + # Apache sets REMOTE_USER to the string '(null)' when using the Rewrite* method for passing REMOTE_USER and a user is # un-authenticated. Any other possible values need to go here as well. path_info = environ.get('PATH_INFO', '') @@ -88,6 +119,7 @@ <p>Contact your local Galaxy tool shed administrator. """ return self.error( start_response, title, message ) + def error( self, start_response, title="Access denied", message="Contact your local Galaxy tool shed administrator." ): start_response( '403 Forbidden', [('Content-type', 'text/html')] ) return [errorpage % (title, message)] Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: 4 new changesets
by commits-noreply＠bitbucket.org 01 Jan '15

01 Jan '15

4 new commits in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/a8cf78bd3629/ Changeset: a8cf78bd3629 User: jmchilton Date: 2015-01-01 15:02:23+00:00 Summary: PEP-8 fix for lib/tool_shed/tools/tool_version_manager.py. Affected #: 1 file diff -r bb548455022c7ad21c74ba61b4fe37b606737db7 -r a8cf78bd3629e4d9fa28e2641b670efb78ba2a56 lib/tool_shed/tools/tool_version_manager.py --- a/lib/tool_shed/tools/tool_version_manager.py +++ b/lib/tool_shed/tools/tool_version_manager.py @@ -1,5 +1,4 @@ import logging -import os from galaxy.model.orm import and_ https://bitbucket.org/galaxy/galaxy-central/commits/81a99e6f570e/ Changeset: 81a99e6f570e User: jmchilton Date: 2015-01-01 15:02:23+00:00 Summary: Unit test to verify tool panel handles multiple successive TS install... In particular that the integrated panel reflects all the installed tools but that they are groupped correctly in ToolBox._tool_panel. Affected #: 3 files diff -r a8cf78bd3629e4d9fa28e2641b670efb78ba2a56 -r 81a99e6f570ef0b0cb6511bfd9ee4dd767490fef test/unit/tool_shed_unit_tests/test_tool_panel_manager.py --- a/test/unit/tool_shed_unit_tests/test_tool_panel_manager.py +++ b/test/unit/tool_shed_unit_tests/test_tool_panel_manager.py @@ -5,6 +5,10 @@ from tools.test_toolbox import BaseToolBoxTestCase from tool_shed.galaxy_install.tools import tool_panel_manager +from tool_shed.tools import tool_version_manager + +DEFAULT_GUID = "123456" + class ToolPanelManagerTestCase( BaseToolBoxTestCase ): @@ -34,15 +38,14 @@ assert len( toolbox._tool_panel ) == 2 def test_add_tool_to_panel( self ): - self._init_tool() - self._add_config( """<toolbox tool_path="%s"></toolbox>""" % self.test_directory ) - tool_path = os.path.join(self.test_directory, "tool.xml") - self.tool.guid = "123456" - new_tools = [{"guid": "123456", "tool_config": tool_path}] + self._init_ts_tool( guid=DEFAULT_GUID ) + self._init_dynamic_tool_conf() + tool_path = self._tool_path() + new_tools = [{"guid": DEFAULT_GUID, "tool_config": tool_path}] repository_tools_tups = [ ( tool_path, - "123456", + DEFAULT_GUID, self.tool, ) ] @@ -63,6 +66,51 @@ ) self._verify_tool_confs() + def test_add_twice( self ): + self._init_dynamic_tool_conf() + tool_versions = {} + previous_guid = None + for v in "1", "2", "3": + changeset = "0123456789abcde%s" % v + guid = DEFAULT_GUID + ("v%s" % v) + tool = self._init_ts_tool( guid=guid, filename="tool_v%s.xml" % v ) + tool_path = self._tool_path( name="tool_v%s.xml" % v ) + new_tools = [{"guid": guid, "tool_config": tool_path}] + tool_shed_repository = self._repo_install( changeset ) + repository_tools_tups = [ + ( + tool_path, + guid, + tool, + ) + ] + _, section = self.toolbox.get_section("tid1", create_if_needed=True) + tpm = self.tpm + tool_panel_dict = tpm.generate_tool_panel_dict_for_new_install( + tool_dicts=new_tools, + tool_section=section, + ) + if previous_guid: + tool_versions[ guid ] = previous_guid + self.tvm.handle_tool_versions( [tool_versions], tool_shed_repository ) + tpm.add_to_tool_panel( + repository_name="example", + repository_clone_url="github.com", + changeset_revision=changeset, + repository_tools_tups=repository_tools_tups, + owner="galaxyproject", + shed_tool_conf="tool_conf.xml", + tool_panel_dict=tool_panel_dict, + ) + self._verify_tool_confs() + section = self.toolbox._tool_panel["tid1"] + # New GUID replaced old one in tool panel but both + # appear in integrated tool panel. + if previous_guid: + assert ("tool_%s" % previous_guid) not in section.panel_items() + assert ("tool_%s" % guid) in self.toolbox._integrated_tool_panel["tid1"].panel_items() + previous_guid = guid + def test_deactivate_in_section( self ): self._setup_two_versions_remove_one( section=True, uninstall=False ) self._verify_version_2_removed_from_panel( ) @@ -154,6 +202,19 @@ message = message_template % ( filename, open( filename, "r" ).read() ) raise AssertionError( message ) + def _init_dynamic_tool_conf( self ): + # Add a dynamic tool conf (such as a ToolShed managed one) to list of configs. + self._add_config( """<toolbox tool_path="%s"></toolbox>""" % self.test_directory ) + + def _init_ts_tool( self, guid=DEFAULT_GUID, **kwds ): + tool = self._init_tool( **kwds ) + tool.guid = guid + return tool + @property def tpm( self ): return tool_panel_manager.ToolPanelManager( self.app ) + + @property + def tvm( self ): + return tool_version_manager.ToolVersionManager( self.app ) diff -r a8cf78bd3629e4d9fa28e2641b670efb78ba2a56 -r 81a99e6f570ef0b0cb6511bfd9ee4dd767490fef test/unit/tools/test_toolbox.py --- a/test/unit/tools/test_toolbox.py +++ b/test/unit/tools/test_toolbox.py @@ -123,6 +123,10 @@ path = os.path.join( self.test_directory, name ) return path + def _tool_path( self, name="tool.xml" ): + path = os.path.join( self.test_directory, name ) + return path + def __reindex( self ): self.reindexed = True diff -r a8cf78bd3629e4d9fa28e2641b670efb78ba2a56 -r 81a99e6f570ef0b0cb6511bfd9ee4dd767490fef test/unit/tools_support.py --- a/test/unit/tools_support.py +++ b/test/unit/tools_support.py @@ -72,7 +72,7 @@ contents_template = string.Template( tool_contents ) tool_contents = contents_template.safe_substitute( dict( version=version ) ) self.__write_tool( tool_contents ) - self.__setup_tool( ) + return self.__setup_tool( ) def _init_app_for_tools( self ): self.app.config.drmaa_external_runjob_script = "" @@ -85,6 +85,7 @@ self.tool = Tool( self.tool_file, tool_source, self.app ) if getattr( self, "tool_action", None ): self.tool.tool_action = self.tool_action + return self.tool def __write_tool( self, contents ): open( self.tool_file, "w" ).write( contents ) https://bitbucket.org/galaxy/galaxy-central/commits/d3484078d199/ Changeset: d3484078d199 User: jmchilton Date: 2015-01-01 15:02:23+00:00 Summary: Tweak InstallManger handle_repository_contents to clarify... ... that this method is only used within this class. Affected #: 1 file diff -r 81a99e6f570ef0b0cb6511bfd9ee4dd767490fef -r d3484078d199cedc593d64e651a7c8c2d51d5f75 lib/tool_shed/galaxy_install/install_manager.py --- a/lib/tool_shed/galaxy_install/install_manager.py +++ b/lib/tool_shed/galaxy_install/install_manager.py @@ -501,8 +501,8 @@ repo_info_dicts = [ repo_info_dict ] return repository_revision_dict, repo_info_dicts - def handle_repository_contents( self, tool_shed_repository, tool_path, repository_clone_url, relative_install_dir, - tool_shed=None, tool_section=None, shed_tool_conf=None, reinstalling=False ): + def __handle_repository_contents( self, tool_shed_repository, tool_path, repository_clone_url, relative_install_dir, + tool_shed=None, tool_section=None, shed_tool_conf=None, reinstalling=False ): """ Generate the metadata for the installed tool shed repository, among other things. This method is called when an administrator is installing a new repository or @@ -839,14 +839,14 @@ create=False ) hg_util.pull_repository( repo, repository_clone_url, current_changeset_revision ) hg_util.update_repository( repo, ctx_rev=current_ctx_rev ) - self.handle_repository_contents( tool_shed_repository=tool_shed_repository, - tool_path=tool_path, - repository_clone_url=repository_clone_url, - relative_install_dir=relative_install_dir, - tool_shed=tool_shed_repository.tool_shed, - tool_section=tool_section, - shed_tool_conf=shed_tool_conf, - reinstalling=reinstalling ) + self.__handle_repository_contents( tool_shed_repository=tool_shed_repository, + tool_path=tool_path, + repository_clone_url=repository_clone_url, + relative_install_dir=relative_install_dir, + tool_shed=tool_shed_repository.tool_shed, + tool_section=tool_section, + shed_tool_conf=shed_tool_conf, + reinstalling=reinstalling ) self.install_model.context.refresh( tool_shed_repository ) metadata = tool_shed_repository.metadata if 'tools' in metadata: https://bitbucket.org/galaxy/galaxy-central/commits/84d5a7279073/ Changeset: 84d5a7279073 User: jmchilton Date: 2015-01-01 15:02:23+00:00 Summary: Bugfix: Populte tool lineage stuff prior to install. During tool shed tool installs populate lineage information in tool shed install database prior to adding tool to the tool panel so that the lineage information may be used to group tools. Previously I believe these tools would only be correctly groupped after restarting Galaxy. Affected #: 1 file diff -r d3484078d199cedc593d64e651a7c8c2d51d5f75 -r 84d5a72790735ae4da03f27735c52182bb925d9b lib/tool_shed/galaxy_install/install_manager.py --- a/lib/tool_shed/galaxy_install/install_manager.py +++ b/lib/tool_shed/galaxy_install/install_manager.py @@ -40,6 +40,8 @@ log = logging.getLogger( __name__ ) +FAILED_TO_FETCH_VERSIONS = object() + class InstallToolDependencyManager( object ): @@ -502,7 +504,8 @@ return repository_revision_dict, repo_info_dicts def __handle_repository_contents( self, tool_shed_repository, tool_path, repository_clone_url, relative_install_dir, - tool_shed=None, tool_section=None, shed_tool_conf=None, reinstalling=False ): + tool_shed=None, tool_section=None, shed_tool_conf=None, reinstalling=False, + tool_versions_response=None ): """ Generate the metadata for the installed tool shed repository, among other things. This method is called when an administrator is installing a new repository or @@ -530,6 +533,10 @@ tool_shed_repository.tool_shed_status = tool_shed_status_dict self.install_model.context.add( tool_shed_repository ) self.install_model.context.flush() + if tool_versions_response and tool_versions_response is not FAILED_TO_FETCH_VERSIONS: + tool_version_dicts = tool_versions_response + tvm = tool_version_manager.ToolVersionManager( self.app ) + tvm.handle_tool_versions( tool_version_dicts, tool_shed_repository ) if 'tool_dependencies' in irmm_metadata_dict and not reinstalling: tool_dependency_util.create_tool_dependency_objects( self.app, tool_shed_repository, @@ -839,6 +846,7 @@ create=False ) hg_util.pull_repository( repo, repository_clone_url, current_changeset_revision ) hg_util.update_repository( repo, ctx_rev=current_ctx_rev ) + tool_versions_response = fetch_tool_versions( self.app, tool_shed_repository ) self.__handle_repository_contents( tool_shed_repository=tool_shed_repository, tool_path=tool_path, repository_clone_url=repository_clone_url, @@ -846,6 +854,7 @@ tool_shed=tool_shed_repository.tool_shed, tool_section=tool_section, shed_tool_conf=shed_tool_conf, + tool_versions_response=tool_versions_response, reinstalling=reinstalling ) self.install_model.context.refresh( tool_shed_repository ) metadata = tool_shed_repository.metadata @@ -853,18 +862,7 @@ # Get the tool_versions from the tool shed for each tool in the installed change set. self.update_tool_shed_repository_status( tool_shed_repository, self.install_model.ToolShedRepository.installation_status.SETTING_TOOL_VERSIONS ) - tool_shed_url = common_util.get_tool_shed_url_from_tool_shed_registry( self.app, str( tool_shed_repository.tool_shed ) ) - params = '?name=%s&owner=%s&changeset_revision=%s' % ( str( tool_shed_repository.name ), - str( tool_shed_repository.owner ), - str( tool_shed_repository.changeset_revision ) ) - url = common_util.url_join( tool_shed_url, - '/repository/get_tool_versions%s' % params ) - text = common_util.tool_shed_get( self.app, tool_shed_url, url ) - if text: - tool_version_dicts = json.loads( text ) - tvm = tool_version_manager.ToolVersionManager( self.app ) - tvm.handle_tool_versions( tool_version_dicts, tool_shed_repository ) - else: + if tool_versions_response is FAILED_TO_FETCH_VERSIONS: if not error_message: error_message = "" error_message += "Version information for the tools included in the <b>%s</b> repository is missing. " % tool_shed_repository.name @@ -958,3 +956,27 @@ tool_shed_repository.error_message = str( error_message ) self.install_model.context.add( tool_shed_repository ) self.install_model.context.flush() + + +def fetch_tool_versions( app, tool_shed_repository ): + """ Fetch a data structure describing tool shed versions from the tool shed + corresponding to a tool_shed_repository object. + """ + failed_to_fetch = False + try: + tool_shed_url = common_util.get_tool_shed_url_from_tool_shed_registry( app, str( tool_shed_repository.tool_shed ) ) + params = '?name=%s&owner=%s&changeset_revision=%s' % ( str( tool_shed_repository.name ), + str( tool_shed_repository.owner ), + str( tool_shed_repository.changeset_revision ) ) + url = common_util.url_join( tool_shed_url, + '/repository/get_tool_versions%s' % params ) + text = common_util.tool_shed_get( app, tool_shed_url, url ) + if text: + return json.loads( text ) + else: + failed_to_fetch = True + except Exception: + failed_to_fetch = True + if failed_to_fetch: + log.exception("Failed to fetch tool shed repository verion information.") + return FAILED_TO_FETCH_VERSIONS Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: 9 new changesets
by commits-noreply＠bitbucket.org 31 Dec '14

31 Dec '14

9 new commits in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/08f8850853d0/ Changeset: 08f8850853d0 User: jmchilton Date: 2014-12-31 23:21:10+00:00 Summary: Allow multiple tools with the same id in ToolBox. How to use: 1.) Place multiple tools with different IDs in your tool conf. 2.) ... ummm ... no step 2 - just use the tools. Implementation: The Tool Shed allows tool lineages by assigning each tool version a GUID and tracking versions in a database. This implementation works by simply allowing the ToolBox to contain multiple tools with the same ID and orders them by the version specified by the tool author. To track enable this a second tool lineage has been introduced that just uses tool versions instead of a database (non-toolshed installed tools are not longer placed into the Tool Shed install database). The ToolBox has been updated to allow multiple versions per tool id (defaulting to the 'latest' version for all operations which do not specify a version). Both jobs and workflow steps would track tool versions but did not use that version when fetching tools from the Toolbox - these components have been updated to try to use the tool version. Unit tests working through most of the ToolBox and tool panel have been added, as well as functional tests exercising the tools API and to ensure workflows now at least attempt to respect tool versions (still kind of silently switches versions in some cases). Manual tests against the new tool form seem to demonstrate the tool switching and tool re-running work with only minor changes to the tools API and the job handler. Affected #: 17 files diff -r 02efb1123d469986a051e7a8015c9d56635efd8a -r 08f8850853d004bf8a456a147436a6694d0a1a11 lib/galaxy/jobs/__init__.py --- a/lib/galaxy/jobs/__init__.py +++ b/lib/galaxy/jobs/__init__.py @@ -711,7 +711,7 @@ self.job_id = job.id self.session_id = job.session_id self.user_id = job.user_id - self.tool = queue.app.toolbox.get_tool( job.tool_id, exact=True ) + self.tool = queue.app.toolbox.get_tool( job.tool_id, job.tool_version, exact=True ) self.queue = queue self.app = queue.app self.sa_session = self.app.model.context diff -r 02efb1123d469986a051e7a8015c9d56635efd8a -r 08f8850853d004bf8a456a147436a6694d0a1a11 lib/galaxy/jobs/handler.py --- a/lib/galaxy/jobs/handler.py +++ b/lib/galaxy/jobs/handler.py @@ -119,7 +119,7 @@ & ( model.Job.handler == self.app.config.server_name ) ).all() for job in jobs_at_startup: - if not self.app.toolbox.has_tool( job.tool_id, exact=True ): + if not self.app.toolbox.has_tool( job.tool_id, job.tool_version, exact=True ): log.warning( "(%s) Tool '%s' removed from tool config, unable to recover job" % ( job.id, job.tool_id ) ) self.job_wrapper( job ).fail( 'This tool was disabled before the job completed. Please contact your Galaxy administrator.' ) elif job.job_runner_name is not None and job.job_runner_external_id is None: diff -r 02efb1123d469986a051e7a8015c9d56635efd8a -r 08f8850853d004bf8a456a147436a6694d0a1a11 lib/galaxy/tools/__init__.py --- a/lib/galaxy/tools/__init__.py +++ b/lib/galaxy/tools/__init__.py @@ -126,6 +126,10 @@ # shed_tool_conf.xml file. self._dynamic_tool_confs = [] self._tools_by_id = {} + # Tool lineages can contain chains of related tools with different ids + # so each will be present once in the above dictionary. The following + # dictionary can instead hold multiple tools with different versions. + self._tool_versions_by_id = {} self._workflows_by_id = {} # In-memory dictionary that defines the layout of the tool panel. self._tool_panel = odict() @@ -352,7 +356,8 @@ inserted = True if not inserted: # Check the tool's installed versions. - for lineage_id in tool.lineage_ids: + for tool_lineage_version in tool.lineage.get_versions(): + lineage_id = tool_lineage_version.id lineage_id_key = 'tool_%s' % lineage_id for index, integrated_panel_key in enumerate( self._integrated_tool_panel.keys() ): if lineage_id_key == integrated_panel_key: @@ -503,20 +508,26 @@ def get_tool( self, tool_id, tool_version=None, get_all_versions=False, exact=False ): """Attempt to locate a tool in the tool box.""" + if tool_version: + tool_version = str( tool_version ) + if get_all_versions and exact: raise AssertionError("Cannot specify get_tool with both get_all_versions and exact as True") if tool_id in self._tools_by_id and not get_all_versions: + if tool_version and tool_version in self._tool_versions_by_id[ tool_id ]: + return self._tool_versions_by_id[ tool_id ][ tool_version ] #tool_id exactly matches an available tool by id (which is 'old' tool_id or guid) return self._tools_by_id[ tool_id ] #exact tool id match not found, or all versions requested, search for other options, e.g. migrated tools or different versions rval = [] tool_lineage = self._lineage_map.get( tool_id ) if tool_lineage: - tool_version_ids = tool_lineage.get_version_ids( ) - for tool_version_id in tool_version_ids: - if tool_version_id in self._tools_by_id: - rval.append( self._tools_by_id[ tool_version_id ] ) + lineage_tool_versions = tool_lineage.get_versions( ) + for lineage_tool_version in lineage_tool_versions: + lineage_tool = self._tool_from_lineage_version( lineage_tool_version ) + if lineage_tool: + rval.append( lineage_tool ) if not rval: #still no tool, do a deeper search and try to match by old ids for tool in self._tools_by_id.itervalues(): @@ -561,11 +572,12 @@ """Get all loaded tools associated by lineage to the tool whose id is tool_id.""" tool_lineage = self._lineage_map.get( tool_id ) if tool_lineage: - tool_version_ids = tool_lineage.get_version_ids( ) + lineage_tool_versions = tool_lineage.get_versions( ) available_tool_versions = [] - for tool_version_id in tool_version_ids: - if tool_version_id in self._tools_by_id: - available_tool_versions.append( self._tools_by_id[ tool_version_id ] ) + for lineage_tool_version in lineage_tool_versions: + tool = self._tool_from_lineage_version( lineage_tool_version ) + if tool: + available_tool_versions.append( tool ) return available_tool_versions else: if tool_id in self._tools_by_id: @@ -746,7 +758,6 @@ tool_lineage = self._lineage_map.register( tool, tool_shed_repository=tool_shed_repository ) # Load the tool's lineage ids. tool.lineage = tool_lineage - tool.lineage_ids = tool_lineage.get_version_ids( ) self._tool_tag_manager.handle_tags( tool.id, elem ) self.__add_tool( tool, load_panel_dict, panel_dict ) # Always load the tool into the integrated_panel_dict, or it will not be included in the integrated_tool_panel.xml file. @@ -925,7 +936,20 @@ return tool def register_tool( self, tool ): - self._tools_by_id[ tool.id ] = tool + tool_id = tool.id + version = tool.version or None + if tool_id not in self._tool_versions_by_id: + self._tool_versions_by_id[ tool_id ] = { version: tool } + else: + self._tool_versions_by_id[ tool_id ][ version ] = tool + if tool_id in self._tools_by_id: + related_tool = self._tools_by_id[ tool_id ] + # This one becomes the default un-versioned tool + # if newer. + if self._newer_tool( tool, related_tool ): + self._tools_by_id[ tool_id ] = tool + else: + self._tools_by_id[ tool_id ] = tool def package_tool( self, trans, tool_id ): """ @@ -1202,9 +1226,11 @@ if not hasattr( tool, "lineage" ): return None tool_lineage = tool.lineage - lineage_ids = tool_lineage.get_version_ids( reverse=True ) - for lineage_id in lineage_ids: - if lineage_id in self._tools_by_id: + lineage_tool_versions = tool_lineage.get_versions( reverse=True ) + for lineage_tool_version in lineage_tool_versions: + lineage_tool = self._tool_from_lineage_version( lineage_tool_version ) + if lineage_tool: + lineage_id = lineage_tool.id loaded_version_key = 'tool_%s' % lineage_id if loaded_version_key in panel_dict: return panel_dict[ loaded_version_key ] @@ -1214,7 +1240,22 @@ """ Return True if tool1 is considered "newer" given its own lineage description. """ - return tool1.lineage_ids.index( tool1.id ) > tool1.lineage_ids.index( tool2.id ) + if not hasattr( tool1, "lineage" ): + return True + lineage_tool_versions = tool1.lineage.get_versions() + for lineage_tool_version in lineage_tool_versions: + lineage_tool = self._tool_from_lineage_version( lineage_tool_version ) + if lineage_tool is tool1: + return False + if lineage_tool is tool2: + return True + return True + + def _tool_from_lineage_version( self, lineage_tool_version ): + if lineage_tool_version.id_based: + return self._tools_by_id.get( lineage_tool_version.id, None ) + else: + return self._tool_versions_by_id.get( lineage_tool_version.id, {} ).get( lineage_tool_version.version, None ) def _filter_for_panel( item, filters, context ): diff -r 02efb1123d469986a051e7a8015c9d56635efd8a -r 08f8850853d004bf8a456a147436a6694d0a1a11 lib/galaxy/tools/toolbox/lineages/factory.py --- a/lib/galaxy/tools/toolbox/lineages/factory.py +++ b/lib/galaxy/tools/toolbox/lineages/factory.py @@ -1,4 +1,5 @@ from .tool_shed import ToolShedLineage +from .stock import StockLineage class LineageMap(object): @@ -13,7 +14,10 @@ tool_id = tool.id if tool_id not in self.lineage_map: tool_shed_repository = kwds.get("tool_shed_repository", None) - lineage = ToolShedLineage.from_tool(self.app, tool, tool_shed_repository) + if tool_shed_repository: + lineage = ToolShedLineage.from_tool(self.app, tool, tool_shed_repository) + else: + lineage = StockLineage.from_tool( tool ) self.lineage_map[tool_id] = lineage return self.lineage_map[tool_id] diff -r 02efb1123d469986a051e7a8015c9d56635efd8a -r 08f8850853d004bf8a456a147436a6694d0a1a11 lib/galaxy/tools/toolbox/lineages/interface.py --- a/lib/galaxy/tools/toolbox/lineages/interface.py +++ b/lib/galaxy/tools/toolbox/lineages/interface.py @@ -8,7 +8,35 @@ __metaclass__ = ABCMeta @abstractmethod - def get_version_ids( self, reverse=False ): - """ Return an ordered list of lineages in this chain, from - oldest to newest. + def get_versions( self, reverse=False ): + """ Return an ordered list of lineages (ToolLineageVersion) in this + chain, from oldest to newest. """ + + +class ToolLineageVersion(object): + """ Represents a single tool in a lineage. If lineage is based + around GUIDs that somehow encode the version (either using GUID + or a simple tool id and a version). """ + + def __init__(self, id, version): + self.id = id + self.version = version + + @staticmethod + def from_id_and_verion( id, version ): + assert version is not None + return ToolLineageVersion( id, version ) + + @staticmethod + def from_guid( guid ): + return ToolLineageVersion( guid, None ) + + @property + def id_based( self ): + """ Return True if the lineage is defined by GUIDs (in this + case the indexer of the tools (i.e. the ToolBox) should ignore + the tool_version (because it is encoded in the GUID and managed + externally). + """ + return self.version is None diff -r 02efb1123d469986a051e7a8015c9d56635efd8a -r 08f8850853d004bf8a456a147436a6694d0a1a11 lib/galaxy/tools/toolbox/lineages/stock.py --- /dev/null +++ b/lib/galaxy/tools/toolbox/lineages/stock.py @@ -0,0 +1,44 @@ +import threading + +from distutils.version import LooseVersion + +from .interface import ToolLineage +from .interface import ToolLineageVersion + + +class StockLineage(ToolLineage): + """ Simple tool's loaded directly from file system with lineage + determined solely by distutil's LooseVersion naming scheme. + """ + lineages_by_id = {} + lock = threading.Lock() + + def __init__(self, tool_id, **kwds): + self.tool_id = tool_id + self.tool_versions = set() + + @staticmethod + def from_tool( tool ): + tool_id = tool.id + lineages_by_id = StockLineage.lineages_by_id + with StockLineage.lock: + if tool_id not in lineages_by_id: + lineages_by_id[ tool_id ] = StockLineage( tool_id ) + lineage = lineages_by_id[ tool_id ] + lineage.register_version( tool.version ) + return lineage + + def register_version( self, tool_version ): + assert tool_version is not None + self.tool_versions.add( tool_version ) + + def get_versions( self, reverse=False ): + versions = [ ToolLineageVersion( self.tool_id, v ) for v in self.tool_versions ] + # Sort using LooseVersion which defines an appropriate __cmp__ + # method for comparing tool versions. + return sorted( versions, key=_to_loose_version ) + + +def _to_loose_version( tool_lineage_version ): + version = str( tool_lineage_version.version ) + return LooseVersion( version ) diff -r 02efb1123d469986a051e7a8015c9d56635efd8a -r 08f8850853d004bf8a456a147436a6694d0a1a11 lib/galaxy/tools/toolbox/lineages/tool_shed.py --- a/lib/galaxy/tools/toolbox/lineages/tool_shed.py +++ b/lib/galaxy/tools/toolbox/lineages/tool_shed.py @@ -1,4 +1,5 @@ from .interface import ToolLineage +from .interface import ToolLineageVersion from galaxy.model.tool_shed_install import ToolVersion @@ -32,6 +33,9 @@ tool_version = self.app.install_model.context.query( ToolVersion ).get( self.tool_version_id ) return tool_version.get_version_ids( self.app, reverse=reverse ) + def get_versions( self, reverse=False ): + return map( ToolLineageVersion.from_guid, self.get_version_ids( reverse=reverse ) ) + def get_install_tool_version( app, tool_id ): return app.install_model.context.query( diff -r 02efb1123d469986a051e7a8015c9d56635efd8a -r 08f8850853d004bf8a456a147436a6694d0a1a11 lib/galaxy/workflow/modules.py --- a/lib/galaxy/workflow/modules.py +++ b/lib/galaxy/workflow/modules.py @@ -467,10 +467,10 @@ type = "tool" - def __init__( self, trans, tool_id ): + def __init__( self, trans, tool_id, tool_version=None ): self.trans = trans self.tool_id = tool_id - self.tool = trans.app.toolbox.get_tool( tool_id ) + self.tool = trans.app.toolbox.get_tool( tool_id, tool_version=tool_version ) self.post_job_actions = {} self.workflow_outputs = [] self.state = None @@ -493,11 +493,14 @@ @classmethod def from_dict( Class, trans, d, secure=True ): tool_id = d[ 'tool_id' ] - module = Class( trans, tool_id ) + tool_version = str( d.get( 'tool_version', None ) ) + module = Class( trans, tool_id, tool_version=tool_version ) module.state = galaxy.tools.DefaultToolState() if module.tool is not None: if d.get('tool_version', 'Unspecified') != module.get_tool_version(): - module.version_changes.append( "%s: using version '%s' instead of version '%s' indicated in this workflow." % ( tool_id, d.get( 'tool_version', 'Unspecified' ), module.get_tool_version() ) ) + message = "%s: using version '%s' instead of version '%s' indicated in this workflow." % ( tool_id, d.get( 'tool_version', 'Unspecified' ), module.get_tool_version() ) + log.debug(message) + module.version_changes.append(message) module.state.decode( d[ "tool_state" ], module.tool, module.trans.app, secure=secure ) module.errors = d.get( "tool_errors", None ) module.post_job_actions = d.get( "post_job_actions", {} ) @@ -519,9 +522,12 @@ # This step has its state saved in the config field due to the # tool being previously unavailable. return module_factory.from_dict(trans, loads(step.config), secure=False) - module = Class( trans, tool_id ) + tool_version = step.tool_version + module = Class( trans, tool_id, tool_version=tool_version ) if step.tool_version and (step.tool_version != module.tool.version): - module.version_changes.append("%s: using version '%s' instead of version '%s' indicated in this workflow." % (tool_id, module.tool.version, step.tool_version)) + message = "%s: using version '%s' instead of version '%s' indicated in this workflow." % (tool_id, module.tool.version, step.tool_version) + log.debug(message) + module.version_changes.append(message) module.recover_state( step.tool_inputs ) module.errors = step.tool_errors module.workflow_outputs = step.workflow_outputs @@ -723,7 +729,7 @@ return state, step_errors def execute( self, trans, progress, invocation, step ): - tool = trans.app.toolbox.get_tool( step.tool_id ) + tool = trans.app.toolbox.get_tool( step.tool_id, tool_version=step.tool_version ) tool_state = step.state collections_to_match = self._find_collections_to_match( tool, progress, step ) diff -r 02efb1123d469986a051e7a8015c9d56635efd8a -r 08f8850853d004bf8a456a147436a6694d0a1a11 test/api/test_tools.py --- a/test/api/test_tools.py +++ b/test/api/test_tools.py @@ -165,6 +165,18 @@ output1_content = self.dataset_populator.get_history_dataset_content( history_id, dataset=output1 ) self.assertEqual( output1_content.strip(), "Cat1Testlistified" ) + @skip_without_tool( "multiple_versions" ) + def test_run_by_versions( self ): + for version in ["0.1", "0.2"]: + # Run simple non-upload tool with an input data parameter. + history_id = self.dataset_populator.new_history() + inputs = dict() + outputs = self._run_and_get_outputs( tool_id="multiple_versions", history_id=history_id, inputs=inputs, tool_version=version ) + self.assertEquals( len( outputs ), 1 ) + output1 = outputs[ 0 ] + output1_content = self.dataset_populator.get_history_dataset_content( history_id, dataset=output1 ) + self.assertEqual( output1_content.strip(), "Version " + version ) + @skip_without_tool( "cat1" ) def test_run_cat1_single_meta_wrapper( self ): # Wrap input in a no-op meta parameter wrapper like Sam is planning to @@ -650,8 +662,8 @@ def _cat1_outputs( self, history_id, inputs ): return self._run_outputs( self._run_cat1( history_id, inputs ) ) - def _run_and_get_outputs( self, tool_id, history_id, inputs ): - return self._run_outputs( self._run( tool_id, history_id, inputs ) ) + def _run_and_get_outputs( self, tool_id, history_id, inputs, tool_version=None ): + return self._run_outputs( self._run( tool_id, history_id, inputs, tool_version=tool_version ) ) def _run_outputs( self, create_response ): self._assert_status_code_is( create_response, 200 ) @@ -660,12 +672,14 @@ def _run_cat1( self, history_id, inputs, assert_ok=False ): return self._run( 'cat1', history_id, inputs, assert_ok=assert_ok ) - def _run( self, tool_id, history_id, inputs, assert_ok=False ): + def _run( self, tool_id, history_id, inputs, assert_ok=False, tool_version=None ): payload = self.dataset_populator.run_tool_payload( tool_id=tool_id, inputs=inputs, history_id=history_id, ) + if tool_version is not None: + payload[ "tool_version" ] = tool_version create_response = self._post( "tools", data=payload ) if assert_ok: self._assert_status_code_is( create_response, 200 ) diff -r 02efb1123d469986a051e7a8015c9d56635efd8a -r 08f8850853d004bf8a456a147436a6694d0a1a11 test/api/test_workflows.py --- a/test/api/test_workflows.py +++ b/test/api/test_workflows.py @@ -444,6 +444,28 @@ def test_run_workflow( self ): self.__run_cat_workflow( inputs_by='step_id' ) + @skip_without_tool( "multiple_versions" ) + def test_run_versioned_tools( self ): + history_01_id = self.dataset_populator.new_history() + workflow_version_01 = self._upload_yaml_workflow( """ +- tool_id: multiple_versions + tool_version: "0.1" + state: + inttest: 0 +""" ) + self.__invoke_workflow( history_01_id, workflow_version_01 ) + self.dataset_populator.wait_for_history( history_01_id, assert_ok=True ) + + history_02_id = self.dataset_populator.new_history() + workflow_version_02 = self._upload_yaml_workflow( """ +- tool_id: multiple_versions + tool_version: "0.2" + state: + inttest: 1 +""" ) + self.__invoke_workflow( history_02_id, workflow_version_02 ) + self.dataset_populator.wait_for_history( history_02_id, assert_ok=True ) + def __run_cat_workflow( self, inputs_by ): workflow = self.workflow_populator.load_workflow( name="test_for_run" ) workflow["steps"]["0"]["uuid"] = str(uuid4()) @@ -910,7 +932,7 @@ self._assert_status_code_is( hda_info_response, 200 ) self.assertEquals( hda_info_response.json()[ "metadata_data_lines" ], lines ) - def __invoke_workflow( self, history_id, workflow_id, inputs, assert_ok=True ): + def __invoke_workflow( self, history_id, workflow_id, inputs={}, assert_ok=True ): workflow_request = dict( history="hist_id=%s" % history_id, ) diff -r 02efb1123d469986a051e7a8015c9d56635efd8a -r 08f8850853d004bf8a456a147436a6694d0a1a11 test/functional/tools/multiple_versions_v01.xml --- /dev/null +++ b/test/functional/tools/multiple_versions_v01.xml @@ -0,0 +1,11 @@ +<tool id="multiple_versions" name="multiple_versions" version="0.1"> + <command> + echo "Version 0.1" > $out_file1 + </command> + <inputs> + <param name="inttest" value="1" type="integer" /> + </inputs> + <outputs> + <data name="out_file1" format="txt" /> + </outputs> +</tool> diff -r 02efb1123d469986a051e7a8015c9d56635efd8a -r 08f8850853d004bf8a456a147436a6694d0a1a11 test/functional/tools/multiple_versions_v02.xml --- /dev/null +++ b/test/functional/tools/multiple_versions_v02.xml @@ -0,0 +1,11 @@ +<tool id="multiple_versions" name="multiple_versions" version="0.2"> + <command> + echo "Version 0.2" > $out_file1 + </command> + <inputs> + <param name="inttest" value="1" type="integer" /> + </inputs> + <outputs> + <data name="out_file1" format="txt" /> + </outputs> +</tool> diff -r 02efb1123d469986a051e7a8015c9d56635efd8a -r 08f8850853d004bf8a456a147436a6694d0a1a11 test/functional/tools/samples_tool_conf.xml --- a/test/functional/tools/samples_tool_conf.xml +++ b/test/functional/tools/samples_tool_conf.xml @@ -36,6 +36,9 @@ <tool file="collection_two_paired.xml" /><tool file="collection_optional_param.xml" /> + <tool file="multiple_versions_v01.xml" /> + <tool file="multiple_versions_v02.xml" /> + \n') - for key, item in self._integrated_tool_panel.items(): + for key, item_type, item in self._integrated_tool_panel.panel_items_iter(): if item: - if isinstance( item, Tool ): + if item_type == panel_item_types.TOOL: os.write( fd, ' <tool id="%s" />\n' % item.id ) - elif isinstance( item, Workflow ): + elif item_type == panel_item_types.WORKFLOW: os.write( fd, ' <workflow id="%s" />\n' % item.id ) - elif isinstance( item, ToolSectionLabel ): + elif item_type == panel_item_types.LABEL: label_id = item.id or '' label_text = item.text or '' label_version = item.version or '' os.write( fd, ' <label id="%s" text="%s" version="%s" />\n' % ( label_id, label_text, label_version ) ) - elif isinstance( item, ToolSection ): + elif item_type == panel_item_types.SECTION: section_id = item.id or '' section_name = item.name or '' section_version = item.version or '' os.write( fd, ' <section id="%s" name="%s" version="%s">\n' % ( section_id, section_name, section_version ) ) - for section_key, section_item in item.elems.items(): - if isinstance( section_item, Tool ): + for section_key, section_item_type, section_item in item.panel_items_iter(): + if section_item_type == panel_item_types.TOOL: if section_item: os.write( fd, ' <tool id="%s" />\n' % section_item.id ) - elif isinstance( section_item, Workflow ): + elif section_item_type == panel_item_types.WORKFLOW: if section_item: os.write( fd, ' <workflow id="%s" />\n' % section_item.id ) - elif isinstance( section_item, ToolSectionLabel ): + elif section_item_type == panel_item_types.LABEL: if section_item: label_id = section_item.id or '' label_text = section_item.text or '' @@ -709,10 +710,9 @@ else: del has_elems[ tool_key ] if remove_from_config: - if hasattr( integrated_has_elems, "elems" ): - integrated_has_elems = integrated_has_elems.elems - if tool_key in integrated_has_elems: - del integrated_has_elems[ tool_key ] + itegrated_items = integrated_has_elems.panel_items() + if tool_key in itegrated_items: + del itegrated_items[ tool_key ] if section_key: _, tool_section = self.get_section( section_key ) @@ -1174,8 +1174,8 @@ """ context = Bunch( toolbox=self, trans=trans ) filters = self._filter_factory.build_filters( trans ) - for elt in self._tool_panel.itervalues(): - elt = _filter_for_panel( elt, filters, context ) + for _, item_type, elt in self._tool_panel.panel_items_iter(): + elt = _filter_for_panel( elt, item_type, filters, context ) if elt: yield elt @@ -1241,7 +1241,7 @@ return self._tool_versions_by_id.get( lineage_tool_version.id, {} ).get( lineage_tool_version.version, None ) -def _filter_for_panel( item, filters, context ): +def _filter_for_panel( item, item_type, filters, context ): """ Filters tool panel elements so that only those that are compatible with provided filters are kept. @@ -1251,13 +1251,13 @@ if not filter_method( context, filter_item ): return False return True - if isinstance( item, Tool ): + if item_type == panel_item_types.TOOL: if _apply_filter( item, filters[ 'tool' ] ): return item - elif isinstance( item, ToolSectionLabel ): + elif item_type == panel_item_types.LABEL: if _apply_filter( item, filters[ 'label' ] ): return item - elif isinstance( item, ToolSection ): + elif item_type == panel_item_types.SECTION: # Filter section item-by-item. Only show a label if there are # non-filtered tools below it. @@ -1265,14 +1265,14 @@ cur_label_key = None tools_under_label = False filtered_elems = item.elems.copy() - for key, section_item in item.elems.items(): - if isinstance( section_item, Tool ): + for key, section_item_type, section_item in item.panel_items_iter(): + if section_item_type == panel_item_types.TOOL: # Filter tool. if _apply_filter( section_item, filters[ 'tool' ] ): tools_under_label = True else: del filtered_elems[ key ] - elif isinstance( section_item, ToolSectionLabel ): + elif section_item_type == panel_item_types.LABEL: # If there is a label and it does not have tools, # remove it. if ( cur_label_key and not tools_under_label ) or not _apply_filter( section_item, filters[ 'label' ] ): diff -r 8527f5a00f196ac8b641c57808d26edf305dc16a -r 24aa2e802a351882695cb98d772001cc2a1316d8 lib/galaxy/tools/toolbox/__init__.py --- a/lib/galaxy/tools/toolbox/__init__.py +++ b/lib/galaxy/tools/toolbox/__init__.py @@ -5,6 +5,12 @@ from .panel import ToolPanelElements from .panel import ToolSection from .panel import ToolSectionLabel +from .panel import panel_item_types - -__all__ = ["ToolPanelElements", "tool_tag_manager", "ToolSection", "ToolSectionLabel"] +__all__ = [ + "ToolPanelElements", + "tool_tag_manager", + "ToolSection", + "ToolSectionLabel", + "panel_item_types" +] diff -r 8527f5a00f196ac8b641c57808d26edf305dc16a -r 24aa2e802a351882695cb98d772001cc2a1316d8 lib/galaxy/tools/toolbox/panel.py --- a/lib/galaxy/tools/toolbox/panel.py +++ b/lib/galaxy/tools/toolbox/panel.py @@ -1,8 +1,46 @@ +from abc import abstractmethod + from galaxy.util.odict import odict +from galaxy.util import bunch from galaxy.model.item_attrs import Dictifiable -class ToolSection( object, Dictifiable ): +panel_item_types = bunch.Bunch( + TOOL="TOOL", + WORKFLOW="WORKFLOW", + SECTION="SECTION", + LABEL="LABEL", +) + + +class HasPanelItems: + """ + """ + + @abstractmethod + def panel_items( self ): + """ Return an ordered dictionary-like object describing tool panel + items (such as workflows, tools, labels, and sections). + """ + + def panel_items_iter( self ): + """ Iterate through panel items each represented as a tuple of + (panel_key, panel_type, panel_content). + """ + for panel_key, panel_value in self.panel_items().iteritems(): + if panel_value is None: + continue + panel_type = panel_item_types.SECTION + if panel_key.startswith("tool_"): + panel_type = panel_item_types.TOOL + elif panel_key.startswith("label_"): + panel_type = panel_item_types.LABEL + elif panel_key.startswith("workflow_"): + panel_type = panel_item_types.WORKFLOW + yield (panel_key, panel_type, panel_value) + + +class ToolSection( object, Dictifiable, HasPanelItems ): """ A group of tools with similar type/purpose that will be displayed as a group in the user interface. @@ -42,6 +80,9 @@ return section_dict + def panel_items( self ): + return self.elems + class ToolSectionLabel( object, Dictifiable ): """ @@ -63,7 +104,7 @@ return super( ToolSectionLabel, self ).to_dict() -class ToolPanelElements( odict ): +class ToolPanelElements( odict, HasPanelItems ): """ Represents an ordered dictionary of tool entries - abstraction used both by tool panel itself (normal and integrated) and its sections. """ @@ -118,3 +159,6 @@ def append_section( self, key, section_elems ): self[ key ] = section_elems + + def panel_items( self ): + return self https://bitbucket.org/galaxy/galaxy-central/commits/edacabfca615/ Changeset: edacabfca615 User: jmchilton Date: 2014-12-31 23:21:10+00:00 Summary: Move 95% of ToolBox into its own module. Including logic related to panels and sections, the integrated tool panels, labels, etc.... Leave a variant of ToolBox in galaxy.tools that needs to know about Tool runtime stuff (Tool creation, actions, and dependency management). Affected #: 3 files diff -r 24aa2e802a351882695cb98d772001cc2a1316d8 -r edacabfca615542ca435a97fc1afcc99daa470d5 lib/galaxy/tools/__init__.py --- a/lib/galaxy/tools/__init__.py +++ b/lib/galaxy/tools/__init__.py @@ -9,9 +9,6 @@ import os import re import shutil -import string -import tarfile -import tempfile import threading import types import urllib @@ -27,12 +24,10 @@ from elementtree import ElementTree from mako.template import Template from paste import httpexceptions -from sqlalchemy import and_ from galaxy import jobs, model from galaxy.datatypes.metadata import JobExternalOutputMetadataWrapper from galaxy import exceptions -from galaxy.tools.toolbox import watcher from galaxy.tools.actions import DefaultToolAction from galaxy.tools.actions.upload import UploadToolAction from galaxy.tools.actions.data_source import DataSourceToolAction @@ -48,17 +43,11 @@ from galaxy.tools.parameters.input_translation import ToolInputTranslator from galaxy.tools.parameters.output import ToolOutputActionGroup from galaxy.tools.parameters.validation import LateValidationError -from galaxy.tools.toolbox.filters import FilterFactory from galaxy.tools.test import parse_tests from galaxy.tools.parser import get_tool_source from galaxy.tools.parser.xml import XmlPageSource -from galaxy.tools.toolbox import ToolPanelElements -from galaxy.tools.toolbox import ToolSection -from galaxy.tools.toolbox import ToolSectionLabel -from galaxy.tools.toolbox import panel_item_types -from galaxy.tools.toolbox import tool_tag_manager -from galaxy.tools.toolbox.lineages import LineageMap -from galaxy.util import listify, parse_xml, rst_to_html, string_as_bool, string_to_object +from galaxy.tools.toolbox import AbstractToolBox +from galaxy.util import rst_to_html, string_as_bool, string_to_object from galaxy.tools.parameters.meta import expand_meta_parameters from galaxy.util.bunch import Bunch from galaxy.util.expressions import ExpressionContext @@ -66,11 +55,7 @@ from galaxy.util.odict import odict from galaxy.util.template import fill_template from galaxy.web import url_for -from galaxy.web.form_builder import SelectField -from galaxy.web.framework.helpers import escape from galaxy.model.item_attrs import Dictifiable -from galaxy.model import Workflow -from tool_shed.util import common_util from tool_shed.util import shed_util_common as suc from .loader import template_macro_params, raw_tool_xml_tree, imported_macro_paths from .execute import execute as execute_job @@ -102,774 +87,30 @@ HELP_UNINITIALIZED = threading.Lock() -INTEGRATED_TOOL_PANEL_DESCRIPTION = """ -This is Galaxy's integrated tool panel and probably should not be modified -directly. It will be regenerated each time Galaxy starts up. To modify locally -managed tools (e.g. from tool_conf.xml) modify that file directly and restart -Galaxy. Whenever possible Tool Shed managed tools (e.g. from shed_tool_conf.xml) -should be managed from within the Galaxy interface of via is UI - but if changes -are nessecary (such as to hide a tool or re-assign its section) modify that file -and restart Galaxy. -""" - class ToolNotFoundException( Exception ): pass -class ToolBox( object, Dictifiable ): - """Container for a collection of tools""" +class ToolBox( AbstractToolBox ): + """ A derivative of AbstractToolBox with knowledge about Tool internals - + how to construct them, action types, dependency management, etc.... + """ def __init__( self, config_filenames, tool_root_dir, app ): - """ - Create a toolbox from the config files named by `config_filenames`, using - `tool_root_dir` as the base directory for finding individual tool config files. - """ - # The _dynamic_tool_confs list contains dictionaries storing - # information about the tools defined in each shed-related - # shed_tool_conf.xml file. - self._dynamic_tool_confs = [] - self._tools_by_id = {} - # Tool lineages can contain chains of related tools with different ids - # so each will be present once in the above dictionary. The following - # dictionary can instead hold multiple tools with different versions. - self._tool_versions_by_id = {} - self._workflows_by_id = {} - # In-memory dictionary that defines the layout of the tool panel. - self._tool_panel = ToolPanelElements() - self._index = 0 - self.data_manager_tools = odict() - self._lineage_map = LineageMap( app ) - # File that contains the XML section and tool tags from all tool panel config files integrated into a - # single file that defines the tool panel layout. This file can be changed by the Galaxy administrator - # (in a way similar to the single tool_conf.xml file in the past) to alter the layout of the tool panel. - self._integrated_tool_panel_config = app.config.integrated_tool_panel_config - # In-memory dictionary that defines the layout of the tool_panel.xml file on disk. - self._integrated_tool_panel = ToolPanelElements() - self._integrated_tool_panel_config_has_contents = os.path.exists( self._integrated_tool_panel_config ) and os.stat( self._integrated_tool_panel_config ).st_size > 0 - if self._integrated_tool_panel_config_has_contents: - self._load_integrated_tool_panel_keys() - # The following refers to the tool_path config setting for backward compatibility. The shed-related - # (e.g., shed_tool_conf.xml) files include the tool_path attribute within the <toolbox> tag. - self._tool_root_dir = tool_root_dir - self.app = app - self._tool_watcher = watcher.get_watcher( self, app.config ) - self._filter_factory = FilterFactory( self ) + super( ToolBox, self ).__init__( + config_filenames=config_filenames, + tool_root_dir=tool_root_dir, + app=app, + ) self._init_dependency_manager() - self._tool_tag_manager = tool_tag_manager( app ) - self._init_tools_from_configs( config_filenames ) - if self.app.name == 'galaxy' and self._integrated_tool_panel_config_has_contents: - # Load self._tool_panel based on the order in self._integrated_tool_panel. - self._load_tool_panel() - if app.config.update_integrated_tool_panel: - # Write the current in-memory integrated_tool_panel to the integrated_tool_panel.xml file. - # This will cover cases where the Galaxy administrator manually edited one or more of the tool panel - # config files, adding or removing locally developed tools or workflows. The value of integrated_tool_panel - # will be False when things like functional tests are the caller. - self._write_integrated_tool_panel_config_file() - - def _init_tools_from_configs( self, config_filenames ): - """ Read through all tool config files and initialize tools in each - with init_tools_from_config below. - """ - self._tool_tag_manager.reset_tags() - config_filenames = listify( config_filenames ) - for config_filename in config_filenames: - if os.path.isdir( config_filename ): - directory_contents = sorted( os.listdir( config_filename ) ) - directory_config_files = [ config_file for config_file in directory_contents if config_file.endswith( ".xml" ) ] - config_filenames.remove( config_filename ) - config_filenames.extend( directory_config_files ) - for config_filename in config_filenames: - try: - self._init_tools_from_config( config_filename ) - except: - log.exception( "Error loading tools defined in config %s", config_filename ) - - def _init_tools_from_config( self, config_filename ): - """ - Read the configuration file and load each tool. The following tags are currently supported: - - .. raw:: xml - - <toolbox> - <tool file="data_source/upload.xml"/> # tools outside sections - <label text="Basic Tools" id="basic_tools" /> # labels outside sections - <workflow id="529fd61ab1c6cc36" /> # workflows outside sections - <section name="Get Data" id="getext"> # sections - <tool file="data_source/biomart.xml" /> # tools inside sections - <label text="In Section" id="in_section" /> # labels inside sections - <workflow id="adb5f5c93f827949" /> # workflows inside sections - </section> - </toolbox> - - """ - log.info( "Parsing the tool configuration %s" % config_filename ) - tree = parse_xml( config_filename ) - root = tree.getroot() - tool_path = root.get( 'tool_path' ) - if tool_path: - # We're parsing a shed_tool_conf file since we have a tool_path attribute. - parsing_shed_tool_conf = True - # Keep an in-memory list of xml elements to enable persistence of the changing tool config. - config_elems = [] - else: - parsing_shed_tool_conf = False - tool_path = self.__resolve_tool_path(tool_path, config_filename) - # Only load the panel_dict under certain conditions. - load_panel_dict = not self._integrated_tool_panel_config_has_contents - for _, elem in enumerate( root ): - index = self._index - self._index += 1 - if parsing_shed_tool_conf: - config_elems.append( elem ) - self.load_item( - elem, - tool_path=tool_path, - load_panel_dict=load_panel_dict, - guid=elem.get( 'guid' ), - index=index, - internal=True - ) - - if parsing_shed_tool_conf: - shed_tool_conf_dict = dict( config_filename=config_filename, - tool_path=tool_path, - config_elems=config_elems ) - self._dynamic_tool_confs.append( shed_tool_conf_dict ) - - def load_item( self, elem, tool_path, panel_dict=None, integrated_panel_dict=None, load_panel_dict=True, guid=None, index=None, internal=False ): - item_type = elem.tag - if item_type not in ['tool', 'section'] and not internal: - # External calls from tool shed code cannot load labels or tool - # directories. - return - - if panel_dict is None: - panel_dict = self._tool_panel - if integrated_panel_dict is None: - integrated_panel_dict = self._integrated_tool_panel - if item_type == 'tool': - self._load_tool_tag_set( elem, panel_dict=panel_dict, integrated_panel_dict=integrated_panel_dict, tool_path=tool_path, load_panel_dict=load_panel_dict, guid=guid, index=index ) - elif item_type == 'workflow': - self._load_workflow_tag_set( elem, panel_dict=panel_dict, integrated_panel_dict=integrated_panel_dict, load_panel_dict=load_panel_dict, index=index ) - elif item_type == 'section': - self._load_section_tag_set( elem, tool_path=tool_path, load_panel_dict=load_panel_dict, index=index ) - elif item_type == 'label': - self._load_label_tag_set( elem, panel_dict=panel_dict, integrated_panel_dict=integrated_panel_dict, load_panel_dict=load_panel_dict, index=index ) - elif item_type == 'tool_dir': - self._load_tooldir_tag_set( elem, panel_dict, tool_path, integrated_panel_dict, load_panel_dict=load_panel_dict ) - - def get_shed_config_dict_by_filename( self, filename, default=None ): - for shed_config_dict in self._dynamic_tool_confs: - if shed_config_dict[ 'config_filename' ] == filename: - return shed_config_dict - return default - - def update_shed_config( self, shed_conf, integrated_panel_changes=True ): - """ Update the in-memory descriptions of tools and write out the changes - to integrated tool panel unless we are just deactivating a tool (since - that doesn't affect that file). - """ - app = self.app - for index, my_shed_tool_conf in enumerate( self._dynamic_tool_confs ): - if shed_conf['config_filename'] == my_shed_tool_conf['config_filename']: - self._dynamic_tool_confs[ index ] = shed_conf - if integrated_panel_changes and app.config.update_integrated_tool_panel: - # Write the current in-memory version of the integrated_tool_panel.xml file to disk. - self._write_integrated_tool_panel_config_file() - app.reindex_tool_search() - - def get_section( self, section_id, new_label=None, create_if_needed=False ): - tool_panel_section_key = str( section_id ) - if tool_panel_section_key in self._tool_panel: - # Appending a tool to an existing section in toolbox._tool_panel - tool_section = self._tool_panel[ tool_panel_section_key ] - log.debug( "Appending to tool panel section: %s" % str( tool_section.name ) ) - elif create_if_needed: - # Appending a new section to toolbox._tool_panel - if new_label is None: - # This might add an ugly section label to the tool panel, but, oh well... - new_label = section_id - section_dict = { - 'name': new_label, - 'id': section_id, - 'version': '', - } - tool_section = ToolSection( section_dict ) - self._tool_panel[ tool_panel_section_key ] = tool_section - log.debug( "Loading new tool panel section: %s" % str( tool_section.name ) ) - else: - tool_section = None - return tool_panel_section_key, tool_section - - def get_integrated_section_for_tool( self, tool ): - tool_id = tool.id - for key, item_type, item in self._integrated_tool_panel.panel_items_iter(): - if item: - if item_type == panel_item_types.TOOL: - if item.id == tool_id: - return '', '' - if item_type == panel_item_types.SECTION: - section_id = item.id or '' - section_name = item.name or '' - for section_key, section_item_type, section_item in item.panel_items_iter(): - if section_item_type == panel_item_types.TOOL: - if section_item: - if section_item.id == tool_id: - return section_id, section_name - return None, None - - def __resolve_tool_path(self, tool_path, config_filename): - if not tool_path: - # Default to backward compatible config setting. - tool_path = self._tool_root_dir - else: - # Allow use of __tool_conf_dir__ in toolbox config files. - tool_conf_dir = os.path.dirname(config_filename) - tool_path_vars = {"tool_conf_dir": tool_conf_dir} - tool_path = string.Template(tool_path).safe_substitute(tool_path_vars) - return tool_path - - def __add_tool_to_tool_panel( self, tool, panel_component, section=False ): - # See if a version of this tool is already loaded into the tool panel. - # The value of panel_component will be a ToolSection (if the value of - # section=True) or self._tool_panel (if section=False). - tool_id = str( tool.id ) - tool = self._tools_by_id[ tool_id ] - if section: - panel_dict = panel_component.elems - else: - panel_dict = panel_component - - related_tool = self._lineage_in_panel( panel_dict, tool=tool ) - if related_tool: - if self._newer_tool( tool, related_tool ): - panel_dict.replace_tool( - previous_tool_id=related_tool.id, - new_tool_id=tool_id, - tool=tool, - ) - log.debug( "Loaded tool id: %s, version: %s into tool panel." % ( tool.id, tool.version ) ) - else: - inserted = False - index = self._integrated_tool_panel.index_of_tool_id( tool_id ) - if index: - panel_dict.insert_tool( index, tool ) - inserted = True - if not inserted: - # Check the tool's installed versions. - for tool_lineage_version in tool.lineage.get_versions(): - lineage_id = tool_lineage_version.id - index = self._integrated_tool_panel.index_of_tool_id( lineage_id ) - if index: - panel_dict.insert_tool( index, tool ) - inserted = True - if not inserted: - if ( - tool.guid is None or - tool.tool_shed is None or - tool.repository_name is None or - tool.repository_owner is None or - tool.installed_changeset_revision is None - ): - # We have a tool that was not installed from the Tool - # Shed, but is also not yet defined in - # integrated_tool_panel.xml, so append it to the tool - # panel. - panel_dict.append_tool( tool ) - log.debug( "Loaded tool id: %s, version: %s into tool panel.." % ( tool.id, tool.version ) ) - else: - # We are in the process of installing the tool. - - tool_lineage = self._lineage_map.get( tool_id ) - already_loaded = self._lineage_in_panel( panel_dict, tool_lineage=tool_lineage ) is not None - if not already_loaded: - # If the tool is not defined in integrated_tool_panel.xml, append it to the tool panel. - panel_dict.append_tool( tool ) - log.debug( "Loaded tool id: %s, version: %s into tool panel...." % ( tool.id, tool.version ) ) - - def _load_tool_panel( self ): - for key, item_type, val in self._integrated_tool_panel.panel_items_iter(): - if item_type == panel_item_types.TOOL: - tool_id = key.replace( 'tool_', '', 1 ) - if tool_id in self._tools_by_id: - self.__add_tool_to_tool_panel( val, self._tool_panel, section=False ) - elif item_type == panel_item_types.WORKFLOW: - workflow_id = key.replace( 'workflow_', '', 1 ) - if workflow_id in self._workflows_by_id: - workflow = self._workflows_by_id[ workflow_id ] - self._tool_panel[ key ] = workflow - log.debug( "Loaded workflow: %s %s" % ( workflow_id, workflow.name ) ) - elif item_type == panel_item_types.LABEL: - self._tool_panel[ key ] = val - elif item_type == panel_item_types.SECTION: - section_dict = { - 'id': val.id or '', - 'name': val.name or '', - 'version': val.version or '', - } - section = ToolSection( section_dict ) - log.debug( "Loading section: %s" % section_dict.get( 'name' ) ) - for section_key, section_item_type, section_val in val.panel_items_iter(): - if section_item_type == panel_item_types.TOOL: - tool_id = section_key.replace( 'tool_', '', 1 ) - if tool_id in self._tools_by_id: - self.__add_tool_to_tool_panel( section_val, section, section=True ) - elif section_item_type == panel_item_types.WORKFLOW: - workflow_id = section_key.replace( 'workflow_', '', 1 ) - if workflow_id in self._workflows_by_id: - workflow = self._workflows_by_id[ workflow_id ] - section.elems[ section_key ] = workflow - log.debug( "Loaded workflow: %s %s" % ( workflow_id, workflow.name ) ) - elif section_item_type == panel_item_types.LABEL: - if section_val: - section.elems[ section_key ] = section_val - log.debug( "Loaded label: %s" % ( section_val.text ) ) - self._tool_panel[ key ] = section - - def _load_integrated_tool_panel_keys( self ): - """ - Load the integrated tool panel keys, setting values for tools and - workflows to None. The values will be reset when the various tool - panel config files are parsed, at which time the tools and workflows - are loaded. - """ - tree = parse_xml( self._integrated_tool_panel_config ) - root = tree.getroot() - for elem in root: - key = elem.get( 'id' ) - if elem.tag == 'tool': - self._integrated_tool_panel.stub_tool( key ) - elif elem.tag == 'workflow': - self._integrated_tool_panel.stub_workflow( key ) - elif elem.tag == 'section': - section = ToolSection( elem ) - for section_elem in elem: - section_id = section_elem.get( 'id' ) - if section_elem.tag == 'tool': - section.elems.stub_tool( section_id ) - elif section_elem.tag == 'workflow': - section.elems.stub_workflow( section_id ) - elif section_elem.tag == 'label': - section.elems.stub_label( section_id ) - self._integrated_tool_panel.append_section( key, section ) - elif elem.tag == 'label': - section.stub_label( key ) - - def _write_integrated_tool_panel_config_file( self ): - """ - Write the current in-memory version of the integrated_tool_panel.xml file to disk. Since Galaxy administrators - use this file to manage the tool panel, we'll not use xml_to_string() since it doesn't write XML quite right. - """ - fd, filename = tempfile.mkstemp() - os.write( fd, '<?xml version="1.0"?>\n' ) - os.write( fd, '<toolbox>\n' ) - os.write( fd, ' \n') - for key, item_type, item in self._integrated_tool_panel.panel_items_iter(): - if item: - if item_type == panel_item_types.TOOL: - os.write( fd, ' <tool id="%s" />\n' % item.id ) - elif item_type == panel_item_types.WORKFLOW: - os.write( fd, ' <workflow id="%s" />\n' % item.id ) - elif item_type == panel_item_types.LABEL: - label_id = item.id or '' - label_text = item.text or '' - label_version = item.version or '' - os.write( fd, ' <label id="%s" text="%s" version="%s" />\n' % ( label_id, label_text, label_version ) ) - elif item_type == panel_item_types.SECTION: - section_id = item.id or '' - section_name = item.name or '' - section_version = item.version or '' - os.write( fd, ' <section id="%s" name="%s" version="%s">\n' % ( section_id, section_name, section_version ) ) - for section_key, section_item_type, section_item in item.panel_items_iter(): - if section_item_type == panel_item_types.TOOL: - if section_item: - os.write( fd, ' <tool id="%s" />\n' % section_item.id ) - elif section_item_type == panel_item_types.WORKFLOW: - if section_item: - os.write( fd, ' <workflow id="%s" />\n' % section_item.id ) - elif section_item_type == panel_item_types.LABEL: - if section_item: - label_id = section_item.id or '' - label_text = section_item.text or '' - label_version = section_item.version or '' - os.write( fd, ' <label id="%s" text="%s" version="%s" />\n' % ( label_id, label_text, label_version ) ) - os.write( fd, ' </section>\n' ) - os.write( fd, '</toolbox>\n' ) - os.close( fd ) - shutil.move( filename, os.path.abspath( self._integrated_tool_panel_config ) ) - os.chmod( self._integrated_tool_panel_config, 0644 ) - - def get_tool( self, tool_id, tool_version=None, get_all_versions=False, exact=False ): - """Attempt to locate a tool in the tool box.""" - if tool_version: - tool_version = str( tool_version ) - - if get_all_versions and exact: - raise AssertionError("Cannot specify get_tool with both get_all_versions and exact as True") - - if tool_id in self._tools_by_id and not get_all_versions: - if tool_version and tool_version in self._tool_versions_by_id[ tool_id ]: - return self._tool_versions_by_id[ tool_id ][ tool_version ] - #tool_id exactly matches an available tool by id (which is 'old' tool_id or guid) - return self._tools_by_id[ tool_id ] - #exact tool id match not found, or all versions requested, search for other options, e.g. migrated tools or different versions - rval = [] - tool_lineage = self._lineage_map.get( tool_id ) - if tool_lineage: - lineage_tool_versions = tool_lineage.get_versions( ) - for lineage_tool_version in lineage_tool_versions: - lineage_tool = self._tool_from_lineage_version( lineage_tool_version ) - if lineage_tool: - rval.append( lineage_tool ) - if not rval: - #still no tool, do a deeper search and try to match by old ids - for tool in self._tools_by_id.itervalues(): - if tool.old_id == tool_id: - rval.append( tool ) - if rval: - if get_all_versions: - return rval - else: - if tool_version: - #return first tool with matching version - for tool in rval: - if tool.version == tool_version: - return tool - #No tool matches by version, simply return the first available tool found - return rval[0] - #We now likely have a Toolshed guid passed in, but no supporting database entries - #If the tool exists by exact id and is loaded then provide exact match within a list - if tool_id in self._tools_by_id: - return[ self._tools_by_id[ tool_id ] ] - return None - - def has_tool( self, tool_id, exact=False ): - return self.get_tool( tool_id, exact=exact ) is not None - - def get_tool_id( self, tool_id ): - """ Take a tool id (potentially from a different Galaxy instance or that - is no longer loaded - and find the closest match to the currently loaded - tools (using get_tool for inexact matches which currently returns the oldest - tool shed installed tool with the same short id). - """ - if tool_id not in self._tools_by_id: - tool = self.get_tool( tool_id ) - if tool: - tool_id = tool.id - else: - tool_id = None - # else exact match - leave unmodified. - return tool_id - - def get_loaded_tools_by_lineage( self, tool_id ): - """Get all loaded tools associated by lineage to the tool whose id is tool_id.""" - tool_lineage = self._lineage_map.get( tool_id ) - if tool_lineage: - lineage_tool_versions = tool_lineage.get_versions( ) - available_tool_versions = [] - for lineage_tool_version in lineage_tool_versions: - tool = self._tool_from_lineage_version( lineage_tool_version ) - if tool: - available_tool_versions.append( tool ) - return available_tool_versions - else: - if tool_id in self._tools_by_id: - tool = self._tools_by_id[ tool_id ] - return [ tool ] - return [] @property def tools_by_id( self ): # Deprecated method, TODO - eliminate calls to this in test/. return self._tools_by_id - def tools( self ): - return self._tools_by_id.iteritems() - - def __elem_to_tool_shed_repository( self, elem ): - # The tool is contained in an installed tool shed repository, so load - # the tool only if the repository has not been marked deleted. - tool_shed = elem.find( "tool_shed" ).text - repository_name = elem.find( "repository_name" ).text - repository_owner = elem.find( "repository_owner" ).text - installed_changeset_revision_elem = elem.find( "installed_changeset_revision" ) - if installed_changeset_revision_elem is None: - # Backward compatibility issue - the tag used to be named 'changeset_revision'. - installed_changeset_revision_elem = elem.find( "changeset_revision" ) - installed_changeset_revision = installed_changeset_revision_elem.text - tool_shed_repository = self.__get_tool_shed_repository( tool_shed, - repository_name, - repository_owner, - installed_changeset_revision ) - return tool_shed_repository - - def __get_tool_shed_repository( self, tool_shed, name, owner, installed_changeset_revision ): - # We store only the port, if one exists, in the database. - tool_shed = common_util.remove_protocol_from_tool_shed_url( tool_shed ) - return self.app.install_model.context.query( self.app.install_model.ToolShedRepository ) \ - .filter( and_( self.app.install_model.ToolShedRepository.table.c.tool_shed == tool_shed, - self.app.install_model.ToolShedRepository.table.c.name == name, - self.app.install_model.ToolShedRepository.table.c.owner == owner, - self.app.install_model.ToolShedRepository.table.c.installed_changeset_revision == installed_changeset_revision ) ) \ - .first() - - def get_tool_components( self, tool_id, tool_version=None, get_loaded_tools_by_lineage=False, set_selected=False ): - """ - Retrieve all loaded versions of a tool from the toolbox and return a select list enabling - selection of a different version, the list of the tool's loaded versions, and the specified tool. - """ - toolbox = self - tool_version_select_field = None - tools = [] - tool = None - # Backwards compatibility for datasource tools that have default tool_id configured, but which - # are now using only GALAXY_URL. - tool_ids = listify( tool_id ) - for tool_id in tool_ids: - if get_loaded_tools_by_lineage: - tools = toolbox.get_loaded_tools_by_lineage( tool_id ) - else: - tools = toolbox.get_tool( tool_id, tool_version=tool_version, get_all_versions=True ) - if tools: - tool = toolbox.get_tool( tool_id, tool_version=tool_version, get_all_versions=False ) - if len( tools ) > 1: - tool_version_select_field = self.build_tool_version_select_field( tools, tool.id, set_selected ) - break - return tool_version_select_field, tools, tool - - def dynamic_confs( self, include_migrated_tool_conf=False ): - confs = [] - for dynamic_tool_conf_dict in self._dynamic_tool_confs: - dynamic_tool_conf_filename = dynamic_tool_conf_dict[ 'config_filename' ] - if include_migrated_tool_conf or (dynamic_tool_conf_filename != self.app.config.migrated_tools_config): - confs.append( dynamic_tool_conf_dict ) - return confs - - def dynamic_conf_filenames( self, include_migrated_tool_conf=False ): - """ Return list of dynamic tool configuration filenames (shed_tools). - These must be used with various dynamic tool configuration update - operations (e.g. with update_shed_config). - """ - for dynamic_tool_conf_dict in self.dynamic_confs( include_migrated_tool_conf=include_migrated_tool_conf ): - yield dynamic_tool_conf_dict[ 'config_filename' ] - - def build_tool_version_select_field( self, tools, tool_id, set_selected ): - """Build a SelectField whose options are the ids for the received list of tools.""" - options = [] - refresh_on_change_values = [] - for tool in tools: - options.insert( 0, ( tool.version, tool.id ) ) - refresh_on_change_values.append( tool.id ) - select_field = SelectField( name='tool_id', refresh_on_change=True, refresh_on_change_values=refresh_on_change_values ) - for option_tup in options: - selected = set_selected and option_tup[ 1 ] == tool_id - if selected: - select_field.add_option( 'version %s' % option_tup[ 0 ], option_tup[ 1 ], selected=True ) - else: - select_field.add_option( 'version %s' % option_tup[ 0 ], option_tup[ 1 ] ) - return select_field - - def remove_from_panel( self, tool_id, section_key='', remove_from_config=True ): - - def remove_from_dict( has_elems, integrated_has_elems ): - tool_key = 'tool_%s' % str( tool_id ) - available_tool_versions = self.get_loaded_tools_by_lineage( tool_id ) - if tool_key in has_elems: - if available_tool_versions: - available_tool_versions.reverse() - replacement_tool_key = None - replacement_tool_version = None - # Since we are going to remove the tool from the section, replace it with - # the newest loaded version of the tool. - for available_tool_version in available_tool_versions: - available_tool_section_id, available_tool_section_name = available_tool_version.get_panel_section() - # I suspect "available_tool_version.id in has_elems.keys()" doesn't - # belong in the following line or at least I don't understand what - # purpose it might serve. -John - if available_tool_version.id in has_elems.keys() or (available_tool_section_id == section_key): - replacement_tool_key = 'tool_%s' % str( available_tool_version.id ) - replacement_tool_version = available_tool_version - break - if replacement_tool_key and replacement_tool_version: - # Get the index of the tool_key in the tool_section. - for tool_panel_index, key in enumerate( has_elems.keys() ): - if key == tool_key: - break - # Remove the tool from the tool panel. - del has_elems[ tool_key ] - # Add the replacement tool at the same location in the tool panel. - has_elems.insert( tool_panel_index, - replacement_tool_key, - replacement_tool_version ) - else: - del has_elems[ tool_key ] - else: - del has_elems[ tool_key ] - if remove_from_config: - itegrated_items = integrated_has_elems.panel_items() - if tool_key in itegrated_items: - del itegrated_items[ tool_key ] - - if section_key: - _, tool_section = self.get_section( section_key ) - if tool_section: - remove_from_dict( tool_section.elems, self._integrated_tool_panel.get( section_key, {} ) ) - else: - remove_from_dict( self._tool_panel, self._integrated_tool_panel ) - - def _load_tool_tag_set( self, elem, panel_dict, integrated_panel_dict, tool_path, load_panel_dict, guid=None, index=None ): - try: - path = elem.get( "file" ) - repository_id = None - - tool_shed_repository = None - can_load_into_panel_dict = True - if guid is not None: - # The tool is contained in an installed tool shed repository, so load - # the tool only if the repository has not been marked deleted. - tool_shed_repository = self.__elem_to_tool_shed_repository( elem ) - if tool_shed_repository: - # Only load tools if the repository is not deactivated or uninstalled. - can_load_into_panel_dict = not tool_shed_repository.deleted - repository_id = self.app.security.encode_id( tool_shed_repository.id ) - # Else there is not yet a tool_shed_repository record, we're in the process of installing - # a new repository, so any included tools can be loaded into the tool panel. - tool = self.load_tool( os.path.join( tool_path, path ), guid=guid, repository_id=repository_id ) - if string_as_bool(elem.get( 'hidden', False )): - tool.hidden = True - key = 'tool_%s' % str( tool.id ) - if can_load_into_panel_dict: - if guid is not None: - tool.tool_shed = tool_shed_repository.tool_shed - tool.repository_name = tool_shed_repository.name - tool.repository_owner = tool_shed_repository.owner - tool.installed_changeset_revision = tool_shed_repository.installed_changeset_revision - tool.guid = guid - tool.version = elem.find( "version" ).text - # Make sure the tool has a tool_version. - tool_lineage = self._lineage_map.register( tool, tool_shed_repository=tool_shed_repository ) - # Load the tool's lineage ids. - tool.lineage = tool_lineage - self._tool_tag_manager.handle_tags( tool.id, elem ) - self.__add_tool( tool, load_panel_dict, panel_dict ) - # Always load the tool into the integrated_panel_dict, or it will not be included in the integrated_tool_panel.xml file. - integrated_panel_dict.update_or_append( index, key, tool ) - except IOError: - log.error( "Error reading tool configuration file from path: %s." % path ) - except Exception: - log.exception( "Error reading tool from path: %s" % path ) - - def __add_tool( self, tool, load_panel_dict, panel_dict ): - # Allow for the same tool to be loaded into multiple places in the - # tool panel. We have to handle the case where the tool is contained - # in a repository installed from the tool shed, and the Galaxy - # administrator has retrieved updates to the installed repository. In - # this case, the tool may have been updated, but the version was not - # changed, so the tool should always be reloaded here. We used to - # only load the tool if it was not found in self._tools_by_id, but - # performing that check did not enable this scenario. - self.register_tool( tool ) - if load_panel_dict: - self.__add_tool_to_tool_panel( tool, panel_dict, section=isinstance( panel_dict, ToolSection ) ) - - def _load_workflow_tag_set( self, elem, panel_dict, integrated_panel_dict, load_panel_dict, index=None ): - try: - # TODO: should id be encoded? - workflow_id = elem.get( 'id' ) - workflow = self._load_workflow( workflow_id ) - self._workflows_by_id[ workflow_id ] = workflow - key = 'workflow_' + workflow_id - if load_panel_dict: - panel_dict[ key ] = workflow - # Always load workflows into the integrated_panel_dict. - integrated_panel_dict.update_or_append( index, key, workflow ) - except: - log.exception( "Error loading workflow: %s" % workflow_id ) - - def _load_label_tag_set( self, elem, panel_dict, integrated_panel_dict, load_panel_dict, index=None ): - label = ToolSectionLabel( elem ) - key = 'label_' + label.id - if load_panel_dict: - panel_dict[ key ] = label - integrated_panel_dict.update_or_append( index, key, label ) - - def _load_section_tag_set( self, elem, tool_path, load_panel_dict, index=None ): - key = elem.get( "id" ) - if key in self._tool_panel: - section = self._tool_panel[ key ] - elems = section.elems - else: - section = ToolSection( elem ) - elems = section.elems - if key in self._integrated_tool_panel: - integrated_section = self._integrated_tool_panel[ key ] - integrated_elems = integrated_section.elems - else: - integrated_section = ToolSection( elem ) - integrated_elems = integrated_section.elems - for sub_index, sub_elem in enumerate( elem ): - self.load_item( - sub_elem, - tool_path=tool_path, - panel_dict=elems, - integrated_panel_dict=integrated_elems, - load_panel_dict=load_panel_dict, - guid=sub_elem.get( 'guid' ), - index=sub_index, - internal=True, - ) - if load_panel_dict: - self._tool_panel[ key ] = section - # Always load sections into the integrated_tool_panel. - self._integrated_tool_panel.update_or_append( index, key, integrated_section ) - - def _load_tooldir_tag_set(self, sub_elem, elems, tool_path, integrated_elems, load_panel_dict): - directory = os.path.join( tool_path, sub_elem.attrib.get("dir") ) - recursive = string_as_bool( sub_elem.attrib.get("recursive", True) ) - self.__watch_directory( directory, elems, integrated_elems, load_panel_dict, recursive ) - - def __watch_directory( self, directory, elems, integrated_elems, load_panel_dict, recursive): - - def quick_load( tool_file, async=True ): - try: - tool = self.load_tool( tool_file ) - self.__add_tool( tool, load_panel_dict, elems ) - # Always load the tool into the integrated_panel_dict, or it will not be included in the integrated_tool_panel.xml file. - key = 'tool_%s' % str( tool.id ) - integrated_elems[ key ] = tool - - if async: - self._load_tool_panel() - - if self.app.config.update_integrated_tool_panel: - # Write the current in-memory integrated_tool_panel to the integrated_tool_panel.xml file. - # This will cover cases where the Galaxy administrator manually edited one or more of the tool panel - # config files, adding or removing locally developed tools or workflows. The value of integrated_tool_panel - # will be False when things like functional tests are the caller. - self._write_integrated_tool_panel_config_file() - return tool.id - except Exception: - log.exception("Failed to load potential tool %s." % tool_file) - return None - - tool_loaded = False - for name in os.listdir( directory ): - child_path = os.path.join(directory, name) - if os.path.isdir(child_path) and recursive: - self.__watch_directory(child_path, elems, integrated_elems, load_panel_dict, recursive) - elif name.endswith( ".xml" ): - quick_load( child_path, async=False ) - tool_loaded = True - if tool_loaded: - self._tool_watcher.watch_directory( directory, quick_load ) - - def load_tool( self, config_file, guid=None, repository_id=None, **kwds ): - """Load a single tool from the file named by `config_file` and return an instance of `Tool`.""" - # Parse XML configuration file and get the root element + def create_tool( self, config_file, repository_id=None, guid=None, **kwds ): tool_source = get_tool_source( config_file, getattr( self.app.config, "enable_beta_tool_formats", False ) ) # Allow specifying a different tool subclass to instantiate tool_module = tool_source.parse_tool_module() @@ -904,254 +145,8 @@ ToolClass = Tool tool = ToolClass( config_file, tool_source, self.app, guid=guid, repository_id=repository_id, **kwds ) - tool_id = tool.id - if not tool_id.startswith("__"): - # do not monitor special tools written to tmp directory - no reason - # to monitor such a large directory. - self._tool_watcher.watch_file( config_file, tool.id ) return tool - def load_hidden_tool( self, config_file, **kwds ): - """ Load a hidden tool (in this context meaning one that does not - appear in the tool panel) and register it in _tools_by_id. - """ - tool = self.load_tool( config_file, **kwds ) - self.register_tool( tool ) - return tool - - def register_tool( self, tool ): - tool_id = tool.id - version = tool.version or None - if tool_id not in self._tool_versions_by_id: - self._tool_versions_by_id[ tool_id ] = { version: tool } - else: - self._tool_versions_by_id[ tool_id ][ version ] = tool - if tool_id in self._tools_by_id: - related_tool = self._tools_by_id[ tool_id ] - # This one becomes the default un-versioned tool - # if newer. - if self._newer_tool( tool, related_tool ): - self._tools_by_id[ tool_id ] = tool - else: - self._tools_by_id[ tool_id ] = tool - - def package_tool( self, trans, tool_id ): - """ - Create a tarball with the tool's xml, help images, and test data. - :param trans: the web transaction - :param tool_id: the tool ID from app.toolbox - :returns: tuple of tarball filename, success True/False, message/None - """ - # Make sure the tool is actually loaded. - if tool_id not in self._tools_by_id: - return None, False, "No tool with id %s" % escape( tool_id ) - else: - tool = self._tools_by_id[ tool_id ] - tarball_files = [] - temp_files = [] - tool_xml = file( os.path.abspath( tool.config_file ), 'r' ).read() - # Retrieve tool help images and rewrite the tool's xml into a temporary file with the path - # modified to be relative to the repository root. - image_found = False - if tool.help is not None: - tool_help = tool.help._source - # Check each line of the rendered tool help for an image tag that points to a location under static/ - for help_line in tool_help.split( '\n' ): - image_regex = re.compile( 'img alt="[^"]+" src="\${static_path}/([^"]+)"' ) - matches = re.search( image_regex, help_line ) - if matches is not None: - tool_help_image = matches.group(1) - tarball_path = tool_help_image - filesystem_path = os.path.abspath( os.path.join( trans.app.config.root, 'static', tool_help_image ) ) - if os.path.exists( filesystem_path ): - tarball_files.append( ( filesystem_path, tarball_path ) ) - image_found = True - tool_xml = tool_xml.replace( '${static_path}/%s' % tarball_path, tarball_path ) - # If one or more tool help images were found, add the modified tool XML to the tarball instead of the original. - if image_found: - fd, new_tool_config = tempfile.mkstemp( suffix='.xml' ) - os.close( fd ) - file( new_tool_config, 'w' ).write( tool_xml ) - tool_tup = ( os.path.abspath( new_tool_config ), os.path.split( tool.config_file )[-1] ) - temp_files.append( os.path.abspath( new_tool_config ) ) - else: - tool_tup = ( os.path.abspath( tool.config_file ), os.path.split( tool.config_file )[-1] ) - tarball_files.append( tool_tup ) - # TODO: This feels hacky. - tool_command = tool.command.strip().split()[0] - tool_path = os.path.dirname( os.path.abspath( tool.config_file ) ) - # Add the tool XML to the tuple that will be used to populate the tarball. - if os.path.exists( os.path.join( tool_path, tool_command ) ): - tarball_files.append( ( os.path.join( tool_path, tool_command ), tool_command ) ) - # Find and add macros and code files. - for external_file in tool.get_externally_referenced_paths( os.path.abspath( tool.config_file ) ): - external_file_abspath = os.path.abspath( os.path.join( tool_path, external_file ) ) - tarball_files.append( ( external_file_abspath, external_file ) ) - if os.path.exists( os.path.join( tool_path, "Dockerfile" ) ): - tarball_files.append( ( os.path.join( tool_path, "Dockerfile" ), "Dockerfile" ) ) - # Find tests, and check them for test data. - tests = tool.tests - if tests is not None: - for test in tests: - # Add input file tuples to the list. - for input in test.inputs: - for input_value in test.inputs[ input ]: - input_path = os.path.abspath( os.path.join( 'test-data', input_value ) ) - if os.path.exists( input_path ): - td_tup = ( input_path, os.path.join( 'test-data', input_value ) ) - tarball_files.append( td_tup ) - # And add output file tuples to the list. - for label, filename, _ in test.outputs: - output_filepath = os.path.abspath( os.path.join( 'test-data', filename ) ) - if os.path.exists( output_filepath ): - td_tup = ( output_filepath, os.path.join( 'test-data', filename ) ) - tarball_files.append( td_tup ) - for param in tool.input_params: - # Check for tool data table definitions. - if hasattr( param, 'options' ): - if hasattr( param.options, 'tool_data_table' ): - data_table = param.options.tool_data_table - if hasattr( data_table, 'filenames' ): - data_table_definitions = [] - for data_table_filename in data_table.filenames: - # FIXME: from_shed_config seems to always be False. - if not data_table.filenames[ data_table_filename ][ 'from_shed_config' ]: - tar_file = data_table.filenames[ data_table_filename ][ 'filename' ] + '.sample' - sample_file = os.path.join( data_table.filenames[ data_table_filename ][ 'tool_data_path' ], - tar_file ) - # Use the .sample file, if one exists. If not, skip this data table. - if os.path.exists( sample_file ): - tarfile_path, tarfile_name = os.path.split( tar_file ) - tarfile_path = os.path.join( 'tool-data', tarfile_name ) - tarball_files.append( ( sample_file, tarfile_path ) ) - data_table_definitions.append( data_table.xml_string ) - if len( data_table_definitions ) > 0: - # Put the data table definition XML in a temporary file. - table_definition = '<?xml version="1.0" encoding="utf-8"?>\n<tables>\n %s</tables>' - table_definition = table_definition % '\n'.join( data_table_definitions ) - fd, table_conf = tempfile.mkstemp() - os.close( fd ) - file( table_conf, 'w' ).write( table_definition ) - tarball_files.append( ( table_conf, os.path.join( 'tool-data', 'tool_data_table_conf.xml.sample' ) ) ) - temp_files.append( table_conf ) - # Create the tarball. - fd, tarball_archive = tempfile.mkstemp( suffix='.tgz' ) - os.close( fd ) - tarball = tarfile.open( name=tarball_archive, mode='w:gz' ) - # Add the files from the previously generated list. - for fspath, tarpath in tarball_files: - tarball.add( fspath, arcname=tarpath ) - tarball.close() - # Delete any temporary files that were generated. - for temp_file in temp_files: - os.remove( temp_file ) - return tarball_archive, True, None - return None, False, "An unknown error occurred." - - def reload_tool_by_id( self, tool_id ): - """ - Attempt to reload the tool identified by 'tool_id', if successful - replace the old tool. - """ - if tool_id not in self._tools_by_id: - message = "No tool with id %s" % escape( tool_id ) - status = 'error' - else: - old_tool = self._tools_by_id[ tool_id ] - new_tool = self.load_tool( old_tool.config_file ) - # The tool may have been installed from a tool shed, so set the tool shed attributes. - # Since the tool version may have changed, we don't override it here. - new_tool.id = old_tool.id - new_tool.guid = old_tool.guid - new_tool.tool_shed = old_tool.tool_shed - new_tool.repository_name = old_tool.repository_name - new_tool.repository_owner = old_tool.repository_owner - new_tool.installed_changeset_revision = old_tool.installed_changeset_revision - new_tool.old_id = old_tool.old_id - # Replace old_tool with new_tool in self._tool_panel - tool_key = 'tool_' + tool_id - for key, val in self._tool_panel.items(): - if key == tool_key: - self._tool_panel[ key ] = new_tool - break - elif key.startswith( 'section' ): - if tool_key in val.elems: - self._tool_panel[ key ].elems[ tool_key ] = new_tool - break - self._tools_by_id[ tool_id ] = new_tool - message = "Reloaded the tool:<br/>" - message += "<b>name:</b> %s<br/>" % old_tool.name - message += "<b>id:</b> %s<br/>" % old_tool.id - message += "<b>version:</b> %s" % old_tool.version - status = 'done' - return message, status - - def remove_tool_by_id( self, tool_id, remove_from_panel=True ): - """ - Attempt to remove the tool identified by 'tool_id'. Ignores - tool lineage - so to remove a tool with potentially multiple - versions send remove_from_panel=False and handle the logic of - promoting the next newest version of the tool into the panel - if needed. - """ - if tool_id not in self._tools_by_id: - message = "No tool with id %s" % escape( tool_id ) - status = 'error' - else: - tool = self._tools_by_id[ tool_id ] - del self._tools_by_id[ tool_id ] - if remove_from_panel: - tool_key = 'tool_' + tool_id - for key, val in self._tool_panel.items(): - if key == tool_key: - del self._tool_panel[ key ] - break - elif key.startswith( 'section' ): - if tool_key in val.elems: - del self._tool_panel[ key ].elems[ tool_key ] - break - if tool_id in self.data_manager_tools: - del self.data_manager_tools[ tool_id ] - #TODO: do we need to manually remove from the integrated panel here? - message = "Removed the tool:<br/>" - message += "<b>name:</b> %s<br/>" % tool.name - message += "<b>id:</b> %s<br/>" % tool.id - message += "<b>version:</b> %s" % tool.version - status = 'done' - return message, status - - def get_sections( self ): - for k, v in self._tool_panel.items(): - if isinstance( v, ToolSection ): - yield (v.id, v.name) - - def find_section_id( self, tool_panel_section_id ): - """ - Find the section ID referenced by the key or return '' indicating - no such section id. - """ - if not tool_panel_section_id: - tool_panel_section_id = '' - else: - if tool_panel_section_id not in self._tool_panel: - # Hack introduced without comment in a29d54619813d5da992b897557162a360b8d610c- - # not sure why it is needed. - fixed_tool_panel_section_id = 'section_%s' % tool_panel_section_id - if fixed_tool_panel_section_id in self._tool_panel: - tool_panel_section_id = fixed_tool_panel_section_id - else: - tool_panel_section_id = '' - return tool_panel_section_id - - def _load_workflow( self, workflow_id ): - """ - Return an instance of 'Workflow' identified by `id`, - which is encoded in the tool panel. - """ - id = self.app.security.decode_id( workflow_id ) - stored = self.app.model.context.query( self.app.model.StoredWorkflow ).get( id ) - return stored.latest_workflow - def _init_dependency_manager( self ): self.dependency_manager = build_dependency_manager( self.app.config ) @@ -1162,138 +157,6 @@ if isinstance( tool.tool_action, UploadToolAction ): self.reload_tool_by_id( tool_id ) - @property - def sa_session( self ): - """ - Returns a SQLAlchemy session - """ - return self.app.model.context - - def tool_panel_contents( self, trans, **kwds ): - """ Filter tool_panel contents for displaying for user. - """ - context = Bunch( toolbox=self, trans=trans ) - filters = self._filter_factory.build_filters( trans ) - for _, item_type, elt in self._tool_panel.panel_items_iter(): - elt = _filter_for_panel( elt, item_type, filters, context ) - if elt: - yield elt - - def to_dict( self, trans, in_panel=True, **kwds ): - """ - to_dict toolbox. - """ - if in_panel: - panel_elts = list( self.tool_panel_contents( trans, **kwds ) ) - # Produce panel. - rval = [] - kwargs = dict( - trans=trans, - link_details=True - ) - for elt in panel_elts: - rval.append( elt.to_dict( **kwargs ) ) - else: - tools = [] - for id, tool in self._tools_by_id.items(): - tools.append( tool.to_dict( trans, link_details=True ) ) - rval = tools - - return rval - - def _lineage_in_panel( self, panel_dict, tool=None, tool_lineage=None ): - """ If tool with same lineage already in panel (or section) - find - and return it. Otherwise return None. - """ - if tool_lineage is None: - assert tool is not None - if not hasattr( tool, "lineage" ): - return None - tool_lineage = tool.lineage - lineage_tool_versions = tool_lineage.get_versions( reverse=True ) - for lineage_tool_version in lineage_tool_versions: - lineage_tool = self._tool_from_lineage_version( lineage_tool_version ) - if lineage_tool: - lineage_id = lineage_tool.id - if panel_dict.has_tool_with_id( lineage_id ): - return panel_dict.get_tool_with_id( lineage_id ) - return None - - def _newer_tool( self, tool1, tool2 ): - """ Return True if tool1 is considered "newer" given its own lineage - description. - """ - if not hasattr( tool1, "lineage" ): - return True - lineage_tool_versions = tool1.lineage.get_versions() - for lineage_tool_version in lineage_tool_versions: - lineage_tool = self._tool_from_lineage_version( lineage_tool_version ) - if lineage_tool is tool1: - return False - if lineage_tool is tool2: - return True - return True - - def _tool_from_lineage_version( self, lineage_tool_version ): - if lineage_tool_version.id_based: - return self._tools_by_id.get( lineage_tool_version.id, None ) - else: - return self._tool_versions_by_id.get( lineage_tool_version.id, {} ).get( lineage_tool_version.version, None ) - - -def _filter_for_panel( item, item_type, filters, context ): - """ - Filters tool panel elements so that only those that are compatible - with provided filters are kept. - """ - def _apply_filter( filter_item, filter_list ): - for filter_method in filter_list: - if not filter_method( context, filter_item ): - return False - return True - if item_type == panel_item_types.TOOL: - if _apply_filter( item, filters[ 'tool' ] ): - return item - elif item_type == panel_item_types.LABEL: - if _apply_filter( item, filters[ 'label' ] ): - return item - elif item_type == panel_item_types.SECTION: - # Filter section item-by-item. Only show a label if there are - # non-filtered tools below it. - - if _apply_filter( item, filters[ 'section' ] ): - cur_label_key = None - tools_under_label = False - filtered_elems = item.elems.copy() - for key, section_item_type, section_item in item.panel_items_iter(): - if section_item_type == panel_item_types.TOOL: - # Filter tool. - if _apply_filter( section_item, filters[ 'tool' ] ): - tools_under_label = True - else: - del filtered_elems[ key ] - elif section_item_type == panel_item_types.LABEL: - # If there is a label and it does not have tools, - # remove it. - if ( cur_label_key and not tools_under_label ) or not _apply_filter( section_item, filters[ 'label' ] ): - del filtered_elems[ cur_label_key ] - - # Reset attributes for new label. - cur_label_key = key - tools_under_label = False - - # Handle last label. - if cur_label_key and not tools_under_label: - del filtered_elems[ cur_label_key ] - - # Only return section if there are elements. - if len( filtered_elems ) != 0: - copy = item.copy() - copy.elems = filtered_elems - return copy - - return None - class DefaultToolState( object ): """ diff -r 24aa2e802a351882695cb98d772001cc2a1316d8 -r edacabfca615542ca435a97fc1afcc99daa470d5 lib/galaxy/tools/toolbox/__init__.py --- a/lib/galaxy/tools/toolbox/__init__.py +++ b/lib/galaxy/tools/toolbox/__init__.py @@ -1,16 +1,15 @@ """ API for this module containing functionality related to the tool box. """ -from .tags import tool_tag_manager -from .panel import ToolPanelElements +from .panel import panel_item_types from .panel import ToolSection from .panel import ToolSectionLabel -from .panel import panel_item_types + +from .base import AbstractToolBox __all__ = [ - "ToolPanelElements", - "tool_tag_manager", "ToolSection", "ToolSectionLabel", - "panel_item_types" + "panel_item_types", + "AbstractToolBox", ] This diff is so big that we needed to truncate the remainder. https://bitbucket.org/galaxy/galaxy-central/commits/cb673f24cd75/ Changeset: cb673f24cd75 User: jmchilton Date: 2014-12-31 23:21:10+00:00 Summary: Rework galaxy.tools' galaxy.jobs import. This version of the import I believe is less likely to cause circular import problems (http://effbot.org/zone/import-confusion.htm) Affected #: 1 file diff -r edacabfca615542ca435a97fc1afcc99daa470d5 -r cb673f24cd75ef491aa850373704535691b0a9dd lib/galaxy/tools/__init__.py --- a/lib/galaxy/tools/__init__.py +++ b/lib/galaxy/tools/__init__.py @@ -25,7 +25,7 @@ from mako.template import Template from paste import httpexceptions -from galaxy import jobs, model +from galaxy import model from galaxy.datatypes.metadata import JobExternalOutputMetadataWrapper from galaxy import exceptions from galaxy.tools.actions import DefaultToolAction @@ -69,6 +69,7 @@ DatasetListWrapper, DatasetCollectionWrapper, ) +import galaxy.jobs log = logging.getLogger( __name__ ) @@ -2127,7 +2128,7 @@ json_params = {} json_params[ 'param_dict' ] = self._prepare_json_param_dict( param_dict ) # it would probably be better to store the original incoming parameters here, instead of the Galaxy modified ones? json_params[ 'output_data' ] = [] - json_params[ 'job_config' ] = dict( GALAXY_DATATYPES_CONF_FILE=param_dict.get( 'GALAXY_DATATYPES_CONF_FILE' ), GALAXY_ROOT_DIR=param_dict.get( 'GALAXY_ROOT_DIR' ), TOOL_PROVIDED_JOB_METADATA_FILE=jobs.TOOL_PROVIDED_JOB_METADATA_FILE ) + json_params[ 'job_config' ] = dict( GALAXY_DATATYPES_CONF_FILE=param_dict.get( 'GALAXY_DATATYPES_CONF_FILE' ), GALAXY_ROOT_DIR=param_dict.get( 'GALAXY_ROOT_DIR' ), TOOL_PROVIDED_JOB_METADATA_FILE=galaxy.jobs.TOOL_PROVIDED_JOB_METADATA_FILE ) json_filename = None for i, ( out_name, data ) in enumerate( out_data.iteritems() ): #use wrapped dataset to access certain values @@ -2177,7 +2178,7 @@ json_params = {} json_params[ 'param_dict' ] = self._prepare_json_param_dict( param_dict ) # it would probably be better to store the original incoming parameters here, instead of the Galaxy modified ones? json_params[ 'output_data' ] = [] - json_params[ 'job_config' ] = dict( GALAXY_DATATYPES_CONF_FILE=param_dict.get( 'GALAXY_DATATYPES_CONF_FILE' ), GALAXY_ROOT_DIR=param_dict.get( 'GALAXY_ROOT_DIR' ), TOOL_PROVIDED_JOB_METADATA_FILE=jobs.TOOL_PROVIDED_JOB_METADATA_FILE ) + json_params[ 'job_config' ] = dict( GALAXY_DATATYPES_CONF_FILE=param_dict.get( 'GALAXY_DATATYPES_CONF_FILE' ), GALAXY_ROOT_DIR=param_dict.get( 'GALAXY_ROOT_DIR' ), TOOL_PROVIDED_JOB_METADATA_FILE=galaxy.jobs.TOOL_PROVIDED_JOB_METADATA_FILE ) json_filename = None for i, ( out_name, data ) in enumerate( out_data.iteritems() ): #use wrapped dataset to access certain values https://bitbucket.org/galaxy/galaxy-central/commits/bb548455022c/ Changeset: bb548455022c User: jmchilton Date: 2014-12-31 23:21:10+00:00 Summary: Mostly back out of 54b7789. It changes the behavior in a subtle way under error conditions. Previously if the on-database config file became out of sync with the tool shed database - Galaxy would loose lineage information (tools would be duplicated in side panel) but all tools from the config file would still be loaded up. This change preventing all tools from loading at all in those scenarios so I am undoing it. Affected #: 1 file diff -r cb673f24cd75ef491aa850373704535691b0a9dd -r bb548455022c7ad21c74ba61b4fe37b606737db7 lib/galaxy/tools/toolbox/base.py --- a/lib/galaxy/tools/toolbox/base.py +++ b/lib/galaxy/tools/toolbox/base.py @@ -521,23 +521,6 @@ def tools( self ): return self._tools_by_id.iteritems() - def __elem_to_tool_shed_repository( self, elem ): - # The tool is contained in an installed tool shed repository, so load - # the tool only if the repository has not been marked deleted. - tool_shed = elem.find( "tool_shed" ).text - repository_name = elem.find( "repository_name" ).text - repository_owner = elem.find( "repository_owner" ).text - installed_changeset_revision_elem = elem.find( "installed_changeset_revision" ) - if installed_changeset_revision_elem is None: - # Backward compatibility issue - the tag used to be named 'changeset_revision'. - installed_changeset_revision_elem = elem.find( "changeset_revision" ) - installed_changeset_revision = installed_changeset_revision_elem.text - tool_shed_repository = self.__get_tool_shed_repository( tool_shed, - repository_name, - repository_owner, - installed_changeset_revision ) - return tool_shed_repository - def __get_tool_shed_repository( self, tool_shed, name, owner, installed_changeset_revision ): # We store only the port, if one exists, in the database. tool_shed = common_util.remove_protocol_from_tool_shed_url( tool_shed ) @@ -662,7 +645,19 @@ if guid is not None: # The tool is contained in an installed tool shed repository, so load # the tool only if the repository has not been marked deleted. - tool_shed_repository = self.__elem_to_tool_shed_repository( elem ) + tool_shed = elem.find( "tool_shed" ).text + repository_name = elem.find( "repository_name" ).text + repository_owner = elem.find( "repository_owner" ).text + installed_changeset_revision_elem = elem.find( "installed_changeset_revision" ) + if installed_changeset_revision_elem is None: + # Backward compatibility issue - the tag used to be named 'changeset_revision'. + installed_changeset_revision_elem = elem.find( "changeset_revision" ) + installed_changeset_revision = installed_changeset_revision_elem.text + tool_shed_repository = self.__get_tool_shed_repository( tool_shed, + repository_name, + repository_owner, + installed_changeset_revision ) + if tool_shed_repository: # Only load tools if the repository is not deactivated or uninstalled. can_load_into_panel_dict = not tool_shed_repository.deleted @@ -675,10 +670,10 @@ key = 'tool_%s' % str( tool.id ) if can_load_into_panel_dict: if guid is not None: - tool.tool_shed = tool_shed_repository.tool_shed - tool.repository_name = tool_shed_repository.name - tool.repository_owner = tool_shed_repository.owner - tool.installed_changeset_revision = tool_shed_repository.installed_changeset_revision + tool.tool_shed = tool_shed + tool.repository_name = repository_name + tool.repository_owner = repository_owner + tool.installed_changeset_revision = installed_changeset_revision tool.guid = guid tool.version = elem.find( "version" ).text # Make sure the tool has a tool_version. Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: jmchilton: Bugfix for functional tests broken with 68daf6d.
by commits-noreply＠bitbucket.org 30 Dec '14

30 Dec '14

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/02efb1123d46/ Changeset: 02efb1123d46 User: jmchilton Date: 2014-12-30 14:43:28+00:00 Summary: Bugfix for functional tests broken with 68daf6d. Affected #: 1 file diff -r 87825915116972a38b2f0d18943ef2959d674f97 -r 02efb1123d469986a051e7a8015c9d56635efd8a lib/galaxy/tools/__init__.py --- a/lib/galaxy/tools/__init__.py +++ b/lib/galaxy/tools/__init__.py @@ -573,6 +573,11 @@ return [ tool ] return [] + @property + def tools_by_id( self ): + # Deprecated method, TODO - eliminate calls to this in test/. + return self._tools_by_id + def tools( self ): return self._tools_by_id.iteritems() Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: 5 new changesets
by commits-noreply＠bitbucket.org 29 Dec '14

29 Dec '14

5 new commits in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/d62272c55cf4/ Changeset: d62272c55cf4 User: jmchilton Date: 2014-12-30 04:07:01+00:00 Summary: Override ToolSectionLabel.to_dict so don't need to_dict_helper. I feel this is cleaner and it makes ToolSection and ToolSectionLabel easier to move into a new module. Affected #: 1 file diff -r fa404b077f67ff492db5d8c3405ea57cdb2d1420 -r d62272c55cf4d99fcb5f22111de7a4b5e104190e lib/galaxy/tools/__init__.py --- a/lib/galaxy/tools/__init__.py +++ b/lib/galaxy/tools/__init__.py @@ -113,16 +113,6 @@ pass -def to_dict_helper( obj, kwargs ): - """ Helper function that provides the appropriate kwargs to to_dict an object. """ - - # Label.to_dict cannot have kwargs. - if isinstance( obj, ToolSectionLabel ): - kwargs = {} - - return obj.to_dict( **kwargs ) - - class ToolBox( object, Dictifiable ): """Container for a collection of tools""" @@ -1189,7 +1179,7 @@ link_details=True ) for elt in panel_elts: - rval.append( to_dict_helper( elt, kwargs ) ) + rval.append( elt.to_dict( **kwargs ) ) else: tools = [] for id, tool in self._tools_by_id.items(): @@ -1311,7 +1301,7 @@ link_details=link_details ) for elt in self.elems.values(): - section_elts.append( to_dict_helper( elt, kwargs ) ) + section_elts.append( elt.to_dict( **kwargs ) ) section_dict[ 'elems' ] = section_elts return section_dict @@ -1333,6 +1323,9 @@ self.id = elem.get( "id" ) self.version = elem.get( "version" ) or '' + def to_dict( self, **kwds ): + return super( ToolSectionLabel, self ).to_dict() + class DefaultToolState( object ): """ https://bitbucket.org/galaxy/galaxy-central/commits/7b64cb79e2b0/ Changeset: 7b64cb79e2b0 User: jmchilton Date: 2014-12-30 04:07:01+00:00 Summary: Various ToolBox filtering improvements and tests. - Allow overriding the base module location for ToolBox filters (needed for OS packages, etc...). - Allow scanning multiple base modules for filters. - Unit tests for module loading functionality, custom tool, label, and section filters, default hidden and require_login filters. Affected #: 5 files diff -r d62272c55cf4d99fcb5f22111de7a4b5e104190e -r 7b64cb79e2b0fc3a7bd2366ca8af6f140ed78a08 lib/galaxy/config.py --- a/lib/galaxy/config.py +++ b/lib/galaxy/config.py @@ -83,6 +83,7 @@ # The value of migrated_tools_config is the file reserved for containing only those tools that have been eliminated from the distribution # and moved to the tool shed. self.integrated_tool_panel_config = resolve_path( kwargs.get( 'integrated_tool_panel_config', 'integrated_tool_panel.xml' ), self.root ) + self.toolbox_filter_base_modules = listify( kwargs.get( "toolbox_filter_base_modules", "galaxy.tools.filters" ) ) self.tool_filters = listify( kwargs.get( "tool_filters", [] ), do_strip=True ) self.tool_label_filters = listify( kwargs.get( "tool_label_filters", [] ), do_strip=True ) self.tool_section_filters = listify( kwargs.get( "tool_section_filters", [] ), do_strip=True ) diff -r d62272c55cf4d99fcb5f22111de7a4b5e104190e -r 7b64cb79e2b0fc3a7bd2366ca8af6f140ed78a08 lib/galaxy/tools/filters/__init__.py --- a/lib/galaxy/tools/filters/__init__.py +++ b/lib/galaxy/tools/filters/__init__.py @@ -1,6 +1,8 @@ +from copy import deepcopy +import sys import logging + from galaxy.util import listify -from copy import deepcopy log = logging.getLogger( __name__ ) @@ -20,6 +22,7 @@ self.default_filters = dict( tool=[ _not_hidden, _handle_requires_login ], section=[], label=[] ) # Add dynamic filters to these default filters. config = toolbox.app.config + self.__base_modules = listify( getattr( config, "toolbox_filter_base_modules", "galaxy.tools.filters" ) ) self.__init_filters( "tool", getattr( config, "tool_filters", "" ), self.default_filters ) self.__init_filters( "section", getattr( config, "tool_section_filters", "" ), self.default_filters ) self.__init_filters( "label", getattr( config, "tool_label_filters", "" ), self.default_filters ) @@ -65,14 +68,27 @@ if ":" in filter_name: # Should be a submodule of filters (e.g. examples:restrict_development_tools) (module_name, function_name) = filter_name.rsplit(":", 1) - module = __import__( module_name.strip(), globals() ) - function = getattr( module, function_name.strip() ) + function = self._import_filter( module_name, function_name ) else: # No module found, just load a function from this file or # one that has be explicitly imported. function = getattr( globals(), filter_name.strip() ) return function + def _import_filter( self, module_name, function_name ): + function_name = function_name.strip() + for base_module in self.__base_modules: + full_module_name = "%s.%s" % ( base_module, module_name.strip() ) + try: + __import__( full_module_name ) + except ImportError: + # log.debug("Failed to load module.", exc_info=True) + continue + module = sys.modules[ full_module_name ] + if hasattr( module, function_name ): + return getattr( module, function_name ) + raise Exception("Failed to find filter %s.%s" % (module_name, function_name)) + ## Stock Filter Functions def _not_hidden( context, tool ): diff -r d62272c55cf4d99fcb5f22111de7a4b5e104190e -r 7b64cb79e2b0fc3a7bd2366ca8af6f140ed78a08 test/unit/tools/filter_modules/filtermod.py --- /dev/null +++ b/test/unit/tools/filter_modules/filtermod.py @@ -0,0 +1,22 @@ +""" Test filters used by test_toolbox_filters.py. +""" + + +def filter_tool( context, tool ): + """Test Filter Tool""" + return False + + +def filter_section( context, section ): + """Test Filter Section""" + return False + + +def filter_label_1( context, label ): + """Test Filter Label 1""" + return False + + +def filter_label_2( context, label ): + """Test Filter Label 2""" + return False diff -r d62272c55cf4d99fcb5f22111de7a4b5e104190e -r 7b64cb79e2b0fc3a7bd2366ca8af6f140ed78a08 test/unit/tools/test_toolbox_filters.py --- /dev/null +++ b/test/unit/tools/test_toolbox_filters.py @@ -0,0 +1,79 @@ +from galaxy.util.bunch import Bunch + +from galaxy.tools.filters import FilterFactory + + +def test_stock_filtering_requires_login_tools( ): + anonymous_user_trans = mock_trans( has_user=False ) + filters = filter_factory( {} ).build_filters( mock_trans() )[ 'tool' ] + assert not is_filtered( filters, anonymous_user_trans, mock_tool( require_login=False ) ) + assert is_filtered( filters, anonymous_user_trans, mock_tool( require_login=True ) ) + + logged_in_trans = mock_trans( has_user=True ) + filters = filter_factory( {} ).build_filters( logged_in_trans )[ 'tool' ] + assert not is_filtered( filters, logged_in_trans, mock_tool( require_login=True ) ) + + +def test_stock_filtering_hidden_tools( ): + filters = filter_factory( {} ).build_filters( mock_trans() )[ 'tool' ] + assert not is_filtered( filters, mock_trans(), mock_tool( hidden=False ) ) + assert is_filtered( filters, mock_trans(), mock_tool( hidden=True ) ) + + +def test_trackster_filtering( ): + filters = filter_factory( {} ).build_filters( mock_trans(), trackster=True )[ 'tool' ] + # Ekkk... is trackster_conf broken? Why is it showing up. + # assert is_filtered( filters, mock_trans(), mock_tool( trackster_conf=False ) ) + assert not is_filtered( filters, mock_trans(), mock_tool( trackster_conf=True ) ) + + +def test_custom_filters(): + filters = filter_factory().build_filters( mock_trans() ) + tool_filters = filters[ "tool" ] + # TODO: the fact that -1 is the custom filter is an implementation + # detail that should not be tested here. + assert tool_filters[ -1 ].__doc__ == "Test Filter Tool" + + section_filters = filters[ "section" ] + assert section_filters[ 0 ].__doc__ == "Test Filter Section" + + label_filters = filters[ "label" ] + assert label_filters[ 0 ].__doc__ == "Test Filter Label 1" + assert label_filters[ 1 ].__doc__ == "Test Filter Label 2" + + +def filter_factory(config_dict=None): + if config_dict is None: + config_dict = dict( + tool_filters=["filtermod:filter_tool"], + tool_section_filters=["filtermod:filter_section"], + tool_label_filters=["filtermod:filter_label_1", "filtermod:filter_label_2"], + ) + config = Bunch( **config_dict ) + config.toolbox_filter_base_modules = "galaxy.tools.filters,tools.filter_modules" + app = Bunch(config=config) + toolbox = Bunch(app=app) + return FilterFactory(toolbox) + + +def is_filtered( filters, trans, tool ): + context = Bunch( trans=trans ) + return not all( map( lambda filter: filter( context, tool ), filters ) ) + + +def mock_tool( require_login=False, hidden=False, trackster_conf=False ): + tool = Bunch( + require_login=require_login, + hidden=hidden, + trackster_conf=trackster_conf, + ) + return tool + + +def mock_trans( has_user=True ): + trans = Bunch( ) + if has_user: + trans.user = Bunch(preferences={}) + else: + trans.user = None + return trans https://bitbucket.org/galaxy/galaxy-central/commits/4a96516a503c/ Changeset: 4a96516a503c User: jmchilton Date: 2014-12-30 04:07:01+00:00 Summary: Move galaxy.tools.watcher into galaxy.tools.toolbox.watcher. Affected #: 4 files diff -r 7b64cb79e2b0fc3a7bd2366ca8af6f140ed78a08 -r 4a96516a503cc3bfae4d26b431f8ad8c67496074 lib/galaxy/tools/__init__.py --- a/lib/galaxy/tools/__init__.py +++ b/lib/galaxy/tools/__init__.py @@ -32,7 +32,7 @@ from galaxy import jobs, model from galaxy.datatypes.metadata import JobExternalOutputMetadataWrapper from galaxy import exceptions -from galaxy.tools import watcher +from galaxy.tools.toolbox import watcher from galaxy.tools.actions import DefaultToolAction from galaxy.tools.actions.upload import UploadToolAction from galaxy.tools.actions.data_source import DataSourceToolAction diff -r 7b64cb79e2b0fc3a7bd2366ca8af6f140ed78a08 -r 4a96516a503cc3bfae4d26b431f8ad8c67496074 lib/galaxy/tools/toolbox/watcher.py --- /dev/null +++ b/lib/galaxy/tools/toolbox/watcher.py @@ -0,0 +1,99 @@ +import os.path +try: + from watchdog.events import FileSystemEventHandler + from watchdog.observers import Observer + can_watch = True +except ImportError: + FileSystemEventHandler = object + can_watch = False + +import logging +log = logging.getLogger( __name__ ) + + +def get_watcher(toolbox, config): + watch_tools = getattr(config, "watch_tools", False) + if watch_tools: + return ToolWatcher(toolbox) + else: + return NullWatcher() + + +class ToolWatcher(object): + + def __init__(self, toolbox): + if not can_watch: + raise Exception("Watchdog library unavailble, cannot watch tools.") + self.toolbox = toolbox + self.tool_file_ids = {} + self.tool_dir_callbacks = {} + self.monitored_dirs = {} + self.observer = Observer() + self.event_handler = ToolFileEventHandler(self) + self.start() + + def start(self): + self.observer.start() + + def monitor(self, dir): + self.observer.schedule(self.event_handler, dir, recursive=False) + + def watch_file(self, tool_file, tool_id): + tool_file = os.path.abspath( tool_file ) + self.tool_file_ids[tool_file] = tool_id + tool_dir = os.path.dirname( tool_file ) + if tool_dir not in self.monitored_dirs: + self.monitored_dirs[ tool_dir ] = tool_dir + self.monitor( tool_dir ) + + def watch_directory(self, tool_dir, callback): + tool_dir = os.path.abspath( tool_dir ) + self.tool_dir_callbacks[tool_dir] = callback + if tool_dir not in self.monitored_dirs: + self.monitored_dirs[ tool_dir ] = tool_dir + self.monitor( tool_dir ) + + +class ToolFileEventHandler(FileSystemEventHandler): + + def __init__(self, tool_watcher): + self.tool_watcher = tool_watcher + + def on_any_event(self, event): + self._handle(event) + + def _handle(self, event): + # modified events will only have src path, move events will + # have dest_path and src_path but we only care about dest. So + # look at dest if it exists else use src. + path = getattr( event, 'dest_path', None ) or event.src_path + path = os.path.abspath( path ) + tool_id = self.tool_watcher.tool_file_ids.get( path, None ) + if tool_id: + try: + self.tool_watcher.toolbox.reload_tool_by_id(tool_id) + except Exception: + pass + elif path.endswith(".xml"): + directory = os.path.dirname( path ) + dir_callback = self.tool_watcher.tool_dir_callbacks.get( directory, None ) + if dir_callback: + tool_file = event.src_path + tool_id = dir_callback( tool_file ) + if tool_id: + self.tool_watcher.tool_file_ids[ tool_file ] = tool_id + + +class NullWatcher(object): + + def start(self): + pass + + def shutdown(self): + pass + + def watch_file(self, tool_file, tool_id): + pass + + def watch_directory(self, tool_dir, callback): + pass diff -r 7b64cb79e2b0fc3a7bd2366ca8af6f140ed78a08 -r 4a96516a503cc3bfae4d26b431f8ad8c67496074 lib/galaxy/tools/watcher.py --- a/lib/galaxy/tools/watcher.py +++ /dev/null @@ -1,99 +0,0 @@ -import os.path -try: - from watchdog.events import FileSystemEventHandler - from watchdog.observers import Observer - can_watch = True -except ImportError: - FileSystemEventHandler = object - can_watch = False - -import logging -log = logging.getLogger( __name__ ) - - -def get_watcher(toolbox, config): - watch_tools = getattr(config, "watch_tools", False) - if watch_tools: - return ToolWatcher(toolbox) - else: - return NullWatcher() - - -class ToolWatcher(object): - - def __init__(self, toolbox): - if not can_watch: - raise Exception("Watchdog library unavailble, cannot watch tools.") - self.toolbox = toolbox - self.tool_file_ids = {} - self.tool_dir_callbacks = {} - self.monitored_dirs = {} - self.observer = Observer() - self.event_handler = ToolFileEventHandler(self) - self.start() - - def start(self): - self.observer.start() - - def monitor(self, dir): - self.observer.schedule(self.event_handler, dir, recursive=False) - - def watch_file(self, tool_file, tool_id): - tool_file = os.path.abspath( tool_file ) - self.tool_file_ids[tool_file] = tool_id - tool_dir = os.path.dirname( tool_file ) - if tool_dir not in self.monitored_dirs: - self.monitored_dirs[ tool_dir ] = tool_dir - self.monitor( tool_dir ) - - def watch_directory(self, tool_dir, callback): - tool_dir = os.path.abspath( tool_dir ) - self.tool_dir_callbacks[tool_dir] = callback - if tool_dir not in self.monitored_dirs: - self.monitored_dirs[ tool_dir ] = tool_dir - self.monitor( tool_dir ) - - -class ToolFileEventHandler(FileSystemEventHandler): - - def __init__(self, tool_watcher): - self.tool_watcher = tool_watcher - - def on_any_event(self, event): - self._handle(event) - - def _handle(self, event): - # modified events will only have src path, move events will - # have dest_path and src_path but we only care about dest. So - # look at dest if it exists else use src. - path = getattr( event, 'dest_path', None ) or event.src_path - path = os.path.abspath( path ) - tool_id = self.tool_watcher.tool_file_ids.get( path, None ) - if tool_id: - try: - self.tool_watcher.toolbox.reload_tool_by_id(tool_id) - except Exception: - pass - elif path.endswith(".xml"): - directory = os.path.dirname( path ) - dir_callback = self.tool_watcher.tool_dir_callbacks.get( directory, None ) - if dir_callback: - tool_file = event.src_path - tool_id = dir_callback( tool_file ) - if tool_id: - self.tool_watcher.tool_file_ids[ tool_file ] = tool_id - - -class NullWatcher(object): - - def start(self): - pass - - def shutdown(self): - pass - - def watch_file(self, tool_file, tool_id): - pass - - def watch_directory(self, tool_dir, callback): - pass diff -r 7b64cb79e2b0fc3a7bd2366ca8af6f140ed78a08 -r 4a96516a503cc3bfae4d26b431f8ad8c67496074 test/unit/tools/test_watcher.py --- a/test/unit/tools/test_watcher.py +++ b/test/unit/tools/test_watcher.py @@ -4,7 +4,7 @@ import tempfile import time -from galaxy.tools import watcher +from galaxy.tools.toolbox import watcher from galaxy.util import bunch https://bitbucket.org/galaxy/galaxy-central/commits/507483359a67/ Changeset: 507483359a67 User: jmchilton Date: 2014-12-30 04:07:01+00:00 Summary: Remove lib/galaxy/tools/filters/examples.py likely incorrectly added in d816d66. Affected #: 1 file diff -r 4a96516a503cc3bfae4d26b431f8ad8c67496074 -r 507483359a67da800ce7a56536d31b7911211474 lib/galaxy/tools/filters/examples.py --- a/lib/galaxy/tools/filters/examples.py +++ /dev/null @@ -1,115 +0,0 @@ -import logging -log = logging.getLogger( __name__ ) - - -def restrict_encode( content, tool ): - """ - Disable the random interval ENCODE tool - - This tool filter will disable all the ENCODE tool when enabled. - """ - if tool.id == 'random_intervals1': - return False - return True - - -def restrict_text( content, section ): - """ - Disable Text sections - - This tool filter will disable all Tools groups under a 'Text' section when enabled. - """ - if section.name.find('Text') != -1: - return False - return True - - -def restrict_upload_to_admins( context, tool ): - """ - Disable Upload tool for all non-admin users. - - This tool filter will hide the upload tool from all users except admin - users. - - This can be enabled by renaming this file to examples.py and adding - the following to the ``app:main`` section of ``galaxy.ini``: - - tool_filters = examples:restrict_upload_to_admins - """ - if tool.name == "Upload File": - return context.trans.user_is_admin() - return True - - -def disable_gatk( context, tool ): - """ - This tool filter will disable all gatk tools when enabled. - - This can be enabled by renaming this file to examples.py and adding the following to the - ``app:main`` section of ``galaxy.ini``: - - tool_filters = examples:disable_gatk - """ - return not any( [ requirement.name == "gatk" for requirement in tool.requirements ] ) - - -def explicit_user_mapping( context, section ): - """ - This tool section filter uses an explicit mapping to describe what users can view - which tool sections. Anonymous users will only be able to view the "Get Data" - tool section (with id getext). This can be enabled by renaming this file to - examples.py and adding the following to the ``app:main`` section of - ``galaxy.ini``: - - tool_section_filters = examples:explicit_user_mapping - """ - users_sections = { - None: [ "getext" ], - "bob(a)example.com": [ "getext", "textutil", "filter" ], - "mary(a)example.com": [ "getext", "textutil", "filter", "ngs" ], - } - user = context.trans.user - email = user and user.email - valid_sections = users_sections.get( email, [] ) - return section.id in valid_sections - - -DEVELOPERS = [ "mary(a)example.com" ] - - -def restrict_development_tools( context, tool ): - """ - This tool filter will disable all tools with the string alpha appearing in - the version for all users except those explicitly appearing in the DEVELOPERS list - defined above. This can be enabled by renaming this file to examples.py and - adding the following to the ``app:main`` section of ``galaxy.ini``: - - tool_filters = examples:restrict_development_tools - """ - version = tool.version - user = context.trans.user - email = user and user.email - return "alpha" not in version or email in DEVELOPERS - - -def per_host_tool_sections( context, section ): - """ - This tool section filter results in different sections being display based on - the URL the user is making the request to. This could allow a single Galaxy instance - to seem like several different instances hosting different tools based on the URL used - to access the Galxy. This can be enabled by renaming this file to examples.py and adding - the following to the ``app:main`` section of ``galaxy.ini``: - - tool_section_filters = examples:per_host_tool_sections - """ - host = context.trans.request.host - # Core tools used by all virtual hosts. - valid_sections = [ "getext", "textutil", "filter" ] - if "ngs.galaxy.example.com" in host: - valid_sections += [ "ngs" ] - elif "microarray.galaxy.example.com" in host: - valid_sections += [ "microarray" ] - elif "proteomics.galaxy.example.com" in host: - valid_sections += [ "proteomics" ] - return section.id in valid_sections - https://bitbucket.org/galaxy/galaxy-central/commits/878259151169/ Changeset: 878259151169 User: jmchilton Date: 2014-12-30 04:07:01+00:00 Summary: Move galaxy.tools.filters to galaxy.tools.toolbox.filters. Leave empty module in galaxy.tools.filters and modify config to ensure backward compatibility (filters in this old directory will continue to work for now). Update config/galaxy.ini.sample with more discussion of ToolBox filter. Affected #: 8 files diff -r 507483359a67da800ce7a56536d31b7911211474 -r 87825915116972a38b2f0d18943ef2959d674f97 config/galaxy.ini.sample --- a/config/galaxy.ini.sample +++ b/config/galaxy.ini.sample @@ -910,17 +910,27 @@ #cache_user_job_count = False # ToolBox filtering -# Modules from lib/galaxy/tools/filters/ can be specified in the following lines. -# tool_* filters will be applied for all users and can not be changed by them. -# user_tool_* filters will be shown under user preferences and can be toogled -# on and off by runtime -#tool_filters -#tool_label_filters -#tool_section_filters + +# Modules from lib/galaxy/tools/toolbox/filters/ can be specified in +# the following lines. tool_* filters will be applied for all users +# and can not be changed by them. user_tool_* filters will be shown +# under user preferences and can be toogled on and off by +# runtime. Example shown below are not real defaults (no custom +# filters applied by defualt) but can be enabled with by renaming the +# example.py.sample in the filters directory to example.py. + +#tool_filters = +#tool_label_filters = +#tool_section_filters = #user_tool_filters = examples:restrict_upload_to_admins, examples:restrict_encode #user_tool_section_filters = examples:restrict_text #user_tool_label_filters = examples:restrict_upload_to_admins, examples:restrict_encode +# The base modules that are searched for modules as described above +# can be modified and modules external to Galaxy can be searched by +# modifying the following option. +#toolbox_filter_base_modules = galaxy.tools.toolbox.filters,galaxy.tools.filters + # Galaxy Application Internal Message Queue # Galaxy uses AMQP internally TODO more documentation on what for. diff -r 507483359a67da800ce7a56536d31b7911211474 -r 87825915116972a38b2f0d18943ef2959d674f97 lib/galaxy/config.py --- a/lib/galaxy/config.py +++ b/lib/galaxy/config.py @@ -83,7 +83,7 @@ # The value of migrated_tools_config is the file reserved for containing only those tools that have been eliminated from the distribution # and moved to the tool shed. self.integrated_tool_panel_config = resolve_path( kwargs.get( 'integrated_tool_panel_config', 'integrated_tool_panel.xml' ), self.root ) - self.toolbox_filter_base_modules = listify( kwargs.get( "toolbox_filter_base_modules", "galaxy.tools.filters" ) ) + self.toolbox_filter_base_modules = listify( kwargs.get( "toolbox_filter_base_modules", "galaxy.tools.filters,galaxy.tools.toolbox.filters" ) ) self.tool_filters = listify( kwargs.get( "tool_filters", [] ), do_strip=True ) self.tool_label_filters = listify( kwargs.get( "tool_label_filters", [] ), do_strip=True ) self.tool_section_filters = listify( kwargs.get( "tool_section_filters", [] ), do_strip=True ) diff -r 507483359a67da800ce7a56536d31b7911211474 -r 87825915116972a38b2f0d18943ef2959d674f97 lib/galaxy/tools/__init__.py --- a/lib/galaxy/tools/__init__.py +++ b/lib/galaxy/tools/__init__.py @@ -48,7 +48,7 @@ from galaxy.tools.parameters.input_translation import ToolInputTranslator from galaxy.tools.parameters.output import ToolOutputActionGroup from galaxy.tools.parameters.validation import LateValidationError -from galaxy.tools.filters import FilterFactory +from galaxy.tools.toolbox.filters import FilterFactory from galaxy.tools.test import parse_tests from galaxy.tools.parser import get_tool_source from galaxy.tools.parser.xml import XmlPageSource diff -r 507483359a67da800ce7a56536d31b7911211474 -r 87825915116972a38b2f0d18943ef2959d674f97 lib/galaxy/tools/filters/__init__.py --- a/lib/galaxy/tools/filters/__init__.py +++ b/lib/galaxy/tools/filters/__init__.py @@ -1,103 +1,5 @@ -from copy import deepcopy -import sys -import logging +"""Deprecated module for toolbox filters. -from galaxy.util import listify - -log = logging.getLogger( __name__ ) - - -class FilterFactory( object ): - """ - An instance of this class is responsible for filtering the list - of tools presented to a given user in a given context. - """ - - def __init__( self, toolbox ): - self.toolbox = toolbox - - # Prepopulate dict containing filters that are always checked, - # other filters that get checked depending on context (e.g. coming from - # trackster or no user found are added in build filters). - self.default_filters = dict( tool=[ _not_hidden, _handle_requires_login ], section=[], label=[] ) - # Add dynamic filters to these default filters. - config = toolbox.app.config - self.__base_modules = listify( getattr( config, "toolbox_filter_base_modules", "galaxy.tools.filters" ) ) - self.__init_filters( "tool", getattr( config, "tool_filters", "" ), self.default_filters ) - self.__init_filters( "section", getattr( config, "tool_section_filters", "" ), self.default_filters ) - self.__init_filters( "label", getattr( config, "tool_label_filters", "" ), self.default_filters ) - - def build_filters( self, trans, **kwds ): - """ - Build list of filters to check tools against given current context. - """ - filters = deepcopy( self.default_filters ) - if trans.user: - for name, value in trans.user.preferences.items(): - if value.strip(): - user_filters = listify( value, do_strip=True ) - category = '' - if name == 'toolbox_tool_filters': - category = "tool" - elif name == 'toolbox_section_filters': - category = "section" - elif name == 'toolbox_label_filters': - category = "label" - if category: - validate = getattr( trans.app.config, 'user_%s_filters' % category, [] ) - self.__init_filters( category, user_filters, filters, validate=validate ) - else: - if kwds.get( "trackster", False ): - filters[ "tool" ].append( _has_trackster_conf ) - - return filters - - def __init_filters( self, key, filters, toolbox_filters, validate=None ): - for filter in filters: - if validate is None or filter in validate or filter in self.default_filters: - filter_function = self.__build_filter_function( filter ) - toolbox_filters[ key ].append( filter_function ) - else: - log.warning( "Refusing to load %s filter '%s' which is not defined in config", key, filter ) - return toolbox_filters - - def __build_filter_function( self, filter_name ): - """Obtain python function (importing a submodule if needed) - corresponding to filter_name. - """ - if ":" in filter_name: - # Should be a submodule of filters (e.g. examples:restrict_development_tools) - (module_name, function_name) = filter_name.rsplit(":", 1) - function = self._import_filter( module_name, function_name ) - else: - # No module found, just load a function from this file or - # one that has be explicitly imported. - function = getattr( globals(), filter_name.strip() ) - return function - - def _import_filter( self, module_name, function_name ): - function_name = function_name.strip() - for base_module in self.__base_modules: - full_module_name = "%s.%s" % ( base_module, module_name.strip() ) - try: - __import__( full_module_name ) - except ImportError: - # log.debug("Failed to load module.", exc_info=True) - continue - module = sys.modules[ full_module_name ] - if hasattr( module, function_name ): - return getattr( module, function_name ) - raise Exception("Failed to find filter %s.%s" % (module_name, function_name)) - - -## Stock Filter Functions -def _not_hidden( context, tool ): - return not tool.hidden - - -def _handle_requires_login( context, tool ): - return not tool.require_login or context.trans.user - - -def _has_trackster_conf( context, tool ): - return tool.trackster_conf +Filters placed in this module will still work - but filters should be +moved to lib/galaxy/tools/toolbox/filters. +""" diff -r 507483359a67da800ce7a56536d31b7911211474 -r 87825915116972a38b2f0d18943ef2959d674f97 lib/galaxy/tools/filters/examples.py.sample --- a/lib/galaxy/tools/filters/examples.py.sample +++ /dev/null @@ -1,87 +0,0 @@ -import logging -log = logging.getLogger( __name__ ) - - -def restrict_upload_to_admins( context, tool ): - """ - This tool filter will hide the upload tool from all users except admin - users. This can be enabled by renaming this file to examples.py and adding - the following to the ``app:main`` section of ``galaxy.ini``: - - tool_filters = examples:restrict_upload_to_admins - """ - if tool.name == "Upload File": - return context.trans.user_is_admin() - return True - - -def disable_gatk( context, tool ): - """ - This tool filter will disable all gatk tools when enabled. This can be - enabled by renaming this file to examples.py and adding the following to the - ``app:main`` section of ``galaxy.ini``: - - tool_filters = examples:disable_gatk - """ - return not any( [ requirement.name == "gatk" for requirement in tool.requirements ] ) - - -def explicit_user_mapping( context, section ): - """ - This tool section filter uses an explicit mapping to describe what users can view - which tool sections. Anonymous users will only be able to view the "Get Data" - tool section (with id getext). This can be enabled by renaming this file to - examples.py and adding the following to the ``app:main`` section of - ``galaxy.ini``: - - tool_section_filters = examples:explicit_user_mapping - """ - users_sections = { - None: [ "getext" ], - "bob(a)example.com": [ "getext", "textutil", "filter" ], - "mary(a)example.com": [ "getext", "textutil", "filter", "ngs" ], - } - user = context.trans.user - email = user and user.email - valid_sections = users_sections.get( email, [] ) - return section.id in valid_sections - - -DEVELOPERS = [ "mary(a)example.com" ] - - -def restrict_development_tools( context, tool ): - """ - This tool filter will disable all tools with the string alpha appearing in - the version for all users except those explicitly appearing in the DEVELOPERS list - defined above. This can be enabled by renaming this file to examples.py and - adding the following to the ``app:main`` section of ``galaxy.ini``: - - tool_filters = examples:restrict_development_tools - """ - version = tool.version - user = context.trans.user - email = user and user.email - return "alpha" not in version or email in DEVELOPERS - - -def per_host_tool_sections( context, section ): - """ - This tool section filter results in different sections being display based on - the URL the user is making the request to. This could allow a single Galaxy instance - to seem like several different instances hosting different tools based on the URL used - to access the Galxy. This can be enabled by renaming this file to examples.py and adding - the following to the ``app:main`` section of ``galaxy.ini``: - - tool_section_filters = examples:per_host_tool_sections - """ - host = context.trans.request.host - # Core tools used by all virtual hosts. - valid_sections = [ "getext", "textutil", "filter" ] - if "ngs.galaxy.example.com" in host: - valid_sections += [ "ngs" ] - elif "microarray.galaxy.example.com" in host: - valid_sections += [ "microarray" ] - elif "proteomics.galaxy.example.com" in host: - valid_sections += [ "proteomics" ] - return section.id in valid_sections diff -r 507483359a67da800ce7a56536d31b7911211474 -r 87825915116972a38b2f0d18943ef2959d674f97 lib/galaxy/tools/toolbox/filters/__init__.py --- /dev/null +++ b/lib/galaxy/tools/toolbox/filters/__init__.py @@ -0,0 +1,103 @@ +from copy import deepcopy +import sys +import logging + +from galaxy.util import listify + +log = logging.getLogger( __name__ ) + + +class FilterFactory( object ): + """ + An instance of this class is responsible for filtering the list + of tools presented to a given user in a given context. + """ + + def __init__( self, toolbox ): + self.toolbox = toolbox + + # Prepopulate dict containing filters that are always checked, + # other filters that get checked depending on context (e.g. coming from + # trackster or no user found are added in build filters). + self.default_filters = dict( tool=[ _not_hidden, _handle_requires_login ], section=[], label=[] ) + # Add dynamic filters to these default filters. + config = toolbox.app.config + self.__base_modules = listify( getattr( config, "toolbox_filter_base_modules", "galaxy.tools.filters" ) ) + self.__init_filters( "tool", getattr( config, "tool_filters", "" ), self.default_filters ) + self.__init_filters( "section", getattr( config, "tool_section_filters", "" ), self.default_filters ) + self.__init_filters( "label", getattr( config, "tool_label_filters", "" ), self.default_filters ) + + def build_filters( self, trans, **kwds ): + """ + Build list of filters to check tools against given current context. + """ + filters = deepcopy( self.default_filters ) + if trans.user: + for name, value in trans.user.preferences.items(): + if value.strip(): + user_filters = listify( value, do_strip=True ) + category = '' + if name == 'toolbox_tool_filters': + category = "tool" + elif name == 'toolbox_section_filters': + category = "section" + elif name == 'toolbox_label_filters': + category = "label" + if category: + validate = getattr( trans.app.config, 'user_%s_filters' % category, [] ) + self.__init_filters( category, user_filters, filters, validate=validate ) + else: + if kwds.get( "trackster", False ): + filters[ "tool" ].append( _has_trackster_conf ) + + return filters + + def __init_filters( self, key, filters, toolbox_filters, validate=None ): + for filter in filters: + if validate is None or filter in validate or filter in self.default_filters: + filter_function = self.__build_filter_function( filter ) + toolbox_filters[ key ].append( filter_function ) + else: + log.warning( "Refusing to load %s filter '%s' which is not defined in config", key, filter ) + return toolbox_filters + + def __build_filter_function( self, filter_name ): + """Obtain python function (importing a submodule if needed) + corresponding to filter_name. + """ + if ":" in filter_name: + # Should be a submodule of filters (e.g. examples:restrict_development_tools) + (module_name, function_name) = filter_name.rsplit(":", 1) + function = self._import_filter( module_name, function_name ) + else: + # No module found, just load a function from this file or + # one that has be explicitly imported. + function = getattr( globals(), filter_name.strip() ) + return function + + def _import_filter( self, module_name, function_name ): + function_name = function_name.strip() + for base_module in self.__base_modules: + full_module_name = "%s.%s" % ( base_module, module_name.strip() ) + try: + __import__( full_module_name ) + except ImportError: + # log.debug("Failed to load module.", exc_info=True) + continue + module = sys.modules[ full_module_name ] + if hasattr( module, function_name ): + return getattr( module, function_name ) + raise Exception("Failed to find filter %s.%s" % (module_name, function_name)) + + +## Stock Filter Functions +def _not_hidden( context, tool ): + return not tool.hidden + + +def _handle_requires_login( context, tool ): + return not tool.require_login or context.trans.user + + +def _has_trackster_conf( context, tool ): + return tool.trackster_conf diff -r 507483359a67da800ce7a56536d31b7911211474 -r 87825915116972a38b2f0d18943ef2959d674f97 lib/galaxy/tools/toolbox/filters/examples.py.sample --- /dev/null +++ b/lib/galaxy/tools/toolbox/filters/examples.py.sample @@ -0,0 +1,87 @@ +import logging +log = logging.getLogger( __name__ ) + + +def restrict_upload_to_admins( context, tool ): + """ + This tool filter will hide the upload tool from all users except admin + users. This can be enabled by renaming this file to examples.py and adding + the following to the ``app:main`` section of ``galaxy.ini``: + + tool_filters = examples:restrict_upload_to_admins + """ + if tool.name == "Upload File": + return context.trans.user_is_admin() + return True + + +def disable_gatk( context, tool ): + """ + This tool filter will disable all gatk tools when enabled. This can be + enabled by renaming this file to examples.py and adding the following to the + ``app:main`` section of ``galaxy.ini``: + + tool_filters = examples:disable_gatk + """ + return not any( [ requirement.name == "gatk" for requirement in tool.requirements ] ) + + +def explicit_user_mapping( context, section ): + """ + This tool section filter uses an explicit mapping to describe what users can view + which tool sections. Anonymous users will only be able to view the "Get Data" + tool section (with id getext). This can be enabled by renaming this file to + examples.py and adding the following to the ``app:main`` section of + ``galaxy.ini``: + + tool_section_filters = examples:explicit_user_mapping + """ + users_sections = { + None: [ "getext" ], + "bob(a)example.com": [ "getext", "textutil", "filter" ], + "mary(a)example.com": [ "getext", "textutil", "filter", "ngs" ], + } + user = context.trans.user + email = user and user.email + valid_sections = users_sections.get( email, [] ) + return section.id in valid_sections + + +DEVELOPERS = [ "mary(a)example.com" ] + + +def restrict_development_tools( context, tool ): + """ + This tool filter will disable all tools with the string alpha appearing in + the version for all users except those explicitly appearing in the DEVELOPERS list + defined above. This can be enabled by renaming this file to examples.py and + adding the following to the ``app:main`` section of ``galaxy.ini``: + + tool_filters = examples:restrict_development_tools + """ + version = tool.version + user = context.trans.user + email = user and user.email + return "alpha" not in version or email in DEVELOPERS + + +def per_host_tool_sections( context, section ): + """ + This tool section filter results in different sections being display based on + the URL the user is making the request to. This could allow a single Galaxy instance + to seem like several different instances hosting different tools based on the URL used + to access the Galxy. This can be enabled by renaming this file to examples.py and adding + the following to the ``app:main`` section of ``galaxy.ini``: + + tool_section_filters = examples:per_host_tool_sections + """ + host = context.trans.request.host + # Core tools used by all virtual hosts. + valid_sections = [ "getext", "textutil", "filter" ] + if "ngs.galaxy.example.com" in host: + valid_sections += [ "ngs" ] + elif "microarray.galaxy.example.com" in host: + valid_sections += [ "microarray" ] + elif "proteomics.galaxy.example.com" in host: + valid_sections += [ "proteomics" ] + return section.id in valid_sections diff -r 507483359a67da800ce7a56536d31b7911211474 -r 87825915116972a38b2f0d18943ef2959d674f97 test/unit/tools/test_toolbox_filters.py --- a/test/unit/tools/test_toolbox_filters.py +++ b/test/unit/tools/test_toolbox_filters.py @@ -1,6 +1,6 @@ from galaxy.util.bunch import Bunch -from galaxy.tools.filters import FilterFactory +from galaxy.tools.toolbox.filters import FilterFactory def test_stock_filtering_requires_login_tools( ): Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: jmchilton: Bugfix for 267fa4b when tool tags are enabled.
by commits-noreply＠bitbucket.org 29 Dec '14

29 Dec '14

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/fa404b077f67/ Changeset: fa404b077f67 User: jmchilton Date: 2014-12-29 19:25:12+00:00 Summary: Bugfix for 267fa4b when tool tags are enabled. Thanks to JJ for the bug report - apparently at least one person has tool tags enabled :). Affected #: 1 file diff -r e84f0ccd8533d76d3ee3f5b7fa739f096e9aeb41 -r fa404b077f67ff492db5d8c3405ea57cdb2d1420 lib/galaxy/tools/toolbox/tags.py --- a/lib/galaxy/tools/toolbox/tags.py +++ b/lib/galaxy/tools/toolbox/tags.py @@ -44,7 +44,7 @@ def __init__( self, app ): self.app = app - self.sa_session = app.sa_session + self.sa_session = app.model.context def reset_tags( self ): log.info("removing all tool tag associations (" + str( self.sa_session.query( self.app.model.ToolTagAssociation ).count() ) + ")" ) Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: guerler: Add classic hidden_data field to tool form
by commits-noreply＠bitbucket.org 29 Dec '14

29 Dec '14

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/e84f0ccd8533/ Changeset: e84f0ccd8533 User: guerler Date: 2014-12-29 19:08:35+00:00 Summary: Add classic hidden_data field to tool form Affected #: 3 files diff -r dc14223818bd8b8a908e056f361f2be1e8972d91 -r e84f0ccd8533d76d3ee3f5b7fa739f096e9aeb41 client/galaxy/scripts/mvc/tools/tools-section.js --- a/client/galaxy/scripts/mvc/tools/tools-section.js +++ b/client/galaxy/scripts/mvc/tools/tools-section.js @@ -316,6 +316,11 @@ field = this._fieldHidden(input_def); break; + // hidden field + case 'hidden_data': + field = this._fieldHidden(input_def); + break; + // integer field case 'integer': field = this._fieldSlider(input_def); diff -r dc14223818bd8b8a908e056f361f2be1e8972d91 -r e84f0ccd8533d76d3ee3f5b7fa739f096e9aeb41 static/scripts/mvc/tools/tools-section.js --- a/static/scripts/mvc/tools/tools-section.js +++ b/static/scripts/mvc/tools/tools-section.js @@ -316,6 +316,11 @@ field = this._fieldHidden(input_def); break; + // hidden field + case 'hidden_data': + field = this._fieldHidden(input_def); + break; + // integer field case 'integer': field = this._fieldSlider(input_def); diff -r dc14223818bd8b8a908e056f361f2be1e8972d91 -r e84f0ccd8533d76d3ee3f5b7fa739f096e9aeb41 static/scripts/packed/mvc/tools/tools-section.js --- a/static/scripts/packed/mvc/tools/tools-section.js +++ b/static/scripts/packed/mvc/tools/tools-section.js @@ -1,1 +1,1 @@ -define(["utils/utils","mvc/ui/ui-table","mvc/ui/ui-misc","mvc/tools/tools-repeat","mvc/tools/tools-select-content","mvc/tools/tools-input"],function(d,b,g,c,a,e){var f=Backbone.View.extend({initialize:function(i,h){this.app=i;this.inputs=h.inputs;h.cls_tr="section-row";this.table=new b.View(h);this.setElement(this.table.$el);this.render()},render:function(){this.table.delAll();for(var h in this.inputs){this._add(this.inputs[h])}},_add:function(j){var i=this;var h=jQuery.extend(true,{},j);h.id=j.id=d.uuid();this.app.input_list[h.id]=h;var k=h.type;switch(k){case"conditional":this._addConditional(h);break;case"repeat":this._addRepeat(h);break;default:this._addRow(h)}},_addConditional:function(h){var j=this;h.test_param.id=h.id;var m=this._addRow(h.test_param);m.options.onchange=function(t){var p=j.app.tree.matchCase(h,t);for(var r in h.cases){var w=h.cases[r];var u=h.id+"-section-"+r;var o=j.table.get(u);var v=false;for(var q in w.inputs){var s=w.inputs[q].type;if(s&&s!=="hidden"){v=true;break}}if(r==p&&v){o.fadeIn("fast")}else{o.hide()}}j.app.trigger("refresh")};for(var l in h.cases){var k=h.id+"-section-"+l;var n=new f(this.app,{inputs:h.cases[l].inputs,cls:"ui-table-plain"});n.$el.addClass("ui-table-form-section");this.table.add(n.$el);this.table.append(k)}m.trigger("change")},_addRepeat:function(o){var r=this;var p=0;function m(i,t){var s=o.id+"-section-"+(p++);var u=null;if(t){u=function(){k.del(s);k.retitle(o.title);r.app.trigger("refresh")}}var v=new f(r.app,{inputs:i,cls:"ui-table-plain"});k.add({id:s,title:o.title,$el:v.$el,ondel:u});k.retitle(o.title)}var k=new c.View({title_new:o.title,max:o.max,onnew:function(){m(o.inputs,true);r.app.trigger("refresh")}});var h=o.min;var q=_.size(o.cache);for(var l=0;l<Math.max(q,h);l++){var n=null;if(l<q){n=o.cache[l]}else{n=o.inputs}m(n,l>=h)}var j=new e(this.app,{label:o.title,help:o.help,field:k});j.$el.addClass("ui-table-form-section");this.table.add(j.$el);this.table.append(o.id)},_addRow:function(h){var k=h.id;var i=this._createField(h);this.app.field_list[k]=i;var j=new e(this.app,{label:h.label,optional:h.optional,help:h.help,field:i});this.app.element_list[k]=j;this.table.add(j.$el);this.table.append(k);return i},_createField:function(h){var i=null;switch(h.type){case"text":i=this._fieldText(h);break;case"select":i=this._fieldSelect(h);break;case"data":i=this._fieldData(h);break;case"data_collection":i=this._fieldData(h);break;case"data_column":h.error_text="Missing columns in referenced dataset.";i=this._fieldSelect(h);break;case"hidden":i=this._fieldHidden(h);break;case"integer":i=this._fieldSlider(h);break;case"float":i=this._fieldSlider(h);break;case"boolean":i=this._fieldBoolean(h);break;case"genomebuild":h.searchable=true;i=this._fieldSelect(h);break;case"drill_down":i=this._fieldDrilldown(h);break;case"baseurl":i=this._fieldHidden(h);break;default:this.app.incompatible=true;if(h.options){i=this._fieldSelect(h)}else{i=this._fieldText(h)}console.debug("tools-form::_addRow() : Auto matched field type ("+h.type+").")}if(h.value!==undefined){i.value(h.value)}return i},_fieldData:function(h){var i=this;return new a.View(this.app,{id:"field-"+h.id,extensions:h.extensions,multiple:h.multiple,type:h.type,data:h.options,onchange:function(){i.app.trigger("refresh")}})},_fieldSelect:function(h){var k=[];for(var l in h.options){var m=h.options[l];k.push({label:m[0],value:m[1]})}var n=g.Select;switch(h.display){case"checkboxes":n=g.Checkbox;break;case"radio":n=g.Radio;break}var j=this;return new n.View({id:"field-"+h.id,data:k,error_text:h.error_text||"No options available",multiple:h.multiple,searchable:h.searchable,onchange:function(){j.app.trigger("refresh")}})},_fieldDrilldown:function(h){var i=this;return new g.Drilldown.View({id:"field-"+h.id,data:h.options,display:h.display,onchange:function(){i.app.trigger("refresh")}})},_fieldText:function(h){var i=this;return new g.Input({id:"field-"+h.id,area:h.area,onchange:function(){i.app.trigger("refresh")}})},_fieldSlider:function(h){return new g.Slider.View({id:"field-"+h.id,precise:h.type=="float",min:h.min,max:h.max})},_fieldHidden:function(h){return new g.Hidden({id:"field-"+h.id})},_fieldBoolean:function(h){return new g.RadioButton.View({id:"field-"+h.id,data:[{label:"Yes",value:"true"},{label:"No",value:"false"}]})}});return{View:f}}); \ No newline at end of file +define(["utils/utils","mvc/ui/ui-table","mvc/ui/ui-misc","mvc/tools/tools-repeat","mvc/tools/tools-select-content","mvc/tools/tools-input"],function(d,b,g,c,a,e){var f=Backbone.View.extend({initialize:function(i,h){this.app=i;this.inputs=h.inputs;h.cls_tr="section-row";this.table=new b.View(h);this.setElement(this.table.$el);this.render()},render:function(){this.table.delAll();for(var h in this.inputs){this._add(this.inputs[h])}},_add:function(j){var i=this;var h=jQuery.extend(true,{},j);h.id=j.id=d.uuid();this.app.input_list[h.id]=h;var k=h.type;switch(k){case"conditional":this._addConditional(h);break;case"repeat":this._addRepeat(h);break;default:this._addRow(h)}},_addConditional:function(h){var j=this;h.test_param.id=h.id;var m=this._addRow(h.test_param);m.options.onchange=function(t){var p=j.app.tree.matchCase(h,t);for(var r in h.cases){var w=h.cases[r];var u=h.id+"-section-"+r;var o=j.table.get(u);var v=false;for(var q in w.inputs){var s=w.inputs[q].type;if(s&&s!=="hidden"){v=true;break}}if(r==p&&v){o.fadeIn("fast")}else{o.hide()}}j.app.trigger("refresh")};for(var l in h.cases){var k=h.id+"-section-"+l;var n=new f(this.app,{inputs:h.cases[l].inputs,cls:"ui-table-plain"});n.$el.addClass("ui-table-form-section");this.table.add(n.$el);this.table.append(k)}m.trigger("change")},_addRepeat:function(o){var r=this;var p=0;function m(i,t){var s=o.id+"-section-"+(p++);var u=null;if(t){u=function(){k.del(s);k.retitle(o.title);r.app.trigger("refresh")}}var v=new f(r.app,{inputs:i,cls:"ui-table-plain"});k.add({id:s,title:o.title,$el:v.$el,ondel:u});k.retitle(o.title)}var k=new c.View({title_new:o.title,max:o.max,onnew:function(){m(o.inputs,true);r.app.trigger("refresh")}});var h=o.min;var q=_.size(o.cache);for(var l=0;l<Math.max(q,h);l++){var n=null;if(l<q){n=o.cache[l]}else{n=o.inputs}m(n,l>=h)}var j=new e(this.app,{label:o.title,help:o.help,field:k});j.$el.addClass("ui-table-form-section");this.table.add(j.$el);this.table.append(o.id)},_addRow:function(h){var k=h.id;var i=this._createField(h);this.app.field_list[k]=i;var j=new e(this.app,{label:h.label,optional:h.optional,help:h.help,field:i});this.app.element_list[k]=j;this.table.add(j.$el);this.table.append(k);return i},_createField:function(h){var i=null;switch(h.type){case"text":i=this._fieldText(h);break;case"select":i=this._fieldSelect(h);break;case"data":i=this._fieldData(h);break;case"data_collection":i=this._fieldData(h);break;case"data_column":h.error_text="Missing columns in referenced dataset.";i=this._fieldSelect(h);break;case"hidden":i=this._fieldHidden(h);break;case"hidden_data":i=this._fieldHidden(h);break;case"integer":i=this._fieldSlider(h);break;case"float":i=this._fieldSlider(h);break;case"boolean":i=this._fieldBoolean(h);break;case"genomebuild":h.searchable=true;i=this._fieldSelect(h);break;case"drill_down":i=this._fieldDrilldown(h);break;case"baseurl":i=this._fieldHidden(h);break;default:this.app.incompatible=true;if(h.options){i=this._fieldSelect(h)}else{i=this._fieldText(h)}console.debug("tools-form::_addRow() : Auto matched field type ("+h.type+").")}if(h.value!==undefined){i.value(h.value)}return i},_fieldData:function(h){var i=this;return new a.View(this.app,{id:"field-"+h.id,extensions:h.extensions,multiple:h.multiple,type:h.type,data:h.options,onchange:function(){i.app.trigger("refresh")}})},_fieldSelect:function(h){var k=[];for(var l in h.options){var m=h.options[l];k.push({label:m[0],value:m[1]})}var n=g.Select;switch(h.display){case"checkboxes":n=g.Checkbox;break;case"radio":n=g.Radio;break}var j=this;return new n.View({id:"field-"+h.id,data:k,error_text:h.error_text||"No options available",multiple:h.multiple,searchable:h.searchable,onchange:function(){j.app.trigger("refresh")}})},_fieldDrilldown:function(h){var i=this;return new g.Drilldown.View({id:"field-"+h.id,data:h.options,display:h.display,onchange:function(){i.app.trigger("refresh")}})},_fieldText:function(h){var i=this;return new g.Input({id:"field-"+h.id,area:h.area,onchange:function(){i.app.trigger("refresh")}})},_fieldSlider:function(h){return new g.Slider.View({id:"field-"+h.id,precise:h.type=="float",min:h.min,max:h.max})},_fieldHidden:function(h){return new g.Hidden({id:"field-"+h.id})},_fieldBoolean:function(h){return new g.RadioButton.View({id:"field-"+h.id,data:[{label:"Yes",value:"true"},{label:"No",value:"false"}]})}});return{View:f}}); \ No newline at end of file Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: 2 new changesets
by commits-noreply＠bitbucket.org 29 Dec '14

29 Dec '14

2 new commits in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/68daf6dfc8b4/ Changeset: 68daf6dfc8b4 User: jmchilton Date: 2014-12-29 17:23:55+00:00 Summary: Update attribute and method names in ToolBox to reflect encapsulation efforts. Using ToolBox._xxx instead of ToolBox.__xxx because realistically ToolBox is still much to large to grok and so I imagine it will need to be broken up even more (base class focused on just the panel details perhaps - or mixins - etc...). Affected #: 3 files diff -r 6268351b242feb5c2167b35996e264e8a51c11a9 -r 68daf6dfc8b405acbdfc64fd21d520178aff1ac4 lib/galaxy/tools/__init__.py --- a/lib/galaxy/tools/__init__.py +++ b/lib/galaxy/tools/__init__.py @@ -131,49 +131,50 @@ Create a toolbox from the config files named by `config_filenames`, using `tool_root_dir` as the base directory for finding individual tool config files. """ - # The shed_tool_confs list contains dictionaries storing information about the tools defined in each - # shed-related shed_tool_conf.xml file. - self.shed_tool_confs = [] - self.tools_by_id = {} - self.workflows_by_id = {} + # The _dynamic_tool_confs list contains dictionaries storing + # information about the tools defined in each shed-related + # shed_tool_conf.xml file. + self._dynamic_tool_confs = [] + self._tools_by_id = {} + self._workflows_by_id = {} # In-memory dictionary that defines the layout of the tool panel. - self.tool_panel = odict() - self.index = 0 + self._tool_panel = odict() + self._index = 0 self.data_manager_tools = odict() - self.lineage_map = LineageMap( app ) + self._lineage_map = LineageMap( app ) # File that contains the XML section and tool tags from all tool panel config files integrated into a # single file that defines the tool panel layout. This file can be changed by the Galaxy administrator # (in a way similar to the single tool_conf.xml file in the past) to alter the layout of the tool panel. - self.integrated_tool_panel_config = app.config.integrated_tool_panel_config + self._integrated_tool_panel_config = app.config.integrated_tool_panel_config # In-memory dictionary that defines the layout of the tool_panel.xml file on disk. - self.integrated_tool_panel = odict() - self.integrated_tool_panel_config_has_contents = os.path.exists( self.integrated_tool_panel_config ) and os.stat( self.integrated_tool_panel_config ).st_size > 0 - if self.integrated_tool_panel_config_has_contents: - self.load_integrated_tool_panel_keys() + self._integrated_tool_panel = odict() + self._integrated_tool_panel_config_has_contents = os.path.exists( self._integrated_tool_panel_config ) and os.stat( self._integrated_tool_panel_config ).st_size > 0 + if self._integrated_tool_panel_config_has_contents: + self._load_integrated_tool_panel_keys() # The following refers to the tool_path config setting for backward compatibility. The shed-related # (e.g., shed_tool_conf.xml) files include the tool_path attribute within the <toolbox> tag. - self.tool_root_dir = tool_root_dir + self._tool_root_dir = tool_root_dir self.app = app - self.tool_watcher = watcher.get_watcher( self, app.config ) - self.filter_factory = FilterFactory( self ) - self.init_dependency_manager() - self.tool_tag_manager = tool_tag_manager( app ) - self.init_tools_from_configs( config_filenames ) - if self.app.name == 'galaxy' and self.integrated_tool_panel_config_has_contents: - # Load self.tool_panel based on the order in self.integrated_tool_panel. - self.load_tool_panel() + self._tool_watcher = watcher.get_watcher( self, app.config ) + self._filter_factory = FilterFactory( self ) + self._init_dependency_manager() + self._tool_tag_manager = tool_tag_manager( app ) + self._init_tools_from_configs( config_filenames ) + if self.app.name == 'galaxy' and self._integrated_tool_panel_config_has_contents: + # Load self._tool_panel based on the order in self._integrated_tool_panel. + self._load_tool_panel() if app.config.update_integrated_tool_panel: # Write the current in-memory integrated_tool_panel to the integrated_tool_panel.xml file. # This will cover cases where the Galaxy administrator manually edited one or more of the tool panel # config files, adding or removing locally developed tools or workflows. The value of integrated_tool_panel # will be False when things like functional tests are the caller. - self.write_integrated_tool_panel_config_file() - - def init_tools_from_configs( self, config_filenames ): + self._write_integrated_tool_panel_config_file() + + def _init_tools_from_configs( self, config_filenames ): """ Read through all tool config files and initialize tools in each with init_tools_from_config below. """ - self.tool_tag_manager.reset_tags() + self._tool_tag_manager.reset_tags() config_filenames = listify( config_filenames ) for config_filename in config_filenames: if os.path.isdir( config_filename ): @@ -183,11 +184,11 @@ config_filenames.extend( directory_config_files ) for config_filename in config_filenames: try: - self.init_tools_from_config( config_filename ) + self._init_tools_from_config( config_filename ) except: log.exception( "Error loading tools defined in config %s", config_filename ) - def init_tools_from_config( self, config_filename ): + def _init_tools_from_config( self, config_filename ): """ Read the configuration file and load each tool. The following tags are currently supported: @@ -218,10 +219,10 @@ parsing_shed_tool_conf = False tool_path = self.__resolve_tool_path(tool_path, config_filename) # Only load the panel_dict under certain conditions. - load_panel_dict = not self.integrated_tool_panel_config_has_contents + load_panel_dict = not self._integrated_tool_panel_config_has_contents for _, elem in enumerate( root ): - index = self.index - self.index += 1 + index = self._index + self._index += 1 if parsing_shed_tool_conf: config_elems.append( elem ) self.load_item( @@ -237,7 +238,7 @@ shed_tool_conf_dict = dict( config_filename=config_filename, tool_path=tool_path, config_elems=config_elems ) - self.shed_tool_confs.append( shed_tool_conf_dict ) + self._dynamic_tool_confs.append( shed_tool_conf_dict ) def load_item( self, elem, tool_path, panel_dict=None, integrated_panel_dict=None, load_panel_dict=True, guid=None, index=None, internal=False ): item_type = elem.tag @@ -247,22 +248,22 @@ return if panel_dict is None: - panel_dict = self.tool_panel + panel_dict = self._tool_panel if integrated_panel_dict is None: - integrated_panel_dict = self.integrated_tool_panel + integrated_panel_dict = self._integrated_tool_panel if item_type == 'tool': - self.load_tool_tag_set( elem, panel_dict=panel_dict, integrated_panel_dict=integrated_panel_dict, tool_path=tool_path, load_panel_dict=load_panel_dict, guid=guid, index=index ) + self._load_tool_tag_set( elem, panel_dict=panel_dict, integrated_panel_dict=integrated_panel_dict, tool_path=tool_path, load_panel_dict=load_panel_dict, guid=guid, index=index ) elif item_type == 'workflow': - self.load_workflow_tag_set( elem, panel_dict=panel_dict, integrated_panel_dict=integrated_panel_dict, load_panel_dict=load_panel_dict, index=index ) + self._load_workflow_tag_set( elem, panel_dict=panel_dict, integrated_panel_dict=integrated_panel_dict, load_panel_dict=load_panel_dict, index=index ) elif item_type == 'section': - self.load_section_tag_set( elem, tool_path=tool_path, load_panel_dict=load_panel_dict, index=index ) + self._load_section_tag_set( elem, tool_path=tool_path, load_panel_dict=load_panel_dict, index=index ) elif item_type == 'label': - self.load_label_tag_set( elem, panel_dict=panel_dict, integrated_panel_dict=integrated_panel_dict, load_panel_dict=load_panel_dict, index=index ) + self._load_label_tag_set( elem, panel_dict=panel_dict, integrated_panel_dict=integrated_panel_dict, load_panel_dict=load_panel_dict, index=index ) elif item_type == 'tool_dir': - self.load_tooldir_tag_set( elem, panel_dict, tool_path, integrated_panel_dict, load_panel_dict=load_panel_dict ) + self._load_tooldir_tag_set( elem, panel_dict, tool_path, integrated_panel_dict, load_panel_dict=load_panel_dict ) def get_shed_config_dict_by_filename( self, filename, default=None ): - for shed_config_dict in self.shed_tool_confs: + for shed_config_dict in self._dynamic_tool_confs: if shed_config_dict[ 'config_filename' ] == filename: return shed_config_dict return default @@ -273,22 +274,22 @@ that doesn't affect that file). """ app = self.app - for index, my_shed_tool_conf in enumerate( self.shed_tool_confs ): + for index, my_shed_tool_conf in enumerate( self._dynamic_tool_confs ): if shed_conf['config_filename'] == my_shed_tool_conf['config_filename']: - self.shed_tool_confs[ index ] = shed_conf + self._dynamic_tool_confs[ index ] = shed_conf if integrated_panel_changes and app.config.update_integrated_tool_panel: # Write the current in-memory version of the integrated_tool_panel.xml file to disk. - self.write_integrated_tool_panel_config_file() + self._write_integrated_tool_panel_config_file() app.reindex_tool_search() def get_section( self, section_id, new_label=None, create_if_needed=False ): tool_panel_section_key = str( section_id ) - if tool_panel_section_key in self.tool_panel: - # Appending a tool to an existing section in toolbox.tool_panel - tool_section = self.tool_panel[ tool_panel_section_key ] + if tool_panel_section_key in self._tool_panel: + # Appending a tool to an existing section in toolbox._tool_panel + tool_section = self._tool_panel[ tool_panel_section_key ] log.debug( "Appending to tool panel section: %s" % str( tool_section.name ) ) elif create_if_needed: - # Appending a new section to toolbox.tool_panel + # Appending a new section to toolbox._tool_panel if new_label is None: # This might add an ugly section label to the tool panel, but, oh well... new_label = section_id @@ -298,7 +299,7 @@ 'version': '', } tool_section = ToolSection( section_dict ) - self.tool_panel[ tool_panel_section_key ] = tool_section + self._tool_panel[ tool_panel_section_key ] = tool_section log.debug( "Loading new tool panel section: %s" % str( tool_section.name ) ) else: tool_section = None @@ -306,7 +307,7 @@ def get_integrated_section_for_tool( self, tool ): tool_id = tool.id - for key, item in self.integrated_tool_panel.items(): + for key, item in self._integrated_tool_panel.items(): if item: if isinstance( item, Tool ): if item.id == tool_id: @@ -324,7 +325,7 @@ def __resolve_tool_path(self, tool_path, config_filename): if not tool_path: # Default to backward compatible config setting. - tool_path = self.tool_root_dir + tool_path = self._tool_root_dir else: # Allow use of __tool_conf_dir__ in toolbox config files. tool_conf_dir = os.path.dirname(config_filename) @@ -335,9 +336,9 @@ def __add_tool_to_tool_panel( self, tool, panel_component, section=False ): # See if a version of this tool is already loaded into the tool panel. # The value of panel_component will be a ToolSection (if the value of - # section=True) or self.tool_panel (if section=False). + # section=True) or self._tool_panel (if section=False). tool_id = str( tool.id ) - tool = self.tools_by_id[ tool_id ] + tool = self._tools_by_id[ tool_id ] if section: panel_dict = panel_component.elems else: @@ -355,7 +356,7 @@ inserted = False key = 'tool_%s' % tool.id # The value of panel_component is the in-memory tool panel dictionary. - for index, integrated_panel_key in enumerate( self.integrated_tool_panel.keys() ): + for index, integrated_panel_key in enumerate( self._integrated_tool_panel.keys() ): if key == integrated_panel_key: panel_dict.insert( index, key, tool ) inserted = True @@ -363,7 +364,7 @@ # Check the tool's installed versions. for lineage_id in tool.lineage_ids: lineage_id_key = 'tool_%s' % lineage_id - for index, integrated_panel_key in enumerate( self.integrated_tool_panel.keys() ): + for index, integrated_panel_key in enumerate( self._integrated_tool_panel.keys() ): if lineage_id_key == integrated_panel_key: panel_dict.insert( index, key, tool ) inserted = True @@ -383,27 +384,28 @@ log.debug( "Loaded tool id: %s, version: %s into tool panel.." % ( tool.id, tool.version ) ) else: # We are in the process of installing the tool. - tool_lineage = self.lineage_map.get( tool_id ) + + tool_lineage = self._lineage_map.get( tool_id ) already_loaded = self._lineage_in_panel( panel_dict, tool_lineage=tool_lineage ) is not None if not already_loaded: # If the tool is not defined in integrated_tool_panel.xml, append it to the tool panel. panel_dict[ key ] = tool log.debug( "Loaded tool id: %s, version: %s into tool panel...." % ( tool.id, tool.version ) ) - def load_tool_panel( self ): - for key, val in self.integrated_tool_panel.items(): + def _load_tool_panel( self ): + for key, val in self._integrated_tool_panel.items(): if isinstance( val, Tool ): tool_id = key.replace( 'tool_', '', 1 ) - if tool_id in self.tools_by_id: - self.__add_tool_to_tool_panel( val, self.tool_panel, section=False ) + if tool_id in self._tools_by_id: + self.__add_tool_to_tool_panel( val, self._tool_panel, section=False ) elif isinstance( val, Workflow ): workflow_id = key.replace( 'workflow_', '', 1 ) - if workflow_id in self.workflows_by_id: - workflow = self.workflows_by_id[ workflow_id ] - self.tool_panel[ key ] = workflow + if workflow_id in self._workflows_by_id: + workflow = self._workflows_by_id[ workflow_id ] + self._tool_panel[ key ] = workflow log.debug( "Loaded workflow: %s %s" % ( workflow_id, workflow.name ) ) elif isinstance( val, ToolSectionLabel ): - self.tool_panel[ key ] = val + self._tool_panel[ key ] = val elif isinstance( val, ToolSection ): section_dict = { 'id': val.id or '', @@ -415,36 +417,36 @@ for section_key, section_val in val.elems.items(): if isinstance( section_val, Tool ): tool_id = section_key.replace( 'tool_', '', 1 ) - if tool_id in self.tools_by_id: + if tool_id in self._tools_by_id: self.__add_tool_to_tool_panel( section_val, section, section=True ) elif isinstance( section_val, Workflow ): workflow_id = section_key.replace( 'workflow_', '', 1 ) - if workflow_id in self.workflows_by_id: - workflow = self.workflows_by_id[ workflow_id ] + if workflow_id in self._workflows_by_id: + workflow = self._workflows_by_id[ workflow_id ] section.elems[ section_key ] = workflow log.debug( "Loaded workflow: %s %s" % ( workflow_id, workflow.name ) ) elif isinstance( section_val, ToolSectionLabel ): if section_val: section.elems[ section_key ] = section_val log.debug( "Loaded label: %s" % ( section_val.text ) ) - self.tool_panel[ key ] = section - - def load_integrated_tool_panel_keys( self ): + self._tool_panel[ key ] = section + + def _load_integrated_tool_panel_keys( self ): """ Load the integrated tool panel keys, setting values for tools and workflows to None. The values will be reset when the various tool panel config files are parsed, at which time the tools and workflows are loaded. """ - tree = parse_xml( self.integrated_tool_panel_config ) + tree = parse_xml( self._integrated_tool_panel_config ) root = tree.getroot() for elem in root: if elem.tag == 'tool': key = 'tool_%s' % elem.get( 'id' ) - self.integrated_tool_panel[ key ] = None + self._integrated_tool_panel[ key ] = None elif elem.tag == 'workflow': key = 'workflow_%s' % elem.get( 'id' ) - self.integrated_tool_panel[ key ] = None + self._integrated_tool_panel[ key ] = None elif elem.tag == 'section': section = ToolSection( elem ) for section_elem in elem: @@ -458,12 +460,12 @@ key = 'label_%s' % section_elem.get( 'id' ) section.elems[ key ] = None key = elem.get( 'id' ) - self.integrated_tool_panel[ key ] = section + self._integrated_tool_panel[ key ] = section elif elem.tag == 'label': key = 'label_%s' % elem.get( 'id' ) - self.integrated_tool_panel[ key ] = None - - def write_integrated_tool_panel_config_file( self ): + self._integrated_tool_panel[ key ] = None + + def _write_integrated_tool_panel_config_file( self ): """ Write the current in-memory version of the integrated_tool_panel.xml file to disk. Since Galaxy administrators use this file to manage the tool panel, we'll not use xml_to_string() since it doesn't write XML quite right. @@ -474,7 +476,7 @@ os.write( fd, ' \n') - for key, item in self.integrated_tool_panel.items(): + for key, item in self._integrated_tool_panel.items(): if item: if isinstance( item, Tool ): os.write( fd, ' <tool id="%s" />\n' % item.id ) @@ -506,28 +508,28 @@ os.write( fd, ' </section>\n' ) os.write( fd, '</toolbox>\n' ) os.close( fd ) - shutil.move( filename, os.path.abspath( self.integrated_tool_panel_config ) ) - os.chmod( self.integrated_tool_panel_config, 0644 ) + shutil.move( filename, os.path.abspath( self._integrated_tool_panel_config ) ) + os.chmod( self._integrated_tool_panel_config, 0644 ) def get_tool( self, tool_id, tool_version=None, get_all_versions=False, exact=False ): """Attempt to locate a tool in the tool box.""" if get_all_versions and exact: raise AssertionError("Cannot specify get_tool with both get_all_versions and exact as True") - if tool_id in self.tools_by_id and not get_all_versions: + if tool_id in self._tools_by_id and not get_all_versions: #tool_id exactly matches an available tool by id (which is 'old' tool_id or guid) - return self.tools_by_id[ tool_id ] + return self._tools_by_id[ tool_id ] #exact tool id match not found, or all versions requested, search for other options, e.g. migrated tools or different versions rval = [] - tool_lineage = self.lineage_map.get( tool_id ) + tool_lineage = self._lineage_map.get( tool_id ) if tool_lineage: tool_version_ids = tool_lineage.get_version_ids( ) for tool_version_id in tool_version_ids: - if tool_version_id in self.tools_by_id: - rval.append( self.tools_by_id[ tool_version_id ] ) + if tool_version_id in self._tools_by_id: + rval.append( self._tools_by_id[ tool_version_id ] ) if not rval: #still no tool, do a deeper search and try to match by old ids - for tool in self.tools_by_id.itervalues(): + for tool in self._tools_by_id.itervalues(): if tool.old_id == tool_id: rval.append( tool ) if rval: @@ -543,8 +545,8 @@ return rval[0] #We now likely have a Toolshed guid passed in, but no supporting database entries #If the tool exists by exact id and is loaded then provide exact match within a list - if tool_id in self.tools_by_id: - return[ self.tools_by_id[ tool_id ] ] + if tool_id in self._tools_by_id: + return[ self._tools_by_id[ tool_id ] ] return None def has_tool( self, tool_id, exact=False ): @@ -556,7 +558,7 @@ tools (using get_tool for inexact matches which currently returns the oldest tool shed installed tool with the same short id). """ - if tool_id not in self.tools_by_id: + if tool_id not in self._tools_by_id: tool = self.get_tool( tool_id ) if tool: tool_id = tool.id @@ -567,22 +569,22 @@ def get_loaded_tools_by_lineage( self, tool_id ): """Get all loaded tools associated by lineage to the tool whose id is tool_id.""" - tool_lineage = self.lineage_map.get( tool_id ) + tool_lineage = self._lineage_map.get( tool_id ) if tool_lineage: tool_version_ids = tool_lineage.get_version_ids( ) available_tool_versions = [] for tool_version_id in tool_version_ids: - if tool_version_id in self.tools_by_id: - available_tool_versions.append( self.tools_by_id[ tool_version_id ] ) + if tool_version_id in self._tools_by_id: + available_tool_versions.append( self._tools_by_id[ tool_version_id ] ) return available_tool_versions else: - if tool_id in self.tools_by_id: - tool = self.tools_by_id[ tool_id ] + if tool_id in self._tools_by_id: + tool = self._tools_by_id[ tool_id ] return [ tool ] return [] def tools( self ): - return self.tools_by_id.iteritems() + return self._tools_by_id.iteritems() def __elem_to_tool_shed_repository( self, elem ): # The tool is contained in an installed tool shed repository, so load @@ -637,7 +639,7 @@ def dynamic_confs( self, include_migrated_tool_conf=False ): confs = [] - for dynamic_tool_conf_dict in self.shed_tool_confs: + for dynamic_tool_conf_dict in self._dynamic_tool_confs: dynamic_tool_conf_filename = dynamic_tool_conf_dict[ 'config_filename' ] if include_migrated_tool_conf or (dynamic_tool_conf_filename != self.app.config.migrated_tools_config): confs.append( dynamic_tool_conf_dict ) @@ -712,11 +714,11 @@ if section_key: _, tool_section = self.get_section( section_key ) if tool_section: - remove_from_dict( tool_section.elems, self.integrated_tool_panel.get( section_key, {} ) ) + remove_from_dict( tool_section.elems, self._integrated_tool_panel.get( section_key, {} ) ) else: - remove_from_dict( self.tool_panel, self.integrated_tool_panel ) - - def load_tool_tag_set( self, elem, panel_dict, integrated_panel_dict, tool_path, load_panel_dict, guid=None, index=None ): + remove_from_dict( self._tool_panel, self._integrated_tool_panel ) + + def _load_tool_tag_set( self, elem, panel_dict, integrated_panel_dict, tool_path, load_panel_dict, guid=None, index=None ): try: path = elem.get( "file" ) repository_id = None @@ -746,11 +748,11 @@ tool.guid = guid tool.version = elem.find( "version" ).text # Make sure the tool has a tool_version. - tool_lineage = self.lineage_map.register( tool, tool_shed_repository=tool_shed_repository ) + tool_lineage = self._lineage_map.register( tool, tool_shed_repository=tool_shed_repository ) # Load the tool's lineage ids. tool.lineage = tool_lineage tool.lineage_ids = tool_lineage.get_version_ids( ) - self.tool_tag_manager.handle_tags( tool.id, elem ) + self._tool_tag_manager.handle_tags( tool.id, elem ) self.__add_tool( tool, load_panel_dict, panel_dict ) # Always load the tool into the integrated_panel_dict, or it will not be included in the integrated_tool_panel.xml file. if key in integrated_panel_dict or index is None: @@ -769,18 +771,18 @@ # administrator has retrieved updates to the installed repository. In # this case, the tool may have been updated, but the version was not # changed, so the tool should always be reloaded here. We used to - # only load the tool if it was not found in self.tools_by_id, but + # only load the tool if it was not found in self._tools_by_id, but # performing that check did not enable this scenario. self.register_tool( tool ) if load_panel_dict: self.__add_tool_to_tool_panel( tool, panel_dict, section=isinstance( panel_dict, ToolSection ) ) - def load_workflow_tag_set( self, elem, panel_dict, integrated_panel_dict, load_panel_dict, index=None ): + def _load_workflow_tag_set( self, elem, panel_dict, integrated_panel_dict, load_panel_dict, index=None ): try: # TODO: should id be encoded? workflow_id = elem.get( 'id' ) - workflow = self.load_workflow( workflow_id ) - self.workflows_by_id[ workflow_id ] = workflow + workflow = self._load_workflow( workflow_id ) + self._workflows_by_id[ workflow_id ] = workflow key = 'workflow_' + workflow_id if load_panel_dict: panel_dict[ key ] = workflow @@ -792,7 +794,7 @@ except: log.exception( "Error loading workflow: %s" % workflow_id ) - def load_label_tag_set( self, elem, panel_dict, integrated_panel_dict, load_panel_dict, index=None ): + def _load_label_tag_set( self, elem, panel_dict, integrated_panel_dict, load_panel_dict, index=None ): label = ToolSectionLabel( elem ) key = 'label_' + label.id if load_panel_dict: @@ -802,16 +804,16 @@ else: integrated_panel_dict.insert( index, key, label ) - def load_section_tag_set( self, elem, tool_path, load_panel_dict, index=None ): + def _load_section_tag_set( self, elem, tool_path, load_panel_dict, index=None ): key = elem.get( "id" ) - if key in self.tool_panel: - section = self.tool_panel[ key ] + if key in self._tool_panel: + section = self._tool_panel[ key ] elems = section.elems else: section = ToolSection( elem ) elems = section.elems - if key in self.integrated_tool_panel: - integrated_section = self.integrated_tool_panel[ key ] + if key in self._integrated_tool_panel: + integrated_section = self._integrated_tool_panel[ key ] integrated_elems = integrated_section.elems else: integrated_section = ToolSection( elem ) @@ -828,14 +830,14 @@ internal=True, ) if load_panel_dict: - self.tool_panel[ key ] = section + self._tool_panel[ key ] = section # Always load sections into the integrated_tool_panel. - if key in self.integrated_tool_panel or index is None: - self.integrated_tool_panel[ key ] = integrated_section + if key in self._integrated_tool_panel or index is None: + self._integrated_tool_panel[ key ] = integrated_section else: - self.integrated_tool_panel.insert( index, key, integrated_section ) - - def load_tooldir_tag_set(self, sub_elem, elems, tool_path, integrated_elems, load_panel_dict): + self._integrated_tool_panel.insert( index, key, integrated_section ) + + def _load_tooldir_tag_set(self, sub_elem, elems, tool_path, integrated_elems, load_panel_dict): directory = os.path.join( tool_path, sub_elem.attrib.get("dir") ) recursive = string_as_bool( sub_elem.attrib.get("recursive", True) ) self.__watch_directory( directory, elems, integrated_elems, load_panel_dict, recursive ) @@ -851,14 +853,14 @@ integrated_elems[ key ] = tool if async: - self.load_tool_panel() + self._load_tool_panel() if self.app.config.update_integrated_tool_panel: # Write the current in-memory integrated_tool_panel to the integrated_tool_panel.xml file. # This will cover cases where the Galaxy administrator manually edited one or more of the tool panel # config files, adding or removing locally developed tools or workflows. The value of integrated_tool_panel # will be False when things like functional tests are the caller. - self.write_integrated_tool_panel_config_file() + self._write_integrated_tool_panel_config_file() return tool.id except Exception: log.exception("Failed to load potential tool %s." % tool_file) @@ -873,7 +875,7 @@ quick_load( child_path, async=False ) tool_loaded = True if tool_loaded: - self.tool_watcher.watch_directory( directory, quick_load ) + self._tool_watcher.watch_directory( directory, quick_load ) def load_tool( self, config_file, guid=None, repository_id=None, **kwds ): """Load a single tool from the file named by `config_file` and return an instance of `Tool`.""" @@ -916,19 +918,19 @@ if not tool_id.startswith("__"): # do not monitor special tools written to tmp directory - no reason # to monitor such a large directory. - self.tool_watcher.watch_file( config_file, tool.id ) + self._tool_watcher.watch_file( config_file, tool.id ) return tool def load_hidden_tool( self, config_file, **kwds ): """ Load a hidden tool (in this context meaning one that does not - appear in the tool panel) and register it in tools_by_id. + appear in the tool panel) and register it in _tools_by_id. """ tool = self.load_tool( config_file, **kwds ) self.register_tool( tool ) return tool def register_tool( self, tool ): - self.tools_by_id[ tool.id ] = tool + self._tools_by_id[ tool.id ] = tool def package_tool( self, trans, tool_id ): """ @@ -938,10 +940,10 @@ :returns: tuple of tarball filename, success True/False, message/None """ # Make sure the tool is actually loaded. - if tool_id not in self.tools_by_id: + if tool_id not in self._tools_by_id: return None, False, "No tool with id %s" % escape( tool_id ) else: - tool = self.tools_by_id[ tool_id ] + tool = self._tools_by_id[ tool_id ] tarball_files = [] temp_files = [] tool_xml = file( os.path.abspath( tool.config_file ), 'r' ).read() @@ -1048,11 +1050,11 @@ Attempt to reload the tool identified by 'tool_id', if successful replace the old tool. """ - if tool_id not in self.tools_by_id: + if tool_id not in self._tools_by_id: message = "No tool with id %s" % escape( tool_id ) status = 'error' else: - old_tool = self.tools_by_id[ tool_id ] + old_tool = self._tools_by_id[ tool_id ] new_tool = self.load_tool( old_tool.config_file ) # The tool may have been installed from a tool shed, so set the tool shed attributes. # Since the tool version may have changed, we don't override it here. @@ -1063,17 +1065,17 @@ new_tool.repository_owner = old_tool.repository_owner new_tool.installed_changeset_revision = old_tool.installed_changeset_revision new_tool.old_id = old_tool.old_id - # Replace old_tool with new_tool in self.tool_panel + # Replace old_tool with new_tool in self._tool_panel tool_key = 'tool_' + tool_id - for key, val in self.tool_panel.items(): + for key, val in self._tool_panel.items(): if key == tool_key: - self.tool_panel[ key ] = new_tool + self._tool_panel[ key ] = new_tool break elif key.startswith( 'section' ): if tool_key in val.elems: - self.tool_panel[ key ].elems[ tool_key ] = new_tool + self._tool_panel[ key ].elems[ tool_key ] = new_tool break - self.tools_by_id[ tool_id ] = new_tool + self._tools_by_id[ tool_id ] = new_tool message = "Reloaded the tool:<br/>" message += "<b>name:</b> %s<br/>" % old_tool.name message += "<b>id:</b> %s<br/>" % old_tool.id @@ -1089,21 +1091,21 @@ promoting the next newest version of the tool into the panel if needed. """ - if tool_id not in self.tools_by_id: + if tool_id not in self._tools_by_id: message = "No tool with id %s" % escape( tool_id ) status = 'error' else: - tool = self.tools_by_id[ tool_id ] - del self.tools_by_id[ tool_id ] + tool = self._tools_by_id[ tool_id ] + del self._tools_by_id[ tool_id ] if remove_from_panel: tool_key = 'tool_' + tool_id - for key, val in self.tool_panel.items(): + for key, val in self._tool_panel.items(): if key == tool_key: - del self.tool_panel[ key ] + del self._tool_panel[ key ] break elif key.startswith( 'section' ): if tool_key in val.elems: - del self.tool_panel[ key ].elems[ tool_key ] + del self._tool_panel[ key ].elems[ tool_key ] break if tool_id in self.data_manager_tools: del self.data_manager_tools[ tool_id ] @@ -1116,7 +1118,7 @@ return message, status def get_sections( self ): - for k, v in self.tool_panel.items(): + for k, v in self._tool_panel.items(): if isinstance( v, ToolSection ): yield (v.id, v.name) @@ -1128,17 +1130,17 @@ if not tool_panel_section_id: tool_panel_section_id = '' else: - if tool_panel_section_id not in self.tool_panel: + if tool_panel_section_id not in self._tool_panel: # Hack introduced without comment in a29d54619813d5da992b897557162a360b8d610c- # not sure why it is needed. fixed_tool_panel_section_id = 'section_%s' % tool_panel_section_id - if fixed_tool_panel_section_id in self.tool_panel: + if fixed_tool_panel_section_id in self._tool_panel: tool_panel_section_id = fixed_tool_panel_section_id else: tool_panel_section_id = '' return tool_panel_section_id - def load_workflow( self, workflow_id ): + def _load_workflow( self, workflow_id ): """ Return an instance of 'Workflow' identified by `id`, which is encoded in the tool panel. @@ -1147,13 +1149,13 @@ stored = self.app.model.context.query( self.app.model.StoredWorkflow ).get( id ) return stored.latest_workflow - def init_dependency_manager( self ): + def _init_dependency_manager( self ): self.dependency_manager = build_dependency_manager( self.app.config ) def handle_datatypes_changed( self ): """ Refresh upload tools when new datatypes are added. """ - for tool_id in self.tools_by_id: - tool = self.tools_by_id[ tool_id ] + for tool_id in self._tools_by_id: + tool = self._tools_by_id[ tool_id ] if isinstance( tool.tool_action, UploadToolAction ): self.reload_tool_by_id( tool_id ) @@ -1168,8 +1170,8 @@ """ Filter tool_panel contents for displaying for user. """ context = Bunch( toolbox=self, trans=trans ) - filters = self.filter_factory.build_filters( trans ) - for elt in self.tool_panel.itervalues(): + filters = self._filter_factory.build_filters( trans ) + for elt in self._tool_panel.itervalues(): elt = _filter_for_panel( elt, filters, context ) if elt: yield elt @@ -1190,7 +1192,7 @@ rval.append( to_dict_helper( elt, kwargs ) ) else: tools = [] - for id, tool in self.tools_by_id.items(): + for id, tool in self._tools_by_id.items(): tools.append( tool.to_dict( trans, link_details=True ) ) rval = tools @@ -1205,7 +1207,7 @@ tool_lineage = tool.lineage lineage_ids = tool_lineage.get_version_ids( reverse=True ) for lineage_id in lineage_ids: - if lineage_id in self.tools_by_id: + if lineage_id in self._tools_by_id: loaded_version_key = 'tool_%s' % lineage_id if loaded_version_key in panel_dict: return panel_dict[ loaded_version_key ] diff -r 6268351b242feb5c2167b35996e264e8a51c11a9 -r 68daf6dfc8b405acbdfc64fd21d520178aff1ac4 test/unit/tool_shed_unit_tests/test_tool_panel_manager.py --- a/test/unit/tool_shed_unit_tests/test_tool_panel_manager.py +++ b/test/unit/tool_shed_unit_tests/test_tool_panel_manager.py @@ -18,20 +18,20 @@ assert section_id == "tid" assert len( section.elems ) == 1 # tool.xml assert section.id == "tid" - assert len( toolbox.tool_panel ) == 1 + assert len( toolbox._tool_panel ) == 1 section_id, section = tpm.handle_tool_panel_section( toolbox, new_tool_panel_section_label="tid2" ) assert section_id == "tid2" assert len( section.elems ) == 0 # new section assert section.id == "tid2" - assert len( toolbox.tool_panel ) == 2 + assert len( toolbox._tool_panel ) == 2 # Test re-fetch new section by same id. section_id, section = tpm.handle_tool_panel_section( toolbox, new_tool_panel_section_label="tid2" ) assert section_id == "tid2" assert len( section.elems ) == 0 # new section assert section.id == "tid2" - assert len( toolbox.tool_panel ) == 2 + assert len( toolbox._tool_panel ) == 2 def test_add_tool_to_panel( self ): self._init_tool() @@ -77,7 +77,7 @@ assert not all_versions # Check that tool panel has reverted to old value... - section = self.toolbox.tool_panel["tid"] + section = self.toolbox._tool_panel["tid"] assert len(section.elems) == 0 def test_uninstall_in_section( self ): @@ -85,7 +85,7 @@ self._verify_version_2_removed_from_panel( ) # Not in tool conf because it was uninstalled. assert "github.com/galaxyproject/example/test_tool/0.2" not in open(os.path.join(self.test_directory, "tool_conf.xml"), "r").read() - assert "tool_github.com/galaxyproject/example/test_tool/0.2" not in self.toolbox.integrated_tool_panel["tid"].elems + assert "tool_github.com/galaxyproject/example/test_tool/0.2" not in self.toolbox._integrated_tool_panel["tid"].elems self._verify_tool_confs() def test_deactivate_outside_section( self ): @@ -109,7 +109,7 @@ assert not all_versions # Check that tool panel has reverted to old value... - section = self.toolbox.tool_panel["tid"] + section = self.toolbox._tool_panel["tid"] assert len(section.elems) == 0 def _setup_two_versions_remove_one( self, section, uninstall ): @@ -126,14 +126,14 @@ # Check that tool panel has reverted to old value... if section: - section = self.toolbox.tool_panel["tid"] + section = self.toolbox._tool_panel["tid"] assert len(section.elems) == 1 assert section.elems.values()[0].id == "github.com/galaxyproject/example/test_tool/0.1" - assert "github.com/galaxyproject/example/test_tool/0.2" not in self.toolbox.integrated_tool_panel["tid"].elems + assert "github.com/galaxyproject/example/test_tool/0.2" not in self.toolbox._integrated_tool_panel["tid"].elems else: - self.toolbox.tool_panel.values()[0].id == "github.com/galaxyproject/example/test_tool/0.1" - assert "github.com/galaxyproject/example/test_tool/0.2" not in self.toolbox.integrated_tool_panel + self.toolbox._tool_panel.values()[0].id == "github.com/galaxyproject/example/test_tool/0.1" + assert "github.com/galaxyproject/example/test_tool/0.2" not in self.toolbox._integrated_tool_panel def _remove_guids( self, guids, uninstall, shed_tool_conf="tool_conf.xml" ): self.tpm.remove_guids( diff -r 6268351b242feb5c2167b35996e264e8a51c11a9 -r 68daf6dfc8b405acbdfc64fd21d520178aff1ac4 test/unit/tools/test_toolbox.py --- a/test/unit/tools/test_toolbox.py +++ b/test/unit/tools/test_toolbox.py @@ -182,7 +182,7 @@ self.__verify_two_test_tools( ) # Assert only newer version of the tool loaded into the panel. - section = self.toolbox.tool_panel["tid"] + section = self.toolbox._tool_panel["tid"] assert len(section.elems) == 1 assert section.elems.values()[0].id == "github.com/galaxyproject/example/test_tool/0.2" @@ -193,7 +193,7 @@ self.__verify_two_test_tools( ) # Assert tools merged in tool panel. - assert len( self.toolbox.tool_panel ) == 1 + assert len( self.toolbox._tool_panel ) == 1 def test_update_shed_conf(self): self.__setup_shed_tool_conf() @@ -228,8 +228,8 @@ stored_workflow = self.__test_workflow() encoded_id = self.app.security.encode_id( stored_workflow.id ) self._add_config( """<toolbox><workflow id="%s" /></toolbox>""" % encoded_id ) - assert len( self.toolbox.tool_panel ) == 1 - panel_workflow = self.toolbox.tool_panel.values()[ 0 ] + assert len( self.toolbox._tool_panel ) == 1 + panel_workflow = self.toolbox._tool_panel.values()[ 0 ] assert panel_workflow == stored_workflow.latest_workflow # TODO: test to_dict with workflows @@ -237,21 +237,21 @@ stored_workflow = self.__test_workflow() encoded_id = self.app.security.encode_id( stored_workflow.id ) self._add_config( """<toolbox><section id="tid" name="TID"><workflow id="%s" /></section></toolbox>""" % encoded_id ) - assert len( self.toolbox.tool_panel ) == 1 - section = self.toolbox.tool_panel[ 'tid' ] + assert len( self.toolbox._tool_panel ) == 1 + section = self.toolbox._tool_panel[ 'tid' ] assert len( section.elems ) == 1 panel_workflow = section.elems.values()[ 0 ] assert panel_workflow == stored_workflow.latest_workflow def test_label_in_panel( self ): self._add_config( """<toolbox><label id="lab1" text="Label 1" /><label id="lab2" text="Label 2" /></toolbox>""" ) - assert len( self.toolbox.tool_panel ) == 2 - self.__check_test_labels( self.toolbox.tool_panel ) + assert len( self.toolbox._tool_panel ) == 2 + self.__check_test_labels( self.toolbox._tool_panel ) def test_label_in_section( self ): self._add_config( """<toolbox><section id="tid" name="TID"><label id="lab1" text="Label 1" /><label id="lab2" text="Label 2" /></section></toolbox>""" ) - assert len( self.toolbox.tool_panel ) == 1 - section = self.toolbox.tool_panel[ 'tid' ] + assert len( self.toolbox._tool_panel ) == 1 + section = self.toolbox._tool_panel[ 'tid' ] self.__check_test_labels( section.elems ) def __check_test_labels( self, panel_dict ): https://bitbucket.org/galaxy/galaxy-central/commits/dc14223818bd/ Changeset: dc14223818bd User: jmchilton Date: 2014-12-29 17:50:27+00:00 Summary: Hack to fix bug introduced in 806202d. I may be trying to juggle to many competing branches working on same component. Affected #: 1 file diff -r 68daf6dfc8b405acbdfc64fd21d520178aff1ac4 -r dc14223818bd8b8a908e056f361f2be1e8972d91 lib/galaxy/tools/__init__.py --- a/lib/galaxy/tools/__init__.py +++ b/lib/galaxy/tools/__init__.py @@ -1204,6 +1204,8 @@ """ if tool_lineage is None: assert tool is not None + if not hasattr( tool, "lineage" ): + return None tool_lineage = tool.lineage lineage_ids = tool_lineage.get_version_ids( reverse=True ) for lineage_id in lineage_ids: Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: jmchilton: Shield ToolBox details from Tool.
by commits-noreply＠bitbucket.org 29 Dec '14

29 Dec '14

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/6268351b242f/ Changeset: 6268351b242f User: jmchilton Date: 2014-12-29 17:14:41+00:00 Summary: Shield ToolBox details from Tool. Better seperation of concerns. Affected #: 1 file diff -r 069b1c6c5f67e74c88d8109ba89cb52ed28deee4 -r 6268351b242feb5c2167b35996e264e8a51c11a9 lib/galaxy/tools/__init__.py --- a/lib/galaxy/tools/__init__.py +++ b/lib/galaxy/tools/__init__.py @@ -304,6 +304,23 @@ tool_section = None return tool_panel_section_key, tool_section + def get_integrated_section_for_tool( self, tool ): + tool_id = tool.id + for key, item in self.integrated_tool_panel.items(): + if item: + if isinstance( item, Tool ): + if item.id == tool_id: + return '', '' + if isinstance( item, ToolSection ): + section_id = item.id or '' + section_name = item.name or '' + for section_key, section_item in item.elems.items(): + if isinstance( section_item, Tool ): + if section_item: + if section_item.id == tool_id: + return section_id, section_name + return None, None + def __resolve_tool_path(self, tool_path, config_filename): if not tool_path: # Default to backward compatible config setting. @@ -1553,20 +1570,7 @@ return self.app.job_config.get_destination(self.__get_job_tool_configuration(job_params=job_params).destination) def get_panel_section( self ): - for key, item in self.app.toolbox.integrated_tool_panel.items(): - if item: - if isinstance( item, Tool ): - if item.id == self.id: - return '', '' - if isinstance( item, ToolSection ): - section_id = item.id or '' - section_name = item.name or '' - for section_key, section_item in item.elems.items(): - if isinstance( section_item, Tool ): - if section_item: - if section_item.id == self.id: - return section_id, section_name - return None, None + return self.app.toolbox.get_integrated_section_for_tool( self ) def parse( self, tool_source, guid=None ): """ Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: jmchilton: Bugfix: Apply tool panel filtering in workflow editor.
by commits-noreply＠bitbucket.org 29 Dec '14

29 Dec '14

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/069b1c6c5f67/ Changeset: 069b1c6c5f67 User: jmchilton Date: 2014-12-29 16:48:38+00:00 Summary: Bugfix: Apply tool panel filtering in workflow editor. Bug originally reported by Eric here - http://dev.list.galaxyproject.org/Bug-Toolbox-filters-not-applied-in-workfl… (though this only fixes the superficial bug of them showing up in the workflow editor - and does not yet implement "strong" toolbox filters as requested). This is actually a broader change that centralizes logic for listing out tool panel contents when displaying components to the user and the added filtering comes about from that. It also eliminates the last places Galaxy was accessing toolbox.tool_panel directly (in those mako templates). Affected #: 4 files diff -r ad5094376c1042a3c5aeab793e326cebcc9818b9 -r 069b1c6c5f67e74c88d8109ba89cb52ed28deee4 lib/galaxy/tools/__init__.py --- a/lib/galaxy/tools/__init__.py +++ b/lib/galaxy/tools/__init__.py @@ -1147,24 +1147,22 @@ """ return self.app.model.context + def tool_panel_contents( self, trans, **kwds ): + """ Filter tool_panel contents for displaying for user. + """ + context = Bunch( toolbox=self, trans=trans ) + filters = self.filter_factory.build_filters( trans ) + for elt in self.tool_panel.itervalues(): + elt = _filter_for_panel( elt, filters, context ) + if elt: + yield elt + def to_dict( self, trans, in_panel=True, **kwds ): """ to_dict toolbox. """ - - context = Bunch( toolbox=self, trans=trans, **kwds ) if in_panel: - panel_elts = [ val for val in self.tool_panel.itervalues() ] - - filters = self.filter_factory.build_filters( trans, **kwds ) - - filtered_panel_elts = [] - for index, elt in enumerate( panel_elts ): - elt = _filter_for_panel( elt, filters, context ) - if elt: - filtered_panel_elts.append( elt ) - panel_elts = filtered_panel_elts - + panel_elts = list( self.tool_panel_contents( trans, **kwds ) ) # Produce panel. rval = [] kwargs = dict( diff -r ad5094376c1042a3c5aeab793e326cebcc9818b9 -r 069b1c6c5f67e74c88d8109ba89cb52ed28deee4 templates/admin/package_tool.mako --- a/templates/admin/package_tool.mako +++ b/templates/admin/package_tool.mako @@ -26,7 +26,7 @@ Tool to bundle: </label><select name="tool_id"> - %for key, val in toolbox.tool_panel.items(): + %for val in toolbox.tool_panel_contents( trans ): %if isinstance( val, Tool ): <option value="${val.id|h}">${val.name|h}</option> %elif isinstance( val, ToolSection ): diff -r ad5094376c1042a3c5aeab793e326cebcc9818b9 -r 069b1c6c5f67e74c88d8109ba89cb52ed28deee4 templates/admin/reload_tool.mako --- a/templates/admin/reload_tool.mako +++ b/templates/admin/reload_tool.mako @@ -26,7 +26,7 @@ Tool to reload: </label><select name="tool_id"> - %for key, val in toolbox.tool_panel.items(): + %for val in toolbox.tool_panel_contents( trans ): %if isinstance( val, Tool ): <option value="${val.id}">${val.name|h}</option> %elif isinstance( val, ToolSection ): diff -r ad5094376c1042a3c5aeab793e326cebcc9818b9 -r 069b1c6c5f67e74c88d8109ba89cb52ed28deee4 templates/webapps/galaxy/workflow/editor.mako --- a/templates/webapps/galaxy/workflow/editor.mako +++ b/templates/webapps/galaxy/workflow/editor.mako @@ -284,7 +284,7 @@ <img src="${h.url_for('/static/images/loading_small_white_bg.gif')}" id="search-spinner" class="search-spinner" /></div><div class="toolSectionList"> - %for key, val in app.toolbox.tool_panel.items(): + %for val in app.toolbox.tool_panel_contents( trans ): <div class="toolSectionWrapper"> %if isinstance( val, Tool ): ${render_tool( val, False )} Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: jmchilton: Fix unit tests for 7cee92f.
by commits-noreply＠bitbucket.org 29 Dec '14

29 Dec '14

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/ad5094376c10/ Changeset: ad5094376c10 User: jmchilton Date: 2014-12-29 16:10:41+00:00 Summary: Fix unit tests for 7cee92f. Affected #: 1 file diff -r 806202db57a7502c2aa5476f2db46f7aae75f439 -r ad5094376c1042a3c5aeab793e326cebcc9818b9 test/unit/tools/test_toolbox.py --- a/test/unit/tools/test_toolbox.py +++ b/test/unit/tools/test_toolbox.py @@ -197,13 +197,13 @@ def test_update_shed_conf(self): self.__setup_shed_tool_conf() - self.toolbox.update_shed_config( 0, {} ) + self.toolbox.update_shed_config( { "config_filename": "tool_conf.xml" } ) assert self.reindexed self.assert_integerated_tool_panel(exists=True) def test_update_shed_conf_deactivate_only(self): self.__setup_shed_tool_conf() - self.toolbox.update_shed_config( 0, {}, integrated_panel_changes=False ) + self.toolbox.update_shed_config( { "config_filename": "tool_conf.xml" }, integrated_panel_changes=False ) assert self.reindexed # No changes, should be regenerated self.assert_integerated_tool_panel(exists=False) Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: 2 new changesets
by commits-noreply＠bitbucket.org 29 Dec '14

29 Dec '14

2 new commits in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/b26dc879b09c/ Changeset: b26dc879b09c User: jmchilton Date: 2014-12-29 15:28:37+00:00 Summary: Abstract tool lineage handling out of ToolBox. Start work on infrastructure to support tool lineages not defined within the tool shed install model. Affected #: 5 files diff -r 68255ebe3f54500aa26db814c41f67f382731462 -r b26dc879b09c639b47c76cb97b067073a3c956e6 lib/galaxy/tools/__init__.py --- a/lib/galaxy/tools/__init__.py +++ b/lib/galaxy/tools/__init__.py @@ -53,6 +53,7 @@ from galaxy.tools.parser import get_tool_source from galaxy.tools.parser.xml import XmlPageSource from galaxy.tools.toolbox import tool_tag_manager +from galaxy.tools.toolbox.lineages import LineageMap from galaxy.util import listify, parse_xml, rst_to_html, string_as_bool, string_to_object from galaxy.tools.parameters.meta import expand_meta_parameters from galaxy.util.bunch import Bunch @@ -139,6 +140,7 @@ self.tool_panel = odict() self.index = 0 self.data_manager_tools = odict() + self.lineage_map = LineageMap( app ) # File that contains the XML section and tool tags from all tool panel config files integrated into a # single file that defines the tool panel layout. This file can be changed by the Galaxy administrator # (in a way similar to the single tool_conf.xml file in the past) to alter the layout of the tool panel. @@ -371,8 +373,8 @@ log.debug( "Loaded tool id: %s, version: %s into tool panel.." % ( tool.id, tool.version ) ) else: # We are in the process of installing the tool. - tool_version = self.__get_tool_version( tool_id ) - tool_lineage_ids = tool_version.get_version_ids( self.app, reverse=True ) + tool_lineage = self.lineage_map.get( tool_id ) + tool_lineage_ids = tool_lineage.get_version_ids( reverse=True ) for lineage_id in tool_lineage_ids: if lineage_id in self.tools_by_id: loaded_version_key = 'tool_%s' % lineage_id @@ -512,9 +514,9 @@ return self.tools_by_id[ tool_id ] #exact tool id match not found, or all versions requested, search for other options, e.g. migrated tools or different versions rval = [] - tv = self.__get_tool_version( tool_id ) - if tv: - tool_version_ids = tv.get_version_ids( self.app ) + tool_lineage = self.lineage_map.get( tool_id ) + if tool_lineage: + tool_version_ids = tool_lineage.get_version_ids( ) for tool_version_id in tool_version_ids: if tool_version_id in self.tools_by_id: rval.append( self.tools_by_id[ tool_version_id ] ) @@ -560,9 +562,9 @@ def get_loaded_tools_by_lineage( self, tool_id ): """Get all loaded tools associated by lineage to the tool whose id is tool_id.""" - tv = self.__get_tool_version( tool_id ) - if tv: - tool_version_ids = tv.get_version_ids( self.app ) + tool_lineage = self.lineage_map.get( tool_id ) + if tool_lineage: + tool_version_ids = tool_lineage.get_version_ids( ) available_tool_versions = [] for tool_version_id in tool_version_ids: if tool_version_id in self.tools_by_id: @@ -574,12 +576,6 @@ return [ tool ] return [] - def __get_tool_version( self, tool_id ): - """Return a ToolVersion if one exists for the tool_id""" - return self.app.install_model.context.query( self.app.install_model.ToolVersion ) \ - .filter( self.app.install_model.ToolVersion.table.c.tool_id == tool_id ) \ - .first() - def tools( self ): return self.tools_by_id.iteritems() @@ -745,12 +741,10 @@ tool.guid = guid tool.version = elem.find( "version" ).text # Make sure the tool has a tool_version. - if not self.__get_tool_version( tool.id ): - tool_version = self.app.install_model.ToolVersion( tool_id=tool.id, tool_shed_repository=tool_shed_repository ) - self.app.install_model.context.add( tool_version ) - self.app.install_model.context.flush() + tool_lineage = self.lineage_map.register( tool, tool_shed_repository=tool_shed_repository ) # Load the tool's lineage ids. - tool.lineage_ids = tool.tool_version.get_version_ids( self.app ) + tool.lineage = tool_lineage + tool.lineage_ids = tool_lineage.get_version_ids( ) self.tool_tag_manager.handle_tags( tool.id, elem ) self.__add_tool( tool, load_panel_dict, panel_dict ) # Always load the tool into the integrated_panel_dict, or it will not be included in the integrated_tool_panel.xml file. @@ -1493,14 +1487,6 @@ return [] @property - def tool_version_ids( self ): - # If we have versions, return a list of their tool_ids. - tool_version = self.tool_version - if tool_version: - return tool_version.get_version_ids( self.app ) - return [] - - @property def tool_shed_repository( self ): # If this tool is included in an installed tool shed repository, return it. if self.tool_shed: diff -r 68255ebe3f54500aa26db814c41f67f382731462 -r b26dc879b09c639b47c76cb97b067073a3c956e6 lib/galaxy/tools/toolbox/lineages/__init__.py --- /dev/null +++ b/lib/galaxy/tools/toolbox/lineages/__init__.py @@ -0,0 +1,5 @@ +from .factory import LineageMap +from .interface import ToolLineage + + +__all__ = ["LineageMap", "ToolLineage"] diff -r 68255ebe3f54500aa26db814c41f67f382731462 -r b26dc879b09c639b47c76cb97b067073a3c956e6 lib/galaxy/tools/toolbox/lineages/factory.py --- /dev/null +++ b/lib/galaxy/tools/toolbox/lineages/factory.py @@ -0,0 +1,28 @@ +from .tool_shed import ToolShedLineage + + +class LineageMap(object): + """ Map each unique tool id to a lineage object. + """ + + def __init__(self, app): + self.lineage_map = {} + self.app = app + + def register(self, tool, **kwds): + tool_id = tool.id + if tool_id not in self.lineage_map: + tool_shed_repository = kwds.get("tool_shed_repository", None) + lineage = ToolShedLineage.from_tool(self.app, tool, tool_shed_repository) + self.lineage_map[tool_id] = lineage + return self.lineage_map[tool_id] + + def get(self, tool_id): + if tool_id not in self.lineage_map: + lineage = ToolShedLineage.from_tool_id( self.app, tool_id ) + if lineage: + self.lineage_map[tool_id] = lineage + + return self.lineage_map.get(tool_id, None) + +__all__ = ["LineageMap"] diff -r 68255ebe3f54500aa26db814c41f67f382731462 -r b26dc879b09c639b47c76cb97b067073a3c956e6 lib/galaxy/tools/toolbox/lineages/interface.py --- /dev/null +++ b/lib/galaxy/tools/toolbox/lineages/interface.py @@ -0,0 +1,14 @@ +from abc import ABCMeta +from abc import abstractmethod + + +class ToolLineage(object): + """ + """ + __metaclass__ = ABCMeta + + @abstractmethod + def get_version_ids( self, reverse=False ): + """ Return an ordered list of lineages in this chain, from + oldest to newest. + """ diff -r 68255ebe3f54500aa26db814c41f67f382731462 -r b26dc879b09c639b47c76cb97b067073a3c956e6 lib/galaxy/tools/toolbox/lineages/tool_shed.py --- /dev/null +++ b/lib/galaxy/tools/toolbox/lineages/tool_shed.py @@ -0,0 +1,43 @@ +from .interface import ToolLineage + +from galaxy.model.tool_shed_install import ToolVersion + + +class ToolShedLineage(ToolLineage): + """ Representation of tool lineage derived from tool shed repository + installations. """ + + def __init__(self, app, tool_version): + self.app = app + self.tool_version_id = tool_version.id + + @staticmethod + def from_tool( app, tool, tool_shed_repository ): + # Make sure the tool has a tool_version. + if not get_install_tool_version( app, tool.id ): + tool_version = ToolVersion( tool_id=tool.id, tool_shed_repository=tool_shed_repository ) + app.install_model.context.add( tool_version ) + app.install_model.context.flush() + return ToolShedLineage( app, tool.tool_version ) + + @staticmethod + def from_tool_id( app, tool_id ): + tool_version = get_install_tool_version( app, tool_id ) + if tool_version: + return ToolShedLineage( app, tool_version ) + else: + return None + + def get_version_ids( self, reverse=False ): + tool_version = self.app.install_model.context.query( ToolVersion ).get( self.tool_version_id ) + return tool_version.get_version_ids( self.app, reverse=reverse ) + + +def get_install_tool_version( app, tool_id ): + return app.install_model.context.query( + app.install_model.ToolVersion + ).filter( + app.install_model.ToolVersion.table.c.tool_id == tool_id + ).first() + +__all__ = [ "ToolShedLineage" ] https://bitbucket.org/galaxy/galaxy-central/commits/806202db57a7/ Changeset: 806202db57a7 User: jmchilton Date: 2014-12-29 15:28:37+00:00 Summary: Abstract out higher-level helpers in ToolBox for lineage reasoning. Reduces code duplication slightly and simplifies __add_tool_to_tool_panel (cyclomatic complexity reduced from 20 to 14). Affected #: 1 file diff -r b26dc879b09c639b47c76cb97b067073a3c956e6 -r 806202db57a7502c2aa5476f2db46f7aae75f439 lib/galaxy/tools/__init__.py --- a/lib/galaxy/tools/__init__.py +++ b/lib/galaxy/tools/__init__.py @@ -325,20 +325,13 @@ panel_dict = panel_component.elems else: panel_dict = panel_component - already_loaded = False - loaded_version_key = None - lineage_id = None - for lineage_id in tool.lineage_ids: - if lineage_id in self.tools_by_id: - loaded_version_key = 'tool_%s' % lineage_id - if loaded_version_key in panel_dict: - already_loaded = True - break - if already_loaded: - if tool.lineage_ids.index( tool_id ) > tool.lineage_ids.index( lineage_id ): + related_tool = self._lineage_in_panel( panel_dict, tool=tool ) + if related_tool: + if self._newer_tool( tool, related_tool ): + lineage_id = "tool_%s" % related_tool.id + index = panel_dict.keys().index( lineage_id ) + del panel_dict[ lineage_id ] key = 'tool_%s' % tool.id - index = panel_dict.keys().index( loaded_version_key ) - del panel_dict[ loaded_version_key ] panel_dict.insert( index, key, tool ) log.debug( "Loaded tool id: %s, version: %s into tool panel." % ( tool.id, tool.version ) ) else: @@ -374,12 +367,7 @@ else: # We are in the process of installing the tool. tool_lineage = self.lineage_map.get( tool_id ) - tool_lineage_ids = tool_lineage.get_version_ids( reverse=True ) - for lineage_id in tool_lineage_ids: - if lineage_id in self.tools_by_id: - loaded_version_key = 'tool_%s' % lineage_id - if loaded_version_key in panel_dict: - already_loaded = True + already_loaded = self._lineage_in_panel( panel_dict, tool_lineage=tool_lineage ) is not None if not already_loaded: # If the tool is not defined in integrated_tool_panel.xml, append it to the tool panel. panel_dict[ key ] = tool @@ -1193,6 +1181,27 @@ return rval + def _lineage_in_panel( self, panel_dict, tool=None, tool_lineage=None ): + """ If tool with same lineage already in panel (or section) - find + and return it. Otherwise return None. + """ + if tool_lineage is None: + assert tool is not None + tool_lineage = tool.lineage + lineage_ids = tool_lineage.get_version_ids( reverse=True ) + for lineage_id in lineage_ids: + if lineage_id in self.tools_by_id: + loaded_version_key = 'tool_%s' % lineage_id + if loaded_version_key in panel_dict: + return panel_dict[ loaded_version_key ] + return None + + def _newer_tool( self, tool1, tool2 ): + """ Return True if tool1 is considered "newer" given its own lineage + description. + """ + return tool1.lineage_ids.index( tool1.id ) > tool1.lineage_ids.index( tool2.id ) + def _filter_for_panel( item, filters, context ): """ Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: 12 new changesets
by commits-noreply＠bitbucket.org 29 Dec '14

29 Dec '14

12 new commits in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/8896bf67ff7a/ Changeset: 8896bf67ff7a User: jmchilton Date: 2014-12-29 14:11:54+00:00 Summary: Correct sample value for migrated tool config. Affected #: 1 file diff -r bfd679408c84a4c6f1d2b1a09eec1dab9c2d212b -r 8896bf67ff7ae6f4379ff0e18c30559a20cf6584 config/galaxy.ini.sample --- a/config/galaxy.ini.sample +++ b/config/galaxy.ini.sample @@ -148,7 +148,7 @@ # Tool config maintained by tool migration scripts. If you use the migration # scripts to install tools that have been migrated to the tool shed upon a new # release, they will be added to this tool config file. -#migrated_tools_config = migrated_tools_conf.xml +#migrated_tools_config = config/migrated_tools_conf.xml # File that contains the XML section and tool tags from all tool panel config # files integrated into a single file that defines the tool panel layout. This https://bitbucket.org/galaxy/galaxy-central/commits/d87f3d474c9a/ Changeset: d87f3d474c9a User: jmchilton Date: 2014-12-29 14:11:54+00:00 Summary: Small simplification to tool_panel_manager.py. Affected #: 1 file diff -r 8896bf67ff7ae6f4379ff0e18c30559a20cf6584 -r d87f3d474c9ab798e88286f80c09cef2bc41451a lib/tool_shed/galaxy_install/tools/tool_panel_manager.py --- a/lib/tool_shed/galaxy_install/tools/tool_panel_manager.py +++ b/lib/tool_shed/galaxy_install/tools/tool_panel_manager.py @@ -243,22 +243,13 @@ tool_file_path, tup_guid, tool = repository_tool_tup if tup_guid == guid: break - if inside_section: - tool_elem = self.generate_tool_elem( tool_shed, - repository_name, - changeset_revision, - owner, - tool_file_path, - tool, - tool_section ) - else: - tool_elem = self.generate_tool_elem( tool_shed, - repository_name, - changeset_revision, - owner, - tool_file_path, - tool, - None ) + tool_elem = self.generate_tool_elem( tool_shed, + repository_name, + changeset_revision, + owner, + tool_file_path, + tool, + tool_section if inside_section else None ) if inside_section: if section_in_elem_list: elem_list[ index ] = tool_section https://bitbucket.org/galaxy/galaxy-central/commits/000cd07f16f9/ Changeset: 000cd07f16f9 User: jmchilton Date: 2014-12-29 14:11:54+00:00 Summary: Reused get_section method to hide ToolBox.tool_panel from install_manager.py. Affected #: 4 files diff -r d87f3d474c9ab798e88286f80c09cef2bc41451a -r 000cd07f16f991cb78025e23f0872bafd58b9daf lib/galaxy/tools/__init__.py --- a/lib/galaxy/tools/__init__.py +++ b/lib/galaxy/tools/__init__.py @@ -277,13 +277,13 @@ self.write_integrated_tool_panel_config_file() app.reindex_tool_search() - def get_or_create_section( self, section_id, new_label=None ): + def get_section( self, section_id, new_label=None, create_if_needed=False ): tool_panel_section_key = str( section_id ) if tool_panel_section_key in self.tool_panel: # Appending a tool to an existing section in toolbox.tool_panel tool_section = self.tool_panel[ tool_panel_section_key ] log.debug( "Appending to tool panel section: %s" % str( tool_section.name ) ) - else: + elif create_if_needed: # Appending a new section to toolbox.tool_panel if new_label is None: # This might add an ugly section label to the tool panel, but, oh well... @@ -296,6 +296,8 @@ tool_section = ToolSection( section_dict ) self.tool_panel[ tool_panel_section_key ] = tool_section log.debug( "Loading new tool panel section: %s" % str( tool_section.name ) ) + else: + tool_section = None return tool_panel_section_key, tool_section def __resolve_tool_path(self, tool_path, config_filename): diff -r d87f3d474c9ab798e88286f80c09cef2bc41451a -r 000cd07f16f991cb78025e23f0872bafd58b9daf lib/tool_shed/galaxy_install/install_manager.py --- a/lib/tool_shed/galaxy_install/install_manager.py +++ b/lib/tool_shed/galaxy_install/install_manager.py @@ -809,12 +809,10 @@ def install_tool_shed_repository( self, tool_shed_repository, repo_info_dict, tool_panel_section_key, shed_tool_conf, tool_path, install_tool_dependencies, reinstalling=False ): if tool_panel_section_key: - try: - tool_section = self.app.toolbox.tool_panel[ tool_panel_section_key ] - except KeyError: + _, tool_section = self.app.toolbox.get_section( tool_panel_section_key ) + if tool_section is None: log.debug( 'Invalid tool_panel_section_key "%s" specified. Tools will be loaded outside of sections in the tool panel.', str( tool_panel_section_key ) ) - tool_section = None else: tool_section = None if isinstance( repo_info_dict, basestring ): diff -r d87f3d474c9ab798e88286f80c09cef2bc41451a -r 000cd07f16f991cb78025e23f0872bafd58b9daf lib/tool_shed/galaxy_install/tools/tool_panel_manager.py --- a/lib/tool_shed/galaxy_install/tools/tool_panel_manager.py +++ b/lib/tool_shed/galaxy_install/tools/tool_panel_manager.py @@ -297,7 +297,7 @@ return tool_section def get_or_create_tool_section( self, toolbox, tool_panel_section_id, new_tool_panel_section_label=None ): - return toolbox.get_or_create_section( section_id=tool_panel_section_id, new_label=new_tool_panel_section_label ) + return toolbox.get_section( section_id=tool_panel_section_id, new_label=new_tool_panel_section_label, create_if_needed=True ) def get_shed_tool_conf_dict( self, shed_tool_conf ): """ diff -r d87f3d474c9ab798e88286f80c09cef2bc41451a -r 000cd07f16f991cb78025e23f0872bafd58b9daf test/unit/tool_shed_unit_tests/test_tool_panel_manager.py --- a/test/unit/tool_shed_unit_tests/test_tool_panel_manager.py +++ b/test/unit/tool_shed_unit_tests/test_tool_panel_manager.py @@ -46,7 +46,7 @@ self.tool, ) ] - _, section = self.toolbox.get_or_create_section("tid1") + _, section = self.toolbox.get_section("tid1", create_if_needed=True) tpm = self.tpm tool_panel_dict = tpm.generate_tool_panel_dict_for_new_install( tool_dicts=new_tools, https://bitbucket.org/galaxy/galaxy-central/commits/739ca42b8555/ Changeset: 739ca42b8555 User: jmchilton Date: 2014-12-29 14:11:54+00:00 Summary: PEP-8 fixes for various Tool Shed components. - lib/tool_shed/util/tool_util.py - lib/tool_shed/galaxy_install/tool_migration_manager.py. - lib/tool_shed/util/shed_util_common.py. - lib/galaxy/model/tool_shed_install/__init__.py. Affected #: 4 files diff -r 000cd07f16f991cb78025e23f0872bafd58b9daf -r 739ca42b855530e8293cbb25298e1d4f6ef03d15 lib/galaxy/model/tool_shed_install/__init__.py --- a/lib/galaxy/model/tool_shed_install/__init__.py +++ b/lib/galaxy/model/tool_shed_install/__init__.py @@ -11,9 +11,9 @@ class ToolShedRepository( object ): dict_collection_visible_keys = ( 'id', 'tool_shed', 'name', 'owner', 'installed_changeset_revision', 'changeset_revision', 'ctx_rev', 'includes_datatypes', - 'tool_shed_status', 'deleted', 'uninstalled', 'dist_to_shed', 'status', 'error_message' ) + 'tool_shed_status', 'deleted', 'uninstalled', 'dist_to_shed', 'status', 'error_message' ) dict_element_visible_keys = ( 'id', 'tool_shed', 'name', 'owner', 'installed_changeset_revision', 'changeset_revision', 'ctx_rev', 'includes_datatypes', - 'tool_shed_status', 'deleted', 'uninstalled', 'dist_to_shed', 'status', 'error_message' ) + 'tool_shed_status', 'deleted', 'uninstalled', 'dist_to_shed', 'status', 'error_message' ) installation_status = Bunch( NEW='New', CLONING='Cloning', SETTING_TOOL_VERSIONS='Setting tool versions', @@ -356,12 +356,12 @@ for rd_tup in rd_tups: if len( rd_tup ) == 5: tool_shed, name, owner, changeset_revision, prior_installation_required, only_if_compiling_contained_td = \ - common_util.parse_repository_dependency_tuple( rd_tup, contains_error=False ) + common_util.parse_repository_dependency_tuple( rd_tup, contains_error=False ) if asbool( prior_installation_required ): required_rd_tups_that_must_be_installed.append( ( tool_shed, name, owner, changeset_revision, 'True', 'False' ) ) elif len( rd_tup ) == 6: tool_shed, name, owner, changeset_revision, prior_installation_required, only_if_compiling_contained_td = \ - common_util.parse_repository_dependency_tuple( rd_tup, contains_error=False ) + common_util.parse_repository_dependency_tuple( rd_tup, contains_error=False ) # The repository dependency will only be required to be previously installed if it does not fall into the category of # a repository that must be installed only so that its contained tool dependency can be used for compiling the tool # dependency of the dependent repository. @@ -493,12 +493,14 @@ class RepositoryRepositoryDependencyAssociation( object ): + def __init__( self, tool_shed_repository_id=None, repository_dependency_id=None ): self.tool_shed_repository_id = tool_shed_repository_id self.repository_dependency_id = repository_dependency_id class RepositoryDependency( object ): + def __init__( self, tool_shed_repository_id=None ): self.tool_shed_repository_id = tool_shed_repository_id @@ -647,6 +649,7 @@ class ToolVersionAssociation( object ): + def __init__( self, id=None, tool_id=None, parent_id=None ): self.id = id self.tool_id = tool_id @@ -654,6 +657,7 @@ class MigrateTools( object ): + def __init__( self, repository_id=None, repository_path=None, version=None ): self.repository_id = repository_id self.repository_path = repository_path diff -r 000cd07f16f991cb78025e23f0872bafd58b9daf -r 739ca42b855530e8293cbb25298e1d4f6ef03d15 lib/tool_shed/galaxy_install/tool_migration_manager.py --- a/lib/tool_shed/galaxy_install/tool_migration_manager.py +++ b/lib/tool_shed/galaxy_install/tool_migration_manager.py @@ -171,8 +171,6 @@ clone_dir = os.path.join( self.tool_path, relative_clone_dir ) if not self.__iscloned( clone_dir ): repository_clone_url = os.path.join( self.tool_shed_url, 'repos', owner, name ) - relative_install_dir = os.path.join( relative_clone_dir, name ) - install_dir = os.path.join( clone_dir, name ) ctx_rev = suc.get_ctx_rev( self.app, self.tool_shed_url, name, owner, changeset_revision ) tool_shed_repository = suc.create_or_update_tool_shed_repository( app=self.app, name=name, @@ -406,7 +404,7 @@ tool_configs_to_filter.append( tool_config ) else: print 'The tool "%s" (%s) has not been enabled because it is not defined in a proprietary tool config (%s).' \ - % ( guid, tool_config, ", ".join( self.proprietary_tool_confs or [] ) ) + % ( guid, tool_config, ", ".join( self.proprietary_tool_confs or [] ) ) if tool_configs_to_filter: lock = threading.Lock() lock.acquire( True ) @@ -421,7 +419,7 @@ repository=tool_shed_repository, changeset_revision=tool_shed_repository.changeset_revision, repository_clone_url=repository_clone_url, - shed_config_dict = self.shed_config_dict, + shed_config_dict=self.shed_config_dict, relative_install_dir=relative_install_dir, repository_files_dir=None, resetting_all_metadata_on_repository=False, @@ -504,11 +502,11 @@ datatypes_config = hg_util.get_config_from_disk( suc.DATATYPES_CONFIG_FILENAME, repo_install_dir ) # Load proprietary data types required by tools. The value of override is not # important here since the Galaxy server will be started after this installation - #completes. + # completes. converter_path, display_path = \ cdl.alter_config_and_load_prorietary_datatypes( datatypes_config, repo_install_dir, - override=False ) + override=False ) if converter_path or display_path: # Create a dictionary of tool shed repository related information. repository_dict = \ @@ -583,23 +581,23 @@ tool_version_dicts = json.loads( text ) tvm.handle_tool_versions( tool_version_dicts, tool_shed_repository ) else: - # Set the tool versions since they seem to be missing for this repository in the tool shed. - # CRITICAL NOTE: These default settings may not properly handle all parent/child associations. + # Set the tool versions since they seem to be missing + # for this repository in the tool shed. CRITICAL NOTE: + # These default settings may not properly handle all + # parent/child associations. for tool_dict in metadata_dict[ 'tools' ]: - flush_needed = False tool_id = tool_dict[ 'guid' ] old_tool_id = tool_dict[ 'id' ] - tool_version = tool_dict[ 'version' ] tool_version_using_old_id = tvm.get_tool_version( old_tool_id ) tool_version_using_guid = tvm.get_tool_version( tool_id ) if not tool_version_using_old_id: tool_version_using_old_id = self.app.install_model.ToolVersion( tool_id=old_tool_id, - tool_shed_repository=tool_shed_repository ) + tool_shed_repository=tool_shed_repository ) self.app.install_model.context.add( tool_version_using_old_id ) self.app.install_model.context.flush() if not tool_version_using_guid: tool_version_using_guid = self.app.install_model.ToolVersion( tool_id=tool_id, - tool_shed_repository=tool_shed_repository ) + tool_shed_repository=tool_shed_repository ) self.app.install_model.context.add( tool_version_using_guid ) self.app.install_model.context.flush() # Associate the two versions as parent / child. @@ -640,7 +638,6 @@ ordered_tsr_ids = [] processed_tsr_ids = [] prior_install_required_dict = self.get_prior_install_required_dict( tool_shed_repositories, repository_dependencies_dict ) - tsr_ids = [ tool_shed_repository.id for tool_shed_repository in tool_shed_repositories ] while len( processed_tsr_ids ) != len( prior_install_required_dict.keys() ): tsr_id = suc.get_next_prior_import_or_install_required_dict_entry( prior_install_required_dict, processed_tsr_ids ) processed_tsr_ids.append( tsr_id ) diff -r 000cd07f16f991cb78025e23f0872bafd58b9daf -r 739ca42b855530e8293cbb25298e1d4f6ef03d15 lib/tool_shed/util/shed_util_common.py --- a/lib/tool_shed/util/shed_util_common.py +++ b/lib/tool_shed/util/shed_util_common.py @@ -3,6 +3,7 @@ import os import re import shutil +import socket import string import sqlalchemy.orm.exc @@ -81,6 +82,7 @@ '${host}' """ + def create_or_update_tool_shed_repository( app, name, description, installed_changeset_revision, ctx_rev, repository_clone_url, metadata_dict, status, current_changeset_revision=None, owner='', dist_to_shed=False ): """ @@ -118,8 +120,8 @@ owner, installed_changeset_revision ) if tool_shed_repository: - log.debug( "Updating an existing row for repository '%s' in the tool_shed_repository table, status set to '%s'." % \ - ( str( name ), str( status ) ) ) + log.debug( "Updating an existing row for repository '%s' in the tool_shed_repository table, status set to '%s'." % + ( str( name ), str( status ) ) ) tool_shed_repository.description = description tool_shed_repository.changeset_revision = current_changeset_revision tool_shed_repository.ctx_rev = ctx_rev @@ -129,8 +131,8 @@ tool_shed_repository.uninstalled = uninstalled tool_shed_repository.status = status else: - log.debug( "Adding new row for repository '%s' in the tool_shed_repository table, status set to '%s'." % \ - ( str( name ), str( status ) ) ) + log.debug( "Adding new row for repository '%s' in the tool_shed_repository table, status set to '%s'." % + ( str( name ), str( status ) ) ) tool_shed_repository = \ app.install_model.ToolShedRepository( tool_shed=tool_shed, name=name, @@ -149,6 +151,7 @@ context.flush() return tool_shed_repository + def extract_components_from_tuple( repository_components_tuple ): '''Extract the repository components from the provided tuple in a backward-compatible manner.''' toolshed = repository_components_tuple[ 0 ] @@ -164,6 +167,7 @@ components_list = [ toolshed, name, owner, changeset_revision, prior_installation_required, only_if_compiling_contained_td ] return components_list + def generate_sharable_link_for_repository_in_tool_shed( repository, changeset_revision=None ): """Generate the URL for sharing a repository that is in the tool shed.""" base_url = url_for( '/', qualified=True ).rstrip( '/' ) @@ -173,6 +177,7 @@ sharable_url += '/%s' % changeset_revision return sharable_url + def generate_tool_guid( repository_clone_url, tool ): """ Generate a guid for the installed tool. It is critical that this guid matches the guid for @@ -182,6 +187,7 @@ tmp_url = common_util.remove_protocol_and_user_from_clone_url( repository_clone_url ) return '%s/%s/%s' % ( tmp_url, tool.id, tool.version ) + def generate_tool_shed_repository_install_dir( repository_clone_url, changeset_revision ): """ Generate a repository installation directory that guarantees repositories with the same @@ -196,6 +202,7 @@ tool_shed_url = common_util.remove_port_from_tool_shed_url( tool_shed_url ) return common_util.url_join( tool_shed_url, 'repos', repo_path, changeset_revision ) + def get_absolute_path_to_file_in_repository( repo_files_dir, file_name ): """Return the absolute path to a specified disk file contained in a repository.""" stripped_file_name = basic_util.strip_path( file_name ) @@ -207,19 +214,22 @@ return os.path.abspath( os.path.join( root, name ) ) return file_path + def get_categories( app ): """Get all categories from the database.""" sa_session = app.model.context.current return sa_session.query( app.model.Category ) \ - .filter( app.model.Category.table.c.deleted==False ) \ + .filter( app.model.Category.table.c.deleted == False ) \ .order_by( app.model.Category.table.c.name ) \ .all() + def get_category( app, id ): """Get a category from the database.""" sa_session = app.model.context.current return sa_session.query( app.model.Category ).get( app.security.decode_id( id ) ) + def get_category_by_name( app, name ): """Get a category from the database via name.""" sa_session = app.model.context.current @@ -228,6 +238,7 @@ except sqlalchemy.orm.exc.NoResultFound: return None + def get_ctx_rev( app, tool_shed_url, name, owner, changeset_revision ): """ Send a request to the tool shed to retrieve the ctx_rev for a repository defined by the @@ -240,6 +251,7 @@ ctx_rev = common_util.tool_shed_get( app, tool_shed_url, url ) return ctx_rev + def get_current_repository_metadata_for_changeset_revision( app, repository, changeset_revision ): encoded_repository_id = app.security.encode_id( repository.id ) repository_metadata = get_repository_metadata_by_changeset_revision( app, @@ -261,6 +273,7 @@ return repository_metadata return None + def get_ids_of_tool_shed_repositories_being_installed( app, as_string=False ): installing_repository_ids = [] new_status = app.install_model.ToolShedRepository.installation_status.NEW @@ -280,6 +293,7 @@ return ','.join( installing_repository_ids ) return installing_repository_ids + def get_latest_downloadable_changeset_revision( app, repository, repo ): repository_tip = repository.tip( app ) repository_metadata = get_repository_metadata_by_changeset_revision( app, app.security.encode_id( repository.id ), repository_tip ) @@ -290,6 +304,7 @@ return changeset_revisions[ -1 ] return hg_util.INITIAL_CHANGELOG_HASH + def get_next_downloadable_changeset_revision( repository, repo, after_changeset_revision ): """ Return the installable changeset_revision in the repository changelog after the changeset to which @@ -311,6 +326,7 @@ found_after_changeset_revision = True return None + def get_next_prior_import_or_install_required_dict_entry( prior_required_dict, processed_tsr_ids ): """ This method is used in the Tool Shed when exporting a repository and its dependencies, and in Galaxy @@ -344,6 +360,7 @@ continue return key + def get_ordered_metadata_changeset_revisions( repository, repo, downloadable=True ): """ Return an ordered list of changeset_revisions that are associated with metadata @@ -366,6 +383,7 @@ sorted_changeset_revisions = [ str( changeset_tup[ 1 ] ) for changeset_tup in sorted_changeset_tups ] return sorted_changeset_revisions + def get_prior_import_or_install_required_dict( app, tsr_ids, repo_info_dicts ): """ This method is used in the Tool Shed when exporting a repository and its dependencies, @@ -390,6 +408,7 @@ prior_import_or_install_required_dict[ encoded_repository_id ] = prior_import_or_install_ids return prior_import_or_install_required_dict + def get_repo_info_tuple_contents( repo_info_tuple ): """Take care in handling the repo_info_tuple as it evolves over time as new tool shed features are introduced.""" if len( repo_info_tuple ) == 6: @@ -399,6 +418,7 @@ description, repository_clone_url, changeset_revision, ctx_rev, repository_owner, repository_dependencies, tool_dependencies = repo_info_tuple return description, repository_clone_url, changeset_revision, ctx_rev, repository_owner, repository_dependencies, tool_dependencies + def get_repository_and_repository_dependencies_from_repo_info_dict( app, repo_info_dict ): """Return a tool_shed_repository or repository record defined by the information in the received repo_info_dict.""" repository_name = repo_info_dict.keys()[ 0 ] @@ -414,6 +434,7 @@ repository = get_repository_by_name_and_owner( app, repository_name, repository_owner ) return repository, repository_dependencies + def get_repository_by_id( app, id ): """Get a repository from the database via id.""" if is_tool_shed_client( app ): @@ -422,28 +443,31 @@ sa_session = app.model.context.current return sa_session.query( app.model.Repository ).get( app.security.decode_id( id ) ) + def get_repository_by_name( app, name ): """Get a repository from the database via name.""" repository_query = get_repository_query( app ) return repository_query.filter_by( name=name ).first() + def get_repository_by_name_and_owner( app, name, owner ): """Get a repository from the database via name and owner""" repository_query = get_repository_query( app ) if is_tool_shed_client( app ): return repository_query \ - .filter( and_( app.install_model.ToolShedRepository.table.c.name == name, - app.install_model.ToolShedRepository.table.c.owner == owner ) ) \ - .first() + .filter( and_( app.install_model.ToolShedRepository.table.c.name == name, + app.install_model.ToolShedRepository.table.c.owner == owner ) ) \ + .first() # We're in the tool shed. user = get_user_by_username( app, owner ) if user: return repository_query \ - .filter( and_( app.model.Repository.table.c.name == name, - app.model.Repository.table.c.user_id == user.id ) ) \ - .first() + .filter( and_( app.model.Repository.table.c.name == name, + app.model.Repository.table.c.user_id == user.id ) ) \ + .first() return None + def get_repository_dependency_types( repository_dependencies ): """ Inspect the received list of repository_dependencies tuples and return boolean values @@ -470,6 +494,7 @@ break return has_repository_dependencies, has_repository_dependencies_only_if_compiling_contained_td + def get_repository_for_dependency_relationship( app, tool_shed, name, owner, changeset_revision ): """ Return an installed tool_shed_repository database record that is defined by either the current changeset @@ -509,6 +534,7 @@ changeset_revision=text ) return repository + def get_repository_file_contents( file_path ): """Return the display-safe contents of a repository file for display in a browser.""" if checkers.is_gzip( file_path ): @@ -543,6 +569,7 @@ beginning_on_size_error=True ) return safe_str + def get_repository_files( folder_path ): """Return the file hierarchy of a tool shed repository.""" contents = [] @@ -558,6 +585,7 @@ contents.sort() return contents + def get_repository_from_refresh_on_change( app, **kwd ): # The changeset_revision_select_field in several grids performs a refresh_on_change which sends in request parameters like # changeset_revison_1, changeset_revision_2, etc. One of the many select fields on the grid performed the refresh_on_change, @@ -575,6 +603,7 @@ # This should never be reached - raise an exception? return v, None + def get_repository_ids_requiring_prior_import_or_install( app, tsr_ids, repository_dependencies ): """ This method is used in the Tool Shed when exporting a repository and its dependencies, @@ -593,11 +622,11 @@ continue for rd_tup in rd_tups: tool_shed, \ - name, \ - owner, \ - changeset_revision, \ - prior_installation_required, \ - only_if_compiling_contained_td = \ + name, \ + owner, \ + changeset_revision, \ + prior_installation_required, \ + only_if_compiling_contained_td = \ common_util.parse_repository_dependency_tuple( rd_tup ) # If only_if_compiling_contained_td is False, then the repository dependency # is not required to be installed prior to the dependent repository even if @@ -626,11 +655,13 @@ prior_tsr_ids.append( encoded_repository_id ) return prior_tsr_ids + def get_repository_in_tool_shed( app, id ): """Get a repository on the tool shed side from the database via id.""" sa_session = app.model.context.current return sa_session.query( app.model.Repository ).get( app.security.decode_id( id ) ) + def get_repository_metadata_by_changeset_revision( app, id, changeset_revision ): """Get metadata for a specified repository change set from the database.""" # Make sure there are no duplicate records, and return the single unique record for the changeset_revision. @@ -652,6 +683,7 @@ return all_metadata_records[ 0 ] return None + def get_repository_owner( cleaned_repository_url ): """Gvien a "cleaned" repository clone URL, return the owner of the repository.""" items = cleaned_repository_url.split( '/repos/' ) @@ -660,12 +692,13 @@ repo_path = repo_path.replace( '/', '', 1 ) return repo_path.lstrip( '/' ).split( '/' )[ 0 ] + def get_repository_owner_from_clone_url( repository_clone_url ): """Given a repository clone URL, return the owner of the repository.""" tmp_url = common_util.remove_protocol_and_user_from_clone_url( repository_clone_url ) - tool_shed = tmp_url.split( '/repos/' )[ 0 ].rstrip( '/' ) return get_repository_owner( tmp_url ) + def get_repository_query( app ): if is_tool_shed_client( app ): query = app.install_model.context.query( app.install_model.ToolShedRepository ) @@ -673,6 +706,7 @@ query = app.model.context.query( app.model.Repository ) return query + def get_tool_panel_config_tool_path_install_dir( app, repository ): """ Return shed-related tool panel config, the tool_path configured in it, and the relative path to @@ -690,12 +724,13 @@ # Just pick a semi-random shed config. for shed_config_dict in app.toolbox.shed_tool_confs: if ( repository.dist_to_shed and shed_config_dict[ 'config_filename' ] == app.config.migrated_tools_config ) \ - or ( not repository.dist_to_shed and shed_config_dict[ 'config_filename' ] != app.config.migrated_tools_config ): + or ( not repository.dist_to_shed and shed_config_dict[ 'config_filename' ] != app.config.migrated_tools_config ): break shed_tool_conf = shed_config_dict[ 'config_filename' ] tool_path = shed_config_dict[ 'tool_path' ] return shed_tool_conf, tool_path, relative_install_dir + def get_tool_path_by_shed_tool_conf_filename( app, shed_tool_conf ): """ Return the tool_path config setting for the received shed_tool_conf file by searching the tool box's in-memory list of shed_tool_confs for the @@ -711,10 +746,12 @@ return shed_tool_conf_dict[ 'tool_path' ] return None + def get_tool_shed_from_clone_url( repository_clone_url ): tmp_url = common_util.remove_protocol_and_user_from_clone_url( repository_clone_url ) return tmp_url.split( '/repos/' )[ 0 ].rstrip( '/' ) + def get_tool_shed_repository_by_id( app, repository_id ): """Return a tool shed repository database record defined by the id.""" # This method is used only in Galaxy, not the tool shed. @@ -722,6 +759,7 @@ .filter( app.install_model.ToolShedRepository.table.c.id == app.security.decode_id( repository_id ) ) \ .first() + def get_tool_shed_repository_by_shed_name_owner_changeset_revision( app, tool_shed, name, owner, changeset_revision ): """ Return a tool shed repository database record defined by the combination of a tool_shed, repository name, @@ -733,11 +771,12 @@ tool_shed = common_util.remove_protocol_from_tool_shed_url( tool_shed ) return repository_query \ .filter( and_( app.install_model.ToolShedRepository.table.c.tool_shed == tool_shed, - app.install_model.ToolShedRepository.table.c.name == name, - app.install_model.ToolShedRepository.table.c.owner == owner, - app.install_model.ToolShedRepository.table.c.changeset_revision == changeset_revision ) ) \ + app.install_model.ToolShedRepository.table.c.name == name, + app.install_model.ToolShedRepository.table.c.owner == owner, + app.install_model.ToolShedRepository.table.c.changeset_revision == changeset_revision ) ) \ .first() + def get_tool_shed_repository_by_shed_name_owner_installed_changeset_revision( app, tool_shed, name, owner, installed_changeset_revision ): """ @@ -750,11 +789,12 @@ tool_shed = common_util.remove_protocol_from_tool_shed_url( tool_shed ) return repository_query \ .filter( and_( app.install_model.ToolShedRepository.table.c.tool_shed == tool_shed, - app.install_model.ToolShedRepository.table.c.name == name, - app.install_model.ToolShedRepository.table.c.owner == owner, - app.install_model.ToolShedRepository.table.c.installed_changeset_revision == installed_changeset_revision ) ) \ + app.install_model.ToolShedRepository.table.c.name == name, + app.install_model.ToolShedRepository.table.c.owner == owner, + app.install_model.ToolShedRepository.table.c.installed_changeset_revision == installed_changeset_revision ) ) \ .first() + def get_tool_shed_status_for_installed_repository( app, repository ): """ Send a request to the tool shed to retrieve information about newer installable repository revisions, @@ -775,8 +815,8 @@ except HTTPError, e: # This should handle backward compatility to the Galaxy 12/20/12 release. We used to only handle updates for an installed revision # using a boolean value. - log.debug( "Error attempting to get tool shed status for installed repository %s: %s\nAttempting older 'check_for_updates' method.\n" % \ - ( str( repository.name ), str( e ) ) ) + log.debug( "Error attempting to get tool shed status for installed repository %s: %s\nAttempting older 'check_for_updates' method.\n" % + ( str( repository.name ), str( e ) ) ) params = '?name=%s&owner=%s&changeset_revision=%s&from_update_manager=True' % ( str( repository.name ), str( repository.owner ), str( repository.changeset_revision ) ) @@ -793,6 +833,7 @@ log.exception( "Error attempting to get tool shed status for installed repository %s: %s" % ( str( repository.name ), str( e ) ) ) return {} + def get_tool_shed_repository_status_label( app, tool_shed_repository=None, name=None, owner=None, changeset_revision=None, repository_clone_url=None ): """Return a color-coded label for the status of the received tool-shed_repository installed into Galaxy.""" if tool_shed_repository is None: @@ -842,6 +883,7 @@ status_label = 'unknown status' return '<div class="count-box state-color-%s">%s</div>' % ( bgcolor, status_label ) + def get_updated_changeset_revisions( app, name, owner, changeset_revision ): """ Return a string of comma-separated changeset revision hashes for all available updates to the received changeset @@ -863,6 +905,7 @@ return changeset_hashes_str return '' + def get_updated_changeset_revisions_from_tool_shed( app, tool_shed_url, name, owner, changeset_revision ): """ Get all appropriate newer changeset revisions for the repository defined by @@ -880,11 +923,13 @@ text = common_util.tool_shed_get( app, tool_shed_url, url ) return text + def get_user( app, id ): """Get a user from the database by id.""" sa_session = app.model.context.current return sa_session.query( app.model.User ).get( app.security.decode_id( id ) ) + def get_user_by_username( app, username ): """Get a user from the database by username.""" sa_session = app.model.context.current @@ -893,9 +938,10 @@ .filter( app.model.User.table.c.username == username ) \ .one() return user - except Exception, e: + except Exception: return None + def handle_email_alerts( app, host, repository, content_alert_str='', new_repo_alert=False, admin_only=False ): """ There are 2 complementary features that enable a tool shed user to receive email notification: @@ -947,7 +993,7 @@ admin_body = string.Template( template ).safe_substitute( host=host, sharable_link=sharable_link, repository_name=repository.name, - revision='%s:%s' %( str( ctx.rev() ), ctx ), + revision='%s:%s' % ( str( ctx.rev() ), ctx ), display_date=display_date, description=ctx.description(), username=username, @@ -955,7 +1001,7 @@ body = string.Template( template ).safe_substitute( host=host, sharable_link=sharable_link, repository_name=repository.name, - revision='%s:%s' %( str( ctx.rev() ), ctx ), + revision='%s:%s' % ( str( ctx.rev() ), ctx ), display_date=display_date, description=ctx.description(), username=username, @@ -985,9 +1031,10 @@ util.send_mail( frm, to, subject, admin_body, app.config ) else: util.send_mail( frm, to, subject, body, app.config ) - except Exception, e: + except Exception: log.exception( "An error occurred sending a tool shed repository update alert by email." ) + def have_shed_tool_conf_for_install( app ): if not app.toolbox.shed_tool_confs: return False @@ -999,6 +1046,7 @@ return True return False + def is_tool_shed_client( app ): """ The tool shed and clients to the tool (i.e. Galaxy) require a lot @@ -1008,6 +1056,7 @@ """ return hasattr( app, "install_model" ) + def open_repository_files_folder( folder_path ): """ Return a list of dictionaries, each of which contains information for a file or directory contained @@ -1026,14 +1075,15 @@ is_folder = True if filename: full_path = os.path.join( folder_path, filename ) - node = { "title" : filename, - "isFolder" : is_folder, - "isLazy" : is_folder, - "tooltip" : full_path, - "key" : full_path } + node = { "title": filename, + "isFolder": is_folder, + "isLazy": is_folder, + "tooltip": full_path, + "key": full_path } folder_contents.append( node ) return folder_contents + def repository_was_previously_installed( app, tool_shed_url, repository_name, repo_info_tuple, from_tip=False ): """ Find out if a repository is already installed into Galaxy - there are several scenarios where this @@ -1077,6 +1127,7 @@ return tool_shed_repository, previous_changeset_revision return None, None + def set_image_paths( app, encoded_repository_id, text ): """ Handle tool help image display for tools that are contained in repositories in @@ -1106,6 +1157,7 @@ text = re.sub( r'\.\. image:: (?!https?://)/?(.+)', r'.. image:: %s/\1' % route_to_images, text ) return text + def set_repository_attributes( app, repository, status, error_message, deleted, uninstalled, remove_from_disk=False ): if remove_from_disk: relative_install_dir = repository.repo_path( app ) @@ -1123,6 +1175,7 @@ app.install_model.context.add( repository ) app.install_model.context.flush() + def tool_shed_is_this_tool_shed( toolshed_base_url ): """Determine if a tool shed is the current tool shed.""" cleaned_toolshed_base_url = common_util.remove_protocol_from_tool_shed_url( toolshed_base_url ) diff -r 000cd07f16f991cb78025e23f0872bafd58b9daf -r 739ca42b855530e8293cbb25298e1d4f6ef03d15 lib/tool_shed/util/tool_util.py --- a/lib/tool_shed/util/tool_util.py +++ b/lib/tool_shed/util/tool_util.py @@ -5,15 +5,14 @@ import galaxy.tools from galaxy import util from galaxy.datatypes import checkers -from galaxy.tools import parameters from galaxy.util.expressions import ExpressionContext from galaxy.web.form_builder import SelectField -from galaxy.tools.actions.upload import UploadToolAction from tool_shed.util import basic_util log = logging.getLogger( __name__ ) + def build_shed_tool_conf_select_field( app ): """Build a SelectField whose options are the keys in app.toolbox.shed_tool_confs.""" options = [] @@ -30,6 +29,7 @@ select_field.add_option( option_tup[ 0 ], option_tup[ 1 ] ) return select_field + def build_tool_panel_section_select_field( app ): """Build a SelectField whose options are the sections of the current in-memory toolbox.""" options = [] @@ -41,6 +41,7 @@ select_field.add_option( option_tup[ 0 ], option_tup[ 1 ] ) return select_field + def copy_sample_file( app, filename, dest_path=None ): """ Copy xxx.sample to dest_path/xxx.sample and dest_path/xxx. The default value for dest_path @@ -61,6 +62,7 @@ if not os.path.exists( os.path.join( dest_path, copied_file ) ): shutil.copy( full_source_path, os.path.join( dest_path, copied_file ) ) + def copy_sample_files( app, sample_files, tool_path=None, sample_files_copied=None, dest_path=None ): """ Copy all appropriate files to dest_path in the local Galaxy environment that have not @@ -75,11 +77,12 @@ filename_sans_path = os.path.split( filename )[ 1 ] if filename_sans_path not in filenames_not_to_copy and filename not in sample_files_copied: if tool_path: - filename=os.path.join( tool_path, filename ) + filename = os.path.join( tool_path, filename ) # Attempt to ensure we're copying an appropriate file. if is_data_index_sample_file( filename ): copy_sample_file( app, filename, dest_path=dest_path ) + def generate_message_for_invalid_tools( app, invalid_file_tups, repository, metadata_dict, as_html=True, displaying_invalid_tool=False ): if as_html: @@ -124,6 +127,7 @@ message += "%s%s%s - %s%s" % ( bold_start, tool_file, bold_end, correction_msg, new_line ) return message + def get_headers( fname, sep, count=60, is_multi_byte=False ): """Returns a list with the first 'count' lines split by 'sep'.""" headers = [] @@ -137,6 +141,7 @@ break return headers + def get_tool_path_install_dir( partial_install_dir, shed_tool_conf_dict, tool_dict, config_elems ): for elem in config_elems: if elem.tag == 'tool': @@ -153,6 +158,7 @@ return tool_path, relative_install_dir return None, None + def handle_missing_index_file( app, tool_path, sample_files, repository_tools_tups, sample_files_copied ): """ Inspect each tool to see if it has any input parameters that are dynamically @@ -180,6 +186,7 @@ repository_tools_tups[ index ] = ( tup_path, guid, repository_tool ) return repository_tools_tups, sample_files_copied + def is_column_based( fname, sep='\t', skip=0, is_multi_byte=False ): """See if the file is column based with respect to a separator.""" headers = get_headers( fname, sep, is_multi_byte=is_multi_byte ) @@ -199,6 +206,7 @@ return False return True + def is_data_index_sample_file( file_path ): """ Attempt to determine if a .sample file is appropriate for copying to ~/tool-data when @@ -224,6 +232,7 @@ # Default to copying the file if none of the above are true. return True + def new_state( trans, tool, invalid=False ): """Create a new `DefaultToolState` for the received tool. Only inputs on the first page will be initialized.""" state = galaxy.tools.DefaultToolState() @@ -240,9 +249,10 @@ state.inputs[ input.name ] = [] return state + def panel_entry_per_tool( tool_section_dict ): # Return True if tool_section_dict looks like this. - # {<Tool guid> : + # {<Tool guid> : # [{ tool_config : <tool_config_file>, # id: <ToolSection id>, # version : <ToolSection version>, @@ -258,6 +268,7 @@ return True return False + def reload_upload_tools( app ): if hasattr( app, 'toolbox' ): app.toolbox.handle_datatypes_changed() https://bitbucket.org/galaxy/galaxy-central/commits/35d456f7edff/ Changeset: 35d456f7edff User: jmchilton Date: 2014-12-29 14:11:54+00:00 Summary: Method to abstract away ToolBox.shed_tool_confs from tool_util.py. Start a gradual process of referring to these as dynamic tool confs instead of shed tool confs. I would like like to make them more generally Galaxy-managed tool configurations and distinguish them from deployer-managed configurations - rather then having overtly ToolShed-only logic in ToolBox. Affected #: 2 files diff -r 739ca42b855530e8293cbb25298e1d4f6ef03d15 -r 35d456f7edffef9a8e4cd825503c3782f2be3617 lib/galaxy/tools/__init__.py --- a/lib/galaxy/tools/__init__.py +++ b/lib/galaxy/tools/__init__.py @@ -632,6 +632,16 @@ break return tool_version_select_field, tools, tool + def dynamic_conf_filenames( self, include_migrated_tool_conf=False ): + """ Return list of dynamic tool configuration filenames (shed_tools). + These must be used with various dynamic tool configuration update + operations (e.g. with update_shed_config). + """ + for dynamic_tool_conf_dict in self.shed_tool_confs: + dynamic_tool_conf_filename = dynamic_tool_conf_dict[ 'config_filename' ] + if include_migrated_tool_conf or (dynamic_tool_conf_filename != self.app.config.migrated_tools_config): + yield dynamic_tool_conf_filename + def build_tool_version_select_field( self, tools, tool_id, set_selected ): """Build a SelectField whose options are the ids for the received list of tools.""" options = [] diff -r 739ca42b855530e8293cbb25298e1d4f6ef03d15 -r 35d456f7edffef9a8e4cd825503c3782f2be3617 lib/tool_shed/util/tool_util.py --- a/lib/tool_shed/util/tool_util.py +++ b/lib/tool_shed/util/tool_util.py @@ -16,14 +16,12 @@ def build_shed_tool_conf_select_field( app ): """Build a SelectField whose options are the keys in app.toolbox.shed_tool_confs.""" options = [] - for shed_tool_conf_dict in app.toolbox.shed_tool_confs: - shed_tool_conf_filename = shed_tool_conf_dict[ 'config_filename' ] - if shed_tool_conf_filename != app.config.migrated_tools_config: - if shed_tool_conf_filename.startswith( './' ): - option_label = shed_tool_conf_filename.replace( './', '', 1 ) - else: - option_label = shed_tool_conf_filename - options.append( ( option_label, shed_tool_conf_filename ) ) + for dynamic_tool_conf_filename in app.toolbox.dynamic_conf_filenames(): + if dynamic_tool_conf_filename.startswith( './' ): + option_label = dynamic_tool_conf_filename.replace( './', '', 1 ) + else: + option_label = dynamic_tool_conf_filename + options.append( ( option_label, dynamic_tool_conf_filename ) ) select_field = SelectField( name='shed_tool_conf' ) for option_tup in options: select_field.add_option( option_tup[ 0 ], option_tup[ 1 ] ) https://bitbucket.org/galaxy/galaxy-central/commits/83dd43611918/ Changeset: 83dd43611918 User: jmchilton Date: 2014-12-29 14:11:54+00:00 Summary: Method to abstract away ToolBox.tool_panel from tool_util.py. Now only thing accessing ToolBox.tool_panel outside ToolBox itself is ToolPanelManager. Affected #: 2 files diff -r 35d456f7edffef9a8e4cd825503c3782f2be3617 -r 83dd436119182a688824218b0efda74e55e045c9 lib/galaxy/tools/__init__.py --- a/lib/galaxy/tools/__init__.py +++ b/lib/galaxy/tools/__init__.py @@ -1059,6 +1059,11 @@ status = 'done' return message, status + def get_sections( self ): + for k, v in self.tool_panel.items(): + if isinstance( v, ToolSection ): + yield (v.id, v.name) + def find_section_id( self, tool_panel_section_id ): """ Find the section ID referenced by the key or return '' indicating diff -r 35d456f7edffef9a8e4cd825503c3782f2be3617 -r 83dd436119182a688824218b0efda74e55e045c9 lib/tool_shed/util/tool_util.py --- a/lib/tool_shed/util/tool_util.py +++ b/lib/tool_shed/util/tool_util.py @@ -31,9 +31,8 @@ def build_tool_panel_section_select_field( app ): """Build a SelectField whose options are the sections of the current in-memory toolbox.""" options = [] - for k, v in app.toolbox.tool_panel.items(): - if isinstance( v, galaxy.tools.ToolSection ): - options.append( ( v.name, v.id ) ) + for section_id, section_name in app.toolbox.get_sections(): + options.append( ( section_name, section_id ) ) select_field = SelectField( name='tool_panel_section_id', display='radio' ) for option_tup in options: select_field.add_option( option_tup[ 0 ], option_tup[ 1 ] ) https://bitbucket.org/galaxy/galaxy-central/commits/6021c963ec92/ Changeset: 6021c963ec92 User: jmchilton Date: 2014-12-29 14:11:54+00:00 Summary: Reuse new ToolBox.get_section method in ToolPanelManager. This is a higher-level method and eliminates a couple more places where Tool Shed code is directly acceessing ToolBox.tool_panel. Affected #: 1 file diff -r 83dd436119182a688824218b0efda74e55e045c9 -r 6021c963ec925698353b134d070b403d2309609f lib/tool_shed/galaxy_install/tools/tool_panel_manager.py --- a/lib/tool_shed/galaxy_install/tools/tool_panel_manager.py +++ b/lib/tool_shed/galaxy_install/tools/tool_panel_manager.py @@ -325,8 +325,7 @@ tool_panel_section_id=section_id, new_tool_panel_section_label=new_tool_panel_section_label ) elif tool_panel_section_id: - tool_panel_section_key = str( tool_panel_section_id ) - tool_section = toolbox.tool_panel[ tool_panel_section_key ] + tool_panel_section_key, tool_section = toolbox.get_section( tool_panel_section_id) else: return None, None return tool_panel_section_key, tool_section @@ -495,8 +494,8 @@ # Remove the tool sub-element from the section element. config_elem.remove( tool_elem ) # Remove the tool from the section in the in-memory tool panel. - if section_key in toolbox.tool_panel: - tool_section = toolbox.tool_panel[ section_key ] + _, tool_section = toolbox.get_section( section_key ) + if tool_section: remove_from_panel( tool_elem, tool_section.elems, toolbox.integrated_tool_panel.get( section_key, {} ), section_key=section_key ) if len( config_elem ) < 1: # Keep a list of all empty section elements so they can be removed. https://bitbucket.org/galaxy/galaxy-central/commits/482b027099f5/ Changeset: 482b027099f5 User: jmchilton Date: 2014-12-29 14:11:54+00:00 Summary: Refactor logic for removing tools from panels into ToolBox. May seem sort of like Tool Shed specific stuff - but I think it can be reused elsewhere down the road. Regardless of this however as of this commit no external components directly access ToolBox.tool_panel or ToolBox.integrated_tool_panel - which I think is important - these should be internal details of the ToolBox (or newer modules it leverages). Affected #: 2 files diff -r 6021c963ec925698353b134d070b403d2309609f -r 482b027099f5aa0e15de928d093c6347a5297e18 lib/galaxy/tools/__init__.py --- a/lib/galaxy/tools/__init__.py +++ b/lib/galaxy/tools/__init__.py @@ -658,6 +658,55 @@ select_field.add_option( 'version %s' % option_tup[ 0 ], option_tup[ 1 ] ) return select_field + def remove_from_panel( self, tool_id, section_key='', remove_from_config=True ): + + def remove_from_dict( has_elems, integrated_has_elems ): + tool_key = 'tool_%s' % str( tool_id ) + available_tool_versions = self.get_loaded_tools_by_lineage( tool_id ) + if tool_key in has_elems: + if available_tool_versions: + available_tool_versions.reverse() + replacement_tool_key = None + replacement_tool_version = None + # Since we are going to remove the tool from the section, replace it with + # the newest loaded version of the tool. + for available_tool_version in available_tool_versions: + available_tool_section_id, available_tool_section_name = available_tool_version.get_panel_section() + # I suspect "available_tool_version.id in has_elems.keys()" doesn't + # belong in the following line or at least I don't understand what + # purpose it might serve. -John + if available_tool_version.id in has_elems.keys() or (available_tool_section_id == section_key): + replacement_tool_key = 'tool_%s' % str( available_tool_version.id ) + replacement_tool_version = available_tool_version + break + if replacement_tool_key and replacement_tool_version: + # Get the index of the tool_key in the tool_section. + for tool_panel_index, key in enumerate( has_elems.keys() ): + if key == tool_key: + break + # Remove the tool from the tool panel. + del has_elems[ tool_key ] + # Add the replacement tool at the same location in the tool panel. + has_elems.insert( tool_panel_index, + replacement_tool_key, + replacement_tool_version ) + else: + del has_elems[ tool_key ] + else: + del has_elems[ tool_key ] + if remove_from_config: + if hasattr( integrated_has_elems, "elems" ): + integrated_has_elems = integrated_has_elems.elems + if tool_key in integrated_has_elems: + del integrated_has_elems[ tool_key ] + + if section_key: + _, tool_section = self.get_section( section_key ) + if tool_section: + remove_from_dict( tool_section.elems, self.integrated_tool_panel.get( section_key, {} ) ) + else: + remove_from_dict( self.tool_panel, self.integrated_tool_panel ) + def load_tool_tag_set( self, elem, panel_dict, integrated_panel_dict, tool_path, load_panel_dict, guid=None, index=None ): try: path = elem.get( "file" ) diff -r 6021c963ec925698353b134d070b403d2309609f -r 482b027099f5aa0e15de928d093c6347a5297e18 lib/tool_shed/galaxy_install/tools/tool_panel_manager.py --- a/lib/tool_shed/galaxy_install/tools/tool_panel_manager.py +++ b/lib/tool_shed/galaxy_install/tools/tool_panel_manager.py @@ -426,50 +426,6 @@ def remove_guids( self, guids_to_remove, shed_tool_conf, uninstall ): toolbox = self.app.toolbox - - def remove_from_panel( tool_elem, has_elems, integrated_has_elems, section_key='' ): - # Hide tool from panel and promote next oldest version if - # available. - guid = tool_elem.get( 'guid' ) - tool_key = 'tool_%s' % str( guid ) - available_tool_versions = toolbox.get_loaded_tools_by_lineage( guid ) - if tool_key in has_elems: - if available_tool_versions: - available_tool_versions.reverse() - replacement_tool_key = None - replacement_tool_version = None - # Since we are going to remove the tool from the section, replace it with - # the newest loaded version of the tool. - for available_tool_version in available_tool_versions: - available_tool_section_id, available_tool_section_name = available_tool_version.get_panel_section() - # I suspect "available_tool_version.id in has_elems.keys()" doesn't - # belong in the following line or at least I don't understand what - # purpose it might serve. -John - if available_tool_version.id in has_elems.keys() or (available_tool_section_id == section_key): - replacement_tool_key = 'tool_%s' % str( available_tool_version.id ) - replacement_tool_version = available_tool_version - break - if replacement_tool_key and replacement_tool_version: - # Get the index of the tool_key in the tool_section. - for tool_panel_index, key in enumerate( has_elems.keys() ): - if key == tool_key: - break - # Remove the tool from the tool panel. - del has_elems[ tool_key ] - # Add the replacement tool at the same location in the tool panel. - has_elems.insert( tool_panel_index, - replacement_tool_key, - replacement_tool_version ) - else: - del has_elems[ tool_key ] - else: - del has_elems[ tool_key ] - if uninstall: - if hasattr( integrated_has_elems, "elems" ): - integrated_has_elems = integrated_has_elems.elems - if tool_key in integrated_has_elems: - del integrated_has_elems[ tool_key ] - # Remove the tools from the toolbox's tools_by_id dictionary. for guid_to_remove in guids_to_remove: # remove_from_tool_panel to false, will handling that logic below. @@ -494,17 +450,14 @@ # Remove the tool sub-element from the section element. config_elem.remove( tool_elem ) # Remove the tool from the section in the in-memory tool panel. - _, tool_section = toolbox.get_section( section_key ) - if tool_section: - remove_from_panel( tool_elem, tool_section.elems, toolbox.integrated_tool_panel.get( section_key, {} ), section_key=section_key ) + toolbox.remove_from_panel( tool_elem.get( "guid" ), section_key=section_key, remove_from_config=uninstall ) if len( config_elem ) < 1: # Keep a list of all empty section elements so they can be removed. config_elems_to_remove.append( config_elem ) elif config_elem.tag == 'tool': guid = config_elem.get( 'guid' ) if guid in guids_to_remove: - # get_panel_section return '' for un-sectioned tools. - remove_from_panel( config_elem, toolbox.tool_panel, toolbox.integrated_tool_panel, section_key='' ) + toolbox.remove_from_panel( guid, section_key='', remove_from_config=uninstall ) config_elems_to_remove.append( config_elem ) for config_elem in config_elems_to_remove: # Remove the element from the in-memory list of elements. https://bitbucket.org/galaxy/galaxy-central/commits/600d39d9bbac/ Changeset: 600d39d9bbac User: jmchilton Date: 2014-12-29 14:11:54+00:00 Summary: Update calls to ToolPanelManager.get_shed_tool_conf_dict for clarity... ... in other words to note where index in the resulting tuple is not actually used. Doing this to enable hiding this implementation detail (index) from components in subsequent commits. Affected #: 3 files diff -r 482b027099f5aa0e15de928d093c6347a5297e18 -r 600d39d9bbac52eb4321e65d16d1740a3f7c0871 lib/tool_shed/galaxy_install/install_manager.py --- a/lib/tool_shed/galaxy_install/install_manager.py +++ b/lib/tool_shed/galaxy_install/install_manager.py @@ -730,7 +730,7 @@ shed_tool_conf = install_options.get( 'shed_tool_conf', None ) if shed_tool_conf: # Get the tool_path setting. - index, shed_conf_dict = self.tpm.get_shed_tool_conf_dict( shed_tool_conf ) + _, shed_conf_dict = self.tpm.get_shed_tool_conf_dict( shed_tool_conf ) tool_path = shed_conf_dict[ 'tool_path' ] else: # Pick a semi-random shed-related tool panel configuration file and get the tool_path setting. diff -r 482b027099f5aa0e15de928d093c6347a5297e18 -r 600d39d9bbac52eb4321e65d16d1740a3f7c0871 lib/tool_shed/galaxy_install/metadata/installed_repository_metadata_manager.py --- a/lib/tool_shed/galaxy_install/metadata/installed_repository_metadata_manager.py +++ b/lib/tool_shed/galaxy_install/metadata/installed_repository_metadata_manager.py @@ -66,7 +66,7 @@ in the received tool shed repository metadata. """ repository_tools_tups = [] - index, shed_conf_dict = self.tpm.get_shed_tool_conf_dict( self.metadata_dict.get( 'shed_config_filename' ) ) + _, shed_conf_dict = self.tpm.get_shed_tool_conf_dict( self.metadata_dict.get( 'shed_config_filename' ) ) if 'tools' in self.metadata_dict: for tool_dict in self.metadata_dict[ 'tools' ]: load_relative_path = relative_path = tool_dict.get( 'tool_config', None ) diff -r 482b027099f5aa0e15de928d093c6347a5297e18 -r 600d39d9bbac52eb4321e65d16d1740a3f7c0871 lib/tool_shed/galaxy_install/tool_migration_manager.py --- a/lib/tool_shed/galaxy_install/tool_migration_manager.py +++ b/lib/tool_shed/galaxy_install/tool_migration_manager.py @@ -78,7 +78,7 @@ self.tool_shed_url = common_util.get_tool_shed_url_from_tool_shed_registry( self.app, defined_tool_shed_url ) self.tool_shed = common_util.remove_protocol_and_port_from_tool_shed_url( self.tool_shed_url ) self.repository_owner = common_util.REPOSITORY_OWNER - index, self.shed_config_dict = self.tpm.get_shed_tool_conf_dict( self.migrated_tools_config ) + _, self.shed_config_dict = self.tpm.get_shed_tool_conf_dict( self.migrated_tools_config ) # Since tool migration scripts can be executed any number of times, we need to # make sure the appropriate tools are defined in tool_conf.xml. If no tools # associated with the migration stage are defined, no repositories will be installed https://bitbucket.org/galaxy/galaxy-central/commits/7cee92f360c0/ Changeset: 7cee92f360c0 User: jmchilton Date: 2014-12-29 14:11:54+00:00 Summary: Update ToolBox.update_shed_config to use config_filename instead of index. Seems a more robust index, simplifies ToolPanelManager, etc.... Affected #: 2 files diff -r 600d39d9bbac52eb4321e65d16d1740a3f7c0871 -r 7cee92f360c0ca15d0f3f443c2e80a629d5eab1b lib/galaxy/tools/__init__.py --- a/lib/galaxy/tools/__init__.py +++ b/lib/galaxy/tools/__init__.py @@ -265,13 +265,15 @@ return shed_config_dict return default - def update_shed_config( self, shed_conf_index, shed_conf, integrated_panel_changes=True ): + def update_shed_config( self, shed_conf, integrated_panel_changes=True ): """ Update the in-memory descriptions of tools and write out the changes to integrated tool panel unless we are just deactivating a tool (since that doesn't affect that file). """ app = self.app - self.shed_tool_confs[ shed_conf_index ] = shed_conf + for index, my_shed_tool_conf in enumerate( self.shed_tool_confs ): + if shed_conf['config_filename'] == my_shed_tool_conf['config_filename']: + self.shed_tool_confs[ index ] = shed_conf if integrated_panel_changes and app.config.update_integrated_tool_panel: # Write the current in-memory version of the integrated_tool_panel.xml file to disk. self.write_integrated_tool_panel_config_file() diff -r 600d39d9bbac52eb4321e65d16d1740a3f7c0871 -r 7cee92f360c0ca15d0f3f443c2e80a629d5eab1b lib/tool_shed/galaxy_install/tools/tool_panel_manager.py --- a/lib/tool_shed/galaxy_install/tools/tool_panel_manager.py +++ b/lib/tool_shed/galaxy_install/tools/tool_panel_manager.py @@ -40,7 +40,7 @@ shed_tool_conf, tool_panel_dict, new_install=True ): """A tool shed repository is being installed or updated so handle tool panel alterations accordingly.""" # We need to change the in-memory version and the file system version of the shed_tool_conf file. - index, shed_tool_conf_dict = self.get_shed_tool_conf_dict( shed_tool_conf ) + _, shed_tool_conf_dict = self.get_shed_tool_conf_dict( shed_tool_conf ) tool_path = shed_tool_conf_dict[ 'tool_path' ] # Generate the list of ElementTree Element objects for each section or tool. elem_list = self.generate_tool_panel_elem_list( repository_name, @@ -66,7 +66,7 @@ ) # Replace the old list of in-memory config_elems with the new list for this shed_tool_conf_dict. shed_tool_conf_dict[ 'config_elems' ] = config_elems - self.app.toolbox.update_shed_config( index, shed_tool_conf_dict ) + self.app.toolbox.update_shed_config( shed_tool_conf_dict ) def config_elems_to_xml_file( self, config_elems, config_filename, tool_path ): """ @@ -430,7 +430,7 @@ for guid_to_remove in guids_to_remove: # remove_from_tool_panel to false, will handling that logic below. toolbox.remove_tool_by_id( guid_to_remove, remove_from_panel=False ) - index, shed_tool_conf_dict = self.get_shed_tool_conf_dict( shed_tool_conf ) + _, shed_tool_conf_dict = self.get_shed_tool_conf_dict( shed_tool_conf ) if uninstall: # Remove from the shed_tool_conf file on disk. self.remove_from_shed_tool_config( shed_tool_conf_dict, guids_to_remove ) @@ -464,4 +464,4 @@ config_elems.remove( config_elem ) # Update the config_elems of the in-memory shed_tool_conf_dict. shed_tool_conf_dict[ 'config_elems' ] = config_elems - toolbox.update_shed_config( index, shed_tool_conf_dict, integrated_panel_changes=uninstall ) + toolbox.update_shed_config( shed_tool_conf_dict, integrated_panel_changes=uninstall ) https://bitbucket.org/galaxy/galaxy-central/commits/22db1ba32b58/ Changeset: 22db1ba32b58 User: jmchilton Date: 2014-12-29 14:11:54+00:00 Summary: Simplifications for ToolPanelManager.get_shed_tool_conf_dict. None of the components using this method are using the index any more - so eliminate it. Also introduce a new ToolBox method to hide the details of ToolBox.shed_tool_confs from this method. Affected #: 6 files diff -r 7cee92f360c0ca15d0f3f443c2e80a629d5eab1b -r 22db1ba32b58093b68662db3f9f7f513e20c2bdf lib/galaxy/tools/__init__.py --- a/lib/galaxy/tools/__init__.py +++ b/lib/galaxy/tools/__init__.py @@ -634,15 +634,21 @@ break return tool_version_select_field, tools, tool + def dynamic_confs( self, include_migrated_tool_conf=False ): + confs = [] + for dynamic_tool_conf_dict in self.shed_tool_confs: + dynamic_tool_conf_filename = dynamic_tool_conf_dict[ 'config_filename' ] + if include_migrated_tool_conf or (dynamic_tool_conf_filename != self.app.config.migrated_tools_config): + confs.append( dynamic_tool_conf_dict ) + return confs + def dynamic_conf_filenames( self, include_migrated_tool_conf=False ): """ Return list of dynamic tool configuration filenames (shed_tools). These must be used with various dynamic tool configuration update operations (e.g. with update_shed_config). """ - for dynamic_tool_conf_dict in self.shed_tool_confs: - dynamic_tool_conf_filename = dynamic_tool_conf_dict[ 'config_filename' ] - if include_migrated_tool_conf or (dynamic_tool_conf_filename != self.app.config.migrated_tools_config): - yield dynamic_tool_conf_filename + for dynamic_tool_conf_dict in self.dynamic_confs( include_migrated_tool_conf=include_migrated_tool_conf ): + yield dynamic_tool_conf_dict[ 'config_filename' ] def build_tool_version_select_field( self, tools, tool_id, set_selected ): """Build a SelectField whose options are the ids for the received list of tools.""" diff -r 7cee92f360c0ca15d0f3f443c2e80a629d5eab1b -r 22db1ba32b58093b68662db3f9f7f513e20c2bdf lib/tool_shed/galaxy_install/install_manager.py --- a/lib/tool_shed/galaxy_install/install_manager.py +++ b/lib/tool_shed/galaxy_install/install_manager.py @@ -730,7 +730,7 @@ shed_tool_conf = install_options.get( 'shed_tool_conf', None ) if shed_tool_conf: # Get the tool_path setting. - _, shed_conf_dict = self.tpm.get_shed_tool_conf_dict( shed_tool_conf ) + shed_conf_dict = self.tpm.get_shed_tool_conf_dict( shed_tool_conf ) tool_path = shed_conf_dict[ 'tool_path' ] else: # Pick a semi-random shed-related tool panel configuration file and get the tool_path setting. diff -r 7cee92f360c0ca15d0f3f443c2e80a629d5eab1b -r 22db1ba32b58093b68662db3f9f7f513e20c2bdf lib/tool_shed/galaxy_install/metadata/installed_repository_metadata_manager.py --- a/lib/tool_shed/galaxy_install/metadata/installed_repository_metadata_manager.py +++ b/lib/tool_shed/galaxy_install/metadata/installed_repository_metadata_manager.py @@ -66,7 +66,7 @@ in the received tool shed repository metadata. """ repository_tools_tups = [] - _, shed_conf_dict = self.tpm.get_shed_tool_conf_dict( self.metadata_dict.get( 'shed_config_filename' ) ) + shed_conf_dict = self.tpm.get_shed_tool_conf_dict( self.metadata_dict.get( 'shed_config_filename' ) ) if 'tools' in self.metadata_dict: for tool_dict in self.metadata_dict[ 'tools' ]: load_relative_path = relative_path = tool_dict.get( 'tool_config', None ) diff -r 7cee92f360c0ca15d0f3f443c2e80a629d5eab1b -r 22db1ba32b58093b68662db3f9f7f513e20c2bdf lib/tool_shed/galaxy_install/tool_migration_manager.py --- a/lib/tool_shed/galaxy_install/tool_migration_manager.py +++ b/lib/tool_shed/galaxy_install/tool_migration_manager.py @@ -78,7 +78,7 @@ self.tool_shed_url = common_util.get_tool_shed_url_from_tool_shed_registry( self.app, defined_tool_shed_url ) self.tool_shed = common_util.remove_protocol_and_port_from_tool_shed_url( self.tool_shed_url ) self.repository_owner = common_util.REPOSITORY_OWNER - _, self.shed_config_dict = self.tpm.get_shed_tool_conf_dict( self.migrated_tools_config ) + self.shed_config_dict = self.tpm.get_shed_tool_conf_dict( self.migrated_tools_config ) # Since tool migration scripts can be executed any number of times, we need to # make sure the appropriate tools are defined in tool_conf.xml. If no tools # associated with the migration stage are defined, no repositories will be installed diff -r 7cee92f360c0ca15d0f3f443c2e80a629d5eab1b -r 22db1ba32b58093b68662db3f9f7f513e20c2bdf lib/tool_shed/galaxy_install/tools/tool_panel_manager.py --- a/lib/tool_shed/galaxy_install/tools/tool_panel_manager.py +++ b/lib/tool_shed/galaxy_install/tools/tool_panel_manager.py @@ -40,7 +40,7 @@ shed_tool_conf, tool_panel_dict, new_install=True ): """A tool shed repository is being installed or updated so handle tool panel alterations accordingly.""" # We need to change the in-memory version and the file system version of the shed_tool_conf file. - _, shed_tool_conf_dict = self.get_shed_tool_conf_dict( shed_tool_conf ) + shed_tool_conf_dict = self.get_shed_tool_conf_dict( shed_tool_conf ) tool_path = shed_tool_conf_dict[ 'tool_path' ] # Generate the list of ElementTree Element objects for each section or tool. elem_list = self.generate_tool_panel_elem_list( repository_name, @@ -304,13 +304,13 @@ Return the in-memory version of the shed_tool_conf file, which is stored in the config_elems entry in the shed_tool_conf_dict associated with the file. """ - for index, shed_tool_conf_dict in enumerate( self.app.toolbox.shed_tool_confs ): + for shed_tool_conf_dict in self.app.toolbox.dynamic_confs( include_migrated_tool_conf=True ): if shed_tool_conf == shed_tool_conf_dict[ 'config_filename' ]: - return index, shed_tool_conf_dict + return shed_tool_conf_dict else: file_name = basic_util.strip_path( shed_tool_conf_dict[ 'config_filename' ] ) if shed_tool_conf == file_name: - return index, shed_tool_conf_dict + return shed_tool_conf_dict def handle_tool_panel_section( self, toolbox, tool_panel_section_id=None, new_tool_panel_section_label=None ): """Return a ToolSection object retrieved from the current in-memory tool_panel.""" @@ -430,7 +430,7 @@ for guid_to_remove in guids_to_remove: # remove_from_tool_panel to false, will handling that logic below. toolbox.remove_tool_by_id( guid_to_remove, remove_from_panel=False ) - _, shed_tool_conf_dict = self.get_shed_tool_conf_dict( shed_tool_conf ) + shed_tool_conf_dict = self.get_shed_tool_conf_dict( shed_tool_conf ) if uninstall: # Remove from the shed_tool_conf file on disk. self.remove_from_shed_tool_config( shed_tool_conf_dict, guids_to_remove ) diff -r 7cee92f360c0ca15d0f3f443c2e80a629d5eab1b -r 22db1ba32b58093b68662db3f9f7f513e20c2bdf test/unit/tools_support.py --- a/test/unit/tools_support.py +++ b/test/unit/tools_support.py @@ -100,6 +100,7 @@ admin_users="mary(a)example.com", len_file_path=os.path.join( 'tool-data', 'shared', 'ucsc', 'chrom' ), builds_file_path=os.path.join( 'tool-data', 'shared', 'ucsc', 'builds.txt.sample' ), + migrated_tools_config=os.path.join(test_directory, "migrated_tools_conf.xml"), ) # Setup some attributes for downstream extension by specific tests. https://bitbucket.org/galaxy/galaxy-central/commits/68255ebe3f54/ Changeset: 68255ebe3f54 User: jmchilton Date: 2014-12-29 14:11:54+00:00 Summary: Reuse new ToolBox.dynamic_confs method other places... ... to shield ToolBox.shed_tool_confs from other components including Tool Shed install models, shed_util_common.py, and install_manager.py. After this commit - shed_tool_confs is only accessed directly by the ToolBox itself. Affected #: 3 files diff -r 22db1ba32b58093b68662db3f9f7f513e20c2bdf -r 68255ebe3f54500aa26db814c41f67f382731462 lib/galaxy/model/tool_shed_install/__init__.py --- a/lib/galaxy/model/tool_shed_install/__init__.py +++ b/lib/galaxy/model/tool_shed_install/__init__.py @@ -99,7 +99,7 @@ if not self.shed_config_filename: self.guess_shed_config( app, default=default ) if self.shed_config_filename: - for shed_tool_conf_dict in app.toolbox.shed_tool_confs: + for shed_tool_conf_dict in app.toolbox.dynamic_confs( include_migrated_tool_conf=True ): if self.shed_config_filename == shed_tool_conf_dict[ 'config_filename' ]: return shed_tool_conf_dict return default @@ -118,7 +118,7 @@ metadata = self.metadata or {} for tool in metadata.get( 'tools', [] ): tool_ids.append( tool.get( 'guid' ) ) - for shed_tool_conf_dict in app.toolbox.shed_tool_confs: + for shed_tool_conf_dict in app.toolbox.dynamic_confs( include_migrated_tool_conf=True ): name = shed_tool_conf_dict[ 'config_filename' ] for elem in shed_tool_conf_dict[ 'config_elems' ]: if elem.tag == 'tool': @@ -137,7 +137,7 @@ if self.includes_datatypes: # We need to search by file paths here, which is less desirable. tool_shed = common_util.remove_protocol_and_port_from_tool_shed_url( self.tool_shed ) - for shed_tool_conf_dict in app.toolbox.shed_tool_confs: + for shed_tool_conf_dict in app.toolbox.dynamic_confs( include_migrated_tool_conf=True ): tool_path = shed_tool_conf_dict[ 'tool_path' ] relative_path = os.path.join( tool_path, tool_shed, 'repos', self.owner, self.name, self.installed_changeset_revision ) if os.path.exists( relative_path ): @@ -284,7 +284,7 @@ def repo_path( self, app ): tool_shed = common_util.remove_protocol_and_port_from_tool_shed_url( self.tool_shed ) - for index, shed_tool_conf_dict in enumerate( app.toolbox.shed_tool_confs ): + for shed_tool_conf_dict in app.toolbox.dynamic_confs( include_migrated_tool_conf=True ): tool_path = shed_tool_conf_dict[ 'tool_path' ] relative_path = os.path.join( tool_path, tool_shed, 'repos', self.owner, self.name, self.installed_changeset_revision ) if os.path.exists( relative_path ): diff -r 22db1ba32b58093b68662db3f9f7f513e20c2bdf -r 68255ebe3f54500aa26db814c41f67f382731462 lib/tool_shed/galaxy_install/install_manager.py --- a/lib/tool_shed/galaxy_install/install_manager.py +++ b/lib/tool_shed/galaxy_install/install_manager.py @@ -733,15 +733,13 @@ shed_conf_dict = self.tpm.get_shed_tool_conf_dict( shed_tool_conf ) tool_path = shed_conf_dict[ 'tool_path' ] else: - # Pick a semi-random shed-related tool panel configuration file and get the tool_path setting. - for shed_config_dict in self.app.toolbox.shed_tool_confs: - # Don't use migrated_tools_conf.xml. - if shed_config_dict[ 'config_filename' ] != self.app.config.migrated_tools_config: - break + # Don't use migrated_tools_conf.xml. + try: + shed_config_dict = self.app.toolbox.dynamic_confs( include_migrated_tool_conf=False )[ 0 ] + except IndexError: + raise exceptions.RequestParameterMissingException( "Missing required parameter 'shed_tool_conf'." ) shed_tool_conf = shed_config_dict[ 'config_filename' ] tool_path = shed_config_dict[ 'tool_path' ] - if not shed_tool_conf: - raise exceptions.RequestParameterMissingException( "Missing required parameter 'shed_tool_conf'." ) tool_panel_section_id = self.app.toolbox.find_section_id( install_options.get( 'tool_panel_section_id', '' ) ) # Build the dictionary of information necessary for creating tool_shed_repository database records # for each repository being installed. diff -r 22db1ba32b58093b68662db3f9f7f513e20c2bdf -r 68255ebe3f54500aa26db814c41f67f382731462 lib/tool_shed/util/shed_util_common.py --- a/lib/tool_shed/util/shed_util_common.py +++ b/lib/tool_shed/util/shed_util_common.py @@ -722,7 +722,7 @@ shed_config_dict = repository.get_shed_config_dict( app ) if not shed_config_dict: # Just pick a semi-random shed config. - for shed_config_dict in app.toolbox.shed_tool_confs: + for shed_config_dict in app.toolbox.dynamic_confs( include_migrated_tool_conf=True ): if ( repository.dist_to_shed and shed_config_dict[ 'config_filename' ] == app.config.migrated_tools_config ) \ or ( not repository.dist_to_shed and shed_config_dict[ 'config_filename' ] != app.config.migrated_tools_config ): break @@ -736,7 +736,7 @@ Return the tool_path config setting for the received shed_tool_conf file by searching the tool box's in-memory list of shed_tool_confs for the dictionary whose config_filename key has a value matching the received shed_tool_conf. """ - for shed_tool_conf_dict in app.toolbox.shed_tool_confs: + for shed_tool_conf_dict in app.toolbox.dynamic_confs( include_migrated_tool_conf=True ): config_filename = shed_tool_conf_dict[ 'config_filename' ] if config_filename == shed_tool_conf: return shed_tool_conf_dict[ 'tool_path' ] @@ -1036,15 +1036,7 @@ def have_shed_tool_conf_for_install( app ): - if not app.toolbox.shed_tool_confs: - return False - migrated_tools_conf_path, migrated_tools_conf_name = os.path.split( app.config.migrated_tools_config ) - for shed_tool_conf_dict in app.toolbox.shed_tool_confs: - shed_tool_conf = shed_tool_conf_dict[ 'config_filename' ] - shed_tool_conf_path, shed_tool_conf_name = os.path.split( shed_tool_conf ) - if shed_tool_conf_name != migrated_tools_conf_name: - return True - return False + return bool( app.toolbox.dynamic_confs( include_migrated_tool_conf=False ) ) def is_tool_shed_client( app ): Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: jmchilton: Bugfix for 44dfa6f and typos in unit tests that caused it to go undetected.
by commits-noreply＠bitbucket.org 28 Dec '14

28 Dec '14

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/bfd679408c84/ Changeset: bfd679408c84 User: jmchilton Date: 2014-12-29 04:25:56+00:00 Summary: Bugfix for 44dfa6f and typos in unit tests that caused it to go undetected. Affected #: 3 files diff -r 1912dec92b728b974094456be8eb6d84d34a7ca6 -r bfd679408c84a4c6f1d2b1a09eec1dab9c2d212b lib/tool_shed/galaxy_install/tools/tool_panel_manager.py --- a/lib/tool_shed/galaxy_install/tools/tool_panel_manager.py +++ b/lib/tool_shed/galaxy_install/tools/tool_panel_manager.py @@ -475,8 +475,10 @@ else: del has_elems[ tool_key ] if uninstall: - if tool_key in toolbox.integrated_tool_panel: - del toolbox.integrated_tool_panel[ tool_key ] + if hasattr( integrated_has_elems, "elems" ): + integrated_has_elems = integrated_has_elems.elems + if tool_key in integrated_has_elems: + del integrated_has_elems[ tool_key ] # Remove the tools from the toolbox's tools_by_id dictionary. for guid_to_remove in guids_to_remove: diff -r 1912dec92b728b974094456be8eb6d84d34a7ca6 -r bfd679408c84a4c6f1d2b1a09eec1dab9c2d212b test/unit/tool_shed_unit_tests/test_tool_panel_manager.py --- a/test/unit/tool_shed_unit_tests/test_tool_panel_manager.py +++ b/test/unit/tool_shed_unit_tests/test_tool_panel_manager.py @@ -67,10 +67,10 @@ self._setup_two_versions_remove_one( section=True, uninstall=False ) self._verify_version_2_removed_from_panel( ) # Still in tool conf since not uninstalled only deactivated... - assert "github.com/galaxyproect/example/test_tool/0.2" in open(os.path.join(self.test_directory, "tool_conf.xml"), "r").read() + assert "github.com/galaxyproject/example/test_tool/0.2" in open(os.path.join(self.test_directory, "tool_conf.xml"), "r").read() self._verify_tool_confs() - self._remove_guids( ["github.com/galaxyproect/example/test_tool/0.1"], uninstall=False ) + self._remove_guids( ["github.com/galaxyproject/example/test_tool/0.1"], uninstall=False ) # Now no versions of this tool are returned by toolbox. all_versions = self.toolbox.get_tool( "test_tool", get_all_versions=True ) @@ -84,24 +84,25 @@ self._setup_two_versions_remove_one( section=True, uninstall=True ) self._verify_version_2_removed_from_panel( ) # Not in tool conf because it was uninstalled. - assert "github.com/galaxyproect/example/test_tool/0.2" not in open(os.path.join(self.test_directory, "tool_conf.xml"), "r").read() + assert "github.com/galaxyproject/example/test_tool/0.2" not in open(os.path.join(self.test_directory, "tool_conf.xml"), "r").read() + assert "tool_github.com/galaxyproject/example/test_tool/0.2" not in self.toolbox.integrated_tool_panel["tid"].elems self._verify_tool_confs() def test_deactivate_outside_section( self ): self._setup_two_versions_remove_one( section=False, uninstall=False ) self._verify_version_2_removed_from_panel( section=False ) # Still in tool conf since not uninstalled only deactivated... - assert "github.com/galaxyproect/example/test_tool/0.2" in open(os.path.join(self.test_directory, "tool_conf.xml"), "r").read() + assert "github.com/galaxyproject/example/test_tool/0.2" in open(os.path.join(self.test_directory, "tool_conf.xml"), "r").read() self._verify_tool_confs() def test_uninstall_outside_section( self ): self._setup_two_versions_remove_one( section=False, uninstall=True ) self._verify_version_2_removed_from_panel( section=False ) # Still in tool conf since not uninstalled only deactivated... - assert "github.com/galaxyproect/example/test_tool/0.2" not in open(os.path.join(self.test_directory, "tool_conf.xml"), "r").read() + assert "github.com/galaxyproject/example/test_tool/0.2" not in open(os.path.join(self.test_directory, "tool_conf.xml"), "r").read() self._verify_tool_confs() - self._remove_guids( ["github.com/galaxyproect/example/test_tool/0.1"], uninstall=True ) + self._remove_guids( ["github.com/galaxyproject/example/test_tool/0.1"], uninstall=True ) # Now no versions of this tool are returned by toolbox. all_versions = self.toolbox.get_tool( "test_tool", get_all_versions=True ) @@ -116,7 +117,7 @@ self._setup_two_versions_in_config( section=True ) self._setup_two_versions() self.toolbox - self._remove_guids( ["github.com/galaxyproect/example/test_tool/0.2"], uninstall=uninstall ) + self._remove_guids( ["github.com/galaxyproject/example/test_tool/0.2"], uninstall=uninstall ) def _verify_version_2_removed_from_panel( self, section=True ): # Check that test_tool now only has one version... @@ -127,12 +128,12 @@ if section: section = self.toolbox.tool_panel["tid"] assert len(section.elems) == 1 - assert section.elems.values()[0].id == "github.com/galaxyproect/example/test_tool/0.1" + assert section.elems.values()[0].id == "github.com/galaxyproject/example/test_tool/0.1" - assert "github.com/galaxyproect/example/test_tool/0.2" not in self.toolbox.integrated_tool_panel["tid"].elems + assert "github.com/galaxyproject/example/test_tool/0.2" not in self.toolbox.integrated_tool_panel["tid"].elems else: - self.toolbox.tool_panel.values()[0].id == "github.com/galaxyproect/example/test_tool/0.1" - assert "github.com/galaxyproect/example/test_tool/0.2" not in self.toolbox.integrated_tool_panel + self.toolbox.tool_panel.values()[0].id == "github.com/galaxyproject/example/test_tool/0.1" + assert "github.com/galaxyproject/example/test_tool/0.2" not in self.toolbox.integrated_tool_panel def _remove_guids( self, guids, uninstall, shed_tool_conf="tool_conf.xml" ): self.tpm.remove_guids( diff -r 1912dec92b728b974094456be8eb6d84d34a7ca6 -r bfd679408c84a4c6f1d2b1a09eec1dab9c2d212b test/unit/tools/test_toolbox.py --- a/test/unit/tools/test_toolbox.py +++ b/test/unit/tools/test_toolbox.py @@ -10,12 +10,12 @@ CONFIG_TEST_TOOL_VERSION_TEMPLATE = string.Template( - """ <tool file="tool.xml" guid="github.com/galaxyproect/example/test_tool/0.${version}"> + """ <tool file="tool.xml" guid="github.com/galaxyproject/example/test_tool/0.${version}"><tool_shed>github.com</tool_shed><repository_name>example</repository_name><repository_owner>galaxyproject</repository_owner><installed_changeset_revision>${version}</installed_changeset_revision> - <id>github.com/galaxyproect/example/test_tool/0.${version}</id> + <id>github.com/galaxyproject/example/test_tool/0.${version}</id><version>0.${version}</version></tool> """ @@ -72,14 +72,14 @@ def _setup_two_versions( self ): repository1 = self._repo_install( changeset="1" ) version1 = tool_shed_install.ToolVersion() - version1.tool_id = "github.com/galaxyproect/example/test_tool/0.1" + version1.tool_id = "github.com/galaxyproject/example/test_tool/0.1" version1.repository = repository1 self.app.install_model.context.add( version1 ) self.app.install_model.context.flush( ) repository2 = self._repo_install( changeset="2" ) version2 = tool_shed_install.ToolVersion() - version2.tool_id = "github.com/galaxyproect/example/test_tool/0.2" + version2.tool_id = "github.com/galaxyproject/example/test_tool/0.2" version2.repository = repository2 self.app.install_model.context.add( version2 ) @@ -184,7 +184,7 @@ # Assert only newer version of the tool loaded into the panel. section = self.toolbox.tool_panel["tid"] assert len(section.elems) == 1 - assert section.elems.values()[0].id == "github.com/galaxyproect/example/test_tool/0.2" + assert section.elems.values()[0].id == "github.com/galaxyproject/example/test_tool/0.2" def test_group_tools_out_of_section( self ): self._init_tool() @@ -212,10 +212,10 @@ self._init_tool() self._setup_two_versions_in_config( ) self._setup_two_versions() - assert self.toolbox.get_tool_id( "test_tool" ) == "github.com/galaxyproect/example/test_tool/0.1" - assert self.toolbox.get_tool_id( "github.com/galaxyproect/example/test_tool/0.1" ) == "github.com/galaxyproect/example/test_tool/0.1" - assert self.toolbox.get_tool_id( "github.com/galaxyproect/example/test_tool/0.2" ) == "github.com/galaxyproect/example/test_tool/0.2" - assert self.toolbox.get_tool_id( "github.com/galaxyproect/example/test_tool/0.3" ) is None + assert self.toolbox.get_tool_id( "test_tool" ) == "github.com/galaxyproject/example/test_tool/0.1" + assert self.toolbox.get_tool_id( "github.com/galaxyproject/example/test_tool/0.1" ) == "github.com/galaxyproject/example/test_tool/0.1" + assert self.toolbox.get_tool_id( "github.com/galaxyproject/example/test_tool/0.2" ) == "github.com/galaxyproject/example/test_tool/0.2" + assert self.toolbox.get_tool_id( "github.com/galaxyproject/example/test_tool/0.3" ) is None def test_tool_dir( self ): self._init_tool() @@ -285,14 +285,14 @@ # Verify lineage_ids on both tools is correctly ordered. for version in ["0.1", "0.2"]: - guid = "github.com/galaxyproect/example/test_tool/" + version + guid = "github.com/galaxyproject/example/test_tool/" + version lineage_ids = self.toolbox.get_tool( guid ).lineage_ids - assert lineage_ids[ 0 ] == "github.com/galaxyproect/example/test_tool/0.1" - assert lineage_ids[ 1 ] == "github.com/galaxyproect/example/test_tool/0.2" + assert lineage_ids[ 0 ] == "github.com/galaxyproject/example/test_tool/0.1" + assert lineage_ids[ 1 ] == "github.com/galaxyproject/example/test_tool/0.2" # Test tool_version attribute. - assert self.toolbox.get_tool( "test_tool", tool_version="0.1" ).guid == "github.com/galaxyproect/example/test_tool/0.1" - assert self.toolbox.get_tool( "test_tool", tool_version="0.2" ).guid == "github.com/galaxyproect/example/test_tool/0.2" + assert self.toolbox.get_tool( "test_tool", tool_version="0.1" ).guid == "github.com/galaxyproject/example/test_tool/0.1" + assert self.toolbox.get_tool( "test_tool", tool_version="0.2" ).guid == "github.com/galaxyproject/example/test_tool/0.2" def __remove_itp( self ): os.remove( os.path) Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0