- galaxy-commits - lists.galaxyproject.org

commit/galaxy-central: jmchilton: Expose improved sample tracking to tools for implicit map/reduce ops.
by commits-noreply＠bitbucket.org 02 Feb '15

02 Feb '15

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/b9ec027d4b1e/ Changeset: b9ec027d4b1e User: jmchilton Date: 2015-02-02 21:29:42+00:00 Summary: Expose improved sample tracking to tools for implicit map/reduce ops. Tools may now use $input.element_identifier during tool evalution for input 'data' parameters with the following semantics: - If the input was specified as a single dataset by the user - this just fallbacks to providing the $input.name. - If the input was mapped over a collection (to produce many jobs) or if the input is a 'multiple="true"' input that was provided a collection - the $input.element_identifier will be the element identifier for the corresponding collection item (generally much more useful the dataset name - since if preserved throughout workflows). 'data_collection' parameters already can access this kind of information - but it is something of a best practice to use simple 'data' parameters since they are compatible with more traditional un-collected datasets. This commit really needs more comments - but Philip Mabon has been patiently waiting for this functionality for a long time. Affected #: 9 files diff -r ff3c5721f74857cf689966abfdeb0dc9390bf605 -r b9ec027d4b1e1b67f26632a55076dbf53e3225bd lib/galaxy/model/__init__.py --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -2685,6 +2685,16 @@ return instances @property + def dataset_elements( self ): + elements = [] + for element in self.elements: + if element.is_collection: + elements.extend( element.child_collection.dataset_elements ) + else: + elements.append( element ) + return elements + + @property def state( self ): # TODO: DatasetCollection state handling... return 'ok' diff -r ff3c5721f74857cf689966abfdeb0dc9390bf605 -r b9ec027d4b1e1b67f26632a55076dbf53e3225bd lib/galaxy/tools/actions/__init__.py --- a/lib/galaxy/tools/actions/__init__.py +++ b/lib/galaxy/tools/actions/__init__.py @@ -191,6 +191,10 @@ if data.dbkey not in [None, '?']: input_dbkey = data.dbkey + identifier = getattr( data, "element_identifier", None ) + if identifier is not None: + incoming[ "%s|__identifier__" % name ] = identifier + # Collect chromInfo dataset and add as parameters to incoming ( chrom_info, db_dataset ) = trans.app.genome_builds.get_chrom_info( input_dbkey, trans=trans, custom_build_hack_get_len_from_fasta_conversion=tool.id != 'CONVERTER_fasta_to_len' ) if db_dataset: diff -r ff3c5721f74857cf689966abfdeb0dc9390bf605 -r b9ec027d4b1e1b67f26632a55076dbf53e3225bd lib/galaxy/tools/evaluation.py --- a/lib/galaxy/tools/evaluation.py +++ b/lib/galaxy/tools/evaluation.py @@ -154,7 +154,7 @@ if isinstance( input, DataToolParameter ) and input.multiple: dataset_instances = input_values[ input.name ] if isinstance( dataset_instances, model.HistoryDatasetCollectionAssociation ): - dataset_instances = dataset_instances.collection.dataset_instances[:] + dataset_instances = dataset_instances.collection.dataset_elements[:] input_values[ input.name ] = \ DatasetListWrapper( dataset_instances, dataset_paths=input_dataset_paths, @@ -199,6 +199,9 @@ tool=self, name=input.name ) + identifier_key = "%s|__identifier__" % input.name + if identifier_key in param_dict: + wrapper_kwds["identifier"] = param_dict[identifier_key] if dataset: #A None dataset does not have a filename real_path = dataset.file_name diff -r ff3c5721f74857cf689966abfdeb0dc9390bf605 -r b9ec027d4b1e1b67f26632a55076dbf53e3225bd lib/galaxy/tools/parameters/meta.py --- a/lib/galaxy/tools/parameters/meta.py +++ b/lib/galaxy/tools/parameters/meta.py @@ -14,6 +14,13 @@ execution). """ + to_remove = [] + for key in incoming.keys(): + if key.endswith("|__identifier__"): + to_remove.append(key) + for key in to_remove: + incoming.pop(key) + def classify_unmodified_parameter( input_key ): value = incoming[ input_key ] if isinstance( value, dict ) and 'values' in value: @@ -118,7 +125,11 @@ subcollection_elements = subcollections.split_dataset_collection_instance( hdc, subcollection_type ) return subcollection_elements else: - hdas = hdc.collection.dataset_instances + hdas = [] + for element in hdc.collection.dataset_elements: + hda = element.dataset_instance + hda.element_identifier = element.element_identifier + hdas.append( hda ) return hdas diff -r ff3c5721f74857cf689966abfdeb0dc9390bf605 -r b9ec027d4b1e1b67f26632a55076dbf53e3225bd lib/galaxy/tools/wrappers.py --- a/lib/galaxy/tools/wrappers.py +++ b/lib/galaxy/tools/wrappers.py @@ -187,7 +187,7 @@ def items( self ): return iter( [ ( k, self.get( k ) ) for k, v in self.metadata.items() ] ) - def __init__( self, dataset, datatypes_registry=None, tool=None, name=None, dataset_path=None ): + def __init__( self, dataset, datatypes_registry=None, tool=None, name=None, dataset_path=None, identifier=None ): if not dataset: try: # TODO: allow this to work when working with grouping @@ -205,6 +205,14 @@ self.datatypes_registry = datatypes_registry self.false_path = getattr( dataset_path, "false_path", None ) self.false_extra_files_path = getattr( dataset_path, "false_extra_files_path", None ) + self._element_identifier = identifier + + @property + def element_identifier( self ): + identifier = self._element_identifier + if identifier is None: + identifier = self.name + return identifier @property def is_collection( self ): @@ -270,6 +278,10 @@ datasets = [datasets] def to_wrapper( dataset ): + if hasattr(dataset, "element_identifier"): + element = dataset + dataset = element.dataset_instance + kwargs["identifier"] = element.element_identifier return self._dataset_wrapper( dataset, dataset_paths, **kwargs ) list.__init__( self, map( to_wrapper, datasets ) ) diff -r ff3c5721f74857cf689966abfdeb0dc9390bf605 -r b9ec027d4b1e1b67f26632a55076dbf53e3225bd test/api/test_tools.py --- a/test/api/test_tools.py +++ b/test/api/test_tools.py @@ -553,6 +553,93 @@ self.assertEquals( output1_content.strip(), "123" ) self.assertEquals( output2_content.strip(), "456" ) + @skip_without_tool( "identifier_single" ) + def test_identifier_in_map( self ): + history_id = self.dataset_populator.new_history() + hdca_id = self.__build_pair( history_id, [ "123", "456" ] ) + inputs = { + "input1": { 'batch': True, 'values': [ { 'src': 'hdca', 'id': hdca_id } ] }, + } + create_response = self._run( "identifier_single", history_id, inputs ) + self._assert_status_code_is( create_response, 200 ) + create = create_response.json() + outputs = create[ 'outputs' ] + jobs = create[ 'jobs' ] + implicit_collections = create[ 'implicit_collections' ] + self.assertEquals( len( jobs ), 2 ) + self.assertEquals( len( outputs ), 2 ) + self.assertEquals( len( implicit_collections ), 1 ) + output1 = outputs[ 0 ] + output2 = outputs[ 1 ] + output1_content = self.dataset_populator.get_history_dataset_content( history_id, dataset=output1 ) + output2_content = self.dataset_populator.get_history_dataset_content( history_id, dataset=output2 ) + self.assertEquals( output1_content.strip(), "forward" ) + self.assertEquals( output2_content.strip(), "reverse" ) + + @skip_without_tool( "identifier_single" ) + def test_identifier_outside_map( self ): + history_id = self.dataset_populator.new_history() + new_dataset1 = self.dataset_populator.new_dataset( history_id, content='123' ) + inputs = { + "input1": { 'src': 'hda', 'id': new_dataset1["id"] }, + } + create_response = self._run( "identifier_single", history_id, inputs ) + self._assert_status_code_is( create_response, 200 ) + create = create_response.json() + outputs = create[ 'outputs' ] + jobs = create[ 'jobs' ] + implicit_collections = create[ 'implicit_collections' ] + self.assertEquals( len( jobs ), 1 ) + self.assertEquals( len( outputs ), 1 ) + self.assertEquals( len( implicit_collections ), 0 ) + output1 = outputs[ 0 ] + output1_content = self.dataset_populator.get_history_dataset_content( history_id, dataset=output1 ) + self.assertEquals( output1_content.strip(), "Pasted Entry" ) + + @skip_without_tool( "identifier_multiple" ) + def test_identifier_in_multiple_reduce( self ): + history_id = self.dataset_populator.new_history() + hdca_id = self.__build_pair( history_id, [ "123", "456" ] ) + inputs = { + "input1": { 'src': 'hdca', 'id': hdca_id }, + } + create_response = self._run( "identifier_multiple", history_id, inputs ) + self._assert_status_code_is( create_response, 200 ) + create = create_response.json() + outputs = create[ 'outputs' ] + jobs = create[ 'jobs' ] + implicit_collections = create[ 'implicit_collections' ] + self.assertEquals( len( jobs ), 1 ) + self.assertEquals( len( outputs ), 1 ) + self.assertEquals( len( implicit_collections ), 0 ) + output1 = outputs[ 0 ] + output1_content = self.dataset_populator.get_history_dataset_content( history_id, dataset=output1 ) + self.assertEquals( output1_content.strip(), "forward\nreverse" ) + + @skip_without_tool( "identifier_multiple" ) + def test_identifier_with_multiple_normal_datasets( self ): + history_id = self.dataset_populator.new_history() + new_dataset1 = self.dataset_populator.new_dataset( history_id, content='123' ) + new_dataset2 = self.dataset_populator.new_dataset( history_id, content='456' ) + inputs = { + "input1": [ + { 'src': 'hda', 'id': new_dataset1["id"] }, + { 'src': 'hda', 'id': new_dataset2["id"] } + ] + } + create_response = self._run( "identifier_multiple", history_id, inputs ) + self._assert_status_code_is( create_response, 200 ) + create = create_response.json() + outputs = create[ 'outputs' ] + jobs = create[ 'jobs' ] + implicit_collections = create[ 'implicit_collections' ] + self.assertEquals( len( jobs ), 1 ) + self.assertEquals( len( outputs ), 1 ) + self.assertEquals( len( implicit_collections ), 0 ) + output1 = outputs[ 0 ] + output1_content = self.dataset_populator.get_history_dataset_content( history_id, dataset=output1 ) + self.assertEquals( output1_content.strip(), "Pasted Entry\nPasted Entry" ) + @skip_without_tool( "cat1" ) def test_map_over_nested_collections_legacy( self ): history_id = self.dataset_populator.new_history() diff -r ff3c5721f74857cf689966abfdeb0dc9390bf605 -r b9ec027d4b1e1b67f26632a55076dbf53e3225bd test/functional/tools/identifier_multiple.xml --- /dev/null +++ b/test/functional/tools/identifier_multiple.xml @@ -0,0 +1,15 @@ +<tool id="identifier_multiple" name="identifier_multiple"> + <command> + #for $input in $input1# + echo '$input.element_identifier' >> 'output1'; + #end for# + </command> + <inputs> + <param type="data" name="input1" label="Input 1" multiple="true" /> + </inputs> + <outputs> + <data name="output1" type="tabular" from_work_dir="output1" /> + </outputs> + <tests> + </tests> +</tool> diff -r ff3c5721f74857cf689966abfdeb0dc9390bf605 -r b9ec027d4b1e1b67f26632a55076dbf53e3225bd test/functional/tools/identifier_single.xml --- /dev/null +++ b/test/functional/tools/identifier_single.xml @@ -0,0 +1,13 @@ +<tool id="identifier_single" name="identifier_single"> + <command> + echo '$input1.element_identifier' > 'output1' + </command> + <inputs> + <param type="data" name="input1" label="Input 1" /> + </inputs> + <outputs> + <data name="output1" type="tabular" from_work_dir="output1" /> + </outputs> + <tests> + </tests> +</tool> diff -r ff3c5721f74857cf689966abfdeb0dc9390bf605 -r b9ec027d4b1e1b67f26632a55076dbf53e3225bd test/functional/tools/samples_tool_conf.xml --- a/test/functional/tools/samples_tool_conf.xml +++ b/test/functional/tools/samples_tool_conf.xml @@ -30,6 +30,8 @@ <tool file="validation_default.xml" /><tool file="validation_sanitizer.xml" /><tool file="validation_repeat.xml" /> + <tool file="identifier_single.xml" /> + <tool file="identifier_multiple.xml" /><tool file="collection_paired_test.xml" /><tool file="collection_nested_test.xml" /><tool file="collection_mixed_param.xml" /> Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: dan: Fix for preventing non-admins from running data managers via the api.
by commits-noreply＠bitbucket.org 02 Feb '15

02 Feb '15

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/ff3c5721f748/ Changeset: ff3c5721f748 User: dan Date: 2015-02-02 20:57:18+00:00 Summary: Fix for preventing non-admins from running data managers via the api. Affected #: 4 files diff -r 8e5b9caba8ff493c25684ebdb6f69b819c34dca6 -r ff3c5721f74857cf689966abfdeb0dc9390bf605 lib/galaxy/tools/__init__.py --- a/lib/galaxy/tools/__init__.py +++ b/lib/galaxy/tools/__init__.py @@ -550,6 +550,12 @@ def get_panel_section( self ): return self.app.toolbox.get_integrated_section_for_tool( self ) + def allow_user_access( self, user ): + """ + :returns: bool -- Whether the user is allowed to access the tool. + """ + return True + def parse( self, tool_source, guid=None ): """ Read tool configuration from the element `root` and fill in `self`. @@ -2799,6 +2805,7 @@ self.data_manager_id = self.id def exec_after_process( self, app, inp_data, out_data, param_dict, job=None, **kwds ): + assert self.allow_user_access( job.user ), "You must be an admin to access this tool." #run original exec_after_process super( DataManagerTool, self ).exec_after_process( app, inp_data, out_data, param_dict, job=job, **kwds ) #process results of tool @@ -2822,6 +2829,7 @@ return history user = trans.user assert user, 'You must be logged in to use this tool.' + assert self.allow_user_access( user ), "You must be an admin to access this tool." history = user.data_manager_histories if not history: #create @@ -2841,6 +2849,17 @@ history = None return history + def allow_user_access( self, user ): + """ + :returns: bool -- Whether the user is allowed to access the tool. + Data Manager tools are only accessible to admins. + """ + if super( DataManagerTool, self ).allow_user_access( user ) and self.app.config.is_admin_user( user ): + return True + if user: + user = user.id + log.debug( "User (%s) attempted to access a data manager tool (%s), but is not an admin.", user, self.id ) + return False # Populate tool_type to ToolClass mappings tool_types = {} diff -r 8e5b9caba8ff493c25684ebdb6f69b819c34dca6 -r ff3c5721f74857cf689966abfdeb0dc9390bf605 lib/galaxy/tools/actions/__init__.py --- a/lib/galaxy/tools/actions/__init__.py +++ b/lib/galaxy/tools/actions/__init__.py @@ -155,6 +155,7 @@ submitting the job to the job queue. If history is not specified, use trans.history as destination for tool's output datasets. """ + assert tool.allow_user_access( trans.user ), "User (%s) is not allowed to access this tool." % ( trans.user ) # Set history. if not history: history = tool.get_default_history_by_trans( trans, create=True ) diff -r 8e5b9caba8ff493c25684ebdb6f69b819c34dca6 -r ff3c5721f74857cf689966abfdeb0dc9390bf605 lib/galaxy/webapps/galaxy/api/tools.py --- a/lib/galaxy/webapps/galaxy/api/tools.py +++ b/lib/galaxy/webapps/galaxy/api/tools.py @@ -67,7 +67,7 @@ """ io_details = util.string_as_bool( kwd.get( 'io_details', False ) ) link_details = util.string_as_bool( kwd.get( 'link_details', False ) ) - tool = self._get_tool( id ) + tool = self._get_tool( id, user=trans.user ) return tool.to_dict( trans, io_details=io_details, link_details=link_details ) @_future_expose_api_anonymous @@ -98,7 +98,7 @@ @_future_expose_api_anonymous def citations( self, trans, id, **kwds ): - tool = self._get_tool( id ) + tool = self._get_tool( id, user=trans.user ) rval = [] for citation in tool.citations: rval.append( citation.to_dict( 'bibtex' ) ) @@ -132,7 +132,7 @@ # Get tool. tool_version = payload.get( 'tool_version', None ) tool = trans.app.toolbox.get_tool( payload[ 'tool_id' ] , tool_version ) if 'tool_id' in payload else None - if not tool: + if not tool or not tool.allow_user_access( trans.user ): trans.response.status = 404 return { "message": { "type": "error", "text" : trans.app.model.Dataset.conversion_messages.NO_TOOL } } @@ -230,10 +230,10 @@ # # -- Helper methods -- # - def _get_tool( self, id, tool_version=None ): + def _get_tool( self, id, tool_version=None, user=None ): id = urllib.unquote_plus( id ) tool = self.app.toolbox.get_tool( id, tool_version ) - if not tool: + if not tool or not tool.allow_user_access( user ): raise exceptions.ObjectNotFound("Could not find tool with id '%s'" % id) return tool @@ -300,7 +300,7 @@ # original_job = self.hda_manager.creating_job( original_dataset ) tool = trans.app.toolbox.get_tool( original_job.tool_id ) - if not tool: + if not tool or not tool.allow_user_access( trans.user ): return trans.app.model.Dataset.conversion_messages.NO_TOOL tool_params = dict( [ ( p.name, p.value ) for p in original_job.parameters ] ) diff -r 8e5b9caba8ff493c25684ebdb6f69b819c34dca6 -r ff3c5721f74857cf689966abfdeb0dc9390bf605 lib/galaxy/webapps/galaxy/controllers/tool_runner.py --- a/lib/galaxy/webapps/galaxy/controllers/tool_runner.py +++ b/lib/galaxy/webapps/galaxy/controllers/tool_runner.py @@ -65,7 +65,7 @@ get_loaded_tools_by_lineage=False, set_selected=refreshed_on_change ) # No tool matching the tool id, display an error (shouldn't happen) - if not tool: + if not tool or not tool.allow_user_access( trans.user ): log.error( "index called with tool id '%s' but no such tool exists", tool_id ) trans.log_event( "Tool id '%s' does not exist" % tool_id ) trans.response.status = 404 @@ -191,6 +191,8 @@ # This is expected so not an exception. tool_id_version_message = '' error( "This dataset was created by an obsolete tool (%s). Can't re-run." % tool_id ) + if not tool.allow_user_access( trans.user ): + error( "The requested tool is unknown." ) # Can't rerun upload, external data sources, et cetera. Workflow compatible will proxy this for now if not tool.is_workflow_compatible: error( "The '%s' tool does not currently support rerunning." % tool.name ) Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: carlfeberhard: Core: error with 403 if user is not logged in *and* there is no session (gen. api call with no api key)
by commits-noreply＠bitbucket.org 02 Feb '15

02 Feb '15

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/8e5b9caba8ff/ Changeset: 8e5b9caba8ff User: carlfeberhard Date: 2015-02-02 20:37:34+00:00 Summary: Core: error with 403 if user is not logged in *and* there is no session (gen. api call with no api key) Affected #: 1 file diff -r e56a78608c79097a6227f8c06cba8a45580a2a7d -r 8e5b9caba8ff493c25684ebdb6f69b819c34dca6 lib/galaxy/web/framework/decorators.py --- a/lib/galaxy/web/framework/decorators.py +++ b/lib/galaxy/web/framework/decorators.py @@ -206,7 +206,10 @@ # TODO: Document this branch, when can this happen, # I don't understand it. return __api_error_response( trans, err_msg=trans.error_message ) - if user_required and trans.anonymous: + # error if user required and anon + # error if anon and no session + if ( ( trans.anonymous and user_required ) + or ( trans.anonymous and not trans.galaxy_session ) ): error_code = error_codes.USER_NO_API_KEY # Use error codes default error message. err_msg = "API authentication required for this request" @@ -219,8 +222,10 @@ return __api_error_response( trans, status_code=400, err_code=error_code ) trans.response.set_content_type( JSON_CONTENT_TYPE ) + # send 'do not cache' headers to handle IE's caching of ajax get responses trans.response.headers[ 'Cache-Control' ] = "max-age=0,no-cache,no-store" + # TODO: Refactor next block out into a helper procedure. # Perform api_run_as processing, possibly changing identity if 'payload' in kwargs and 'run_as' in kwargs['payload']: Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: dannon: Merged in nsoranzo/galaxy-central (pull request #656)
by commits-noreply＠bitbucket.org 02 Feb '15

02 Feb '15

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/e56a78608c79/ Changeset: e56a78608c79 User: dannon Date: 2015-02-02 20:22:32+00:00 Summary: Merged in nsoranzo/galaxy-central (pull request #656) Merge rolling_restart.sh in run.sh by adding --wait parameter. Affected #: 2 files diff -r 8d74daa932a06f7bbd3e357f8216c8f58ae55965 -r e56a78608c79097a6227f8c06cba8a45580a2a7d rolling_restart.sh --- a/rolling_restart.sh +++ b/rolling_restart.sh @@ -2,60 +2,4 @@ cd `dirname $0` -# If there is a .venv/ directory, assume it contains a virtualenv that we -# should run this instance in. -if [ -d .venv ]; -then - . .venv/bin/activate -fi - -python ./scripts/check_python.py -[ $? -ne 0 ] && exit 1 - -./scripts/common_startup.sh - -if [ -n "$GALAXY_UNIVERSE_CONFIG_DIR" ]; then - python ./scripts/build_universe_config.py "$GALAXY_UNIVERSE_CONFIG_DIR" -fi - -if [ -z "$GALAXY_CONFIG_FILE" ]; then - if [ -f universe_wsgi.ini ]; then - GALAXY_CONFIG_FILE=universe_wsgi.ini - elif [ -f config/galaxy.ini ]; then - GALAXY_CONFIG_FILE=config/galaxy.ini - else - GALAXY_CONFIG_FILE=config/galaxy.ini.sample - fi - export GALAXY_CONFIG_FILE -fi - -servers=`sed -n 's/^\[server:$.*$\]/\1/ p' $GALAXY_CONFIG_FILE | xargs echo` -for server in $servers; do - # If there's a pid - if [ -e $server.pid ]; then - # Then kill it - echo "Killing $server" - pid=`cat $server.pid` - kill $pid - else - # Otherwise just continue - echo "$server not running" - fi - # Start the server (and background) (should this be nohup'd?) - python ./scripts/paster.py serve $GALAXY_CONFIG_FILE --server-name=$server --pid-file=$server.pid --log-file=$server.log --daemon $@ - while true; do - sleep 1 - printf "." - # Grab the current pid from the pid file - if ! current_pid_in_file=$(cat $server.pid); then - echo "A Galaxy process died, interrupting" >&2 - exit 1 - fi - # Search for all pids in the logs and tail for the last one - latest_pid=`egrep '^Starting server in PID [0-9]+\.$' $server.log -o | sed 's/Starting server in PID //g;s/\.$//g' | tail -n 1` - # If they're equivalent, then the current pid file agrees with our logs - # and we've succesfully started - [ -n "$latest_pid" ] && [ $latest_pid -eq $current_pid_in_file ] && break - done - echo -done +GALAXY_RUN_ALL=1 ./run.sh restart --wait diff -r 8d74daa932a06f7bbd3e357f8216c8f58ae55965 -r e56a78608c79097a6227f8c06cba8a45580a2a7d run.sh --- a/run.sh +++ b/run.sh @@ -44,15 +44,38 @@ if [ -n "$GALAXY_RUN_ALL" ]; then servers=`sed -n 's/^\[server:$.*$\]/\1/ p' $GALAXY_CONFIG_FILE | xargs echo` - daemon=`echo "$@" | grep -q daemon` + echo "$@" | grep -q 'daemon\|restart' if [ $? -ne 0 ]; then - echo 'ERROR: $GALAXY_RUN_ALL cannot be used without the `--daemon` or `--stop-daemon` arguments to run.sh' + echo 'ERROR: $GALAXY_RUN_ALL cannot be used without the `--daemon`, `--stop-daemon` or `restart` arguments to run.sh' exit 1 fi + (echo "$@" | grep -q -e '--daemon\|restart') && (echo "$@" | grep -q -e '--wait') + WAIT=$? + ARGS=`echo "$@" | sed 's/--wait//'` for server in $servers; do - echo "Handling $server with log file $server.log..." - python ./scripts/paster.py serve $GALAXY_CONFIG_FILE --server-name=$server --pid-file=$server.pid --log-file=$server.log $@ + if [ $WAIT -eq 0 ]; then + python ./scripts/paster.py serve $GALAXY_CONFIG_FILE --server-name=$server --pid-file=$server.pid --log-file=$server.log $ARGS + while true; do + sleep 1 + printf "." + # Grab the current pid from the pid file + if ! current_pid_in_file=$(cat $server.pid); then + echo "A Galaxy process died, interrupting" >&2 + exit 1 + fi + # Search for all pids in the logs and tail for the last one + latest_pid=`egrep '^Starting server in PID [0-9]+\.$' $server.log -o | sed 's/Starting server in PID //g;s/\.$//g' | tail -n 1` + # If they're equivalent, then the current pid file agrees with our logs + # and we've succesfully started + [ -n "$latest_pid" ] && [ $latest_pid -eq $current_pid_in_file ] && break + done + echo + else + echo "Handling $server with log file $server.log..." + python ./scripts/paster.py serve $GALAXY_CONFIG_FILE --server-name=$server --pid-file=$server.pid --log-file=$server.log $@ + fi done else + # Handle only 1 server, whose name can be specified with --server-name parameter (defaults to "main") python ./scripts/paster.py serve $GALAXY_CONFIG_FILE $@ fi Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: 4 new changesets
by commits-noreply＠bitbucket.org 02 Feb '15

02 Feb '15

4 new commits in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/1333f32a5063/ Changeset: 1333f32a5063 User: nsoranzo Date: 2015-02-02 15:46:48+00:00 Summary: Allow "GALAXY_RUN_ALL=1 ./run.sh restart" since --daemon is implied. Affected #: 1 file diff -r 5aca28ab8cfbbc1769f68fcfacf3dfbf9352ccde -r 1333f32a50639a2d2dfaca327b20520da358721c run.sh --- a/run.sh +++ b/run.sh @@ -44,9 +44,9 @@ if [ -n "$GALAXY_RUN_ALL" ]; then servers=`sed -n 's/^\[server:$.*$\]/\1/ p' $GALAXY_CONFIG_FILE | xargs echo` - daemon=`echo "$@" | grep -q daemon` + echo "$@" | grep -q 'daemon\|restart' if [ $? -ne 0 ]; then - echo 'ERROR: $GALAXY_RUN_ALL cannot be used without the `--daemon` or `--stop-daemon` arguments to run.sh' + echo 'ERROR: $GALAXY_RUN_ALL cannot be used without the `--daemon`, `--stop-daemon` or `restart` arguments to run.sh' exit 1 fi for server in $servers; do @@ -54,5 +54,6 @@ python ./scripts/paster.py serve $GALAXY_CONFIG_FILE --server-name=$server --pid-file=$server.pid --log-file=$server.log $@ done else + # Handle only 1 server, whose name can be specified with --server-name parameter (defaults to "main") python ./scripts/paster.py serve $GALAXY_CONFIG_FILE $@ fi https://bitbucket.org/galaxy/galaxy-central/commits/6d11484ceb7d/ Changeset: 6d11484ceb7d User: nsoranzo Date: 2015-02-02 16:08:13+00:00 Summary: Merge rolling_restart.sh in run.sh by adding --wait parameter. Also add the following combination: GALAXY_RUN_ALL=1 ./run.sh --daemon --wait which, by preventing servers from starting simultaneously, is useful when multiple servers are configured before creating the database, or when updating to a new Galaxy release containing a database upgrade. Affected #: 2 files diff -r 1333f32a50639a2d2dfaca327b20520da358721c -r 6d11484ceb7d797f5571df24a147d3f59f8ee553 rolling_restart.sh --- a/rolling_restart.sh +++ b/rolling_restart.sh @@ -1,61 +1,3 @@ #!/bin/sh -cd `dirname $0` - -# If there is a .venv/ directory, assume it contains a virtualenv that we -# should run this instance in. -if [ -d .venv ]; -then - . .venv/bin/activate -fi - -python ./scripts/check_python.py -[ $? -ne 0 ] && exit 1 - -./scripts/common_startup.sh - -if [ -n "$GALAXY_UNIVERSE_CONFIG_DIR" ]; then - python ./scripts/build_universe_config.py "$GALAXY_UNIVERSE_CONFIG_DIR" -fi - -if [ -z "$GALAXY_CONFIG_FILE" ]; then - if [ -f universe_wsgi.ini ]; then - GALAXY_CONFIG_FILE=universe_wsgi.ini - elif [ -f config/galaxy.ini ]; then - GALAXY_CONFIG_FILE=config/galaxy.ini - else - GALAXY_CONFIG_FILE=config/galaxy.ini.sample - fi - export GALAXY_CONFIG_FILE -fi - -servers=`sed -n 's/^\[server:$.*$\]/\1/ p' $GALAXY_CONFIG_FILE | xargs echo` -for server in $servers; do - # If there's a pid - if [ -e $server.pid ]; then - # Then kill it - echo "Killing $server" - pid=`cat $server.pid` - kill $pid - else - # Otherwise just continue - echo "$server not running" - fi - # Start the server (and background) (should this be nohup'd?) - python ./scripts/paster.py serve $GALAXY_CONFIG_FILE --server-name=$server --pid-file=$server.pid --log-file=$server.log --daemon $@ - while true; do - sleep 1 - printf "." - # Grab the current pid from the pid file - if ! current_pid_in_file=$(cat $server.pid); then - echo "A Galaxy process died, interrupting" >&2 - exit 1 - fi - # Search for all pids in the logs and tail for the last one - latest_pid=`egrep '^Starting server in PID [0-9]+\.$' $server.log -o | sed 's/Starting server in PID //g;s/\.$//g' | tail -n 1` - # If they're equivalent, then the current pid file agrees with our logs - # and we've succesfully started - [ -n "$latest_pid" ] && [ $latest_pid -eq $current_pid_in_file ] && break - done - echo -done +GALAXY_RUN_ALL=1 ./run.sh restart --wait diff -r 1333f32a50639a2d2dfaca327b20520da358721c -r 6d11484ceb7d797f5571df24a147d3f59f8ee553 run.sh --- a/run.sh +++ b/run.sh @@ -49,9 +49,31 @@ echo 'ERROR: $GALAXY_RUN_ALL cannot be used without the `--daemon`, `--stop-daemon` or `restart` arguments to run.sh' exit 1 fi + (echo "$@" | grep -q -e '--daemon\|restart') && (echo "$@" | grep -q -e '--wait') + WAIT=$? + ARGS=`echo "$@" | sed 's/--wait//'` for server in $servers; do - echo "Handling $server with log file $server.log..." - python ./scripts/paster.py serve $GALAXY_CONFIG_FILE --server-name=$server --pid-file=$server.pid --log-file=$server.log $@ + if [ $WAIT -eq 0 ]; then + python ./scripts/paster.py serve $GALAXY_CONFIG_FILE --server-name=$server --pid-file=$server.pid --log-file=$server.log $ARGS + while true; do + sleep 1 + printf "." + # Grab the current pid from the pid file + if ! current_pid_in_file=$(cat $server.pid); then + echo "A Galaxy process died, interrupting" >&2 + exit 1 + fi + # Search for all pids in the logs and tail for the last one + latest_pid=`egrep '^Starting server in PID [0-9]+\.$' $server.log -o | sed 's/Starting server in PID //g;s/\.$//g' | tail -n 1` + # If they're equivalent, then the current pid file agrees with our logs + # and we've succesfully started + [ -n "$latest_pid" ] && [ $latest_pid -eq $current_pid_in_file ] && break + done + echo + else + echo "Handling $server with log file $server.log..." + python ./scripts/paster.py serve $GALAXY_CONFIG_FILE --server-name=$server --pid-file=$server.pid --log-file=$server.log $@ + fi done else # Handle only 1 server, whose name can be specified with --server-name parameter (defaults to "main") https://bitbucket.org/galaxy/galaxy-central/commits/d9c2dce96ec4/ Changeset: d9c2dce96ec4 User: nsoranzo Date: 2015-02-02 17:30:03+00:00 Summary: Restore a line deleted by mistake. Thanks John. Affected #: 1 file diff -r 6d11484ceb7d797f5571df24a147d3f59f8ee553 -r d9c2dce96ec4376816a0bf23b4ba956616b63a48 rolling_restart.sh --- a/rolling_restart.sh +++ b/rolling_restart.sh @@ -1,3 +1,5 @@ #!/bin/sh +cd `dirname $0` + GALAXY_RUN_ALL=1 ./run.sh restart --wait https://bitbucket.org/galaxy/galaxy-central/commits/e56a78608c79/ Changeset: e56a78608c79 User: dannon Date: 2015-02-02 20:22:32+00:00 Summary: Merged in nsoranzo/galaxy-central (pull request #656) Merge rolling_restart.sh in run.sh by adding --wait parameter. Affected #: 2 files diff -r 8d74daa932a06f7bbd3e357f8216c8f58ae55965 -r e56a78608c79097a6227f8c06cba8a45580a2a7d rolling_restart.sh --- a/rolling_restart.sh +++ b/rolling_restart.sh @@ -2,60 +2,4 @@ cd `dirname $0` -# If there is a .venv/ directory, assume it contains a virtualenv that we -# should run this instance in. -if [ -d .venv ]; -then - . .venv/bin/activate -fi - -python ./scripts/check_python.py -[ $? -ne 0 ] && exit 1 - -./scripts/common_startup.sh - -if [ -n "$GALAXY_UNIVERSE_CONFIG_DIR" ]; then - python ./scripts/build_universe_config.py "$GALAXY_UNIVERSE_CONFIG_DIR" -fi - -if [ -z "$GALAXY_CONFIG_FILE" ]; then - if [ -f universe_wsgi.ini ]; then - GALAXY_CONFIG_FILE=universe_wsgi.ini - elif [ -f config/galaxy.ini ]; then - GALAXY_CONFIG_FILE=config/galaxy.ini - else - GALAXY_CONFIG_FILE=config/galaxy.ini.sample - fi - export GALAXY_CONFIG_FILE -fi - -servers=`sed -n 's/^\[server:$.*$\]/\1/ p' $GALAXY_CONFIG_FILE | xargs echo` -for server in $servers; do - # If there's a pid - if [ -e $server.pid ]; then - # Then kill it - echo "Killing $server" - pid=`cat $server.pid` - kill $pid - else - # Otherwise just continue - echo "$server not running" - fi - # Start the server (and background) (should this be nohup'd?) - python ./scripts/paster.py serve $GALAXY_CONFIG_FILE --server-name=$server --pid-file=$server.pid --log-file=$server.log --daemon $@ - while true; do - sleep 1 - printf "." - # Grab the current pid from the pid file - if ! current_pid_in_file=$(cat $server.pid); then - echo "A Galaxy process died, interrupting" >&2 - exit 1 - fi - # Search for all pids in the logs and tail for the last one - latest_pid=`egrep '^Starting server in PID [0-9]+\.$' $server.log -o | sed 's/Starting server in PID //g;s/\.$//g' | tail -n 1` - # If they're equivalent, then the current pid file agrees with our logs - # and we've succesfully started - [ -n "$latest_pid" ] && [ $latest_pid -eq $current_pid_in_file ] && break - done - echo -done +GALAXY_RUN_ALL=1 ./run.sh restart --wait diff -r 8d74daa932a06f7bbd3e357f8216c8f58ae55965 -r e56a78608c79097a6227f8c06cba8a45580a2a7d run.sh --- a/run.sh +++ b/run.sh @@ -44,15 +44,38 @@ if [ -n "$GALAXY_RUN_ALL" ]; then servers=`sed -n 's/^\[server:$.*$\]/\1/ p' $GALAXY_CONFIG_FILE | xargs echo` - daemon=`echo "$@" | grep -q daemon` + echo "$@" | grep -q 'daemon\|restart' if [ $? -ne 0 ]; then - echo 'ERROR: $GALAXY_RUN_ALL cannot be used without the `--daemon` or `--stop-daemon` arguments to run.sh' + echo 'ERROR: $GALAXY_RUN_ALL cannot be used without the `--daemon`, `--stop-daemon` or `restart` arguments to run.sh' exit 1 fi + (echo "$@" | grep -q -e '--daemon\|restart') && (echo "$@" | grep -q -e '--wait') + WAIT=$? + ARGS=`echo "$@" | sed 's/--wait//'` for server in $servers; do - echo "Handling $server with log file $server.log..." - python ./scripts/paster.py serve $GALAXY_CONFIG_FILE --server-name=$server --pid-file=$server.pid --log-file=$server.log $@ + if [ $WAIT -eq 0 ]; then + python ./scripts/paster.py serve $GALAXY_CONFIG_FILE --server-name=$server --pid-file=$server.pid --log-file=$server.log $ARGS + while true; do + sleep 1 + printf "." + # Grab the current pid from the pid file + if ! current_pid_in_file=$(cat $server.pid); then + echo "A Galaxy process died, interrupting" >&2 + exit 1 + fi + # Search for all pids in the logs and tail for the last one + latest_pid=`egrep '^Starting server in PID [0-9]+\.$' $server.log -o | sed 's/Starting server in PID //g;s/\.$//g' | tail -n 1` + # If they're equivalent, then the current pid file agrees with our logs + # and we've succesfully started + [ -n "$latest_pid" ] && [ $latest_pid -eq $current_pid_in_file ] && break + done + echo + else + echo "Handling $server with log file $server.log..." + python ./scripts/paster.py serve $GALAXY_CONFIG_FILE --server-name=$server --pid-file=$server.pid --log-file=$server.log $@ + fi done else + # Handle only 1 server, whose name can be specified with --server-name parameter (defaults to "main") python ./scripts/paster.py serve $GALAXY_CONFIG_FILE $@ fi Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: carlfeberhard: Managers: re-implement url_for as overridable service, add tests for history contents
by commits-noreply＠bitbucket.org 02 Feb '15

02 Feb '15

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/8d74daa932a0/ Changeset: 8d74daa932a0 User: carlfeberhard Date: 2015-02-02 19:13:54+00:00 Summary: Managers: re-implement url_for as overridable service, add tests for history contents Affected #: 4 files diff -r d1b39530bd1ffa10c482e7f7c49b2b41c5a847a1 -r 8d74daa932a06f7bbd3e357f8216c8f58ae55965 lib/galaxy/managers/base.py --- a/lib/galaxy/managers/base.py +++ b/lib/galaxy/managers/base.py @@ -503,7 +503,7 @@ item_dict = MySerializer.serialize( trans, my_item, keys_to_serialize ) """ #: 'service' to use for getting urls - use class var to allow overriding when testing - #url_service = + url_for = staticmethod( routes.url_for ) def __init__( self, app ): """ @@ -544,7 +544,7 @@ If `include_keys_from` is a proper view name, extend `key_list` by the list in that view. """ - key_list += self.views.get( include_keys_from, [] ) + key_list = list( set( key_list + self.views.get( include_keys_from, [] ) ) ) self.views[ view_name ] = key_list self.serializable_keyset.update( key_list ) return key_list @@ -590,10 +590,6 @@ id = getattr( item, key ) return self.app.security.encode_id( id ) if id is not None else None - @staticmethod - def url_for( *args, **kwargs ): - return routes.url_for( *args, **kwargs ) - # serializing to a view where a view is a predefied list of keys to serialize def serialize_to_view( self, trans, item, view=None, keys=None, default_view=None ): """ diff -r d1b39530bd1ffa10c482e7f7c49b2b41c5a847a1 -r 8d74daa932a06f7bbd3e357f8216c8f58ae55965 test/unit/managers/mock.py --- a/test/unit/managers/mock.py +++ b/test/unit/managers/mock.py @@ -21,6 +21,14 @@ class OpenObject( object ): pass +class MockApp( object ): + def __init__( self, **kwargs ): + self.config = MockAppConfig( **kwargs ) + self.security = self.config.security + self.object_store = objectstore.build_object_store_from_config( self.config ) + self.model = mapping.init( "/tmp", "sqlite:///:memory:", create_tables=True, object_store=self.object_store ) + self.security_agent = self.model.security_agent + self.visualizations_registry = MockVisualizationsRegistry() class MockAppConfig( Bunch ): def __init__( self, **kwargs ): @@ -40,15 +48,6 @@ self.expose_dataset_path = True self.allow_user_dataset_purge = True self.enable_old_display_applications = True - - -class MockApp( object ): - def __init__( self, **kwargs ): - self.config = MockAppConfig( **kwargs ) - self.security = self.config.security - self.object_store = objectstore.build_object_store_from_config( self.config ) - self.model = mapping.init( "/tmp", "sqlite:///:memory:", create_tables=True, object_store=self.object_store ) - self.security_agent = self.model.security_agent class MockWebapp( object ): def __init__( self, **kwargs ): @@ -92,3 +91,7 @@ template = template_lookup.get_template( filename ) template.output_encoding = 'utf-8' return template.render( **kwargs ) + +class MockVisualizationsRegistry( object ): + def get_visualizations( self, trans, target ): + return [] diff -r d1b39530bd1ffa10c482e7f7c49b2b41c5a847a1 -r 8d74daa932a06f7bbd3e357f8216c8f58ae55965 test/unit/managers/test_HistoryManager.py --- a/test/unit/managers/test_HistoryManager.py +++ b/test/unit/managers/test_HistoryManager.py @@ -312,15 +312,12 @@ # ============================================================================= # web.url_for doesn't work well in the framework testable_url_for = lambda *a, **k: '(fake url): %s, %s' % ( a, k ) -HistorySerializer.url_for = testable_url_for -hdas.HDASerializer.url_for = testable_url_for +HistorySerializer.url_for = staticmethod( testable_url_for ) +hdas.HDASerializer.url_for = staticmethod( testable_url_for ) class HistorySerializerTestCase( BaseTestCase ): - def assertHasKeys( self, obj, key_list ): - self.assertEqual( sorted( obj.keys() ), sorted( key_list ) ) - def set_up_managers( self ): super( HistorySerializerTestCase, self ).set_up_managers() self.history_mgr = HistoryManager( self.app ) @@ -333,15 +330,15 @@ self.log( 'should have a summary view' ) summary_view = self.history_serializer.serialize_to_view( self.trans, history1, view='summary' ) - self.assertHasKeys( summary_view, self.history_serializer.views[ 'summary' ] ) + self.assertKeys( summary_view, self.history_serializer.views[ 'summary' ] ) self.log( 'should have a detailed view' ) detailed_view = self.history_serializer.serialize_to_view( self.trans, history1, view='detailed' ) - self.assertHasKeys( detailed_view, self.history_serializer.views[ 'detailed' ] ) + self.assertKeys( detailed_view, self.history_serializer.views[ 'detailed' ] ) self.log( 'should have the summary view as default view' ) default_view = self.history_serializer.serialize_to_view( self.trans, history1, default_view='summary' ) - self.assertHasKeys( summary_view, self.history_serializer.views[ 'summary' ] ) + self.assertKeys( summary_view, self.history_serializer.views[ 'summary' ] ) self.log( 'should have a serializer for all serializable keys' ) need_no_serializers = ( basestring, bool, type( None ) ) @@ -360,13 +357,13 @@ self.log( 'should be able to use keys with views' ) serialized = self.history_serializer.serialize_to_view( self.trans, history1, view='summary', keys=[ 'state_ids', 'user_id' ] ) - self.assertHasKeys( serialized, + self.assertKeys( serialized, self.history_serializer.views[ 'summary' ] + [ 'state_ids', 'user_id' ] ) self.log( 'should be able to use keys on their own' ) serialized = self.history_serializer.serialize_to_view( self.trans, history1, keys=[ 'state_ids', 'user_id' ] ) - self.assertHasKeys( serialized, [ 'state_ids', 'user_id' ] ) + self.assertKeys( serialized, [ 'state_ids', 'user_id' ] ) def test_sharable( self ): user2 = self.user_mgr.create( self.trans, **user2_data ) @@ -375,7 +372,7 @@ self.log( 'should have a serializer for all SharableModel keys' ) sharable_attrs = [ 'user_id', 'username_and_slug', 'importable', 'published', 'slug' ] serialized = self.history_serializer.serialize( self.trans, history1, sharable_attrs ) - self.assertHasKeys( serialized, sharable_attrs ) + self.assertKeys( serialized, sharable_attrs ) def test_purgable( self ): user2 = self.user_mgr.create( self.trans, **user2_data ) @@ -398,8 +395,6 @@ self.assertEqual( serialized[ 'deleted' ], True ) self.assertEqual( serialized[ 'purged' ], True ) - #pprint.pprint( self.history_serializer.serialize( self.trans, history1, [ 'contents' ] ) ) - def test_history_serializers( self ): user2 = self.user_mgr.create( self.trans, **user2_data ) history1 = self.history_mgr.create( self.trans, name='history1', user=user2 ) @@ -435,6 +430,8 @@ self.assertIsInstance( serialized[ 'hdas' ], list ) self.assertIsInstance( serialized[ 'hdas' ][0], basestring ) + serialized = self.history_serializer.serialize( self.trans, history1, [ 'contents' ] ) + self.assertHasKeys( serialized[ 'contents' ][0], [ 'id', 'name', 'peek', 'create_time' ]) # ============================================================================= class HistoryDeserializerTestCase( BaseTestCase ): diff -r d1b39530bd1ffa10c482e7f7c49b2b41c5a847a1 -r 8d74daa932a06f7bbd3e357f8216c8f58ae55965 test/unit/managers/test_ModelManager.py --- a/test/unit/managers/test_ModelManager.py +++ b/test/unit/managers/test_ModelManager.py @@ -75,6 +75,17 @@ def log( self, *args, **kwargs ): print( *args, **kwargs ) + # ---- additional test types + def assertKeys( self, obj, key_list ): + self.assertEqual( sorted( obj.keys() ), sorted( key_list ) ) + + def assertHasKeys( self, obj, key_list ): + for key in key_list: + if key not in obj: + self.fail( 'Missing key: ' + key ) + else: + self.assertTrue( True, 'keys found in object' ) + # ============================================================================= if __name__ == '__main__': Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: natefoo: Update tag latest_2015.01.13 for changeset fd75aaee91cf
by commits-noreply＠bitbucket.org 02 Feb '15

02 Feb '15

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/96aabc4e66f1/ Changeset: 96aabc4e66f1 Branch: stable User: natefoo Date: 2015-02-02 19:00:23+00:00 Summary: Update tag latest_2015.01.13 for changeset fd75aaee91cf Affected #: 1 file diff -r fd75aaee91cf3e8a0916689dfea72e0c752c447c -r 96aabc4e66f193d4bb3b7b37d8ad4dce671f1b04 .hgtags --- a/.hgtags +++ b/.hgtags @@ -22,4 +22,4 @@ 2092948937ac30ef82f71463a235c66d34987088 release_2014.10.06 782fa60fc65488aea0c618d723e9a63d42caf865 latest_2014.10.06 2e8dd2949dd3eee0f56f9a3a5ebf1b2baca24aee release_2015.01.13 -c5e7535b4d229dbbe52d48ea35a27ab601205b7b latest_2015.01.13 +fd75aaee91cf3e8a0916689dfea72e0c752c447c latest_2015.01.13 Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: natefoo: Remove GeneTrack tag from eggs.ini as well.
by commits-noreply＠bitbucket.org 02 Feb '15

02 Feb '15

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/d1b39530bd1f/ Changeset: d1b39530bd1f User: natefoo Date: 2015-02-02 18:42:29+00:00 Summary: Remove GeneTrack tag from eggs.ini as well. Affected #: 1 file diff -r eca1b4b64574cf09b4cbe35c4a8b103c1b455a79 -r d1b39530bd1ffa10c482e7f7c49b2b41c5a847a1 eggs.ini --- a/eggs.ini +++ b/eggs.ini @@ -82,7 +82,6 @@ psycopg2 = _9.2.4_static pysqlite = _3.6.17_static MySQL_python = _5.1.41_static -GeneTrack = _dev_48da9e998f0caf01c5be731e926f4b0481f658f0 pysam = _kanwei_b10f6e722e9a ; dependency source urls, necessary for scrambling. for an explanation, see Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: carlfeberhard: Managers: add shorthand for adding a serializer view, back off of class-level url service, more testing
by commits-noreply＠bitbucket.org 02 Feb '15

02 Feb '15

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/eca1b4b64574/ Changeset: eca1b4b64574 User: carlfeberhard Date: 2015-02-02 18:33:06+00:00 Summary: Managers: add shorthand for adding a serializer view, back off of class-level url service, more testing Affected #: 10 files diff -r 620437dab02aa2bf8508292bc0df4b2e1066264e -r eca1b4b64574cf09b4cbe35c4a8b103c1b455a79 lib/galaxy/managers/base.py --- a/lib/galaxy/managers/base.py +++ b/lib/galaxy/managers/base.py @@ -29,6 +29,9 @@ pkg_resources.require( "SQLAlchemy >= 0.4" ) import sqlalchemy +pkg_resources.require("Routes") +import routes + from galaxy import exceptions from galaxy import model from galaxy import web @@ -490,15 +493,17 @@ that should be called for those keys. E.g. { 'x' : lambda trans, item, key: item.x, ... } + Note: if a key to serialize is not listed in the Serializer.serializable_keyset + or serializers, it will not be returned. + To serialize call: my_serializer = MySerializer( app ) ... keys_to_serialize = [ 'id', 'name', 'attr1', 'attr2', ... ] item_dict = MySerializer.serialize( trans, my_item, keys_to_serialize ) - # if a key to serialize is not listed in the Serializer.serializable_keys or serializers, it will not be added """ #: 'service' to use for getting urls - use class var to allow overriding when testing - url_for = web.url_for + #url_service = def __init__( self, app ): """ @@ -506,14 +511,17 @@ """ self.app = app + # a list of valid serializable keys that can use the default (string) serializer + # this allows us to: 'mention' the key without adding the default serializer + # TODO: we may want to eventually error if a key is requested + # that is in neither serializable_keyset or serializers + self.serializable_keyset = set([]) # a map of dictionary keys to the functions (often lambdas) that create the values for those keys self.serializers = {} - # a list of valid serializable keys that can use the default (string) serializer - # this allows us to: 'mention' the key without adding the default serializer - # NOTE: if a key is requested that is in neither serializable_keys or serializers, it is not returned - self.serializable_keys = [] # add subclass serializers defined there self.add_serializers() + # update the keyset by the serializers (removing the responsibility from subclasses) + self.serializable_keyset.update( self.serializers.keys() ) # views are collections of serializable attributes (a named array of keys) # inspired by model.dict_{view}_visible_keys @@ -528,6 +536,19 @@ # to be overridden in subclasses pass + def add_view( self, view_name, key_list, include_keys_from=None ): + """ + Add the list of serializable attributes `key_list` to the serializer's + view dictionary under the key `view_name`. + + If `include_keys_from` is a proper view name, extend `key_list` by + the list in that view. + """ + key_list += self.views.get( include_keys_from, [] ) + self.views[ view_name ] = key_list + self.serializable_keyset.update( key_list ) + return key_list + def serialize( self, trans, item, keys ): """ Serialize the model `item` to a dictionary. @@ -542,7 +563,7 @@ # check both serializers and serializable keys if key in self.serializers: returned[ key ] = self.serializers[ key ]( trans, item, key ) - elif key in self.serializable_keys: + elif key in self.serializable_keyset: returned[ key ] = self.default_serializer( trans, item, key ) # ignore bad/unreg keys return returned @@ -569,6 +590,10 @@ id = getattr( item, key ) return self.app.security.encode_id( id ) if id is not None else None + @staticmethod + def url_for( *args, **kwargs ): + return routes.url_for( *args, **kwargs ) + # serializing to a view where a view is a predefied list of keys to serialize def serialize_to_view( self, trans, item, view=None, keys=None, default_view=None ): """ @@ -628,7 +653,7 @@ self.app = app self.deserializers = {} - self.deserializable_keys = [] + self.deserializable_keyset = set([]) self.add_deserializers() # a sub object that can validate incoming values self.validate = ModelValidator( self.app ) diff -r 620437dab02aa2bf8508292bc0df4b2e1066264e -r eca1b4b64574cf09b4cbe35c4a8b103c1b455a79 lib/galaxy/managers/datasets.py --- a/lib/galaxy/managers/datasets.py +++ b/lib/galaxy/managers/datasets.py @@ -192,8 +192,8 @@ def __init__( self, app ): super( DatasetSerializer, self ).__init__( app ) - # most of these views build/add to the previous view - summary_view = [ + self.default_view = 'summary' + self.add_view( 'summary', [ 'id', 'create_time', 'update_time', 'state', @@ -203,19 +203,13 @@ #'extra_files_path', 'file_size', 'total_size', 'uuid', - ] + ]) # could do visualizations and/or display_apps - self.serializable_keys = summary_view + [ - ] - self.views = { - 'summary' : summary_view, - } - self.default_view = 'summary' - def add_serializers( self ): super( DatasetSerializer, self ).add_serializers() deletable.PurgableSerializerMixin.add_serializers( self ) + self.serializers.update({ 'id' : self.serialize_id, 'create_time' : self.serialize_date, diff -r 620437dab02aa2bf8508292bc0df4b2e1066264e -r eca1b4b64574cf09b4cbe35c4a8b103c1b455a79 lib/galaxy/managers/deletable.py --- a/lib/galaxy/managers/deletable.py +++ b/lib/galaxy/managers/deletable.py @@ -35,7 +35,7 @@ class DeletableSerializerMixin( object ): def add_serializers( self ): - pass + self.serializable_keyset.add( 'deleted' ) # TODO: these are of questionable value if we don't want to enable users to delete/purge via update @@ -87,6 +87,7 @@ def add_serializers( self ): DeletableSerializerMixin.add_serializers( self ) + self.serializable_keyset.add( 'purged' ) class PurgableDeserializerMixin( DeletableDeserializerMixin ): diff -r 620437dab02aa2bf8508292bc0df4b2e1066264e -r eca1b4b64574cf09b4cbe35c4a8b103c1b455a79 lib/galaxy/managers/hdas.py --- a/lib/galaxy/managers/hdas.py +++ b/lib/galaxy/managers/hdas.py @@ -315,8 +315,8 @@ super( HDASerializer, self ).__init__( app ) self.hda_manager = HDAManager( app ) - # most of these views build/add to the previous view - summary_view = [ + self.default_view = 'summary' + self.add_view( 'summary', [ 'id', 'name', 'history_id', 'hid', # why include if model_class is there? @@ -325,12 +325,8 @@ 'state', 'extension', 'deleted', 'purged', 'visible', 'resubmitted', 'type', 'url' - ] - inaccessible = [ - 'id', 'name', 'history_id', 'hid', 'history_content_type', - 'state', 'deleted', 'visible' - ] - detailed_view = summary_view + [ + ]) + self.add_view( 'detailed', [ 'model_class', 'history_id', 'hid', # why include if model_class is there? @@ -362,19 +358,17 @@ 'annotation', 'tags', 'api_type' - ] - extended_view = detailed_view + [ + ], include_keys_from='summary' ) + + self.add_view( 'extended', [ 'tool_version', 'parent_id', 'designation', - ] + ], include_keys_from='detailed' ) - self.serializable_keys = extended_view + [ - ] - self.views = { - 'summary' : summary_view, - 'detailed' : detailed_view, - 'extended' : extended_view, - } - self.default_view = 'summary' + # keyset returned to create show a dataset where the owner has no access + self.add_view( 'inaccessible', [ + 'id', 'name', 'history_id', 'hid', 'history_content_type', + 'state', 'deleted', 'visible' + ]) def add_serializers( self ): super( HDASerializer, self ).add_serializers() @@ -597,4 +591,4 @@ 'importable' : self.deserialize_bool, }) - self.deserializable_keys.extend( self.deserializers.keys() ) + self.deserializable_keyset.update( self.deserializers.keys() ) diff -r 620437dab02aa2bf8508292bc0df4b2e1066264e -r eca1b4b64574cf09b4cbe35c4a8b103c1b455a79 lib/galaxy/managers/histories.py --- a/lib/galaxy/managers/histories.py +++ b/lib/galaxy/managers/histories.py @@ -249,12 +249,13 @@ def __init__( self, app ): super( HistorySerializer, self ).__init__( app ) + self.history_manager = HistoryManager( app ) - self.hda_manager = hdas.HDAManager( app ) self.hda_serializer = hdas.HDASerializer( app ) - summary_view = [ + self.default_view = 'summary' + self.add_view( 'summary', [ 'id', 'model_class', 'name', @@ -266,9 +267,8 @@ 'published', 'annotation', 'tags', - ] - # in the Historys' case, each of these views includes the keys from the previous - detailed_view = summary_view + [ + ]) + self.add_view( 'detailed', [ 'contents_url', #'hdas', 'empty', @@ -281,16 +281,8 @@ 'state', 'state_details', 'state_ids', - ] - extended_view = detailed_view + [ - ] - self.serializable_keys = extended_view + [] - self.views = { - 'summary' : summary_view, - 'detailed' : detailed_view, - 'extended' : extended_view, - } - self.default_view = 'summary' + # in the Historys' case, each of these views includes the keys from the previous + ], include_keys_from='summary' ) #assumes: outgoing to json.dumps and sanitized def add_serializers( self ): @@ -300,19 +292,19 @@ self.serializers.update({ 'model_class' : lambda *a: 'History', 'id' : self.serialize_id, - 'count' : lambda trans, item, key: len( item.datasets ), 'create_time' : self.serialize_date, 'update_time' : self.serialize_date, 'size' : lambda t, i, k: int( i.get_disk_size() ), 'nice_size' : lambda t, i, k: i.get_disk_size( nice_size=True ), + 'state' : lambda t, i, k: self.history_manager.get_history_state( t, i ), 'url' : lambda t, i, k: self.url_for( 'history', id=t.security.encode_id( i.id ) ), 'contents_url' : lambda t, i, k: self.url_for( 'history_contents', history_id=t.security.encode_id( i.id ) ), + 'empty' : lambda t, i, k: len( i.datasets ) <= 0, - + 'count' : lambda trans, item, key: len( item.datasets ), 'hdas' : lambda t, i, k: [ t.security.encode_id( hda.id ) for hda in i.datasets ], - 'state' : lambda t, i, k: self.history_manager.get_history_state( t, i ), 'state_details' : lambda t, i, k: self.history_manager.get_state_counts( t, i ), 'state_ids' : lambda t, i, k: self.history_manager.get_state_ids( t, i ), 'contents' : self.serialize_contents diff -r 620437dab02aa2bf8508292bc0df4b2e1066264e -r eca1b4b64574cf09b4cbe35c4a8b103c1b455a79 lib/galaxy/managers/pages.py --- a/lib/galaxy/managers/pages.py +++ b/lib/galaxy/managers/pages.py @@ -46,17 +46,9 @@ super( PageSerializer, self ).__init__( app ) self.page_manager = PageManager( app ) - summary_view = [ - ] - # in the Pages' case, each of these views includes the keys from the previous - detailed_view = summary_view + [ - ] - self.serializable_keys = detailed_view + [] - self.views = { - 'summary': summary_view, - 'detailed': detailed_view - } self.default_view = 'summary' + self.add_view( 'summary', [] ) + self.add_view( 'detailed', [] ) def add_serializers( self ): super( PageSerializer, self ).add_serializers() @@ -79,4 +71,4 @@ super( PageDeserializer, self ).add_deserializers() self.deserializers.update({ }) - self.deserializable_keys = self.deserializers.keys() + self.deserializable_keyset.update( self.deserializers.keys() ) diff -r 620437dab02aa2bf8508292bc0df4b2e1066264e -r eca1b4b64574cf09b4cbe35c4a8b103c1b455a79 lib/galaxy/managers/sharable.py --- a/lib/galaxy/managers/sharable.py +++ b/lib/galaxy/managers/sharable.py @@ -319,7 +319,7 @@ 'user_id' : self.serialize_id, 'username_and_slug' : self.serialize_username_and_slug }) - self.serializable_keys.extend([ + self.serializable_keyset.update([ 'importable', 'published', 'slug' ]) diff -r 620437dab02aa2bf8508292bc0df4b2e1066264e -r eca1b4b64574cf09b4cbe35c4a8b103c1b455a79 lib/galaxy/managers/users.py --- a/lib/galaxy/managers/users.py +++ b/lib/galaxy/managers/users.py @@ -202,28 +202,26 @@ """ super( UserSerializer, self ).__init__() - summary_view = [ + self.default_view = 'summary' + self.add_view( 'summary', [ 'id', 'email', 'username' - ] - # in the Historys' case, each of these views includes the keys from the previous - detailed_view = summary_view + [ - 'update_time', 'create_time', - 'total_disk_usage', 'nice_total_disk_usage', - 'deleted', 'purged', + ]) + self.add_view( 'detailed', [ + 'update_time', + 'create_time', + 'total_disk_usage', + 'nice_total_disk_usage', + 'deleted', + 'purged', 'active' - ] - extended_view = detailed_view + [ - #'preferences', - #'tags', # all tags - #'annotations' # all annotations - ] - self.serializable_keys = extended_view - self.views = { - 'summary' : summary_view, - 'detailed' : detailed_view, - 'extended' : extended_view, - } - self.default_view = 'summary' + ], include_keys_from='summary' ) + #self.add_view( 'summary', [ + # 'preferences', + # # all tags + # 'tags', + # # all annotations + # 'annotations' + #], include_keys_from='detailed' ) def add_serializers( self ): self.serializers.update({ diff -r 620437dab02aa2bf8508292bc0df4b2e1066264e -r eca1b4b64574cf09b4cbe35c4a8b103c1b455a79 lib/galaxy/managers/visualizations.py --- a/lib/galaxy/managers/visualizations.py +++ b/lib/galaxy/managers/visualizations.py @@ -48,17 +48,9 @@ super( VisualizationSerializer, self ).__init__( app ) self.visualizations_manager = VisualizationManager( app ) - summary_view = [ - ] - # in the Visualizations' case, each of these views includes the keys from the previous - detailed_view = summary_view + [ - ] - self.serializable_keys = detailed_view + [] - self.views = { - 'summary' : summary_view, - 'detailed' : detailed_view - } self.default_view = 'summary' + self.add_view( 'summary', [] ) + self.add_view( 'detailed', [] ) def add_serializers( self ): super( VisualizationSerializer, self ).add_serializers() @@ -81,4 +73,4 @@ super( VisualizationDeserializer, self ).add_deserializers() self.deserializers.update({ }) - self.deserializable_keys = self.deserializers.keys() + self.deserializable_keyset.update( self.deserializers.keys() ) diff -r 620437dab02aa2bf8508292bc0df4b2e1066264e -r eca1b4b64574cf09b4cbe35c4a8b103c1b455a79 test/unit/managers/test_HistoryManager.py --- a/test/unit/managers/test_HistoryManager.py +++ b/test/unit/managers/test_HistoryManager.py @@ -339,17 +339,13 @@ detailed_view = self.history_serializer.serialize_to_view( self.trans, history1, view='detailed' ) self.assertHasKeys( detailed_view, self.history_serializer.views[ 'detailed' ] ) - self.log( 'should have a extended view' ) - extended_view = self.history_serializer.serialize_to_view( self.trans, history1, view='extended' ) - self.assertHasKeys( extended_view, self.history_serializer.views[ 'extended' ] ) - self.log( 'should have the summary view as default view' ) default_view = self.history_serializer.serialize_to_view( self.trans, history1, default_view='summary' ) self.assertHasKeys( summary_view, self.history_serializer.views[ 'summary' ] ) self.log( 'should have a serializer for all serializable keys' ) need_no_serializers = ( basestring, bool, type( None ) ) - for key in self.history_serializer.serializable_keys: + for key in self.history_serializer.serializable_keyset: instantiated_attribute = getattr( history1, key, None ) if not ( ( key in self.history_serializer.serializers ) or ( isinstance( instantiated_attribute, need_no_serializers ) ) ): @@ -372,24 +368,61 @@ keys=[ 'state_ids', 'user_id' ] ) self.assertHasKeys( serialized, [ 'state_ids', 'user_id' ] ) - def test_serializers( self ): - # size - # nice size - pass + def test_sharable( self ): + user2 = self.user_mgr.create( self.trans, **user2_data ) + history1 = self.history_mgr.create( self.trans, name='history1', user=user2 ) + + self.log( 'should have a serializer for all SharableModel keys' ) + sharable_attrs = [ 'user_id', 'username_and_slug', 'importable', 'published', 'slug' ] + serialized = self.history_serializer.serialize( self.trans, history1, sharable_attrs ) + self.assertHasKeys( serialized, sharable_attrs ) + + def test_purgable( self ): + user2 = self.user_mgr.create( self.trans, **user2_data ) + history1 = self.history_mgr.create( self.trans, name='history1', user=user2 ) + + self.log( 'deleted and purged should be returned in their default states' ) + keys = [ 'deleted', 'purged' ] + serialized = self.history_serializer.serialize( self.trans, history1, keys ) + self.assertEqual( serialized[ 'deleted' ], False ) + self.assertEqual( serialized[ 'purged' ], False ) + + self.log( 'deleted and purged should return their current state' ) + self.history_mgr.delete( self.trans, history1 ) + serialized = self.history_serializer.serialize( self.trans, history1, keys ) + self.assertEqual( serialized[ 'deleted' ], True ) + self.assertEqual( serialized[ 'purged' ], False ) + + self.history_mgr.purge( self.trans, history1 ) + serialized = self.history_serializer.serialize( self.trans, history1, keys ) + self.assertEqual( serialized[ 'deleted' ], True ) + self.assertEqual( serialized[ 'purged' ], True ) + + #pprint.pprint( self.history_serializer.serialize( self.trans, history1, [ 'contents' ] ) ) + + def test_history_serializers( self ): + user2 = self.user_mgr.create( self.trans, **user2_data ) + history1 = self.history_mgr.create( self.trans, name='history1', user=user2 ) + + serialized = self.history_serializer.serialize( self.trans, history1, [ 'size', 'nice_size' ]) + self.assertIsInstance( serialized[ 'size' ], int ) + self.assertIsInstance( serialized[ 'nice_size' ], basestring ) def test_contents( self ): user2 = self.user_mgr.create( self.trans, **user2_data ) history1 = self.history_mgr.create( self.trans, name='history1', user=user2 ) self.log( 'a history with no contents should be properly reflected in empty, etc.' ) - keys = [ 'empty', 'count', 'state_ids', 'state_details', 'state' ] + keys = [ 'empty', 'count', 'state_ids', 'state_details', 'state', 'hdas' ] serialized = self.history_serializer.serialize( self.trans, history1, keys ) self.assertEqual( serialized[ 'state' ], 'new' ) self.assertEqual( serialized[ 'empty' ], True ) self.assertEqual( serialized[ 'count' ], 0 ) self.assertEqual( sum( serialized[ 'state_details' ].values() ), 0 ) self.assertEqual( serialized[ 'state_ids' ][ 'ok' ], [] ) + self.assertIsInstance( serialized[ 'hdas' ], list ) + self.log( 'a history with contents should be properly reflected in empty, etc.' ) hda1 = self.hda_mgr.create( self.trans, history=history1, hid=1 ) self.hda_mgr.update( self.trans, hda1, dict( state='ok' ) ) @@ -399,8 +432,9 @@ self.assertEqual( serialized[ 'count' ], 1 ) self.assertEqual( serialized[ 'state_details' ][ 'ok' ], 1 ) self.assertIsInstance( serialized[ 'state_ids' ][ 'ok' ], list ) + self.assertIsInstance( serialized[ 'hdas' ], list ) + self.assertIsInstance( serialized[ 'hdas' ][0], basestring ) - #pprint.pprint( self.history_serializer.serialize( self.trans, history1, [ 'contents' ] ) ) # ============================================================================= class HistoryDeserializerTestCase( BaseTestCase ): Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: dannon: Merged in dan/galaxy-central-prs (pull request #643)
by commits-noreply＠bitbucket.org 02 Feb '15

02 Feb '15

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/620437dab02a/ Changeset: 620437dab02a User: dannon Date: 2015-02-02 17:04:23+00:00 Summary: Merged in dan/galaxy-central-prs (pull request #643) Add ZebrafishMine Data Source Tool. Affected #: 3 files diff -r 5aca28ab8cfbbc1769f68fcfacf3dfbf9352ccde -r 620437dab02aa2bf8508292bc0df4b2e1066264e config/tool_conf.xml.main --- a/config/tool_conf.xml.main +++ b/config/tool_conf.xml.main @@ -15,6 +15,7 @@ <tool file="data_source/yeastmine.xml" /><tool file="data_source/worm_modencode.xml" /><tool file="data_source/wormbase.xml" /> + <tool file="data_source/zebrafishmine.xml" /><tool file="data_source/eupathdb.xml" /><tool file="genomespace/genomespace_file_browser_prod.xml" /><tool file="genomespace/genomespace_importer.xml" /> diff -r 5aca28ab8cfbbc1769f68fcfacf3dfbf9352ccde -r 620437dab02aa2bf8508292bc0df4b2e1066264e config/tool_conf.xml.sample --- a/config/tool_conf.xml.sample +++ b/config/tool_conf.xml.sample @@ -22,6 +22,7 @@ <tool file="data_source/worm_modencode.xml" /><tool file="data_source/wormbase.xml" /><tool file="data_source/wormbase_test.xml" /> + <tool file="data_source/zebrafishmine.xml" /><tool file="data_source/eupathdb.xml" /><tool file="data_source/hbvar.xml" /><tool file="genomespace/genomespace_file_browser_prod.xml" /> diff -r 5aca28ab8cfbbc1769f68fcfacf3dfbf9352ccde -r 620437dab02aa2bf8508292bc0df4b2e1066264e tools/data_source/zebrafishmine.xml --- /dev/null +++ b/tools/data_source/zebrafishmine.xml @@ -0,0 +1,20 @@ +<?xml version="1.0"?> +<tool name="ZebrafishMine" id="zebrafishmine" tool_type="data_source"> + <description>server</description> + <command interpreter="python">data_source.py $output $__app__.config.output_size_limit</command> + <inputs action="http://zebrafishmine.org/begin.do" check_values="false" method="get"> + <display>go to ZebrafishMine server $GALAXY_URL</display> + </inputs> + <request_param_translation> + <request_param galaxy_name="data_type" remote_name="data_type" missing="auto" > + <value_translation> + <value galaxy_value="auto" remote_value="txt" /> + </value_translation> + </request_param> + </request_param_translation> + <uihints minwidth="800"/> + <outputs> + <data name="output" format="txt" /> + </outputs> + <options sanitize="False"/> +</tool> Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0