- galaxy-commits - lists.galaxyproject.org

[galaxyproject/usegalaxy-playbook] 1d0a9a: Update test to 658833a7988fe335a5aaaac9709f9e6dfed...
by GitHub 25 Feb '15

25 Feb '15

Branch: refs/heads/master Home: https://github.com/galaxyproject/usegalaxy-playbook Commit: 1d0a9a2d58d4d1dfcfbbbbd08a9053c8e0bd8107 https://github.com/galaxyproject/usegalaxy-playbook/commit/1d0a9a2d58d4d1df… Author: Dannon Baker <dannon.baker(a)gmail.com> Date: 2015-02-25 (Wed, 25 Feb 2015) Changed paths: M files/galaxy/test.galaxyproject.org/var/integrated_tool_panel.xml M files/galaxy/test.galaxyproject.org/var/shed_tool_conf.xml M stage/group_vars/all.yml Log Message: ----------- Update test to 658833a7988fe335a5aaaac9709f9e6dfed4a7b2

1 0

commit/galaxy-central: 4 new changesets
by commits-noreply＠bitbucket.org 25 Feb '15

25 Feb '15

4 new commits in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/1177b255f17d/ Changeset: 1177b255f17d User: martenson Date: 2015-02-23 22:42:17+00:00 Summary: add API endpoint to create TS repository Affected #: 3 files diff -r a12fc23ff13785d2c2ef7d223dca100ef334ea87 -r 1177b255f17d80190849fc64a5ce9bd88bbe7fd7 lib/galaxy/webapps/tool_shed/api/repositories.py --- a/lib/galaxy/webapps/tool_shed/api/repositories.py +++ b/lib/galaxy/webapps/tool_shed/api/repositories.py @@ -10,8 +10,10 @@ from galaxy import util from galaxy import web +from galaxy import exceptions from galaxy.datatypes import checkers from galaxy.model.orm import and_ +from galaxy.web import _future_expose_api as expose_api from galaxy.web.base.controller import BaseAPIController from galaxy.web.base.controller import HTTPBadRequest from galaxy.web.framework.helpers import time_ago @@ -31,6 +33,7 @@ from tool_shed.util import shed_util_common as suc from tool_shed.util import tool_util + log = logging.getLogger( __name__ ) @@ -577,13 +580,72 @@ id=trans.security.encode_id( repository.id ) ) return repository_dict + @expose_api + def create( self, trans, payload, **kwd ): + """ + create( self, trans, payload, **kwd ) + * POST /api/repositories: + Creates a new repository. + Only ``name`` and ``synopsis`` parameters are required. + + :param payload: dictionary structure containing:: + 'name': new repo's name (required) + 'synopsis': new repo's synopsis (required) + 'description': new repo's description (optional) + 'remote_repository_url': new repo's remote repo (optional) + 'homepage_url': new repo's homepage url (optional) + 'category_ids[]': list of existing encoded TS category ids + the new repo should be associated with (optional) + 'type': new repo's type, defaults to ``unrestricted`` (optional) + + :type payload: dict + + :returns: detailed repository information + :rtype: dict + + :raises: RequestParameterMissingException + """ + params = util.Params( payload ) + name = util.restore_text( params.get( 'name', None ) ) + if not name: + raise exceptions.RequestParameterMissingException( "Missing required parameter 'name'." ) + synopsis = util.restore_text( params.get( 'synopsis', None ) ) + if not synopsis: + raise exceptions.RequestParameterMissingException( "Missing required parameter 'synopsis'." ) + + description = util.restore_text( params.get( 'description', '' ) ) + remote_repository_url = util.restore_text( params.get( 'remote_repository_url', '' ) ) + homepage_url = util.restore_text( params.get( 'homepage_url', '' ) ) + category_ids = util.listify( params.get( 'category_ids[]', '' ) ) + selected_categories = [ trans.security.decode_id( id ) for id in category_ids ] + + repo_type = kwd.get( 'type', rt_util.UNRESTRICTED ) + if repo_type not in rt_util.types: + raise exceptions.RequestParameterInvalidException( 'This repository type is not valid' ) + + invalid_message = repository_util.validate_repository_name( trans.app, name, trans.user ) + if invalid_message: + raise exceptions.RequestParameterInvalidException( invalid_message ) + + repo, message = repository_util.create_repository( app=trans.app, + name=name, + type=repo_type, + description=synopsis, + long_description=description, + user_id = trans.user.id, + category_ids=category_ids, + remote_repository_url=remote_repository_url, + homepage_url=homepage_url ) + + return repo.to_dict( view='element', value_mapper=self.__get_value_mapper( trans ) ) + @web.expose_api def create_changeset_revision( self, trans, id, payload, **kwd ): """ POST /api/repositories/{encoded_repository_id}/changeset_revision Create a new tool shed repository commit - leaving PUT on parent - resource open for updating meta-attirbutes of the repository (and + resource open for updating meta-attributes of the repository (and Galaxy doesn't allow PUT multipart data anyway https://trello.com/c/CQwmCeG6) diff -r a12fc23ff13785d2c2ef7d223dca100ef334ea87 -r 1177b255f17d80190849fc64a5ce9bd88bbe7fd7 lib/galaxy/webapps/tool_shed/buildapp.py --- a/lib/galaxy/webapps/tool_shed/buildapp.py +++ b/lib/galaxy/webapps/tool_shed/buildapp.py @@ -129,6 +129,11 @@ controller='repositories', action='create_changeset_revision', conditions=dict( method=[ "POST" ] ) ) + webapp.mapper.connect( 'create_repository', + '/api/repositories', + controller='repositories', + action='create', + conditions=dict( method=[ "POST" ] ) ) webapp.finalize_config() # Wrap the webapp in some useful middleware diff -r a12fc23ff13785d2c2ef7d223dca100ef334ea87 -r 1177b255f17d80190849fc64a5ce9bd88bbe7fd7 lib/tool_shed/util/repository_util.py --- a/lib/tool_shed/util/repository_util.py +++ b/lib/tool_shed/util/repository_util.py @@ -313,8 +313,11 @@ return associations_dict def validate_repository_name( app, name, user ): - # Repository names must be unique for each user, must be at least four characters - # in length and must contain only lower-case letters, numbers, and the '_' character. + """ + Validate whether the given name qualifies as a new TS repo name. + Repository names must be unique for each user, must be at least two characters + in length and must contain only lower-case letters, numbers, and the '_' character. + """ if name in [ 'None', None, '' ]: return 'Enter the required repository name.' if name in [ 'repos' ]: @@ -322,13 +325,13 @@ check_existing = suc.get_repository_by_name_and_owner( app, name, user.username ) if check_existing is not None: if check_existing.deleted: - return 'You have a deleted repository named %s, so choose a different name.' % name + return 'You own a deleted repository named %s, please choose a different name.' % escape( name ) else: - return "You already have a repository named %s, so choose a different name." % name + return "You already own a repository named %s, please choose a different name." % escape( name ) if len( name ) < 2: return "Repository names must be at least 2 characters in length." if len( name ) > 80: return "Repository names cannot be more than 80 characters in length." if not( VALID_REPOSITORYNAME_RE.match( name ) ): - return "Repository names must contain only lower-case letters, numbers and underscore _." + return "Repository names must contain only lower-case letters, numbers and underscore." return '' https://bitbucket.org/galaxy/galaxy-central/commits/05d1b7925348/ Changeset: 05d1b7925348 User: jmchilton Date: 2015-02-24 20:37:45+00:00 Summary: Fix tool shed-y workflowing for ce1f85a0f09689e86fce1839390a5a00a730321f. Affected #: 1 file diff -r 1177b255f17d80190849fc64a5ce9bd88bbe7fd7 -r 05d1b792534806bd064865d813edcff012dbf81f lib/tool_shed/util/workflow_util.py --- a/lib/tool_shed/util/workflow_util.py +++ b/lib/tool_shed/util/workflow_util.py @@ -6,10 +6,10 @@ import galaxy.tools import galaxy.tools.parameters -import galaxy.webapps.galaxy.controllers.workflow from galaxy.util import json from galaxy.util.sanitize_html import sanitize_html from galaxy.workflow.render import WorkflowCanvas +from galaxy.workflow.steps import attach_ordered_steps from galaxy.workflow.modules import module_types from galaxy.workflow.modules import ToolModule from galaxy.workflow.modules import WorkflowModuleFactory @@ -312,7 +312,7 @@ step.input_connections.append( conn ) del step.temp_input_connections # Order the steps if possible. - galaxy.webapps.galaxy.controllers.workflow.attach_ordered_steps( workflow, steps ) + attach_ordered_steps( workflow, steps ) # Return the in-memory Workflow object for display or later persistence to the Galaxy database. return workflow, missing_tool_tups https://bitbucket.org/galaxy/galaxy-central/commits/20bf9ebc1cd1/ Changeset: 20bf9ebc1cd1 User: carlfeberhard Date: 2015-02-24 22:20:10+00:00 Summary: Fix to layering of dropdown buttons and menus Affected #: 3 files diff -r 05d1b792534806bd064865d813edcff012dbf81f -r 20bf9ebc1cd143d29bb651caf1994f1d9c2db05a static/style/blue/base.css --- a/static/style/blue/base.css +++ b/static/style/blue/base.css @@ -2030,8 +2030,8 @@ .list-panel .controls .list-actions:after{clear:both} .list-panel .controls .list-actions:before,.list-panel .controls .list-actions:after{content:" ";display:table;} .list-panel .controls .list-actions:after{clear:both} -.list-panel .controls .list-actions .btn{padding-top:2px;padding-bottom:2px;font-size:90%;z-index:inherit} -.list-panel .controls .list-actions .list-action-menu{float:right}.list-panel .controls .list-actions .list-action-menu .dropdown-menu{z-index:inherit} +.list-panel .controls .list-actions .btn{padding-top:2px;padding-bottom:2px;font-size:90%} +.list-panel .controls .list-actions .list-action-menu{float:right} .list-panel .list-items .list-item:not(:last-child){border-bottom-width:0px} .list-panel .empty-message{display:none;margin:0px} .list-item .details .list-panel{margin-top:8px;border-radius:3px;background:white;padding:4px}.list-item .details .list-panel .list-items{border:1px solid #bfbfbf;border-radius:3px}.list-item .details .list-panel .list-items .list-item:first-child{border-top-width:0px;border-radius:3px 3px 0px 0px} @@ -2138,9 +2138,10 @@ .multi-panel-history .control-column.control-column-right{text-align:right} .multi-panel-history .outer-middle{overflow:auto} .multi-panel-history .middle{min-width:100%;margin:0px 0px 0px 0px;background-color:white;padding:8px} -.multi-panel-history .history-column{width:312px;margin:0 8px 0 0}.multi-panel-history .history-column:first-child{position:fixed;z-index:2}.multi-panel-history .history-column:first-child .history-panel{border:1px solid black;box-shadow:4px 4px 4px rgba(96,96,96,0.3)} +.multi-panel-history .history-column{width:312px;margin:0 8px 0 0}.multi-panel-history .history-column:first-child{position:fixed;z-index:10}.multi-panel-history .history-column:first-child .history-panel{border:1px solid black;box-shadow:4px 4px 4px rgba(96,96,96,0.3)} .multi-panel-history .history-column:nth-child(2){margin-left:320px} .multi-panel-history .history-column:last-child{margin-right:0px} +.multi-panel-history .history-column .dropdown-menu{z-index:inherit} .multi-panel-history .history-column .panel-controls{width:100%;height:24px;border-radius:3px;background-color:white;text-align:center;flex:0 0 auto;-webkit-align-self:auto;-ms-flex-item-align:auto;align-self:auto}.multi-panel-history .history-column .panel-controls .btn{height:20px;line-height:normal;font-size:90%;padding-top:0px;padding-bottom:0px} .multi-panel-history .history-column .panel-controls .pull-left .btn{margin-right:4px} .multi-panel-history .history-column .panel-controls .pull-right .btn{margin-left:4px} diff -r 05d1b792534806bd064865d813edcff012dbf81f -r 20bf9ebc1cd143d29bb651caf1994f1d9c2db05a static/style/src/less/history.less --- a/static/style/src/less/history.less +++ b/static/style/src/less/history.less @@ -744,7 +744,7 @@ // current history &:first-child { position: fixed; - z-index : 2; + z-index : 10; // visually differentiate the current history .history-panel { @@ -759,6 +759,9 @@ &:last-child { margin-right: 0px; } + .dropdown-menu { + z-index: inherit; + } .panel-controls { width: 100%; diff -r 05d1b792534806bd064865d813edcff012dbf81f -r 20bf9ebc1cd143d29bb651caf1994f1d9c2db05a static/style/src/less/list-item.less --- a/static/style/src/less/list-item.less +++ b/static/style/src/less/list-item.less @@ -167,13 +167,9 @@ padding-top: 2px; padding-bottom: 2px; font-size: 90%; - z-index: inherit; } .list-action-menu { float: right; - .dropdown-menu { - z-index: inherit; - } } } } https://bitbucket.org/galaxy/galaxy-central/commits/658833a7988f/ Changeset: 658833a7988f User: dannon Date: 2015-02-24 23:55:50+00:00 Summary: More tweaks to mutable capabilities, adding back in a copy and stripping pickle from MutationList Affected #: 1 file diff -r 20bf9ebc1cd143d29bb651caf1994f1d9c2db05a -r 658833a7988fe335a5aaaac9709f9e6dfed4a7b2 lib/galaxy/model/custom_types.py --- a/lib/galaxy/model/custom_types.py +++ b/lib/galaxy/model/custom_types.py @@ -1,4 +1,5 @@ import binascii +import copy import json import logging import uuid @@ -62,6 +63,12 @@ else: return self.impl + def copy_value( self, value ): + return copy.deepcopy( value ) + + def compare_values( self, x, y ): + return ( x == y ) + class MutationObj(Mutable): """ @@ -196,12 +203,6 @@ list.remove(self, value) self.changed() - def __getstate__(self): - return list(self) - - def __setstate__(self, state): - self.update(state) - MutationObj.associate_with(JSONType) Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

[galaxyproject/usegalaxy-playbook] f65108: Fetch modified integrated_tool_panel.xml from Main
by GitHub 24 Feb '15

24 Feb '15

Branch: refs/heads/master Home: https://github.com/galaxyproject/usegalaxy-playbook Commit: f65108118e2fa0f18274944b21084b1234937889 https://github.com/galaxyproject/usegalaxy-playbook/commit/f65108118e2fa0f1… Author: Nate Coraor <nate(a)bx.psu.edu> Date: 2015-02-24 (Tue, 24 Feb 2015) Changed paths: M files/galaxy/usegalaxy.org/var/integrated_tool_panel.xml Log Message: ----------- Fetch modified integrated_tool_panel.xml from Main Commit: cf4260106a86f13850876a14db8976fbf1c719d9 https://github.com/galaxyproject/usegalaxy-playbook/commit/cf4260106a86f138… Author: Nate Coraor <nate(a)bx.psu.edu> Date: 2015-02-24 (Tue, 24 Feb 2015) Changed paths: M templates/galaxy/test.galaxyproject.org/config/job_conf.xml.j2 M templates/galaxy/usegalaxy.org/config/job_conf.xml.j2 Log Message: ----------- Switch Test/Main to use a slurm-drmaa compiled without optimization, to assist with tracking down the pthread deadlock. Compare: https://github.com/galaxyproject/usegalaxy-playbook/compare/eb8faf9d51a1...…

1 0

[galaxyproject/usegalaxy-playbook] eb8faf: Unfortunately, metadata failure is still considere...
by GitHub 24 Feb '15

24 Feb '15

Branch: refs/heads/master Home: https://github.com/galaxyproject/usegalaxy-playbook Commit: eb8faf9d51a1b7cffa7f75bdf5b9bc4d5495f05b https://github.com/galaxyproject/usegalaxy-playbook/commit/eb8faf9d51a1b7cf… Author: Nate Coraor <nate(a)bx.psu.edu> Date: 2015-02-24 (Tue, 24 Feb 2015) Changed paths: M stage/group_vars/galaxyservers.yml Log Message: ----------- Unfortunately, metadata failure is still considered a job "success", and the jobdir is removed. So, set Test to never remove jobdirs while we debug metadata failures.

1 0

[galaxyproject/usegalaxy-playbook] bf3597: Update Test
by GitHub 24 Feb '15

24 Feb '15

Branch: refs/heads/master Home: https://github.com/galaxyproject/usegalaxy-playbook Commit: bf3597a0e5a22b66969f58b22cf08ce5c507bcbb https://github.com/galaxyproject/usegalaxy-playbook/commit/bf3597a0e5a22b66… Author: Nate Coraor <nate(a)bx.psu.edu> Date: 2015-02-24 (Tue, 24 Feb 2015) Changed paths: M stage/group_vars/all.yml Log Message: ----------- Update Test

1 0

commit/galaxy-central: 2 new changesets
by commits-noreply＠bitbucket.org 24 Feb '15

24 Feb '15

2 new commits in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/99fef2a32e19/ Changeset: 99fef2a32e19 Branch: release_15.03 User: dan Date: 2015-02-24 17:02:42+00:00 Summary: Fix for installing .loc files from the toolshed to respect shed_tool_data_path. Mention shed_tool_data_path in galaxy.ini.sample. Affected #: 2 files diff -r d1c3d84a7e06553cf948fcbd8c00b45bebcfb059 -r 99fef2a32e19ce5a8cf6137c1da2dbd9ddbb0536 config/galaxy.ini.sample --- a/config/galaxy.ini.sample +++ b/config/galaxy.ini.sample @@ -217,6 +217,10 @@ # https://wiki.galaxyproject.org/Admin/DataIntegration #tool_data_path = tool-data +# Directory where Tool Data Table related files will be placed +# when installed from a ToolShed. Defaults to tool_data_path +#shed_tool_data_path = tool-data + # File containing old-style genome builds #builds_file_path = tool-data/shared/ucsc/builds.txt diff -r d1c3d84a7e06553cf948fcbd8c00b45bebcfb059 -r 99fef2a32e19ce5a8cf6137c1da2dbd9ddbb0536 lib/tool_shed/tools/data_table_manager.py --- a/lib/tool_shed/tools/data_table_manager.py +++ b/lib/tool_shed/tools/data_table_manager.py @@ -103,7 +103,7 @@ try: new_table_elems, message = self.app.tool_data_tables \ .add_new_entries_from_config_file( config_filename=filename, - tool_data_path=self.app.config.tool_data_path, + tool_data_path=self.app.config.shed_tool_data_path, shed_tool_data_table_config=self.app.config.shed_tool_data_table_config, persist=persist ) if message: https://bitbucket.org/galaxy/galaxy-central/commits/4a42638e8cd4/ Changeset: 4a42638e8cd4 Branch: release_15.03 User: dannon Date: 2015-02-24 19:50:02+00:00 Summary: Apply custom_types bugfixes from dev branch to release. Affected #: 1 file diff -r 99fef2a32e19ce5a8cf6137c1da2dbd9ddbb0536 -r 4a42638e8cd45652db9dc1f1eed54be6aa2c20b1 lib/galaxy/model/custom_types.py --- a/lib/galaxy/model/custom_types.py +++ b/lib/galaxy/model/custom_types.py @@ -1,25 +1,23 @@ -from sqlalchemy.types import * +import binascii +import json +import logging +import uuid -import json -import pickle -import copy -import uuid -import binascii -from galaxy.util.bunch import Bunch +from galaxy import eggs +eggs.require("SQLAlchemy") +import sqlalchemy + from galaxy.util.aliaspickler import AliasPickleModule +from sqlalchemy.types import CHAR, LargeBinary, String, TypeDecorator +from sqlalchemy.ext.mutable import Mutable -# For monkeypatching BIGINT -import sqlalchemy.dialects.sqlite -import sqlalchemy.dialects.postgresql -import sqlalchemy.dialects.mysql - -import logging log = logging.getLogger( __name__ ) # Default JSON encoder and decoder json_encoder = json.JSONEncoder( sort_keys=True ) json_decoder = json.JSONDecoder( ) + def _sniffnfix_pg9_hex(value): """ Sniff for and fix postgres 9 hex decoding issue @@ -31,39 +29,32 @@ return binascii.unhexlify( value[2:] ) else: return value - except Exception, ex: + except Exception: return value -class JSONType( TypeDecorator ): + +class JSONType(sqlalchemy.types.TypeDecorator): """ - Defines a JSONType for SQLAlchemy. Takes a primitive as input and - JSONifies it. This should replace PickleType throughout Galaxy. + Represents an immutable structure as a json-encoded string. + + If default is, for example, a dict, then a NULL value in the + database will be exposed as an empty dict. """ + + # TODO: Figure out why this is a large binary, and provide a migratino to + # something like sqlalchemy.String, or even better, when applicable, native + # sqlalchemy.dialects.postgresql.JSON impl = LargeBinary - def process_bind_param( self, value, dialect ): - if value is None: - return None - return json_encoder.encode( value ) + def process_bind_param(self, value, dialect): + if value is not None: + value = json_encoder.encode(value) + return value - def process_result_value( self, value, dialect ): + def process_result_value(self, value, dialect): if value is not None: - try: - return json_decoder.decode( str( _sniffnfix_pg9_hex( value ) ) ) - except Exception, e: - log.error( 'Failed to decode JSON (%s): %s', value, e ) - return None - - def copy_value( self, value ): - # return json_decoder.decode( json_encoder.encode( value ) ) - return copy.deepcopy( value ) - - def compare_values( self, x, y ): - # return json_encoder.encode( x ) == json_encoder.encode( y ) - return ( x == y ) - - def is_mutable( self ): - return True + value = json_decoder.decode( str( _sniffnfix_pg9_hex( value ) ) ) + return value def load_dialect_impl(self, dialect): if dialect.name == "mysql": @@ -72,10 +63,153 @@ return self.impl +class MutationObj(Mutable): + """ + Mutable JSONType for SQLAlchemy from original gist: + https://gist.github.com/dbarnett/1730610 + + Using minor changes from this fork of the gist: + https://gist.github.com/miracle2k/52a031cced285ba9b8cd + + And other minor changes to make it work for us. + """ + @classmethod + def coerce(cls, key, value): + if isinstance(value, dict) and not isinstance(value, MutationDict): + return MutationDict.coerce(key, value) + if isinstance(value, list) and not isinstance(value, MutationList): + return MutationList.coerce(key, value) + return value + + @classmethod + def _listen_on_attribute(cls, attribute, coerce, parent_cls): + key = attribute.key + if parent_cls is not attribute.class_: + return + + # rely on "propagate" here + parent_cls = attribute.class_ + + def load(state, *args): + val = state.dict.get(key, None) + if coerce: + val = cls.coerce(key, val) + state.dict[key] = val + if isinstance(val, cls): + val._parents[state.obj()] = key + + def set(target, value, oldvalue, initiator): + if not isinstance(value, cls): + value = cls.coerce(key, value) + if isinstance(value, cls): + value._parents[target.obj()] = key + if isinstance(oldvalue, cls): + oldvalue._parents.pop(target.obj(), None) + return value + + def pickle(state, state_dict): + val = state.dict.get(key, None) + if isinstance(val, cls): + if 'ext.mutable.values' not in state_dict: + state_dict['ext.mutable.values'] = [] + state_dict['ext.mutable.values'].append(val) + + def unpickle(state, state_dict): + if 'ext.mutable.values' in state_dict: + for val in state_dict['ext.mutable.values']: + val._parents[state.obj()] = key + + sqlalchemy.event.listen(parent_cls, 'load', load, raw=True, propagate=True) + sqlalchemy.event.listen(parent_cls, 'refresh', load, raw=True, propagate=True) + sqlalchemy.event.listen(attribute, 'set', set, raw=True, retval=True, propagate=True) + sqlalchemy.event.listen(parent_cls, 'pickle', pickle, raw=True, propagate=True) + sqlalchemy.event.listen(parent_cls, 'unpickle', unpickle, raw=True, propagate=True) + + +class MutationDict(MutationObj, dict): + @classmethod + def coerce(cls, key, value): + """Convert plain dictionary to MutationDict""" + self = MutationDict((k, MutationObj.coerce(key, v)) for (k, v) in value.items()) + self._key = key + return self + + def __setitem__(self, key, value): + if hasattr(self, '_key'): + value = MutationObj.coerce(self._key, value) + dict.__setitem__(self, key, value) + self.changed() + + def __delitem__(self, key): + dict.__delitem__(self, key) + self.changed() + + def __getstate__(self): + return dict(self) + + def __setstate__(self, state): + self.update(state) + + +class MutationList(MutationObj, list): + @classmethod + def coerce(cls, key, value): + """Convert plain list to MutationList""" + self = MutationList((MutationObj.coerce(key, v) for v in value)) + self._key = key + return self + + def __setitem__(self, idx, value): + list.__setitem__(self, idx, MutationObj.coerce(self._key, value)) + self.changed() + + def __setslice__(self, start, stop, values): + list.__setslice__(self, start, stop, (MutationObj.coerce(self._key, v) for v in values)) + self.changed() + + def __delitem__(self, idx): + list.__delitem__(self, idx) + self.changed() + + def __delslice__(self, start, stop): + list.__delslice__(self, start, stop) + self.changed() + + def append(self, value): + list.append(self, MutationObj.coerce(self._key, value)) + self.changed() + + def insert(self, idx, value): + list.insert(self, idx, MutationObj.coerce(self._key, value)) + self.changed() + + def extend(self, values): + list.extend(self, (MutationObj.coerce(self._key, v) for v in values)) + self.changed() + + def pop(self, *args, **kw): + value = list.pop(self, *args, **kw) + self.changed() + return value + + def remove(self, value): + list.remove(self, value) + self.changed() + + def __getstate__(self): + return list(self) + + def __setstate__(self, state): + self.update(state) + + +MutationObj.associate_with(JSONType) + metadata_pickler = AliasPickleModule( { - ( "cookbook.patterns", "Bunch" ) : ( "galaxy.util.bunch" , "Bunch" ) + ( "cookbook.patterns", "Bunch" ): ( "galaxy.util.bunch", "Bunch" ) } ) + class MetadataType( JSONType ): """ Backward compatible metadata type. Can read pickles or JSON, but always @@ -96,6 +230,7 @@ ret = None return ret + class UUIDType(TypeDecorator): """ Platform-independent UUID type. @@ -129,31 +264,9 @@ class TrimmedString( TypeDecorator ): impl = String + def process_bind_param( self, value, dialect ): """Automatically truncate string values""" if self.impl.length and value is not None: value = value[0:self.impl.length] return value - - -#class BigInteger( Integer ): - #""" - #A type for bigger ``int`` integers. - - #Typically generates a ``BIGINT`` in DDL, and otherwise acts like - #a normal :class:`Integer` on the Python side. - - #""" - -#class BIGINT( BigInteger ): - #"""The SQL BIGINT type.""" - -#class SLBigInteger( BigInteger ): - #def get_col_spec( self ): - #return "BIGINT" - -#sqlalchemy.dialects.sqlite.SLBigInteger = SLBigInteger -#sqlalchemy.dialects.sqlite.colspecs[BigInteger] = SLBigInteger -#sqlalchemy.dialects.sqlite.ischema_names['BIGINT'] = SLBigInteger -#sqlalchemy.dialects.postgres.colspecs[BigInteger] = sqlalchemy.dialects.postgres.PGBigInteger -#sqlalchemy.dialects.mysql.colspecs[BigInteger] = sqlalchemy.dialects.mysql.MSBigInteger Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: 6 new changesets
by commits-noreply＠bitbucket.org 24 Feb '15

24 Feb '15

6 new commits in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/59cd12ceafc4/ Changeset: 59cd12ceafc4 User: dannon Date: 2015-02-24 16:38:18+00:00 Summary: Cleanup of comments, doc of customtypes. Affected #: 1 file diff -r 476c79d7b00a5edde57847ca1882d7e7781030b6 -r 59cd12ceafc4628e05600be46939124d4ca59c08 lib/galaxy/model/custom_types.py --- a/lib/galaxy/model/custom_types.py +++ b/lib/galaxy/model/custom_types.py @@ -33,15 +33,6 @@ return value -# Mutable JSONType for SQLAlchemy from original gist: -# https://gist.github.com/dbarnett/1730610 -# -# Also using minor changes from this fork of the gist: -# https://gist.github.com/miracle2k/52a031cced285ba9b8cd -# -# And other minor changes to make it work for us. - - class JSONType(sqlalchemy.types.TypeDecorator): """Represents an immutable structure as a json-encoded string. @@ -69,6 +60,15 @@ class MutationObj(Mutable): + """ + Mutable JSONType for SQLAlchemy from original gist: + https://gist.github.com/dbarnett/1730610 + + Using minor changes from this fork of the gist: + https://gist.github.com/miracle2k/52a031cced285ba9b8cd + + And other minor changes to make it work for us. + """ @classmethod def coerce(cls, key, value): if isinstance(value, dict) and not isinstance(value, MutationDict): @@ -192,13 +192,6 @@ MutationObj.associate_with(JSONType) -"""A type to encode/decode JSON on the fly - -sqltype is the string type for the underlying DB column:: - - Column(JSON) -""" - metadata_pickler = AliasPickleModule( { ( "cookbook.patterns", "Bunch" ): ( "galaxy.util.bunch", "Bunch" ) } ) https://bitbucket.org/galaxy/galaxy-central/commits/834b3e714755/ Changeset: 834b3e714755 User: dan Date: 2015-02-24 16:50:58+00:00 Summary: Fix for installing .loc files from the toolshed to respect shed_tool_data_path. Mention shed_tool_data_path in galaxy.ini.sample. Affected #: 2 files diff -r 59cd12ceafc4628e05600be46939124d4ca59c08 -r 834b3e71475518902f071d4066d69e707f752730 config/galaxy.ini.sample --- a/config/galaxy.ini.sample +++ b/config/galaxy.ini.sample @@ -217,6 +217,10 @@ # https://wiki.galaxyproject.org/Admin/DataIntegration #tool_data_path = tool-data +# Directory where Tool Data Table related files will be placed +# when installed from a ToolShed. Defaults to tool_data_path +#shed_tool_data_path = tool-data + # File containing old-style genome builds #builds_file_path = tool-data/shared/ucsc/builds.txt diff -r 59cd12ceafc4628e05600be46939124d4ca59c08 -r 834b3e71475518902f071d4066d69e707f752730 lib/tool_shed/tools/data_table_manager.py --- a/lib/tool_shed/tools/data_table_manager.py +++ b/lib/tool_shed/tools/data_table_manager.py @@ -103,7 +103,7 @@ try: new_table_elems, message = self.app.tool_data_tables \ .add_new_entries_from_config_file( config_filename=filename, - tool_data_path=self.app.config.tool_data_path, + tool_data_path=self.app.config.shed_tool_data_path, shed_tool_data_table_config=self.app.config.shed_tool_data_table_config, persist=persist ) if message: https://bitbucket.org/galaxy/galaxy-central/commits/b154ff11d2ea/ Changeset: b154ff11d2ea User: jmchilton Date: 2015-02-24 17:01:35+00:00 Summary: Fix unit tests for dde2fc9572be6d909e22dda510520639db5260bf. Thanks to @ dannon for the bug report. Affected #: 1 file diff -r 834b3e71475518902f071d4066d69e707f752730 -r b154ff11d2ea00d6978da97ee60bb1013f4f7f4e test/unit/jobs/test_job_configuration.py --- a/test/unit/jobs/test_job_configuration.py +++ b/test/unit/jobs/test_job_configuration.py @@ -21,6 +21,7 @@ job_config_file=os.path.join( self.temp_directory, "job_conf.xml" ), use_tasked_jobs=False, job_resource_params_file="/tmp/fake_absent_path", + config_dict={}, ) self.__write_config_from( SIMPLE_JOB_CONF ) self.app = bunch.Bunch( config=self.config, job_metrics=MockJobMetrics() ) https://bitbucket.org/galaxy/galaxy-central/commits/64b4a5b00522/ Changeset: 64b4a5b00522 User: dannon Date: 2015-02-24 18:47:29+00:00 Summary: Fix for JSONType getstate/setstate with pickle (exhibited error during uploading of a file). Affected #: 1 file diff -r b154ff11d2ea00d6978da97ee60bb1013f4f7f4e -r 64b4a5b005224c5639482de4eda52adcfa25df10 lib/galaxy/model/custom_types.py --- a/lib/galaxy/model/custom_types.py +++ b/lib/galaxy/model/custom_types.py @@ -143,6 +143,12 @@ dict.__delitem__(self, key) self.changed() + def __getstate__(self): + return dict(self) + + def __setstate__(self, state): + self.update(state) + class MutationList(MutationObj, list): @classmethod https://bitbucket.org/galaxy/galaxy-central/commits/9f3d45b2e032/ Changeset: 9f3d45b2e032 User: dannon Date: 2015-02-24 18:56:19+00:00 Summary: Add getstate/setstate for pickling to the MutationList as well, though I'm not sure we use this anywhere. Future-proofing. Affected #: 1 file diff -r 64b4a5b005224c5639482de4eda52adcfa25df10 -r 9f3d45b2e032b7d208c1e7e021ab3460795272fe lib/galaxy/model/custom_types.py --- a/lib/galaxy/model/custom_types.py +++ b/lib/galaxy/model/custom_types.py @@ -195,6 +195,12 @@ list.remove(self, value) self.changed() + def __getstate__(self): + return list(self) + + def __setstate__(self, state): + self.update(state) + MutationObj.associate_with(JSONType) https://bitbucket.org/galaxy/galaxy-central/commits/a12fc23ff137/ Changeset: a12fc23ff137 User: dannon Date: 2015-02-24 19:18:24+00:00 Summary: Commentary in custom_types w/ a TODO. Remove irrelevant comments about OrderedDict, since we're not relying on that. Affected #: 1 file diff -r 9f3d45b2e032b7d208c1e7e021ab3460795272fe -r a12fc23ff13785d2c2ef7d223dca100ef334ea87 lib/galaxy/model/custom_types.py --- a/lib/galaxy/model/custom_types.py +++ b/lib/galaxy/model/custom_types.py @@ -34,12 +34,16 @@ class JSONType(sqlalchemy.types.TypeDecorator): - """Represents an immutable structure as a json-encoded string. + """ + Represents an immutable structure as a json-encoded string. If default is, for example, a dict, then a NULL value in the database will be exposed as an empty dict. """ + # TODO: Figure out why this is a large binary, and provide a migratino to + # something like sqlalchemy.String, or even better, when applicable, native + # sqlalchemy.dialects.postgresql.JSON impl = LargeBinary def process_bind_param(self, value, dialect): @@ -131,9 +135,6 @@ return self def __setitem__(self, key, value): - # Due to the way OrderedDict works, this is called during __init__. - # At this time we don't have a key set, but what is more, the value - # being set has already been coerced. So special case this and skip. if hasattr(self, '_key'): value = MutationObj.coerce(self._key, value) dict.__setitem__(self, key, value) Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

[galaxyproject/usegalaxy-playbook] 7a7a4e: update mutable confs
by GitHub 24 Feb '15

24 Feb '15

Branch: refs/heads/master Home: https://github.com/galaxyproject/usegalaxy-playbook Commit: 7a7a4e4d8ef3b238d8ef93d00c77372fef237c04 https://github.com/galaxyproject/usegalaxy-playbook/commit/7a7a4e4d8ef3b238… Author: Martin Cech <cech.marten(a)gmail.com> Date: 2015-02-24 (Tue, 24 Feb 2015) Changed paths: M files/galaxy/usegalaxy.org/var/integrated_tool_panel.xml M files/galaxy/usegalaxy.org/var/shed_tool_conf.xml M files/galaxy/usegalaxy.org/var/shed_tool_data_table_conf.xml Log Message: ----------- update mutable confs

1 0

[galaxyproject/usegalaxy-playbook] f84180: Update test to 476c79d7b00a5edde57847ca1882d7e7781...
by GitHub 24 Feb '15

24 Feb '15

Branch: refs/heads/master Home: https://github.com/galaxyproject/usegalaxy-playbook Commit: f841803b0b0a3b0b8dd8d2bb4f42845be982f341 https://github.com/galaxyproject/usegalaxy-playbook/commit/f841803b0b0a3b0b… Author: Dannon Baker <dannon.baker(a)gmail.com> Date: 2015-02-24 (Tue, 24 Feb 2015) Changed paths: M files/galaxy/test.galaxyproject.org/var/integrated_tool_panel.xml M files/galaxy/test.galaxyproject.org/var/shed_tool_conf.xml M files/galaxy/test.galaxyproject.org/var/shed_tool_data_table_conf.xml M stage/group_vars/all.yml Log Message: ----------- Update test to 476c79d7b00a5edde57847ca1882d7e7781030b6

1 0

commit/galaxy-central: 2 new changesets
by commits-noreply＠bitbucket.org 24 Feb '15

24 Feb '15

2 new commits in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/525b1fc16a58/ Changeset: 525b1fc16a58 Branch: release_15.03 User: jmchilton Date: 2015-02-22 05:32:46+00:00 Summary: Fix for watching tools on some more interesting file systems. Affected #: 3 files diff -r 33fd43f7590f17ed611ce07178cae8c2b70525d0 -r 525b1fc16a587e91d60cdc4f9152f8e881b7e7de config/galaxy.ini.sample --- a/config/galaxy.ini.sample +++ b/config/galaxy.ini.sample @@ -176,11 +176,15 @@ # install from in the admin interface (.sample used if default does not exist). #tool_sheds_config_file = config/tool_sheds_conf.xml -# Enable monitoring of tools and tool directories listed in any tool config file -# specified in tool_config_file option. If changes are found, tools are -# automatically reloaded. -# Watchdog ( https://pypi.python.org/pypi/watchdog ) must be installed and -# available to Galaxy to use this option. +# Set to True to Enable monitoring of tools and tool directories +# listed in any tool config file specified in tool_config_file option. +# If changes are found, tools are automatically reloaded. Watchdog ( +# https://pypi.python.org/pypi/watchdog ) must be installed and +# available to Galaxy to use this option. Other options include 'auto' +# which will attempt to watch tools if the watchdog library is available +# but won't fail to load Galaxy if it is not and 'polling' which will use +# a less efficient monitoring scheme that may work in wider range of scenarios +# then the watchdog default montiory. #watch_tools = False # Enable automatic polling of relative tool sheds to see if any updates diff -r 33fd43f7590f17ed611ce07178cae8c2b70525d0 -r 525b1fc16a587e91d60cdc4f9152f8e881b7e7de lib/galaxy/config.py --- a/lib/galaxy/config.py +++ b/lib/galaxy/config.py @@ -272,7 +272,7 @@ self.ftp_upload_site = kwargs.get( 'ftp_upload_site', None ) self.allow_library_path_paste = kwargs.get( 'allow_library_path_paste', False ) self.disable_library_comptypes = kwargs.get( 'disable_library_comptypes', '' ).lower().split( ',' ) - self.watch_tools = string_as_bool( kwargs.get( 'watch_tools', False ) ) + self.watch_tools = kwargs.get( 'watch_tools', 'false' ) # On can mildly speed up Galaxy startup time by disabling index of help, # not needed on production systems but useful if running many functional tests. self.index_tool_help = string_as_bool( kwargs.get( "index_tool_help", True ) ) diff -r 33fd43f7590f17ed611ce07178cae8c2b70525d0 -r 525b1fc16a587e91d60cdc4f9152f8e881b7e7de lib/galaxy/tools/toolbox/watcher.py --- a/lib/galaxy/tools/toolbox/watcher.py +++ b/lib/galaxy/tools/toolbox/watcher.py @@ -2,9 +2,11 @@ try: from watchdog.events import FileSystemEventHandler from watchdog.observers import Observer + from watchdog.observers.polling import PollingObserver can_watch = True except ImportError: FileSystemEventHandler = object + PollingObserver = object can_watch = False import logging @@ -12,23 +14,34 @@ def get_watcher(toolbox, config): - watch_tools = getattr(config, "watch_tools", False) - if watch_tools: + watch_tools_val = str( getattr(config, "watch_tools", False) ).lower() + if watch_tools_val in ( 'true', 'yes', 'on' ): return ToolWatcher(toolbox) + elif watch_tools_val == "auto": + try: + return ToolWatcher(toolbox) + except Exception: + log.info("Failed to load ToolWatcher (watchdog is likely unavailable) - proceeding without tool monitoring.") + return NullWatcher() + elif watch_tools_val == "polling": + log.info("Using less ineffecient polling toolbox watcher.") + return ToolWatcher(toolbox, observer_class=PollingObserver) else: return NullWatcher() class ToolWatcher(object): - def __init__(self, toolbox): + def __init__(self, toolbox, observer_class=None): if not can_watch: raise Exception("Watchdog library unavailble, cannot watch tools.") + if observer_class is None: + observer_class = Observer self.toolbox = toolbox self.tool_file_ids = {} self.tool_dir_callbacks = {} self.monitored_dirs = {} - self.observer = Observer() + self.observer = observer_class() self.event_handler = ToolFileEventHandler(self) self.start() https://bitbucket.org/galaxy/galaxy-central/commits/d1c3d84a7e06/ Changeset: d1c3d84a7e06 Branch: release_15.03 User: jmchilton Date: 2015-02-24 04:09:19+00:00 Summary: Comment out broken liftOver tests. liftOver loc file handling is problematic (https://trello.com/c/A6I6zQvF) If it weren't this wouldn't be a problem. I could fix the API functional test framework to mimic the older form driven approach but it would take a week and would be rendered moot by fixing liftOver. Unfortunately, fixing liftOver would likely break a lot of existing workflows and tool reruns so I am not fixing that either. Only option left, admittedly a crappy one, is to comment out the tests. Affected #: 1 file diff -r 525b1fc16a587e91d60cdc4f9152f8e881b7e7de -r d1c3d84a7e06553cf948fcbd8c00b45bebcfb059 tools/extract/liftOver_wrapper.xml --- a/tools/extract/liftOver_wrapper.xml +++ b/tools/extract/liftOver_wrapper.xml @@ -62,6 +62,7 @@ <requirement type="package">ucsc_tools</requirement></requirements><tests> + </tests><help> .. class:: warningmark Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0