December 2011 - galaxy-commits - lists.galaxyproject.org

commit/galaxy-central: jgoecks: Add hidden attribute to tool outputs so that an output can be hidden by default.
by Bitbucket 19 Dec '11

19 Dec '11

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/changeset/303741fda0ec/ changeset: 303741fda0ec user: jgoecks date: 2011-12-19 19:40:18 summary: Add hidden attribute to tool outputs so that an output can be hidden by default. affected #: 2 files diff -r 10e0d2765f24c32f840be414d88bf813321e41b5 -r 303741fda0ec4ce0fc7b42a0334a6c84f44c7a1f lib/galaxy/tools/__init__.py --- a/lib/galaxy/tools/__init__.py +++ b/lib/galaxy/tools/__init__.py @@ -316,7 +316,7 @@ """ def __init__( self, name, format=None, format_source=None, metadata_source=None, - parent=None, label=None, filters = None, actions = None ): + parent=None, label=None, filters = None, actions = None, hidden=False ): self.name = name self.format = format self.format_source = format_source @@ -325,6 +325,7 @@ self.label = label self.filters = filters or [] self.actions = actions + self.hidden = hidden # Tuple emulation @@ -677,6 +678,7 @@ output.count = int( data_elem.get("count", 1) ) output.filters = data_elem.findall( 'filter' ) output.from_work_dir = data_elem.get("from_work_dir", None) + output.hidden = data_elem.get("hidden", None) in [ 'true', 'True' ] output.tool = self output.actions = ToolOutputActionGroup( output, data_elem.find( 'actions' ) ) self.outputs[ output.name ] = output diff -r 10e0d2765f24c32f840be414d88bf813321e41b5 -r 303741fda0ec4ce0fc7b42a0334a6c84f44c7a1f lib/galaxy/tools/actions/__init__.py --- a/lib/galaxy/tools/actions/__init__.py +++ b/lib/galaxy/tools/actions/__init__.py @@ -286,6 +286,8 @@ if str( getattr( check, when_elem.get( 'attribute' ) ) ) == when_elem.get( 'value', None ): ext = when_elem.get( 'format', ext ) data = trans.app.model.HistoryDatasetAssociation( extension=ext, create_dataset=True, sa_session=trans.sa_session ) + if output.hidden: + data.visible = False # Commit the dataset immediately so it gets database assigned unique id trans.sa_session.add( data ) trans.sa_session.flush() Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: greg: Only persist the datatypes registry when the load_datatypes() method is called. Tools and metadata setting processes that create their own datatypes registry will use the existing temporary file rather than persisting the registry themselves.
by Bitbucket 18 Dec '11

18 Dec '11

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/changeset/10e0d2765f24/ changeset: 10e0d2765f24 user: greg date: 2011-12-18 19:49:26 summary: Only persist the datatypes registry when the load_datatypes() method is called. Tools and metadata setting processes that create their own datatypes registry will use the existing temporary file rather than persisting the registry themselves. affected #: 4 files diff -r 4ee7b5f6b23db6f92bfe6b895bb6db5f4cf4d7b1 -r 10e0d2765f24c32f840be414d88bf813321e41b5 lib/galaxy/datatypes/registry.py --- a/lib/galaxy/datatypes/registry.py +++ b/lib/galaxy/datatypes/registry.py @@ -174,6 +174,9 @@ self.log.debug( 'Loaded sniffer for datatype: %s' % dtype ) except Exception, exc: self.log.warning( 'Error appending datatype %s to sniff_order, problem: %s' % ( dtype, str( exc ) ) ) + # Persist the xml form of the registry into a temporary file so that it + # can be loaded from the command line by tools and set_metadata processing. + self.to_xml_file() # Default values. if not self.datatypes_by_extension: self.datatypes_by_extension = { @@ -466,4 +469,3 @@ os.write( fd, '</sniffers>\n' ) os.write( fd, '</datatypes>\n' ) os.close( fd ) - return os.path.abspath( filename ) diff -r 4ee7b5f6b23db6f92bfe6b895bb6db5f4cf4d7b1 -r 10e0d2765f24c32f840be414d88bf813321e41b5 lib/galaxy/jobs/__init__.py --- a/lib/galaxy/jobs/__init__.py +++ b/lib/galaxy/jobs/__init__.py @@ -868,7 +868,7 @@ if config_root is None: config_root = self.app.config.root if datatypes_config is None: - datatypes_config = self.app.datatypes_registry.to_xml_file() + datatypes_config = self.app.datatypes_registry.xml_filename return self.external_output_metadata.setup_external_metadata( [ output_dataset_assoc.dataset for output_dataset_assoc in job.output_datasets ], self.sa_session, exec_dir = exec_dir, diff -r 4ee7b5f6b23db6f92bfe6b895bb6db5f4cf4d7b1 -r 10e0d2765f24c32f840be414d88bf813321e41b5 lib/galaxy/tools/__init__.py --- a/lib/galaxy/tools/__init__.py +++ b/lib/galaxy/tools/__init__.py @@ -1672,8 +1672,7 @@ # For the upload tool, we need to know the root directory and the # datatypes conf path, so we can load the datatypes registry param_dict['__root_dir__'] = param_dict['GALAXY_ROOT_DIR'] = os.path.abspath( self.app.config.root ) - datatypes_config = self.app.datatypes_registry.to_xml_file() - param_dict['__datatypes_config__'] = param_dict['GALAXY_DATATYPES_CONF_FILE'] = datatypes_config + param_dict['__datatypes_config__'] = param_dict['GALAXY_DATATYPES_CONF_FILE'] = self.app.datatypes_registry.xml_filename # Return the dictionary of parameters return param_dict diff -r 4ee7b5f6b23db6f92bfe6b895bb6db5f4cf4d7b1 -r 10e0d2765f24c32f840be414d88bf813321e41b5 lib/galaxy/tools/actions/metadata.py --- a/lib/galaxy/tools/actions/metadata.py +++ b/lib/galaxy/tools/actions/metadata.py @@ -51,7 +51,7 @@ dataset_files_path = trans.app.model.Dataset.file_path, output_fnames = None, config_root = None, - datatypes_config = trans.app.datatypes_registry.to_xml_file(), + datatypes_config = trans.app.datatypes_registry.xml_filename, job_metadata = None, kwds = { 'overwrite' : overwrite } ) incoming[ '__SET_EXTERNAL_METADATA_COMMAND_LINE__' ] = cmd_line Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: greg: Always set / return the absolute path of the file to which the datatypes registry is persisted.
by Bitbucket 18 Dec '11

18 Dec '11

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/changeset/4ee7b5f6b23d/ changeset: 4ee7b5f6b23d user: greg date: 2011-12-18 16:52:02 summary: Always set / return the absolute path of the file to which the datatypes registry is persisted. affected #: 2 files diff -r da4342bc04c779a50f50431fbec87d3d0c56576b -r 4ee7b5f6b23db6f92bfe6b895bb6db5f4cf4d7b1 lib/galaxy/datatypes/registry.py --- a/lib/galaxy/datatypes/registry.py +++ b/lib/galaxy/datatypes/registry.py @@ -441,7 +441,7 @@ pass self.xml_filename = None fd, filename = tempfile.mkstemp() - self.xml_filename = filename + self.xml_filename = os.path.abspath( filename ) if self.converters_path_attr: converters_path_str = ' converters_path="%s"' % self.converters_path_attr else: @@ -466,4 +466,4 @@ os.write( fd, '</sniffers>\n' ) os.write( fd, '</datatypes>\n' ) os.close( fd ) - return filename + return os.path.abspath( filename ) diff -r da4342bc04c779a50f50431fbec87d3d0c56576b -r 4ee7b5f6b23db6f92bfe6b895bb6db5f4cf4d7b1 lib/galaxy/tools/__init__.py --- a/lib/galaxy/tools/__init__.py +++ b/lib/galaxy/tools/__init__.py @@ -1673,7 +1673,7 @@ # datatypes conf path, so we can load the datatypes registry param_dict['__root_dir__'] = param_dict['GALAXY_ROOT_DIR'] = os.path.abspath( self.app.config.root ) datatypes_config = self.app.datatypes_registry.to_xml_file() - param_dict['__datatypes_config__'] = param_dict['GALAXY_DATATYPES_CONF_FILE'] = os.path.abspath( datatypes_config ) + param_dict['__datatypes_config__'] = param_dict['GALAXY_DATATYPES_CONF_FILE'] = datatypes_config # Return the dictionary of parameters return param_dict Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: greg: Send the persisted data types registry instead of None as the value of the datatypes_config parameter to the setup_external_metadata method when creating the cmd_line in the SetMetadataToolAction.execute() method. This eliminates the use of the hard-coded 'datatypes_conf.xml' file name when setting metadata externally.
by Bitbucket 18 Dec '11

18 Dec '11

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/changeset/da4342bc04c7/ changeset: da4342bc04c7 user: greg date: 2011-12-18 15:16:52 summary: Send the persisted data types registry instead of None as the value of the datatypes_config parameter to the setup_external_metadata method when creating the cmd_line in the SetMetadataToolAction.execute() method. This eliminates the use of the hard-coded 'datatypes_conf.xml' file name when setting metadata externally. affected #: 1 file diff -r be0a75fd565845be5d6a9719552013548bb6a82f -r da4342bc04c779a50f50431fbec87d3d0c56576b lib/galaxy/tools/actions/metadata.py --- a/lib/galaxy/tools/actions/metadata.py +++ b/lib/galaxy/tools/actions/metadata.py @@ -51,7 +51,7 @@ dataset_files_path = trans.app.model.Dataset.file_path, output_fnames = None, config_root = None, - datatypes_config = None, + datatypes_config = trans.app.datatypes_registry.to_xml_file(), job_metadata = None, kwds = { 'overwrite' : overwrite } ) incoming[ '__SET_EXTERNAL_METADATA_COMMAND_LINE__' ] = cmd_line Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: jgoecks: Trackster: fix bug in 13ba6909faae that prevented display in Dense mode.
by Bitbucket 17 Dec '11

17 Dec '11

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/changeset/be0a75fd5658/ changeset: be0a75fd5658 user: jgoecks date: 2011-12-17 20:29:23 summary: Trackster: fix bug in 13ba6909faae that prevented display in Dense mode. affected #: 1 file diff -r e65f4e9539144c882f4f07355a033f53a1963ddd -r be0a75fd565845be5d6a9719552013548bb6a82f static/scripts/trackster.js --- a/static/scripts/trackster.js +++ b/static/scripts/trackster.js @@ -3966,10 +3966,8 @@ return this.summary_draw_height + SUMMARY_TREE_TOP_PADDING; } else { - var rows_required = 1; - if (mode === "no_detail" || mode === "Squish" || mode === "Pack") { - var rows_required = this.incremental_slots(w_scale, result.data, mode); - } + // All other modes require slotting. + var rows_required = this.incremental_slots(w_scale, result.data, mode); // HACK: use dummy painter to get required height. Painter should be extended so that get_required_height // works as a static function. var dummy_painter = new (this.painter)(null, null, null, this.prefs, mode); Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: greg: Always attempt to remove previously written temporary xml files when persisting the current datatypes registry.
by Bitbucket 16 Dec '11

16 Dec '11

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/changeset/e65f4e953914/ changeset: e65f4e953914 user: greg date: 2011-12-16 23:08:35 summary: Always attempt to remove previously written temporary xml files when persisting the current datatypes registry. affected #: 2 files diff -r 48b3531465ee90f26680291971c837670ad0b7f0 -r e65f4e9539144c882f4f07355a033f53a1963ddd lib/galaxy/app.py --- a/lib/galaxy/app.py +++ b/lib/galaxy/app.py @@ -126,8 +126,7 @@ try: # If the datatypes registry was persisted, attempt to # remove the temporary file in which it was written. - tmp_filename = self.datatypes_registry.xml_filename - if tmp_filename: - os.unlink( tmp_filename ) + if self.datatypes_registry.xml_filename is not None: + os.unlink( self.datatypes_registry.xml_filename ) except: pass diff -r 48b3531465ee90f26680291971c837670ad0b7f0 -r e65f4e9539144c882f4f07355a033f53a1963ddd lib/galaxy/datatypes/registry.py --- a/lib/galaxy/datatypes/registry.py +++ b/lib/galaxy/datatypes/registry.py @@ -432,6 +432,14 @@ rval[ 'auto' ] = rval[ 'txt' ] return rval def to_xml_file( self ): + if self.xml_filename is not None: + # If persisted previously, attempt to remove + # the temporary file in which we were written. + try: + os.unlink( self.xml_filename ) + except: + pass + self.xml_filename = None fd, filename = tempfile.mkstemp() self.xml_filename = filename if self.converters_path_attr: Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: greg: Enhance the datatypes registry so that it can be persisted as an xml file, which is then used for all tools instead of the datatypes config file.
by Bitbucket 16 Dec '11

16 Dec '11

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/changeset/48b3531465ee/ changeset: 48b3531465ee user: greg date: 2011-12-16 22:10:06 summary: Enhance the datatypes registry so that it can be persisted as an xml file, which is then used for all tools instead of the datatypes config file. affected #: 11 files diff -r c6764f7a359cf26c81cfc14ff0e9e94e93b6c82a -r 48b3531465ee90f26680291971c837670ad0b7f0 lib/galaxy/app.py --- a/lib/galaxy/app.py +++ b/lib/galaxy/app.py @@ -118,9 +118,16 @@ self.job_stop_queue = self.job_manager.job_stop_queue # Initialize the external service types self.external_service_types = external_service_types.ExternalServiceTypesCollection( self.config.external_service_type_config_file, self.config.external_service_type_path, self ) - def shutdown( self ): self.job_manager.shutdown() self.object_store.shutdown() if self.heartbeat: self.heartbeat.shutdown() + try: + # If the datatypes registry was persisted, attempt to + # remove the temporary file in which it was written. + tmp_filename = self.datatypes_registry.xml_filename + if tmp_filename: + os.unlink( tmp_filename ) + except: + pass diff -r c6764f7a359cf26c81cfc14ff0e9e94e93b6c82a -r 48b3531465ee90f26680291971c837670ad0b7f0 lib/galaxy/datatypes/metadata.py --- a/lib/galaxy/datatypes/metadata.py +++ b/lib/galaxy/datatypes/metadata.py @@ -544,6 +544,7 @@ if config_root is None: config_root = os.path.abspath( os.getcwd() ) if datatypes_config is None: + raise Exception( 'In setup_external_metadata, the received datatypes_config is None.' ) datatypes_config = 'datatypes_conf.xml' metadata_files_list = [] for dataset in datasets: diff -r c6764f7a359cf26c81cfc14ff0e9e94e93b6c82a -r 48b3531465ee90f26680291971c837670ad0b7f0 lib/galaxy/datatypes/registry.py --- a/lib/galaxy/datatypes/registry.py +++ b/lib/galaxy/datatypes/registry.py @@ -27,9 +27,15 @@ self.sniff_order = [] self.upload_file_formats = [] self.display_applications = odict() #map a display application id to a display application + self.converters_path_attr = None self.datatype_converters_path = None + self.indexers_path_attr = None self.datatype_indexers_path = None + self.display_path_attr = None self.display_applications_path = None + self.datatype_elems = [] + self.sniffer_elems = [] + self.xml_filename = None def load_datatypes( self, root_dir=None, config=None, imported_module=None ): if root_dir and config: inherit_display_application_by_class = [] @@ -45,16 +51,21 @@ # files installed with repositories from tool sheds must use the same paths. However, we # may discover at some future time that allowing for multiple paths is more optimal. if not self.datatype_converters_path: - self.datatype_converters_path = os.path.join( root_dir, registration.get( 'converters_path', 'lib/galaxy/datatypes/converters' ) ) + self.converters_path_attr = registration.get( 'converters_path', 'lib/galaxy/datatypes/converters' ) + self.datatype_converters_path = os.path.join( root_dir, self.converters_path_attr ) if not os.path.isdir( self.datatype_converters_path ): raise ConfigurationError( "Directory does not exist: %s" % self.datatype_converters_path ) if not self.datatype_indexers_path: - self.datatype_indexers_path = os.path.join( root_dir, registration.get( 'indexers_path', 'lib/galaxy/datatypes/indexers' ) ) + self.indexers_path_attr = registration.get( 'indexers_path', 'lib/galaxy/datatypes/indexers' ) + self.datatype_indexers_path = os.path.join( root_dir, self.indexers_path_attr ) if not os.path.isdir( self.datatype_indexers_path ): raise ConfigurationError( "Directory does not exist: %s" % self.datatype_indexers_path ) if not self.display_applications_path: - self.display_applications_path = os.path.join( root_dir, registration.get( 'display_path', 'display_applications' ) ) + self.display_path_attr = registration.get( 'display_path', 'display_applications' ) + self.display_applications_path = os.path.join( root_dir, self.display_path_attr ) for elem in registration.findall( 'datatype' ): + # Keep an in-memory list of datatype elems to enable persistence. + self.datatype_elems.append( elem ) try: extension = elem.get( 'extension', None ) dtype = elem.get( 'type', None ) @@ -147,6 +158,8 @@ sniffers = root.find( 'sniffers' ) if sniffers: for elem in sniffers.findall( 'sniffer' ): + # Keep an in-memory list of sniffer elems to enable persistence. + self.sniffer_elems.append( elem ) dtype = elem.get( 'type', None ) if dtype: try: @@ -418,3 +431,31 @@ if 'auto' not in rval and 'txt' in rval: #need to manually add 'auto' datatype rval[ 'auto' ] = rval[ 'txt' ] return rval + def to_xml_file( self ): + fd, filename = tempfile.mkstemp() + self.xml_filename = filename + if self.converters_path_attr: + converters_path_str = ' converters_path="%s"' % self.converters_path_attr + else: + converters_path_str = '' + if self.indexers_path_attr: + indexers_path_str = ' indexers_path="%s"' % self.indexers_path_attr + else: + indexers_path_str = '' + if self.display_path_attr: + display_path_str = ' display_path="%s"' % self.display_path_attr + else: + display_path_str = '' + os.write( fd, '<?xml version="1.0"?>\n' ) + os.write( fd, '<datatypes>\n' ) + os.write( fd, '<registration%s%s%s>\n' % ( converters_path_str, indexers_path_str, display_path_str ) ) + for elem in self.datatype_elems: + os.write( fd, '%s' % galaxy.util.xml_to_string( elem ) ) + os.write( fd, '</registration>\n' ) + os.write( fd, '<sniffers>\n' ) + for elem in self.sniffer_elems: + os.write( fd, '%s' % galaxy.util.xml_to_string( elem ) ) + os.write( fd, '</sniffers>\n' ) + os.write( fd, '</datatypes>\n' ) + os.close( fd ) + return filename diff -r c6764f7a359cf26c81cfc14ff0e9e94e93b6c82a -r 48b3531465ee90f26680291971c837670ad0b7f0 lib/galaxy/jobs/__init__.py --- a/lib/galaxy/jobs/__init__.py +++ b/lib/galaxy/jobs/__init__.py @@ -868,7 +868,7 @@ if config_root is None: config_root = self.app.config.root if datatypes_config is None: - datatypes_config = self.app.config.datatypes_config + datatypes_config = self.app.datatypes_registry.to_xml_file() return self.external_output_metadata.setup_external_metadata( [ output_dataset_assoc.dataset for output_dataset_assoc in job.output_datasets ], self.sa_session, exec_dir = exec_dir, diff -r c6764f7a359cf26c81cfc14ff0e9e94e93b6c82a -r 48b3531465ee90f26680291971c837670ad0b7f0 lib/galaxy/tool_shed/__init__.py --- a/lib/galaxy/tool_shed/__init__.py +++ b/lib/galaxy/tool_shed/__init__.py @@ -14,7 +14,8 @@ def load_datatypes( self ): for tool_shed_repository in self.sa_session.query( self.model.ToolShedRepository ) \ .filter( and_( self.model.ToolShedRepository.table.c.includes_datatypes==True, - self.model.ToolShedRepository.table.c.deleted==False ) ): + self.model.ToolShedRepository.table.c.deleted==False ) ) \ + .order_by( self.model.ToolShedRepository.table.c.id ): metadata = tool_shed_repository.metadata datatypes_config = metadata[ 'datatypes_config' ] full_path = os.path.abspath( datatypes_config ) diff -r c6764f7a359cf26c81cfc14ff0e9e94e93b6c82a -r 48b3531465ee90f26680291971c837670ad0b7f0 lib/galaxy/tools/__init__.py --- a/lib/galaxy/tools/__init__.py +++ b/lib/galaxy/tools/__init__.py @@ -1672,7 +1672,8 @@ # For the upload tool, we need to know the root directory and the # datatypes conf path, so we can load the datatypes registry param_dict['__root_dir__'] = param_dict['GALAXY_ROOT_DIR'] = os.path.abspath( self.app.config.root ) - param_dict['__datatypes_config__'] = param_dict['GALAXY_DATATYPES_CONF_FILE'] = os.path.abspath( self.app.config.datatypes_config ) + datatypes_config = self.app.datatypes_registry.to_xml_file() + param_dict['__datatypes_config__'] = param_dict['GALAXY_DATATYPES_CONF_FILE'] = os.path.abspath( datatypes_config ) # Return the dictionary of parameters return param_dict diff -r c6764f7a359cf26c81cfc14ff0e9e94e93b6c82a -r 48b3531465ee90f26680291971c837670ad0b7f0 lib/galaxy/util/__init__.py --- a/lib/galaxy/util/__init__.py +++ b/lib/galaxy/util/__init__.py @@ -107,10 +107,28 @@ ElementInclude.include(root) return tree -def xml_to_string(elem): - """Returns an string from and xml tree""" - text = ElementTree.tostring(elem) - return text +def xml_to_string( elem, pretty=False ): + """Returns a string from an xml tree""" + if pretty: + return ElementTree.tostring( pretty_print_xml( elem ) ) + return ElementTree.tostring( elem ) + +def pretty_print_xml( elem, level=0 ): + pad = ' ' + i = "\n" + level * pad + if len( elem ): + if not elem.text or not elem.text.strip(): + elem.text = i + pad + pad + if not elem.tail or not elem.tail.strip(): + elem.tail = i + for e in elem: + pretty_print_xml( e, level + 1 ) + if not elem.tail or not elem.tail.strip(): + elem.tail = i + else: + if level and ( not elem.tail or not elem.tail.strip() ): + elem.tail = i + pad + return elem # characters that are valid valid_chars = set(string.letters + string.digits + " -=_.()/+*^,:?!") diff -r c6764f7a359cf26c81cfc14ff0e9e94e93b6c82a -r 48b3531465ee90f26680291971c837670ad0b7f0 lib/galaxy/util/none_like.py --- a/lib/galaxy/util/none_like.py +++ b/lib/galaxy/util/none_like.py @@ -21,6 +21,7 @@ self.ext = self.extension = ext self.dbkey = dbkey if datatypes_registry is None: + # Default Value Required for unit tests datatypes_registry = Registry() datatypes_registry.load_datatypes() self.datatype = datatypes_registry.get_datatype_by_extension( ext ) diff -r c6764f7a359cf26c81cfc14ff0e9e94e93b6c82a -r 48b3531465ee90f26680291971c837670ad0b7f0 lib/galaxy/util/shed_util.py --- a/lib/galaxy/util/shed_util.py +++ b/lib/galaxy/util/shed_util.py @@ -9,7 +9,7 @@ pkg_resources.require( 'elementtree' ) from elementtree import ElementTree, ElementInclude -from elementtree.ElementTree import Element, SubElement, tostring +from elementtree.ElementTree import Element, SubElement log = logging.getLogger( __name__ ) @@ -39,7 +39,7 @@ if line.startswith( '</toolbox>' ): # We're at the end of the original config file, so add our entry. new_shed_tool_conf.write( ' ' ) - new_shed_tool_conf.write( tostring( pretty_print_xml( tool_panel_entry ) ) ) + new_shed_tool_conf.write( util.xml_to_string( tool_panel_entry, pretty=True ) ) new_shed_tool_conf.write( line ) else: new_shed_tool_conf.write( line ) @@ -554,22 +554,6 @@ log.debug( "Adding new row (or updating an existing row) for repository '%s' in the tool_shed_repository table." % name ) create_or_update_tool_shed_repository( app, name, description, changeset_revision, repository_clone_url, metadata_dict ) return metadata_dict -def pretty_print_xml( elem, level=0 ): - pad = ' ' - i = "\n" + level * pad - if len( elem ): - if not elem.text or not elem.text.strip(): - elem.text = i + pad + pad - if not elem.tail or not elem.tail.strip(): - elem.tail = i - for e in elem: - pretty_print_xml( e, level + 1 ) - if not elem.tail or not elem.tail.strip(): - elem.tail = i - else: - if level and ( not elem.tail or not elem.tail.strip() ): - elem.tail = i + pad - return elem def pull_repository( current_working_dir, repo_files_dir, name ): # Pull the latest possible contents to the repository. log.debug( "Pulling latest updates to the repository named '%s'" % name ) diff -r c6764f7a359cf26c81cfc14ff0e9e94e93b6c82a -r 48b3531465ee90f26680291971c837670ad0b7f0 lib/galaxy/web/base/controller.py --- a/lib/galaxy/web/base/controller.py +++ b/lib/galaxy/web/base/controller.py @@ -22,10 +22,6 @@ from Cheetah.Template import Template -pkg_resources.require( 'elementtree' ) -from elementtree import ElementTree, ElementInclude -from elementtree.ElementTree import Element, SubElement, tostring - log = logging.getLogger( __name__ ) # States for passing messages diff -r c6764f7a359cf26c81cfc14ff0e9e94e93b6c82a -r 48b3531465ee90f26680291971c837670ad0b7f0 lib/galaxy/webapps/community/app.py --- a/lib/galaxy/webapps/community/app.py +++ b/lib/galaxy/webapps/community/app.py @@ -16,6 +16,7 @@ config.configure_logging( self.config ) # Set up datatypes registry self.datatypes_registry = galaxy.datatypes.registry.Registry() + # TODO: Handle datatypes included in repositories - the following will only load datatypes_conf.xml. self.datatypes_registry.load_datatypes( self.config.root, self.config.datatypes_config ) # Determine the database url if self.config.database_connection: Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: dan: Add Picard Interval List to BED6 converter.
by Bitbucket 16 Dec '11

16 Dec '11

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/changeset/c6764f7a359c/ changeset: c6764f7a359c user: dan date: 2011-12-16 20:39:40 summary: Add Picard Interval List to BED6 converter. affected #: 3 files diff -r 11710de1b1fbf52ae3a9c860a7d36e76acf5e060 -r c6764f7a359cf26c81cfc14ff0e9e94e93b6c82a datatypes_conf.xml.sample --- a/datatypes_conf.xml.sample +++ b/datatypes_conf.xml.sample @@ -101,7 +101,9 @@ <display file="ensembl/ensembl_interval_as_bed.xml" inherit="True"/><display file="gbrowse/gbrowse_interval_as_bed.xml" inherit="True"/></datatype> - <datatype extension="picard_interval_list" type="galaxy.datatypes.data:Text" subclass="True" display_in_upload="True"/> + <datatype extension="picard_interval_list" type="galaxy.datatypes.tabular:Tabular" subclass="True" display_in_upload="True"> + <converter file="picard_interval_list_to_bed6_converter.xml" target_datatype="bed6"/> + </datatype><datatype extension="gatk_interval" type="galaxy.datatypes.data:Text" subclass="True" display_in_upload="True"/><datatype extension="gatk_dbsnp" type="galaxy.datatypes.tabular:Tabular" subclass="True" display_in_upload="True"/><datatype extension="gatk_tranche" type="galaxy.datatypes.tabular:Tabular" subclass="True" display_in_upload="True"/> diff -r 11710de1b1fbf52ae3a9c860a7d36e76acf5e060 -r c6764f7a359cf26c81cfc14ff0e9e94e93b6c82a lib/galaxy/datatypes/converters/picard_interval_list_to_bed6_converter.py --- /dev/null +++ b/lib/galaxy/datatypes/converters/picard_interval_list_to_bed6_converter.py @@ -0,0 +1,44 @@ +#!/usr/bin/env python +#Dan Blankenberg + +import sys + +assert sys.version_info[:2] >= ( 2, 5 ) +HEADER_STARTS_WITH = ( '@' ) + +def __main__(): + input_name = sys.argv[1] + output_name = sys.argv[2] + skipped_lines = 0 + first_skipped_line = 0 + header_lines = 0 + out = open( output_name, 'w' ) + i = 0 + for i, line in enumerate( open( input_name ) ): + complete_interval = False + line = line.rstrip( '\r\n' ) + if line: + if line.startswith( HEADER_STARTS_WITH ): + header_lines += 1 + else: + try: + elems = line.split( '\t' ) + if len( elems ) >= 5: + complete_interval = True + out.write( '%s\t%s\t%s\t%s\t0\t%s\n' % ( elems[0], int(elems[1])-1, elems[2], elems[4], elems[3] ) ) + except Exception, e: + print e + skipped_lines += 1 + if not first_skipped_line: + first_skipped_line = i + 1 + else: + skipped_lines += 1 + if not first_skipped_line: + first_skipped_line = i + 1 + out.close() + info_msg = "%i lines converted to BED. " % ( i + 1 - skipped_lines ) + if skipped_lines > 0: + info_msg += "Skipped %d blank/comment/invalid lines starting with line #%d." % ( skipped_lines, first_skipped_line ) + print info_msg + +if __name__ == "__main__": __main__() diff -r 11710de1b1fbf52ae3a9c860a7d36e76acf5e060 -r c6764f7a359cf26c81cfc14ff0e9e94e93b6c82a lib/galaxy/datatypes/converters/picard_interval_list_to_bed6_converter.xml --- /dev/null +++ b/lib/galaxy/datatypes/converters/picard_interval_list_to_bed6_converter.xml @@ -0,0 +1,12 @@ +<tool id="CONVERTER_picard_interval_list_to_bed6" name="Convert Picard Interval List to BED6" version="1.0.0"> + <description>converter</description> + <command interpreter="python">picard_interval_list_to_bed6_converter.py "$input" "$output"</command> + <inputs> + <param name="input" type="data" format="picard_interval_list" label="Picard Interval List file"/> + </inputs> + <outputs> + <data name="output" format="bed6"/> + </outputs> + <help> + </help> +</tool> Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: dan: Add Maximal Information-based Nonparametric Exploration (MINE) tool.
by Bitbucket 16 Dec '11

16 Dec '11

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/changeset/11710de1b1fb/ changeset: 11710de1b1fb user: dan date: 2011-12-16 18:54:21 summary: Add Maximal Information-based Nonparametric Exploration (MINE) tool. affected #: 3 files diff -r 42b4bb82e006c42f7f2049c2b8460af5e08ddc24 -r 11710de1b1fbf52ae3a9c860a7d36e76acf5e060 tool_conf.xml.sample --- a/tool_conf.xml.sample +++ b/tool_conf.xml.sample @@ -149,6 +149,7 @@ <tool file="stats/plot_from_lda.xml" /><tool file="regVariation/t_test_two_samples.xml" /><tool file="regVariation/compute_q_values.xml" /> + <tool file="stats/MINE.xml" /><label text="GFF" id="gff" /><tool file="stats/count_gff_features.xml" /> diff -r 42b4bb82e006c42f7f2049c2b8460af5e08ddc24 -r 11710de1b1fbf52ae3a9c860a7d36e76acf5e060 tools/stats/MINE.xml --- /dev/null +++ b/tools/stats/MINE.xml @@ -0,0 +1,82 @@ +<tool id="maximal_information_based_nonparametric_exploration" name="MINE" version="0.0.1"> + <description>- Maximal Information-based Nonparametric Exploration</description> + <requirements> + <requirement type="package" version="1.0">MINE</requirement> + </requirements> + <command interpreter="python">mine_wrapper.py + --jar "${GALAXY_DATA_INDEX_DIR}/shared/jars/mine/MINE.jar" + + --infile "${input_file}" + + #if str( $master_variable_type.master_variable_type_selector ) in [ 'allPairs', 'adjacentPairs' ]: + --master_variable "${master_variable_type.master_variable_type_selector}" + #else: + --master_variable "${master_variable_type.master_variable}" + #end if + + --cv "${cv}" + + --exp "${exp}" + + --c "${c}" + + ##--gc ##skip + + + #if str( $master_variable_type.master_variable_type_selector ) != 'allPairs' and $master_variable_type.permute: + --permute + #end if + + --output_results "${output_results}" + + --output_log "${output_log}" + </command> + <inputs> + <param name="input_file" type="data" format="csv" label="CSV file" /> + + <conditional name="master_variable_type"> + <param name="master_variable_type_selector" type="select" label="Choose the master variable type"> + <option value="allPairs">allPairs</option> + <option value="adjacentPairs">adjacentPairs</option> + <option value="compare_against_ith" selected="True">compare against i-th</option> + </param> + <when value="compare_against_ith"> + <param type="integer" value="0" name="master_variable" /> + <param type="boolean" truevalue="--permute" false_value="" name="permute" checked="False" /> + </when> + <when value="adjacentPairs"> + <param type="boolean" truevalue="--permute" false_value="" name="permute" checked="False" /> + </when> + </conditional> + + <param type="float" value="0" name="cv" /> + + <param type="float" value="0.6" name="exp" /> + + <param type="float" value="15" name="c" /> + + </inputs> + <outputs> + <data format="csv" name="output_results" label="${tool.name} on ${on_string} (Results)" /> + <data format="txt" name="output_log" label="${tool.name} on ${on_string} (log)" /> + </outputs> + <tests> +  + </tests> + <help> +**What it does** + +Applies the Maximal Information-based Nonparametric Exploration strategy to an input dataset. + +See http://www.exploredata.net/ for more information. + +------ + +**Citation** + +For the underlying tool, please cite `David N. Reshef, Yakir A. Reshef, Hilary K. Finucane5, Sharon R. Grossman, Gilean McVean, Peter J. Turnbaugh, Eric S. Lander, Michael Mitzenmacher, Pardis C. Sabeti Detecting Novel Associations in Large Data Sets. Science. 2011 Dec. <http://www.sciencemag.org/content/334/6062/1518>`_ + +If you use this tool in Galaxy, please cite Blankenberg D, et al. *In preparation.* + + </help> +</tool> diff -r 42b4bb82e006c42f7f2049c2b8460af5e08ddc24 -r 11710de1b1fbf52ae3a9c860a7d36e76acf5e060 tools/stats/mine_wrapper.py --- /dev/null +++ b/tools/stats/mine_wrapper.py @@ -0,0 +1,85 @@ +#!/usr/bin/env python +#Dan Blankenberg + +""" +A wrapper script for running the MINE.jar commands. +""" + +import sys, optparse, os, tempfile, subprocess, shutil + +CHUNK_SIZE = 2**20 #1mb + +BASE_NAME = "galaxy_mime_file.txt" +JOB_ID = "galaxy_mine" + +def cleanup_before_exit( tmp_dir ): + if tmp_dir and os.path.exists( tmp_dir ): + print os.listdir( tmp_dir ) + shutil.rmtree( tmp_dir ) + +def open_file_from_option( filename, mode = 'rb' ): + if filename: + return open( filename, mode = mode ) + return None + + +def __main__(): + #Parse Command Line + parser = optparse.OptionParser() + parser.add_option( '-j', '--jar', dest='jar', action='store', type="string", help='Location of JAR file' ) + parser.add_option( '-i', '--infile', dest='infile', action='store', type="string", help='infile' ) + parser.add_option( '-m', '--master_variable', dest='master_variable', action='store', type="string", help='master_variable' ) + parser.add_option( '-v', '--cv', dest='cv', action='store', type="string", help='cv' ) + parser.add_option( '-e', '--exp', dest='exp', action='store', type="string", help='exp' ) + parser.add_option( '-c', '--c', dest='c', action='store', type="string", help='c' ) + parser.add_option( '-p', '--permute', dest='permute', action='store_true', default=False, help='permute' ) + parser.add_option( '-o', '--output_results', dest='output_results', action='store', type="string", help='output_results' ) + parser.add_option( '-l', '--output_log', dest='output_log', action='store', type="string", help='output_log' ) + parser.add_option( '', '--stdout', dest='stdout', action='store', type="string", default=None, help='If specified, the output of stdout will be written to this file.' ) + parser.add_option( '', '--stderr', dest='stderr', action='store', type="string", default=None, help='If specified, the output of stderr will be written to this file.' ) + (options, args) = parser.parse_args() + + tmp_dir = tempfile.mkdtemp( prefix='tmp-MINE-' ) + tmp_input_name = os.path.join( tmp_dir, BASE_NAME ) + if options.permute: + permute = "-permute" + else: + permute = "" + + os.symlink( options.infile, tmp_input_name ) + + cmd = 'java -jar "%s" "%s" %s -cv%s -exp%s -c%s %s "%s"' % ( options.jar, tmp_input_name, options.master_variable, options.cv, options.exp, options.c, permute, JOB_ID ) + print cmd + + #set up stdout and stderr output options + stdout = open_file_from_option( options.stdout, mode = 'wb' ) + stderr = open_file_from_option( options.stderr, mode = 'wb' ) + #if no stderr file is specified, we'll use our own + if stderr is None: + stderr = tempfile.NamedTemporaryFile( prefix="MINE-stderr-", dir=tmp_dir ) + + proc = subprocess.Popen( args=cmd, stdout=stdout, stderr=stderr, shell=True, cwd=tmp_dir ) + return_code = proc.wait() + + if return_code: + stderr_target = sys.stderr + else: + stderr_target = sys.stdout + stderr.flush() + stderr.seek(0) + while True: + chunk = stderr.read( CHUNK_SIZE ) + if chunk: + stderr_target.write( chunk ) + else: + break + stderr.close() + + print os.listdir( tmp_dir ) + + shutil.move( '%s,%s,Results.csv' % ( tmp_input_name, JOB_ID ), options.output_results ) + shutil.move( '%s,%s,Status.csv' % ( tmp_input_name, JOB_ID ), options.output_log ) + + cleanup_before_exit( tmp_dir ) + +if __name__=="__main__": __main__() Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: greg: 1) Add a new InstalledRepositoryManager class which currently enables loading datatypes from previously installed tool shed repositories into the datatypes registry. This component will enable additional features in the future.
by Bitbucket 16 Dec '11

16 Dec '11

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/changeset/42b4bb82e006/ changeset: 42b4bb82e006 user: greg date: 2011-12-16 17:34:35 summary: 1) Add a new InstalledRepositoryManager class which currently enables loading datatypes from previously installed tool shed repositories into the datatypes registry. This component will enable additional features in the future. 2) Enhance the Iinstall_repository method in the admin_toolshed controller to skip displaying the page for selecting a tool panel section if no tools exist in any of the repositories being installed. 3) Eliminate duplicate generate_datatypes_metadata generate_tool_metadata generate_workflow_metadata methods and use those now contained in shed_util.py. 4) Fixes for handling tool shed repository metadata for repositories that do not include any tools. affected #: 10 files diff -r f37d9a0a1afc72945e1283195739fc31418dc934 -r 42b4bb82e006c42f7f2049c2b8460af5e08ddc24 lib/galaxy/app.py --- a/lib/galaxy/app.py +++ b/lib/galaxy/app.py @@ -22,7 +22,7 @@ self.config = config.Configuration( **kwargs ) self.config.check() config.configure_logging( self.config ) - # Set up datatypes registry + # Initialize the datatypes registry to the default data types included in self.config.datatypes_config. self.datatypes_registry = galaxy.datatypes.registry.Registry() self.datatypes_registry.load_datatypes( self.config.root, self.config.datatypes_config ) galaxy.model.set_datatypes_registry( self.datatypes_registry ) @@ -68,6 +68,10 @@ if self.config.get_bool( 'enable_tool_shed_check', False ): from tool_shed import update_manager self.update_manager = update_manager.UpdateManager( self ) + # Manage installed tool shed repositories + self.installed_repository_manager = galaxy.tool_shed.InstalledRepositoryManager( self ) + # Add additional datatypes from installed tool shed repositories to the datatypes registry. + self.installed_repository_manager.load_datatypes() # Load datatype converters self.datatypes_registry.load_datatype_converters( self.toolbox ) # Load history import/export tools diff -r f37d9a0a1afc72945e1283195739fc31418dc934 -r 42b4bb82e006c42f7f2049c2b8460af5e08ddc24 lib/galaxy/tool_shed/__init__.py --- a/lib/galaxy/tool_shed/__init__.py +++ b/lib/galaxy/tool_shed/__init__.py @@ -1,3 +1,22 @@ """ -Classes encapsulating the relationships between Galaxy and Galaxy tool sheds. -""" \ No newline at end of file +Classes encapsulating the management of repositories installed from Galaxy tool sheds. +""" +import os, logging +from galaxy.model.orm import * + +log = logging.getLogger(__name__) + +class InstalledRepositoryManager( object ): + def __init__( self, app ): + self.app = app + self.model = self.app.model + self.sa_session = self.model.context.current + def load_datatypes( self ): + for tool_shed_repository in self.sa_session.query( self.model.ToolShedRepository ) \ + .filter( and_( self.model.ToolShedRepository.table.c.includes_datatypes==True, + self.model.ToolShedRepository.table.c.deleted==False ) ): + metadata = tool_shed_repository.metadata + datatypes_config = metadata[ 'datatypes_config' ] + full_path = os.path.abspath( datatypes_config ) + self.app.datatypes_registry.load_datatypes( self.app.config.root, full_path ) + \ No newline at end of file diff -r f37d9a0a1afc72945e1283195739fc31418dc934 -r 42b4bb82e006c42f7f2049c2b8460af5e08ddc24 lib/galaxy/tool_shed/install_manager.py --- a/lib/galaxy/tool_shed/install_manager.py +++ b/lib/galaxy/tool_shed/install_manager.py @@ -158,4 +158,13 @@ if os.path.exists( clone_dir ): installed = True break + if not installed: + full_path = os.path.abspath( clone_dir ) + # We may have a repository that contains no tools. + if os.path.exists( full_path ): + for root, dirs, files in os.walk( full_path ): + if '.hg' in dirs: + # Assume that the repository has been installed if we find a .hg directory. + installed = True + break return installed diff -r f37d9a0a1afc72945e1283195739fc31418dc934 -r 42b4bb82e006c42f7f2049c2b8460af5e08ddc24 lib/galaxy/util/shed_util.py --- a/lib/galaxy/util/shed_util.py +++ b/lib/galaxy/util/shed_util.py @@ -109,6 +109,7 @@ Update the received metadata_dict with changes that have been applied to the received datatypes_config. This method is used by the InstallManager, which does not have access to trans. + TODO: Handle converters, indexers, sniffers, etc... """ # Parse datatypes_config. tree = ElementTree.parse( datatypes_config ) @@ -125,13 +126,29 @@ registration = root.find( 'registration' ) if registration: for elem in registration.findall( 'datatype' ): - extension = elem.get( 'extension', None ) + datatypes_dict = {} + display_in_upload = elem.get( 'display_in_upload', None ) + if display_in_upload: + datatypes_dict[ 'display_in_upload' ] = display_in_upload dtype = elem.get( 'type', None ) + if dtype: + datatypes_dict[ 'dtype' ] = dtype + extension = elem.get( 'extension', None ) + if extension: + datatypes_dict[ 'extension' ] = extension + max_optional_metadata_filesize = elem.get( 'max_optional_metadata_filesize', None ) + if max_optional_metadata_filesize: + datatypes_dict[ 'max_optional_metadata_filesize' ] = max_optional_metadata_filesize mimetype = elem.get( 'mimetype', None ) - datatypes.append( dict( extension=extension, - dtype=dtype, - mimetype=mimetype ) ) - metadata_dict[ 'datatypes' ] = datatypes + if mimetype: + datatypes_dict[ 'mimetype' ] = mimetype + subclass = elem.get( 'subclass', None ) + if subclass: + datatypes_dict[ 'subclass' ] = subclass + if datatypes_dict: + datatypes.append( datatypes_dict ) + if datatypes: + metadata_dict[ 'datatypes' ] = datatypes return metadata_dict def generate_metadata( toolbox, relative_install_dir, repository_clone_url ): """ @@ -426,46 +443,52 @@ # This method is used by the InstallManager, which does not have access to trans. imported_module = None # Parse datatypes_config. - tree = parse_xml( datatypes_config ) + tree = util.parse_xml( datatypes_config ) datatypes_config_root = tree.getroot() relative_path_to_datatype_file_name = None datatype_files = datatypes_config_root.find( 'datatype_files' ) - # Currently only a single datatype_file is supported. For example: - # <datatype_files> - # <datatype_file name="gmap.py"/> - # </datatype_files> - for elem in datatype_files.findall( 'datatype_file' ): - datatype_file_name = elem.get( 'name', None ) - if datatype_file_name: - # Find the file in the installed repository. - for root, dirs, files in os.walk( relative_intall_dir ): - if root.find( '.hg' ) < 0: - for name in files: - if name == datatype_file_name: - relative_path_to_datatype_file_name = os.path.join( root, name ) - break - break - if relative_path_to_datatype_file_name: - relative_head, relative_tail = os.path.split( relative_path_to_datatype_file_name ) - registration = datatypes_config_root.find( 'registration' ) - # Get the module by parsing the <datatype> tag. - for elem in registration.findall( 'datatype' ): - # A 'type' attribute is currently required. The attribute - # should be something like: type="gmap:GmapDB". - dtype = elem.get( 'type', None ) - if dtype: - fields = dtype.split( ':' ) - datatype_module = fields[0] - datatype_class_name = fields[1] - # Since we currently support only a single datatype_file, - # we have what we need. + if datatype_files: + # Currently only a single datatype_file is supported. For example: + # <datatype_files> + # <datatype_file name="gmap.py"/> + # </datatype_files> + for elem in datatype_files.findall( 'datatype_file' ): + datatype_file_name = elem.get( 'name', None ) + if datatype_file_name: + # Find the file in the installed repository. + for root, dirs, files in os.walk( relative_intall_dir ): + if root.find( '.hg' ) < 0: + for name in files: + if name == datatype_file_name: + relative_path_to_datatype_file_name = os.path.join( root, name ) + break break - try: - sys.path.insert( 0, relative_head ) - imported_module = __import__( datatype_module ) - sys.path.pop( 0 ) - except Exception, e: - log.debug( "Exception importing datatypes code file included in installed repository: %s" % str( e ) ) + if relative_path_to_datatype_file_name: + relative_head, relative_tail = os.path.split( relative_path_to_datatype_file_name ) + registration = datatypes_config_root.find( 'registration' ) + # Get the module by parsing the <datatype> tag. + for elem in registration.findall( 'datatype' ): + # A 'type' attribute is currently required. The attribute + # should be something like: type="gmap:GmapDB". + dtype = elem.get( 'type', None ) + if dtype: + fields = dtype.split( ':' ) + datatype_module = fields[0] + datatype_class_name = fields[1] + # Since we currently support only a single datatype_file, + # we have what we need. + break + try: + sys.path.insert( 0, relative_head ) + imported_module = __import__( datatype_module ) + sys.path.pop( 0 ) + except Exception, e: + log.debug( "Exception importing datatypes code file included in installed repository: %s" % str( e ) ) + else: + # The repository includes a datayptes_conf.xml file, but no code file that + # contains data type classes. This implies that the data types in datayptes_conf.xml + # are all subclasses of data types that are in the distribution. + imported_module = None app.datatypes_registry.load_datatypes( root_dir=app.config.root, config=datatypes_config, imported_module=imported_module ) def load_repository_contents( app, name, description, owner, changeset_revision, tool_path, repository_clone_url, relative_install_dir, current_working_dir, tmp_name, tool_section=None, shed_tool_conf=None, new_install=True ): diff -r f37d9a0a1afc72945e1283195739fc31418dc934 -r 42b4bb82e006c42f7f2049c2b8460af5e08ddc24 lib/galaxy/web/controllers/admin_toolshed.py --- a/lib/galaxy/web/controllers/admin_toolshed.py +++ b/lib/galaxy/web/controllers/admin_toolshed.py @@ -191,13 +191,20 @@ repo_info_dict = kwd[ 'repo_info_dict' ] new_tool_panel_section = kwd.get( 'new_tool_panel_section', '' ) tool_panel_section = kwd.get( 'tool_panel_section', '' ) - if kwd.get( 'select_tool_panel_section_button', False ): - shed_tool_conf = kwd[ 'shed_tool_conf' ] + includes_tools = util.string_as_bool( kwd.get( 'includes_tools', False ) ) + if not includes_tools or ( includes_tools and kwd.get( 'select_tool_panel_section_button', False ) ): + if includes_tools: + shed_tool_conf = kwd[ 'shed_tool_conf' ] + else: + # If installing a repository that includes no tools, get the relative + # tool_path from the file to which the install_tool_config_file config + # setting points. + shed_tool_conf = trans.app.config.install_tool_config # Get the tool path. for k, tool_path in trans.app.toolbox.shed_tool_confs.items(): if k == shed_tool_conf: break - if new_tool_panel_section or tool_panel_section: + if includes_tools and ( new_tool_panel_section or tool_panel_section ): if new_tool_panel_section: section_id = new_tool_panel_section.lower().replace( ' ', '_' ) new_section_key = 'section_%s' % str( section_id ) @@ -290,6 +297,7 @@ tool_shed_url=tool_shed_url, repo_info_dict=repo_info_dict, shed_tool_conf=shed_tool_conf, + includes_tools=includes_tools, shed_tool_conf_select_field=shed_tool_conf_select_field, tool_panel_section_select_field=tool_panel_section_select_field, new_tool_panel_section=new_tool_panel_section, diff -r f37d9a0a1afc72945e1283195739fc31418dc934 -r 42b4bb82e006c42f7f2049c2b8460af5e08ddc24 lib/galaxy/webapps/community/controllers/common.py --- a/lib/galaxy/webapps/community/controllers/common.py +++ b/lib/galaxy/webapps/community/controllers/common.py @@ -4,6 +4,7 @@ from galaxy.tools import * from galaxy.util.json import from_json_string, to_json_string from galaxy.util.hash_util import * +from galaxy.util.shed_util import generate_datatypes_metadata, generate_tool_metadata, generate_workflow_metadata from galaxy.web.base.controller import * from galaxy.webapps.community import model from galaxy.model.orm import * @@ -149,6 +150,7 @@ .order_by( trans.model.RepositoryMetadata.table.c.id.desc() ) \ .first() def generate_clone_url( trans, repository_id ): + """Generate the URL for cloning a repository.""" repository = get_repository( trans, repository_id ) protocol, base = trans.request.base.split( '://' ) if trans.user: @@ -220,54 +222,6 @@ correction_msg += "Upload a file named <b>%s.sample</b> to the repository to correct this error." % str( index_tail ) invalid_files.append( ( name, correction_msg ) ) return can_set_metadata, invalid_files -def generate_tool_metadata( trans, id, changeset_revision, tool_config, tool, metadata_dict ): - """ - Update the received metadata_dict with changes that have been - applied to the received tool. - """ - repository = get_repository( trans, id ) - # Handle tool.requirements. - tool_requirements = [] - for tr in tool.requirements: - name=tr.name - type=tr.type - if type == 'fabfile': - version = None - fabfile = tr.fabfile - method = tr.method - else: - version = tr.version - fabfile = None - method = None - requirement_dict = dict( name=name, - type=type, - version=version, - fabfile=fabfile, - method=method ) - tool_requirements.append( requirement_dict ) - # Handle tool.tests. - tool_tests = [] - if tool.tests: - for ttb in tool.tests: - test_dict = dict( name=ttb.name, - required_files=ttb.required_files, - inputs=ttb.inputs, - outputs=ttb.outputs ) - tool_tests.append( test_dict ) - tool_dict = dict( id=tool.id, - guid = generate_tool_guid( trans, repository, tool ), - name=tool.name, - version=tool.version, - description=tool.description, - version_string_cmd = tool.version_string_cmd, - tool_config=tool_config, - requirements=tool_requirements, - tests=tool_tests ) - if 'tools' in metadata_dict: - metadata_dict[ 'tools' ].append( tool_dict ) - else: - metadata_dict[ 'tools' ] = [ tool_dict ] - return metadata_dict def new_tool_metadata_required( trans, id, metadata_dict ): """ Compare the last saved metadata for each tool in the repository with the new metadata @@ -309,16 +263,6 @@ # The received metadata_dict includes no metadata for tools, so a new repository_metadata table # record is not needed. return False -def generate_workflow_metadata( trans, id, changeset_revision, exported_workflow_dict, metadata_dict ): - """ - Update the received metadata_dict with changes that have been applied - to the received exported_workflow_dict. Store everything in the database. - """ - if 'workflows' in metadata_dict: - metadata_dict[ 'workflows' ].append( exported_workflow_dict ) - else: - metadata_dict[ 'workflows' ] = [ exported_workflow_dict ] - return metadata_dict def new_workflow_metadata_required( trans, id, metadata_dict ): """ Currently everything about an exported workflow except the name is hard-coded, so there's @@ -337,34 +281,6 @@ # The received metadata_dict includes no metadata for workflows, so a new repository_metadata table # record is not needed. return False -def generate_datatypes_metadata( trans, id, changeset_revision, datatypes_config, metadata_dict ): - """ - Update the received metadata_dict with changes that have been applied - to the received datatypes_config. - """ - # Parse datatypes_config. - tree = ElementTree.parse( datatypes_config ) - root = tree.getroot() - ElementInclude.include( root ) - repository_datatype_code_files = [] - datatype_files = root.find( 'datatype_files' ) - if datatype_files: - for elem in datatype_files.findall( 'datatype_file' ): - name = elem.get( 'name', None ) - repository_datatype_code_files.append( name ) - metadata_dict[ 'datatype_files' ] = repository_datatype_code_files - datatypes = [] - registration = root.find( 'registration' ) - if registration: - for elem in registration.findall( 'datatype' ): - extension = elem.get( 'extension', None ) - dtype = elem.get( 'type', None ) - mimetype = elem.get( 'mimetype', None ) - datatypes.append( dict( extension=extension, - dtype=dtype, - mimetype=mimetype ) ) - metadata_dict[ 'datatypes' ] = datatypes - return metadata_dict def generate_metadata_for_repository_tip( trans, id, ctx, changeset_revision, repo_dir ): # Browse the repository tip files on disk to generate metadata. This is faster than # the generate_metadata_for_changeset_revision() method below because fctx.data() does @@ -382,7 +298,7 @@ datatypes_config = os.path.abspath( os.path.join( root, name ) ) break if datatypes_config: - metadata_dict = generate_datatypes_metadata( trans, id, changeset_revision, datatypes_config, metadata_dict ) + metadata_dict = generate_datatypes_metadata( datatypes_config, metadata_dict ) # Find all special .sample files. for root, dirs, files in os.walk( repo_dir ): if root.find( '.hg' ) < 0: @@ -409,19 +325,19 @@ if can_set_metadata: # Update the list of metadata dictionaries for tools in metadata_dict. tool_config = os.path.join( root, name ) - metadata_dict = generate_tool_metadata( trans, id, changeset_revision, tool_config, tool, metadata_dict ) + repository_clone_url = generate_clone_url( trans, id ) + metadata_dict = generate_tool_metadata( tool_config, tool, repository_clone_url, metadata_dict ) # Find all exported workflows elif name.endswith( '.ga' ): try: - full_path = os.path.abspath( os.path.join( root, name ) ) + relative_path = os.path.join( root, name ) # Convert workflow data from json - fp = open( full_path, 'rb' ) + fp = open( relative_path, 'rb' ) workflow_text = fp.read() fp.close() exported_workflow_dict = from_json_string( workflow_text ) if 'a_galaxy_workflow' in exported_workflow_dict and exported_workflow_dict[ 'a_galaxy_workflow' ] == 'true': - # Update the list of metadata dictionaries for workflows in metadata_dict. - metadata_dict = generate_workflow_metadata( trans, id, changeset_revision, exported_workflow_dict, metadata_dict ) + metadata_dict = generate_workflow_metadata( relative_path, exported_workflow_dict, metadata_dict ) except Exception, e: invalid_files.append( ( name, str( e ) ) ) return metadata_dict, invalid_files @@ -438,7 +354,7 @@ datatypes_config = fctx.data() break if datatypes_config: - metadata_dict = generate_datatypes_metadata( trans, id, changeset_revision, datatypes_config, metadata_dict ) + metadata_dict = generate_datatypes_metadata( datatypes_config, metadata_dict ) # Get all tool config file names from the hgweb url, something like: # /repos/test/convert_chars1/file/e58dcf0026c7/convert_characters.xml for filename in ctx: @@ -469,7 +385,8 @@ # anything, but may result in a bit of confusion when maintaining the code / data over time. # IMPORTANT NOTE: Here we are assuming that since the current change set is not the repository # tip, we do not have to handle any .loc.sample files since they would have been handled previously. - metadata_dict = generate_tool_metadata( trans, id, changeset_revision, filename, tool, metadata_dict ) + repository_clone_url = generate_clone_url( trans, id ) + metadata_dict = generate_tool_metadata( filename, tool, repository_clone_url, metadata_dict ) try: os.unlink( tmp_filename ) except: @@ -481,8 +398,7 @@ workflow_text = fctx.data() exported_workflow_dict = from_json_string( workflow_text ) if 'a_galaxy_workflow' in exported_workflow_dict and exported_workflow_dict[ 'a_galaxy_workflow' ] == 'true': - # Update the list of metadata dictionaries for workflows in metadata_dict. - metadata_dict = generate_workflow_metadata( trans, id, changeset_revision, exported_workflow_dict, metadata_dict ) + metadata_dict = generate_workflow_metadata( '', exported_workflow_dict, metadata_dict ) except Exception, e: invalid_files.append( ( name, str( e ) ) ) return metadata_dict, invalid_files @@ -510,12 +426,18 @@ if len( repository.downloadable_revisions ) == 1: handle_email_alerts( trans, repository, content_alert_str='', new_repo_alert=True, admin_only=False ) else: - # Update the last saved repository_metadata table row. repository_metadata = get_latest_repository_metadata( trans, id ) - repository_metadata.changeset_revision = changeset_revision - repository_metadata.metadata = metadata_dict - trans.sa_session.add( repository_metadata ) - trans.sa_session.flush() + if repository_metadata: + # Update the last saved repository_metadata table row. + repository_metadata.changeset_revision = changeset_revision + repository_metadata.metadata = metadata_dict + trans.sa_session.add( repository_metadata ) + trans.sa_session.flush() + else: + # There are no tools in the repository, and we're setting metadat on the repository tip. + repository_metadata = trans.model.RepositoryMetadata( repository.id, changeset_revision, metadata_dict ) + trans.sa_session.add( repository_metadata ) + trans.sa_session.flush() else: # We're re-generating metadata for an old repository revision. repository_metadata = get_repository_metadata_by_changeset_revision( trans, id, changeset_revision ) diff -r f37d9a0a1afc72945e1283195739fc31418dc934 -r 42b4bb82e006c42f7f2049c2b8460af5e08ddc24 lib/galaxy/webapps/community/controllers/repository.py --- a/lib/galaxy/webapps/community/controllers/repository.py +++ b/lib/galaxy/webapps/community/controllers/repository.py @@ -436,9 +436,9 @@ **kwd ) ) if operation == "install": galaxy_url = trans.get_cookie( name='toolshedgalaxyurl' ) - encoded_repo_info_dict = self.__encode_repo_info_dict( trans, webapp, util.listify( item_id ) ) - url = '%s/admin_toolshed/install_repository?tool_shed_url=%s&webapp=%s&repo_info_dict=%s' % \ - ( galaxy_url, url_for( '', qualified=True ), webapp, encoded_repo_info_dict ) + encoded_repo_info_dict, includes_tools = self.__encode_repo_info_dict( trans, webapp, util.listify( item_id ) ) + url = '%s/admin_toolshed/install_repository?tool_shed_url=%s&webapp=%s&repo_info_dict=%s&includes_tools=%s' % \ + ( galaxy_url, url_for( '', qualified=True ), webapp, encoded_repo_info_dict, str( includes_tools ) ) return trans.response.send_redirect( url ) else: # This can only occur when there is a multi-select grid with check boxes and an operation, @@ -512,9 +512,9 @@ **kwd ) ) if operation == "install": galaxy_url = trans.get_cookie( name='toolshedgalaxyurl' ) - encoded_repo_info_dict = self.__encode_repo_info_dict( trans, webapp, util.listify( item_id ) ) - url = '%s/admin_toolshed/install_repository?tool_shed_url=%s&webapp=%s&repo_info_dict=%s' % \ - ( galaxy_url, url_for( '', qualified=True ), webapp, encoded_repo_info_dict ) + encoded_repo_info_dict, includes_tools = self.__encode_repo_info_dict( trans, webapp, util.listify( item_id ) ) + url = '%s/admin_toolshed/install_repository?tool_shed_url=%s&webapp=%s&repo_info_dict=%s&includes_tools=%s' % \ + ( galaxy_url, url_for( '', qualified=True ), webapp, encoded_repo_info_dict, str( includes_tools ) ) return trans.response.send_redirect( url ) else: # This can only occur when there is a multi-select grid with check boxes and an operation, @@ -707,14 +707,17 @@ return match_tuples def __encode_repo_info_dict( self, trans, webapp, repository_metadata_ids ): repo_info_dict = {} + includes_tools = False for repository_metadata_id in repository_metadata_ids: repository_metadata = get_repository_metadata_by_id( trans, repository_metadata_id ) + if not includes_tools and 'tools' in repository_metadata.metadata: + includes_tools = True repository = get_repository( trans, trans.security.encode_id( repository_metadata.repository_id ) ) repository_id = trans.security.encode_id( repository.id ) changeset_revision = repository_metadata.changeset_revision repository_clone_url = generate_clone_url( trans, repository_id ) repo_info_dict[ repository.name ] = ( repository.description, repository_clone_url, changeset_revision ) - return encode( repo_info_dict ) + return encode( repo_info_dict ), includes_tools @web.expose def preview_tools_in_changeset( self, trans, repository_id, **kwd ): params = util.Params( kwd ) @@ -755,12 +758,16 @@ repository_clone_url = generate_clone_url( trans, repository_id ) repository = get_repository( trans, repository_id ) changeset_revision = util.restore_text( params.get( 'changeset_revision', repository.tip ) ) + repository_metadata = get_repository_metadata_by_changeset_revision( trans, repository_id, changeset_revision ) + # Tell the caller if the repository includes Galaxy tools so the page + # enabling selection of the tool panel section can be displayed. + includes_tools = 'tools' in repository_metadata.metadata repo_info_dict = {} repo_info_dict[ repository.name ] = ( repository.description, repository_clone_url, changeset_revision ) encoded_repo_info_dict = encode( repo_info_dict ) # Redirect back to local Galaxy to perform install. - url = '%s/admin_toolshed/install_repository?tool_shed_url=%s&repo_info_dict=%s' % \ - ( galaxy_url, url_for( '', qualified=True ), encoded_repo_info_dict ) + url = '%s/admin_toolshed/install_repository?tool_shed_url=%s&repo_info_dict=%s&includes_tools=%s' % \ + ( galaxy_url, url_for( '', qualified=True ), encoded_repo_info_dict, str( includes_tools ) ) return trans.response.send_redirect( url ) @web.expose def check_for_updates( self, trans, **kwd ): diff -r f37d9a0a1afc72945e1283195739fc31418dc934 -r 42b4bb82e006c42f7f2049c2b8460af5e08ddc24 templates/admin/tool_shed_repository/common.mako --- a/templates/admin/tool_shed_repository/common.mako +++ b/templates/admin/tool_shed_repository/common.mako @@ -128,25 +128,22 @@ <table class="grid"><tr><td><b>extension</b></td> - <td><b>dtype</b></td> + <td><b>type</b></td><td><b>mimetype</b></td> + <td><b>subclass</b></td></tr> %for datatypes_dict in datatypes_dicts: - <% - extension = datatypes_dict[ 'extension' ] - dtype = datatypes_dict[ 'dtype' ] - mimetype = datatypes_dict[ 'mimetype' ] + <% + extension = datatypes_dict.get( 'extension', ' ' ) + dtype = datatypes_dict.get( 'dtype', ' ' ) + mimetype = datatypes_dict.get( 'mimetype', ' ' ) + subclass = datatypes_dict.get( 'subclass', ' ' ) %><tr><td>${extension}</td><td>${dtype}</td> - <td> - %if mimetype: - ${mimetype} - %else: -   - %endif - </td> + <td>${mimetype}</td> + <td>${subclass}</td></tr> %endfor </table> diff -r f37d9a0a1afc72945e1283195739fc31418dc934 -r 42b4bb82e006c42f7f2049c2b8460af5e08ddc24 templates/admin/tool_shed_repository/select_tool_panel_section.mako --- a/templates/admin/tool_shed_repository/select_tool_panel_section.mako +++ b/templates/admin/tool_shed_repository/select_tool_panel_section.mako @@ -25,7 +25,7 @@ <div class="toolForm"><div class="toolFormTitle">Choose tool panel section to contain installed tools (optional)</div><div class="toolFormBody"> - <form name="select_tool_panel_section" id="select_tool_panel_section" action="${h.url_for( controller='admin_toolshed', action='install_repository', tool_shed_url=tool_shed_url, repo_info_dict=repo_info_dict )}" method="post" > + <form name="select_tool_panel_section" id="select_tool_panel_section" action="${h.url_for( controller='admin_toolshed', action='install_repository', tool_shed_url=tool_shed_url, repo_info_dict=repo_info_dict, includes_tools=includes_tools )}" method="post" > %if shed_tool_conf_select_field: <div class="form-row"><label>Shed tool configuration file:</label> diff -r f37d9a0a1afc72945e1283195739fc31418dc934 -r 42b4bb82e006c42f7f2049c2b8460af5e08ddc24 templates/webapps/community/repository/common.mako --- a/templates/webapps/community/repository/common.mako +++ b/templates/webapps/community/repository/common.mako @@ -209,25 +209,22 @@ <table class="grid"><tr><td><b>extension</b></td> - <td><b>dtype</b></td> + <td><b>type</b></td><td><b>mimetype</b></td> + <td><b>subclass</b></td></tr> %for datatypes_dict in datatypes_dicts: - <% - extension = datatypes_dict[ 'extension' ] - dtype = datatypes_dict[ 'dtype' ] - mimetype = datatypes_dict[ 'mimetype' ] + <% + extension = datatypes_dict.get( 'extension', ' ' ) + dtype = datatypes_dict.get( 'dtype', ' ' ) + mimetype = datatypes_dict.get( 'mimetype', ' ' ) + subclass = datatypes_dict.get( 'subclass', ' ' ) %><tr><td>${extension}</td><td>${dtype}</td> - <td> - %if mimetype: - ${mimetype} - %else: -   - %endif - </td> + <td>${mimetype}</td> + <td>${subclass}</td></tr> %endfor </table> Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0