- galaxy-commits - lists.galaxyproject.org

commit/galaxy-central: greg: Enhance the process of deactivating / activating installed tool shed repositories by handling the deactivation / reactivation of proprietary datatypes, datatype converters and display applications.
by Bitbucket 17 Jan '12

17 Jan '12

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/changeset/1a4be690f503/ changeset: 1a4be690f503 user: greg date: 2012-01-17 16:54:15 summary: Enhance the process of deactivating / activating installed tool shed repositories by handling the deactivation / reactivation of proprietary datatypes, datatype converters and display applications. affected #: 5 files diff -r e3f4aca57f838e3b6fb6dc114a25ea62b5f34317 -r 1a4be690f5038e78a6f5abe21b9552293b911a53 lib/galaxy/datatypes/registry.py --- a/lib/galaxy/datatypes/registry.py +++ b/lib/galaxy/datatypes/registry.py @@ -48,13 +48,22 @@ self.datatype_elems = [] self.sniffer_elems = [] self.xml_filename = None - def load_datatypes( self, root_dir=None, config=None, imported_modules=None ): + def load_datatypes( self, root_dir=None, config=None, imported_modules=None, deactivate=False ): + """ + Parse a datatypes XML file located at root_dir/config. If imported_modules is received, it + is a list of imported datatypes class files included in an installed tool shed repository. + If deactivate is received as True, an installed tool shed repository that includes proprietary + datatypes is being deactivated, so relevant loaded datatypes will be removed from the registry. + """ if root_dir and config: # Parse datatypes_conf.xml tree = galaxy.util.parse_xml( config ) root = tree.getroot() # Load datatypes and converters from config - self.log.debug( 'Loading datatypes from %s' % config ) + if deactivate: + self.log.debug( 'Deactivating datatypes from %s' % config ) + else: + self.log.debug( 'Loading datatypes from %s' % config ) registration = root.find( 'registration' ) # Set default paths defined in local datatypes_conf.xml. if not self.converters_path: @@ -66,8 +75,6 @@ self.display_path_attr = registration.get( 'display_path', 'display_applications' ) self.display_applications_path = os.path.join( root_dir, self.display_path_attr ) for elem in registration.findall( 'datatype' ): - # Keep an in-memory list of datatype elems to enable persistence. - self.datatype_elems.append( elem ) try: extension = elem.get( 'extension', None ) dtype = elem.get( 'type', None ) @@ -75,9 +82,30 @@ mimetype = elem.get( 'mimetype', None ) display_in_upload = elem.get( 'display_in_upload', False ) make_subclass = galaxy.util.string_as_bool( elem.get( 'subclass', False ) ) + if deactivate: + # We are deactivating an installed tool shed repository, so eliminate the + # datatype elem from the in-memory list of datatype elems. + for in_memory_elem in self.datatype_elems: + in_memory_extension = in_memory_elem.get( 'extension', None ) + if in_memory_extension == extension: + in_memory_dtype = elem.get( 'type', None ) + in_memory_type_extension = elem.get( 'type_extension', None ) + in_memory_mimetype = elem.get( 'mimetype', None ) + in_memory_display_in_upload = elem.get( 'display_in_upload', False ) + in_memory_make_subclass = galaxy.util.string_as_bool( elem.get( 'subclass', False ) ) + if in_memory_dtype == dtype and in_memory_type_extension == type_extension and in_memory_mimetype == mimetype \ + and in_memory_display_in_upload == display_in_upload and in_memory_make_subclass == make_subclass: + self.datatype_elems.remove( in_memory_elem ) + else: + # Keep an in-memory list of datatype elems to enable persistence. + self.datatype_elems.append( elem ) if extension and extension in self.datatypes_by_extension: - self.log.debug( "Ignoring datatype with extension '%s' from '%s' because the registry already includes a datatype with that extension." \ - % ( extension, config ) ) + if deactivate: + # We are deactivating an installed tool shed repository, so eliminate the datatype from the registry. + self.log.debug( "Removing datatype with extension '%s' from the registry." % extension ) + del self.datatypes_by_extension[ extension ] + else: + self.log.debug( "Ignoring datatype with extension '%s' from '%s' because the registry already contains a datatype with that extension." % ( extension, config ) ) elif extension and ( dtype or type_extension ): if dtype: fields = dtype.split( ':' ) @@ -144,7 +172,10 @@ if elem not in self.display_app_containers: self.display_app_containers.append( elem ) except Exception, e: - self.log.warning( 'Error loading datatype "%s", problem: %s' % ( extension, str( e ) ) ) + if deactivate: + self.log.warning( 'Error deactivating datatype "%s": %s' % ( extension, str( e ) ) ) + else: + self.log.warning( 'Error loading datatype "%s": %s' % ( extension, str( e ) ) ) # Load datatype sniffers from the config sniffers = root.find( 'sniffers' ) if sniffers: @@ -158,13 +189,20 @@ datatype_module = fields[0] datatype_class = fields[1] module = __import__( datatype_module ) - for comp in datatype_module.split('.')[1:]: - module = getattr(module, comp) - aclass = getattr( module, datatype_class )() - self.sniff_order.append( aclass ) - self.log.debug( 'Loaded sniffer for datatype: %s' % dtype ) + for comp in datatype_module.split( '.' )[ 1: ]: + module = getattr( module, comp ) + aclass = getattr( module, datatype_class )() + if deactivate: + self.sniff_order.remove( aclass ) + self.log.debug( 'Deactivated sniffer for datatype: %s' % dtype ) + else: + self.sniff_order.append( aclass ) + self.log.debug( 'Loaded sniffer for datatype: %s' % dtype ) except Exception, exc: - self.log.warning( 'Error appending datatype %s to sniff_order, problem: %s' % ( dtype, str( exc ) ) ) + if deactivate: + self.log.warning( 'Error deactivating sniffer for datatype %s: %s' % ( dtype, str( exc ) ) ) + else: + self.log.warning( 'Error appending sniffer for datatype %s to sniff_order: %s' % ( dtype, str( exc ) ) ) # Persist the xml form of the registry into a temporary file so that it # can be loaded from the command line by tools and set_metadata processing. self.to_xml_file() @@ -316,9 +354,13 @@ setattr(newdata, key, value) newdata.ext = ext return newdata - def load_datatype_converters( self, toolbox, converter_path=None ): - """Adds datatype converters from self.converters to the calling app's toolbox""" - if converter_path: + def load_datatype_converters( self, toolbox, installed_repository_dict=None, deactivate=False ): + """ + If deactivate is False, add datatype converters from self.converters or self.proprietary_converters + to the calling app's toolbox. If deactivate is True, eliminates relevant converters from the calling + app's toolbox. + """ + if installed_repository_dict: # Load converters defined by datatypes_conf.xml # included in installed tool shed repository. converters = self.proprietary_converters @@ -329,21 +371,51 @@ tool_config = elem[0] source_datatype = elem[1] target_datatype = elem[2] - if converter_path: + if installed_repository_dict: + converter_path = installed_repository_dict[ 'converter_path' ] config_path = os.path.join( converter_path, tool_config ) else: config_path = os.path.join( self.converters_path, tool_config ) try: converter = toolbox.load_tool( config_path ) - toolbox.tools_by_id[ converter.id ] = converter - if source_datatype not in self.datatype_converters: - self.datatype_converters[ source_datatype ] = odict() - self.datatype_converters[ source_datatype ][ target_datatype ] = converter - self.log.debug( "Loaded converter: %s", converter.id ) + if installed_repository_dict: + # If the converter is included in an installed tool shed repository, set the tool + # shed related tool attributes. + converter.tool_shed = installed_repository_dict[ 'tool_shed' ] + converter.repository_name = installed_repository_dict[ 'repository_name' ] + converter.repository_owner = installed_repository_dict[ 'repository_owner' ] + converter.installed_changeset_revision = installed_repository_dict[ 'installed_changeset_revision' ] + converter.old_id = converter.id + # The converter should be included in the list of tools defined in tool_dicts. + tool_dicts = installed_repository_dict[ 'tool_dicts' ] + for tool_dict in tool_dicts: + if tool_dict[ 'id' ] == converter.id: + converter.guid = tool_dict[ 'guid' ] + converter.id = tool_dict[ 'guid' ] + break + if deactivate: + del toolbox.tools_by_id[ converter.id ] + if source_datatype in self.datatype_converters: + del self.datatype_converters[ source_datatype ][ target_datatype ] + self.log.debug( "Deactivated converter: %s", converter.id ) + else: + toolbox.tools_by_id[ converter.id ] = converter + if source_datatype not in self.datatype_converters: + self.datatype_converters[ source_datatype ] = odict() + self.datatype_converters[ source_datatype ][ target_datatype ] = converter + self.log.debug( "Loaded converter: %s", converter.id ) except Exception, e: - self.log.exception( "Error loading converter (%s): %s" % ( config_path, str( e ) ) ) - def load_display_applications( self, display_path=None ): - if display_path: + if deactivate: + self.log.exception( "Error deactivating converter (%s): %s" % ( config_path, str( e ) ) ) + else: + self.log.exception( "Error loading converter (%s): %s" % ( config_path, str( e ) ) ) + def load_display_applications( self, installed_repository_dict=None, deactivate=False ): + """ + If deactivate is False, add display applications from self.display_app_containers or + self.proprietary_display_app_containers to appropriate datatypes. If deactivate is + True, eliminates relevant display applications from appropriate datatypes. + """ + if installed_repository_dict: # Load display applications defined by datatypes_conf.xml # included in installed tool shed repository. datatype_elems = self.proprietary_display_app_containers @@ -354,7 +426,8 @@ extension = elem.get( 'extension', None ) for display_app in elem.findall( 'display' ): display_file = display_app.get( 'file', None ) - if display_path: + if installed_repository_dict: + display_path = installed_repository_dict[ 'display_path' ] config_path = os.path.join( display_path, display_file ) else: config_path = os.path.join( self.display_applications_path, display_file ) @@ -363,15 +436,43 @@ display_app = DisplayApplication.from_file( config_path, self ) if display_app: if display_app.id in self.display_applications: - # If we already loaded this display application, we'll use the first one loaded. - display_app = self.display_applications[ display_app.id ] - self.log.debug( "Loaded display application '%s' for datatype '%s', inherit=%s" % ( display_app.id, extension, inherit ) ) - self.display_applications[ display_app.id ] = display_app - self.datatypes_by_extension[ extension ].add_display_application( display_app ) - if inherit and ( self.datatypes_by_extension[ extension ], display_app ) not in self.inherit_display_application_by_class: - self.inherit_display_application_by_class.append( ( self.datatypes_by_extension[extension], display_app ) ) + if deactivate: + del self.display_applications[ display_app.id ] + else: + # If we already loaded this display application, we'll use the first one loaded. + display_app = self.display_applications[ display_app.id ] + elif installed_repository_dict: + # If the display application is included in an installed tool shed repository, + # set the tool shed related tool attributes. + display_app.tool_shed = installed_repository_dict[ 'tool_shed' ] + display_app.repository_name = installed_repository_dict[ 'repository_name' ] + display_app.repository_owner = installed_repository_dict[ 'repository_owner' ] + display_app.installed_changeset_revision = installed_repository_dict[ 'installed_changeset_revision' ] + display_app.old_id = display_app.id + # The converter should be included in the list of tools defined in tool_dicts. + tool_dicts = installed_repository_dict[ 'tool_dicts' ] + for tool_dict in tool_dicts: + if tool_dict[ 'id' ] == display_app.id: + display_app.guid = tool_dict[ 'guid' ] + display_app.id = tool_dict[ 'guid' ] + break + if deactivate: + del self.display_applications[ display_app.id ] + del self.datatypes_by_extension[ extension ].display_applications[ display_app.id ] + if inherit and ( self.datatypes_by_extension[ extension ], display_app ) in self.inherit_display_application_by_class: + self.inherit_display_application_by_class.remove( ( self.datatypes_by_extension[ extension ], display_app ) ) + self.log.debug( "Deactivated display application '%s' for datatype '%s'." % ( display_app.id, extension ) ) + else: + self.display_applications[ display_app.id ] = display_app + self.datatypes_by_extension[ extension ].add_display_application( display_app ) + if inherit and ( self.datatypes_by_extension[ extension ], display_app ) not in self.inherit_display_application_by_class: + self.inherit_display_application_by_class.append( ( self.datatypes_by_extension[ extension ], display_app ) ) + self.log.debug( "Loaded display application '%s' for datatype '%s', inherit=%s." % ( display_app.id, extension, inherit ) ) except Exception, e: - self.log.exception( "Error loading display application (%s): %s" % ( config_path, str( e ) ) ) + if deactivate: + self.log.exception( "Error deactivating display application (%s): %s" % ( config_path, str( e ) ) ) + else: + self.log.exception( "Error loading display application (%s): %s" % ( config_path, str( e ) ) ) # Handle display_application subclass inheritance. for extension, d_type1 in self.datatypes_by_extension.iteritems(): for d_type2, display_app in self.inherit_display_application_by_class: @@ -381,9 +482,9 @@ d_type1.add_display_application( display_app ) def load_external_metadata_tool( self, toolbox ): """Adds a tool which is used to set external metadata""" - #we need to be able to add a job to the queue to set metadata. The queue will currently only accept jobs with an associated tool. - #We'll create a special tool to be used for Auto-Detecting metadata; this is less than ideal, but effective - #Properly building a tool without relying on parsing an XML file is near impossible...so we'll create a temporary file + # We need to be able to add a job to the queue to set metadata. The queue will currently only accept jobs with an associated + # tool. We'll create a special tool to be used for Auto-Detecting metadata; this is less than ideal, but effective + # Properly building a tool without relying on parsing an XML file is near impossible...so we'll create a temporary file tool_xml_text = """ <tool id="__SET_METADATA__" name="Set External Metadata" version="1.0.1" tool_type="set_metadata"><type class="SetMetadataTool" module="galaxy.tools"/> diff -r e3f4aca57f838e3b6fb6dc114a25ea62b5f34317 -r 1a4be690f5038e78a6f5abe21b9552293b911a53 lib/galaxy/tool_shed/__init__.py --- a/lib/galaxy/tool_shed/__init__.py +++ b/lib/galaxy/tool_shed/__init__.py @@ -2,7 +2,7 @@ Classes encapsulating the management of repositories installed from Galaxy tool sheds. """ import os, logging -from galaxy.util.shed_util import load_datatypes +from galaxy.util.shed_util import create_repository_dict_for_proprietary_datatypes, load_datatype_items from galaxy.model.orm import * log = logging.getLogger(__name__) @@ -23,11 +23,5 @@ path_items = datatypes_config.split( 'repos' ) relative_install_dir = '%srepos/%s/%s/%s' % \ ( path_items[0], tool_shed_repository.owner, tool_shed_repository.name, tool_shed_repository.installed_changeset_revision ) - converter_path, display_path = load_datatypes( self.app, datatypes_config, relative_install_dir ) - if converter_path: - # Load proprietary datatype converters - self.app.datatypes_registry.load_datatype_converters( self.app.toolbox, converter_path=converter_path ) - if display_path: - # Load proprietary datatype display applications - app.datatypes_registry.load_display_applications( display_path=display_path ) + load_datatype_items( self.app, tool_shed_repository, relative_install_dir ) \ No newline at end of file diff -r e3f4aca57f838e3b6fb6dc114a25ea62b5f34317 -r 1a4be690f5038e78a6f5abe21b9552293b911a53 lib/galaxy/tool_shed/install_manager.py --- a/lib/galaxy/tool_shed/install_manager.py +++ b/lib/galaxy/tool_shed/install_manager.py @@ -84,6 +84,7 @@ relative_install_dir=relative_install_dir, current_working_dir=current_working_dir, tmp_name=tmp_name, + tool_shed=self.tool_shed, tool_section=tool_section, shed_tool_conf=self.install_tool_config, new_install=True ) diff -r e3f4aca57f838e3b6fb6dc114a25ea62b5f34317 -r 1a4be690f5038e78a6f5abe21b9552293b911a53 lib/galaxy/util/shed_util.py --- a/lib/galaxy/util/shed_util.py +++ b/lib/galaxy/util/shed_util.py @@ -88,6 +88,14 @@ # overwrite it in case it contains stuff proprietary to the local instance. if not os.path.exists( os.path.join( tool_data_path, loc_file ) ): shutil.copy( os.path.abspath( filename ), os.path.join( tool_data_path, loc_file ) ) +def create_repository_dict_for_proprietary_datatypes( tool_shed, name, owner, installed_changeset_revision, tool_dicts, converter_path=None, display_path=None ): + return dict( tool_shed=tool_shed, + repository_name=name, + repository_owner=owner, + installed_changeset_revision=installed_changeset_revision, + tool_dicts=tool_dicts, + converter_path=converter_path, + display_path=display_path ) def create_or_update_tool_shed_repository( app, name, description, changeset_revision, repository_clone_url, metadata_dict, owner='' ): # This method is used by the InstallManager, which does not have access to trans. sa_session = app.model.context.current @@ -464,7 +472,28 @@ error = tmp_stderr.read() tmp_stderr.close() log.debug( 'Problem installing dependencies for tool "%s"\n%s' % ( repository_tool.name, error ) ) -def load_datatypes( app, datatypes_config, relative_install_dir ): +def load_datatype_items( app, repository, relative_install_dir, deactivate=False ): + # Load proprietary datatypes. + metadata = repository.metadata + datatypes_config = metadata.get( 'datatypes_config', None ) + if datatypes_config: + converter_path, display_path = load_datatypes( app, datatypes_config, relative_install_dir, deactivate=deactivate ) + if converter_path or display_path: + # Create a dictionary of tool shed repository related information. + repository_dict = create_repository_dict_for_proprietary_datatypes( tool_shed=repository.tool_shed, + name=repository.name, + owner=repository.owner, + installed_changeset_revision=repository.installed_changeset_revision, + tool_dicts=metadata.get( 'tools', [] ), + converter_path=converter_path, + display_path=display_path ) + if converter_path: + # Load or deactivate proprietary datatype converters + app.datatypes_registry.load_datatype_converters( app.toolbox, installed_repository_dict=repository_dict, deactivate=deactivate ) + if display_path: + # Load or deactivate proprietary datatype display applications + app.datatypes_registry.load_display_applications( installed_repository_dict=repository_dict, deactivate=deactivate ) +def load_datatypes( app, datatypes_config, relative_install_dir, deactivate=False ): # This method is used by the InstallManager, which does not have access to trans. def __import_module( relative_path, datatype_module ): sys.path.insert( 0, relative_path ) @@ -569,10 +598,10 @@ # are all subclasses of data types that are in the distribution. imported_modules = [] # Load proprietary datatypes - app.datatypes_registry.load_datatypes( root_dir=app.config.root, config=datatypes_config, imported_modules=imported_modules ) + app.datatypes_registry.load_datatypes( root_dir=app.config.root, config=datatypes_config, imported_modules=imported_modules, deactivate=deactivate ) return converter_path, display_path def load_repository_contents( app, repository_name, description, owner, changeset_revision, tool_path, repository_clone_url, relative_install_dir, - current_working_dir, tmp_name, tool_section=None, shed_tool_conf=None, new_install=True ): + current_working_dir, tmp_name, tool_shed=None, tool_section=None, shed_tool_conf=None, new_install=True ): # This method is used by the InstallManager, which does not have access to trans. # Generate the metadata for the installed tool shed repository. It is imperative that # the installed repository is updated to the desired changeset_revision before metadata @@ -592,16 +621,6 @@ section_name = '' tool_section_dict = dict( id=section_id, version=section_version, name=section_name ) metadata_dict = generate_metadata( app.toolbox, relative_install_dir, repository_clone_url, tool_section_dict=tool_section_dict ) - if 'datatypes_config' in metadata_dict: - datatypes_config = os.path.abspath( metadata_dict[ 'datatypes_config' ] ) - # Load data types required by tools. - converter_path, display_path = load_datatypes( app, datatypes_config, relative_install_dir ) - if converter_path: - # Load proprietary datatype converters - app.datatypes_registry.load_datatype_converters( app.toolbox, converter_path=converter_path ) - if display_path: - # Load proprietary datatype display applications - app.datatypes_registry.load_display_applications( display_path=display_path ) if 'tools' in metadata_dict: repository_tools_tups = get_repository_tools_tups( app, metadata_dict ) if repository_tools_tups: @@ -632,6 +651,25 @@ os.unlink( tmp_name ) except: pass + if 'datatypes_config' in metadata_dict: + datatypes_config = os.path.abspath( metadata_dict[ 'datatypes_config' ] ) + # Load data types required by tools. + converter_path, display_path = load_datatypes( app, datatypes_config, relative_install_dir ) + if converter_path or display_path: + # Create a dictionary of tool shed repository related information. + repository_dict = create_repository_dict_for_proprietary_datatypes( tool_shed=tool_shed, + name=repository_name, + owner=owner, + installed_changeset_revision=changeset_revision, + tool_dicts=metadata_dict.get( 'tools', [] ), + converter_path=converter_path, + display_path=display_path ) + if converter_path: + # Load proprietary datatype converters + app.datatypes_registry.load_datatype_converters( app.toolbox, installed_repository_dict=repository_dict ) + if display_path: + # Load proprietary datatype display applications + app.datatypes_registry.load_display_applications( installed_repository_dict=repository_dict ) # Add a new record to the tool_shed_repository table if one doesn't # already exist. If one exists but is marked deleted, undelete it. log.debug( "Adding new row (or updating an existing row) for repository '%s' in the tool_shed_repository table." % repository_name ) diff -r e3f4aca57f838e3b6fb6dc114a25ea62b5f34317 -r 1a4be690f5038e78a6f5abe21b9552293b911a53 lib/galaxy/web/controllers/admin_toolshed.py --- a/lib/galaxy/web/controllers/admin_toolshed.py +++ b/lib/galaxy/web/controllers/admin_toolshed.py @@ -258,6 +258,7 @@ returncode, tmp_name = update_repository( current_working_dir, relative_install_dir, changeset_revision ) if returncode == 0: owner = get_repository_owner( clean_repository_clone_url( repository_clone_url ) ) + tool_shed = clean_tool_shed_url( tool_shed_url ) metadata_dict = load_repository_contents( app=trans.app, repository_name=name, description=description, @@ -268,6 +269,7 @@ relative_install_dir=relative_install_dir, current_working_dir=current_working_dir, tmp_name=tmp_name, + tool_shed=tool_shed, tool_section=tool_section, shed_tool_conf=shed_tool_conf, new_install=True ) @@ -328,6 +330,7 @@ remove_from_disk_checked = CheckboxField.is_checked( remove_from_disk ) repository = get_repository( trans, kwd[ 'id' ] ) if params.get( 'deactivate_or_uninstall_repository_button', False ): + # Deatcivate repository tools. shed_tool_conf, tool_path, relative_install_dir = self.__get_tool_path_and_relative_install_dir( trans, repository ) metadata = repository.metadata repository_tools_tups = get_repository_tools_tups( trans.app, metadata ) @@ -366,6 +369,8 @@ tool_path=tool_path, owner=repository.owner, add_to_config=False ) + # Deactivate proprietary datatypes. + load_datatype_items( trans.app, repository, relative_install_dir, deactivate=True ) if remove_from_disk_checked: # TODO: Remove repository from disk and alter the shed tool config. message = 'The repository named %s has been uninstalled.' % repository.name @@ -413,10 +418,6 @@ repository_tools_tups = get_repository_tools_tups( trans.app, metadata ) tool_panel_section = metadata[ 'tool_panel_section' ] section_id = tool_panel_section[ 'id' ] - # Generate the list of tool panel keys derived from the tools included in the repository. - #repository_tool_panel_keys = [] - #if repository_tools_tups: - # repository_tool_panel_keys = [ 'tool_%s' % repository_tools_tup[ 1 ] for repository_tools_tup in repository_tools_tups ] if section_id in [ '' ]: # If the repository includes tools, reload them into the tool panel outside of any sections. self.__activate( trans, repository, repository_tools_tups, tool_section=None, section_key=None ) @@ -447,11 +448,14 @@ id_attr = elem.get( 'id' ) if id_attr == section_id: tool_section = elem + # Load the tools. trans.app.toolbox.load_section_tag_set( tool_section, trans.app.toolbox.tool_panel, tool_path ) section_loaded = True break if section_loaded: break + # Load proprietary datatypes. + load_datatype_items( trans.app, repository, relative_install_dir ) message = 'The repository named %s has been activated.' % repository.name status = 'done' return trans.response.send_redirect( web.url_for( controller='admin_toolshed', @@ -459,6 +463,7 @@ message=message, status=status ) ) def __activate( self, trans, repository, repository_tools_tups, tool_section=None, section_key=None ): + # Load tools. if tool_section: elems = tool_section.elems for repository_tools_tup in repository_tools_tups: @@ -475,16 +480,18 @@ if tool_section: if tool.id not in elems: elems[ 'tool_%s' % tool.id ] = tool - log.debug( "Reactivated tool: %s %s" % ( tool.id, tool.version ) ) + log.debug( "Reactivated tool id: %s, version: %s" % ( tool.id, tool.version ) ) else: if tool.id not in trans.app.toolbox.tools_by_id: # Allow for the same tool to be loaded into multiple places in the tool panel. trans.app.toolbox.tools_by_id[ tool.id ] = tool trans.app.toolbox.tool_panel[ 'tool_%s' % tool.id ] = tool - log.debug( "Reactivated tool: %s %s" % ( tool.id, tool.version ) ) + log.debug( "Reactivated tool id: %s, version: %s" % ( tool.id, tool.version ) ) if tool_section: trans.app.toolbox.tool_panel[ section_key ] = tool_section log.debug( "Appended reactivated tool to section: %s" % tool_section.name ) + shed_tool_conf, tool_path, relative_install_dir = self.__get_tool_path_and_relative_install_dir( trans, repository ) + load_datatype_items( trans.app, repository, relative_install_dir ) @web.expose @web.require_admin def manage_repository( self, trans, **kwd ): @@ -556,6 +563,7 @@ if returncode == 0: # Update the repository metadata. repository_clone_url = os.path.join( tool_shed_url, 'repos', owner, name ) + tool_shed = clean_tool_shed_url( tool_shed_url ) metadata_dict = load_repository_contents( app=trans.app, name=name, description=repository.description, @@ -566,6 +574,7 @@ relative_install_dir=relative_install_dir, current_working_dir=current_working_dir, tmp_name=tmp_name, + tool_shed=tool_shed, tool_section=None, shed_tool_conf=None, new_install=False ) Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: jgoecks: Remove spurious export in run functional tests scripts.
by Bitbucket 17 Jan '12

17 Jan '12

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/changeset/e3f4aca57f83/ changeset: e3f4aca57f83 user: jgoecks date: 2012-01-17 15:24:37 summary: Remove spurious export in run functional tests scripts. affected #: 1 file diff -r 6b635cb8191d0ce91e60f8e2cc5ec1606b1c1720 -r e3f4aca57f838e3b6fb6dc114a25ea62b5f34317 run_functional_tests.sh --- a/run_functional_tests.sh +++ b/run_functional_tests.sh @@ -1,7 +1,6 @@ #!/bin/sh # A good place to look for nose info: http://somethingaboutorange.com/mrl/projects/nose/ -export PATH=/usr/local/bin:$PATH rm -f run_functional_tests.log if [ ! $1 ]; then Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: jgoecks: Update functional tests to be compatible with 0d1d62b8be2e
by Bitbucket 17 Jan '12

17 Jan '12

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/changeset/6b635cb8191d/ changeset: 6b635cb8191d user: jgoecks date: 2012-01-17 15:05:11 summary: Update functional tests to be compatible with 0d1d62b8be2e affected #: 2 files diff -r 2272101e8c292bd33e78cdcee37bac4a62cd5cb4 -r 6b635cb8191d0ce91e60f8e2cc5ec1606b1c1720 test/base/twilltestcase.py --- a/test/base/twilltestcase.py +++ b/test/base/twilltestcase.py @@ -831,12 +831,12 @@ previously_created = True except: try: - self.check_page_for_string( 'This user name is not available' ) + self.check_page_for_string( 'Public name is taken; please choose another' ) username_taken = True except: try: # Note that we're only checking if the usr name is >< 4 chars here... - self.check_page_for_string( 'User name must be at least 4 characters in length' ) + self.check_page_for_string( 'Public name must be at least 4 characters in length' ) invalid_username = True except: pass @@ -1218,12 +1218,12 @@ previously_created = True except: try: - self.check_page_for_string( 'This user name is not available' ) + self.check_page_for_string( 'Public name is taken; please choose another' ) username_taken = True except: try: # Note that we're only checking if the usr name is >< 4 chars here... - self.check_page_for_string( 'User name must be at least 4 characters in length' ) + self.check_page_for_string( 'Public name must be at least 4 characters in length' ) invalid_username = True except: pass diff -r 2272101e8c292bd33e78cdcee37bac4a62cd5cb4 -r 6b635cb8191d0ce91e60f8e2cc5ec1606b1c1720 test/functional/test_admin_features.py --- a/test/functional/test_admin_features.py +++ b/test/functional/test_admin_features.py @@ -32,7 +32,7 @@ webapp='galaxy', referer='' ) if not username_taken: - raise AssertionError, "The user name (%s) is already being used by another user, but no error was displayed" \ + raise AssertionError, "The public name (%s) is already being used by another user, but no error was displayed" \ % 'admin-user' # Test setting the user name to an invalid one. Note that the account must not exist in order # for this test to work as desired, so the email we're passing is important... @@ -42,7 +42,7 @@ webapp='galaxy', referer='' ) if not invalid_username: - raise AssertionError, "The user name (%s) is is invalid, but no error was displayed" % username + raise AssertionError, "The public name (%s) is is invalid, but no error was displayed" % username previously_created, username_taken, invalid_username = self.create_new_account_as_admin( email=email, password=password, username='regular-user3', Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: jgoecks: Subset GFF datasets correctly when running tools in Trackster.
by Bitbucket 17 Jan '12

17 Jan '12

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/changeset/2272101e8c29/ changeset: 2272101e8c29 user: jgoecks date: 2012-01-17 04:40:13 summary: Subset GFF datasets correctly when running tools in Trackster. affected #: 1 file diff -r ec85ba64a5b751f53c2288fdbd27ae228a8d89c3 -r 2272101e8c292bd33e78cdcee37bac4a62cd5cb4 lib/galaxy/visualization/tracks/data_providers.py --- a/lib/galaxy/visualization/tracks/data_providers.py +++ b/lib/galaxy/visualization/tracks/data_providers.py @@ -857,7 +857,7 @@ reader = GFFReaderWrapper( source, fix_strand=True ) feature = reader.next() for interval in feature.intervals: - out.write(interval.raw_line + '\n') + out.write( '\t'.join( interval.fields ) + '\n' ) out.close() def get_iterator( self, chrom, start, end ): Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: jgoecks: Trackster: true replacement of a track with a group when running a tool.
by Bitbucket 16 Jan '12

16 Jan '12

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/changeset/ec85ba64a5b7/ changeset: ec85ba64a5b7 user: jgoecks date: 2012-01-16 23:36:04 summary: Trackster: true replacement of a track with a group when running a tool. affected #: 1 file diff -r a5703335e20544613416329c2fbd2a5c7af965e4 -r ec85ba64a5b751f53c2288fdbd27ae228a8d89c3 static/scripts/trackster.js --- a/static/scripts/trackster.js +++ b/static/scripts/trackster.js @@ -974,9 +974,8 @@ if (update_html) { old_drawable.container_div.replaceWith(new_drawable.container_div); } - return true; - } - return false; + } + return index; }, /** * Remove drawable from this collection. @@ -1839,18 +1838,13 @@ // Create new group. var group = new DrawableGroup(this.name, this.track.view); + // Replace track with group. + var index = current_track.container.replace_drawable(current_track, group, false); + + // Update HTML. // FIXME: this is ugly way to replace a track with a group -- make this easier via // a Drawable or DrawableCollection function. - // NOTE: doing this: - // current_track.container.replace_drawable(current_track, group, true); - // causes problems with sliders, so it's not used right now. - - // Add group to view. - current_track.view.add_drawable(group); - current_track.view.content_div.append(group.container_div); - - // Move track from view to group. - current_track.container.remove_drawable(current_track); + group.container_div.insertBefore(current_track.view.content_div.children()[index]); group.add_drawable(current_track); current_track.container_div.appendTo(group.content_div); container = group; Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: natefoo: Avoid annoying errors produced by the Googlebot hitting the import library datasets to history page (it doesn't have a history).
by Bitbucket 16 Jan '12

16 Jan '12

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/changeset/a8b733b58fec/ changeset: a8b733b58fec user: natefoo date: 2012-01-16 18:38:03 summary: Avoid annoying errors produced by the Googlebot hitting the import library datasets to history page (it doesn't have a history). affected #: 1 file diff -r 096f4040e90d3516c8ad6093f4e2fcd618c43f6b -r a8b733b58fec15f2aa664eb2b3f7e72e10b622bc lib/galaxy/web/controllers/library_common.py --- a/lib/galaxy/web/controllers/library_common.py +++ b/lib/galaxy/web/controllers/library_common.py @@ -1956,7 +1956,12 @@ # to the lddas in order for the menu optin to be available. ldda = trans.sa_session.query( trans.model.LibraryDatasetDatasetAssociation ).get( ldda_id ) source_lddas.append( ldda ) - target_histories = [ current_history ] + if current_history is not None: + target_histories = [ current_history ] + else: + target_histories = [] + message = 'You must have a history before you can import datasets. You can do this by <a href="%s" target="_top">loading the analysis interface</a>.' % url_for(controller='root') + status = 'error' if user: target_histories = user.active_histories if action == 'import_to_current_history' and library_id: Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: natefoo: Filesystem free space checking for the object store.
by Bitbucket 16 Jan '12

16 Jan '12

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/changeset/096f4040e90d/ changeset: 096f4040e90d user: natefoo date: 2012-01-16 18:04:45 summary: Filesystem free space checking for the object store. affected #: 5 files diff -r d3b6789cbc90c65ca78a5205547a1b6d4bc39064 -r 096f4040e90d3516c8ad6093f4e2fcd618c43f6b lib/galaxy/jobs/__init__.py --- a/lib/galaxy/jobs/__init__.py +++ b/lib/galaxy/jobs/__init__.py @@ -9,6 +9,7 @@ from galaxy.util.json import from_json_string from galaxy.util.expressions import ExpressionContext from galaxy.jobs.actions.post import ActionBox +from galaxy.exceptions import ObjectInvalid from sqlalchemy.sql.expression import and_, or_ @@ -321,9 +322,12 @@ # directory to be set before prepare is run, or else premature deletion # and job recovery fail. # Create the working dir if necessary - self.app.object_store.create(job, base_dir='job_work', dir_only=True, extra_dir=str(self.job_id)) - self.working_directory = self.app.object_store.get_filename(job, base_dir='job_work', dir_only=True, extra_dir=str(self.job_id)) - log.debug('(%s) Working directory for job is: %s' % (self.job_id, self.working_directory)) + try: + self.app.object_store.create(job, base_dir='job_work', dir_only=True, extra_dir=str(self.job_id)) + self.working_directory = self.app.object_store.get_filename(job, base_dir='job_work', dir_only=True, extra_dir=str(self.job_id)) + log.debug('(%s) Working directory for job is: %s' % (self.job_id, self.working_directory)) + except ObjectInvalid: + raise Exception('Unable to create job working directory, job failure') self.output_paths = None self.output_dataset_paths = None self.tool_provided_job_metadata = None diff -r d3b6789cbc90c65ca78a5205547a1b6d4bc39064 -r 096f4040e90d3516c8ad6093f4e2fcd618c43f6b lib/galaxy/objectstore/__init__.py --- a/lib/galaxy/objectstore/__init__.py +++ b/lib/galaxy/objectstore/__init__.py @@ -8,6 +8,7 @@ import time import random import shutil +import statvfs import logging import threading import subprocess @@ -40,7 +41,6 @@ def shutdown(self): self.running = False - self.extra_dirs = {} def exists(self, obj, base_dir=None, dir_only=False, extra_dir=None, extra_dir_at_root=False, alt_name=None): """ @@ -170,6 +170,12 @@ See `exists` method for the description of the fields. """ raise NotImplementedError() + + def get_store_usage_percent(self): + """ + Return the percentage indicating how full the store is + """ + raise NotImplementedError() ## def get_staging_command( id ): ## """ @@ -342,6 +348,9 @@ def get_object_url(self, obj, **kwargs): return None + def get_store_usage_percent(self): + st = os.statvfs(self.file_path) + return (float(st.f_blocks - st.f_bavail)/st.f_blocks) * 100 class CachingObjectStore(ObjectStore): @@ -831,6 +840,9 @@ log.warning("Trouble generating URL for dataset '%s': %s" % (rel_path, ex)) return None + def get_store_usage_percent(self): + return 0.0 + class DistributedObjectStore(ObjectStore): """ @@ -847,18 +859,29 @@ "'distributed_object_store_config_file')" self.backends = {} self.weighted_backend_ids = [] + self.original_weighted_backend_ids = [] + self.max_percent_full = {} + self.global_max_percent_full = 0.0 random.seed() self.__parse_distributed_config(config) + if self.global_max_percent_full or filter(lambda x: x is not None, self.max_percent_full.values()): + self.sleeper = Sleeper() + self.filesystem_monitor_thread = threading.Thread(target=self.__filesystem_monitor) + self.filesystem_monitor_thread.start() + log.info("Filesystem space monitor started") + def __parse_distributed_config(self, config): tree = util.parse_xml(self.distributed_config) root = tree.getroot() log.debug('Loading backends for distributed object store from %s' % self.distributed_config) + self.global_max_percent_full = float(root.get('maxpctfull', 0)) for elem in [ e for e in root if e.tag == 'backend' ]: id = elem.get('id') weight = int(elem.get('weight', 1)) + maxpctfull = float(elem.get('maxpctfull', 0)) if elem.get('type', 'disk'): path = None extra_dirs = {} @@ -869,6 +892,7 @@ type = sub.get('type') extra_dirs[type] = sub.get('path') self.backends[id] = DiskObjectStore(config, file_path=path, extra_dirs=extra_dirs) + self.max_percent_full[id] = maxpctfull log.debug("Loaded disk backend '%s' with weight %s and file_path: %s" % (id, weight, path)) if extra_dirs: log.debug(" Extra directories:") @@ -879,13 +903,26 @@ # sequence the number of times equalling weight, then randomly # choose a backend from that sequence at creation self.weighted_backend_ids.append(id) + self.original_weighted_backend_ids = self.weighted_backend_ids + + def __filesystem_monitor(self): + while self.running: + new_weighted_backend_ids = self.original_weighted_backend_ids + for id, backend in self.backends.items(): + maxpct = self.max_percent_full[id] or self.global_max_percent_full + pct = backend.get_store_usage_percent() + if pct > maxpct: + new_weighted_backend_ids = filter(lambda x: x != id, new_weighted_backend_ids) + self.weighted_backend_ids = new_weighted_backend_ids + self.sleeper.sleep(120) # Test free space every 2 minutes + + def shutdown(self): + super(DistributedObjectStore, self).shutdown() + self.sleeper.wake() def exists(self, obj, **kwargs): return self.__call_method('exists', obj, False, False, **kwargs) - #def store_id(self, obj, **kwargs): - # return self.__get_store_id_for(obj, **kwargs)[0] - def file_ready(self, obj, **kwargs): return self.__call_method('file_ready', obj, False, False, **kwargs) @@ -894,8 +931,11 @@ create() is the only method in which obj.object_store_id may be None """ if obj.object_store_id is None or not self.exists(obj, **kwargs): - if obj.object_store_id is None or obj.object_store_id not in self.backends: - obj.object_store_id = random.choice(self.weighted_backend_ids) + if obj.object_store_id is None or obj.object_store_id not in self.weighted_backend_ids: + try: + obj.object_store_id = random.choice(self.weighted_backend_ids) + except IndexError: + raise ObjectInvalid() object_session( obj ).add( obj ) object_session( obj ).flush() log.debug("Selected backend '%s' for creation of %s %s" % (obj.object_store_id, obj.__class__.__name__, obj.id)) diff -r d3b6789cbc90c65ca78a5205547a1b6d4bc39064 -r 096f4040e90d3516c8ad6093f4e2fcd618c43f6b lib/galaxy/tools/__init__.py --- a/lib/galaxy/tools/__init__.py +++ b/lib/galaxy/tools/__init__.py @@ -1142,7 +1142,11 @@ return "tool_form.mako", dict( errors=errors, tool_state=state, incoming=incoming, error_message=error_message ) # If we've completed the last page we can execute the tool elif state.page == self.last_page: - _, out_data = self.execute( trans, incoming=params, history=history ) + try: + _, out_data = self.execute( trans, incoming=params, history=history ) + except Exception, e: + log.exception('Exception caught while attempting tool execution:') + return 'message.mako', dict( status='error', message='Error executing tool: %s' % str(e), refresh_frames=[] ) try: assert isinstance( out_data, odict ) return 'tool_executed.mako', dict( out_data=out_data ) diff -r d3b6789cbc90c65ca78a5205547a1b6d4bc39064 -r 096f4040e90d3516c8ad6093f4e2fcd618c43f6b lib/galaxy/tools/actions/__init__.py --- a/lib/galaxy/tools/actions/__init__.py +++ b/lib/galaxy/tools/actions/__init__.py @@ -6,6 +6,7 @@ from galaxy.util.template import fill_template from galaxy.util.none_like import NoneDataset from galaxy.web import url_for +from galaxy.exceptions import ObjectInvalid import galaxy.tools from types import * @@ -295,7 +296,10 @@ # created in the "default" store, all others will be created in # the same store as the first. data.dataset.object_store_id = object_store_id - trans.app.object_store.create( data.dataset ) + try: + trans.app.object_store.create( data.dataset ) + except ObjectInvalid: + raise Exception('Unable to create output dataset: object store is full') object_store_id = data.dataset.object_store_id # these will be the same thing after the first output # This may not be neccesary with the new parent/child associations data.designation = name diff -r d3b6789cbc90c65ca78a5205547a1b6d4bc39064 -r 096f4040e90d3516c8ad6093f4e2fcd618c43f6b lib/galaxy/tools/actions/upload_common.py --- a/lib/galaxy/tools/actions/upload_common.py +++ b/lib/galaxy/tools/actions/upload_common.py @@ -5,6 +5,7 @@ from galaxy.datatypes import sniff from galaxy.util.json import to_json_string from galaxy.model.orm import eagerload_all +from galaxy.exceptions import ObjectInvalid import logging log = logging.getLogger( __name__ ) @@ -328,7 +329,10 @@ # Create an empty file immediately if not dataset.dataset.external_filename: dataset.dataset.object_store_id = object_store_id - trans.app.object_store.create( dataset.dataset ) + try: + trans.app.object_store.create( dataset.dataset ) + except ObjectInvalid: + raise Exception('Unable to create output dataset: object store is full') object_store_id = dataset.dataset.object_store_id trans.sa_session.add( dataset ) # open( dataset.file_name, "w" ).close() Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: greg: Fix for reloading a tool in the tool panel - make sure to keep the tool's tool shed related attributes when the tool is reloaded. Also make sure to send the guid attribute when loading a tool into the tool panel if the tool is contained in an installed tool shed repository.
by Bitbucket 16 Jan '12

16 Jan '12

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/changeset/b36526f8edbc/ changeset: b36526f8edbc user: greg date: 2012-01-16 16:43:33 summary: Fix for reloading a tool in the tool panel - make sure to keep the tool's tool shed related attributes when the tool is reloaded. Also make sure to send the guid attribute when loading a tool into the tool panel if the tool is contained in an installed tool shed repository. affected #: 3 files diff -r e2ac92093afda6743ea68766be96c9c7a454568d -r b36526f8edbc2776225b5b9a8da7657771186025 lib/galaxy/tools/__init__.py --- a/lib/galaxy/tools/__init__.py +++ b/lib/galaxy/tools/__init__.py @@ -163,6 +163,7 @@ tool.repository_name = repository_name tool.repository_owner = repository_owner tool.installed_changeset_revision = installed_changeset_revision + tool.guid = guid tool.old_id = elem.find( "id" ).text tool.version = elem.find( "version" ).text if self.app.config.get_bool( 'enable_tool_tags', False ): @@ -261,6 +262,15 @@ else: old_tool = self.tools_by_id[ tool_id ] new_tool = self.load_tool( old_tool.config_file ) + # The tool may have been installed from a tool shed, so set the tool shed attributes. + # Since the tool version may have changed, we don't override it here. + new_tool.id = old_tool.id + new_tool.guid = old_tool.guid + new_tool.tool_shed = old_tool.tool_shed + new_tool.repository_name = old_tool.repository_name + new_tool.repository_owner = old_tool.repository_owner + new_tool.installed_changeset_revision = old_tool.installed_changeset_revision + new_tool.old_id = old_tool.old_id # Replace old_tool with new_tool in self.tool_panel tool_key = 'tool_' + tool_id for key, val in self.tool_panel.items(): @@ -268,11 +278,9 @@ self.tool_panel[ key ] = new_tool break elif key.startswith( 'section' ): - section = val - for section_key, section_val in section.elems.items(): - if section_key == tool_key: - self.tool_panel[ key ].elems[ section_key ] = new_tool - break + if tool_key in val.elems: + self.tool_panel[ key ].elems[ tool_key ] = new_tool + break self.tools_by_id[ tool_id ] = new_tool message = "Reloaded the tool: " message += "name: %s " % old_tool.name @@ -462,18 +470,19 @@ self.repository_name = None self.repository_owner = None self.installed_changeset_revision = None + # The tool.id value will be the value of guid, but we'll keep the + # guid attribute since it is useful to have. + self.guid = guid self.old_id = None self.version = None # Parse XML element containing configuration self.parse( root, guid=guid ) - @property def sa_session( self ): """ Returns a SQLAlchemy session """ return self.app.model.context - def parse( self, root, guid=None ): """ Read tool configuration from the element `root` and fill in `self`. @@ -625,7 +634,6 @@ self.trackster_conf = TracksterConfig.parse( trackster_conf ) else: self.trackster_conf = None - def parse_inputs( self, root ): """ Parse the "<inputs>" element and create appropriate `ToolParameter`s. @@ -683,7 +691,6 @@ if not isinstance( param, ( HiddenToolParameter, BaseURLToolParameter ) ): self.input_required = True break - def parse_help( self, root ): """ Parse the help text for the tool. Formatted in reStructuredText. @@ -715,7 +722,6 @@ # Pad out help pages to match npages ... could this be done better? while len( self.help_by_page ) < self.npages: self.help_by_page.append( self.help ) - def parse_outputs( self, root ): """ Parse <outputs> elements and fill in self.outputs (keyed by name) @@ -739,7 +745,6 @@ output.tool = self output.actions = ToolOutputActionGroup( output, data_elem.find( 'actions' ) ) self.outputs[ output.name ] = output - def parse_tests( self, tests_elem ): """ Parse any "<test>" elements, create a `ToolTestBuilder` for each and @@ -853,7 +858,6 @@ test.error = True test.exception = e self.tests.append( test ) - def parse_input_page( self, input_elem, enctypes ): """ Parse a page of inputs. This basically just calls 'parse_input_elem', @@ -868,7 +872,6 @@ else: display = None return display, inputs - def parse_input_elem( self, parent_elem, enctypes, context=None ): """ Parse a parent element whose children are inputs -- these could be @@ -960,7 +963,6 @@ param.ref_input = context[ param.data_ref ] self.input_params.append( param ) return rval - def parse_param_elem( self, input_elem, enctypes, context ): """ Parse a single "<param>" element and return a ToolParameter instance. @@ -976,7 +978,6 @@ for name in param.get_dependencies(): context[ name ].refresh_on_change = True return param - def parse_requirements( self, requirements_elem ): """ Parse each requirement from the <requirements> element and add to @@ -999,7 +1000,6 @@ method = None requirement = ToolRequirement( name=name, type=type, version=version, fabfile=fabfile, method=method ) self.requirements.append( requirement ) - def check_workflow_compatible( self ): """ Determine if a tool can be used in workflows. External tools and the @@ -1020,7 +1020,6 @@ # TODO: Anyway to capture tools that dynamically change their own # outputs? return True - def new_state( self, trans, all_pages=False ): """ Create a new `DefaultToolState` for this tool. It will be initialized @@ -1037,7 +1036,6 @@ inputs = self.inputs_by_page[ 0 ] self.fill_in_new_state( trans, inputs, state.inputs ) return state - def fill_in_new_state( self, trans, inputs, state, context=None ): """ Fill in a tool state dictionary with default values for all parameters @@ -1046,7 +1044,6 @@ context = ExpressionContext( state, context ) for input in inputs.itervalues(): state[ input.name ] = input.get_initial_value( trans, context ) - def get_param_html_map( self, trans, page=0, other_values={} ): """ Return a dictionary containing the HTML representation of each @@ -1062,14 +1059,12 @@ raise Exception( "'get_param_html_map' only supported for simple paramters" ) rval[key] = param.get_html( trans, other_values=other_values ) return rval - def get_param( self, key ): """ Returns the parameter named `key` or None if there is no such parameter. """ return self.inputs.get( key, None ) - def get_hook(self, name): """ Returns an object from the code file referenced by `code_namespace` @@ -1082,7 +1077,6 @@ elif name in self.code_namespace: return self.code_namespace[name] return None - def visit_inputs( self, value, callback ): """ Call the function `callback` on each parameter of this tool. Visits @@ -1099,7 +1093,6 @@ callback( "", input, value[input.name] ) else: input.visit_inputs( "", value[input.name], callback ) - def handle_input( self, trans, incoming, history=None ): """ Process incoming parameters for this tool from the dict `incoming`, @@ -1177,7 +1170,6 @@ pass # Just a refresh, render the form with updated state and errors. return 'tool_form.mako', dict( errors=errors, tool_state=state ) - def find_fieldstorage( self, x ): if isinstance( x, FieldStorage ): raise InterruptedUpload( None ) @@ -1185,7 +1177,6 @@ [ self.find_fieldstorage( y ) for y in x.values() ] elif type( x ) is types.ListType: [ self.find_fieldstorage( y ) for y in x ] - def handle_interrupted( self, trans, inputs ): """ Upon handling inputs, if it appears that we have received an incomplete @@ -1216,7 +1207,6 @@ return 'message.mako', dict( status='error', message='Your upload was interrupted. If this was uninentional, please retry it.', refresh_frames=[], cont=None ) - def update_state( self, trans, inputs, state, incoming, prefix="", context=None, update_only=False, old_errors={}, item_callback=None ): """ @@ -1458,7 +1448,6 @@ else: raise Exception( "Unexpected parameter type" ) return args - def execute( self, trans, incoming={}, set_output_hid=True, history=None, **kwargs ): """ Execute the tool using parameter values in `incoming`. This just @@ -1467,13 +1456,10 @@ when run will build the tool's outputs, e.g. `DefaultToolAction`. """ return self.tool_action.execute( self, trans, incoming=incoming, set_output_hid=set_output_hid, history=history, **kwargs ) - def params_to_strings( self, params, app ): return params_to_strings( self.inputs, params, app ) - def params_from_strings( self, params, app, ignore_errors=False ): return params_from_strings( self.inputs, params, app, ignore_errors ) - def check_and_update_param_values( self, values, trans ): """ Check that all parameters have values, and fill in with default @@ -1483,7 +1469,6 @@ messages = {} self.check_and_update_param_values_helper( self.inputs, values, trans, messages ) return messages - def check_and_update_param_values_helper( self, inputs, values, trans, messages, context=None, prefix="" ): """ Recursive helper for `check_and_update_param_values_helper` @@ -1529,7 +1514,6 @@ else: # Regular tool parameter, no recursion needed pass - def handle_unvalidated_param_values( self, input_values, app ): """ Find any instances of `UnvalidatedValue` within input_values and @@ -1540,7 +1524,6 @@ if not self.check_values: return self.handle_unvalidated_param_values_helper( self.inputs, input_values, app ) - def handle_unvalidated_param_values_helper( self, inputs, input_values, app, context=None, prefix="" ): """ Recursive helper for `handle_unvalidated_param_values` @@ -1582,7 +1565,6 @@ % ( prefix, input.label, e ) raise LateValidationError( message ) input_values[ input.name ] = value - def handle_job_failure_exception( self, e ): """ Called by job.fail when an exception is generated to allow generation @@ -1594,7 +1576,6 @@ if isinstance( e, LateValidationError ): message = e.message return message - def build_param_dict( self, incoming, input_datasets, output_datasets, output_paths, job_working_directory ): """ Build the dictionary of parameters for substituting into the command @@ -1734,7 +1715,6 @@ param_dict['__datatypes_config__'] = param_dict['GALAXY_DATATYPES_CONF_FILE'] = self.app.datatypes_registry.integrated_datatypes_configs # Return the dictionary of parameters return param_dict - def build_param_file( self, param_dict, directory=None ): """ Build temporary file for file based parameter transfer if needed @@ -1754,7 +1734,6 @@ return param_filename else: return None - def build_config_files( self, param_dict, directory=None ): """ Build temporary file for file based parameter transfer if needed @@ -1775,7 +1754,6 @@ param_dict[name] = config_filename config_filenames.append( config_filename ) return config_filenames - def build_command_line( self, param_dict ): """ Build command line to invoke this tool given a populated param_dict @@ -1799,7 +1777,6 @@ command_line = command_line.replace(executable, abs_executable, 1) command_line = self.interpreter + " " + command_line return command_line - def build_dependency_shell_commands( self ): """ Return a list of commands to be run to populate the current @@ -1820,7 +1797,6 @@ else: commands.append( 'PACKAGE_BASE=%s; export PACKAGE_BASE; . %s' % ( base_path, script_file ) ) return commands - def build_redirect_url_params( self, param_dict ): """ Substitute parameter values into self.redirect_url_params @@ -1833,7 +1809,6 @@ # Remove newlines redirect_url_params = redirect_url_params.replace( "\n", " " ).replace( "\r", " " ) return redirect_url_params - def parse_redirect_url( self, data, param_dict ): """ Parse the REDIRECT_URL tool param. Tools that send data to an external @@ -1873,7 +1848,6 @@ USERNAME = 'Anonymous' redirect_url += "&USERNAME=%s" % USERNAME return redirect_url - def call_hook( self, hook_name, *args, **kwargs ): """ Call the custom code hook function identified by 'hook_name' if any, @@ -1886,19 +1860,15 @@ except Exception, e: e.args = ( "Error in '%s' hook '%s', original message: %s" % ( self.name, hook_name, e.args[0] ) ) raise - def exec_before_job( self, app, inp_data, out_data, param_dict={} ): pass - def exec_after_process( self, app, inp_data, out_data, param_dict, job = None ): pass - def job_failed( self, job_wrapper, message, exception = False ): """ Called when a job has failed """ pass - def collect_associated_files( self, output, job_working_directory ): """ Find extra files in the job working directory and move them into @@ -1922,7 +1892,6 @@ shutil.rmtree(temp_file_path) except: continue - def collect_child_datasets( self, output): """ Look for child dataset files, create HDA and attach to parent. @@ -1983,7 +1952,6 @@ self.sa_session.add( child_dataset ) self.sa_session.flush() return children - def collect_primary_datasets( self, output): """ Find any additional datasets generated by a tool and attach (for @@ -2062,12 +2030,10 @@ def _build_GALAXY_URL_parameter( self ): return ToolParameter.build( self, ElementTree.XML( '<param name="GALAXY_URL" type="baseurl" value="/tool_runner?tool_id=%s" />' % self.id ) ) - def parse_inputs( self, root ): Tool.parse_inputs( self, root ) if 'GALAXY_URL' not in self.inputs: self.inputs[ 'GALAXY_URL' ] = self._build_GALAXY_URL_parameter() - def _prepare_datasource_json_list( self, param_list ): rval = [] for value in param_list: @@ -2088,7 +2054,6 @@ else: rval[ key ] = str( value ) return rval - def exec_before_job( self, app, inp_data, out_data, param_dict=None ): if param_dict is None: param_dict = {} @@ -2111,7 +2076,6 @@ cur_dbkey = param_dict.get( cur_base_param_name + 'dkey', dbkey ) cur_info = param_dict.get( cur_base_param_name + 'info', info ) cur_data_type = param_dict.get( cur_base_param_name + 'data_type', data_type ) - if cur_name: data.name = cur_name if not data.info and cur_info: @@ -2122,16 +2086,13 @@ data.extension = cur_data_type file_name = str( wrapped_data ) extra_files_path = str( wrapped_data.files_path ) - data_dict = dict( out_data_name = out_name, ext = data.ext, dataset_id = data.dataset.id, hda_id = data.id, file_name = file_name, extra_files_path = extra_files_path ) - json_params[ 'output_data' ].append( data_dict ) - if json_filename is None: json_filename = file_name out = open( json_filename, 'w' ) @@ -2176,7 +2137,6 @@ dataset.set_peek() self.sa_session.add( dataset ) self.sa_session.flush() - def job_failed( self, job_wrapper, message, exception = False ): job = job_wrapper.sa_session.query( model.Job ).get( job_wrapper.job_id ) if job: @@ -2334,19 +2294,16 @@ self.dataset = dataset self.metadata = self.MetadataWrapper( dataset.metadata ) self.false_path = false_path - def __str__( self ): if self.false_path is not None: return self.false_path else: return self.dataset.file_name - def __getattr__( self, key ): if self.false_path is not None and key == 'file_name': return self.false_path else: return getattr( self.dataset, key ) - def __nonzero__( self ): return bool( self.dataset ) diff -r e2ac92093afda6743ea68766be96c9c7a454568d -r b36526f8edbc2776225b5b9a8da7657771186025 lib/galaxy/util/shed_util.py --- a/lib/galaxy/util/shed_util.py +++ b/lib/galaxy/util/shed_util.py @@ -347,7 +347,7 @@ for tool_dict in metadata_dict[ 'tools' ]: relative_path = tool_dict[ 'tool_config' ] guid = tool_dict[ 'guid' ] - tool = app.toolbox.load_tool( os.path.abspath( relative_path ) ) + tool = app.toolbox.load_tool( os.path.abspath( relative_path ), guid=guid ) repository_tools_tups.append( ( relative_path, guid, tool ) ) return repository_tools_tups def get_tool_id_guid_map( app, tool_id, version, tool_shed, repository_owner, repository_name ): @@ -398,7 +398,7 @@ # TODO: Do more here than logging an exception. log.debug( exception_msg ) # Reload the tool into the local list of repository_tools_tups. - repository_tool = app.toolbox.load_tool( os.path.join( tool_path, tup_path ) ) + repository_tool = app.toolbox.load_tool( os.path.join( tool_path, tup_path ), guid=guid ) repository_tools_tups[ index ] = ( tup_path, repository_tool ) return repository_tools_tups def handle_missing_index_file( app, tool_path, sample_files, repository_tools_tups ): @@ -425,7 +425,7 @@ missing_files_handled.append( missing_tail ) break # Reload the tool into the local list of repository_tools_tups. - repository_tool = app.toolbox.load_tool( os.path.join( tool_path, tup_path ) ) + repository_tool = app.toolbox.load_tool( os.path.join( tool_path, tup_path ), guid=guid ) repository_tools_tups[ index ] = ( tup_path, guid, repository_tool ) return repository_tools_tups def handle_tool_dependencies( current_working_dir, repo_files_dir, repository_tools_tups ): diff -r e2ac92093afda6743ea68766be96c9c7a454568d -r b36526f8edbc2776225b5b9a8da7657771186025 lib/galaxy/web/controllers/admin_toolshed.py --- a/lib/galaxy/web/controllers/admin_toolshed.py +++ b/lib/galaxy/web/controllers/admin_toolshed.py @@ -467,6 +467,7 @@ tool.repository_name = repository.name tool.repository_owner = repository.owner tool.installed_changeset_revision = repository.installed_changeset_revision + tool.guid = guid # Set the tool's old_id to the id used before the tool shed existed. tool.old_id = tool.id # Set the tool's id to the tool shed guid. @@ -610,7 +611,7 @@ for tool_metadata_dict in repository.metadata[ 'tools' ]: if tool_metadata_dict[ 'id' ] == tool_id: metadata = tool_metadata_dict - tool = trans.app.toolbox.load_tool( os.path.abspath( metadata[ 'tool_config' ] ) ) + tool = trans.app.toolbox.load_tool( os.path.abspath( metadata[ 'tool_config' ] ), guid=metadata[ 'guid' ] ) break return trans.fill_template( "/admin/tool_shed_repository/view_tool_metadata.mako", repository=repository, Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: jgoecks: Add href to trackster icons in history pane so that browser treats it as a URL and not an anchor.
by Bitbucket 13 Jan '12

13 Jan '12

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/changeset/e2ac92093afd/ changeset: e2ac92093afd user: jgoecks date: 2012-01-13 22:22:29 summary: Add href to trackster icons in history pane so that browser treats it as a URL and not an anchor. affected #: 1 file diff -r a5e1ad623f390f1daceaa21aaa52868b4ce50c30 -r e2ac92093afda6743ea68766be96c9c7a454568d templates/root/history_common.mako --- a/templates/root/history_common.mako +++ b/templates/root/history_common.mako @@ -220,7 +220,7 @@ else: data_url = h.url_for( controller='tracks', action='list_tracks' ) %> - <a data-url="${data_url}" class="icon-button chart_curve tooltip trackster-add" + <a href="javascript:void(0)" data-url="${data_url}" class="icon-button chart_curve tooltip trackster-add" action-url="${h.url_for( controller='tracks', action='browser', dataset_id=dataset_id)}" new-url="${h.url_for( controller='tracks', action='index', dataset_id=dataset_id, default_dbkey=data.dbkey)}" title="View in Trackster"></a> %endif Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: greg: Fixes for Galaxy instance , => tool shed communciation where Galaxy instance is running behind apache with rewrite rules.
by Bitbucket 13 Jan '12

13 Jan '12

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/changeset/a5e1ad623f39/ changeset: a5e1ad623f39 user: greg date: 2012-01-13 22:09:41 summary: Fixes for Galaxy instance ,=> tool shed communciation where Galaxy instance is running behind apache with rewrite rules. affected #: 2 files diff -r 3d9f0ce7bb6138b20c200203014e60a4e9342160 -r a5e1ad623f390f1daceaa21aaa52868b4ce50c30 lib/galaxy/web/controllers/admin_toolshed.py --- a/lib/galaxy/web/controllers/admin_toolshed.py +++ b/lib/galaxy/web/controllers/admin_toolshed.py @@ -169,21 +169,21 @@ @web.require_admin def find_tools_in_tool_shed( self, trans, **kwd ): tool_shed_url = kwd[ 'tool_shed_url' ] - galaxy_url = url_for( '', qualified=True ) + galaxy_url = url_for( '/', qualified=True ) url = '%s/repository/find_tools?galaxy_url=%s&webapp=galaxy' % ( tool_shed_url, galaxy_url ) return trans.response.send_redirect( url ) @web.expose @web.require_admin def find_workflows_in_tool_shed( self, trans, **kwd ): tool_shed_url = kwd[ 'tool_shed_url' ] - galaxy_url = url_for( '', qualified=True ) + galaxy_url = url_for( '/', qualified=True ) url = '%s/repository/find_workflows?galaxy_url=%s&webapp=galaxy' % ( tool_shed_url, galaxy_url ) return trans.response.send_redirect( url ) @web.expose @web.require_admin def browse_tool_shed( self, trans, **kwd ): tool_shed_url = kwd[ 'tool_shed_url' ] - galaxy_url = url_for( '', qualified=True ) + galaxy_url = url_for( '/', qualified=True ) url = '%s/repository/browse_valid_repositories?galaxy_url=%s&webapp=galaxy' % ( tool_shed_url, galaxy_url ) return trans.response.send_redirect( url ) @web.expose diff -r 3d9f0ce7bb6138b20c200203014e60a4e9342160 -r a5e1ad623f390f1daceaa21aaa52868b4ce50c30 lib/galaxy/webapps/community/controllers/repository.py --- a/lib/galaxy/webapps/community/controllers/repository.py +++ b/lib/galaxy/webapps/community/controllers/repository.py @@ -437,7 +437,7 @@ if operation == "install": galaxy_url = trans.get_cookie( name='toolshedgalaxyurl' ) encoded_repo_info_dict, includes_tools = self.__encode_repo_info_dict( trans, webapp, util.listify( item_id ) ) - url = '%s/admin_toolshed/install_repository?tool_shed_url=%s&webapp=%s&repo_info_dict=%s&includes_tools=%s' % \ + url = '%sadmin_toolshed/install_repository?tool_shed_url=%s&webapp=%s&repo_info_dict=%s&includes_tools=%s' % \ ( galaxy_url, url_for( '', qualified=True ), webapp, encoded_repo_info_dict, str( includes_tools ) ) return trans.response.send_redirect( url ) else: @@ -513,7 +513,7 @@ if operation == "install": galaxy_url = trans.get_cookie( name='toolshedgalaxyurl' ) encoded_repo_info_dict, includes_tools = self.__encode_repo_info_dict( trans, webapp, util.listify( item_id ) ) - url = '%s/admin_toolshed/install_repository?tool_shed_url=%s&webapp=%s&repo_info_dict=%s&includes_tools=%s' % \ + url = '%sadmin_toolshed/install_repository?tool_shed_url=%s&webapp=%s&repo_info_dict=%s&includes_tools=%s' % \ ( galaxy_url, url_for( '', qualified=True ), webapp, encoded_repo_info_dict, str( includes_tools ) ) return trans.response.send_redirect( url ) else: @@ -766,7 +766,7 @@ repo_info_dict[ repository.name ] = ( repository.description, repository_clone_url, changeset_revision ) encoded_repo_info_dict = encode( repo_info_dict ) # Redirect back to local Galaxy to perform install. - url = '%s/admin_toolshed/install_repository?tool_shed_url=%s&repo_info_dict=%s&includes_tools=%s' % \ + url = '%sadmin_toolshed/install_repository?tool_shed_url=%s&repo_info_dict=%s&includes_tools=%s' % \ ( galaxy_url, url_for( '', qualified=True ), encoded_repo_info_dict, str( includes_tools ) ) return trans.response.send_redirect( url ) @web.expose @@ -789,7 +789,7 @@ no_update = 'false' else: # Start building up the url to redirect back to the calling Galaxy instance. - url = '%s/admin_toolshed/update_to_changeset_revision?tool_shed_url=%s' % ( galaxy_url, url_for( '', qualified=True ) ) + url = '%sadmin_toolshed/update_to_changeset_revision?tool_shed_url=%s' % ( galaxy_url, url_for( '', qualified=True ) ) url += '&name=%s&owner=%s&changeset_revision=%s&latest_changeset_revision=' % \ ( repository.name, repository.user.username, changeset_revision ) if changeset_revision == repository.tip: Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0