- galaxy-commits - lists.galaxyproject.org

commit/galaxy-central: greg: Move ~/lib/galaxy/tool_shed to ~/lib/tool_shed, and move the ~/lib/galaxy/util/shed_util componets to ~/lib/tool_shed/util.
by commits-noreply＠bitbucket.org 26 Feb '13

26 Feb '13

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/6f0050c4e06e/ changeset: 6f0050c4e06e user: greg date: 2013-02-26 21:52:51 summary: Move ~/lib/galaxy/tool_shed to ~/lib/tool_shed, and move the ~/lib/galaxy/util/shed_util componets to ~/lib/tool_shed/util. affected #: 77 files diff -r e9e09d1ab51752db592eee71032ec5fc33f6ee57 -r 6f0050c4e06ef6f5fdce3f7326280c07c63ef62e lib/galaxy/app.py --- a/lib/galaxy/app.py +++ b/lib/galaxy/app.py @@ -3,8 +3,8 @@ from galaxy import config, jobs, util, tools, web import galaxy.tools.search import galaxy.tools.data -import galaxy.tool_shed -import galaxy.tool_shed.tool_shed_registry +import tool_shed.galaxy_install +import tool_shed.tool_shed_registry from galaxy.web import security import galaxy.model import galaxy.datatypes.registry @@ -41,7 +41,7 @@ db_url = "sqlite:///%s?isolation_level=IMMEDIATE" % self.config.database # Set up the tool sheds registry if os.path.isfile( self.config.tool_sheds_config ): - self.tool_shed_registry = galaxy.tool_shed.tool_shed_registry.Registry( self.config.root, self.config.tool_sheds_config ) + self.tool_shed_registry = tool_shed.tool_shed_registry.Registry( self.config.root, self.config.tool_sheds_config ) else: self.tool_shed_registry = None log.debug( 'self.config.tool_sheds_config: %s, self.tool_shed_registry: %s', @@ -51,7 +51,7 @@ from galaxy.model.migrate.check import create_or_verify_database create_or_verify_database( db_url, kwargs.get( 'global_conf', {} ).get( '__file__', None ), self.config.database_engine_options, app=self ) # Alert the Galaxy admin to tools that have been moved from the distribution to the tool shed. - from galaxy.tool_shed.migrate.check import verify_tools + from tool_shed.galaxy_install.migrate.check import verify_tools verify_tools( self, db_url, kwargs.get( 'global_conf', {} ).get( '__file__', None ), self.config.database_engine_options ) # Object store manager self.object_store = build_object_store_from_config(self.config) @@ -64,7 +64,7 @@ object_store = self.object_store, trace_logger=self.trace_logger ) # Manage installed tool shed repositories. - self.installed_repository_manager = galaxy.tool_shed.InstalledRepositoryManager( self ) + self.installed_repository_manager = tool_shed.galaxy_install.InstalledRepositoryManager( self ) # Create an empty datatypes registry. self.datatypes_registry = galaxy.datatypes.registry.Registry() # Load proprietary datatypes defined in datatypes_conf.xml files in all installed tool shed repositories. We @@ -105,7 +105,7 @@ self.data_managers = DataManagers( self ) # If enabled, poll respective tool sheds to see if updates are available for any installed tool shed repositories. if self.config.get_bool( 'enable_tool_shed_check', False ): - from tool_shed import update_manager + from tool_shed.galaxy_install import update_manager self.update_manager = update_manager.UpdateManager( self ) # Load proprietary datatype converters and display applications. self.installed_repository_manager.load_proprietary_converters_and_display_applications() diff -r e9e09d1ab51752db592eee71032ec5fc33f6ee57 -r 6f0050c4e06ef6f5fdce3f7326280c07c63ef62e lib/galaxy/model/__init__.py --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -3031,6 +3031,10 @@ pass class ToolShedRepository( object ): + api_collection_visible_keys = ( 'id', 'name', 'tool_shed', 'owner', 'installed_changeset_revision', 'changeset_revision', 'ctx_rev', 'includes_datatypes', + 'update_available', 'deleted', 'uninstalled', 'dist_to_shed', 'status', 'error_message' ) + api_element_visible_keys = ( 'id', 'name', 'tool_shed', 'owner', 'installed_changeset_revision', 'changeset_revision', 'ctx_rev', 'includes_datatypes', + 'update_available', 'deleted', 'uninstalled', 'dist_to_shed', 'status', 'error_message' ) installation_status = Bunch( NEW='New', CLONING='Cloning', SETTING_TOOL_VERSIONS='Setting tool versions', @@ -3066,6 +3070,10 @@ self.dist_to_shed = dist_to_shed self.status = status self.error_message = error_message + def as_dict( self, trans ): + tsr_dict = self.get_api_value( view='element' ) + tsr_dict[ 'id' ] = trans.security.encode_id( self.id ) + return tsr_dict def repo_files_directory( self, app ): repo_path = self.repo_path( app ) if repo_path: @@ -3153,6 +3161,22 @@ if self.shed_config_filename == shed_tool_conf_dict[ 'config_filename' ]: return shed_tool_conf_dict return default + def get_api_value( self, view='collection', value_mapper=None ): + if value_mapper is None: + value_mapper = {} + rval = {} + try: + visible_keys = self.__getattribute__( 'api_' + view + '_visible_keys' ) + except AttributeError: + raise Exception( 'Unknown API view: %s' % view ) + for key in visible_keys: + try: + rval[ key ] = self.__getattribute__( key ) + if key in value_mapper: + rval[ key ] = value_mapper.get( key )( rval[ key ] ) + except AttributeError: + rval[ key ] = None + return rval @property def can_install( self ): return self.status == self.installation_status.NEW diff -r e9e09d1ab51752db592eee71032ec5fc33f6ee57 -r 6f0050c4e06ef6f5fdce3f7326280c07c63ef62e lib/galaxy/model/migrate/versions/0113_update_migrate_tools_table.py --- /dev/null +++ b/lib/galaxy/model/migrate/versions/0113_update_migrate_tools_table.py @@ -0,0 +1,44 @@ +""" +Migration script to update the migrate_tools.repository_path column to point to the new location lib/tool_shed/galaxy_install/migrate. +""" + +from sqlalchemy import * +from sqlalchemy.orm import * +from migrate import * +from migrate.changeset import * + +import datetime +now = datetime.datetime.utcnow +# Need our custom types, but don't import anything else from model +from galaxy.model.custom_types import * + +import sys, logging +log = logging.getLogger( __name__ ) +log.setLevel(logging.DEBUG) +handler = logging.StreamHandler( sys.stdout ) +format = "%(name)s %(levelname)s %(asctime)s %(message)s" +formatter = logging.Formatter( format ) +handler.setFormatter( formatter ) +log.addHandler( handler ) + +metadata = MetaData( migrate_engine ) +db_session = scoped_session( sessionmaker( bind=migrate_engine, autoflush=False, autocommit=True ) ) + +def upgrade(): + print __doc__ + + metadata.reflect() + # Create the table. + try: + cmd = "UPDATE migrate_tools set repository_path='lib/galaxy/tool_shed/migrate';" + db_session.execute( cmd ) + except Exception, e: + log.debug( "Updating migrate_tools.repository_path column to point to the new location lib/tool_shed/galaxy_install/migrate failed: %s" % str( e ) ) + +def downgrade(): + metadata.reflect() + try: + cmd = "UPDATE migrate_tools set repository_path='lib/galaxy/tool_shed/migrate';" + db_session.execute( cmd ) + except Exception, e: + log.debug( "Updating migrate_tools.repository_path column to point to the old location lib/galaxy/tool_shed/migrate failed: %s" % str( e ) ) diff -r e9e09d1ab51752db592eee71032ec5fc33f6ee57 -r 6f0050c4e06ef6f5fdce3f7326280c07c63ef62e lib/galaxy/tool_shed/__init__.py --- a/lib/galaxy/tool_shed/__init__.py +++ /dev/null @@ -1,57 +0,0 @@ -""" -Classes encapsulating the management of repositories installed from Galaxy tool sheds. -""" -import os -import galaxy.util.shed_util -import galaxy.util.shed_util_common -from galaxy.model.orm import and_ - -from galaxy import eggs -import pkg_resources - -pkg_resources.require( 'elementtree' ) -from elementtree import ElementTree, ElementInclude - -class InstalledRepositoryManager( object ): - def __init__( self, app ): - self.app = app - self.model = self.app.model - self.sa_session = self.model.context.current - self.tool_configs = self.app.config.tool_configs - if self.app.config.migrated_tools_config not in self.tool_configs: - self.tool_configs.append( self.app.config.migrated_tools_config ) - self.installed_repository_dicts = [] - def get_repository_install_dir( self, tool_shed_repository ): - for tool_config in self.tool_configs: - tree = ElementTree.parse( tool_config ) - root = tree.getroot() - ElementInclude.include( root ) - tool_path = root.get( 'tool_path', None ) - if tool_path: - tool_shed = galaxy.util.shed_util_common.clean_tool_shed_url( tool_shed_repository.tool_shed ) - relative_path = os.path.join( tool_path, - tool_shed, - 'repos', - tool_shed_repository.owner, - tool_shed_repository.name, - tool_shed_repository.installed_changeset_revision ) - if os.path.exists( relative_path ): - return relative_path - return None - def load_proprietary_datatypes( self ): - for tool_shed_repository in self.sa_session.query( self.model.ToolShedRepository ) \ - .filter( and_( self.model.ToolShedRepository.table.c.includes_datatypes==True, - self.model.ToolShedRepository.table.c.deleted==False ) ) \ - .order_by( self.model.ToolShedRepository.table.c.id ): - relative_install_dir = self.get_repository_install_dir( tool_shed_repository ) - if relative_install_dir: - installed_repository_dict = galaxy.util.shed_util.load_installed_datatypes( self.app, tool_shed_repository, relative_install_dir ) - if installed_repository_dict: - self.installed_repository_dicts.append( installed_repository_dict ) - def load_proprietary_converters_and_display_applications( self, deactivate=False ): - for installed_repository_dict in self.installed_repository_dicts: - if installed_repository_dict[ 'converter_path' ]: - galaxy.util.shed_util.load_installed_datatype_converters( self.app, installed_repository_dict, deactivate=deactivate ) - if installed_repository_dict[ 'display_path' ]: - galaxy.util.shed_util.load_installed_display_applications( self.app, installed_repository_dict, deactivate=deactivate ) - diff -r e9e09d1ab51752db592eee71032ec5fc33f6ee57 -r 6f0050c4e06ef6f5fdce3f7326280c07c63ef62e lib/galaxy/tool_shed/common_util.py --- a/lib/galaxy/tool_shed/common_util.py +++ /dev/null @@ -1,93 +0,0 @@ -import os, urllib2 -from galaxy import util -from galaxy.util.odict import odict -from galaxy.tool_shed import encoding_util - -REPOSITORY_OWNER = 'devteam' - -def check_for_missing_tools( app, tool_panel_configs, latest_tool_migration_script_number ): - # Get the 000x_tools.xml file associated with the current migrate_tools version number. - tools_xml_file_path = os.path.abspath( os.path.join( 'scripts', 'migrate_tools', '%04d_tools.xml' % latest_tool_migration_script_number ) ) - # Parse the XML and load the file attributes for later checking against the proprietary tool_panel_config. - migrated_tool_configs_dict = odict() - tree = util.parse_xml( tools_xml_file_path ) - root = tree.getroot() - tool_shed = root.get( 'name' ) - tool_shed_url = get_tool_shed_url_from_tools_xml_file_path( app, tool_shed ) - # The default behavior is that the tool shed is down. - tool_shed_accessible = False - if tool_shed_url: - for elem in root: - if elem.tag == 'repository': - tool_dependencies = [] - tool_dependencies_dict = {} - repository_name = elem.get( 'name' ) - changeset_revision = elem.get( 'changeset_revision' ) - url = '%s/repository/get_tool_dependencies?name=%s&owner=%s&changeset_revision=%s&from_install_manager=True' % \ - ( tool_shed_url, repository_name, REPOSITORY_OWNER, changeset_revision ) - try: - response = urllib2.urlopen( url ) - text = response.read() - response.close() - tool_shed_accessible = True - except Exception, e: - # Tool shed may be unavailable - we have to set tool_shed_accessible since we're looping. - tool_shed_accessible = False - print "The URL\n%s\nraised the exception:\n%s\n" % ( url, str( e ) ) - if tool_shed_accessible: - if text: - tool_dependencies_dict = encoding_util.tool_shed_decode( text ) - for dependency_key, requirements_dict in tool_dependencies_dict.items(): - tool_dependency_name = requirements_dict[ 'name' ] - tool_dependency_version = requirements_dict[ 'version' ] - tool_dependency_type = requirements_dict[ 'type' ] - tool_dependency_readme = requirements_dict.get( 'readme', '' ) - tool_dependencies.append( ( tool_dependency_name, tool_dependency_version, tool_dependency_type, tool_dependency_readme ) ) - for tool_elem in elem.findall( 'tool' ): - migrated_tool_configs_dict[ tool_elem.get( 'file' ) ] = tool_dependencies - if tool_shed_accessible: - # Parse the proprietary tool_panel_configs (the default is tool_conf.xml) and generate the list of missing tool config file names. - missing_tool_configs_dict = odict() - for tool_panel_config in tool_panel_configs: - tree = util.parse_xml( tool_panel_config ) - root = tree.getroot() - for elem in root: - if elem.tag == 'tool': - missing_tool_configs_dict = check_tool_tag_set( elem, migrated_tool_configs_dict, missing_tool_configs_dict ) - elif elem.tag == 'section': - for section_elem in elem: - if section_elem.tag == 'tool': - missing_tool_configs_dict = check_tool_tag_set( section_elem, migrated_tool_configs_dict, missing_tool_configs_dict ) - else: - exception_msg = '\n\nThe entry for the main Galaxy tool shed at %s is missing from the %s file. ' % ( tool_shed, app.config.tool_sheds_config ) - exception_msg += 'The entry for this tool shed must always be available in this file, so re-add it before attempting to start your Galaxy server.\n' - raise Exception( exception_msg ) - return tool_shed_accessible, missing_tool_configs_dict -def check_tool_tag_set( elem, migrated_tool_configs_dict, missing_tool_configs_dict ): - file_path = elem.get( 'file', None ) - if file_path: - path, name = os.path.split( file_path ) - if name in migrated_tool_configs_dict: - tool_dependencies = migrated_tool_configs_dict[ name ] - missing_tool_configs_dict[ name ] = tool_dependencies - return missing_tool_configs_dict -def get_non_shed_tool_panel_configs( app ): - # Get the non-shed related tool panel configs - there can be more than one, and the default is tool_conf.xml. - config_filenames = [] - for config_filename in app.config.tool_configs: - # Any config file that includes a tool_path attribute in the root tag set like the following is shed-related. - # <toolbox tool_path="../shed_tools"> - tree = util.parse_xml( config_filename ) - root = tree.getroot() - tool_path = root.get( 'tool_path', None ) - if tool_path is None: - config_filenames.append( config_filename ) - return config_filenames -def get_tool_shed_url_from_tools_xml_file_path( app, tool_shed ): - search_str = '://%s' % tool_shed - for shed_name, shed_url in app.tool_shed_registry.tool_sheds.items(): - if shed_url.find( search_str ) >= 0: - if shed_url.endswith( '/' ): - shed_url = shed_url.rstrip( '/' ) - return shed_url - return None diff -r e9e09d1ab51752db592eee71032ec5fc33f6ee57 -r 6f0050c4e06ef6f5fdce3f7326280c07c63ef62e lib/galaxy/tool_shed/encoding_util.py --- a/lib/galaxy/tool_shed/encoding_util.py +++ /dev/null @@ -1,44 +0,0 @@ -import binascii, logging -from galaxy.util.hash_util import hmac_new -from galaxy.util.json import json_fix - -from galaxy import eggs -import pkg_resources - -pkg_resources.require( "simplejson" ) -import simplejson - -log = logging.getLogger( __name__ ) - -encoding_sep = '__esep__' -encoding_sep2 = '__esepii__' - -def tool_shed_decode( value ): - # Extract and verify hash - a, b = value.split( ":" ) - value = binascii.unhexlify( b ) - test = hmac_new( 'ToolShedAndGalaxyMustHaveThisSameKey', value ) - assert a == test - # Restore from string - values = None - try: - values = simplejson.loads( value ) - except Exception, e: - log.debug( "Decoding json value from tool shed for value '%s' threw exception: %s" % ( str( value ), str( e ) ) ) - if values is not None: - try: - return json_fix( values ) - except Exception, e: - log.debug( "Fixing decoded json values '%s' from tool shed threw exception: %s" % ( str( values ), str( e ) ) ) - fixed_values = values - if values is None: - values = value - return values -def tool_shed_encode( val ): - if isinstance( val, dict ): - value = simplejson.dumps( val ) - else: - value = val - a = hmac_new( 'ToolShedAndGalaxyMustHaveThisSameKey', value ) - b = binascii.hexlify( value ) - return "%s:%s" % ( a, b ) \ No newline at end of file diff -r e9e09d1ab51752db592eee71032ec5fc33f6ee57 -r 6f0050c4e06ef6f5fdce3f7326280c07c63ef62e lib/galaxy/tool_shed/install_manager.py --- a/lib/galaxy/tool_shed/install_manager.py +++ /dev/null @@ -1,372 +0,0 @@ -""" -Manage automatic installation of tools configured in the xxx.xml files in ~/scripts/migrate_tools (e.g., 0002_tools.xml). -All of the tools were at some point included in the Galaxy distribution, but are now hosted in the main Galaxy tool shed. -""" -import os, urllib2, tempfile -from galaxy import util -from galaxy.tools import ToolSection -from galaxy.util.json import from_json_string, to_json_string -import galaxy.util.shed_util as shed_util -import galaxy.util.shed_util_common as suc -from galaxy.util.odict import odict -from galaxy.tool_shed import common_util - -class InstallManager( object ): - def __init__( self, app, latest_migration_script_number, tool_shed_install_config, migrated_tools_config, install_dependencies ): - """ - Check tool settings in tool_shed_install_config and install all repositories that are not already installed. The tool - panel configuration file is the received migrated_tools_config, which is the reserved file named migrated_tools_conf.xml. - """ - self.app = app - self.toolbox = self.app.toolbox - self.migrated_tools_config = migrated_tools_config - # If install_dependencies is True but tool_dependency_dir is not set, do not attempt to install but print informative error message. - if install_dependencies and app.config.tool_dependency_dir is None: - message = 'You are attempting to install tool dependencies but do not have a value for "tool_dependency_dir" set in your universe_wsgi.ini ' - message += 'file. Set this location value to the path where you want tool dependencies installed and rerun the migration script.' - raise Exception( message ) - # Get the local non-shed related tool panel configs (there can be more than one, and the default name is tool_conf.xml). - self.proprietary_tool_confs = self.non_shed_tool_panel_configs - self.proprietary_tool_panel_elems = self.get_proprietary_tool_panel_elems( latest_migration_script_number ) - # Set the location where the repositories will be installed by retrieving the tool_path setting from migrated_tools_config. - tree = util.parse_xml( migrated_tools_config ) - root = tree.getroot() - self.tool_path = root.get( 'tool_path' ) - print "Repositories will be installed into configured tool_path location ", str( self.tool_path ) - # Parse tool_shed_install_config to check each of the tools. - self.tool_shed_install_config = tool_shed_install_config - tree = util.parse_xml( tool_shed_install_config ) - root = tree.getroot() - self.tool_shed = suc.clean_tool_shed_url( root.get( 'name' ) ) - self.repository_owner = common_util.REPOSITORY_OWNER - index, self.shed_config_dict = suc.get_shed_tool_conf_dict( app, self.migrated_tools_config ) - # Since tool migration scripts can be executed any number of times, we need to make sure the appropriate tools are defined in - # tool_conf.xml. If no tools associated with the migration stage are defined, no repositories will be installed on disk. - # The default behavior is that the tool shed is down. - tool_shed_accessible = False - tool_panel_configs = common_util.get_non_shed_tool_panel_configs( app ) - if tool_panel_configs: - # The missing_tool_configs_dict contents are something like: - # {'emboss_antigenic.xml': [('emboss', '5.0.0', 'package', '\nreadme blah blah blah\n')]} - tool_shed_accessible, missing_tool_configs_dict = common_util.check_for_missing_tools( app, tool_panel_configs, latest_migration_script_number ) - else: - # It doesn't matter if the tool shed is accessible since there are no migrated tools defined in the local Galaxy instance, but - # we have to set the value of tool_shed_accessible to True so that the value of migrate_tools.version can be correctly set in - # the database. - tool_shed_accessible = True - missing_tool_configs_dict = odict() - if tool_shed_accessible: - if len( self.proprietary_tool_confs ) == 1: - plural = '' - file_names = self.proprietary_tool_confs[ 0 ] - else: - plural = 's' - file_names = ', '.join( self.proprietary_tool_confs ) - if missing_tool_configs_dict: - for repository_elem in root: - self.install_repository( repository_elem, install_dependencies ) - else: - message = "\nNo tools associated with migration stage %s are defined in your " % str( latest_migration_script_number ) - message += "file%s named %s,\nso no repositories will be installed on disk.\n" % ( plural, file_names ) - print message - else: - message = "\nThe main Galaxy tool shed is not currently available, so skipped migration stage %s.\n" % str( latest_migration_script_number ) - message += "Try again later.\n" - print message - def get_guid( self, repository_clone_url, relative_install_dir, tool_config ): - if self.shed_config_dict.get( 'tool_path' ): - relative_install_dir = os.path.join( self.shed_config_dict['tool_path'], relative_install_dir ) - found = False - for root, dirs, files in os.walk( relative_install_dir ): - if root.find( '.hg' ) < 0 and root.find( 'hgrc' ) < 0: - if '.hg' in dirs: - dirs.remove( '.hg' ) - for name in files: - if name == tool_config: - found = True - break - if found: - break - full_path = str( os.path.abspath( os.path.join( root, name ) ) ) - tool = self.toolbox.load_tool( full_path ) - return suc.generate_tool_guid( repository_clone_url, tool ) - def get_proprietary_tool_panel_elems( self, latest_tool_migration_script_number ): - # Parse each config in self.proprietary_tool_confs (the default is tool_conf.xml) and generate a list of Elements that are - # either ToolSection elements or Tool elements. These will be used to generate new entries in the migrated_tools_conf.xml - # file for the installed tools. - tools_xml_file_path = os.path.abspath( os.path.join( 'scripts', 'migrate_tools', '%04d_tools.xml' % latest_tool_migration_script_number ) ) - # Parse the XML and load the file attributes for later checking against the integrated elements from self.proprietary_tool_confs. - migrated_tool_configs = [] - tree = util.parse_xml( tools_xml_file_path ) - root = tree.getroot() - for elem in root: - if elem.tag == 'repository': - for tool_elem in elem: - migrated_tool_configs.append( tool_elem.get( 'file' ) ) - # Parse each file in self.proprietary_tool_confs and generate the integrated list of tool panel Elements that contain them. - tool_panel_elems = [] - for proprietary_tool_conf in self.proprietary_tool_confs: - tree = util.parse_xml( proprietary_tool_conf ) - root = tree.getroot() - for elem in root: - if elem.tag == 'tool': - # Tools outside of sections. - file_path = elem.get( 'file', None ) - if file_path: - name = suc.strip_path( file_path ) - if name in migrated_tool_configs: - if elem not in tool_panel_elems: - tool_panel_elems.append( elem ) - elif elem.tag == 'section': - # Tools contained in a section. - for section_elem in elem: - if section_elem.tag == 'tool': - file_path = section_elem.get( 'file', None ) - if file_path: - name = suc.strip_path( file_path ) - if name in migrated_tool_configs: - # Append the section, not the tool. - if elem not in tool_panel_elems: - tool_panel_elems.append( elem ) - return tool_panel_elems - def get_containing_tool_sections( self, tool_config ): - """ - If tool_config is defined somewhere in self.proprietary_tool_panel_elems, return True and a list of ToolSections in which the - tool is displayed. If the tool is displayed outside of any sections, None is appended to the list. - """ - tool_sections = [] - is_displayed = False - for proprietary_tool_panel_elem in self.proprietary_tool_panel_elems: - if proprietary_tool_panel_elem.tag == 'tool': - # The proprietary_tool_panel_elem looks something like <tool file="emboss_5/emboss_antigenic.xml" />. - proprietary_tool_config = proprietary_tool_panel_elem.get( 'file' ) - proprietary_name = suc.strip_path( proprietary_tool_config ) - if tool_config == proprietary_name: - # The tool is loaded outside of any sections. - tool_sections.append( None ) - if not is_displayed: - is_displayed = True - if proprietary_tool_panel_elem.tag == 'section': - # The proprietary_tool_panel_elem looks something like <section name="EMBOSS" id="EMBOSSLite">. - for section_elem in proprietary_tool_panel_elem: - if section_elem.tag == 'tool': - # The section_elem looks something like <tool file="emboss_5/emboss_antigenic.xml" />. - proprietary_tool_config = section_elem.get( 'file' ) - proprietary_name = suc.strip_path( proprietary_tool_config ) - if tool_config == proprietary_name: - # The tool is loaded inside of the section_elem. - tool_sections.append( ToolSection( proprietary_tool_panel_elem ) ) - if not is_displayed: - is_displayed = True - return is_displayed, tool_sections - def handle_repository_contents( self, tool_shed_repository, repository_clone_url, relative_install_dir, repository_elem, install_dependencies ): - """Generate the metadata for the installed tool shed repository, among other things.""" - tool_panel_dict_for_display = odict() - if self.tool_path: - repo_install_dir = os.path.join( self.tool_path, relative_install_dir ) - else: - repo_install_dir = relative_install_dir - for tool_elem in repository_elem: - # The tool_elem looks something like this: <tool id="EMBOSS: antigenic1" version="5.0.0" file="emboss_antigenic.xml" /> - tool_config = tool_elem.get( 'file' ) - guid = self.get_guid( repository_clone_url, relative_install_dir, tool_config ) - # See if tool_config is defined inside of a section in self.proprietary_tool_panel_elems. - is_displayed, tool_sections = self.get_containing_tool_sections( tool_config ) - if is_displayed: - tool_panel_dict_for_tool_config = shed_util.generate_tool_panel_dict_for_tool_config( guid, tool_config, tool_sections=tool_sections ) - for k, v in tool_panel_dict_for_tool_config.items(): - tool_panel_dict_for_display[ k ] = v - else: - print 'The tool "%s" (%s) has not been enabled because it is not defined in a proprietary tool config (%s).' \ - % ( guid, tool_config, ", ".join( self.proprietary_tool_confs or [] ) ) - metadata_dict, invalid_file_tups = suc.generate_metadata_for_changeset_revision( app=self.app, - repository=tool_shed_repository, - changeset_revision=tool_shed_repository.changeset_revision, - repository_clone_url=repository_clone_url, - shed_config_dict = self.shed_config_dict, - relative_install_dir=relative_install_dir, - repository_files_dir=None, - resetting_all_metadata_on_repository=False, - updating_installed_repository=False, - persist=True ) - tool_shed_repository.metadata = metadata_dict - self.app.sa_session.add( tool_shed_repository ) - self.app.sa_session.flush() - if 'tool_dependencies' in metadata_dict: - # All tool_dependency objects must be created before the tools are processed even if no tool dependencies will be installed. - tool_dependencies = shed_util.create_tool_dependency_objects( self.app, tool_shed_repository, relative_install_dir, set_status=True ) - else: - tool_dependencies = None - if 'tools' in metadata_dict: - sample_files = metadata_dict.get( 'sample_files', [] ) - sample_files = [ str( s ) for s in sample_files ] - tool_index_sample_files = shed_util.get_tool_index_sample_files( sample_files ) - shed_util.copy_sample_files( self.app, tool_index_sample_files, tool_path=self.tool_path ) - sample_files_copied = [ s for s in tool_index_sample_files ] - repository_tools_tups = suc.get_repository_tools_tups( self.app, metadata_dict ) - if repository_tools_tups: - # Handle missing data table entries for tool parameters that are dynamically generated select lists. - repository_tools_tups = shed_util.handle_missing_data_table_entry( self.app, relative_install_dir, self.tool_path, repository_tools_tups ) - # Handle missing index files for tool parameters that are dynamically generated select lists. - repository_tools_tups, sample_files_copied = shed_util.handle_missing_index_file( self.app, - self.tool_path, - sample_files, - repository_tools_tups, - sample_files_copied ) - # Copy remaining sample files included in the repository to the ~/tool-data directory of the local Galaxy instance. - shed_util.copy_sample_files( self.app, sample_files, tool_path=self.tool_path, sample_files_copied=sample_files_copied ) - if install_dependencies and tool_dependencies and 'tool_dependencies' in metadata_dict: - # Install tool dependencies. - shed_util.update_tool_shed_repository_status( self.app, - tool_shed_repository, - self.app.model.ToolShedRepository.installation_status.INSTALLING_TOOL_DEPENDENCIES ) - # Get the tool_dependencies.xml file from disk. - tool_dependencies_config = suc.get_config_from_disk( 'tool_dependencies.xml', repo_install_dir ) - installed_tool_dependencies = shed_util.handle_tool_dependencies( app=self.app, - tool_shed_repository=tool_shed_repository, - tool_dependencies_config=tool_dependencies_config, - tool_dependencies=tool_dependencies ) - for installed_tool_dependency in installed_tool_dependencies: - if installed_tool_dependency.status == self.app.model.ToolDependency.installation_status.ERROR: - print '\nThe following error occurred from the InstallManager while installing tool dependency ', installed_tool_dependency.name, ':' - print installed_tool_dependency.error_message, '\n\n' - shed_util.add_to_tool_panel( self.app, - tool_shed_repository.name, - repository_clone_url, - tool_shed_repository.installed_changeset_revision, - repository_tools_tups, - self.repository_owner, - self.migrated_tools_config, - tool_panel_dict=tool_panel_dict_for_display, - new_install=True ) - if 'datatypes' in metadata_dict: - tool_shed_repository.status = self.app.model.ToolShedRepository.installation_status.LOADING_PROPRIETARY_DATATYPES - if not tool_shed_repository.includes_datatypes: - tool_shed_repository.includes_datatypes = True - self.app.sa_session.add( tool_shed_repository ) - self.app.sa_session.flush() - work_dir = tempfile.mkdtemp() - datatypes_config = suc.get_config_from_disk( 'datatypes_conf.xml', repo_install_dir ) - # Load proprietary data types required by tools. The value of override is not important here since the Galaxy server will be started - # after this installation completes. - converter_path, display_path = shed_util.alter_config_and_load_prorietary_datatypes( self.app, datatypes_config, repo_install_dir, override=False ) #repo_install_dir was relative_install_dir - if converter_path or display_path: - # Create a dictionary of tool shed repository related information. - repository_dict = shed_util.create_repository_dict_for_proprietary_datatypes( tool_shed=self.tool_shed, - name=tool_shed_repository.name, - owner=self.repository_owner, - installed_changeset_revision=tool_shed_repository.installed_changeset_revision, - tool_dicts=metadata_dict.get( 'tools', [] ), - converter_path=converter_path, - display_path=display_path ) - if converter_path: - # Load proprietary datatype converters - self.app.datatypes_registry.load_datatype_converters( self.toolbox, installed_repository_dict=repository_dict ) - if display_path: - # Load proprietary datatype display applications - self.app.datatypes_registry.load_display_applications( installed_repository_dict=repository_dict ) - try: - shutil.rmtree( work_dir ) - except: - pass - def install_repository( self, repository_elem, install_dependencies ): - # Install a single repository, loading contained tools into the tool panel. - name = repository_elem.get( 'name' ) - description = repository_elem.get( 'description' ) - installed_changeset_revision = repository_elem.get( 'changeset_revision' ) - # Install path is of the form: <tool path>/<tool shed>/repos/<repository owner>/<repository name>/<installed changeset revision> - relative_clone_dir = os.path.join( self.tool_shed, 'repos', self.repository_owner, name, installed_changeset_revision ) - clone_dir = os.path.join( self.tool_path, relative_clone_dir ) - if self.__isinstalled( clone_dir ): - print "Skipping automatic install of repository '", name, "' because it has already been installed in location ", clone_dir - else: - tool_shed_url = self.__get_url_from_tool_shed( self.tool_shed ) - repository_clone_url = os.path.join( tool_shed_url, 'repos', self.repository_owner, name ) - relative_install_dir = os.path.join( relative_clone_dir, name ) - install_dir = os.path.join( clone_dir, name ) - ctx_rev = suc.get_ctx_rev( tool_shed_url, name, self.repository_owner, installed_changeset_revision ) - tool_shed_repository = suc.create_or_update_tool_shed_repository( app=self.app, - name=name, - description=description, - installed_changeset_revision=installed_changeset_revision, - ctx_rev=ctx_rev, - repository_clone_url=repository_clone_url, - metadata_dict={}, - status=self.app.model.ToolShedRepository.installation_status.NEW, - current_changeset_revision=None, - owner=self.repository_owner, - dist_to_shed=True ) - shed_util.update_tool_shed_repository_status( self.app, tool_shed_repository, self.app.model.ToolShedRepository.installation_status.CLONING ) - cloned_ok, error_message = suc.clone_repository( repository_clone_url, os.path.abspath( install_dir ), ctx_rev ) - if cloned_ok: - self.handle_repository_contents( tool_shed_repository=tool_shed_repository, - repository_clone_url=repository_clone_url, - relative_install_dir=relative_install_dir, - repository_elem=repository_elem, - install_dependencies=install_dependencies ) - self.app.sa_session.refresh( tool_shed_repository ) - metadata_dict = tool_shed_repository.metadata - if 'tools' in metadata_dict: - shed_util.update_tool_shed_repository_status( self.app, - tool_shed_repository, - self.app.model.ToolShedRepository.installation_status.SETTING_TOOL_VERSIONS ) - # Get the tool_versions from the tool shed for each tool in the installed change set. - url = '%s/repository/get_tool_versions?name=%s&owner=%s&changeset_revision=%s' % \ - ( tool_shed_url, tool_shed_repository.name, self.repository_owner, installed_changeset_revision ) - response = urllib2.urlopen( url ) - text = response.read() - response.close() - if text: - tool_version_dicts = from_json_string( text ) - shed_util.handle_tool_versions( self.app, tool_version_dicts, tool_shed_repository ) - else: - # Set the tool versions since they seem to be missing for this repository in the tool shed. - # CRITICAL NOTE: These default settings may not properly handle all parent/child associations. - for tool_dict in metadata_dict[ 'tools' ]: - flush_needed = False - tool_id = tool_dict[ 'guid' ] - old_tool_id = tool_dict[ 'id' ] - tool_version = tool_dict[ 'version' ] - tool_version_using_old_id = shed_util.get_tool_version( self.app, old_tool_id ) - tool_version_using_guid = shed_util.get_tool_version( self.app, tool_id ) - if not tool_version_using_old_id: - tool_version_using_old_id = self.app.model.ToolVersion( tool_id=old_tool_id, - tool_shed_repository=tool_shed_repository ) - self.app.sa_session.add( tool_version_using_old_id ) - self.app.sa_session.flush() - if not tool_version_using_guid: - tool_version_using_guid = self.app.model.ToolVersion( tool_id=tool_id, - tool_shed_repository=tool_shed_repository ) - self.app.sa_session.add( tool_version_using_guid ) - self.app.sa_session.flush() - # Associate the two versions as parent / child. - tool_version_association = shed_util.get_tool_version_association( self.app, - tool_version_using_old_id, - tool_version_using_guid ) - if not tool_version_association: - tool_version_association = self.app.model.ToolVersionAssociation( tool_id=tool_version_using_guid.id, - parent_id=tool_version_using_old_id.id ) - self.app.sa_session.add( tool_version_association ) - self.app.sa_session.flush() - shed_util.update_tool_shed_repository_status( self.app, tool_shed_repository, self.app.model.ToolShedRepository.installation_status.INSTALLED ) - @property - def non_shed_tool_panel_configs( self ): - return common_util.get_non_shed_tool_panel_configs( self.app ) - def __get_url_from_tool_shed( self, tool_shed ): - # The value of tool_shed is something like: toolshed.g2.bx.psu.edu. We need the URL to this tool shed, which is something like: - # http://toolshed.g2.bx.psu.edu/ - for shed_name, shed_url in self.app.tool_shed_registry.tool_sheds.items(): - if shed_url.find( tool_shed ) >= 0: - if shed_url.endswith( '/' ): - shed_url = shed_url.rstrip( '/' ) - return shed_url - # The tool shed from which the repository was originally installed must no longer be configured in tool_sheds_conf.xml. - return None - def __isinstalled( self, clone_dir ): - full_path = os.path.abspath( clone_dir ) - if os.path.exists( full_path ): - for root, dirs, files in os.walk( full_path ): - if '.hg' in dirs: - # Assume that the repository has been installed if we find a .hg directory. - return True - return False diff -r e9e09d1ab51752db592eee71032ec5fc33f6ee57 -r 6f0050c4e06ef6f5fdce3f7326280c07c63ef62e lib/galaxy/tool_shed/migrate/check.py --- a/lib/galaxy/tool_shed/migrate/check.py +++ /dev/null @@ -1,171 +0,0 @@ -import sys, os, logging, subprocess -from galaxy import eggs -import pkg_resources -pkg_resources.require( "sqlalchemy-migrate" ) - -from migrate.versioning import repository, schema -from sqlalchemy import * -from galaxy.util.odict import odict -from galaxy.tool_shed import common_util - -log = logging.getLogger( __name__ ) - -# Path relative to galaxy -migrate_repository_directory = os.path.dirname( __file__ ).replace( os.getcwd() + os.path.sep, '', 1 ) -migrate_repository = repository.Repository( migrate_repository_directory ) -dialect_to_egg = { - "sqlite" : "pysqlite>=2", - "postgres" : "psycopg2", - "mysql" : "MySQL_python" -} - -def verify_tools( app, url, galaxy_config_file, engine_options={} ): - # Check the value in the migrate_tools.version database table column to verify that the number is in - # sync with the number of version scripts in ~/lib/galaxy/tools/migrate/versions. - dialect = ( url.split( ':', 1 ) )[0] - try: - egg = dialect_to_egg[ dialect ] - try: - pkg_resources.require( egg ) - log.debug( "%s egg successfully loaded for %s dialect" % ( egg, dialect ) ) - except: - # If the module is in the path elsewhere (i.e. non-egg), it'll still load. - log.warning( "%s egg not found, but an attempt will be made to use %s anyway" % ( egg, dialect ) ) - except KeyError: - # Let this go, it could possibly work with db's we don't support - log.error( "database_connection contains an unknown SQLAlchemy database dialect: %s" % dialect ) - # Create engine and metadata - engine = create_engine( url, **engine_options ) - meta = MetaData( bind=engine ) - # The migrate_tools table was created in database version script 0092_add_migrate_tools_table.py. - version_table = Table( "migrate_tools", meta, autoload=True ) - # Verify that the code and the database are in sync. - db_schema = schema.ControlledSchema( engine, migrate_repository ) - latest_tool_migration_script_number = migrate_repository.versions.latest - if latest_tool_migration_script_number != db_schema.version: - # The default behavior is that the tool shed is down. - tool_shed_accessible = False - if app.new_installation: - # New installations will not be missing tools, so we don't need to worry about them. - missing_tool_configs_dict = odict() - else: - tool_panel_configs = common_util.get_non_shed_tool_panel_configs( app ) - if tool_panel_configs: - # The missing_tool_configs_dict contents are something like: - # {'emboss_antigenic.xml': [('emboss', '5.0.0', 'package', '\nreadme blah blah blah\n')]} - tool_shed_accessible, missing_tool_configs_dict = common_util.check_for_missing_tools( app, tool_panel_configs, latest_tool_migration_script_number ) - else: - # It doesn't matter if the tool shed is accessible since there are no migrated tools defined in the local Galaxy instance, but - # we have to set the value of tool_shed_accessible to True so that the value of migrate_tools.version can be correctly set in - # the database. - tool_shed_accessible = True - missing_tool_configs_dict = odict() - have_tool_dependencies = False - for k, v in missing_tool_configs_dict.items(): - if v: - have_tool_dependencies = True - break - config_arg = '' - if os.path.abspath( os.path.join( os.getcwd(), 'universe_wsgi.ini' ) ) != galaxy_config_file: - config_arg = ' -c %s' % galaxy_config_file.replace( os.path.abspath( os.getcwd() ), '.' ) - if not app.config.running_functional_tests: - if tool_shed_accessible: - # Automatically update the value of the migrate_tools.version database table column. - cmd = 'sh manage_tools.sh%s upgrade' % config_arg - proc = subprocess.Popen( args=cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT ) - return_code = proc.wait() - output = proc.stdout.read( 32768 ) - if return_code != 0: - raise Exception( "Error attempting to update the value of migrate_tools.version: %s" % output ) - elif missing_tool_configs_dict: - if len( tool_panel_configs ) == 1: - plural = '' - tool_panel_config_file_names = tool_panel_configs[ 0 ] - else: - plural = 's' - tool_panel_config_file_names = ', '.join( tool_panel_configs ) - msg = "\n>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>" - msg += "\n\nThe list of files at the end of this message refers to tools that are configured to load into the tool panel for\n" - msg += "this Galaxy instance, but have been removed from the Galaxy distribution. These tools and their dependencies can be\n" - msg += "automatically installed from the Galaxy tool shed at http://toolshed.g2.bx.psu.edu.\n\n" - msg += "To skip this process, attempt to start your Galaxy server again (e.g., sh run.sh or whatever you use). If you do this,\n" - msg += "be aware that these tools will no longer be available in your Galaxy tool panel, and entries for each of them should\n" - msg += "be removed from your file%s named %s.\n\n" % ( plural, tool_panel_config_file_names ) - msg += "CRITICAL NOTE IF YOU PLAN TO INSTALL\n" - msg += "The location in which the tool repositories will be installed is the value of the 'tool_path' attribute in the <tool>\n" - msg += 'tag of the file named ./migrated_tool_conf.xml (i.e., <toolbox tool_path="../shed_tools">). The default location\n' - msg += "setting is '../shed_tools', which may be problematic for some cluster environments, so make sure to change it before\n" - msg += "you execute the installation process if appropriate. The configured location must be outside of the Galaxy installation\n" - msg += "directory or it must be in a sub-directory protected by a properly configured .hgignore file if the directory is within\n" - msg += "the Galaxy installation directory hierarchy. This is because tool shed repositories will be installed using mercurial's\n" - msg += "clone feature, which creates .hg directories and associated mercurial repository files. Not having .hgignore properly\n" - msg += "configured could result in undesired behavior when modifying or updating your local Galaxy instance or the tool shed\n" - msg += "repositories if they are in directories that pose conflicts. See mercurial's .hgignore documentation at the following\n" - msg += "URL for details.\n\nhttp://mercurial.selenic.com/wiki/.hgignore\n\n" - if have_tool_dependencies: - msg += "The following tool dependencies can also optionally be installed (see the option flag in the command below). If you\n" - msg += "choose to install them (recommended), they will be installed within the location specified by the 'tool_dependency_dir'\n" - msg += "setting in your main Galaxy configuration file (e.g., uninverse_wsgi.ini).\n" - processed_tool_dependencies = [] - for missing_tool_config, tool_dependencies in missing_tool_configs_dict.items(): - for tool_dependencies_tup in tool_dependencies: - if tool_dependencies_tup not in processed_tool_dependencies: - msg += "------------------------------------\n" - msg += "Tool Dependency\n" - msg += "------------------------------------\n" - msg += "Name: %s, Version: %s, Type: %s\n" % ( tool_dependencies_tup[ 0 ], - tool_dependencies_tup[ 1 ], - tool_dependencies_tup[ 2 ] ) - if tool_dependencies_tup[ 3 ]: - msg += "Requirements and installation information:\n" - msg += "%s\n" % tool_dependencies_tup[ 3 ] - else: - msg += "\n" - msg += "------------------------------------\n" - processed_tool_dependencies.append( tool_dependencies_tup ) - msg += "\n" - msg += "%s" % output.replace( 'done', '' ) - msg += "vvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvv\n" - msg += "sh ./scripts/migrate_tools/%04d_tools.sh\n" % latest_tool_migration_script_number - msg += "^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n\n" - if have_tool_dependencies: - msg += "The tool dependencies listed above will be installed along with the repositories if you add the 'install_dependencies'\n" - msg += "option to the above command like this:\n\n" - msg += "vvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvv\n" - msg += "sh ./scripts/migrate_tools/%04d_tools.sh install_dependencies\n" % latest_tool_migration_script_number - msg += "^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n\n" - msg += "Tool dependencies can be installed after the repositories have been installed as well.\n\n" - msg += "After the installation process finishes, you can start your Galaxy server. As part of this installation process,\n" - msg += "entries for each of the following tool config files will be added to the file named ./migrated_tool_conf.xml, so these\n" - msg += "tools will continue to be loaded into your tool panel. Because of this, existing entries for these files should be\n" - msg += "removed from your file%s named %s, but only after the installation process finishes.\n\n" % ( plural, tool_panel_config_file_names ) - for missing_tool_config, tool_dependencies in missing_tool_configs_dict.items(): - msg += "%s\n" % missing_tool_config - msg += "<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<\n" - raise Exception( msg ) - else: - log.debug( "The main Galaxy tool shed is not currently available, so skipped tool migration %s until next server startup" % db_schema.version ) - else: - log.info( "At migrate_tools version %d" % db_schema.version ) - -def migrate_to_current_version( engine, schema ): - # Changes to get to current version. - changeset = schema.changeset( None ) - for ver, change in changeset: - nextver = ver + changeset.step - log.info( 'Installing tools from version %s -> %s... ' % ( ver, nextver ) ) - old_stdout = sys.stdout - class FakeStdout( object ): - def __init__( self ): - self.buffer = [] - def write( self, s ): - self.buffer.append( s ) - def flush( self ): - pass - sys.stdout = FakeStdout() - try: - schema.runchange( ver, change, changeset.step ) - finally: - for message in "".join( sys.stdout.buffer ).split( "\n" ): - log.info( message ) - sys.stdout = old_stdout diff -r e9e09d1ab51752db592eee71032ec5fc33f6ee57 -r 6f0050c4e06ef6f5fdce3f7326280c07c63ef62e lib/galaxy/tool_shed/migrate/common.py --- a/lib/galaxy/tool_shed/migrate/common.py +++ /dev/null @@ -1,81 +0,0 @@ -import sys, os, ConfigParser -import galaxy.config -import galaxy.datatypes.registry -from galaxy import tools -from galaxy.tools.data import * -import galaxy.model.mapping -import galaxy.tools.search -from galaxy.objectstore import build_object_store_from_config -from galaxy.tool_shed.common_util import * -import galaxy.tool_shed.tool_shed_registry -from galaxy.tool_shed import install_manager - -class MigrateToolsApplication( object ): - """Encapsulates the state of a basic Galaxy Universe application in order to initiate the Install Manager""" - def __init__( self, tools_migration_config ): - install_dependencies = 'install_dependencies' in sys.argv - galaxy_config_file = 'universe_wsgi.ini' - if '-c' in sys.argv: - pos = sys.argv.index( '-c' ) - sys.argv.pop( pos ) - galaxy_config_file = sys.argv.pop( pos ) - if not os.path.exists( galaxy_config_file ): - print "Galaxy config file does not exist (hint: use '-c config.ini' for non-standard locations): %s" % galaxy_config_file - sys.exit( 1 ) - config_parser = ConfigParser.ConfigParser( { 'here':os.getcwd() } ) - config_parser.read( galaxy_config_file ) - galaxy_config_dict = {} - for key, value in config_parser.items( "app:main" ): - galaxy_config_dict[ key ] = value - self.config = galaxy.config.Configuration( **galaxy_config_dict ) - if not self.config.database_connection: - self.config.database_connection = "sqlite:///%s?isolation_level=IMMEDIATE" % self.config.database - self.config.update_integrated_tool_panel = True - self.object_store = build_object_store_from_config( self.config ) - # Setup the database engine and ORM - self.model = galaxy.model.mapping.init( self.config.file_path, - self.config.database_connection, - engine_options={}, - create_tables=False, - object_store=self.object_store ) - # Create an empty datatypes registry. - self.datatypes_registry = galaxy.datatypes.registry.Registry() - # Load the data types in the Galaxy distribution, which are defined in self.config.datatypes_config. - self.datatypes_registry.load_datatypes( self.config.root, self.config.datatypes_config ) - # Initialize tool data tables using the config defined by self.config.tool_data_table_config_path. - self.tool_data_tables = ToolDataTableManager( tool_data_path=self.config.tool_data_path, - config_filename=self.config.tool_data_table_config_path ) - # Load additional entries defined by self.config.shed_tool_data_table_config into tool data tables. - self.tool_data_tables.load_from_config_file( config_filename=self.config.shed_tool_data_table_config, - tool_data_path=self.tool_data_tables.tool_data_path, - from_shed_config=True ) - # Initialize the tools, making sure the list of tool configs includes the reserved migrated_tools_conf.xml file. - tool_configs = self.config.tool_configs - if self.config.migrated_tools_config not in tool_configs: - tool_configs.append( self.config.migrated_tools_config ) - self.toolbox = tools.ToolBox( tool_configs, self.config.tool_path, self ) - # Search support for tools - self.toolbox_search = galaxy.tools.search.ToolBoxSearch( self.toolbox ) - # Set up the tool sheds registry. - if os.path.isfile( self.config.tool_sheds_config ): - self.tool_shed_registry = galaxy.tool_shed.tool_shed_registry.Registry( self.config.root, self.config.tool_sheds_config ) - else: - self.tool_shed_registry = None - # Get the latest tool migration script number to send to the Install manager. - latest_migration_script_number = int( tools_migration_config.split( '_' )[ 0 ] ) - # The value of migrated_tools_config is migrated_tools_conf.xml, and is reserved for containing only those tools that have been - # eliminated from the distribution and moved to the tool shed. A side-effect of instantiating the InstallManager is the automatic - # installation of all appropriate tool shed repositories. - self.install_manager = install_manager.InstallManager( app=self, - latest_migration_script_number=latest_migration_script_number, - tool_shed_install_config=os.path.join( self.config.root, - 'scripts', - 'migrate_tools', - tools_migration_config ), - migrated_tools_config=self.config.migrated_tools_config, - install_dependencies=install_dependencies ) - @property - def sa_session( self ): - return self.model.context.current - def shutdown( self ): - self.object_store.shutdown() diff -r e9e09d1ab51752db592eee71032ec5fc33f6ee57 -r 6f0050c4e06ef6f5fdce3f7326280c07c63ef62e lib/galaxy/tool_shed/migrate/migrate.cfg --- a/lib/galaxy/tool_shed/migrate/migrate.cfg +++ /dev/null @@ -1,20 +0,0 @@ -[db_settings] -# Used to identify which repository this database is versioned under. -# You can use the name of your project. -repository_id=GalaxyTools - -# The name of the database table used to track the schema version. -# This name shouldn't already be used by your project. -# If this is changed once a database is under version control, you'll need to -# change the table name in each database too. -version_table=migrate_tools - -# When committing a change script, Migrate will attempt to generate the -# sql for all supported databases; normally, if one of them fails - probably -# because you don't have that database installed - it is ignored and the -# commit continues, perhaps ending successfully. -# Databases in this list MUST compile successfully during a commit, or the -# entire commit will fail. List the databases your application will actually -# be using to ensure your updates to that database work properly. -# This must be a list; example: ['postgres','sqlite'] -required_dbs=[] \ No newline at end of file diff -r e9e09d1ab51752db592eee71032ec5fc33f6ee57 -r 6f0050c4e06ef6f5fdce3f7326280c07c63ef62e lib/galaxy/tool_shed/migrate/versions/0001_tools.py --- a/lib/galaxy/tool_shed/migrate/versions/0001_tools.py +++ /dev/null @@ -1,9 +0,0 @@ -""" -Initialize the version column of the migrate_tools database table to 1. No tool migrations are handled in this version. -""" -import sys - -def upgrade(): - print __doc__ -def downgrade(): - pass diff -r e9e09d1ab51752db592eee71032ec5fc33f6ee57 -r 6f0050c4e06ef6f5fdce3f7326280c07c63ef62e lib/galaxy/tool_shed/migrate/versions/0002_tools.py --- a/lib/galaxy/tool_shed/migrate/versions/0002_tools.py +++ /dev/null @@ -1,13 +0,0 @@ -""" -The Emboss 5.0.0 tools have been eliminated from the distribution and the Emboss datatypes have been removed from -datatypes_conf.xml.sample. You should remove the Emboss datatypes from your version of datatypes_conf.xml. The -repositories named emboss_5 and emboss_datatypes from the main Galaxy tool shed at http://toolshed.g2.bx.psu.edu -will be installed into your local Galaxy instance at the location discussed above by running the following command. -""" - -import sys - -def upgrade(): - print __doc__ -def downgrade(): - pass diff -r e9e09d1ab51752db592eee71032ec5fc33f6ee57 -r 6f0050c4e06ef6f5fdce3f7326280c07c63ef62e lib/galaxy/tool_shed/migrate/versions/0003_tools.py --- a/lib/galaxy/tool_shed/migrate/versions/0003_tools.py +++ /dev/null @@ -1,12 +0,0 @@ -""" -The freebayes tool has been eliminated from the distribution . The repository named freebayes from the main -Galaxy tool shed at http://toolshed.g2.bx.psu.edu will be installed into your local Galaxy instance at the -location discussed above by running the following command. -""" - -import sys - -def upgrade(): - print __doc__ -def downgrade(): - pass diff -r e9e09d1ab51752db592eee71032ec5fc33f6ee57 -r 6f0050c4e06ef6f5fdce3f7326280c07c63ef62e lib/galaxy/tool_shed/migrate/versions/0004_tools.py --- a/lib/galaxy/tool_shed/migrate/versions/0004_tools.py +++ /dev/null @@ -1,14 +0,0 @@ -""" -The NCBI BLAST+ tools have been eliminated from the distribution. The tools and -datatypes are now available in repositories named ncbi_blast_plus and -blast_datatypes, in the main Galaxy tool shed at http://toolshed.g2.bx.psu.edu. -These repositories will be installed into your local Galaxy instance at the -location discussed above by running the following command. -""" - -import sys - -def upgrade(): - print __doc__ -def downgrade(): - pass diff -r e9e09d1ab51752db592eee71032ec5fc33f6ee57 -r 6f0050c4e06ef6f5fdce3f7326280c07c63ef62e lib/galaxy/tool_shed/migrate/versions/0005_tools.py --- a/lib/galaxy/tool_shed/migrate/versions/0005_tools.py +++ /dev/null @@ -1,15 +0,0 @@ -""" -The tools "Map with BWA for Illumina" and "Map with BWA for SOLiD" have -been eliminated from the distribution. The tools are now available -in the repository named bwa_wrappers from the main Galaxy tool shed at -http://toolshed.g2.bx.psu.edu, and will be installed into your local -Galaxy instance at the location discussed above by running the following -command. -""" - -import sys - -def upgrade(): - print __doc__ -def downgrade(): - pass diff -r e9e09d1ab51752db592eee71032ec5fc33f6ee57 -r 6f0050c4e06ef6f5fdce3f7326280c07c63ef62e lib/galaxy/tool_shed/migrate/versions/0006_tools.py --- a/lib/galaxy/tool_shed/migrate/versions/0006_tools.py +++ /dev/null @@ -1,20 +0,0 @@ -""" -The following tools have been eliminated from the distribution: -FASTQ to BAM, SAM to FASTQ, BAM Index Statistics, Estimate Library -Complexity, Insertion size metrics for PAIRED data, SAM/BAM Hybrid -Selection Metrics, bam/sam Cleaning, Add or Replace Groups, Replace -SAM/BAM Header, Paired Read Mate Fixer, Mark Duplicate reads, -SAM/BAM Alignment Summary Metrics, SAM/BAM GC Bias Metrics, and -Reorder SAM/BAM. The tools are now available in the repository -named picard from the main Galaxy tool shed at -http://toolshed.g2.bx.psu.edu, and will be installed into your -local Galaxy instance at the location discussed above by running -the following command. -""" - -import sys - -def upgrade(): - print __doc__ -def downgrade(): - pass diff -r e9e09d1ab51752db592eee71032ec5fc33f6ee57 -r 6f0050c4e06ef6f5fdce3f7326280c07c63ef62e lib/galaxy/tool_shed/migrate/versions/0007_tools.py --- a/lib/galaxy/tool_shed/migrate/versions/0007_tools.py +++ /dev/null @@ -1,17 +0,0 @@ -""" -The following tools have been eliminated from the distribution: -Map with Bowtie for Illumina, Map with Bowtie for SOLiD, Lastz, -and Lastz paired reads. The tools are now available in the -repositories named bowtie_wrappers, bowtie_color_wrappers, lastz, -and lastz_paired_reads from the main Galaxy tool shed at -http://toolshed.g2.bx.psu.edu, and will be installed into your -local Galaxy instance at the location discussed above by running -the following command. -""" - -import sys - -def upgrade(): - print __doc__ -def downgrade(): - pass diff -r e9e09d1ab51752db592eee71032ec5fc33f6ee57 -r 6f0050c4e06ef6f5fdce3f7326280c07c63ef62e lib/galaxy/tool_shed/tool_dependencies/common_util.py --- a/lib/galaxy/tool_shed/tool_dependencies/common_util.py +++ /dev/null @@ -1,149 +0,0 @@ -import os, shutil, tarfile, urllib2, zipfile -from galaxy.datatypes import checkers - -def create_env_var_dict( elem, tool_dependency_install_dir=None, tool_shed_repository_install_dir=None ): - env_var_name = elem.get( 'name', 'PATH' ) - env_var_action = elem.get( 'action', 'prepend_to' ) - env_var_text = None - if elem.text and elem.text.find( 'REPOSITORY_INSTALL_DIR' ) >= 0: - if tool_shed_repository_install_dir and elem.text.find( '$REPOSITORY_INSTALL_DIR' ) != -1: - env_var_text = elem.text.replace( '$REPOSITORY_INSTALL_DIR', tool_shed_repository_install_dir ) - return dict( name=env_var_name, action=env_var_action, value=env_var_text ) - else: - env_var_text = elem.text.replace( '$REPOSITORY_INSTALL_DIR', tool_dependency_install_dir ) - return dict( name=env_var_name, action=env_var_action, value=env_var_text ) - if elem.text and elem.text.find( 'INSTALL_DIR' ) >= 0: - if tool_dependency_install_dir: - env_var_text = elem.text.replace( '$INSTALL_DIR', tool_dependency_install_dir ) - return dict( name=env_var_name, action=env_var_action, value=env_var_text ) - else: - env_var_text = elem.text.replace( '$INSTALL_DIR', tool_shed_repository_install_dir ) - return dict( name=env_var_name, action=env_var_action, value=env_var_text ) - if elem.text: - # Allow for environment variables that contain neither REPOSITORY_INSTALL_DIR nor INSTALL_DIR since there may be command line - # parameters that are tuned for a Galaxy instance. Allowing them to be set in one location rather than being hard coded into - # each tool config is the best approach. For example: - # <environment_variable name="GATK2_SITE_OPTIONS" action="set_to"> - # "--num_threads 4 --num_cpu_threads_per_data_thread 3 --phone_home STANDARD" - # </environment_variable> - return dict( name=env_var_name, action=env_var_action, value=elem.text) - return None -def create_or_update_env_shell_file( install_dir, env_var_dict ): - env_var_name = env_var_dict[ 'name' ] - env_var_action = env_var_dict[ 'action' ] - env_var_value = env_var_dict[ 'value' ] - if env_var_action == 'prepend_to': - changed_value = '%s:$%s' % ( env_var_value, env_var_name ) - elif env_var_action == 'set_to': - changed_value = '%s' % env_var_value - elif env_var_action == 'append_to': - changed_value = '$%s:%s' % ( env_var_name, env_var_value ) - env_shell_file_path = '%s/env.sh' % install_dir - if os.path.exists( env_shell_file_path ): - write_action = '>>' - else: - write_action = '>' - cmd = "echo '%s=%s; export %s' %s %s;chmod +x %s" % ( env_var_name, - changed_value, - env_var_name, - write_action, - env_shell_file_path, - env_shell_file_path ) - return cmd -def extract_tar( file_name, file_path ): - if isgzip( file_name ) or isbz2( file_name ): - # Open for reading with transparent compression. - tar = tarfile.open( file_name, 'r:*' ) - else: - tar = tarfile.open( file_name ) - tar.extractall( path=file_path ) - tar.close() -def extract_zip( archive_path, extraction_path ): - # TODO: change this method to use zipfile.Zipfile.extractall() when we stop supporting Python 2.5. - if not zipfile_ok( archive_path ): - return False - zip_archive = zipfile.ZipFile( archive_path, 'r' ) - for name in zip_archive.namelist(): - uncompressed_path = os.path.join( extraction_path, name ) - if uncompressed_path.endswith( '/' ): - if not os.path.isdir( uncompressed_path ): - os.makedirs( uncompressed_path ) - else: - file( uncompressed_path, 'wb' ).write( zip_archive.read( name ) ) - zip_archive.close() - return True -def isbz2( file_path ): - return checkers.is_bz2( file_path ) -def isgzip( file_path ): - return checkers.is_gzip( file_path ) -def istar( file_path ): - return tarfile.is_tarfile( file_path ) -def iszip( file_path ): - return checkers.check_zip( file_path ) -def make_directory( full_path ): - if not os.path.exists( full_path ): - os.makedirs( full_path ) -def move_directory_files( current_dir, source_dir, destination_dir ): - source_directory = os.path.abspath( os.path.join( current_dir, source_dir ) ) - destination_directory = os.path.join( destination_dir ) - if not os.path.isdir( destination_directory ): - os.makedirs( destination_directory ) - for file_name in os.listdir( source_directory ): - source_file = os.path.join( source_directory, file_name ) - destination_file = os.path.join( destination_directory, file_name ) - shutil.move( source_file, destination_file ) -def move_file( current_dir, source, destination_dir ): - source_file = os.path.abspath( os.path.join( current_dir, source ) ) - destination_directory = os.path.join( destination_dir ) - if not os.path.isdir( destination_directory ): - os.makedirs( destination_directory ) - shutil.move( source_file, destination_directory ) -def tar_extraction_directory( file_path, file_name ): - """Try to return the correct extraction directory.""" - file_name = file_name.strip() - extensions = [ '.tar.gz', '.tgz', '.tar.bz2', '.tar', '.zip' ] - for extension in extensions: - if file_name.find( extension ) > 0: - dir_name = file_name[ :-len( extension ) ] - if os.path.exists( os.path.abspath( os.path.join( file_path, dir_name ) ) ): - return dir_name - if os.path.exists( os.path.abspath( os.path.join( file_path, file_name ) ) ): - return os.path.abspath( file_path ) - raise ValueError( 'Could not find path to file %s' % os.path.abspath( os.path.join( file_path, file_name ) ) ) -def url_download( install_dir, downloaded_file_name, download_url ): - file_path = os.path.join( install_dir, downloaded_file_name ) - src = None - dst = None - try: - src = urllib2.urlopen( download_url ) - data = src.read() - dst = open( file_path,'wb' ) - dst.write( data ) - except: - if src: - src.close() - if dst: - dst.close() - return os.path.abspath( file_path ) -def zip_extraction_directory( file_path, file_name ): - """Try to return the correct extraction directory.""" - files = [ filename for filename in os.listdir( file_path ) if not filename.endswith( '.zip' ) ] - if len( files ) > 1: - return os.path.abspath( file_path ) - elif len( files ) == 1: - # If there is only on file it should be a directory. - if os.path.isdir( os.path.join( file_path, files[ 0 ] ) ): - return os.path.abspath( os.path.join( file_path, files[ 0 ] ) ) - raise ValueError( 'Could not find directory for the extracted file %s' % os.path.abspath( os.path.join( file_path, file_name ) ) ) -def zipfile_ok( path_to_archive ): - """ - This function is a bit pedantic and not functionally necessary. It checks whether there is no file pointing outside of the extraction, - because ZipFile.extractall() has some potential security holes. See python zipfile documentation for more details. - """ - basename = os.path.realpath( os.path.dirname( path_to_archive ) ) - zip_archive = zipfile.ZipFile( path_to_archive ) - for member in zip_archive.namelist(): - member_path = os.path.realpath( os.path.join( basename, member ) ) - if not member_path.startswith( basename ): - return False - return True diff -r e9e09d1ab51752db592eee71032ec5fc33f6ee57 -r 6f0050c4e06ef6f5fdce3f7326280c07c63ef62e lib/galaxy/tool_shed/tool_dependencies/fabric_util.py --- a/lib/galaxy/tool_shed/tool_dependencies/fabric_util.py +++ /dev/null @@ -1,143 +0,0 @@ -# For Python 2.5 -from __future__ import with_statement - -import os, shutil, tempfile -from contextlib import contextmanager -import common_util - -from galaxy import eggs -import pkg_resources - -pkg_resources.require('ssh' ) -pkg_resources.require( 'Fabric' ) - -from fabric.api import env, lcd, local, settings - -INSTALLATION_LOG = 'INSTALLATION.log' - -def check_fabric_version(): - version = env.version - if int( version.split( "." )[ 0 ] ) < 1: - raise NotImplementedError( "Install Fabric version 1.0 or later." ) -def set_galaxy_environment( galaxy_user, tool_dependency_dir, host='localhost', shell='/bin/bash -l -c' ): - """General Galaxy environment configuration""" - env.user = galaxy_user - env.install_dir = tool_dependency_dir - env.host_string = host - env.shell = shell - env.use_sudo = False - env.safe_cmd = local - return env -@contextmanager -def make_tmp_dir(): - work_dir = tempfile.mkdtemp() - yield work_dir - if os.path.exists( work_dir ): - local( 'rm -rf %s' % work_dir ) -def handle_command( app, tool_dependency, install_dir, cmd ): - sa_session = app.model.context.current - output = local( cmd, capture=True ) - log_results( cmd, output, os.path.join( install_dir, INSTALLATION_LOG ) ) - if output.return_code: - tool_dependency.status = app.model.ToolDependency.installation_status.ERROR - tool_dependency.error_message = str( output.stderr ) - sa_session.add( tool_dependency ) - sa_session.flush() - return output.return_code -def install_and_build_package( app, tool_dependency, actions_dict ): - """Install a Galaxy tool dependency package either via a url or a mercurial or git clone command.""" - sa_session = app.model.context.current - install_dir = actions_dict[ 'install_dir' ] - package_name = actions_dict[ 'package_name' ] - actions = actions_dict.get( 'actions', None ) - filtered_actions = [] - if actions: - with make_tmp_dir() as work_dir: - with lcd( work_dir ): - # The first action in the list of actions will be the one that defines the installation process. There - # are currently only two supported processes; download_by_url and clone via a "shell_command" action type. - action_type, action_dict = actions[ 0 ] - if action_type == 'download_by_url': - # Eliminate the download_by_url action so remaining actions can be processed correctly. - filtered_actions = actions[ 1: ] - url = action_dict[ 'url' ] - if 'target_filename' in action_dict: - downloaded_filename = action_dict[ 'target_filename' ] - else: - downloaded_filename = os.path.split( url )[ -1 ] - downloaded_file_path = common_util.url_download( work_dir, downloaded_filename, url ) - if common_util.istar( downloaded_file_path ): - # <action type="download_by_url">http://sourceforge.net/projects/samtools/files/samtools/0.1.18/samtools-0.1…</action> - common_util.extract_tar( downloaded_file_path, work_dir ) - dir = common_util.tar_extraction_directory( work_dir, downloaded_filename ) - elif common_util.iszip( downloaded_file_path ): - # <action type="download_by_url">http://downloads.sourceforge.net/project/picard/picard-tools/1.56/picard-to…</action> - zip_archive_extracted = common_util.extract_zip( downloaded_file_path, work_dir ) - dir = common_util.zip_extraction_directory( work_dir, downloaded_filename ) - else: - dir = work_dir - elif action_type == 'shell_command': - # <action type="shell_command">git clone --recursive git://github.com/ekg/freebayes.git</action> - # Eliminate the shell_command clone action so remaining actions can be processed correctly. - filtered_actions = actions[ 1: ] - return_code = handle_command( app, tool_dependency, install_dir, action_dict[ 'command' ] ) - if return_code: - return - dir = package_name - else: - # We're handling a complex repository dependency where we only have a set_environment tag set. - # <action type="set_environment"> - # <environment_variable name="PATH" action="prepend_to">$INSTALL_DIR/bin</environment_variable> - # </action> - filtered_actions = [ a for a in actions ] - dir = install_dir - if not os.path.exists( dir ): - os.makedirs( dir ) - # The package has been down-loaded, so we can now perform all of the actions defined for building it. - with lcd( dir ): - for action_tup in filtered_actions: - action_type, action_dict = action_tup - current_dir = os.path.abspath( os.path.join( work_dir, dir ) ) - if action_type == 'make_directory': - common_util.make_directory( full_path=action_dict[ 'full_path' ] ) - elif action_type == 'move_directory_files': - common_util.move_directory_files( current_dir=current_dir, - source_dir=os.path.join( action_dict[ 'source_directory' ] ), - destination_dir=os.path.join( action_dict[ 'destination_directory' ] ) ) - elif action_type == 'move_file': - # TODO: Remove this hack that resets current_dir so that the pre-compiled bwa binary can be found. - # current_dir = '/Users/gvk/workspaces_2008/bwa/bwa-0.5.9' - common_util.move_file( current_dir=current_dir, - source=os.path.join( action_dict[ 'source' ] ), - destination_dir=os.path.join( action_dict[ 'destination' ] ) ) - elif action_type == 'set_environment': - # Currently the only action supported in this category is "environment_variable". - env_var_dicts = action_dict[ 'environment_variable' ] - for env_var_dict in env_var_dicts: - cmd = common_util.create_or_update_env_shell_file( install_dir, env_var_dict ) - return_code = handle_command( app, tool_dependency, install_dir, cmd ) - if return_code: - return - elif action_type == 'shell_command': - with settings( warn_only=True ): - return_code = handle_command( app, tool_dependency, install_dir, action_dict[ 'command' ] ) - if return_code: - return -def log_results( command, fabric_AttributeString, file_path ): - """ - Write attributes of fabric.operations._AttributeString (which is the output of executing command using fabric's local() method) - to a specified log file. - """ - if os.path.exists( file_path ): - logfile = open( file_path, 'ab' ) - else: - logfile = open( file_path, 'wb' ) - logfile.write( "\n#############################################\n" ) - logfile.write( '%s\nSTDOUT\n' % command ) - logfile.write( str( fabric_AttributeString.stdout ) ) - logfile.write( "\n#############################################\n" ) - logfile.write( "\n#############################################\n" ) - logfile.write( '%s\nSTDERR\n' % command ) - logfile.write( str( fabric_AttributeString.stderr ) ) - logfile.write( "\n#############################################\n" ) - logfile.close() This diff is so big that we needed to truncate the remainder. Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: natefoo: Python 2.5 compatibility fix for lwr client updates.
by commits-noreply＠bitbucket.org 26 Feb '13

26 Feb '13

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/e9e09d1ab517/ changeset: e9e09d1ab517 user: natefoo date: 2013-02-26 15:33:31 summary: Python 2.5 compatibility fix for lwr client updates. affected #: 1 file diff -r 993c232d34cda550a5605f87d7416467d4e2c33f -r e9e09d1ab51752db592eee71032ec5fc33f6ee57 lib/galaxy/jobs/runners/lwr_client/transport/standard.py --- a/lib/galaxy/jobs/runners/lwr_client/transport/standard.py +++ b/lib/galaxy/jobs/runners/lwr_client/transport/standard.py @@ -1,6 +1,7 @@ """ LWR HTTP Client layer based on Python Standard Library (urllib2) """ +from __future__ import with_statement import mmap import urllib2 Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: greg: Handle new repository_metadata columns when resetting metadata on repositories in the tool shed.
by commits-noreply＠bitbucket.org 25 Feb '13

25 Feb '13

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/993c232d34cd/ changeset: 993c232d34cd user: greg date: 2013-02-25 22:30:40 summary: Handle new repository_metadata columns when resetting metadata on repositories in the tool shed. affected #: 3 files diff -r ecbfab5f9f1b070bda03520700335d800b8fc761 -r 993c232d34cda550a5605f87d7416467d4e2c33f lib/galaxy/util/shed_util_common.py --- a/lib/galaxy/util/shed_util_common.py +++ b/lib/galaxy/util/shed_util_common.py @@ -34,8 +34,6 @@ REPOSITORY_DATA_MANAGER_CONFIG_FILENAME = "data_manager_conf.xml" MAX_CONTENT_SIZE = 32768 NOT_TOOL_CONFIGS = [ 'datatypes_conf.xml', 'repository_dependencies.xml', 'tool_dependencies.xml', REPOSITORY_DATA_MANAGER_CONFIG_FILENAME ] -GALAXY_ADMIN_TOOL_SHED_CONTROLLER = 'GALAXY_ADMIN_TOOL_SHED_CONTROLLER' -TOOL_SHED_ADMIN_CONTROLLER = 'TOOL_SHED_ADMIN_CONTROLLER' TOOL_TYPES_NOT_IN_TOOL_PANEL = [ 'manage_data' ] VALID_CHARS = set( string.letters + string.digits + "'\"-=_.()/+*^,:?!#[]%\\$@;{}&<>" ) @@ -484,10 +482,11 @@ repository_dependency_id=repository_dependency.id ) trans.sa_session.add( rrda ) trans.sa_session.flush() -def build_repository_ids_select_field( trans, cntrller, name='repository_ids', multiple=True, display='checkboxes' ): +def build_repository_ids_select_field( trans, name='repository_ids', multiple=True, display='checkboxes' ): """Method called from both Galaxy and the Tool Shed to generate the current list of repositories for resetting metadata.""" repositories_select_field = SelectField( name=name, multiple=multiple, display=display ) - if cntrller == TOOL_SHED_ADMIN_CONTROLLER: + if trans.webapp.name == 'community': + # We're in the tool shed. for repository in trans.sa_session.query( trans.model.Repository ) \ .filter( trans.model.Repository.table.c.deleted == False ) \ .order_by( trans.model.Repository.table.c.name, @@ -496,7 +495,8 @@ option_label = '%s (%s)' % ( repository.name, owner ) option_value = '%s' % trans.security.encode_id( repository.id ) repositories_select_field.add_option( option_label, option_value ) - elif cntrller == GALAXY_ADMIN_TOOL_SHED_CONTROLLER: + else: + # We're in Galaxy. for repository in trans.sa_session.query( trans.model.ToolShedRepository ) \ .filter( trans.model.ToolShedRepository.table.c.uninstalled == False ) \ .order_by( trans.model.ToolShedRepository.table.c.name, @@ -899,6 +899,7 @@ if not os.path.exists( os.path.join( dest_path, copied_file ) ): shutil.copy( full_source_path, os.path.join( dest_path, copied_file ) ) def create_or_update_repository_metadata( trans, id, repository, changeset_revision, metadata_dict ): + """Create or update a repository_metadatqa record in the tool shed.""" downloadable = is_downloadable( metadata_dict ) repository_metadata = get_repository_metadata_by_changeset_revision( trans, id, changeset_revision ) if repository_metadata: @@ -909,6 +910,11 @@ changeset_revision=changeset_revision, metadata=metadata_dict, downloadable=downloadable ) + # Always set the default values for the following columns. When resetting all metadata on a repository, this will reset the values. + repository_metadata.tools_functionally_correct = False + repository_metadata.do_not_test = False + repository_metadata.time_last_tested = None + repository_metadata.tool_test_errors = None trans.sa_session.add( repository_metadata ) trans.sa_session.flush() return repository_metadata @@ -2949,7 +2955,7 @@ return True return False def is_downloadable( metadata_dict ): - # NOTE: although repository README files are considered Galaxy utilities, they have no effect on determining if a revision is instakllable. + # NOTE: although repository README files are considered Galaxy utilities, they have no effect on determining if a revision is installable. # See the comments in the compare_readme_files() method. if 'datatypes' in metadata_dict: # We have proprietary datatypes. @@ -3503,7 +3509,8 @@ def reset_all_metadata_on_repository_in_tool_shed( trans, id ): """Reset all metadata on a single repository in a tool shed.""" def reset_all_tool_versions( trans, id, repo ): - changeset_revisions = [] + """Reset tool version lineage for those changeset revisions that include valid tools.""" + changeset_revisions_that_contain_tools = [] for changeset in repo.changelog: changeset_revision = str( repo.changectx( changeset ) ) repository_metadata = get_repository_metadata_by_changeset_revision( trans, id, changeset_revision ) @@ -3511,10 +3518,10 @@ metadata = repository_metadata.metadata if metadata: if metadata.get( 'tools', None ): - changeset_revisions.append( changeset_revision ) - # The list of changeset_revisions is now filtered to contain only those that are downloadable and contain tools. + changeset_revisions_that_contain_tools.append( changeset_revision ) + # The list of changeset_revisions_that_contain_tools is now filtered to contain only those that are downloadable and contain tools. # If a repository includes tools, build a dictionary of { 'tool id' : 'parent tool id' } pairs for each tool in each changeset revision. - for index, changeset_revision in enumerate( changeset_revisions ): + for index, changeset_revision in enumerate( changeset_revisions_that_contain_tools ): tool_versions_dict = {} repository_metadata = get_repository_metadata_by_changeset_revision( trans, id, changeset_revision ) metadata = repository_metadata.metadata @@ -3531,7 +3538,7 @@ tool_dict[ 'id' ], tool_dict[ 'version' ], tool_dict[ 'guid' ], - changeset_revisions[ 0:index ] ) + changeset_revisions_that_contain_tools[ 0:index ] ) tool_versions_dict[ tool_dict[ 'guid' ] ] = parent_id if tool_versions_dict: repository_metadata.tool_versions = tool_versions_dict @@ -3556,7 +3563,7 @@ work_dir = tempfile.mkdtemp() current_changeset_revision = str( repo.changectx( changeset ) ) ctx = repo.changectx( changeset ) - log.debug( "Cloning repository revision: %s", str( ctx.rev() ) ) + log.debug( "Cloning repository changeset revision: %s", str( ctx.rev() ) ) cloned_ok, error_message = clone_repository( repository_clone_url, work_dir, str( ctx.rev() ) ) if cloned_ok: log.debug( "Generating metadata for changset revision: %s", str( ctx.rev() ) ) @@ -3626,9 +3633,11 @@ reset_tool_data_tables( trans.app ) return invalid_file_tups, metadata_dict def reset_metadata_on_selected_repositories( trans, **kwd ): - # This method is called from both Galaxy and the Tool Shed, so the cntrller param is required. + """ + Inspect the repository changelog to reset metadata for all appropriate changeset revisions. This method is called from both Galaxy and the + Tool Shed. + """ repository_ids = util.listify( kwd.get( 'repository_ids', None ) ) - CONTROLLER = kwd[ 'CONTROLLER' ] message = '' status = 'done' if repository_ids: @@ -3636,10 +3645,12 @@ unsuccessful_count = 0 for repository_id in repository_ids: try: - if CONTROLLER == 'TOOL_SHED_ADMIN_CONTROLLER': + if trans.webapp.name == 'community': + # We're in the tool shed. repository = get_repository_in_tool_shed( trans, repository_id ) invalid_file_tups, metadata_dict = reset_all_metadata_on_repository_in_tool_shed( trans, repository_id ) - elif CONTROLLER == 'GALAXY_ADMIN_TOOL_SHED_CONTROLLER': + else: + # We're in Galaxy. repository = get_installed_tool_shed_repository( trans, repository_id ) invalid_file_tups, metadata_dict = reset_all_metadata_on_installed_repository( trans, repository_id ) if invalid_file_tups: diff -r ecbfab5f9f1b070bda03520700335d800b8fc761 -r 993c232d34cda550a5605f87d7416467d4e2c33f lib/galaxy/webapps/community/controllers/admin.py --- a/lib/galaxy/webapps/community/controllers/admin.py +++ b/lib/galaxy/webapps/community/controllers/admin.py @@ -721,12 +721,11 @@ @web.require_admin def reset_metadata_on_selected_repositories_in_tool_shed( self, trans, **kwd ): if 'reset_metadata_on_selected_repositories_button' in kwd: - kwd[ 'CONTROLLER' ] = suc.TOOL_SHED_ADMIN_CONTROLLER message, status = suc.reset_metadata_on_selected_repositories( trans, **kwd ) else: message = util.restore_text( kwd.get( 'message', '' ) ) status = kwd.get( 'status', 'done' ) - repositories_select_field = suc.build_repository_ids_select_field( trans, suc.TOOL_SHED_ADMIN_CONTROLLER ) + repositories_select_field = suc.build_repository_ids_select_field( trans ) return trans.fill_template( '/webapps/community/admin/reset_metadata_on_selected_repositories.mako', repositories_select_field=repositories_select_field, message=message, diff -r ecbfab5f9f1b070bda03520700335d800b8fc761 -r 993c232d34cda550a5605f87d7416467d4e2c33f lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py --- a/lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py +++ b/lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py @@ -1711,12 +1711,11 @@ @web.require_admin def reset_metadata_on_selected_installed_repositories( self, trans, **kwd ): if 'reset_metadata_on_selected_repositories_button' in kwd: - kwd[ 'CONTROLLER' ] = suc.GALAXY_ADMIN_TOOL_SHED_CONTROLLER message, status = suc.reset_metadata_on_selected_repositories( trans, **kwd ) else: message = util.restore_text( kwd.get( 'message', '' ) ) status = kwd.get( 'status', 'done' ) - repositories_select_field = suc.build_repository_ids_select_field( trans, suc.GALAXY_ADMIN_TOOL_SHED_CONTROLLER ) + repositories_select_field = suc.build_repository_ids_select_field( trans ) return trans.fill_template( '/admin/tool_shed_repository/reset_metadata_on_selected_repositories.mako', repositories_select_field=repositories_select_field, message=message, Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: 2 new changesets
by commits-noreply＠bitbucket.org 25 Feb '13

25 Feb '13

2 new commits in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/dd4ecbbbbab3/ changeset: dd4ecbbbbab3 user: jmchilton date: 2013-02-07 21:18:33 summary: Update lwr_client through https://bitbucket.org/jmchilton/lwr/commits/3034b5cb789a6c96b005b838542c6e7…. UChicago reported some issues with the use of mmap in the LWR client for large files. To get around this, I have implemented an optional alternative transport layer for the LWR client that is backed by pycurl instead of urllib2. This can be enabled by setting the environment variable LWR_CURL_TRANSPORT=1 for the Galaxy process. If LWR_CURL_TRANSPORT is set, the python pycurl package must be installed. affected #: 4 files diff -r ce9789a35356da2b2ee4ae723506d5af57a0ce69 -r dd4ecbbbbab33020fb5ff482fc302f092824e514 lib/galaxy/jobs/runners/lwr_client/__init__.py --- a/lib/galaxy/jobs/runners/lwr_client/__init__.py +++ b/lib/galaxy/jobs/runners/lwr_client/__init__.py @@ -5,15 +5,15 @@ This module contains logic for interfacing with an external LWR server. """ -import mmap import os import re import time import urllib -import urllib2 import simplejson +from transport import get_transport + class JobInputs(object): """ @@ -254,6 +254,18 @@ return self.job_inputs.rewritten_command_line +class parseJson(object): + + def __init__(self): + pass + + def __call__(self, func): + def replacement(*args, **kwargs): + response = func(*args, **kwargs) + return simplejson.loads(response) + return replacement + + class Client(object): """ Objects of this client class perform low-level communication with a remote LWR server. @@ -283,9 +295,7 @@ self.remote_host = remote_host self.job_id = job_id self.private_key = private_key - - def _url_open(self, request, data): - return urllib2.urlopen(request, data) + self.transport = get_transport() def __build_url(self, command, args): if self.private_key: @@ -294,29 +304,20 @@ url = self.remote_host + command + "?" + data return url - def __raw_execute(self, command, args={}, data=None): + def __raw_execute(self, command, args={}, data=None, input_path=None, output_path=None): url = self.__build_url(command, args) - request = urllib2.Request(url=url, data=data) - response = self._url_open(request, data) + response = self.transport.execute(url, data=data, input_path=input_path, output_path=output_path) return response - def __raw_execute_and_parse(self, command, args={}, data=None): - response = self.__raw_execute(command, args, data) - return simplejson.loads(response.read()) - + @parseJson() def __upload_file(self, action, path, name=None, contents=None): - input = open(path, 'rb') - try: - mmapped_input = mmap.mmap(input.fileno(), 0, access=mmap.ACCESS_READ) - return self.__upload_contents(action, path, mmapped_input, name) - finally: - input.close() - - def __upload_contents(self, action, path, contents, name=None): if not name: name = os.path.basename(path) args = {"job_id": self.job_id, "name": name} - return self.__raw_execute_and_parse(action, args, contents) + input_path = path + if contents: + input_path = None + return self.__raw_execute(action, args, contents, input_path) def upload_tool_file(self, path): """ @@ -364,7 +365,7 @@ contents : str Rewritten contents of the config file to upload. """ - return self.__upload_contents("upload_config_file", path, contents) + return self.__upload_file("upload_config_file", path, contents=contents) def upload_working_directory_file(self, path): """ @@ -378,9 +379,10 @@ """ return self.__upload_file("upload_working_directory_file", path) + @parseJson() def _get_output_type(self, name): - return self.__raw_execute_and_parse("get_output_type", {"name": name, - "job_id": self.job_id}) + return self.__raw_execute("get_output_type", {"name": name, + "job_id": self.job_id}) def download_work_dir_output(self, source, working_directory, output_path): """ @@ -414,25 +416,19 @@ name = os.path.basename(path) output_type = self._get_output_type(name) if output_type == "direct": - output = open(path, "wb") + output_path = path elif output_type == "task": - output = open(os.path.join(working_directory, name), "wb") + output_path = os.path.join(working_directory, name) else: raise Exception("No remote output found for dataset with path %s" % path) - self.__raw_download_output(name, self.job_id, output_type, output) + self.__raw_download_output(name, self.job_id, output_type, output_path) - def __raw_download_output(self, name, job_id, output_type, output_file): - response = self.__raw_execute("download_output", {"name": name, - "job_id": self.job_id, - "output_type": output_type}) - try: - while True: - buffer = response.read(1024) - if buffer == "": - break - output_file.write(buffer) - finally: - output_file.close() + def __raw_download_output(self, name, job_id, output_type, output_path): + self.__raw_execute("download_output", + {"name": name, + "job_id": self.job_id, + "output_type": output_type}, + output_path=output_path) def launch(self, command_line): """ @@ -463,11 +459,12 @@ return complete_response time.sleep(1) + @parseJson() def raw_check_complete(self): """ Get check_complete response from the remote server. """ - check_complete_response = self.__raw_execute_and_parse("check_complete", {"job_id": self.job_id}) + check_complete_response = self.__raw_execute("check_complete", {"job_id": self.job_id}) return check_complete_response def check_complete(self): @@ -482,11 +479,12 @@ """ self.__raw_execute("clean", {"job_id": self.job_id}) + @parseJson() def setup(self): """ Setup remote LWR server to run this job. """ - return self.__raw_execute_and_parse("setup", {"job_id": self.job_id}) + return self.__raw_execute("setup", {"job_id": self.job_id}) def _read(path): diff -r ce9789a35356da2b2ee4ae723506d5af57a0ce69 -r dd4ecbbbbab33020fb5ff482fc302f092824e514 lib/galaxy/jobs/runners/lwr_client/transport/__init__.py --- /dev/null +++ b/lib/galaxy/jobs/runners/lwr_client/transport/__init__.py @@ -0,0 +1,16 @@ +from standard import Urllib2Transport +from curl import PycurlTransport +import os + + +def get_transport(os_module=os): + use_curl = os_module.getenv('LWR_CURL_TRANSPORT', "0") + ## If LWR_CURL_TRANSPORT is unset or set to 0, use default, + ## else use curl. + if use_curl.isdigit() and not int(use_curl): + return Urllib2Transport() + else: + return PycurlTransport() + + +__all__ = [get_transport] diff -r ce9789a35356da2b2ee4ae723506d5af57a0ce69 -r dd4ecbbbbab33020fb5ff482fc302f092824e514 lib/galaxy/jobs/runners/lwr_client/transport/curl.py --- /dev/null +++ b/lib/galaxy/jobs/runners/lwr_client/transport/curl.py @@ -0,0 +1,42 @@ +from cStringIO import StringIO +try: + from pycurl import Curl +except: + pass +from os.path import getsize + + +PYCURL_UNAVAILABLE_MESSAGE = \ + "You are attempting to use the Pycurl version of the LWR client by pycurl is unavailable." + + +class PycurlTransport(object): + + def execute(self, url, data=None, input_path=None, output_path=None): + buf = self._open_output(output_path) + try: + c = self._new_curl_object() + c.setopt(c.URL, url.encode('ascii')) + c.setopt(c.WRITEFUNCTION, buf.write) + if input_path: + c.setopt(c.UPLOAD, 1) + c.setopt(c.READFUNCTION, open(input_path, 'rb').read) + filesize = getsize(input_path) + c.setopt(c.INFILESIZE, filesize) + if data: + c.setopt(c.POST, 1) + c.setopt(c.POSTFIELDS, data) + c.perform() + if not output_path: + return buf.getvalue() + finally: + buf.close() + + def _new_curl_object(self): + try: + return Curl() + except NameError: + raise ImportError(PYCURL_UNAVAILABLE_MESSAGE) + + def _open_output(self, output_path): + return open(output_path, 'wb') if output_path else StringIO() diff -r ce9789a35356da2b2ee4ae723506d5af57a0ce69 -r dd4ecbbbbab33020fb5ff482fc302f092824e514 lib/galaxy/jobs/runners/lwr_client/transport/standard.py --- /dev/null +++ b/lib/galaxy/jobs/runners/lwr_client/transport/standard.py @@ -0,0 +1,33 @@ +""" +LWR HTTP Client layer based on Python Standard Library (urllib2) +""" +import mmap +import urllib2 + + +class Urllib2Transport(object): + + def _url_open(self, request, data): + return urllib2.urlopen(request, data) + + def execute(self, url, data=None, input_path=None, output_path=None): + request = urllib2.Request(url=url, data=data) + input = None + try: + if input_path: + input = open(input_path, 'rb') + data = mmap.mmap(input.fileno(), 0, access=mmap.ACCESS_READ) + response = self._url_open(request, data) + finally: + if input: + input.close() + if output_path: + with open(output_path, 'wb') as output: + while True: + buffer = response.read(1024) + if buffer == "": + break + output.write(buffer) + return response + else: + return response.read() https://bitbucket.org/galaxy/galaxy-central/commits/ecbfab5f9f1b/ changeset: ecbfab5f9f1b user: natefoo date: 2013-02-25 21:16:30 summary: Merged in jmchilton/galaxy-central-lwr (pull request #118) Implement optional, alternative pycurl backend for LWR client. affected #: 4 files diff -r fa34924860aaa282fe3c3021a257f2523848a6e6 -r ecbfab5f9f1b070bda03520700335d800b8fc761 lib/galaxy/jobs/runners/lwr_client/__init__.py --- a/lib/galaxy/jobs/runners/lwr_client/__init__.py +++ b/lib/galaxy/jobs/runners/lwr_client/__init__.py @@ -5,15 +5,15 @@ This module contains logic for interfacing with an external LWR server. """ -import mmap import os import re import time import urllib -import urllib2 import simplejson +from transport import get_transport + class JobInputs(object): """ @@ -254,6 +254,18 @@ return self.job_inputs.rewritten_command_line +class parseJson(object): + + def __init__(self): + pass + + def __call__(self, func): + def replacement(*args, **kwargs): + response = func(*args, **kwargs) + return simplejson.loads(response) + return replacement + + class Client(object): """ Objects of this client class perform low-level communication with a remote LWR server. @@ -283,9 +295,7 @@ self.remote_host = remote_host self.job_id = job_id self.private_key = private_key - - def _url_open(self, request, data): - return urllib2.urlopen(request, data) + self.transport = get_transport() def __build_url(self, command, args): if self.private_key: @@ -294,29 +304,20 @@ url = self.remote_host + command + "?" + data return url - def __raw_execute(self, command, args={}, data=None): + def __raw_execute(self, command, args={}, data=None, input_path=None, output_path=None): url = self.__build_url(command, args) - request = urllib2.Request(url=url, data=data) - response = self._url_open(request, data) + response = self.transport.execute(url, data=data, input_path=input_path, output_path=output_path) return response - def __raw_execute_and_parse(self, command, args={}, data=None): - response = self.__raw_execute(command, args, data) - return simplejson.loads(response.read()) - + @parseJson() def __upload_file(self, action, path, name=None, contents=None): - input = open(path, 'rb') - try: - mmapped_input = mmap.mmap(input.fileno(), 0, access=mmap.ACCESS_READ) - return self.__upload_contents(action, path, mmapped_input, name) - finally: - input.close() - - def __upload_contents(self, action, path, contents, name=None): if not name: name = os.path.basename(path) args = {"job_id": self.job_id, "name": name} - return self.__raw_execute_and_parse(action, args, contents) + input_path = path + if contents: + input_path = None + return self.__raw_execute(action, args, contents, input_path) def upload_tool_file(self, path): """ @@ -364,7 +365,7 @@ contents : str Rewritten contents of the config file to upload. """ - return self.__upload_contents("upload_config_file", path, contents) + return self.__upload_file("upload_config_file", path, contents=contents) def upload_working_directory_file(self, path): """ @@ -378,9 +379,10 @@ """ return self.__upload_file("upload_working_directory_file", path) + @parseJson() def _get_output_type(self, name): - return self.__raw_execute_and_parse("get_output_type", {"name": name, - "job_id": self.job_id}) + return self.__raw_execute("get_output_type", {"name": name, + "job_id": self.job_id}) def download_work_dir_output(self, source, working_directory, output_path): """ @@ -414,25 +416,19 @@ name = os.path.basename(path) output_type = self._get_output_type(name) if output_type == "direct": - output = open(path, "wb") + output_path = path elif output_type == "task": - output = open(os.path.join(working_directory, name), "wb") + output_path = os.path.join(working_directory, name) else: raise Exception("No remote output found for dataset with path %s" % path) - self.__raw_download_output(name, self.job_id, output_type, output) + self.__raw_download_output(name, self.job_id, output_type, output_path) - def __raw_download_output(self, name, job_id, output_type, output_file): - response = self.__raw_execute("download_output", {"name": name, - "job_id": self.job_id, - "output_type": output_type}) - try: - while True: - buffer = response.read(1024) - if buffer == "": - break - output_file.write(buffer) - finally: - output_file.close() + def __raw_download_output(self, name, job_id, output_type, output_path): + self.__raw_execute("download_output", + {"name": name, + "job_id": self.job_id, + "output_type": output_type}, + output_path=output_path) def launch(self, command_line): """ @@ -463,11 +459,12 @@ return complete_response time.sleep(1) + @parseJson() def raw_check_complete(self): """ Get check_complete response from the remote server. """ - check_complete_response = self.__raw_execute_and_parse("check_complete", {"job_id": self.job_id}) + check_complete_response = self.__raw_execute("check_complete", {"job_id": self.job_id}) return check_complete_response def check_complete(self): @@ -482,11 +479,12 @@ """ self.__raw_execute("clean", {"job_id": self.job_id}) + @parseJson() def setup(self): """ Setup remote LWR server to run this job. """ - return self.__raw_execute_and_parse("setup", {"job_id": self.job_id}) + return self.__raw_execute("setup", {"job_id": self.job_id}) def _read(path): diff -r fa34924860aaa282fe3c3021a257f2523848a6e6 -r ecbfab5f9f1b070bda03520700335d800b8fc761 lib/galaxy/jobs/runners/lwr_client/transport/__init__.py --- /dev/null +++ b/lib/galaxy/jobs/runners/lwr_client/transport/__init__.py @@ -0,0 +1,16 @@ +from standard import Urllib2Transport +from curl import PycurlTransport +import os + + +def get_transport(os_module=os): + use_curl = os_module.getenv('LWR_CURL_TRANSPORT', "0") + ## If LWR_CURL_TRANSPORT is unset or set to 0, use default, + ## else use curl. + if use_curl.isdigit() and not int(use_curl): + return Urllib2Transport() + else: + return PycurlTransport() + + +__all__ = [get_transport] diff -r fa34924860aaa282fe3c3021a257f2523848a6e6 -r ecbfab5f9f1b070bda03520700335d800b8fc761 lib/galaxy/jobs/runners/lwr_client/transport/curl.py --- /dev/null +++ b/lib/galaxy/jobs/runners/lwr_client/transport/curl.py @@ -0,0 +1,42 @@ +from cStringIO import StringIO +try: + from pycurl import Curl +except: + pass +from os.path import getsize + + +PYCURL_UNAVAILABLE_MESSAGE = \ + "You are attempting to use the Pycurl version of the LWR client by pycurl is unavailable." + + +class PycurlTransport(object): + + def execute(self, url, data=None, input_path=None, output_path=None): + buf = self._open_output(output_path) + try: + c = self._new_curl_object() + c.setopt(c.URL, url.encode('ascii')) + c.setopt(c.WRITEFUNCTION, buf.write) + if input_path: + c.setopt(c.UPLOAD, 1) + c.setopt(c.READFUNCTION, open(input_path, 'rb').read) + filesize = getsize(input_path) + c.setopt(c.INFILESIZE, filesize) + if data: + c.setopt(c.POST, 1) + c.setopt(c.POSTFIELDS, data) + c.perform() + if not output_path: + return buf.getvalue() + finally: + buf.close() + + def _new_curl_object(self): + try: + return Curl() + except NameError: + raise ImportError(PYCURL_UNAVAILABLE_MESSAGE) + + def _open_output(self, output_path): + return open(output_path, 'wb') if output_path else StringIO() diff -r fa34924860aaa282fe3c3021a257f2523848a6e6 -r ecbfab5f9f1b070bda03520700335d800b8fc761 lib/galaxy/jobs/runners/lwr_client/transport/standard.py --- /dev/null +++ b/lib/galaxy/jobs/runners/lwr_client/transport/standard.py @@ -0,0 +1,33 @@ +""" +LWR HTTP Client layer based on Python Standard Library (urllib2) +""" +import mmap +import urllib2 + + +class Urllib2Transport(object): + + def _url_open(self, request, data): + return urllib2.urlopen(request, data) + + def execute(self, url, data=None, input_path=None, output_path=None): + request = urllib2.Request(url=url, data=data) + input = None + try: + if input_path: + input = open(input_path, 'rb') + data = mmap.mmap(input.fileno(), 0, access=mmap.ACCESS_READ) + response = self._url_open(request, data) + finally: + if input: + input.close() + if output_path: + with open(output_path, 'wb') as output: + while True: + buffer = response.read(1024) + if buffer == "": + break + output.write(buffer) + return response + else: + return response.read() Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: 3 new changesets
by commits-noreply＠bitbucket.org 23 Feb '13

23 Feb '13

3 new commits in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/26f7825cc871/ changeset: 26f7825cc871 user: jmchilton date: 2013-02-23 08:14:22 summary: Refactor code related to expanding multi inputs to more easily allow for switching between matched and product mode in the same workflow execution. affected #: 1 file diff -r 1c2e1625dd8a419aa1d335ddc19ba34857c05fc4 -r 26f7825cc871c752d1e79bbe8984857d67f1eb0e lib/galaxy/webapps/galaxy/controllers/workflow.py --- a/lib/galaxy/webapps/galaxy/controllers/workflow.py +++ b/lib/galaxy/webapps/galaxy/controllers/workflow.py @@ -2106,22 +2106,27 @@ def _expand_multiple_inputs(kwargs, mode): - (input_combos, multi_inputs) = _build_input_combos(kwargs, mode) + (single_inputs, matched_multi_inputs, multiplied_multi_inputs) = \ + _split_inputs(kwargs, mode) + + # Build up every combination of inputs to be run together. + #input_combos = [single_inputs] + input_combos = _extend_with_matched_combos(single_inputs, matched_multi_inputs) + input_combos = _extend_with_multiplied_combos(input_combos, multiplied_multi_inputs) + + # Input name that are multiply specified + multi_input_keys = \ + matched_multi_inputs.keys() + multiplied_multi_inputs.keys() + for input_combo in input_combos: for key, value in input_combo.iteritems(): kwargs[key] = value - yield (kwargs, multi_inputs.keys()) + yield (kwargs, multi_input_keys) -def _build_input_combos(kwargs, mode): - if mode == "product": - return _build_input_combos_product(kwargs) - else: # mode == "matched" - return _build_input_combos_matched(kwargs) -def _build_input_combos_matched(kwargs): - (single_inputs, multi_inputs) = _split_inputs(kwargs) +def _extend_with_matched_combos(single_inputs, multi_inputs): if len(multi_inputs) == 0: - return ([{}], {}) + return [single_inputs] matched_multi_inputs = [] @@ -2139,11 +2144,12 @@ raise Exception("Failed to match up multi-select inputs, must select equal number of data files in each multiselect") for index, value in enumerate(multi_input_values): matched_multi_inputs[index][multi_input_key] = value - return (matched_multi_inputs, multi_inputs) + return matched_multi_inputs -def _build_input_combos_product(kwargs): - (single_inputs, multi_inputs) = _split_inputs(kwargs) - combos = [single_inputs] + +def _extend_with_multiplied_combos(input_combos, multi_inputs): + combos = input_combos + for multi_input_key, multi_input_value in multi_inputs.iteritems(): iter_combos = [] @@ -2152,14 +2158,18 @@ iter_combos.append(_copy_and_extend_inputs(combo, multi_input_key, input_value)) combos = iter_combos - return (combos, multi_inputs) + return combos + def _copy_and_extend_inputs(inputs, key, value): new_inputs = dict(inputs) new_inputs[key] = value return new_inputs -def _split_inputs(kwargs): + +def _split_inputs(kwargs, mode): + """ + """ input_keys = filter(lambda a: a.endswith('|input'), kwargs) single_inputs = {} multi_inputs = {} @@ -2169,4 +2179,10 @@ multi_inputs[input_key] = input_val else: single_inputs[input_key] = input_val - return (single_inputs, multi_inputs) + matched_multi_inputs = {} + multiplied_multi_inputs = {} + if mode == "product": + multiplied_multi_inputs = multi_inputs + else: + matched_multi_inputs = multi_inputs + return (single_inputs, matched_multi_inputs, multiplied_multi_inputs) https://bitbucket.org/galaxy/galaxy-central/commits/6f30725d5973/ changeset: 6f30725d5973 user: jmchilton date: 2013-02-23 08:14:22 summary: Implement UI multi batch workflow inputs. affected #: 4 files diff -r 26f7825cc871c752d1e79bbe8984857d67f1eb0e -r 6f30725d59732a351ca9b4f6fd7bc14f43c8a4a1 lib/galaxy/webapps/galaxy/controllers/workflow.py --- a/lib/galaxy/webapps/galaxy/controllers/workflow.py +++ b/lib/galaxy/webapps/galaxy/controllers/workflow.py @@ -2110,7 +2110,6 @@ _split_inputs(kwargs, mode) # Build up every combination of inputs to be run together. - #input_combos = [single_inputs] input_combos = _extend_with_matched_combos(single_inputs, matched_multi_inputs) input_combos = _extend_with_multiplied_combos(input_combos, multiplied_multi_inputs) @@ -2172,17 +2171,18 @@ """ input_keys = filter(lambda a: a.endswith('|input'), kwargs) single_inputs = {} - multi_inputs = {} + matched_multi_inputs = {} + multiplied_multi_inputs = {} for input_key in input_keys: input_val = kwargs[input_key] if isinstance(input_val, list): - multi_inputs[input_key] = input_val + input_base = input_key[:-len("|input")] + mode_key = "%s|multi_mode" % input_base + mode = kwargs.get(mode_key, "matched") + if mode == "matched": + matched_multi_inputs[input_key] = input_val + else: + multiplied_multi_inputs[input_key] = input_val else: single_inputs[input_key] = input_val - matched_multi_inputs = {} - multiplied_multi_inputs = {} - if mode == "product": - multiplied_multi_inputs = multi_inputs - else: - matched_multi_inputs = multi_inputs return (single_inputs, matched_multi_inputs, multiplied_multi_inputs) diff -r 26f7825cc871c752d1e79bbe8984857d67f1eb0e -r 6f30725d59732a351ca9b4f6fd7bc14f43c8a4a1 static/images/silk/link.png Binary file static/images/silk/link.png has changed diff -r 26f7825cc871c752d1e79bbe8984857d67f1eb0e -r 6f30725d59732a351ca9b4f6fd7bc14f43c8a4a1 static/images/silk/link_break.png Binary file static/images/silk/link_break.png has changed diff -r 26f7825cc871c752d1e79bbe8984857d67f1eb0e -r 6f30725d59732a351ca9b4f6fd7bc14f43c8a4a1 templates/webapps/galaxy/workflow/run.mako --- a/templates/webapps/galaxy/workflow/run.mako +++ b/templates/webapps/galaxy/workflow/run.mako @@ -33,12 +33,12 @@ } else { select.val($('option:last', select).val()); } + select.siblings('img').hide(); select.removeAttr('multiple').removeAttr('size'); placeholder = 'type to filter'; } else { - // Comment out the following line to multiple batch input workflows in UI. - $('.multiinput').addClass('disabled'); $('.multiinput', select.closest('.form-row')).removeClass('disabled'); + select.siblings('img').show(); select.attr('multiple', 'multiple').attr('size', 8); placeholder = 'type to filter, [enter] to select all'; } @@ -126,6 +126,57 @@ select.after(filter); select.width(new_width); }); + + // Augment hidden fields with icons. + // http://stackoverflow.com/a/2088430 + var imgOn='${h.url_for("/static/images/silk/link.png")}'; + var imgOff='${h.url_for("/static/images/silk/link_break.png")}'; + $(function(){ + $(".multi-mode").each(function(){ + if($(this).val() == "matched") { + $(this).before($(document.createElement("img")) + .attr({src:imgOn,title:'Checkbox', id:$(this).attr("id")}) + .css("display", $(this).css("display")) + .addClass("chkBoxImg")); + } else { + $(this).before($(document.createElement("img")) + .attr({src:imgOff, title:'Checkbox',id:$(this).attr("id")}) + .css("display", $(this).css("display")) + .addClass("chkBoxImg")); + } + }); + $("img.chkBoxImg").click(function(){ + i= $(this).siblings("input[type=hidden]"); + s=$(this).attr("src"); + if(s==imgOn) { + $(this).attr("src",imgOff); + $(i).val("product"); + } else { + $(this).attr("src",imgOn); + $(i).val("matched"); + } + }); + }); + $("#tool_form").submit(function(e) { + var matchLength = -1; + $('span.multiinput_wrap select[name*="|input"]').each(function() { + var value = $(this).val(); + if(value instanceof Array) { + // Multi-value + if($(this).siblings("input[type=hidden]").val() == "matched") { + var length = $(this).val().length; + if(matchLength == -1) { + matchLength = length; + } else if(length != matchLength) { + e.preventDefault(); + alert("Linked inputs must be submitted in equal number."); + return false; + } + } + } + }); + return true; + }); }); </script></%def> @@ -260,6 +311,7 @@ %if step.type == 'data_input': ##Input Dataset Step, wrap for multiinput. <span class='multiinput_wrap'> + <input class="multi-mode" type="hidden" name="${str(step.id)}|multi_mode" id="${str(step.id)}|multi_mode" value="matched" /> ${param.get_html_field( t, value, other_values ).get_html( str(step.id) + "|" + prefix )} </span> %else: @@ -340,30 +392,6 @@ <form id="tool_form" name="tool_form" method="POST"> ## <input type="hidden" name="workflow_name" value="${h.to_unicode( workflow.name ) | h}" /> - -<input type="hidden" name="multiple_input_mode" value="matched" /> - - %if wf_parms: <div class="metadataForm"><div class="metadataFormTitle">Workflow Parameters</div> https://bitbucket.org/galaxy/galaxy-central/commits/fa34924860aa/ changeset: fa34924860aa user: jmchilton date: 2013-02-23 08:14:22 summary: Touch up UI related to multi input batch mode - move icon up by the multi document icon, switch from icon to CSS span, cleanup variable names. affected #: 1 file diff -r 6f30725d59732a351ca9b4f6fd7bc14f43c8a4a1 -r fa34924860aaa282fe3c3021a257f2523848a6e6 templates/webapps/galaxy/workflow/run.mako --- a/templates/webapps/galaxy/workflow/run.mako +++ b/templates/webapps/galaxy/workflow/run.mako @@ -1,5 +1,11 @@ <%inherit file="/base.mako"/> +<style> +/* TODO: Move this block into base.less? base.css? Someone more familiar with GUI should move this. */ +.icon-button.link {background:url(../images/silk/link.png) no-repeat;cursor:pointer;float:none;display:inline-block;margin-left:10px;} +.icon-button.link-broken {background:url(../images/silk/link_break.png) no-repeat;cursor:pointer;float:none;display:inline-block;margin-left:10px;} +</style> + <%def name="javascripts()"> ${parent.javascripts()} ${h.js( "libs/jquery/jquery.autocomplete" )} @@ -33,12 +39,12 @@ } else { select.val($('option:last', select).val()); } - select.siblings('img').hide(); + select.closest('.form-row').children('label').children('span.mode-icon').hide(); select.removeAttr('multiple').removeAttr('size'); placeholder = 'type to filter'; } else { $('.multiinput', select.closest('.form-row')).removeClass('disabled'); - select.siblings('img').show(); + select.closest('.form-row').children('label').children('span.mode-icon').show(); select.attr('multiple', 'multiple').attr('size', 8); placeholder = 'type to filter, [enter] to select all'; } @@ -129,30 +135,25 @@ // Augment hidden fields with icons. // http://stackoverflow.com/a/2088430 - var imgOn='${h.url_for("/static/images/silk/link.png")}'; - var imgOff='${h.url_for("/static/images/silk/link_break.png")}'; $(function(){ $(".multi-mode").each(function(){ if($(this).val() == "matched") { - $(this).before($(document.createElement("img")) - .attr({src:imgOn,title:'Checkbox', id:$(this).attr("id")}) - .css("display", $(this).css("display")) - .addClass("chkBoxImg")); + $(this).closest('.form-row').children('label').append($('<span class="icon-button link mode-icon"></span>') + .attr({id:$(this).attr("id")}) + .css("display", $(this).css("display"))); } else { - $(this).before($(document.createElement("img")) - .attr({src:imgOff, title:'Checkbox',id:$(this).attr("id")}) - .css("display", $(this).css("display")) - .addClass("chkBoxImg")); + $(this).closest('.form-row').children('label').append($('<span class="icon-button link-broken mode-icon"></span>') + .attr({id:$(this).attr("id")}) + .css("display", $(this).css("display"))); } }); - $("img.chkBoxImg").click(function(){ - i= $(this).siblings("input[type=hidden]"); - s=$(this).attr("src"); - if(s==imgOn) { - $(this).attr("src",imgOff); + $("span.mode-icon").click(function(){ + i= $(this).closest('.form-row').find("input[type=hidden]"); + if($(this).hasClass("link")) { + $(this).removeClass("link").addClass("link-broken"); $(i).val("product"); } else { - $(this).attr("src",imgOn); + $(this).removeClass("link-broken").addClass("link"); $(i).val("matched"); } }); Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: 2 new changesets
by commits-noreply＠bitbucket.org 22 Feb '13

22 Feb '13

2 new commits in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/1c2e1625dd8a/ changeset: 1c2e1625dd8a user: dan date: 2013-02-22 23:03:49 summary: Fix for displaying error messages on DataToolParameter where optional=True. affected #: 1 file diff -r 574e22b584eb7ef8b6168902347473ed3adeccac -r 1c2e1625dd8a419aa1d335ddc19ba34857c05fc4 lib/galaxy/tools/parameters/basic.py --- a/lib/galaxy/tools/parameters/basic.py +++ b/lib/galaxy/tools/parameters/basic.py @@ -1555,7 +1555,7 @@ # although, this should never be called in workflow mode right? if trans.workflow_building_mode: return None - if not value: + if not value and not self.optional: raise ValueError( "History does not include a dataset of the required format / build" ) if value in [None, "None"]: return None https://bitbucket.org/galaxy/galaxy-central/commits/c177960e4ed6/ changeset: c177960e4ed6 branch: stable user: dan date: 2013-02-22 23:03:49 summary: Fix for displaying error messages on DataToolParameter where optional=True. affected #: 1 file diff -r 31b09605fcc313b3e93efb927ac328bfcc42ad82 -r c177960e4ed61925a8b6c858e1f3f8d54c93cb37 lib/galaxy/tools/parameters/basic.py --- a/lib/galaxy/tools/parameters/basic.py +++ b/lib/galaxy/tools/parameters/basic.py @@ -1552,7 +1552,7 @@ # although, this should never be called in workflow mode right? if trans.workflow_building_mode: return None - if not value: + if not value and not self.optional: raise ValueError( "History does not include a dataset of the required format / build" ) if value in [None, "None"]: return None Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: greg: Enhance filtering in the repository revisions api.
by commits-noreply＠bitbucket.org 22 Feb '13

22 Feb '13

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/574e22b584eb/ changeset: 574e22b584eb user: greg date: 2013-02-22 22:41:56 summary: Enhance filtering in the repository revisions api. affected #: 2 files diff -r ed6104097dc9b9c519f71a392225e7ed45bfede3 -r 574e22b584eb7ef8b6168902347473ed3adeccac lib/galaxy/webapps/community/api/repository_revision_contents.py --- a/lib/galaxy/webapps/community/api/repository_revision_contents.py +++ b/lib/galaxy/webapps/community/api/repository_revision_contents.py @@ -11,9 +11,11 @@ log = logging.getLogger( __name__ ) def default_value_mapper( trans, repository_metadata ): - return { 'id' : trans.security.encode_id( repository_metadata.id ), - 'repository_id' : trans.security.encode_id( repository_metadata.repository_id ), - 'time_last_tested' : time_ago( repository_metadata.time_last_tested ) } + value_mapper = { 'id' : trans.security.encode_id( repository_metadata.id ), + 'repository_id' : trans.security.encode_id( repository_metadata.repository_id ) } + if repository_metadata.time_last_tested: + value_mapper[ 'time_last_tested' ] = time_ago( repository_metadata.time_last_tested ) + return value_mapper class RepositoryRevisionContentsController( BaseAPIController ): @web.expose_api diff -r ed6104097dc9b9c519f71a392225e7ed45bfede3 -r 574e22b584eb7ef8b6168902347473ed3adeccac lib/galaxy/webapps/community/api/repository_revisions.py --- a/lib/galaxy/webapps/community/api/repository_revisions.py +++ b/lib/galaxy/webapps/community/api/repository_revisions.py @@ -2,6 +2,7 @@ from galaxy.web.framework.helpers import time_ago import galaxy.util.shed_util_common as suc from galaxy import web, util +from galaxy.model.orm import and_, or_ from galaxy.web.base.controller import BaseAPIController from galaxy.web.framework.helpers import is_true @@ -13,29 +14,56 @@ log = logging.getLogger( __name__ ) def default_value_mapper( trans, repository_metadata ): - return { 'id' : trans.security.encode_id( repository_metadata.id ), - 'repository_id' : trans.security.encode_id( repository_metadata.repository_id ), - 'time_last_tested' : time_ago( repository_metadata.time_last_tested ) } + value_mapper = { 'id' : trans.security.encode_id( repository_metadata.id ), + 'repository_id' : trans.security.encode_id( repository_metadata.repository_id ) } + if repository_metadata.time_last_tested: + value_mapper[ 'time_last_tested' ] = time_ago( repository_metadata.time_last_tested ) + return value_mapper class RepositoryRevisionsController( BaseAPIController ): """RESTful controller for interactions with tool shed repository revisions.""" @web.expose_api - def index( self, trans, downloadable=True, **kwd ): + def index( self, trans, **kwd ): """ GET /api/repository_revisions Displays a collection (list) of repository revisions. """ rval = [] - downloadable = util.string_as_bool( downloadable ) + # Build up an anded clause list of filters. + clause_list = [] + # Filter by downloadable if received. + downloadable = kwd.get( 'downloadable', None ) + if downloadable is not None: + clause_list.append( trans.model.RepositoryMetadata.table.c.downloadable == util.string_as_bool( downloadable ) ) + # Filter by tools_functionally_correct if received. + tools_functionally_correct = kwd.get( 'tools_functionally_correct', None ) + if tools_functionally_correct is not None: + clause_list.append( trans.model.RepositoryMetadata.table.c.tools_functionally_correct == util.string_as_bool( tools_functionally_correct ) ) + # Filter by do_not_test if received. + do_not_test = kwd.get( 'do_not_test', None ) + if do_not_test is not None: + clause_list.append( trans.model.RepositoryMetadata.table.c.do_not_test == util.string_as_bool( do_not_test ) ) + # Filter by must_include_tools if received. + must_include_tools = kwd.get( 'must_include_tools', False ) try: query = trans.sa_session.query( trans.app.model.RepositoryMetadata ) \ - .filter( trans.app.model.RepositoryMetadata.table.c.downloadable == downloadable ) \ + .filter( and_( *clause_list ) ) \ .order_by( trans.app.model.RepositoryMetadata.table.c.repository_id ) \ .all() for repository_metadata in query: - item = repository_metadata.get_api_value( view='collection', value_mapper=default_value_mapper( trans, repository_metadata ) ) - item[ 'url' ] = web.url_for( 'repository_revision', id=trans.security.encode_id( repository_metadata.id ) ) - rval.append( item ) + if must_include_tools: + metadata = repository_metadata.metadata + if 'tools' in metadata: + ok_to_return = True + else: + ok_to_return = False + else: + ok_to_return = True + if ok_to_return: + item = repository_metadata.get_api_value( view='collection', + value_mapper=default_value_mapper( trans, repository_metadata ) ) + item[ 'url' ] = web.url_for( 'repository_revision', id=trans.security.encode_id( repository_metadata.id ) ) + rval.append( item ) except Exception, e: rval = "Error in the Tool Shed repository_revisions API in index: " + str( e ) log.error( rval + ": %s" % str( e ) ) @@ -49,7 +77,8 @@ """ try: repository_metadata = suc.get_repository_metadata_by_id( trans, id ) - repository_data = repository_metadata.get_api_value( view='element', value_mapper=default_value_mapper( trans, repository_metadata ) ) + repository_data = repository_metadata.get_api_value( view='element', + value_mapper=default_value_mapper( trans, repository_metadata ) ) repository_data[ 'contents_url' ] = web.url_for( 'repository_revision_contents', repository_metadata_id=id ) except Exception, e: message = "Error in the Tool Shed repository_revisions API in show: %s" % str( e ) Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: greg: Fix for deleting and undeleting repositories in the tool shed: if deleting, all installable revisions are marked as not installable, and if undeleting, all revisions are inspected and those determined to be installable are marked accordingly.
by commits-noreply＠bitbucket.org 22 Feb '13

22 Feb '13

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/ed6104097dc9/ changeset: ed6104097dc9 user: greg date: 2013-02-22 21:19:47 summary: Fix for deleting and undeleting repositories in the tool shed: if deleting, all installable revisions are marked as not installable, and if undeleting, all revisions are inspected and those determined to be installable are marked accordingly. affected #: 1 file diff -r 3bdb4291e5e666c4d4b86e184b4599e61b847864 -r ed6104097dc9b9c519f71a392225e7ed45bfede3 lib/galaxy/webapps/community/controllers/admin.py --- a/lib/galaxy/webapps/community/controllers/admin.py +++ b/lib/galaxy/webapps/community/controllers/admin.py @@ -586,12 +586,17 @@ deleted_repositories = "" for repository_id in ids: repository = suc.get_repository_in_tool_shed( trans, repository_id ) - if not repository.deleted: - repository.deleted = True - trans.sa_session.add( repository ) - trans.sa_session.flush() - count += 1 - deleted_repositories += " %s " % repository.name + if repository: + if not repository.deleted: + # Mark all installable repository_metadata records as not installable. + for repository_metadata in repository.downloadable_revisions: + repository_metadata.downloadable = False + trans.sa_session.add( repository_metadata ) + repository.deleted = True + trans.sa_session.add( repository ) + trans.sa_session.flush() + count += 1 + deleted_repositories += " %s " % repository.name if count: message = "Deleted %d %s: %s" % ( count, inflector.cond_plural( len( ids ), "repository" ), deleted_repositories ) else: @@ -740,12 +745,20 @@ undeleted_repositories = "" for repository_id in ids: repository = suc.get_repository_in_tool_shed( trans, repository_id ) - if repository.deleted: - repository.deleted = False - trans.sa_session.add( repository ) - trans.sa_session.flush() - count += 1 - undeleted_repositories += " %s" % repository.name + if repository: + if repository.deleted: + # Inspect all repository_metadata records to determine those that are installable, and mark them accordingly. + for repository_metadata in repository.metadata_revisions: + metadata = repository_metadata.metadata + if metadata: + if suc.is_downloadable( metadata ): + repository_metadata.downloadable = True + trans.sa_session.add( repository_metadata ) + repository.deleted = False + trans.sa_session.add( repository ) + trans.sa_session.flush() + count += 1 + undeleted_repositories += " %s" % repository.name if count: message = "Undeleted %d %s: %s" % ( count, inflector.cond_plural( count, "repository" ), undeleted_repositories ) else: Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: carlfeberhard: Fix to new-style display_application url mapping/formation; display_applications/link_generator: clean up
by commits-noreply＠bitbucket.org 22 Feb '13

22 Feb '13

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/3bdb4291e5e6/ changeset: 3bdb4291e5e6 user: carlfeberhard date: 2013-02-22 20:58:50 summary: Fix to new-style display_application url mapping/formation; display_applications/link_generator: clean up affected #: 2 files diff -r 5f10f61335fbcfb7982a879edc80325f5a73402f -r 3bdb4291e5e666c4d4b86e184b4599e61b847864 lib/galaxy/datatypes/display_applications/link_generator.py --- a/lib/galaxy/datatypes/display_applications/link_generator.py +++ b/lib/galaxy/datatypes/display_applications/link_generator.py @@ -1,4 +1,6 @@ -"""Separating Transaction based elements of display applications from datatypes. +"""Classes to generate links for display applications. + +Separating Transaction based elements of display applications from datatypes. """ import urllib @@ -10,19 +12,22 @@ from galaxy import util from galaxy.web import url_for - from galaxy.datatypes.interval import Interval, Gff, Wiggle, CustomTrack +#TODO: Ideally, these classes would be instantiated in the trans (or some other semi-persistant fixture) +# Currently, these are instantiated per HDA which is not the best solution + +#TODO: these could be extended to handle file_function and parse/contain the builds.txt files def get_display_app_link_generator( display_app_name ): """Returns an instance of the proper link generator class based on the display_app_name or DisplayAppLinkGenerator - if the name is unrecognized. + if the display_app_name is unrecognized. """ if display_app_name == 'ucsc': return UCSCDisplayAppLinkGenerator() - if display_app_name == 'gbrowse': + elif display_app_name == 'gbrowse': return GBrowseDisplayAppLinkGenerator() return DisplayAppLinkGenerator() @@ -58,9 +63,10 @@ class UCSCDisplayAppLinkGenerator( DisplayAppLinkGenerator ): - """Class for UCSC display application link generators. + """Class for generating links to display data in the + UCSC genome browser. - This class returns UCSC main and test links for the following datatypes: + This class returns links for the following datatypes and their subclasses: Interval, Wiggle, Gff, CustomTrack """ def __init__( self ): @@ -69,7 +75,6 @@ def _link_function_from_datatype( self, datatype ): """Dispatch to proper link generating function based on datatype. """ - # they're all the same if( ( isinstance( datatype, Interval ) ) or ( isinstance( datatype, Wiggle ) ) or ( isinstance( datatype, Gff ) ) @@ -83,8 +88,6 @@ and content of dataset. """ # this is a refactor of Interval.ucsc_links, GFF.ucsc_links, Wiggle.ucsc_links, and CustomTrack.ucsc_links - # ...which are all the same function - #TODO: app vars can be moved into init (and base_url as well) chrom, start, stop = dataset.datatype.get_estimated_display_viewport( dataset ) if chrom is None: @@ -107,10 +110,11 @@ class GBrowseDisplayAppLinkGenerator( DisplayAppLinkGenerator ): - """Class for UCSC display application link generators. + """Class for generating links to display data in the + GBrowse genome browser. - This class returns UCSC main and test links for the following datatypes: - Interval, Wiggle, Gff, CustomTrack + This class returns links for the following datatypes and their subclasses: + Gff, Wiggle """ def __init__( self ): self.display_app_name = 'gbrowse' @@ -118,7 +122,6 @@ def _link_function_from_datatype( self, datatype ): """Dispatch to proper link generating function based on datatype. """ - # they're all the same if( ( isinstance( datatype, Gff ) ) or ( isinstance( datatype, Wiggle ) ) ): return self.gbrowse_links diff -r 5f10f61335fbcfb7982a879edc80325f5a73402f -r 3bdb4291e5e666c4d4b86e184b4599e61b847864 lib/galaxy/webapps/galaxy/api/history_contents.py --- a/lib/galaxy/webapps/galaxy/api/history_contents.py +++ b/lib/galaxy/webapps/galaxy/api/history_contents.py @@ -244,7 +244,7 @@ def get_display_app_url( display_app_link, hda, trans ): web_url_for = routes.URLGenerator( trans.webapp.mapper, trans.environ ) dataset_hash, user_hash = util.encode_dataset_user( trans, hda, None ) - return web_url_for( controller='/dataset', + return web_url_for( controller='dataset', action="display_application", dataset_id=dataset_hash, user_id=user_hash, @@ -280,4 +280,3 @@ display_apps.append( dict( label=hda.datatype.get_display_label( display_app_name ), links=app_links ) ) return display_apps - Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: greg: Enhancements to the tool shed api to enable the curretnly most important updates for repository_metadata records.
by commits-noreply＠bitbucket.org 22 Feb '13

22 Feb '13

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/5f10f61335fb/ changeset: 5f10f61335fb user: greg date: 2013-02-22 20:35:34 summary: Enhancements to the tool shed api to enable the curretnly most important updates for repository_metadata records. affected #: 7 files diff -r 8019917d7c309b2e9d4ca25078a6bda15569f94f -r 5f10f61335fbcfb7982a879edc80325f5a73402f lib/galaxy/webapps/community/api/repositories.py --- a/lib/galaxy/webapps/community/api/repositories.py +++ b/lib/galaxy/webapps/community/api/repositories.py @@ -32,7 +32,7 @@ item[ 'url' ] = web.url_for( 'repository_contents', repository_id=trans.security.encode_id( repository.id ) ) rval.append( item ) except Exception, e: - message = "Error in the Tool Shed API at index: %s" % str( e ) + message = "Error in the Tool Shed repositories API in index: %s" % str( e ) log.error( message, exc_info=True ) trans.response.status = 500 return message @@ -50,8 +50,8 @@ repository_data = repository.get_api_value( view='element', value_mapper=value_mapper ) repository_data[ 'contents_url' ] = web.url_for( 'repository_contents', repository_id=id ) except Exception, e: - message = "Error in the Tool Shed API at show: %s" % str( e ) + message = "Error in the Tool Shed repositories API in show: %s" % str( e ) log.error( message, exc_info=True ) trans.response.status = 500 return message - return repository_data \ No newline at end of file + return repository_data diff -r 8019917d7c309b2e9d4ca25078a6bda15569f94f -r 5f10f61335fbcfb7982a879edc80325f5a73402f lib/galaxy/webapps/community/api/repository_contents.py --- a/lib/galaxy/webapps/community/api/repository_contents.py +++ b/lib/galaxy/webapps/community/api/repository_contents.py @@ -11,21 +11,24 @@ class RepositoryContentsController( BaseAPIController ): @web.expose_api - def index( self, trans, repository_id, **kwd ): + def index( self, trans, **kwd ): """ - GET /api/repositories/{encoded_repsository_id}/contents - Displays a collection (list) of repository contents. + GET /api/repositories/{encoded_repository_id}/contents + Displays a collection (dictionary) of repository contents. :param repository_id: an encoded id string of the `Repository` to inspect """ rval = [] + repository_id = kwd.get( 'repository_id', None ) try: repository = suc.get_repository_in_tool_shed( trans, repository_id ) - repository_dict = repository.as_dict( trans ) + value_mapper={ 'id' : repository_id, + 'user_id' : trans.security.encode_id( repository.user_id ) } + repository_dict = repository.as_dict( value_mapper ) repository_dict[ 'url' ] = web.url_for( 'repository_contents', repository_id=repository_id ) rval.append( repository_dict ) except Exception, e: - message = "Error in repository_contents API: %s" % str( e ) + message = "Error in the Tool Shed repository_contents API in index: %s" % str( e ) log.error( message, exc_info=True ) trans.response.status = 500 return message diff -r 8019917d7c309b2e9d4ca25078a6bda15569f94f -r 5f10f61335fbcfb7982a879edc80325f5a73402f lib/galaxy/webapps/community/api/repository_revision_contents.py --- /dev/null +++ b/lib/galaxy/webapps/community/api/repository_revision_contents.py @@ -0,0 +1,39 @@ +import logging +from galaxy.web.framework.helpers import time_ago +import galaxy.util.shed_util_common as suc +from galaxy import web +from galaxy.web.base.controller import BaseAPIController + +import pkg_resources +pkg_resources.require( "Routes" ) +import routes + +log = logging.getLogger( __name__ ) + +def default_value_mapper( trans, repository_metadata ): + return { 'id' : trans.security.encode_id( repository_metadata.id ), + 'repository_id' : trans.security.encode_id( repository_metadata.repository_id ), + 'time_last_tested' : time_ago( repository_metadata.time_last_tested ) } + +class RepositoryRevisionContentsController( BaseAPIController ): + @web.expose_api + def index( self, trans, **kwd ): + """ + GET /api/repository_revisions/{encoded_repository_metadata_id}/contents + Displays a collection (dictionary) of repository_metadata contents. + + :param repository_metadata_id: an encoded id string of the `RepositoryMetadata` to inspect + """ + rval = [] + repository_metadata_id = kwd.get( 'repository_metadata_id', None ) + try: + repository_metadata = suc.get_repository_metadata_by_id( trans, repository_metadata_id ) + repository_dict = repository_metadata.as_dict( value_mapper=default_value_mapper( trans, repository_metadata ) ) + repository_dict[ 'url' ] = web.url_for( 'repository_revision_contents', repository_metadata_id=repository_metadata_id ) + rval.append( repository_dict ) + except Exception, e: + message = "Error in the Tool Shed repository_revision_contents API in index: %s" % str( e ) + log.error( message, exc_info=True ) + trans.response.status = 500 + return message + return rval diff -r 8019917d7c309b2e9d4ca25078a6bda15569f94f -r 5f10f61335fbcfb7982a879edc80325f5a73402f lib/galaxy/webapps/community/api/repository_revisions.py --- a/lib/galaxy/webapps/community/api/repository_revisions.py +++ b/lib/galaxy/webapps/community/api/repository_revisions.py @@ -1,3 +1,5 @@ +import datetime +from galaxy.web.framework.helpers import time_ago import galaxy.util.shed_util_common as suc from galaxy import web, util from galaxy.web.base.controller import BaseAPIController @@ -10,6 +12,11 @@ log = logging.getLogger( __name__ ) +def default_value_mapper( trans, repository_metadata ): + return { 'id' : trans.security.encode_id( repository_metadata.id ), + 'repository_id' : trans.security.encode_id( repository_metadata.repository_id ), + 'time_last_tested' : time_ago( repository_metadata.time_last_tested ) } + class RepositoryRevisionsController( BaseAPIController ): """RESTful controller for interactions with tool shed repository revisions.""" @web.expose_api @@ -26,13 +33,56 @@ .order_by( trans.app.model.RepositoryMetadata.table.c.repository_id ) \ .all() for repository_metadata in query: - value_mapper={ 'id' : trans.security.encode_id( repository_metadata.id ), - 'repository_id' : trans.security.encode_id( repository_metadata.repository_id ) } - item = repository_metadata.get_api_value( view='collection', value_mapper=value_mapper ) + item = repository_metadata.get_api_value( view='collection', value_mapper=default_value_mapper( trans, repository_metadata ) ) item[ 'url' ] = web.url_for( 'repository_revision', id=trans.security.encode_id( repository_metadata.id ) ) rval.append( item ) except Exception, e: - rval = "Error in repository_revisions API at index: " + str( e ) + rval = "Error in the Tool Shed repository_revisions API in index: " + str( e ) log.error( rval + ": %s" % str( e ) ) trans.response.status = 500 return rval + @web.expose_api + def show( self, trans, id, **kwd ): + """ + GET /api/repository_revisions/{encoded_repository_metadata_id} + Displays information about a repository_metadata record in the Tool Shed. + """ + try: + repository_metadata = suc.get_repository_metadata_by_id( trans, id ) + repository_data = repository_metadata.get_api_value( view='element', value_mapper=default_value_mapper( trans, repository_metadata ) ) + repository_data[ 'contents_url' ] = web.url_for( 'repository_revision_contents', repository_metadata_id=id ) + except Exception, e: + message = "Error in the Tool Shed repository_revisions API in show: %s" % str( e ) + log.error( message, exc_info=True ) + trans.response.status = 500 + return message + return repository_data + @web.expose_api + def update( self, trans, payload, **kwd ): + """ + PUT /api/repository_revisions/{encoded_repository_metadata_id}/{payload} + Updates the value of specified columns of the repository_metadata table based on the key / value pairs in payload. + """ + repository_metadata_id = kwd.get( 'id', None ) + try: + repository_metadata = suc.get_repository_metadata_by_id( trans, repository_metadata_id ) + flush_needed = False + for key, new_value in payload.items(): + if hasattr( repository_metadata, key ): + old_value = getattr( repository_metadata, key ) + setattr( repository_metadata, key, new_value ) + if key in [ 'tools_functionally_correct', 'time_last_tested' ]: + # Automatically update repository_metadata.time_last_tested. + repository_metadata.time_last_tested = datetime.datetime.utcnow() + flush_needed = True + if flush_needed: + trans.sa_session.add( repository_metadata ) + trans.sa_session.flush() + except Exception, e: + message = "Error in the Tool Shed repository_revisions API in update: %s" % str( e ) + log.error( message, exc_info=True ) + trans.response.status = 500 + return message + item = repository_metadata.as_dict( value_mapper=default_value_mapper( trans, repository_metadata ) ) + item[ 'url' ] = web.url_for( 'repository_revision', id=repository_metadata_id ) + return [ item ] diff -r 8019917d7c309b2e9d4ca25078a6bda15569f94f -r 5f10f61335fbcfb7982a879edc80325f5a73402f lib/galaxy/webapps/community/buildapp.py --- a/lib/galaxy/webapps/community/buildapp.py +++ b/lib/galaxy/webapps/community/buildapp.py @@ -77,6 +77,12 @@ name_prefix='repository_', path_prefix='/api/repositories/:repository_id', parent_resources=dict( member_name='repository', collection_name='repositories' ) ) + webapp.api_mapper.resource( 'content', + 'contents', + controller='repository_revision_contents', + name_prefix='repository_revision_', + path_prefix='/api/repository_revisions/:repository_metadata_id', + parent_resources=dict( member_name='repository_revision', collection_name='repository_revisions' ) ) webapp.api_mapper.resource( 'repository', 'repositories', path_prefix='/api' ) webapp.api_mapper.resource( 'repository_revision', 'repository_revisions', path_prefix='/api' ) webapp.finalize_config() diff -r 8019917d7c309b2e9d4ca25078a6bda15569f94f -r 5f10f61335fbcfb7982a879edc80325f5a73402f lib/galaxy/webapps/community/model/__init__.py --- a/lib/galaxy/webapps/community/model/__init__.py +++ b/lib/galaxy/webapps/community/model/__init__.py @@ -133,11 +133,8 @@ self.email_alerts = email_alerts self.times_downloaded = times_downloaded self.deprecated = deprecated - def as_dict( self, trans ): - value_mapper={ 'id' : trans.security.encode_id( self.id ), - 'user_id' : trans.security.encode_id( self.user_id ) } - repository_dict = self.get_api_value( view='element', value_mapper=value_mapper ) - return repository_dict + def as_dict( self, value_mapper=None ): + return self.get_api_value( view='element', value_mapper=value_mapper ) def get_api_value( self, view='collection', value_mapper=None ): if value_mapper is None: value_mapper = {} @@ -194,10 +191,13 @@ fp.close() class RepositoryMetadata( object, APIItem ): - api_collection_visible_keys = ( 'id', 'repository_id', 'changeset_revision', 'malicious', 'downloadable' ) - api_element_visible_keys = ( 'id', 'repository_id', 'changeset_revision', 'malicious', 'downloadable' ) - def __init__( self, repository_id=None, changeset_revision=None, metadata=None, tool_versions=None, malicious=False, downloadable=False, + api_collection_visible_keys = ( 'id', 'repository_id', 'changeset_revision', 'malicious', 'downloadable', 'tools_functionally_correct', + 'do_not_test', 'time_last_tested', 'tool_test_errors' ) + api_element_visible_keys = ( 'id', 'repository_id', 'changeset_revision', 'malicious', 'downloadable', 'tools_functionally_correct', + 'do_not_test', 'time_last_tested', 'tool_test_errors' ) + def __init__( self, id=None, repository_id=None, changeset_revision=None, metadata=None, tool_versions=None, malicious=False, downloadable=False, tools_functionally_correct=False, do_not_test=False, time_last_tested=None, tool_test_errors=None ): + self.id = id self.repository_id = repository_id self.changeset_revision = changeset_revision self.metadata = metadata or dict() @@ -208,6 +208,8 @@ self.do_not_test = do_not_test self.time_last_tested = time_last_tested self.tool_test_errors = tool_test_errors + def as_dict( self, value_mapper=None ): + return self.get_api_value( view='element', value_mapper=value_mapper ) def get_api_value( self, view='collection', value_mapper=None ): if value_mapper is None: value_mapper = {} diff -r 8019917d7c309b2e9d4ca25078a6bda15569f94f -r 5f10f61335fbcfb7982a879edc80325f5a73402f scripts/api/tool_shed_repository_revision_update.py --- /dev/null +++ b/scripts/api/tool_shed_repository_revision_update.py @@ -0,0 +1,37 @@ +#!/usr/bin/env python +""" +PUT/update script to update appropriate values in a repository_metadata table record in the Tool Shed. + +usage: tool_shed_repository_revision_update.py key url key1=value1 key2=value2 ... +""" + +import os, sys +sys.path.insert( 0, os.path.dirname( __file__ ) ) +from common import update + +import pkg_resources +pkg_resources.require( "simplejson" ) + +import simplejson + +to_json_string = simplejson.dumps +from_json_string = simplejson.loads + +data = {} +for key, value in [ kwarg.split( '=', 1 ) for kwarg in sys.argv[ 3: ] ]: + """ + This example script will properly handle updating the value of one or more of the following RepositoryMetadata attributes: + tools_functionally_correct, do_not_test, tool_test_errors + """ + if key in [ 'tools_functionally_correct', 'do_not_test' ]: + if str( value ).lower() in [ 'true', 'yes', 'on' ]: + new_value = True + else: + new_value = False + elif key in [ 'tool_test_errors' ]: + new_value = from_json_string( value ) + else: + new_value = str( value ) + data[ key ] = new_value + +update( sys.argv[ 1 ], sys.argv[ 2 ], data ) Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: dannon: Update documentation for max_data_lines defaulting to 100000.
by commits-noreply＠bitbucket.org 22 Feb '13

22 Feb '13

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/8019917d7c30/ changeset: 8019917d7c30 user: dannon date: 2013-02-22 17:14:37 summary: Update documentation for max_data_lines defaulting to 100000. affected #: 1 file diff -r 6714ad95a7f3dc9096e4630633beab30df54cdd8 -r 8019917d7c309b2e9d4ca25078a6bda15569f94f lib/galaxy/datatypes/tabular.py --- a/lib/galaxy/datatypes/tabular.py +++ b/lib/galaxy/datatypes/tabular.py @@ -35,19 +35,19 @@ data.Text.init_meta( self, dataset, copy_from=copy_from ) def set_meta( self, dataset, overwrite = True, skip = None, max_data_lines = 100000, max_guess_type_data_lines = None, **kwd ): """ - Tries to determine the number of columns as well as those columns - that contain numerical values in the dataset. A skip parameter is - used because various tabular data types reuse this function, and - their data type classes are responsible to determine how many invalid - comment lines should be skipped. Using None for skip will cause skip - to be zero, but the first line will be processed as a header. A + Tries to determine the number of columns as well as those columns that + contain numerical values in the dataset. A skip parameter is used + because various tabular data types reuse this function, and their data + type classes are responsible to determine how many invalid comment + lines should be skipped. Using None for skip will cause skip to be + zero, but the first line will be processed as a header. A max_data_lines parameter is used because various tabular data types reuse this function, and their data type classes are responsible to determine how many data lines should be processed to ensure that the non-optional metadata parameters are properly set; if used, optional metadata parameters will be set to None, unless the entire file has - already been read. Using None (default) for max_data_lines will - process all data lines. + already been read. Using None for max_data_lines will process all data + lines. Items of interest: Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: carlfeberhard: generate display application links outside of datatype classes
by commits-noreply＠bitbucket.org 22 Feb '13

22 Feb '13

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/6714ad95a7f3/ changeset: 6714ad95a7f3 user: carlfeberhard date: 2013-02-22 05:30:55 summary: generate display application links outside of datatype classes affected #: 2 files diff -r 429103684d864f93563cd92c30888bd623fdb80e -r 6714ad95a7f3dc9096e4630633beab30df54cdd8 lib/galaxy/datatypes/display_applications/link_generator.py --- /dev/null +++ b/lib/galaxy/datatypes/display_applications/link_generator.py @@ -0,0 +1,153 @@ +"""Separating Transaction based elements of display applications from datatypes. +""" + +import urllib + +# for the url_for hack +import pkg_resources +pkg_resources.require( "Routes" ) +import routes + +from galaxy import util +from galaxy.web import url_for + +from galaxy.datatypes.interval import Interval, Gff, Wiggle, CustomTrack + + +def get_display_app_link_generator( display_app_name ): + """Returns an instance of the proper link generator class + based on the display_app_name or DisplayAppLinkGenerator + if the name is unrecognized. + """ + if display_app_name == 'ucsc': + return UCSCDisplayAppLinkGenerator() + + if display_app_name == 'gbrowse': + return GBrowseDisplayAppLinkGenerator() + + return DisplayAppLinkGenerator() + + +class DisplayAppLinkGenerator( object ): + """Base class for display application link generators. + + This class returns an empty list of links for all datatypes. + """ + def __init__( self ): + self.display_app_name = '' + + def no_links_available( self, dataset, app, base_url, url_for=url_for ): + """Called when no display application links are available + for this display app name and datatype combination. + """ + return [] + + def _link_function_from_datatype( self, datatype ): + """Dispatch to proper link generating function on datatype. + """ + return self.no_links_available + + def generate_links( self, trans, dataset ): + # here's the hack - which is expensive (time) + web_url_for = routes.URLGenerator( trans.webapp.mapper, trans.environ ) + + link_function = self._link_function_from_datatype( dataset.datatype ) + display_links = link_function( dataset, trans.app, trans.request.base, url_for=web_url_for ) + + return display_links + + +class UCSCDisplayAppLinkGenerator( DisplayAppLinkGenerator ): + """Class for UCSC display application link generators. + + This class returns UCSC main and test links for the following datatypes: + Interval, Wiggle, Gff, CustomTrack + """ + def __init__( self ): + self.display_app_name = 'ucsc' + + def _link_function_from_datatype( self, datatype ): + """Dispatch to proper link generating function based on datatype. + """ + # they're all the same + if( ( isinstance( datatype, Interval ) ) + or ( isinstance( datatype, Wiggle ) ) + or ( isinstance( datatype, Gff ) ) + or ( isinstance( datatype, CustomTrack ) ) ): + return self.ucsc_links + else: + return super( UCSCDisplayAppLinkGenerator, self )._link_function_from_datatype( datatype ) + + def ucsc_links( self, dataset, app, base_url, url_for=url_for ): + """Generate links to UCSC genome browser sites based on the dbkey + and content of dataset. + """ + # this is a refactor of Interval.ucsc_links, GFF.ucsc_links, Wiggle.ucsc_links, and CustomTrack.ucsc_links + # ...which are all the same function + + #TODO: app vars can be moved into init (and base_url as well) + chrom, start, stop = dataset.datatype.get_estimated_display_viewport( dataset ) + if chrom is None: + return [] + ret_val = [] + for site_name, site_url in util.get_ucsc_by_build(dataset.dbkey): + if site_name in app.config.ucsc_display_sites: + internal_url = url_for( controller='dataset', dataset_id=dataset.id, + action='display_at', filename='%s_%s' % ( self.display_app_name, site_name ) ) + base_url = app.config.get( "display_at_callback", base_url ) + display_url = urllib.quote_plus( "%s%s/display_as?id=%i&display_app=%s&authz_method=display_at" + % (base_url, url_for( controller='root' ), dataset.id, self.display_app_name) ) + redirect_url = urllib.quote_plus( "%sdb=%s&position=%s:%s-%s&hgt.customText=%%s" + % (site_url, dataset.dbkey, chrom, start, stop ) ) + + link = '%s?redirect_url=%s&display_url=%s' % ( internal_url, redirect_url, display_url ) + ret_val.append( ( site_name, link ) ) + + return ret_val + + +class GBrowseDisplayAppLinkGenerator( DisplayAppLinkGenerator ): + """Class for UCSC display application link generators. + + This class returns UCSC main and test links for the following datatypes: + Interval, Wiggle, Gff, CustomTrack + """ + def __init__( self ): + self.display_app_name = 'gbrowse' + + def _link_function_from_datatype( self, datatype ): + """Dispatch to proper link generating function based on datatype. + """ + # they're all the same + if( ( isinstance( datatype, Gff ) ) + or ( isinstance( datatype, Wiggle ) ) ): + return self.gbrowse_links + else: + return super( GBrowseDisplayAppLinkGenerator, self )._link_function_from_datatype( datatype ) + + def gbrowse_links( self, dataset, app, base_url, url_for=url_for ): + """Generate links to GBrowse genome browser sites based on the dbkey + and content of dataset. + """ + # when normalized for var names, Gff.gbrowse_links and Wiggle.gbrowse_links are the same + # also: almost identical to ucsc_links except for the 'chr' stripping, sites_by_build, config key + # could be refactored even more + chrom, start, stop = dataset.datatype.get_estimated_display_viewport( dataset ) + if chrom is None: + return [] + ret_val = [] + for site_name, site_url in util.get_gbrowse_sites_by_build( dataset.dbkey ): + if site_name in app.config.gbrowse_display_sites: + # strip chr from seqid + if chrom.startswith( 'chr' ) and len ( chrom ) > 3: + chrom = chrom[3:] + internal_url = url_for( controller='dataset', dataset_id=dataset.id, + action='display_at', filename='%s_%s' % ( self.display_app_name, site_name ) ) + redirect_url = urllib.quote_plus( "%s/?q=%s:%s..%s&eurl=%%s" % ( site_url, chrom, start, stop ) ) + base_url = app.config.get( "display_at_callback", base_url ) + display_url = urllib.quote_plus( "%s%s/display_as?id=%i&display_app=%s&authz_method=display_at" + % ( base_url, url_for( controller='root' ), dataset.id, self.display_app_name ) ) + link = '%s?redirect_url=%s&display_url=%s' % ( internal_url, redirect_url, display_url ) + ret_val.append( ( site_name, link ) ) + + return ret_val diff -r 429103684d864f93563cd92c30888bd623fdb80e -r 6714ad95a7f3dc9096e4630633beab30df54cdd8 lib/galaxy/webapps/galaxy/api/history_contents.py --- a/lib/galaxy/webapps/galaxy/api/history_contents.py +++ b/lib/galaxy/webapps/galaxy/api/history_contents.py @@ -14,6 +14,8 @@ from galaxy.datatypes.display_applications import util from galaxy.datatypes.metadata import FileParameter +from galaxy.datatypes.display_applications.link_generator import get_display_app_link_generator + import pkg_resources pkg_resources.require( "Routes" ) import routes @@ -128,8 +130,6 @@ """ GET /api/histories/{encoded_history_id}/contents/{encoded_content_id} Displays information about a history content (dataset). - - """ hda_dict = {} try: @@ -225,7 +225,8 @@ hda_dict[ 'meta_files' ] = meta_files hda_dict[ 'display_apps' ] = get_display_apps( trans, hda ) - #hda_dict[ 'display_types' ] = get_display_types( trans, hda ) + hda_dict[ 'display_types' ] = get_old_display_applications( trans, hda ) + hda_dict[ 'visualizations' ] = hda.get_visualizations() hda_dict[ 'peek' ] = to_unicode( hda.display_peek() ) @@ -262,21 +263,21 @@ return display_apps -def get_display_types( trans, hda ): - #TODO: make more straightforward (somehow) - #FIXME: need to force a transition to all new-style display applications +def get_old_display_applications( trans, hda ): display_apps = [] - - for display_app in hda.datatype.get_display_types(): + for display_app_name in hda.datatype.get_display_types(): + link_generator = get_display_app_link_generator( display_app_name ) + display_links = link_generator.generate_links( trans, hda ) + app_links = [] - target_frame, display_links = hda.datatype.get_display_links( hda, display_app, trans.app, trans.request.base ) for display_name, display_link in display_links: app_links.append({ - 'target' : target_frame, + 'target' : '_blank', 'href' : display_link, 'text' : display_name }) if app_links: - display_apps.append( dict( label=hda.datatype.get_display_label( display_app ), links=app_links ) ) + display_apps.append( dict( label=hda.datatype.get_display_label( display_app_name ), links=app_links ) ) return display_apps + Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: jgoecks: Trackster: add subsetting functionality to data manager and enable subsetting for line tracks.
by Bitbucket 21 Feb '13

21 Feb '13

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/429103684d86/ changeset: 429103684d86 user: jgoecks date: 2013-02-21 23:10:54 summary: Trackster: add subsetting functionality to data manager and enable subsetting for line tracks. affected #: 4 files diff -r 26d7b89d9f897a28876214c88faed215973e6fcd -r 429103684d864f93563cd92c30888bd623fdb80e lib/galaxy/visualization/data_providers/genome.py --- a/lib/galaxy/visualization/data_providers/genome.py +++ b/lib/galaxy/visualization/data_providers/genome.py @@ -1044,9 +1044,7 @@ return all_dat is not None def get_data( self, chrom, start, end, start_val=0, max_vals=None, num_samples=1000, **kwargs ): - # Subtract 1 because start/end are in 0-based coordinate system but BBI - # provider uses 1-based coordinate system. - start = int( start ) - 1 + start = int( start ) end = int( end ) # Helper function for getting summary data regardless of chromosome diff -r 26d7b89d9f897a28876214c88faed215973e6fcd -r 429103684d864f93563cd92c30888bd623fdb80e static/scripts/viz/trackster/painters.js --- a/static/scripts/viz/trackster/painters.js +++ b/static/scripts/viz/trackster/painters.js @@ -258,7 +258,8 @@ // y becomes the bar height in pixels, which is the negated for canvas coords y = Math.round( y / vertical_range * height_px ); ctx.fillRect(x_scaled, y_zero, delta_x_px, - y ); - } else if (mode === "Intensity") { + } + else if (mode === "Intensity") { var saturation = (y - min_value) / vertical_range, // Range is [pref_color, 255] where saturation = 0 --> 255 and saturation = 1 --> pref color @@ -267,18 +268,21 @@ new_b = Math.round( pref_b + (255 - pref_b) * (1 - saturation) ); ctx.fillStyle = "rgb(" + new_r + "," + new_g + "," + new_b + ")"; ctx.fillRect(x_scaled, 0, delta_x_px, height_px); - } else { + } + else { // console.log(y, track.min_value, track.vertical_range, (y - track.min_value) / track.vertical_range * track.height_px); y = Math.round( height_px - (y - min_value) / vertical_range * height_px ); // console.log(canvas.get(0).height, canvas.get(0).width); if (in_path) { ctx.lineTo(x_scaled, y); - } else { + } + else { in_path = true; if (mode === "Filled") { ctx.moveTo(x_scaled, height_px); ctx.lineTo(x_scaled, y); - } else { + } + else { ctx.moveTo(x_scaled, y); } } @@ -289,7 +293,8 @@ var overflow_x; if (mode === "Histogram" || mode === "Intensity") { overflow_x = delta_x_px; - } else { // Line and Filled, which are points + } + else { // Line and Filled, which are points x_scaled -= 2; // Move it over to the left so it's centered on the point overflow_x = 4; } @@ -308,7 +313,8 @@ ctx.lineTo( 0, y_zero ); } ctx.fill(); - } else { + } + else { ctx.stroke(); } @@ -469,7 +475,7 @@ */ get_row_height: function() { var mode = this.mode, height; - if (mode === "Dense") { + if (mode === "Dense") { height = DENSE_TRACK_HEIGHT; } else if (mode === "no_detail") { diff -r 26d7b89d9f897a28876214c88faed215973e6fcd -r 429103684d864f93563cd92c30888bd623fdb80e static/scripts/viz/trackster/tracks.js --- a/static/scripts/viz/trackster/tracks.js +++ b/static/scripts/viz/trackster/tracks.js @@ -3540,6 +3540,7 @@ on_resize: function() { this.request_draw(true); }, + /** * Set track minimum value. */ @@ -3549,6 +3550,7 @@ this.tile_cache.clear(); this.request_draw(); }, + /** * Set track maximum value. */ @@ -3558,6 +3560,7 @@ this.tile_cache.clear(); this.request_draw(); }, + predraw_init: function() { var track = this; track.vertical_range = undefined; @@ -3618,6 +3621,7 @@ .prependTo(track.container_div); }); }, + /** * Draw LineTrack tile. */ @@ -3632,11 +3636,12 @@ return new Tile(this, region, resolution, canvas, result.data); }, + /** - * LineTrack data cannot currently be subsetted. + * Subset line tracks only if resolution is single-base pair. */ - can_subset: function(data) { - return false; + can_subset: function(entry) { + return (entry.data[1][0] - entry.data[0][0] === 1); } }); diff -r 26d7b89d9f897a28876214c88faed215973e6fcd -r 429103684d864f93563cd92c30888bd623fdb80e static/scripts/viz/visualization.js --- a/static/scripts/viz/visualization.js +++ b/static/scripts/viz/visualization.js @@ -355,25 +355,38 @@ // var key_ary = this.get('key_ary'), obj_cache = this.get('obj_cache'), - key, entry_region; + key, entry_region, is_subregion; for (var i = 0; i < key_ary.length; i++) { key = key_ary[i]; entry_region = new GenomeRegion({from_str: key}); if (entry_region.contains(region)) { + is_subregion = true; + // This entry has data in the requested range. Return if data // is compatible and can be subsetted. entry = obj_cache[key]; if ( is_deferred(entry) || ( this.get('data_mode_compatible')(entry, mode) && this.get('can_subset')(entry) ) ) { this.move_key_to_end(key, i); + + // If there's data, subset it. + if ( !is_deferred(entry) ) { + var subset_entry = this.subset_entry(entry, region); + this.set(region, subset_entry); + entry = subset_entry; + } + return entry; } } } + // FIXME: There _may_ be instances where region is a subregion of another entry but cannot be + // subsetted. For these cases, do not increase length because region will never be found (and + // an infinite loop will occur.) // If needed, extend region to make it minimum size. - if (region.length() < this.attributes.min_region_size) { + if (!is_subregion && region.length() < this.attributes.min_region_size) { // IDEA: alternative heuristic is to find adjacent cache entry to region and use that to extend. // This would prevent bad extensions when zooming in/out while still preserving the behavior // below. @@ -553,7 +566,40 @@ return deferred; }, - + + /** + * Returns entry with only data in the subregion. + */ + subset_entry: function(entry, subregion) { + // Dictionary from entry type to function for subsetting data. + var subset_fns = { + bigwig: function(data, subregion) { + return _.filter(data, function(data_point) { + return data_point[0] >= subregion.get('start') && + data_point[0] <= subregion.get('end'); + }); + }, + 'refseq': function(data, subregion) { + var seq_start = subregion.get('start') - entry.region.get('start'), + seq_end = entry.data.length - ( entry.region.get('end') - subregion.get('end') ); + return entry.data.slice(seq_start, seq_end); + } + }; + + // Subset entry if there is a function for subsetting and regions are not the same. + var subregion_data = entry.data; + if (!entry.region.same(subregion) && entry.dataset_type in subset_fns) { + subregion_data = subset_fns[entry.dataset_type](entry.data, subregion); + } + + // Return entry with subregion's data. + return { + region: subregion, + data: subregion_data, + dataset_type: entry.dataset_type + }; + }, + /** * Get data from the cache. */ @@ -584,24 +630,6 @@ return { data: null }; } return GenomeDataManager.prototype.load_data.call(this, region, mode, resolution, extra_params); - }, - - /** - * Return an entry that includes only data in the subregion. - */ - subset_entry: function(entry, subregion) { - var seq_data = entry.data; - if (!entry.region.same(subregion)) { - // Need to subset sequence data. - var seq_start = subregion.get('start') - entry.region.get('start'), - seq_end = entry.data.length - ( entry.region.get('end') - subregion.get('end') ); - seq_data = entry.data.slice(seq_start, seq_end); - } - - return { - region: subregion, - data: seq_data - }; } }); Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: 2 new changesets
by Bitbucket 21 Feb '13

21 Feb '13

2 new commits in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/26d7b89d9f89/ changeset: 26d7b89d9f89 user: dan date: 2013-02-21 21:26:15 summary: Fix for display_in_upload affected #: 1 file diff -r 8717485b3c6c78f709d352a63255e50ef978f57d -r 26d7b89d9f897a28876214c88faed215973e6fcd lib/galaxy/datatypes/registry.py --- a/lib/galaxy/datatypes/registry.py +++ b/lib/galaxy/datatypes/registry.py @@ -90,7 +90,7 @@ dtype = elem.get( 'type', None ) type_extension = elem.get( 'type_extension', None ) mimetype = elem.get( 'mimetype', None ) - display_in_upload = elem.get( 'display_in_upload', False ) + display_in_upload = galaxy.util.string_as_bool( elem.get( 'display_in_upload', False ) ) make_subclass = galaxy.util.string_as_bool( elem.get( 'subclass', False ) ) # Proprietary datatypes included in installed tool shed repositories will include two special attributes # (proprietary_path and proprietary_datatype_module) if they depend on proprietary datatypes classes. @@ -107,7 +107,7 @@ in_memory_dtype = elem.get( 'type', None ) in_memory_type_extension = elem.get( 'type_extension', None ) in_memory_mimetype = elem.get( 'mimetype', None ) - in_memory_display_in_upload = elem.get( 'display_in_upload', False ) + in_memory_display_in_upload = galaxy.util.string_as_bool( elem.get( 'display_in_upload', False ) ) in_memory_make_subclass = galaxy.util.string_as_bool( elem.get( 'subclass', False ) ) if in_memory_dtype == dtype and in_memory_type_extension == type_extension and in_memory_mimetype == mimetype \ and in_memory_display_in_upload == display_in_upload and in_memory_make_subclass == make_subclass: https://bitbucket.org/galaxy/galaxy-central/commits/31b09605fcc3/ changeset: 31b09605fcc3 branch: stable user: dan date: 2013-02-21 21:26:15 summary: Fix for display_in_upload affected #: 1 file diff -r a0741e6798f28f05bb9bb17b572ce1ea4c62e1f0 -r 31b09605fcc313b3e93efb927ac328bfcc42ad82 lib/galaxy/datatypes/registry.py --- a/lib/galaxy/datatypes/registry.py +++ b/lib/galaxy/datatypes/registry.py @@ -90,7 +90,7 @@ dtype = elem.get( 'type', None ) type_extension = elem.get( 'type_extension', None ) mimetype = elem.get( 'mimetype', None ) - display_in_upload = elem.get( 'display_in_upload', False ) + display_in_upload = galaxy.util.string_as_bool( elem.get( 'display_in_upload', False ) ) make_subclass = galaxy.util.string_as_bool( elem.get( 'subclass', False ) ) # Proprietary datatypes included in installed tool shed repositories will include two special attributes # (proprietary_path and proprietary_datatype_module) if they depend on proprietary datatypes classes. @@ -107,7 +107,7 @@ in_memory_dtype = elem.get( 'type', None ) in_memory_type_extension = elem.get( 'type_extension', None ) in_memory_mimetype = elem.get( 'mimetype', None ) - in_memory_display_in_upload = elem.get( 'display_in_upload', False ) + in_memory_display_in_upload = galaxy.util.string_as_bool( elem.get( 'display_in_upload', False ) ) in_memory_make_subclass = galaxy.util.string_as_bool( elem.get( 'subclass', False ) ) if in_memory_dtype == dtype and in_memory_type_extension == type_extension and in_memory_mimetype == mimetype \ and in_memory_display_in_upload == display_in_upload and in_memory_make_subclass == make_subclass: Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: greg: Minor fixes to the baseline API methods in the tool shed.
by Bitbucket 21 Feb '13

21 Feb '13

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/8717485b3c6c/ changeset: 8717485b3c6c user: greg date: 2013-02-21 21:04:14 summary: Minor fixes to the baseline API methods in the tool shed. affected #: 3 files diff -r 73914d7ce26131b0670f82e3596d2a56494f0480 -r 8717485b3c6c78f709d352a63255e50ef978f57d lib/galaxy/webapps/community/api/repositories.py --- a/lib/galaxy/webapps/community/api/repositories.py +++ b/lib/galaxy/webapps/community/api/repositories.py @@ -26,7 +26,9 @@ .order_by( trans.app.model.Repository.table.c.name ) \ .all() for repository in query: - item = repository.get_api_value( value_mapper={ 'id' : trans.security.encode_id( repository.id ) } ) + value_mapper={ 'id' : trans.security.encode_id( repository.id ), + 'user_id' : trans.security.encode_id( repository.user_id ) } + item = repository.get_api_value( view='collection', value_mapper=value_mapper ) item[ 'url' ] = web.url_for( 'repository_contents', repository_id=trans.security.encode_id( repository.id ) ) rval.append( item ) except Exception, e: @@ -43,7 +45,9 @@ """ try: repository = suc.get_repository_in_tool_shed( trans, id ) - repository_data = repository.get_api_value( view='element', value_mapper={ 'id' : trans.security.encode_id( repository.id ) } ) + value_mapper={ 'id' : trans.security.encode_id( repository.id ), + 'user_id' : trans.security.encode_id( repository.user_id ) } + repository_data = repository.get_api_value( view='element', value_mapper=value_mapper ) repository_data[ 'contents_url' ] = web.url_for( 'repository_contents', repository_id=id ) except Exception, e: message = "Error in the Tool Shed API at show: %s" % str( e ) diff -r 73914d7ce26131b0670f82e3596d2a56494f0480 -r 8717485b3c6c78f709d352a63255e50ef978f57d lib/galaxy/webapps/community/api/repository_revisions.py --- a/lib/galaxy/webapps/community/api/repository_revisions.py +++ b/lib/galaxy/webapps/community/api/repository_revisions.py @@ -26,7 +26,9 @@ .order_by( trans.app.model.RepositoryMetadata.table.c.repository_id ) \ .all() for repository_metadata in query: - item = repository_metadata.get_api_value( value_mapper={ 'id' : trans.security.encode_id( repository_metadata.id ) } ) + value_mapper={ 'id' : trans.security.encode_id( repository_metadata.id ), + 'repository_id' : trans.security.encode_id( repository_metadata.repository_id ) } + item = repository_metadata.get_api_value( view='collection', value_mapper=value_mapper ) item[ 'url' ] = web.url_for( 'repository_revision', id=trans.security.encode_id( repository_metadata.id ) ) rval.append( item ) except Exception, e: diff -r 73914d7ce26131b0670f82e3596d2a56494f0480 -r 8717485b3c6c78f709d352a63255e50ef978f57d lib/galaxy/webapps/community/model/__init__.py --- a/lib/galaxy/webapps/community/model/__init__.py +++ b/lib/galaxy/webapps/community/model/__init__.py @@ -114,14 +114,14 @@ self.prev_session_id = prev_session_id class Repository( object, APIItem ): - api_collection_visible_keys = ( 'id', 'name', 'description', 'user_id', 'private', 'times_downloaded', 'deprecated' ) - api_element_visible_keys = ( 'id', 'name', 'description', 'long_description', 'user_id', 'private', 'times_downloaded', 'deprecated' ) + api_collection_visible_keys = ( 'id', 'name', 'description', 'user_id', 'private', 'deleted', 'times_downloaded', 'deprecated' ) + api_element_visible_keys = ( 'id', 'name', 'description', 'long_description', 'user_id', 'private', 'deleted', 'times_downloaded', 'deprecated' ) file_states = Bunch( NORMAL = 'n', NEEDS_MERGING = 'm', MARKED_FOR_REMOVAL = 'r', MARKED_FOR_ADDITION = 'a', NOT_TRACKED = '?' ) - def __init__( self, id=None, name=None, description=None, long_description=None, user_id=None, private=False, email_alerts=None, + def __init__( self, id=None, name=None, description=None, long_description=None, user_id=None, private=False, deleted=None, email_alerts=None, times_downloaded=0, deprecated=False ): self.id = id self.name = name or "Unnamed repository" @@ -129,12 +129,14 @@ self.long_description = long_description self.user_id = user_id self.private = private + self.deleted = deleted self.email_alerts = email_alerts self.times_downloaded = times_downloaded self.deprecated = deprecated def as_dict( self, trans ): - repository_dict = self.get_api_value( view='element' ) - repository_dict[ 'id' ] = trans.security.encode_id( self.id ) + value_mapper={ 'id' : trans.security.encode_id( self.id ), + 'user_id' : trans.security.encode_id( self.user_id ) } + repository_dict = self.get_api_value( view='element', value_mapper=value_mapper ) return repository_dict def get_api_value( self, view='collection', value_mapper=None ): if value_mapper is None: Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: greg: Baseline API methods for repositories in the tool shed.
by Bitbucket 21 Feb '13

21 Feb '13

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/73914d7ce261/ changeset: 73914d7ce261 user: greg date: 2013-02-21 20:38:53 summary: Baseline API methods for repositories in the tool shed. affected #: 5 files diff -r c32f70e92aa428dcab3d753ef88a5d255272e4f9 -r 73914d7ce26131b0670f82e3596d2a56494f0480 lib/galaxy/webapps/community/api/repositories.py --- /dev/null +++ b/lib/galaxy/webapps/community/api/repositories.py @@ -0,0 +1,53 @@ +import galaxy.util.shed_util_common as suc +from galaxy import web, util +from galaxy.web.base.controller import BaseAPIController +from galaxy.web.framework.helpers import is_true + +import pkg_resources +pkg_resources.require( "Routes" ) +import routes +import logging + +log = logging.getLogger( __name__ ) + +class RepositoriesController( BaseAPIController ): + """RESTful controller for interactions with repositories in the Tool Shed.""" + @web.expose_api + def index( self, trans, deleted=False, **kwd ): + """ + GET /api/repository_revisions + Displays a collection (list) of repositories. + """ + rval = [] + deleted = util.string_as_bool( deleted ) + try: + query = trans.sa_session.query( trans.app.model.Repository ) \ + .filter( trans.app.model.Repository.table.c.deleted == deleted ) \ + .order_by( trans.app.model.Repository.table.c.name ) \ + .all() + for repository in query: + item = repository.get_api_value( value_mapper={ 'id' : trans.security.encode_id( repository.id ) } ) + item[ 'url' ] = web.url_for( 'repository_contents', repository_id=trans.security.encode_id( repository.id ) ) + rval.append( item ) + except Exception, e: + message = "Error in the Tool Shed API at index: %s" % str( e ) + log.error( message, exc_info=True ) + trans.response.status = 500 + return message + return rval + @web.expose_api + def show( self, trans, id, **kwd ): + """ + GET /api/repositories/{encoded_repository_id} + Displays information about a repository in the Tool Shed. + """ + try: + repository = suc.get_repository_in_tool_shed( trans, id ) + repository_data = repository.get_api_value( view='element', value_mapper={ 'id' : trans.security.encode_id( repository.id ) } ) + repository_data[ 'contents_url' ] = web.url_for( 'repository_contents', repository_id=id ) + except Exception, e: + message = "Error in the Tool Shed API at show: %s" % str( e ) + log.error( message, exc_info=True ) + trans.response.status = 500 + return message + return repository_data \ No newline at end of file diff -r c32f70e92aa428dcab3d753ef88a5d255272e4f9 -r 73914d7ce26131b0670f82e3596d2a56494f0480 lib/galaxy/webapps/community/api/repository_contents.py --- /dev/null +++ b/lib/galaxy/webapps/community/api/repository_contents.py @@ -0,0 +1,32 @@ +import logging +import galaxy.util.shed_util_common as suc +from galaxy import web +from galaxy.web.base.controller import BaseAPIController + +import pkg_resources +pkg_resources.require( "Routes" ) +import routes + +log = logging.getLogger( __name__ ) + +class RepositoryContentsController( BaseAPIController ): + @web.expose_api + def index( self, trans, repository_id, **kwd ): + """ + GET /api/repositories/{encoded_repsository_id}/contents + Displays a collection (list) of repository contents. + + :param repository_id: an encoded id string of the `Repository` to inspect + """ + rval = [] + try: + repository = suc.get_repository_in_tool_shed( trans, repository_id ) + repository_dict = repository.as_dict( trans ) + repository_dict[ 'url' ] = web.url_for( 'repository_contents', repository_id=repository_id ) + rval.append( repository_dict ) + except Exception, e: + message = "Error in repository_contents API: %s" % str( e ) + log.error( message, exc_info=True ) + trans.response.status = 500 + return message + return rval diff -r c32f70e92aa428dcab3d753ef88a5d255272e4f9 -r 73914d7ce26131b0670f82e3596d2a56494f0480 lib/galaxy/webapps/community/api/repository_revisions.py --- a/lib/galaxy/webapps/community/api/repository_revisions.py +++ b/lib/galaxy/webapps/community/api/repository_revisions.py @@ -1,5 +1,6 @@ +import galaxy.util.shed_util_common as suc from galaxy import web, util -from galaxy.web.base.controller import BaseController, BaseAPIController +from galaxy.web.base.controller import BaseAPIController from galaxy.web.framework.helpers import is_true import pkg_resources @@ -10,7 +11,7 @@ log = logging.getLogger( __name__ ) class RepositoryRevisionsController( BaseAPIController ): - """RESTful controller for interactions with tool shed repositories.""" + """RESTful controller for interactions with tool shed repository revisions.""" @web.expose_api def index( self, trans, downloadable=True, **kwd ): """ @@ -25,11 +26,11 @@ .order_by( trans.app.model.RepositoryMetadata.table.c.repository_id ) \ .all() for repository_metadata in query: - item = repository_metadata.get_api_value( value_mapper={ 'id' : trans.security.encode_id } ) + item = repository_metadata.get_api_value( value_mapper={ 'id' : trans.security.encode_id( repository_metadata.id ) } ) item[ 'url' ] = web.url_for( 'repository_revision', id=trans.security.encode_id( repository_metadata.id ) ) rval.append( item ) except Exception, e: - rval = "Error in repository_revisions API" + rval = "Error in repository_revisions API at index: " + str( e ) log.error( rval + ": %s" % str( e ) ) trans.response.status = 500 return rval diff -r c32f70e92aa428dcab3d753ef88a5d255272e4f9 -r 73914d7ce26131b0670f82e3596d2a56494f0480 lib/galaxy/webapps/community/buildapp.py --- a/lib/galaxy/webapps/community/buildapp.py +++ b/lib/galaxy/webapps/community/buildapp.py @@ -69,8 +69,15 @@ webapp.add_route( '/:controller/:action', action='index' ) webapp.add_route( '/:action', controller='repository', action='index' ) webapp.add_route( '/repos/*path_info', controller='hg', action='handle_request', path_info='/' ) - # Add the web API + # Add the web API. webapp.add_api_controllers( 'galaxy.webapps.community.api', app ) + webapp.api_mapper.resource( 'content', + 'contents', + controller='repository_contents', + name_prefix='repository_', + path_prefix='/api/repositories/:repository_id', + parent_resources=dict( member_name='repository', collection_name='repositories' ) ) + webapp.api_mapper.resource( 'repository', 'repositories', path_prefix='/api' ) webapp.api_mapper.resource( 'repository_revision', 'repository_revisions', path_prefix='/api' ) webapp.finalize_config() # Wrap the webapp in some useful middleware diff -r c32f70e92aa428dcab3d753ef88a5d255272e4f9 -r 73914d7ce26131b0670f82e3596d2a56494f0480 lib/galaxy/webapps/community/model/__init__.py --- a/lib/galaxy/webapps/community/model/__init__.py +++ b/lib/galaxy/webapps/community/model/__init__.py @@ -114,14 +114,16 @@ self.prev_session_id = prev_session_id class Repository( object, APIItem ): - api_collection_visible_keys = ( 'id', 'name' ) - api_element_visible_keys = ( 'id', 'name', 'description' ) + api_collection_visible_keys = ( 'id', 'name', 'description', 'user_id', 'private', 'times_downloaded', 'deprecated' ) + api_element_visible_keys = ( 'id', 'name', 'description', 'long_description', 'user_id', 'private', 'times_downloaded', 'deprecated' ) file_states = Bunch( NORMAL = 'n', NEEDS_MERGING = 'm', MARKED_FOR_REMOVAL = 'r', MARKED_FOR_ADDITION = 'a', NOT_TRACKED = '?' ) - def __init__( self, name=None, description=None, long_description=None, user_id=None, private=False, email_alerts=None, times_downloaded=0, deprecated=False ): + def __init__( self, id=None, name=None, description=None, long_description=None, user_id=None, private=False, email_alerts=None, + times_downloaded=0, deprecated=False ): + self.id = id self.name = name or "Unnamed repository" self.description = description self.long_description = long_description @@ -130,6 +132,28 @@ self.email_alerts = email_alerts self.times_downloaded = times_downloaded self.deprecated = deprecated + def as_dict( self, trans ): + repository_dict = self.get_api_value( view='element' ) + repository_dict[ 'id' ] = trans.security.encode_id( self.id ) + return repository_dict + def get_api_value( self, view='collection', value_mapper=None ): + if value_mapper is None: + value_mapper = {} + rval = {} + try: + visible_keys = self.__getattribute__( 'api_' + view + '_visible_keys' ) + except AttributeError: + raise Exception( 'Unknown API view: %s' % view ) + for key in visible_keys: + try: + rval[ key ] = self.__getattribute__( key ) + if key in value_mapper: + rval[ key ] = value_mapper.get( key, rval[ key ] ) + except AttributeError: + rval[ key ] = None + if 'user_id' in rval: + rval[ 'owner' ] = self.user.username + return rval def repo_path( self, app ): return app.hgweb_config_manager.get_entry( os.path.join( "repos", self.user.username, self.name ) ) def revision( self, app ): @@ -194,7 +218,7 @@ try: rval[ key ] = self.__getattribute__( key ) if key in value_mapper: - rval[ key ] = value_mapper.get( key )( rval[ key ] ) + rval[ key ] = value_mapper.get( key, rval[ key ] ) except AttributeError: rval[ key ] = None return rval Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: jgoecks: Trackster: performance improvements for drawing reference track data.
by Bitbucket 21 Feb '13

21 Feb '13

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/c32f70e92aa4/ changeset: c32f70e92aa4 user: jgoecks date: 2013-02-21 20:37:38 summary: Trackster: performance improvements for drawing reference track data. affected #: 2 files diff -r b7b000557d128c537dfeb5bb4a9260853463793b -r c32f70e92aa428dcab3d753ef88a5d255272e4f9 static/scripts/viz/trackster/painters.js --- a/static/scripts/viz/trackster/painters.js +++ b/static/scripts/viz/trackster/painters.js @@ -803,6 +803,7 @@ } } } else { + // Not enough detail to draw sequence data. ctx.fillStyle = block_color; // TODO: This is a pretty hack-ish way to fill rectangle based on mode. ctx.fillRect(s_start, y_center + 4, s_end - s_start, SQUISH_FEATURE_HEIGHT); diff -r b7b000557d128c537dfeb5bb4a9260853463793b -r c32f70e92aa428dcab3d753ef88a5d255272e4f9 static/scripts/viz/trackster/tracks.js --- a/static/scripts/viz/trackster/tracks.js +++ b/static/scripts/viz/trackster/tracks.js @@ -3465,11 +3465,11 @@ */ draw_helper: function(force, region, resolution, parent_element, w_scale, kwargs) { if (w_scale > this.view.canvas_manager.char_width_px) { - this.show_contents(); + this.tiles_div.show(); return TiledTrack.prototype.draw_helper.call(this, force, region, resolution, parent_element, w_scale, kwargs); } else { - this.hide_contents(); + this.tiles_div.hide(); return null; } }, @@ -3480,22 +3480,18 @@ * Draw ReferenceTrack tile. */ draw_tile: function(data, ctx, mode, resolution, region, w_scale) { - if (w_scale > this.view.canvas_manager.char_width_px) { - // Try to subset data. - var subset = this.data_manager.subset_entry(data, region), - seq_data = subset.data; - - // Draw sequence data. - var canvas = ctx.canvas; - ctx.font = ctx.canvas.manager.default_font; - ctx.textAlign = "center"; - for (var c = 0, str_len = seq_data.length; c < str_len; c++) { - ctx.fillText(seq_data[c], Math.floor(c * w_scale), 10); - } - return new Tile(this, region, resolution, canvas, subset); + // Try to subset data. + var subset = this.data_manager.subset_entry(data, region), + seq_data = subset.data; + + // Draw sequence data. + var canvas = ctx.canvas; + ctx.font = ctx.canvas.manager.default_font; + ctx.textAlign = "center"; + for (var c = 0, str_len = seq_data.length; c < str_len; c++) { + ctx.fillText(seq_data[c], Math.floor(c * w_scale), 10); } - - this.hide_contents(); + return new Tile(this, region, resolution, canvas, subset); } }); Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: jgoecks: Trackster: subset reference data when available.
by Bitbucket 21 Feb '13

21 Feb '13

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/b7b000557d12/ changeset: b7b000557d12 user: jgoecks date: 2013-02-21 20:11:09 summary: Trackster: subset reference data when available. affected #: 2 files diff -r 5a910304b25466d3781821bc35f7d0a6c1eae28a -r b7b000557d128c537dfeb5bb4a9260853463793b static/scripts/viz/trackster/tracks.js --- a/static/scripts/viz/trackster/tracks.js +++ b/static/scripts/viz/trackster/tracks.js @@ -2915,6 +2915,10 @@ if ( is_deferred( seq_data ) ) { can_draw_now = false; } + else { + // Sequence data is available, subset to get only data in region. + seq_data = view.reference_track.data_manager.subset_entry(seq_data, region); + } } // If we can draw now, do so. @@ -3440,7 +3444,8 @@ this.data_url = reference_url + "/" + this.view.dbkey; this.data_url_extra_params = {reference: true}; this.data_manager = new visualization.ReferenceTrackDataManager({ - data_url: this.data_url + data_url: this.data_url, + can_subset: this.can_subset }); this.hide_contents(); }; @@ -3456,10 +3461,11 @@ can_draw: Drawable.prototype.can_draw, /** - * Only retrieves data and draws tile if reference data can be displayed. + * Retrieves data and draws tile if reference data can be displayed. */ draw_helper: function(force, region, resolution, parent_element, w_scale, kwargs) { if (w_scale > this.view.canvas_manager.char_width_px) { + this.show_contents(); return TiledTrack.prototype.draw_helper.call(this, force, region, resolution, parent_element, w_scale, kwargs); } else { @@ -3468,28 +3474,27 @@ } }, + can_subset: function(data) { return true; }, + /** * Draw ReferenceTrack tile. */ - draw_tile: function(seq, ctx, mode, resolution, region, w_scale) { - var track = this; - + draw_tile: function(data, ctx, mode, resolution, region, w_scale) { if (w_scale > this.view.canvas_manager.char_width_px) { - if (seq.data === null) { - this.hide_contents(); - return; - } + // Try to subset data. + var subset = this.data_manager.subset_entry(data, region), + seq_data = subset.data; + + // Draw sequence data. var canvas = ctx.canvas; ctx.font = ctx.canvas.manager.default_font; - ctx.textAlign = "center"; - seq = seq.data; - for (var c = 0, str_len = seq.length; c < str_len; c++) { - var c_start = Math.floor(c * w_scale); - ctx.fillText(seq[c], c_start, 10); + ctx.textAlign = "center"; + for (var c = 0, str_len = seq_data.length; c < str_len; c++) { + ctx.fillText(seq_data[c], Math.floor(c * w_scale), 10); } - this.show_contents(); - return new Tile(track, region, resolution, canvas, seq); + return new Tile(this, region, resolution, canvas, subset); } + this.hide_contents(); } }); diff -r 5a910304b25466d3781821bc35f7d0a6c1eae28a -r b7b000557d128c537dfeb5bb4a9260853463793b static/scripts/viz/visualization.js --- a/static/scripts/viz/visualization.js +++ b/static/scripts/viz/visualization.js @@ -318,6 +318,8 @@ // Do request. var manager = this, entry = $.getJSON(dataset.url(), params, function (result) { + // Add region to the result. + result.region = region; manager.set_data(region, result); }); @@ -582,7 +584,25 @@ return { data: null }; } return GenomeDataManager.prototype.load_data.call(this, region, mode, resolution, extra_params); - } + }, + + /** + * Return an entry that includes only data in the subregion. + */ + subset_entry: function(entry, subregion) { + var seq_data = entry.data; + if (!entry.region.same(subregion)) { + // Need to subset sequence data. + var seq_start = subregion.get('start') - entry.region.get('start'), + seq_end = entry.data.length - ( entry.region.get('end') - subregion.get('end') ); + seq_data = entry.data.slice(seq_start, seq_end); + } + + return { + region: subregion, + data: seq_data + }; + } }); /** @@ -639,6 +659,16 @@ end: 0, genome: null }, + + /** + * Returns true if this region is the same as a given region. + * It does not test the genome right now. + */ + same: function(region) { + return this.attributes.chrom === region.get('chrom') && + this.attributes.start === region.get('start') && + this.attributes.end === region.get('end'); + }, /** * If from_str specified, use it to initialize attributes. Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: greg: Add styles to the readme container.
by Bitbucket 21 Feb '13

21 Feb '13

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/a0741e6798f2/ changeset: a0741e6798f2 branch: stable user: greg date: 2013-02-21 17:49:21 summary: Add styles to the readme container. affected #: 1 file diff -r 04b4efed14bab6a322c3c93d1785cc3cd398e91f -r a0741e6798f28f05bb9bb17b572ce1ea4c62e1f0 templates/webapps/community/repository/common.mako --- a/templates/webapps/community/repository/common.mako +++ b/templates/webapps/community/repository/common.mako @@ -349,13 +349,25 @@ from galaxy.util.shed_util_common import to_safe_string encoded_id = trans.security.encode_id( readme.id ) %> + <style type="text/css"> + #readme_table{ table-layout:fixed; + width:100%; + overflow-wrap:normal; + overflow:hidden; + border:0px; + word-break:keep-all; + word-wrap:break-word; + line-break:strict; } + </style><tr class="datasetRow" %if parent is not None: parent="${parent}" %endif id="libraryItem-${encoded_id}"><td style="padding-left: ${pad+20}px;"> - ${ to_safe_string( readme.text, to_html=True ) } + <table id="readme_table"> + <tr><td>${ to_safe_string( readme.text, to_html=True ) }</td></tr> + </table></td></tr><% Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: dan: Fix for displaying job_id when preparing a job fails.
by Bitbucket 21 Feb '13

21 Feb '13

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/5a910304b254/ changeset: 5a910304b254 user: dan date: 2013-02-21 17:53:32 summary: Fix for displaying job_id when preparing a job fails. affected #: 1 file diff -r bf84118d56670484db09986435bb79376e8fb52b -r 5a910304b25466d3781821bc35f7d0a6c1eae28a lib/galaxy/jobs/runners/__init__.py --- a/lib/galaxy/jobs/runners/__init__.py +++ b/lib/galaxy/jobs/runners/__init__.py @@ -112,7 +112,7 @@ job_wrapper.prepare() job_wrapper.runner_command_line = self.build_command_line( job_wrapper ) except: - log.exception("(%d) Failure preparing job" % job_id) + log.exception("(%s) Failure preparing job" % job_id) job_wrapper.fail( "failure preparing job", exception=True ) return Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: greg: Add styles to the readme container.
by Bitbucket 21 Feb '13

21 Feb '13

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/bf84118d5667/ changeset: bf84118d5667 user: greg date: 2013-02-21 17:49:21 summary: Add styles to the readme container. affected #: 1 file diff -r 45f1d93124ad4cadff01d8ae0fadfb1db5d8fcb2 -r bf84118d56670484db09986435bb79376e8fb52b templates/webapps/community/repository/common.mako --- a/templates/webapps/community/repository/common.mako +++ b/templates/webapps/community/repository/common.mako @@ -462,13 +462,25 @@ from galaxy.util.shed_util_common import to_safe_string encoded_id = trans.security.encode_id( readme.id ) %> + <style type="text/css"> + #readme_table{ table-layout:fixed; + width:100%; + overflow-wrap:normal; + overflow:hidden; + border:0px; + word-break:keep-all; + word-wrap:break-word; + line-break:strict; } + </style><tr class="datasetRow" %if parent is not None: parent="${parent}" %endif id="libraryItem-${encoded_id}"><td style="padding-left: ${pad+20}px;"> - ${ to_safe_string( readme.text, to_html=True ) } + <table id="readme_table"> + <tr><td>${ to_safe_string( readme.text, to_html=True ) }</td></tr> + </table></td></tr><% Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: 2 new changesets
by Bitbucket 21 Feb '13

21 Feb '13

2 new commits in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/04b4efed14ba/ changeset: 04b4efed14ba branch: stable user: inithello date: 2013-02-21 17:06:11 summary: Changed to_safe_string to use markupsafe.escape for unsafe characters. affected #: 1 file diff -r 7c3df0bcbc222f478b1e21886fbb384d3dc96f31 -r 04b4efed14bab6a322c3c93d1785cc3cd398e91f lib/galaxy/util/shed_util_common.py --- a/lib/galaxy/util/shed_util_common.py +++ b/lib/galaxy/util/shed_util_common.py @@ -30,17 +30,11 @@ log = logging.getLogger( __name__ ) INITIAL_CHANGELOG_HASH = '000000000000' -# Characters that must be html escaped -MAPPED_CHARS = { '>' :'>', - '<' :'<', - '"' : '"', - '&' : '&', - '\'' : ''' } MAX_CONTENT_SIZE = 32768 NOT_TOOL_CONFIGS = [ 'datatypes_conf.xml', 'repository_dependencies.xml', 'tool_dependencies.xml' ] GALAXY_ADMIN_TOOL_SHED_CONTROLLER = 'GALAXY_ADMIN_TOOL_SHED_CONTROLLER' TOOL_SHED_ADMIN_CONTROLLER = 'TOOL_SHED_ADMIN_CONTROLLER' -VALID_CHARS = set( string.letters + string.digits + "'\"-=_.()/+*^,:?!#[]%\\$@;{}" ) +VALID_CHARS = set( string.letters + string.digits + "'\"-=_.()/+*^,:?!#[]%\\$@;{}&<>" ) new_repo_email_alert_template = """ Repository name: ${repository_name} @@ -3420,12 +3414,14 @@ def to_safe_string( text, to_html=True ): """Translates the characters in text to an html string""" if text: + if to_html: + escaped_text = str( markupsafe.escape( text ) ) + else: + escaped_text = str( text ) translated = [] - for c in text: + for c in escaped_text: if c in VALID_CHARS: translated.append( c ) - elif c in MAPPED_CHARS: - translated.append( MAPPED_CHARS[ c ] ) elif c in [ '\n' ]: if to_html: translated.append( '<br/>' ) @@ -3437,8 +3433,6 @@ translated.append( c ) else: translated.append( '' ) - if to_html: - str( markupsafe.escape( ''.join( translated ) ) ) return ''.join( translated ) return text def tool_shed_from_repository_clone_url( repository_clone_url ): https://bitbucket.org/galaxy/galaxy-central/commits/45f1d93124ad/ changeset: 45f1d93124ad user: inithello date: 2013-02-21 17:06:11 summary: Changed to_safe_string to use markupsafe.escape for unsafe characters. affected #: 1 file diff -r 1adf6fdd9c49bda204e5d90f70f10e7ef4ec06bd -r 45f1d93124ad4cadff01d8ae0fadfb1db5d8fcb2 lib/galaxy/util/shed_util_common.py --- a/lib/galaxy/util/shed_util_common.py +++ b/lib/galaxy/util/shed_util_common.py @@ -32,18 +32,12 @@ INITIAL_CHANGELOG_HASH = '000000000000' REPOSITORY_DATA_MANAGER_CONFIG_FILENAME = "data_manager_conf.xml" -# Characters that must be html escaped -MAPPED_CHARS = { '>' :'>', - '<' :'<', - '"' : '"', - '&' : '&', - '\'' : ''' } MAX_CONTENT_SIZE = 32768 NOT_TOOL_CONFIGS = [ 'datatypes_conf.xml', 'repository_dependencies.xml', 'tool_dependencies.xml', REPOSITORY_DATA_MANAGER_CONFIG_FILENAME ] GALAXY_ADMIN_TOOL_SHED_CONTROLLER = 'GALAXY_ADMIN_TOOL_SHED_CONTROLLER' TOOL_SHED_ADMIN_CONTROLLER = 'TOOL_SHED_ADMIN_CONTROLLER' TOOL_TYPES_NOT_IN_TOOL_PANEL = [ 'manage_data' ] -VALID_CHARS = set( string.letters + string.digits + "'\"-=_.()/+*^,:?!#[]%\\$@;{}" ) +VALID_CHARS = set( string.letters + string.digits + "'\"-=_.()/+*^,:?!#[]%\\$@;{}&<>" ) new_repo_email_alert_template = """ Sharable link: ${sharable_link} @@ -3802,12 +3796,14 @@ def to_safe_string( text, to_html=True ): """Translates the characters in text to an html string""" if text: + if to_html: + escaped_text = str( markupsafe.escape( text ) ) + else: + escaped_text = str( text ) translated = [] - for c in text: + for c in escaped_text: if c in VALID_CHARS: translated.append( c ) - elif c in MAPPED_CHARS: - translated.append( MAPPED_CHARS[ c ] ) elif c in [ '\n' ]: if to_html: translated.append( '<br/>' ) @@ -3825,8 +3821,6 @@ translated.append( c ) else: translated.append( '' ) - if to_html: - str( markupsafe.escape( ''.join( translated ) ) ) return ''.join( translated ) return text def tool_shed_from_repository_clone_url( repository_clone_url ): Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: 2 new changesets
by Bitbucket 20 Feb '13

20 Feb '13

2 new commits in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/9697f9fc92b1/ changeset: 9697f9fc92b1 user: inithello date: 2013-02-20 21:51:56 summary: Fix an import in the repository_revisions API controller. affected #: 1 file diff -r 1bd67f03d4f12d0f6cddacb57a6402cf306f8f55 -r 9697f9fc92b1bc8fbfaca72917eeff928c2cc4ac lib/galaxy/webapps/community/api/repository_revisions.py --- a/lib/galaxy/webapps/community/api/repository_revisions.py +++ b/lib/galaxy/webapps/community/api/repository_revisions.py @@ -1,10 +1,11 @@ -from galaxy import web, util, logging +from galaxy import web, util from galaxy.web.base.controller import BaseController, BaseAPIController from galaxy.web.framework.helpers import is_true import pkg_resources pkg_resources.require( "Routes" ) import routes +import logging log = logging.getLogger( __name__ ) https://bitbucket.org/galaxy/galaxy-central/commits/1adf6fdd9c49/ changeset: 1adf6fdd9c49 user: inithello date: 2013-02-20 21:52:10 summary: Add do_not_test, tools_functionally_correct, time_last_tested, and tool_test_errors columns to the repository_metadata table. affected #: 3 files diff -r 9697f9fc92b1bc8fbfaca72917eeff928c2cc4ac -r 1adf6fdd9c49bda204e5d90f70f10e7ef4ec06bd lib/galaxy/webapps/community/model/__init__.py --- a/lib/galaxy/webapps/community/model/__init__.py +++ b/lib/galaxy/webapps/community/model/__init__.py @@ -170,13 +170,18 @@ class RepositoryMetadata( object, APIItem ): api_collection_visible_keys = ( 'id', 'repository_id', 'changeset_revision', 'malicious', 'downloadable' ) api_element_visible_keys = ( 'id', 'repository_id', 'changeset_revision', 'malicious', 'downloadable' ) - def __init__( self, repository_id=None, changeset_revision=None, metadata=None, tool_versions=None, malicious=False, downloadable=False ): + def __init__( self, repository_id=None, changeset_revision=None, metadata=None, tool_versions=None, malicious=False, downloadable=False, + tools_functionally_correct=False, do_not_test=False, time_last_tested=None, tool_test_errors=None ): self.repository_id = repository_id self.changeset_revision = changeset_revision self.metadata = metadata or dict() self.tool_versions = tool_versions or dict() self.malicious = malicious self.downloadable = downloadable + self.tools_functionally_correct = tools_functionally_correct + self.do_not_test = do_not_test + self.time_last_tested = time_last_tested + self.tool_test_errors = tool_test_errors def get_api_value( self, view='collection', value_mapper=None ): if value_mapper is None: value_mapper = {} diff -r 9697f9fc92b1bc8fbfaca72917eeff928c2cc4ac -r 1adf6fdd9c49bda204e5d90f70f10e7ef4ec06bd lib/galaxy/webapps/community/model/mapping.py --- a/lib/galaxy/webapps/community/model/mapping.py +++ b/lib/galaxy/webapps/community/model/mapping.py @@ -130,7 +130,11 @@ Column( "metadata", JSONType, nullable=True ), Column( "tool_versions", JSONType, nullable=True ), Column( "malicious", Boolean, default=False ), - Column( "downloadable", Boolean, default=True ) ) + Column( "downloadable", Boolean, default=True ), + Column( "tools_functionally_correct", Boolean, default=False, index=True ), + Column( "do_not_test", Boolean, default=False, index=True ), + Column( "time_last_tested", DateTime, default=None, nullable=True ), + Column( "tool_test_errors", JSONType, nullable=True ) ) RepositoryReview.table = Table( "repository_review", metadata, Column( "id", Integer, primary_key=True ), diff -r 9697f9fc92b1bc8fbfaca72917eeff928c2cc4ac -r 1adf6fdd9c49bda204e5d90f70f10e7ef4ec06bd lib/galaxy/webapps/community/model/migrate/versions/0016_add_do_not_test_tools_functionally_correct_errors_columns.py --- /dev/null +++ b/lib/galaxy/webapps/community/model/migrate/versions/0016_add_do_not_test_tools_functionally_correct_errors_columns.py @@ -0,0 +1,99 @@ +""" +Migration script to add the tool_test_errors, do_not_test, tools_functionally_correct, and time_last_tested columns to the repository_metadata table. +""" + +from sqlalchemy import * +from sqlalchemy.orm import * +from migrate import * +from migrate.changeset import * + +# Need our custom types, but don't import anything else from model +from galaxy.model.custom_types import * + +import sys, logging +log = logging.getLogger( __name__ ) +log.setLevel(logging.DEBUG) +handler = logging.StreamHandler( sys.stdout ) +format = "%(name)s %(levelname)s %(asctime)s %(message)s" +formatter = logging.Formatter( format ) +handler.setFormatter( formatter ) +log.addHandler( handler ) + +metadata = MetaData( migrate_engine ) +db_session = scoped_session( sessionmaker( bind=migrate_engine, autoflush=False, autocommit=True ) ) + +def upgrade(): + print __doc__ + metadata.reflect() + # Create and initialize tools_functionally_correct, do_not_test, time_last_tested, and tool_test_errors columns in repository_metadata table. + RepositoryMetadata_table = Table( "repository_metadata", metadata, autoload=True ) + c = Column( "tools_functionally_correct", Boolean, default=False, index=True ) + try: + # Create tools_functionally_correct column + c.create( RepositoryMetadata_table ) + assert c is RepositoryMetadata_table.c.tools_functionally_correct + # Initialize. + if migrate_engine.name == 'mysql' or migrate_engine.name == 'sqlite': + default_false = "0" + elif migrate_engine.name == 'postgres': + default_false = "false" + db_session.execute( "UPDATE repository_metadata SET tools_functionally_correct=%s" % default_false ) + except Exception, e: + print "Adding tools_functionally_correct column to the repository_metadata table failed: %s" % str( e ) + log.debug( "Adding tools_functionally_correct column to the repository_metadata table failed: %s" % str( e ) ) + c = Column( "do_not_test", Boolean, default=False, index=True ) + try: + # Create do_not_test column + c.create( RepositoryMetadata_table ) + assert c is RepositoryMetadata_table.c.do_not_test + # Initialize. + if migrate_engine.name == 'mysql' or migrate_engine.name == 'sqlite': + default_false = "0" + elif migrate_engine.name == 'postgres': + default_false = "false" + db_session.execute( "UPDATE repository_metadata SET do_not_test=%s" % default_false ) + except Exception, e: + print "Adding do_not_test column to the repository_metadata table failed: %s" % str( e ) + log.debug( "Adding do_not_test column to the repository_metadata table failed: %s" % str( e ) ) + c = Column( "time_last_tested", DateTime, default=None, nullable=True ) + try: + # Create time_last_tested column + c.create( RepositoryMetadata_table ) + assert c is RepositoryMetadata_table.c.time_last_tested + except Exception, e: + print "Adding time_last_tested column to the repository_metadata table failed: %s" % str( e ) + log.debug( "Adding time_last_tested column to the repository_metadata table failed: %s" % str( e ) ) + c = Column( "tool_test_errors", JSONType, nullable=True ) + try: + pass + # Create tool_test_errors column + c.create( RepositoryMetadata_table ) + assert c is RepositoryMetadata_table.c.tool_test_errors + except Exception, e: + print "Adding tool_test_errors column to the repository_metadata table failed: %s" % str( e ) + log.debug( "Adding tool_test_errors column to the repository_metadata table failed: %s" % str( e ) ) + +def downgrade(): + metadata.reflect() + # Drop tool_test_errors, time_last_tested, do_not_test, and tools_functionally_correct columns from repository_metadata table. + RepositoryMetadata_table = Table( "repository_metadata", metadata, autoload=True ) + try: + RepositoryMetadata_table.c.tool_test_errors.drop() + except Exception, e: + print "Dropping column tool_test_errors from the repository_metadata table failed: %s" % str( e ) + log.debug( "Dropping column tool_test_errors from the repository_metadata table failed: %s" % str( e ) ) + try: + RepositoryMetadata_table.c.time_last_tested.drop() + except Exception, e: + print "Dropping column time_last_tested from the repository_metadata table failed: %s" % str( e ) + log.debug( "Dropping column time_last_tested from the repository_metadata table failed: %s" % str( e ) ) + try: + RepositoryMetadata_table.c.do_not_test.drop() + except Exception, e: + print "Dropping column do_not_test from the repository_metadata table failed: %s" % str( e ) + log.debug( "Dropping column do_not_test from the repository_metadata table failed: %s" % str( e ) ) + try: + RepositoryMetadata_table.c.tools_functionally_correct.drop() + except Exception, e: + print "Dropping column tools_functionally_correct from the repository_metadata table failed: %s" % str( e ) + log.debug( "Dropping column tools_functionally_correct from the repository_metadata table failed: %s" % str( e ) ) Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: greg: Add the API framework to the Galaxy Tool Shed.
by Bitbucket 20 Feb '13

20 Feb '13

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/1bd67f03d4f1/ changeset: 1bd67f03d4f1 user: greg date: 2013-02-20 20:55:26 summary: Add the API framework to the Galaxy Tool Shed. affected #: 10 files diff -r 0fb1af8ce7410cd48c9259962a4e0da528ff4ef6 -r 1bd67f03d4f12d0f6cddacb57a6402cf306f8f55 lib/galaxy/webapps/community/api/repository_revisions.py --- /dev/null +++ b/lib/galaxy/webapps/community/api/repository_revisions.py @@ -0,0 +1,34 @@ +from galaxy import web, util, logging +from galaxy.web.base.controller import BaseController, BaseAPIController +from galaxy.web.framework.helpers import is_true + +import pkg_resources +pkg_resources.require( "Routes" ) +import routes + +log = logging.getLogger( __name__ ) + +class RepositoryRevisionsController( BaseAPIController ): + """RESTful controller for interactions with tool shed repositories.""" + @web.expose_api + def index( self, trans, downloadable=True, **kwd ): + """ + GET /api/repository_revisions + Displays a collection (list) of repository revisions. + """ + rval = [] + downloadable = util.string_as_bool( downloadable ) + try: + query = trans.sa_session.query( trans.app.model.RepositoryMetadata ) \ + .filter( trans.app.model.RepositoryMetadata.table.c.downloadable == downloadable ) \ + .order_by( trans.app.model.RepositoryMetadata.table.c.repository_id ) \ + .all() + for repository_metadata in query: + item = repository_metadata.get_api_value( value_mapper={ 'id' : trans.security.encode_id } ) + item[ 'url' ] = web.url_for( 'repository_revision', id=trans.security.encode_id( repository_metadata.id ) ) + rval.append( item ) + except Exception, e: + rval = "Error in repository_revisions API" + log.error( rval + ": %s" % str( e ) ) + trans.response.status = 500 + return rval diff -r 0fb1af8ce7410cd48c9259962a4e0da528ff4ef6 -r 1bd67f03d4f12d0f6cddacb57a6402cf306f8f55 lib/galaxy/webapps/community/buildapp.py --- a/lib/galaxy/webapps/community/buildapp.py +++ b/lib/galaxy/webapps/community/buildapp.py @@ -69,6 +69,9 @@ webapp.add_route( '/:controller/:action', action='index' ) webapp.add_route( '/:action', controller='repository', action='index' ) webapp.add_route( '/repos/*path_info', controller='hg', action='handle_request', path_info='/' ) + # Add the web API + webapp.add_api_controllers( 'galaxy.webapps.community.api', app ) + webapp.api_mapper.resource( 'repository_revision', 'repository_revisions', path_prefix='/api' ) webapp.finalize_config() # Wrap the webapp in some useful middleware if kwargs.get( 'middleware', True ): diff -r 0fb1af8ce7410cd48c9259962a4e0da528ff4ef6 -r 1bd67f03d4f12d0f6cddacb57a6402cf306f8f55 lib/galaxy/webapps/community/framework/middleware/remoteuser.py --- a/lib/galaxy/webapps/community/framework/middleware/remoteuser.py +++ b/lib/galaxy/webapps/community/framework/middleware/remoteuser.py @@ -75,6 +75,9 @@ elif path_info.startswith( '/api/' ): # The API handles its own authentication via keys return self.app( environ, start_response ) + elif path_info.startswith( '/user/api_keys' ): + # api_keys can be managed when remote_user is in use. + pass else: title = "Access to this Galaxy tool shed is denied" message = """ diff -r 0fb1af8ce7410cd48c9259962a4e0da528ff4ef6 -r 1bd67f03d4f12d0f6cddacb57a6402cf306f8f55 lib/galaxy/webapps/community/model/__init__.py --- a/lib/galaxy/webapps/community/model/__init__.py +++ b/lib/galaxy/webapps/community/model/__init__.py @@ -9,14 +9,20 @@ from galaxy.util.bunch import Bunch from galaxy.util.hash_util import new_secure_hash from galaxy.web.form_builder import * +from galaxy.model.item_attrs import APIItem from galaxy import eggs eggs.require('mercurial') from mercurial import hg, ui log = logging.getLogger( __name__ ) - -class User( object ): + +class APIKeys( object ): + pass + +class User( object, APIItem ): + api_collection_visible_keys = ( 'id', 'email' ) + api_element_visible_keys = ( 'id', 'email', 'username' ) def __init__( self, email=None, password=None ): self.email = email self.password = password @@ -47,12 +53,16 @@ def nice_total_disk_usage( self ): return 0 -class Group( object ): +class Group( object, APIItem ): + api_collection_visible_keys = ( 'id', 'name' ) + api_element_visible_keys = ( 'id', 'name' ) def __init__( self, name = None ): self.name = name self.deleted = False -class Role( object ): +class Role( object, APIItem ): + api_collection_visible_keys = ( 'id', 'name' ) + api_element_visible_keys = ( 'id', 'name', 'description', 'type' ) private_id = None types = Bunch( PRIVATE = 'private', @@ -103,7 +113,9 @@ self.is_valid = is_valid self.prev_session_id = prev_session_id -class Repository( object ): +class Repository( object, APIItem ): + api_collection_visible_keys = ( 'id', 'name' ) + api_element_visible_keys = ( 'id', 'name', 'description' ) file_states = Bunch( NORMAL = 'n', NEEDS_MERGING = 'm', MARKED_FOR_REMOVAL = 'r', @@ -155,7 +167,9 @@ fp.write( line ) fp.close() -class RepositoryMetadata( object ): +class RepositoryMetadata( object, APIItem ): + api_collection_visible_keys = ( 'id', 'repository_id', 'changeset_revision', 'malicious', 'downloadable' ) + api_element_visible_keys = ( 'id', 'repository_id', 'changeset_revision', 'malicious', 'downloadable' ) def __init__( self, repository_id=None, changeset_revision=None, metadata=None, tool_versions=None, malicious=False, downloadable=False ): self.repository_id = repository_id self.changeset_revision = changeset_revision @@ -163,8 +177,26 @@ self.tool_versions = tool_versions or dict() self.malicious = malicious self.downloadable = downloadable + def get_api_value( self, view='collection', value_mapper=None ): + if value_mapper is None: + value_mapper = {} + rval = {} + try: + visible_keys = self.__getattribute__( 'api_' + view + '_visible_keys' ) + except AttributeError: + raise Exception( 'Unknown API view: %s' % view ) + for key in visible_keys: + try: + rval[ key ] = self.__getattribute__( key ) + if key in value_mapper: + rval[ key ] = value_mapper.get( key )( rval[ key ] ) + except AttributeError: + rval[ key ] = None + return rval -class RepositoryReview( object ): +class RepositoryReview( object, APIItem ): + api_collection_visible_keys = ( 'id', 'repository_id', 'changeset_revision', 'user_id', 'rating' ) + api_element_visible_keys = ( 'id', 'repository_id', 'changeset_revision', 'user_id', 'rating' ) approved_states = Bunch( NO='no', YES='yes' ) def __init__( self, repository_id=None, changeset_revision=None, user_id=None, rating=None, deleted=False ): self.repository_id = repository_id @@ -173,7 +205,9 @@ self.rating = rating self.deleted = deleted -class ComponentReview( object ): +class ComponentReview( object, APIItem ): + api_collection_visible_keys = ( 'id', 'repository_review_id', 'component_id', 'private', 'approved', 'rating', 'deleted' ) + api_element_visible_keys = ( 'id', 'repository_review_id', 'component_id', 'private', 'approved', 'rating', 'deleted' ) approved_states = Bunch( NO='no', YES='yes', NA='not_applicable' ) def __init__( self, repository_review_id=None, component_id=None, comment=None, private=False, approved=False, rating=None, deleted=False ): self.repository_review_id = repository_review_id @@ -204,7 +238,9 @@ def set_item( self, repository ): self.repository = repository -class Category( object ): +class Category( object, APIItem ): + api_collection_visible_keys = ( 'id', 'name', 'description', 'deleted' ) + api_element_visible_keys = ( 'id', 'name', 'description', 'deleted' ) def __init__( self, name=None, description=None, deleted=False ): self.name = name self.description = description diff -r 0fb1af8ce7410cd48c9259962a4e0da528ff4ef6 -r 1bd67f03d4f12d0f6cddacb57a6402cf306f8f55 lib/galaxy/webapps/community/model/mapping.py --- a/lib/galaxy/webapps/community/model/mapping.py +++ b/lib/galaxy/webapps/community/model/mapping.py @@ -39,6 +39,12 @@ # Return the current time in UTC without any timezone information now = datetime.datetime.utcnow +APIKeys.table = Table( "api_keys", metadata, + Column( "id", Integer, primary_key=True ), + Column( "create_time", DateTime, default=now ), + Column( "user_id", Integer, ForeignKey( "galaxy_user.id" ), index=True ), + Column( "key", TrimmedString( 32 ), index=True, unique=True ) ) + User.table = Table( "galaxy_user", metadata, Column( "id", Integer, primary_key=True), Column( "create_time", DateTime, default=now ), @@ -186,7 +192,11 @@ # With the tables defined we can define the mappers and setup the relationships between the model objects. assign_mapper( context, User, User.table, properties=dict( active_repositories=relation( Repository, primaryjoin=( ( Repository.table.c.user_id == User.table.c.id ) & ( not_( Repository.table.c.deleted ) ) ), order_by=( Repository.table.c.name ) ), - galaxy_sessions=relation( GalaxySession, order_by=desc( GalaxySession.table.c.update_time ) ) ) ) + galaxy_sessions=relation( GalaxySession, order_by=desc( GalaxySession.table.c.update_time ) ), + api_keys=relation( APIKeys, backref="user", order_by=desc( APIKeys.table.c.create_time ) ) ) ) + +assign_mapper( context, APIKeys, APIKeys.table, + properties = {} ) assign_mapper( context, Group, Group.table, properties=dict( users=relation( UserGroupAssociation ) ) ) diff -r 0fb1af8ce7410cd48c9259962a4e0da528ff4ef6 -r 1bd67f03d4f12d0f6cddacb57a6402cf306f8f55 lib/galaxy/webapps/community/model/migrate/versions/0015_add_api_keys_table.py --- /dev/null +++ b/lib/galaxy/webapps/community/model/migrate/versions/0015_add_api_keys_table.py @@ -0,0 +1,46 @@ +""" +Migration script to add the api_keys table. +""" +import datetime, sys, logging +from sqlalchemy import * +from sqlalchemy.orm import * +from migrate import * +from migrate.changeset import * +from galaxy.model.custom_types import * + +log = logging.getLogger( __name__ ) +log.setLevel(logging.DEBUG) +handler = logging.StreamHandler( sys.stdout ) +format = "%(name)s %(levelname)s %(asctime)s %(message)s" +formatter = logging.Formatter( format ) +handler.setFormatter( formatter ) +log.addHandler( handler ) + +now = datetime.datetime.utcnow + +metadata = MetaData( migrate_engine ) +db_session = scoped_session( sessionmaker( bind=migrate_engine, autoflush=False, autocommit=True ) ) + +metadata = MetaData( migrate_engine ) + +APIKeys_table = Table( "api_keys", metadata, + Column( "id", Integer, primary_key=True ), + Column( "create_time", DateTime, default=now ), + Column( "user_id", Integer, ForeignKey( "galaxy_user.id" ), index=True ), + Column( "key", TrimmedString( 32 ), index=True, unique=True ) ) + +def upgrade(): + print __doc__ + metadata.reflect() + try: + APIKeys_table.create() + except Exception, e: + log.debug( "Creating api_keys table failed: %s" % str( e ) ) + +def downgrade(): + # Load existing tables + metadata.reflect() + try: + APIKeys_table.drop() + except Exception, e: + log.debug( "Dropping api_keys table failed: %s" % str( e ) ) diff -r 0fb1af8ce7410cd48c9259962a4e0da528ff4ef6 -r 1bd67f03d4f12d0f6cddacb57a6402cf306f8f55 templates/user/index.mako --- a/templates/user/index.mako +++ b/templates/user/index.mako @@ -25,6 +25,7 @@ %endif %else: <li><a href="${h.url_for( controller='user', action='manage_user_info', cntrller=cntrller )}">${_('Manage your information')}</a></li> + <li><a href="${h.url_for( controller='user', action='api_keys', cntrller=cntrller )}">${_('Manage your API keys')}</a></li><li><a href="${h.url_for( controller='repository', action='manage_email_alerts', cntrller=cntrller )}">${_('Manage your email alerts')}</a></li><li><a href="${h.url_for( controller='user', action='logout', logout_all=True )}" target="_top">${_('Logout')}</a> ${_('of all user sessions')}</li> %endif diff -r 0fb1af8ce7410cd48c9259962a4e0da528ff4ef6 -r 1bd67f03d4f12d0f6cddacb57a6402cf306f8f55 templates/webapps/community/base_panels.mako --- a/templates/webapps/community/base_panels.mako +++ b/templates/webapps/community/base_panels.mako @@ -105,6 +105,7 @@ menu_options.append( [ _('Logout'), app.config.remote_user_logout_href, "_top" ] ) else: menu_options.append( [ _('Preferences'), h.url_for( controller='/user', action='index', cntrller='user' ), "galaxy_main" ] ) + menu_options.append( [ _('API Keys'), h.url_for( controller='/user', action='api_keys', cntrller='user' ), "galaxy_main" ] ) logout_url = h.url_for( controller='/user', action='logout' ) menu_options.append( [ 'Logout', logout_url, "_top" ] ) menu_options.append( None ) diff -r 0fb1af8ce7410cd48c9259962a4e0da528ff4ef6 -r 1bd67f03d4f12d0f6cddacb57a6402cf306f8f55 templates/webapps/galaxy/user/api_keys.mako --- a/templates/webapps/galaxy/user/api_keys.mako +++ b/templates/webapps/galaxy/user/api_keys.mako @@ -30,11 +30,14 @@ (invalidates old key) %endif <div class="toolParamHelp" style="clear: both;"> - An API key will allow you to access Galaxy via its web - API (documentation forthcoming). Please note that - <strong>this key acts as an alternate means to access - your account, and should be treated with the same care - as your login password</strong>. + <% + if trans.webapp.name == 'galaxy': + webapp_str = 'Galaxy' + else: + webapp_str = 'the Tool Shed' + %> + An API key will allow you to access ${webapp_str} via its web API. Please note that <strong>this key acts as an alternate means + to access your account and should be treated with the same care as your login password</strong>. </div></div></form> Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: inithello: Tool shed functional test enhancement, made the grant_write_access method more robust, added functional test for contacting a repository's owner.
by Bitbucket 20 Feb '13

20 Feb '13

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/0fb1af8ce741/ changeset: 0fb1af8ce741 user: inithello date: 2013-02-20 20:33:04 summary: Tool shed functional test enhancement, made the grant_write_access method more robust, added functional test for contacting a repository's owner. affected #: 3 files diff -r b9c93f559efa7a50cfa445b5f79c579690155589 -r 0fb1af8ce7410cd48c9259962a4e0da528ff4ef6 test/tool_shed/base/twilltestcase.py --- a/test/tool_shed/base/twilltestcase.py +++ b/test/tool_shed/base/twilltestcase.py @@ -587,13 +587,20 @@ self.visit_url( url ) strings_displayed = [ "Role '%s' has been updated" % role.name ] self.check_for_strings( strings_displayed, strings_not_displayed ) - def grant_write_access( self, repository, usernames=[], strings_displayed=[], strings_not_displayed=[] ): + def grant_write_access( self, + repository, + usernames=[], + strings_displayed=[], + strings_not_displayed=[], + post_submit_strings_displayed=[], + post_submit_strings_not_displayed=[] ): self.display_manage_repository_page( repository ) - tc.fv( "3", "allow_push", '-Select one' ) + self.check_for_strings( strings_displayed, strings_not_displayed ) + tc.fv( "user_access", "allow_push", '-Select one' ) for username in usernames: - tc.fv( "3", "allow_push", '+%s' % username ) + tc.fv( "user_access", "allow_push", '+%s' % username ) tc.submit( 'user_access_button' ) - self.check_for_strings( strings_displayed, strings_not_displayed ) + self.check_for_strings( post_submit_strings_displayed, post_submit_strings_not_displayed ) def import_workflow( self, repository, workflow_name, strings_displayed=[], strings_not_displayed=[] ): url = '/admin_toolshed/import_workflow?repository_id=%s&workflow_name=%s' % \ ( self.security.encode_id( repository.id ), tool_shed_encode( workflow_name ) ) @@ -847,6 +854,19 @@ tc.fv( "1", field_name, search_string ) tc.submit() self.check_for_strings( strings_displayed, strings_not_displayed ) + def send_message_to_repository_owner( self, + repository, + message, + strings_displayed=[], + strings_not_displayed=[], + post_submit_strings_displayed=[], + post_submit_strings_not_displayed=[] ): + url = '/repository/contact_owner?id=%s' % self.security.encode_id( repository.id ) + self.visit_url( url ) + self.check_for_strings( strings_displayed, strings_not_displayed ) + tc.fv( 1, 'message', message ) + tc.submit() + self.check_for_strings( post_submit_strings_displayed, post_submit_strings_not_displayed ) def set_repository_deprecated( self, repository, set_deprecated=True, strings_displayed=[], strings_not_displayed=[] ): url = '/repository/deprecate?id=%s&mark_deprecated=%s' % ( self.security.encode_id( repository.id ), set_deprecated ) self.visit_url( url ) diff -r b9c93f559efa7a50cfa445b5f79c579690155589 -r 0fb1af8ce7410cd48c9259962a4e0da528ff4ef6 test/tool_shed/functional/test_0000_basic_repository_features.py --- a/test/tool_shed/functional/test_0000_basic_repository_features.py +++ b/test/tool_shed/functional/test_0000_basic_repository_features.py @@ -9,11 +9,17 @@ '''Test core repository features.''' def test_0000_initiate_users( self ): """Create necessary user accounts and login as an admin user.""" + self.logout() self.login( email=common.test_user_1_email, username=common.test_user_1_name ) test_user_1 = test_db_util.get_user( common.test_user_1_email ) assert test_user_1 is not None, 'Problem retrieving user with email %s from the database' % common.test_user_1_email test_user_1_private_role = test_db_util.get_private_role( test_user_1 ) self.logout() + self.login( email=common.test_user_2_email, username=common.test_user_2_name ) + test_user_2 = test_db_util.get_user( common.test_user_2_email ) + assert test_user_2 is not None, 'Problem retrieving user with email %s from the database' % common.test_user_2_email + test_user_2_private_role = test_db_util.get_private_role( test_user_2 ) + self.logout() self.login( email=common.admin_email, username=common.admin_username ) admin_user = test_db_util.get_user( common.admin_email ) assert admin_user is not None, 'Problem retrieving user with email %s from the database' % common.admin_email @@ -51,8 +57,8 @@ def test_0025_grant_write_access( self ): '''Grant write access to another user''' repository = test_db_util.get_repository_by_name_and_owner( repository_name, common.test_user_1_name ) - self.grant_write_access( repository, usernames=[ common.admin_username ] ) - self.revoke_write_access( repository, common.admin_username ) + self.grant_write_access( repository, usernames=[ common.test_user_2_name ] ) + self.revoke_write_access( repository, common.test_user_2_name ) def test_0030_upload_filtering_1_1_0( self ): """Upload filtering_1.1.0.tar to the repository""" repository = test_db_util.get_repository_by_name_and_owner( repository_name, common.test_user_1_name ) @@ -214,3 +220,22 @@ assert test_user_1 is None, 'Creating user with public name "repos" succeeded.' error_message = 'The term <b>repos</b> is a reserved word in the tool shed, so it cannot be used as a public user name.' self.check_for_strings( strings_displayed=[ error_message ] ) + def test_0105_contact_repository_owner( self ): + '''Fill out and submit the form to contact the owner of a repository.''' + ''' + This test should not actually send the email, since functional tests are designed to function without + any external network connection. The embedded tool shed server these tests are running against has been configured + with an SMTP server address that will not and should not resolve correctly. However, since the successful sending of + the email is the last step in the process, this will verify functional correctness of all preceding steps. + ''' + self.logout() + self.login( email=common.test_user_2_email, username=common.test_user_2_name ) + repository = test_db_util.get_repository_by_name_and_owner( repository_name, common.test_user_1_name ) + message = 'This is a test message.' + strings_displayed = [ 'Contact the owner of the repository named', repository.name, 'streamline appropriate communication' ] + post_submit_strings_displayed = [ 'An error occurred sending your message by email' ] + self.send_message_to_repository_owner( repository=repository, + message=message, + strings_displayed=strings_displayed, + post_submit_strings_displayed=post_submit_strings_displayed ) + diff -r b9c93f559efa7a50cfa445b5f79c579690155589 -r 0fb1af8ce7410cd48c9259962a4e0da528ff4ef6 test/tool_shed/functional_tests.py --- a/test/tool_shed/functional_tests.py +++ b/test/tool_shed/functional_tests.py @@ -199,6 +199,8 @@ new_file_path = new_repos_path, running_functional_tests = True, shed_tool_data_table_config = shed_tool_data_table_conf_file, + smtp_server = 'smtp.dummy.string.tld', + email_from = 'functional@localhost', template_path = 'templates', tool_path=tool_path, tool_parse_help = False, Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0