- galaxy-commits - lists.galaxyproject.org

commit/galaxy-central: 2 new changesets
by commits-noreply＠bitbucket.org 07 Aug '13

07 Aug '13

2 new commits in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/6e72b04fcb58/ Changeset: 6e72b04fcb58 User: abretaud Date: 2013-06-27 11:45:46 Summary: When creating the temp directory for dataset zip downloads, use the Galaxy user's umask and group on the directory. (same idea as 068a043) Affected #: 1 file diff -r 3fa9df444b4b81f94b1c42a033c685a6e23827be -r 6e72b04fcb582208a9a2bb57a942d8f2a582e94c lib/galaxy/datatypes/data.py --- a/lib/galaxy/datatypes/data.py +++ b/lib/galaxy/datatypes/data.py @@ -230,6 +230,7 @@ if (params.do_action == 'zip'): # Can't use mkstemp - the file must not exist first tmpd = tempfile.mkdtemp() + util.umask_fix_perms( tmpd, trans.app.config.umask, 0777, trans.app.config.gid ) tmpf = os.path.join( tmpd, 'library_download.' + params.do_action ) if ziptype == '64': archive = zipfile.ZipFile( tmpf, 'w', zipfile.ZIP_DEFLATED, True ) https://bitbucket.org/galaxy/galaxy-central/commits/446e2fee6b8f/ Changeset: 446e2fee6b8f User: dannon Date: 2013-08-07 20:38:31 Summary: Merged in abretaud/galaxy-central (pull request #191) When creating the temp directory for dataset zip downloads, use the Galaxy user's umask and group on the directory. Affected #: 1 file diff -r b8cf5887ad464707887aaf8381df19dbc67ac697 -r 446e2fee6b8f978472e67031f429aeda6cdfffdd lib/galaxy/datatypes/data.py --- a/lib/galaxy/datatypes/data.py +++ b/lib/galaxy/datatypes/data.py @@ -230,6 +230,7 @@ if (params.do_action == 'zip'): # Can't use mkstemp - the file must not exist first tmpd = tempfile.mkdtemp() + util.umask_fix_perms( tmpd, trans.app.config.umask, 0777, trans.app.config.gid ) tmpf = os.path.join( tmpd, 'library_download.' + params.do_action ) if ziptype == '64': archive = zipfile.ZipFile( tmpf, 'w', zipfile.ZIP_DEFLATED, True ) Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: greg: Relocate the tool shed's container_util.
by commits-noreply＠bitbucket.org 07 Aug '13

07 Aug '13

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/b8cf5887ad46/ Changeset: b8cf5887ad46 User: greg Date: 2013-08-07 20:37:22 Summary: Relocate the tool shed's container_util. Affected #: 11 files diff -r f02a75ce05b71457845c0e5f56f409047a841967 -r b8cf5887ad464707887aaf8381df19dbc67ac697 lib/galaxy/webapps/tool_shed/controllers/repository.py --- a/lib/galaxy/webapps/tool_shed/controllers/repository.py +++ b/lib/galaxy/webapps/tool_shed/controllers/repository.py @@ -17,6 +17,7 @@ from galaxy.util import json from galaxy.model.orm import and_ import tool_shed.util.shed_util_common as suc +from tool_shed.util import container_util from tool_shed.util import encoding_util from tool_shed.util import export_util from tool_shed.util import metadata_util @@ -30,7 +31,6 @@ from tool_shed.util import workflow_util from tool_shed.galaxy_install import repository_util from galaxy.webapps.tool_shed.util import common_util -from galaxy.webapps.tool_shed.util import container_util import galaxy.tools import tool_shed.grids.repository_grids as repository_grids import tool_shed.grids.util as grids_util diff -r f02a75ce05b71457845c0e5f56f409047a841967 -r b8cf5887ad464707887aaf8381df19dbc67ac697 lib/galaxy/webapps/tool_shed/controllers/repository_review.py --- a/lib/galaxy/webapps/tool_shed/controllers/repository_review.py +++ b/lib/galaxy/webapps/tool_shed/controllers/repository_review.py @@ -5,7 +5,7 @@ from sqlalchemy.sql.expression import func from galaxy.model.orm import and_ from galaxy.webapps.tool_shed.util import common_util -from galaxy.webapps.tool_shed.util.container_util import STRSEP +from tool_shed.util.container_util import STRSEP import tool_shed.util.shed_util_common as suc from tool_shed.util import review_util from galaxy.util.odict import odict diff -r f02a75ce05b71457845c0e5f56f409047a841967 -r b8cf5887ad464707887aaf8381df19dbc67ac697 lib/galaxy/webapps/tool_shed/util/container_util.py --- a/lib/galaxy/webapps/tool_shed/util/container_util.py +++ /dev/null @@ -1,1408 +0,0 @@ -import logging -import os -import threading -from galaxy.util import asbool -from galaxy.web.framework.helpers import time_ago -from tool_shed.util import readme_util -import tool_shed.util.shed_util_common as suc - -log = logging.getLogger( __name__ ) - -# String separator -STRSEP = '__ESEP__' - - -class Folder( object ): - """Container object.""" - - def __init__( self, id=None, key=None, label=None, parent=None ): - self.id = id - self.key = key - self.label = label - self.parent = parent - self.description = None - self.datatypes = [] - self.folders = [] - self.invalid_repository_dependencies = [] - self.invalid_tool_dependencies = [] - self.invalid_tools = [] - self.installation_errors = [] - self.current_repository_installation_errors = [] - self.repository_installation_errors = [] - self.tool_dependency_installation_errors = [] - self.valid_tools = [] - self.valid_data_managers = [] - self.invalid_data_managers = [] - self.tool_dependencies = [] - self.failed_tests = [] - self.missing_test_components = [] - self.not_tested = [] - self.passed_tests = [] - self.test_environments = [] - self.repository_dependencies = [] - self.readme_files = [] - self.workflows = [] - - def contains_folder( self, folder ): - for index, contained_folder in enumerate( self.folders ): - if folder == contained_folder: - return index, contained_folder - return 0, None - - def contains_repository_dependency( self, repository_dependency ): - listified_repository_dependency = repository_dependency.listify - for contained_repository_dependency in self.repository_dependencies: - if contained_repository_dependency.listify == listified_repository_dependency: - return True - return False - - def remove_repository_dependency( self, repository_dependency ): - listified_repository_dependency = repository_dependency.listify - for contained_repository_dependency in self.repository_dependencies: - if contained_repository_dependency.listify == listified_repository_dependency: - self.repository_dependencies.remove( contained_repository_dependency ) - - def to_repository_dependency( self, repository_dependency_id ): - toolshed, name, owner, changeset_revision, prior_installation_required = suc.parse_repository_dependency_tuple( self.key.split( STRSEP ) ) - return RepositoryDependency( id=repository_dependency_id, - toolshed=toolshed, - repository_name=name, - repository_owner=owner, - changeset_revision=changeset_revision, - prior_installation_required=asbool( prior_installation_required ) ) - - -class DataManager( object ): - """Data Manager object""" - - def __init__( self, id=None, name=None, version=None, data_tables=None ): - self.id = id - self.name = name - self.version = version - self.data_tables = data_tables - - -class Datatype( object ): - """Datatype object""" - - def __init__( self, id=None, extension=None, type=None, mimetype=None, subclass=None, converters=None, display_app_containers=None ): - self.id = id - self.extension = extension - self.type = type - self.mimetype = mimetype - self.subclass = subclass - self.converters = converters - self.display_app_containers = display_app_containers - - -class FailedTest( object ): - """Failed tool tests object""" - - def __init__( self, id=None, stderr=None, test_id=None, tool_id=None, tool_version=None, traceback=None ): - self.id = id - self.stderr = stderr - self.test_id = test_id - self.tool_id = tool_id - self.tool_version = tool_version - self.traceback = traceback - - -class InvalidDataManager( object ): - """Invalid data Manager object""" - - def __init__( self, id=None, index=None, error=None ): - self.id = id - self.index = index - self.error = error - - -class InvalidRepositoryDependency( object ): - """Invalid repository dependency definition object""" - - def __init__( self, id=None, toolshed=None, repository_name=None, repository_owner=None, changeset_revision=None, prior_installation_required=False, error=None ): - self.id = id - self.toolshed = toolshed - self.repository_name = repository_name - self.repository_owner = repository_owner - self.changeset_revision = changeset_revision - self.prior_installation_required = prior_installation_required - self.error = error - - -class InvalidTool( object ): - """Invalid tool object""" - - def __init__( self, id=None, tool_config=None, repository_id=None, changeset_revision=None, repository_installation_status=None ): - self.id = id - self.tool_config = tool_config - self.repository_id = repository_id - self.changeset_revision = changeset_revision - self.repository_installation_status = repository_installation_status - - -class InvalidToolDependency( object ): - """Invalid tool dependency definition object""" - - def __init__( self, id=None, name=None, version=None, type=None, error=None ): - self.id = id - self.name = name - self.version = version - self.type = type - self.error = error - - -class MissingTestComponent( object ): - """Missing tool test components object""" - - def __init__( self, id=None, missing_components=None, tool_guid=None, tool_id=None, tool_version=None ): - self.id = id - self.missing_components = missing_components - self.tool_guid = tool_guid - self.tool_id = tool_id - self.tool_version = tool_version - - -class NotTested( object ): - """NotTested object""" - - def __init__( self, id=None, reason=None ): - self.id = id - self.reason = reason - - -class PassedTest( object ): - """Passed tool tests object""" - - def __init__( self, id=None, test_id=None, tool_id=None, tool_version=None ): - self.id = id - self.test_id = test_id - self.tool_id = tool_id - self.tool_version = tool_version - - -class ReadMe( object ): - """Readme text object""" - - def __init__( self, id=None, name=None, text=None ): - self.id = id - self.name = name - self.text = text - - -class RepositoryDependency( object ): - """Repository dependency object""" - - def __init__( self, id=None, toolshed=None, repository_name=None, repository_owner=None, changeset_revision=None, prior_installation_required=False, - installation_status=None, tool_shed_repository_id=None ): - self.id = id - self.toolshed = toolshed - self.repository_name = repository_name - self.repository_owner = repository_owner - self.changeset_revision = changeset_revision - self.prior_installation_required = prior_installation_required - self.installation_status = installation_status - self.tool_shed_repository_id = tool_shed_repository_id - - @property - def listify( self ): - return [ self.toolshed, self.repository_name, self.repository_owner, self.changeset_revision, asbool( str( self.prior_installation_required ) ) ] - - -class RepositoryInstallationError( object ): - """Repository installation error object""" - - def __init__( self, id=None, tool_shed=None, name=None, owner=None, changeset_revision=None, error_message=None ): - self.id = id - self.tool_shed = tool_shed - self.name = name - self.owner = owner - self.changeset_revision = changeset_revision - self.error_message = error_message - - -class TestEnvironment( object ): - """Tool test environment object""" - - def __init__( self, id=None, architecture=None, galaxy_database_version=None, galaxy_revision=None, python_version=None, system=None, time_last_tested=None, - tool_shed_database_version=None, tool_shed_mercurial_version=None, tool_shed_revision=None ): - self.id = id - self.architecture = architecture - self.galaxy_database_version = galaxy_database_version - self.galaxy_revision = galaxy_revision - self.python_version = python_version - self.system = system - self.time_last_tested = time_last_tested - self.tool_shed_database_version = tool_shed_database_version - self.tool_shed_mercurial_version = tool_shed_mercurial_version - self.tool_shed_revision = tool_shed_revision - - -class Tool( object ): - """Tool object""" - - def __init__( self, id=None, tool_config=None, tool_id=None, name=None, description=None, version=None, requirements=None, - repository_id=None, changeset_revision=None, repository_installation_status=None ): - self.id = id - self.tool_config = tool_config - self.tool_id = tool_id - self.name = name - self.description = description - self.version = version - self.requirements = requirements - self.repository_id = repository_id - self.changeset_revision = changeset_revision - self.repository_installation_status = repository_installation_status - - -class ToolDependency( object ): - """Tool dependency object""" - - def __init__( self, id=None, name=None, version=None, type=None, readme=None, installation_status=None, repository_id=None, - tool_dependency_id=None, is_orphan=None ): - self.id = id - self.name = name - self.version = version - self.type = type - self.readme = readme - self.installation_status = installation_status - self.repository_id = repository_id - self.tool_dependency_id = tool_dependency_id - self.is_orphan = is_orphan - - @property - def listify( self ): - return [ self.name, self.version, self.type ] - - -class ToolDependencyInstallationError( object ): - """Tool dependency installation error object""" - - def __init__( self, id=None, type=None, name=None, version=None, error_message=None ): - self.id = id - self.type = type - self.name = name - self.version = version - self.error_message = error_message - - -class Workflow( object ): - """Workflow object.""" - - def __init__( self, id=None, workflow_name=None, steps=None, format_version=None, annotation=None, repository_metadata_id=None, repository_id=None ): - # When rendered in the tool shed, repository_metadata_id will have a value and repository_id will be None. When rendered in Galaxy, repository_id - # will have a value and repository_metadata_id will be None. - self.id = id - self.workflow_name = workflow_name - self.steps = steps - self.format_version = format_version - self.annotation = annotation - self.repository_metadata_id = repository_metadata_id - self.repository_id = repository_id - -def add_orphan_settings_to_tool_dependencies( tool_dependencies, orphan_tool_dependencies ): - """Inspect all received tool dependencies and label those that are orphans within the repository.""" - orphan_env_dependencies = orphan_tool_dependencies.get( 'set_environment', None ) - new_tool_dependencies = {} - if tool_dependencies: - for td_key, requirements_dict in tool_dependencies.items(): - if td_key in [ 'set_environment' ]: - # "set_environment": [{"name": "R_SCRIPT_PATH", "type": "set_environment"}] - if orphan_env_dependencies: - new_set_environment_dict_list = [] - for set_environment_dict in requirements_dict: - if set_environment_dict in orphan_env_dependencies: - set_environment_dict[ 'is_orphan' ] = True - else: - set_environment_dict[ 'is_orphan' ] = False - new_set_environment_dict_list.append( set_environment_dict ) - new_tool_dependencies[ td_key ] = new_set_environment_dict_list - else: - new_tool_dependencies[ td_key ] = requirements_dict - else: - # {"R/2.15.1": {"name": "R", "readme": "some string", "type": "package", "version": "2.15.1"} - if td_key in orphan_tool_dependencies: - requirements_dict[ 'is_orphan' ] = True - else: - requirements_dict[ 'is_orphan' ] = False - new_tool_dependencies[ td_key ] = requirements_dict - return new_tool_dependencies - -def build_data_managers_folder( trans, folder_id, data_managers, label=None ): - """Return a folder hierarchy containing Data Managers.""" - if data_managers: - if label is None: - label = "Data Managers" - data_manager_id = 0 - folder_id += 1 - data_managers_root_folder = Folder( id=folder_id, key='root', label='root', parent=None ) - folder_id += 1 - key = "valid_data_managers" - folder = Folder( id=folder_id, key=key, label=label, parent=data_managers_root_folder ) - data_managers_root_folder.folders.append( folder ) - # Insert a header row. - data_manager_id += 1 - data_manager = DataManager( id=data_manager_id, - name='Name', - version='Version', - data_tables='Data Tables' ) - folder.valid_data_managers.append( data_manager ) - for data_manager_dict in data_managers.itervalues(): - data_manager_id += 1 - data_manager = DataManager( id=data_manager_id, - name=data_manager_dict.get( 'name', '' ), - version=data_manager_dict.get( 'version', '' ), - data_tables=", ".join( data_manager_dict.get( 'data_tables', '' ) ) ) - folder.valid_data_managers.append( data_manager ) - else: - data_managers_root_folder = None - return folder_id, data_managers_root_folder - -def build_datatypes_folder( trans, folder_id, datatypes, label='Datatypes' ): - """Return a folder hierarchy containing datatypes.""" - if datatypes: - datatype_id = 0 - folder_id += 1 - datatypes_root_folder = Folder( id=folder_id, key='root', label='root', parent=None ) - folder_id += 1 - folder = Folder( id=folder_id, key='datatypes', label=label, parent=datatypes_root_folder ) - datatypes_root_folder.folders.append( folder ) - # Insert a header row. - datatype_id += 1 - datatype = Datatype( id=datatype_id, - extension='extension', - type='type', - mimetype='mimetype', - subclass='subclass' ) - folder.datatypes.append( datatype ) - for datatypes_dict in datatypes: - # {"converters": - # [{"target_datatype": "gff", - # "tool_config": "bed_to_gff_converter.xml", - # "guid": "localhost:9009/repos/test/bed_to_gff_converter/CONVERTER_bed_to_gff_0/2.0.0"}], - # "display_in_upload": "true", - # "dtype": "galaxy.datatypes.interval:Bed", - # "extension": "bed"} - # TODO: converters and display_app information is not currently rendered. Should it be? - # Handle defined converters, if any. - converters = datatypes_dict.get( 'converters', None ) - if converters: - num_converters = len( converters ) - else: - num_converters = 0 - # Handle defined display applications, if any. - display_app_containers = datatypes_dict.get( 'display_app_containers', None ) - if display_app_containers: - num_display_app_containers = len( display_app_containers ) - else: - num_display_app_containers = 0 - datatype_id += 1 - datatype = Datatype( id=datatype_id, - extension=datatypes_dict.get( 'extension', '' ), - type=datatypes_dict.get( 'dtype', '' ), - mimetype=datatypes_dict.get( 'mimetype', '' ), - subclass=datatypes_dict.get( 'subclass', '' ), - converters=num_converters, - display_app_containers=num_display_app_containers ) - folder.datatypes.append( datatype ) - else: - datatypes_root_folder = None - return folder_id, datatypes_root_folder - -def build_invalid_data_managers_folder( trans, folder_id, data_managers, error_messages=None, label=None ): - """Return a folder hierarchy containing invalid Data Managers.""" - if data_managers or error_messages: - if label is None: - label = "Invalid Data Managers" - data_manager_id = 0 - folder_id += 1 - data_managers_root_folder = Folder( id=folder_id, key='root', label='root', parent=None ) - folder_id += 1 - key = "invalid_data_managers" - folder = Folder( id=folder_id, key=key, label=label, parent=data_managers_root_folder ) - data_managers_root_folder.folders.append( folder ) - # Insert a header row. - data_manager_id += 1 - data_manager = InvalidDataManager( id=data_manager_id, - index='Element Index', - error='Error' ) - folder.invalid_data_managers.append( data_manager ) - if error_messages: - for error_message in error_messages: - data_manager_id += 1 - data_manager = InvalidDataManager( id=data_manager_id, - index=0, - error=error_message ) - folder.invalid_data_managers.append( data_manager ) - has_errors = True - for data_manager_dict in data_managers: - data_manager_id += 1 - data_manager = InvalidDataManager( id=data_manager_id, - index=data_manager_dict.get( 'index', 0 ) + 1, - error=data_manager_dict.get( 'error_message', '' ) ) - folder.invalid_data_managers.append( data_manager ) - has_errors = True - else: - data_managers_root_folder = None - return folder_id, data_managers_root_folder - -def build_invalid_repository_dependencies_root_folder( trans, folder_id, invalid_repository_dependencies_dict ): - """Return a folder hierarchy containing invalid repository dependencies.""" - label = 'Invalid repository dependencies' - if invalid_repository_dependencies_dict: - invalid_repository_dependency_id = 0 - folder_id += 1 - invalid_repository_dependencies_root_folder = Folder( id=folder_id, key='root', label='root', parent=None ) - folder_id += 1 - invalid_repository_dependencies_folder = Folder( id=folder_id, - key='invalid_repository_dependencies', - label=label, - parent=invalid_repository_dependencies_root_folder ) - invalid_repository_dependencies_root_folder.folders.append( invalid_repository_dependencies_folder ) - invalid_repository_dependencies = invalid_repository_dependencies_dict[ 'repository_dependencies' ] - for invalid_repository_dependency in invalid_repository_dependencies: - folder_id += 1 - invalid_repository_dependency_id += 1 - toolshed, name, owner, changeset_revision, prior_installation_required, error = \ - suc.parse_repository_dependency_tuple( invalid_repository_dependency, contains_error=True ) - key = generate_repository_dependencies_key_for_repository( toolshed, name, owner, changeset_revision, prior_installation_required ) - label = "Repository <b>%s</b> revision <b>%s</b> owned by <b>%s</b>" % ( name, changeset_revision, owner ) - folder = Folder( id=folder_id, - key=key, - label=label, - parent=invalid_repository_dependencies_folder ) - ird = InvalidRepositoryDependency( id=invalid_repository_dependency_id, - toolshed=toolshed, - repository_name=name, - repository_owner=owner, - changeset_revision=changeset_revision, - prior_installation_required=asbool( prior_installation_required ), - error=error ) - folder.invalid_repository_dependencies.append( ird ) - invalid_repository_dependencies_folder.folders.append( folder ) - else: - invalid_repository_dependencies_root_folder = None - return folder_id, invalid_repository_dependencies_root_folder - -def build_invalid_tool_dependencies_root_folder( trans, folder_id, invalid_tool_dependencies_dict ): - """Return a folder hierarchy containing invalid tool dependencies.""" - # # INvalid tool dependencies are always packages like: - # {"R/2.15.1": {"name": "R", "readme": "some string", "type": "package", "version": "2.15.1" "error" : "some sting" } - label = 'Invalid tool dependencies' - if invalid_tool_dependencies_dict: - invalid_tool_dependency_id = 0 - folder_id += 1 - invalid_tool_dependencies_root_folder = Folder( id=folder_id, key='root', label='root', parent=None ) - folder_id += 1 - invalid_tool_dependencies_folder = Folder( id=folder_id, - key='invalid_tool_dependencies', - label=label, - parent=invalid_tool_dependencies_root_folder ) - invalid_tool_dependencies_root_folder.folders.append( invalid_tool_dependencies_folder ) - for td_key, requirements_dict in invalid_tool_dependencies_dict.items(): - folder_id += 1 - invalid_tool_dependency_id += 1 - name = requirements_dict[ 'name' ] - type = requirements_dict[ 'type' ] - version = requirements_dict[ 'version' ] - error = requirements_dict[ 'error' ] - key = generate_tool_dependencies_key( name, version, type ) - label = "Version <b>%s</b> of the <b>%s</b><b>%s</b>" % ( version, name, type ) - folder = Folder( id=folder_id, - key=key, - label=label, - parent=invalid_tool_dependencies_folder ) - itd = InvalidToolDependency( id=invalid_tool_dependency_id, - name=name, - version=version, - type=type, - error=error ) - folder.invalid_tool_dependencies.append( itd ) - invalid_tool_dependencies_folder.folders.append( folder ) - else: - invalid_tool_dependencies_root_folder = None - return folder_id, invalid_tool_dependencies_root_folder - -def build_invalid_tools_folder( trans, folder_id, invalid_tool_configs, changeset_revision, repository=None, label='Invalid tools' ): - """Return a folder hierarchy containing invalid tools.""" - # TODO: Should we display invalid tools on the tool panel selection page when installing the repository into Galaxy? - if invalid_tool_configs: - invalid_tool_id = 0 - folder_id += 1 - invalid_tools_root_folder = Folder( id=folder_id, key='root', label='root', parent=None ) - folder_id += 1 - folder = Folder( id=folder_id, key='invalid_tools', label=label, parent=invalid_tools_root_folder ) - invalid_tools_root_folder.folders.append( folder ) - for invalid_tool_config in invalid_tool_configs: - invalid_tool_id += 1 - if repository: - repository_id = repository.id - if trans.webapp.name == 'galaxy': - repository_installation_status = repository.status - else: - repository_installation_status = None - else: - repository_id = None - repository_installation_status = None - invalid_tool = InvalidTool( id=invalid_tool_id, - tool_config=invalid_tool_config, - repository_id=repository_id, - changeset_revision=changeset_revision, - repository_installation_status=repository_installation_status ) - folder.invalid_tools.append( invalid_tool ) - else: - invalid_tools_root_folder = None - return folder_id, invalid_tools_root_folder - -def build_readme_files_folder( trans, folder_id, readme_files_dict, label='Readme files' ): - """Return a folder hierarchy containing readme text files.""" - if readme_files_dict: - multiple_readme_files = len( readme_files_dict ) > 1 - readme_id = 0 - folder_id += 1 - readme_files_root_folder = Folder( id=folder_id, key='root', label='root', parent=None ) - if multiple_readme_files: - folder_id += 1 - readme_files_folder = Folder( id=folder_id, key='readme_files', label=label, parent=readme_files_root_folder ) - readme_files_root_folder.folders.append( readme_files_folder ) - for readme_file_name, readme_file_text in readme_files_dict.items(): - readme_id += 1 - readme = ReadMe( id=readme_id, name=readme_file_name, text=readme_file_text ) - if multiple_readme_files: - folder_id += 1 - folder = Folder( id=folder_id, key=readme.name, label=readme.name, parent=readme_files_folder ) - folder.readme_files.append( readme ) - readme_files_folder.folders.append( folder ) - else: - folder_id += 1 - readme_files_folder = Folder( id=folder_id, key='readme_files', label=readme.name, parent=readme_files_root_folder ) - readme_files_folder.readme_files.append( readme ) - readme_files_root_folder.folders.append( readme_files_folder ) - else: - readme_files_root_folder = None - return folder_id, readme_files_root_folder - -def build_repository_containers_for_galaxy( trans, repository, datatypes, invalid_tools, missing_repository_dependencies, missing_tool_dependencies, - readme_files_dict, repository_dependencies, tool_dependencies, valid_tools, workflows, valid_data_managers, - invalid_data_managers, data_managers_errors, new_install=False, reinstalling=False ): - """Return a dictionary of containers for the received repository's dependencies and readme files for display during installation to Galaxy.""" - containers_dict = dict( datatypes=None, - invalid_tools=None, - missing_tool_dependencies=None, - readme_files=None, - repository_dependencies=None, - missing_repository_dependencies=None, - tool_dependencies=None, - valid_tools=None, - workflows=None, - valid_data_managers=None, - invalid_data_managers=None ) - # Some of the tool dependency folders will include links to display tool dependency information, and some of these links require the repository - # id. However we need to be careful because sometimes the repository object is None. - if repository: - repository_id = repository.id - changeset_revision = repository.changeset_revision - else: - repository_id = None - changeset_revision = None - lock = threading.Lock() - lock.acquire( True ) - try: - folder_id = 0 - # Datatypes container. - if datatypes: - folder_id, datatypes_root_folder = build_datatypes_folder( trans, folder_id, datatypes ) - containers_dict[ 'datatypes' ] = datatypes_root_folder - # Invalid tools container. - if invalid_tools: - folder_id, invalid_tools_root_folder = build_invalid_tools_folder( trans, - folder_id, - invalid_tools, - changeset_revision, - repository=repository, - label='Invalid tools' ) - containers_dict[ 'invalid_tools' ] = invalid_tools_root_folder - # Readme files container. - if readme_files_dict: - folder_id, readme_files_root_folder = build_readme_files_folder( trans, folder_id, readme_files_dict ) - containers_dict[ 'readme_files' ] = readme_files_root_folder - # Installed repository dependencies container. - if repository_dependencies: - if new_install: - label = 'Repository dependencies' - else: - label = 'Installed repository dependencies' - folder_id, repository_dependencies_root_folder = build_repository_dependencies_folder( trans=trans, - folder_id=folder_id, - repository_dependencies=repository_dependencies, - label=label, - installed=True ) - containers_dict[ 'repository_dependencies' ] = repository_dependencies_root_folder - # Missing repository dependencies container. - if missing_repository_dependencies: - folder_id, missing_repository_dependencies_root_folder = \ - build_repository_dependencies_folder( trans=trans, - folder_id=folder_id, - repository_dependencies=missing_repository_dependencies, - label='Missing repository dependencies', - installed=False ) - containers_dict[ 'missing_repository_dependencies' ] = missing_repository_dependencies_root_folder - # Installed tool dependencies container. - if tool_dependencies: - if new_install: - label = 'Tool dependencies' - else: - label = 'Installed tool dependencies' - # We only want to display the Status column if the tool_dependency is missing. - folder_id, tool_dependencies_root_folder = build_tool_dependencies_folder( trans, - folder_id, - tool_dependencies, - label=label, - missing=False, - new_install=new_install, - reinstalling=reinstalling ) - containers_dict[ 'tool_dependencies' ] = tool_dependencies_root_folder - # Missing tool dependencies container. - if missing_tool_dependencies: - # We only want to display the Status column if the tool_dependency is missing. - folder_id, missing_tool_dependencies_root_folder = build_tool_dependencies_folder( trans, - folder_id, - missing_tool_dependencies, - label='Missing tool dependencies', - missing=True, - new_install=new_install, - reinstalling=reinstalling ) - containers_dict[ 'missing_tool_dependencies' ] = missing_tool_dependencies_root_folder - # Valid tools container. - if valid_tools: - folder_id, valid_tools_root_folder = build_tools_folder( trans, - folder_id, - valid_tools, - repository, - changeset_revision, - label='Valid tools' ) - containers_dict[ 'valid_tools' ] = valid_tools_root_folder - # Workflows container. - if workflows: - folder_id, workflows_root_folder = build_workflows_folder( trans=trans, - folder_id=folder_id, - workflows=workflows, - repository_metadata_id=None, - repository_id=repository_id, - label='Workflows' ) - containers_dict[ 'workflows' ] = workflows_root_folder - if valid_data_managers: - folder_id, valid_data_managers_root_folder = build_data_managers_folder( trans=trans, - folder_id=folder_id, - data_managers=valid_data_managers, - label='Valid Data Managers' ) - containers_dict[ 'valid_data_managers' ] = valid_data_managers_root_folder - if invalid_data_managers or data_managers_errors: - folder_id, invalid_data_managers_root_folder = build_invalid_data_managers_folder( trans=trans, - folder_id=folder_id, - data_managers=invalid_data_managers, - error_messages=data_managers_errors, - label='Invalid Data Managers' ) - containers_dict[ 'invalid_data_managers' ] = invalid_data_managers_root_folder - except Exception, e: - log.debug( "Exception in build_repository_containers_for_galaxy: %s" % str( e ) ) - finally: - lock.release() - return containers_dict - -def build_repository_containers_for_tool_shed( trans, repository, changeset_revision, repository_dependencies, repository_metadata, exclude=None ): - """Return a dictionary of containers for the received repository's dependencies and contents for display in the tool shed.""" - if exclude is None: - exclude = [] - containers_dict = dict( datatypes=None, - invalid_tools=None, - readme_files=None, - repository_dependencies=None, - tool_dependencies=None, - valid_tools=None, - workflows=None, - valid_data_managers=None - ) - if repository_metadata: - metadata = repository_metadata.metadata - tool_test_results = repository_metadata.tool_test_results - try: - time_last_tested = time_ago( repository_metadata.time_last_tested ) - except: - time_last_tested = None - lock = threading.Lock() - lock.acquire( True ) - try: - folder_id = 0 - # Datatypes container. - if metadata: - if 'datatypes' not in exclude and 'datatypes' in metadata: - datatypes = metadata[ 'datatypes' ] - folder_id, datatypes_root_folder = build_datatypes_folder( trans, folder_id, datatypes ) - containers_dict[ 'datatypes' ] = datatypes_root_folder - # Invalid repository dependencies container. - if metadata: - if 'invalid_repository_dependencies' not in exclude and 'invalid_repository_dependencies' in metadata: - invalid_repository_dependencies = metadata[ 'invalid_repository_dependencies' ] - folder_id, invalid_repository_dependencies_root_folder = \ - build_invalid_repository_dependencies_root_folder( trans, - folder_id, - invalid_repository_dependencies ) - containers_dict[ 'invalid_repository_dependencies' ] = invalid_repository_dependencies_root_folder - # Invalid tool dependencies container. - if metadata: - if 'invalid_tool_dependencies' not in exclude and 'invalid_tool_dependencies' in metadata: - invalid_tool_dependencies = metadata[ 'invalid_tool_dependencies' ] - folder_id, invalid_tool_dependencies_root_folder = \ - build_invalid_tool_dependencies_root_folder( trans, - folder_id, - invalid_tool_dependencies ) - containers_dict[ 'invalid_tool_dependencies' ] = invalid_tool_dependencies_root_folder - # Invalid tools container. - if metadata: - if 'invalid_tools' not in exclude and 'invalid_tools' in metadata: - invalid_tool_configs = metadata[ 'invalid_tools' ] - folder_id, invalid_tools_root_folder = build_invalid_tools_folder( trans, - folder_id, - invalid_tool_configs, - changeset_revision, - repository=repository, - label='Invalid tools' ) - containers_dict[ 'invalid_tools' ] = invalid_tools_root_folder - # Readme files container. - if metadata: - if 'readme_files' not in exclude and 'readme_files' in metadata: - readme_files_dict = readme_util.build_readme_files_dict( metadata ) - folder_id, readme_files_root_folder = build_readme_files_folder( trans, folder_id, readme_files_dict ) - containers_dict[ 'readme_files' ] = readme_files_root_folder - if 'repository_dependencies' not in exclude: - # Repository dependencies container. - folder_id, repository_dependencies_root_folder = build_repository_dependencies_folder( trans=trans, - folder_id=folder_id, - repository_dependencies=repository_dependencies, - label='Repository dependencies', - installed=False ) - if repository_dependencies_root_folder: - containers_dict[ 'repository_dependencies' ] = repository_dependencies_root_folder - # Tool dependencies container. - if metadata: - if 'tool_dependencies' not in exclude and 'tool_dependencies' in metadata: - tool_dependencies = metadata[ 'tool_dependencies' ] - if trans.webapp.name == 'tool_shed': - if 'orphan_tool_dependencies' in metadata: - orphan_tool_dependencies = metadata[ 'orphan_tool_dependencies' ] - tool_dependencies = add_orphan_settings_to_tool_dependencies( tool_dependencies, orphan_tool_dependencies ) - folder_id, tool_dependencies_root_folder = build_tool_dependencies_folder( trans, - folder_id, - tool_dependencies, - missing=False, - new_install=False ) - containers_dict[ 'tool_dependencies' ] = tool_dependencies_root_folder - # Valid tools container. - if metadata: - if 'tools' not in exclude and 'tools' in metadata: - valid_tools = metadata[ 'tools' ] - folder_id, valid_tools_root_folder = build_tools_folder( trans, - folder_id, - valid_tools, - repository, - changeset_revision, - label='Valid tools' ) - containers_dict[ 'valid_tools' ] = valid_tools_root_folder - # Tool test results container. - if 'tool_test_results' not in exclude and tool_test_results and len( tool_test_results ) > 1: - # Only create and populate this folder if there are actual tool test results to display, since the display of the 'Test environment' - # folder by itself can be misleading. We check for more than a single entry in the tool_test_results dictionary because it may have - # only the "test_environment" entry, but we want at least 1 of "passed_tests", "failed_tests", "installation_errors", "missing_test_components" - # "skipped_tests", "not_tested" or any other entry that may be added in the future. - folder_id, tool_test_results_root_folder = build_tool_test_results_folder( trans, folder_id, tool_test_results, time_last_tested=time_last_tested ) - containers_dict[ 'tool_test_results' ] = tool_test_results_root_folder - # Workflows container. - if metadata: - if 'workflows' not in exclude and 'workflows' in metadata: - workflows = metadata[ 'workflows' ] - folder_id, workflows_root_folder = build_workflows_folder( trans=trans, - folder_id=folder_id, - workflows=workflows, - repository_metadata_id=repository_metadata.id, - repository_id=None, - label='Workflows' ) - containers_dict[ 'workflows' ] = workflows_root_folder - # Valid Data Managers container - if metadata: - if 'data_manager' not in exclude and 'data_manager' in metadata: - data_managers = metadata['data_manager'].get( 'data_managers', None ) - folder_id, data_managers_root_folder = build_data_managers_folder( trans, folder_id, data_managers, label="Data Managers" ) - containers_dict[ 'valid_data_managers' ] = data_managers_root_folder - error_messages = metadata['data_manager'].get( 'error_messages', None ) - data_managers = metadata['data_manager'].get( 'invalid_data_managers', None ) - folder_id, data_managers_root_folder = build_invalid_data_managers_folder( trans, folder_id, data_managers, error_messages, label="Invalid Data Managers" ) - containers_dict[ 'invalid_data_managers' ] = data_managers_root_folder - except Exception, e: - log.debug( "Exception in build_repository_containers_for_tool_shed: %s" % str( e ) ) - finally: - lock.release() - return containers_dict - -def build_repository_dependencies_folder( trans, folder_id, repository_dependencies, label='Repository dependencies', installed=False ): - """Return a folder hierarchy containing repository dependencies.""" - if repository_dependencies: - repository_dependency_id = 0 - folder_id += 1 - # Create the root folder. - repository_dependencies_root_folder = Folder( id=folder_id, key='root', label='root', parent=None ) - folder_id += 1 - # Create the Repository dependencies folder and add it to the root folder. - repository_dependencies_folder_key = repository_dependencies[ 'root_key' ] - repository_dependencies_folder = Folder( id=folder_id, key=repository_dependencies_folder_key, label=label, parent=repository_dependencies_root_folder ) - del repository_dependencies[ 'root_key' ] - # The received repository_dependencies is a dictionary with keys: 'root_key', 'description', and one or more repository_dependency keys. - # We want the description value associated with the repository_dependencies_folder. - repository_dependencies_folder.description = repository_dependencies.get( 'description', None ) - repository_dependencies_root_folder.folders.append( repository_dependencies_folder ) - del repository_dependencies[ 'description' ] - repository_dependencies_folder, folder_id, repository_dependency_id = \ - populate_repository_dependencies_container( trans, repository_dependencies_folder, repository_dependencies, folder_id, repository_dependency_id ) - repository_dependencies_folder = prune_repository_dependencies( repository_dependencies_folder ) - else: - repository_dependencies_root_folder = None - return folder_id, repository_dependencies_root_folder - -def build_tools_folder( trans, folder_id, tool_dicts, repository, changeset_revision, valid=True, label='Valid tools' ): - """Return a folder hierarchy containing valid tools.""" - if tool_dicts: - tool_id = 0 - folder_id += 1 - tools_root_folder = Folder( id=folder_id, key='root', label='root', parent=None ) - folder_id += 1 - folder = Folder( id=folder_id, key='tools', label=label, parent=tools_root_folder ) - if trans.webapp.name == 'galaxy': - folder.description = 'click the name to inspect the tool metadata' - tools_root_folder.folders.append( folder ) - # Insert a header row. - tool_id += 1 - tool = Tool( id=tool_id, - tool_config='', - tool_id='', - name='Name', - description='Description', - version='Version', - requirements='', - repository_id='', - changeset_revision='' ) - folder.valid_tools.append( tool ) - if repository: - repository_id = repository.id - if trans.webapp.name == 'galaxy': - repository_installation_status = repository.status - else: - repository_installation_status = None - else: - repository_id = None - repository_installation_status = None - for tool_dict in tool_dicts: - tool_id += 1 - if 'requirements' in tool_dict: - requirements = tool_dict[ 'requirements' ] - requirements_str = '' - for requirement_dict in requirements: - requirements_str += '%s (%s), ' % ( requirement_dict[ 'name' ], requirement_dict[ 'type' ] ) - requirements_str = requirements_str.rstrip( ', ' ) - else: - requirements_str = 'none' - tool = Tool( id=tool_id, - tool_config=tool_dict[ 'tool_config' ], - tool_id=tool_dict[ 'id' ], - name=tool_dict[ 'name' ], - description=tool_dict[ 'description' ], - version=tool_dict[ 'version' ], - requirements=requirements_str, - repository_id=repository_id, - changeset_revision=changeset_revision, - repository_installation_status=repository_installation_status ) - folder.valid_tools.append( tool ) - else: - tools_root_folder = None - return folder_id, tools_root_folder - -def build_tool_dependencies_folder( trans, folder_id, tool_dependencies, label='Tool dependencies', missing=False, new_install=False, reinstalling=False ): - """Return a folder hierarchy containing tool dependencies.""" - # When we're in Galaxy (not the tool shed) and the tool dependencies are not installed or are in an error state, they are considered missing. The tool - # dependency status will be displayed only if a record exists for the tool dependency in the Galaxy database, but the tool dependency is not installed. - # The value for new_install will be True only if the associated repository in being installed for the first time. This value is used in setting the - # container description. - if tool_dependencies: - tool_dependency_id = 0 - folder_id += 1 - tool_dependencies_root_folder = Folder( id=folder_id, key='root', label='root', parent=None ) - folder_id += 1 - folder = Folder( id=folder_id, key='tool_dependencies', label=label, parent=tool_dependencies_root_folder ) - if trans.webapp.name == 'galaxy': - if new_install or reinstalling: - folder.description = "repository tools require handling of these dependencies" - elif missing and not new_install and not reinstalling: - folder.description = 'click the name to install the missing dependency' - else: - folder.description = 'click the name to browse the dependency installation directory' - tool_dependencies_root_folder.folders.append( folder ) - # Insert a header row. - tool_dependency_id += 1 - if trans.webapp.name == 'galaxy': - tool_dependency = ToolDependency( id=tool_dependency_id, - name='Name', - version='Version', - type='Type', - readme=None, - installation_status='Installation status', - repository_id=None, - tool_dependency_id=None, - is_orphan=None ) - else: - tool_dependency = ToolDependency( id=tool_dependency_id, - name='Name', - version='Version', - type='Type', - readme=None, - installation_status=None, - repository_id=None, - tool_dependency_id=None, - is_orphan='Orphan' ) - folder.tool_dependencies.append( tool_dependency ) - is_orphan_description = "these dependencies may not be required by tools in this repository" - for dependency_key, requirements_dict in tool_dependencies.items(): - tool_dependency_id += 1 - if dependency_key in [ 'set_environment' ]: - for set_environment_dict in requirements_dict: - if trans.webapp.name == 'tool_shed': - is_orphan = set_environment_dict.get( 'is_orphan', False ) - else: - # TODO: handle this is Galaxy - is_orphan = False - if is_orphan: - folder.description = is_orphan_description - name = set_environment_dict.get( 'name', None ) - type = set_environment_dict[ 'type' ] - repository_id = set_environment_dict.get( 'repository_id', None ) - td_id = set_environment_dict.get( 'tool_dependency_id', None ) - if trans.webapp.name == 'galaxy': - installation_status = set_environment_dict.get( 'status', 'Never installed' ) - else: - installation_status = None - tool_dependency = ToolDependency( id=tool_dependency_id, - name=name, - version=None, - type=type, - readme=None, - installation_status=installation_status, - repository_id=repository_id, - tool_dependency_id=td_id, - is_orphan=is_orphan ) - folder.tool_dependencies.append( tool_dependency ) - else: - if trans.webapp.name == 'tool_shed': - is_orphan = requirements_dict.get( 'is_orphan', False ) - else: - # TODO: handle this is Galaxy - is_orphan = False - if is_orphan: - folder.description = is_orphan_description - name = requirements_dict[ 'name' ] - version = requirements_dict[ 'version' ] - type = requirements_dict[ 'type' ] - repository_id = requirements_dict.get( 'repository_id', None ) - td_id = requirements_dict.get( 'tool_dependency_id', None ) - if trans.webapp.name == 'galaxy': - installation_status = requirements_dict.get( 'status', 'Never installed' ) - else: - installation_status = None - tool_dependency = ToolDependency( id=tool_dependency_id, - name=name, - version=version, - type=type, - readme=None, - installation_status=installation_status, - repository_id=repository_id, - tool_dependency_id=td_id, - is_orphan=is_orphan ) - folder.tool_dependencies.append( tool_dependency ) - else: - tool_dependencies_root_folder = None - return folder_id, tool_dependencies_root_folder - -def build_tool_test_results_folder( trans, folder_id, tool_test_results_dict, label='Tool test results', time_last_tested=None ): - """Return a folder hierarchy containing tool dependencies.""" - # This container is displayed only in the tool shed. - if tool_test_results_dict: - folder_id += 1 - tool_test_results_root_folder = Folder( id=folder_id, key='root', label='root', parent=None ) - test_environment_dict = tool_test_results_dict.get( 'test_environment', None ) - if test_environment_dict: - folder_id += 1 - test_results_folder = Folder( id=folder_id, key='test_results', label=label, parent=tool_test_results_root_folder ) - tool_test_results_root_folder.folders.append( test_results_folder ) - folder_id += 1 - folder = Folder( id=folder_id, key='test_environment', label='Automated test environment', parent=test_results_folder ) - test_results_folder.folders.append( folder ) - test_environment = TestEnvironment( id=1, - architecture=test_environment_dict.get( 'architecture', '' ), - galaxy_database_version=test_environment_dict.get( 'galaxy_database_version', '' ), - galaxy_revision=test_environment_dict.get( 'galaxy_revision', '' ), - python_version=test_environment_dict.get( 'python_version', '' ), - system=test_environment_dict.get( 'system', '' ), - time_last_tested=time_last_tested, - tool_shed_database_version=test_environment_dict.get( 'tool_shed_database_version', '' ), - tool_shed_mercurial_version=test_environment_dict.get( 'tool_shed_mercurial_version', '' ), - tool_shed_revision=test_environment_dict.get( 'tool_shed_revision', '' ) ) - folder.test_environments.append( test_environment ) - not_tested_dict = tool_test_results_dict.get( 'not_tested', {} ) - if not_tested_dict: - folder_id += 1 - folder = Folder( id=folder_id, key='not_tested', label='Not tested', parent=test_results_folder ) - test_results_folder.folders.append( folder ) - not_tested_id = 0 - not_tested = NotTested( id=not_tested_id, - reason=not_tested_dict.get( 'reason', '' ) ) - folder.not_tested.append( not_tested ) - passed_tests_dicts = tool_test_results_dict.get( 'passed_tests', [] ) - if passed_tests_dicts: - folder_id += 1 - folder = Folder( id=folder_id, key='passed_tests', label='Tests that passed successfully', parent=test_results_folder ) - test_results_folder.folders.append( folder ) - passed_test_id = 0 - for passed_tests_dict in passed_tests_dicts: - passed_test_id += 1 - passed_test = PassedTest( id=passed_test_id, - test_id=passed_tests_dict.get( 'test_id' '' ), - tool_id=passed_tests_dict.get( 'tool_id', '' ), - tool_version=passed_tests_dict.get( 'tool_version', '' ) ) - folder.passed_tests.append( passed_test ) - failed_tests_dicts = tool_test_results_dict.get( 'failed_tests', [] ) - if failed_tests_dicts: - folder_id += 1 - folder = Folder( id=folder_id, key='failed_tests', label='Tests that failed', parent=test_results_folder ) - test_results_folder.folders.append( folder ) - failed_test_id = 0 - for failed_tests_dict in failed_tests_dicts: - failed_test_id += 1 - failed_test = FailedTest( id=failed_test_id, - stderr=failed_tests_dict.get( 'stderr', '' ), - test_id=failed_tests_dict.get( 'test_id', '' ), - tool_id=failed_tests_dict.get( 'tool_id', '' ), - tool_version=failed_tests_dict.get( 'tool_version', '' ), - traceback=failed_tests_dict.get( 'traceback', '' ) ) - folder.failed_tests.append( failed_test ) - missing_test_components_dicts = tool_test_results_dict.get( 'missing_test_components', [] ) - if missing_test_components_dicts: - folder_id += 1 - folder = Folder( id=folder_id, key='missing_test_components', label='Tools missing tests or test data', parent=test_results_folder ) - test_results_folder.folders.append( folder ) - missing_test_component_id = 0 - for missing_test_components_dict in missing_test_components_dicts: - missing_test_component_id += 1 - missing_test_component = MissingTestComponent( id=missing_test_component_id, - missing_components=missing_test_components_dict.get( 'missing_components', '' ), - tool_guid=missing_test_components_dict.get( 'tool_guid', '' ), - tool_id=missing_test_components_dict.get( 'tool_id', '' ), - tool_version=missing_test_components_dict.get( 'tool_version', '' ) ) - folder.missing_test_components.append( missing_test_component ) - installation_error_dicts = tool_test_results_dict.get( 'installation_errors', {} ) - if installation_error_dicts: - current_repository_errors = installation_error_dicts.get( 'current_repository', [] ) - repository_dependency_errors = installation_error_dicts.get( 'repository_dependencies', [] ) - tool_dependency_errors = installation_error_dicts.get( 'tool_dependencies', [] ) - if current_repository_errors or repository_dependency_errors or tool_dependency_errors: - folder_id += 1 - installation_error_base_folder = Folder( id=folder_id, - key='installation_errors', - label='Installation errors', - parent=test_results_folder ) - if current_repository_errors: - folder_id += 1 - subfolder = Folder( id=folder_id, - key='current_repository_errors', - label='This repository', - parent=installation_error_base_folder ) - repository_error_id = 0 - for repository_error_dict in current_repository_errors: - repository_error_id += 1 - repository_installation_error = RepositoryInstallationError( id=repository_error_id, - tool_shed=repository_error_dict.get( 'tool_shed', '' ), - name=repository_error_dict.get( 'name', '' ), - owner=repository_error_dict.get( 'owner', '' ), - changeset_revision=repository_error_dict.get( 'changeset_revision', '' ), - error_message=repository_error_dict.get( 'error_message', '' ) ) - subfolder.current_repository_installation_errors.append( repository_installation_error ) - installation_error_base_folder.folders.append( subfolder ) - if repository_dependency_errors: - folder_id += 1 - subfolder = Folder( id=folder_id, - key='repository_dependency_errors', - label='Repository dependencies', - parent=installation_error_base_folder ) - repository_error_id = 0 - for repository_error_dict in repository_dependency_errors: - repository_error_id += 1 - repository_installation_error = RepositoryInstallationError( id=repository_error_id, - tool_shed=repository_error_dict.get( 'tool_shed', '' ), - name=repository_error_dict.get( 'name', '' ), - owner=repository_error_dict.get( 'owner', '' ), - changeset_revision=repository_error_dict.get( 'changeset_revision', '' ), - error_message=repository_error_dict.get( 'error_message', '' ) ) - subfolder.repository_installation_errors.append( repository_installation_error ) - installation_error_base_folder.folders.append( subfolder ) - if tool_dependency_errors: - folder_id += 1 - subfolder = Folder( id=folder_id, - key='tool_dependency_errors', - label='Tool dependencies', - parent=installation_error_base_folder ) - tool_dependency_error_id = 0 - for tool_dependency_error_dict in tool_dependency_errors: - tool_dependency_error_id += 1 - tool_dependency_installation_error = ToolDependencyInstallationError( id=tool_dependency_error_id, - type=tool_dependency_error_dict.get( 'type', '' ), - name=tool_dependency_error_dict.get( 'name', '' ), - version=tool_dependency_error_dict.get( 'version', '' ), - error_message=tool_dependency_error_dict.get( 'error_message', '' ) ) - subfolder.tool_dependency_installation_errors.append( tool_dependency_installation_error ) - installation_error_base_folder.folders.append( subfolder ) - test_results_folder.installation_errors.append( installation_error_base_folder ) - else: - tool_test_results_root_folder = None - return folder_id, tool_test_results_root_folder - -def build_workflows_folder( trans, folder_id, workflows, repository_metadata_id=None, repository_id=None, label='Workflows' ): - """ - Return a folder hierarchy containing workflow objects for each workflow dictionary in the received workflows list. When - this method is called from the tool shed, repository_metadata_id will have a value and repository_id will be None. When - this method is called from Galaxy, repository_id will have a value only if the repository is not currenlty being installed - and repository_metadata_id will be None. - """ - if workflows: - workflow_id = 0 - folder_id += 1 - workflows_root_folder = Folder( id=folder_id, key='root', label='root', parent=None ) - folder_id += 1 - folder = Folder( id=folder_id, key='workflows', label=label, parent=workflows_root_folder ) - workflows_root_folder.folders.append( folder ) - # Insert a header row. - workflow_id += 1 - workflow = Workflow( id=workflow_id, - workflow_name='Name', - steps='steps', - format_version='format-version', - annotation='annotation', - repository_metadata_id=repository_metadata_id, - repository_id=repository_id ) - folder.workflows.append( workflow ) - for workflow_tup in workflows: - workflow_dict=workflow_tup[ 1 ] - steps = workflow_dict.get( 'steps', [] ) - if steps: - steps = str( len( steps ) ) - else: - steps = 'unknown' - workflow_id += 1 - workflow = Workflow( id=workflow_id, - workflow_name=workflow_dict.get( 'name', '' ), - steps=steps, - format_version=workflow_dict.get( 'format-version', '' ), - annotation=workflow_dict.get( 'annotation', '' ), - repository_metadata_id=repository_metadata_id, - repository_id=repository_id ) - folder.workflows.append( workflow ) - else: - workflows_root_folder = None - return folder_id, workflows_root_folder - -def cast_empty_repository_dependency_folders( folder, repository_dependency_id ): - """ - Change any empty folders contained within the repository dependencies container into a repository dependency since it has no repository dependencies - of it's own. This method is not used (and may not be needed), but here it is just in case. - """ - if not folder.folders and not folder.repository_dependencies: - repository_dependency_id += 1 - repository_dependency = folder.to_repository_dependency( repository_dependency_id ) - if not folder.parent.contains_repository_dependency( repository_dependency ): - folder.parent.repository_dependencies.append( repository_dependency ) - folder.parent.folders.remove( folder ) - for sub_folder in folder.folders: - return cast_empty_repository_dependency_folders( sub_folder, repository_dependency_id ) - return folder, repository_dependency_id - -def generate_repository_dependencies_folder_label_from_key( repository_name, repository_owner, changeset_revision, prior_installation_required, key ): - """Return a repository dependency label based on the repository dependency key.""" - if key_is_current_repositorys_key( repository_name, repository_owner, changeset_revision, prior_installation_required, key ): - label = 'Repository dependencies' - else: - if prior_installation_required: - prior_installation_required_str = " <i>(prior install required)</i>" - else: - prior_installation_required_str = "" - label = "Repository <b>%s</b> revision <b>%s</b> owned by <b>%s</b>%s" % \ - ( repository_name, changeset_revision, repository_owner, prior_installation_required_str ) - return label - -def generate_repository_dependencies_key_for_repository( toolshed_base_url, repository_name, repository_owner, changeset_revision, prior_installation_required ): - # FIXME: assumes tool shed is current tool shed since repository dependencies across tool sheds is not yet supported. - return '%s%s%s%s%s%s%s%s%s' % ( str( toolshed_base_url ).rstrip( '/' ), - STRSEP, - str( repository_name ), - STRSEP, - str( repository_owner ), - STRSEP, - str( changeset_revision ), - STRSEP, - str( prior_installation_required ) ) - -def generate_tool_dependencies_key( name, version, type ): - return '%s%s%s%s%s' % ( str( name ), STRSEP, str( version ), STRSEP, str( type ) ) - -def get_folder( folder, key ): - if folder.key == key: - return folder - for sub_folder in folder.folders: - return get_folder( sub_folder, key ) - return None - -def get_components_from_key( key ): - # FIXME: assumes tool shed is current tool shed since repository dependencies across tool sheds is not yet supported. - items = key.split( STRSEP ) - toolshed_base_url = items[ 0 ] - repository_name = items[ 1 ] - repository_owner = items[ 2 ] - changeset_revision = items[ 3 ] - if len( items ) == 5: - prior_installation_required = asbool( str( items[ 4 ] ) ) - return toolshed_base_url, repository_name, repository_owner, changeset_revision, prior_installation_required - else: - # For backward compatibility to the 12/20/12 Galaxy release we have to return the following, and callers must handle exceptions. - return toolshed_base_url, repository_name, repository_owner, changeset_revision - -def handle_repository_dependencies_container_entry( trans, repository_dependencies_folder, rd_key, rd_value, folder_id, repository_dependency_id, folder_keys ): - try: - toolshed, repository_name, repository_owner, changeset_revision, prior_installation_required = get_components_from_key( rd_key ) - except ValueError: - # For backward compatibility to the 12/20/12 Galaxy release, default prior_installation_required to False. - toolshed, repository_name, repository_owner, changeset_revision = get_components_from_key( rd_key ) - prior_installation_required = False - folder = get_folder( repository_dependencies_folder, rd_key ) - label = generate_repository_dependencies_folder_label_from_key( repository_name, - repository_owner, - changeset_revision, - prior_installation_required, - repository_dependencies_folder.key ) - if folder: - if rd_key not in folder_keys: - folder_id += 1 - sub_folder = Folder( id=folder_id, key=rd_key, label=label, parent=folder ) - folder.folders.append( sub_folder ) - else: - sub_folder = folder - else: - folder_id += 1 - sub_folder = Folder( id=folder_id, key=rd_key, label=label, parent=repository_dependencies_folder ) - repository_dependencies_folder.folders.append( sub_folder ) - if trans.webapp.name == 'galaxy': - # Insert a header row. - repository_dependency_id += 1 - repository_dependency = RepositoryDependency( id=repository_dependency_id, - repository_name='Name', - changeset_revision='Revision', - repository_owner='Owner', - installation_status='Installation status' ) - # Insert the header row into the folder. - sub_folder.repository_dependencies.append( repository_dependency ) - for repository_dependency in rd_value: - if trans.webapp.name == 'galaxy': - if len( repository_dependency ) == 6: - # Metadata should have been reset on this installed repository, but it wasn't. - tool_shed_repository_id = repository_dependency[ 4 ] - installation_status = repository_dependency[ 5 ] - tool_shed, name, owner, changeset_revision = repository_dependency[ 0:4 ] - # Default prior_installation_required to False. - prior_installation_required = False - repository_dependency = [ tool_shed, name, owner, changeset_revision, prior_installation_required ] - elif len( repository_dependency ) == 7: - # We have a repository dependency tuple that includes a prior_installation_required value. - tool_shed_repository_id = repository_dependency[ 5 ] - installation_status = repository_dependency[ 6 ] - repository_dependency = repository_dependency[ 0:5 ] - else: - tool_shed_repository_id = None - installation_status = 'unknown' - else: - tool_shed_repository_id = None - installation_status = None - can_create_dependency = not is_subfolder_of( sub_folder, repository_dependency ) - if can_create_dependency: - toolshed, repository_name, repository_owner, changeset_revision, prior_installation_required = \ - suc.parse_repository_dependency_tuple( repository_dependency ) - repository_dependency_id += 1 - repository_dependency = RepositoryDependency( id=repository_dependency_id, - toolshed=toolshed, - repository_name=repository_name, - repository_owner=repository_owner, - changeset_revision=changeset_revision, - prior_installation_required=asbool( prior_installation_required ), - installation_status=installation_status, - tool_shed_repository_id=tool_shed_repository_id ) - # Insert the repository_dependency into the folder. - sub_folder.repository_dependencies.append( repository_dependency ) - return repository_dependencies_folder, folder_id, repository_dependency_id - -def is_subfolder_of( folder, repository_dependency ): - toolshed, repository_name, repository_owner, changeset_revision, prior_installation_required = \ - suc.parse_repository_dependency_tuple( repository_dependency ) - key = generate_repository_dependencies_key_for_repository( toolshed, repository_name, repository_owner, changeset_revision, asbool( prior_installation_required ) ) - for sub_folder in folder.folders: - if key == sub_folder.key: - return True - return False - -def key_is_current_repositorys_key( repository_name, repository_owner, changeset_revision, prior_installation_required, key ): - try: - toolshed_base_url, key_name, key_owner, key_changeset_revision, key_prior_installation_required = get_components_from_key( key ) - except ValueError: - # For backward compatibility to the 12/20/12 Galaxy release, default key_prior_installation_required to False. - toolshed_base_url, key_name, key_owner, key_changeset_revision = get_components_from_key( key ) - key_prior_installation_required = False - return repository_name == key_name and \ - repository_owner == key_owner and \ - changeset_revision == key_changeset_revision and \ - prior_installation_required == key_prior_installation_required - -def populate_repository_dependencies_container( trans, repository_dependencies_folder, repository_dependencies, folder_id, repository_dependency_id ): - folder_keys = repository_dependencies.keys() - for key, value in repository_dependencies.items(): - repository_dependencies_folder, folder_id, repository_dependency_id = \ - handle_repository_dependencies_container_entry( trans, repository_dependencies_folder, key, value, folder_id, repository_dependency_id, folder_keys ) - return repository_dependencies_folder, folder_id, repository_dependency_id - -def print_folders( pad, folder ): - # For debugging... - pad_str = '' - for i in range( 1, pad ): - pad_str += ' ' - print '%sid: %s key: %s' % ( pad_str, str( folder.id ), folder.key ) - for repository_dependency in folder.repository_dependencies: - print ' %s%s' % ( pad_str, repository_dependency.listify ) - for sub_folder in folder.folders: - print_folders( pad+5, sub_folder ) - -def prune_repository_dependencies( folder ): - """ - Since the object used to generate a repository dependencies container is a dictionary and not an odict() (it must be json-serialize-able), the - order in which the dictionary is processed to create the container sometimes results in repository dependency entries in a folder that also - includes the repository dependency as a sub-folder (if the repository dependency has it's own repository dependency). This method will remove - all repository dependencies from folder that are also sub-folders of folder. - """ - repository_dependencies = [ rd for rd in folder.repository_dependencies ] - for repository_dependency in repository_dependencies: - listified_repository_dependency = repository_dependency.listify - if is_subfolder_of( folder, listified_repository_dependency ): - repository_dependencies.remove( repository_dependency ) - folder.repository_dependencies = repository_dependencies - for sub_folder in folder.folders: - return prune_repository_dependencies( sub_folder ) - return folder - \ No newline at end of file diff -r f02a75ce05b71457845c0e5f56f409047a841967 -r b8cf5887ad464707887aaf8381df19dbc67ac697 lib/tool_shed/galaxy_install/repository_util.py --- a/lib/tool_shed/galaxy_install/repository_util.py +++ b/lib/tool_shed/galaxy_install/repository_util.py @@ -8,10 +8,10 @@ from galaxy import util from galaxy import web from galaxy.model.orm import or_ -from galaxy.webapps.tool_shed.util import container_util import tool_shed.util.shed_util_common as suc from tool_shed.util import common_util from tool_shed.util import common_install_util +from tool_shed.util import container_util from tool_shed.util import data_manager_util from tool_shed.util import datatype_util from tool_shed.util import encoding_util diff -r f02a75ce05b71457845c0e5f56f409047a841967 -r b8cf5887ad464707887aaf8381df19dbc67ac697 lib/tool_shed/util/common_install_util.py --- a/lib/tool_shed/util/common_install_util.py +++ b/lib/tool_shed/util/common_install_util.py @@ -6,9 +6,9 @@ from galaxy import util from galaxy import web from galaxy.util import json -from galaxy.webapps.tool_shed.util import container_util import tool_shed.util.shed_util_common as suc from tool_shed.util import common_util +from tool_shed.util import container_util from tool_shed.util import encoding_util from tool_shed.util import data_manager_util from tool_shed.util import datatype_util This diff is so big that we needed to truncate the remainder. Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: guerler: Bowtie2: Added quotation marks to group selection and added basic tool test
by commits-noreply＠bitbucket.org 07 Aug '13

07 Aug '13

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/f02a75ce05b7/ Changeset: f02a75ce05b7 User: guerler Date: 2013-08-07 20:17:43 Summary: Bowtie2: Added quotation marks to group selection and added basic tool test Affected #: 4 files diff -r f54e602ca23310b8a237af1388d7bc2a37047d34 -r f02a75ce05b71457845c0e5f56f409047a841967 test-data/bowtie2/phix_genome.fasta --- /dev/null +++ b/test-data/bowtie2/phix_genome.fasta @@ -0,0 +1,78 @@ +>gi|9626372|ref|NC_001422.1| Enterobacteria phage phiX174, complete genome +GAGTTTTATCGCTTCCATGACGCAGAAGTTAACACTTTCGGATATTTCTGATGAGTCGAAAAATTATCTT +GATAAAGCAGGAATTACTACTGCTTGTTTACGAATTAAATCGAAGTGGACTGCTGGCGGAAAATGAGAAA +ATTCGACCTATCCTTGCGCAGCTCGAGAAGCTCTTACTTTGCGACCTTTCGCCATCAACTAACGATTCTG +TCAAAAACTGACGCGTTGGATGAGGAGAAGTGGCTTAATATGCTTGGCACGTTCGTCAAGGACTGGTTTA +GATATGAGTCACATTTTGTTCATGGTAGAGATTCTCTTGTTGACATTTTAAAAGAGCGTGGATTACTATC +TGAGTCCGATGCTGTTCAACCACTAATAGGTAAGAAATCATGAGTCAAGTTACTGAACAATCCGTACGTT +TCCAGACCGCTTTGGCCTCTATTAAGCTCATTCAGGCTTCTGCCGTTTTGGATTTAACCGAAGATGATTT +CGATTTTCTGACGAGTAACAAAGTTTGGATTGCTACTGACCGCTCTCGTGCTCGTCGCTGCGTTGAGGCT +TGCGTTTATGGTACGCTGGACTTTGTGGGATACCCTCGCTTTCCTGCTCCTGTTGAGTTTATTGCTGCCG +TCATTGCTTATTATGTTCATCCCGTCAACATTCAAACGGCCTGTCTCATCATGGAAGGCGCTGAATTTAC +GGAAAACATTATTAATGGCGTCGAGCGTCCGGTTAAAGCCGCTGAATTGTTCGCGTTTACCTTGCGTGTA +CGCGCAGGAAACACTGACGTTCTTACTGACGCAGAAGAAAACGTGCGTCAAAAATTACGTGCGGAAGGAG +TGATGTAATGTCTAAAGGTAAAAAACGTTCTGGCGCTCGCCCTGGTCGTCCGCAGCCGTTGCGAGGTACT +AAAGGCAAGCGTAAAGGCGCTCGTCTTTGGTATGTAGGTGGTCAACAATTTTAATTGCAGGGGCTTCGGC +CCCTTACTTGAGGATAAATTATGTCTAATATTCAAACTGGCGCCGAGCGTATGCCGCATGACCTTTCCCA +TCTTGGCTTCCTTGCTGGTCAGATTGGTCGTCTTATTACCATTTCAACTACTCCGGTTATCGCTGGCGAC +TCCTTCGAGATGGACGCCGTTGGCGCTCTCCGTCTTTCTCCATTGCGTCGTGGCCTTGCTATTGACTCTA +CTGTAGACATTTTTACTTTTTATGTCCCTCATCGTCACGTTTATGGTGAACAGTGGATTAAGTTCATGAA +GGATGGTGTTAATGCCACTCCTCTCCCGACTGTTAACACTACTGGTTATATTGACCATGCCGCTTTTCTT +GGCACGATTAACCCTGATACCAATAAAATCCCTAAGCATTTGTTTCAGGGTTATTTGAATATCTATAACA +ACTATTTTAAAGCGCCGTGGATGCCTGACCGTACCGAGGCTAACCCTAATGAGCTTAATCAAGATGATGC +TCGTTATGGTTTCCGTTGCTGCCATCTCAAAAACATTTGGACTGCTCCGCTTCCTCCTGAGACTGAGCTT +TCTCGCCAAATGACGACTTCTACCACATCTATTGACATTATGGGTCTGCAAGCTGCTTATGCTAATTTGC +ATACTGACCAAGAACGTGATTACTTCATGCAGCGTTACCATGATGTTATTTCTTCATTTGGAGGTAAAAC +CTCTTATGACGCTGACAACCGTCCTTTACTTGTCATGCGCTCTAATCTCTGGGCATCTGGCTATGATGTT +GATGGAACTGACCAAACGTCGTTAGGCCAGTTTTCTGGTCGTGTTCAACAGACCTATAAACATTCTGTGC +CGCGTTTCTTTGTTCCTGAGCATGGCACTATGTTTACTCTTGCGCTTGTTCGTTTTCCGCCTACTGCGAC +TAAAGAGATTCAGTACCTTAACGCTAAAGGTGCTTTGACTTATACCGATATTGCTGGCGACCCTGTTTTG +TATGGCAACTTGCCGCCGCGTGAAATTTCTATGAAGGATGTTTTCCGTTCTGGTGATTCGTCTAAGAAGT +TTAAGATTGCTGAGGGTCAGTGGTATCGTTATGCGCCTTCGTATGTTTCTCCTGCTTATCACCTTCTTGA +AGGCTTCCCATTCATTCAGGAACCGCCTTCTGGTGATTTGCAAGAACGCGTACTTATTCGCCACCATGAT +TATGACCAGTGTTTCCAGTCCGTTCAGTTGTTGCAGTGGAATAGTCAGGTTAAATTTAATGTGACCGTTT +ATCGCAATCTGCCGACCACTCGCGATTCAATCATGACTTCGTGATAAAAGATTGAGTGTGAGGTTATAAC +GCCGAAGCGGTAAAAATTTTAATTTTTGCCGCTGAGGGGTTGACCAAGCGAAGCGCGGTAGGTTTTCTGC +TTAGGAGTTTAATCATGTTTCAGACTTTTATTTCTCGCCATAATTCAAACTTTTTTTCTGATAAGCTGGT +TCTCACTTCTGTTACTCCAGCTTCTTCGGCACCTGTTTTACAGACACCTAAAGCTACATCGTCAACGTTA +TATTTTGATAGTTTGACGGTTAATGCTGGTAATGGTGGTTTTCTTCATTGCATTCAGATGGATACATCTG +TCAACGCCGCTAATCAGGTTGTTTCTGTTGGTGCTGATATTGCTTTTGATGCCGACCCTAAATTTTTTGC +CTGTTTGGTTCGCTTTGAGTCTTCTTCGGTTCCGACTACCCTCCCGACTGCCTATGATGTTTATCCTTTG +AATGGTCGCCATGATGGTGGTTATTATACCGTCAAGGACTGTGTGACTATTGACGTCCTTCCCCGTACGC +CGGGCAATAACGTTTATGTTGGTTTCATGGTTTGGTCTAACTTTACCGCTACTAAATGCCGCGGATTGGT +TTCGCTGAATCAGGTTATTAAAGAGATTATTTGTCTCCAGCCACTTAAGTGAGGTGATTTATGTTTGGTG +CTATTGCTGGCGGTATTGCTTCTGCTCTTGCTGGTGGCGCCATGTCTAAATTGTTTGGAGGCGGTCAAAA +AGCCGCCTCCGGTGGCATTCAAGGTGATGTGCTTGCTACCGATAACAATACTGTAGGCATGGGTGATGCT +GGTATTAAATCTGCCATTCAAGGCTCTAATGTTCCTAACCCTGATGAGGCCGCCCCTAGTTTTGTTTCTG +GTGCTATGGCTAAAGCTGGTAAAGGACTTCTTGAAGGTACGTTGCAGGCTGGCACTTCTGCCGTTTCTGA +TAAGTTGCTTGATTTGGTTGGACTTGGTGGCAAGTCTGCCGCTGATAAAGGAAAGGATACTCGTGATTAT +CTTGCTGCTGCATTTCCTGAGCTTAATGCTTGGGAGCGTGCTGGTGCTGATGCTTCCTCTGCTGGTATGG +TTGACGCCGGATTTGAGAATCAAAAAGAGCTTACTAAAATGCAACTGGACAATCAGAAAGAGATTGCCGA +GATGCAAAATGAGACTCAAAAAGAGATTGCTGGCATTCAGTCGGCGACTTCACGCCAGAATACGAAAGAC +CAGGTATATGCACAAAATGAGATGCTTGCTTATCAACAGAAGGAGTCTACTGCTCGCGTTGCGTCTATTA +TGGAAAACACCAATCTTTCCAAGCAACAGCAGGTTTCCGAGATTATGCGCCAAATGCTTACTCAAGCTCA +AACGGCTGGTCAGTATTTTACCAATGACCAAATCAAAGAAATGACTCGCAAGGTTAGTGCTGAGGTTGAC +TTAGTTCATCAGCAAACGCAGAATCAGCGGTATGGCTCTTCTCATATTGGCGCTACTGCAAAGGATATTT +CTAATGTCGTCACTGATGCTGCTTCTGGTGTGGTTGATATTTTTCATGGTATTGATAAAGCTGTTGCCGA +TACTTGGAACAATTTCTGGAAAGACGGTAAAGCTGATGGTATTGGCTCTAATTTGTCTAGGAAATAACCG +TCAGGATTGACACCCTCCCAATTGTATGTTTTCATGCCTCCAAATCTTGGAGGCTTTTTTATGGTTCGTT +CTTATTACCCTTCTGAATGTCACGCTGATTATTTTGACTTTGAGCGTATCGAGGCTCTTAAACCTGCTAT +TGAGGCTTGTGGCATTTCTACTCTTTCTCAATCCCCAATGCTTGGCTTCCATAAGCAGATGGATAACCGC +ATCAAGCTCTTGGAAGAGATTCTGTCTTTTCGTATGCAGGGCGTTGAGTTCGATAATGGTGATATGTATG +TTGACGGCCATAAGGCTGCTTCTGACGTTCGTGATGAGTTTGTATCTGTTACTGAGAAGTTAATGGATGA +ATTGGCACAATGCTACAATGTGCTCCCCCAACTTGATATTAATAACACTATAGACCACCGCCCCGAAGGG +GACGAAAAATGGTTTTTAGAGAACGAGAAGACGGTTACGCAGTTTTGCCGCAAGCTGGCTGCTGAACGCC +CTCTTAAGGATATTCGCGATGAGTATAATTACCCCAAAAAGAAAGGTATTAAGGATGAGTGTTCAAGATT +GCTGGAGGCCTCCACTATGAAATCGCGTAGAGGCTTTGCTATTCAGCGTTTGATGAATGCAATGCGACAG +GCTCATGCTGATGGTTGGTTTATCGTTTTTGACACTCTCACGTTGGCTGACGACCGATTAGAGGCGTTTT +ATGATAATCCCAATGCTTTGCGTGACTATTTTCGTGATATTGGTCGTATGGTTCTTGCTGCCGAGGGTCG +CAAGGCTAATGATTCACACGCCGACTGCTATCAGTATTTTTGTGTGCCTGAGTATGGTACAGCTAATGGC +CGTCTTCATTTCCATGCGGTGCACTTTATGCGGACACTTCCTACAGGTAGCGTTGACCCTAATTTTGGTC +GTCGGGTACGCAATCGCCGCCAGTTAAATAGCTTGCAAAATACGTGGCCTTATGGTTACAGTATGCCCAT +CGCAGTTCGCTACACGCAGGACGCTTTTTCACGTTCTGGTTGGTTGTGGCCTGTTGATGCTAAAGGTGAG +CCGCTTAAAGCTACCAGTTATATGGCTGTTGGTTTCTATGTGGCTAAATACGTTAACAAAAAGTCAGATA +TGGACCTTGCTGCTAAAGGTCTAGGAGCTAAAGAATGGAACAACTCACTAAAAACCAAGCTGTCGCTACT +TCCCAAGAAGCTGTTCAGAATCAGAATGAGCCGCAACTTCGGGATGAAAATGCTCACAATGACAAATCTG +TCCACGGAGTGCTTAATCCAACTTACCAAGCTGGGTTACGACGCGACGCCGTTCAACCAGATATTGAAGC +AGAACGCAAAAAGAGAGATGAGATTGAGGCTGGGAAAAGTTACTGTAGCCGACGTTTTGGCGGCGCAACC +TGTGACGACAAATCTGCTCAAATTTATGCGCGCTTCGATAAAAATGATTGGCGTATCCAACCTGCA \ No newline at end of file diff -r f54e602ca23310b8a237af1388d7bc2a37047d34 -r f02a75ce05b71457845c0e5f56f409047a841967 test-data/bowtie2/phix_mapped.bam Binary file test-data/bowtie2/phix_mapped.bam has changed diff -r f54e602ca23310b8a237af1388d7bc2a37047d34 -r f02a75ce05b71457845c0e5f56f409047a841967 test-data/bowtie2/phix_reads.fastq --- /dev/null +++ b/test-data/bowtie2/phix_reads.fastq @@ -0,0 +1,260 @@ +@HWI-EAS210R_0001:4:10:890:1882#0/1 +AATCTCATCTCTCTTTTTGCGTTCTGCTTCAATATCTG ++HWI-EAS210R_0001:4:10:890:1882#0/1 +fcceeggggggggggggffghggggggggggggggggg +@HWI-EAS210R_0001:4:10:890:1010#0/1 +GTTGTTTCTGTTGGTGCTGATATTGCTTTTGATGCCGA ++HWI-EAS210R_0001:4:10:890:1010#0/1 +gggggggggggggghgggggggggfggggggggggggg +@HWI-EAS210R_0001:4:10:890:1780#0/1 +GAGGCCTCCACTATGAAATCGCGTAGAGGCTTTGCTAT ++HWI-EAS210R_0001:4:10:890:1780#0/1 +ggfhggggggggggggggggghgggggfggggggdggg +@HWI-EAS210R_0001:4:10:890:1348#0/1 +TTGAGCGTATCGAGGCTCTTAAACCTGCTATTGAGGCT ++HWI-EAS210R_0001:4:10:890:1348#0/1 +ggggggggggggggggggggggggghfggeggggggdg +@HWI-EAS210R_0001:4:10:890:1707#0/1 +AAAAGAGATTGCTGGCATTCAGTCGGCGACTTCACGCC ++HWI-EAS210R_0001:4:10:890:1707#0/1 +gfgfgfgggggggfggeggggffgcgggdfggfggdgg +@HWI-EAS210R_0001:4:10:890:1527#0/1 +CGTACTTATTCGCCACCATGATTATGACCAGTGTTTCC ++HWI-EAS210R_0001:4:10:890:1527#0/1 +gggggggggggggggggggggggggggggggggggggg +@HWI-EAS210R_0001:4:10:890:781#0/1 +CGGAAAACGAACAAGCGCAAGAGTAAACATAGTGCCAT ++HWI-EAS210R_0001:4:10:890:781#0/1 +gggggggggggggggggggggggggggggggghggggg +@HWI-EAS210R_0001:4:10:890:568#0/1 +GTTTATCGCAATCTGCCGACCACTCGCGATTCAATCAT ++HWI-EAS210R_0001:4:10:890:568#0/1 +gggggggfgggggggggggggggggggggggggggggg +@HWI-EAS210R_0001:4:10:890:1365#0/1 +CCAACATAAACATTATTGCCCGGCGTACGAGGAAGGAC ++HWI-EAS210R_0001:4:10:890:1365#0/1 +eggggdggfghggggggggghgfgfegghfggfgggge +@HWI-EAS210R_0001:4:10:890:161#0/1 +GTGATTATCTTGCTGCTGCATTTCCTGAGCTTAATGCT ++HWI-EAS210R_0001:4:10:890:161#0/1 +ggggfg_gggfegggfgeggaefefdbfddeedgcgg` +@HWI-EAS210R_0001:4:10:890:1920#0/1 +TTTATCAATACCATGAAAAATATCAACCACACCAGAAG ++HWI-EAS210R_0001:4:10:890:1920#0/1 +gggggggggggggggggggggffggggggggggggggg +@HWI-EAS210R_0001:4:10:890:61#0/1 +GTCGCGTCGTAACCCAGCTTGGTAAGTTGGATTAAGCA ++HWI-EAS210R_0001:4:10:890:61#0/1 +eede[egfggfggggggggggggegggggfgggggggg +@HWI-EAS210R_0001:4:10:890:1284#0/1 +AAGCGCAAGAGTAAACATAGTGCCATGCTCAGGAACAA ++HWI-EAS210R_0001:4:10:890:1284#0/1 +gggggggggggggggggggggggggggggggggggggg +@HWI-EAS210R_0001:4:10:890:208#0/1 +CAGACCTATAAACATTCTGTGCCGCGTTTCTTTGTTCC ++HWI-EAS210R_0001:4:10:890:208#0/1 +gggeggggggggggggggggggggggggggggddgggg +@HWI-EAS210R_0001:4:10:890:859#0/1 +CAATAGATGTGGTAGAAGTCGTCATTTGGCGAGAAAGC ++HWI-EAS210R_0001:4:10:890:859#0/1 +ggggcgggggggggggdgfggggggggfggggghgggh +@HWI-EAS210R_0001:4:10:890:1947#0/1 +TCAACGCCGCTAATCAGGTTGTTTCTGTTGGTGCTGAT ++HWI-EAS210R_0001:4:10:890:1947#0/1 +ggggggggggggggggggeggggggggggggfgfgggg +@HWI-EAS210R_0001:4:10:890:416#0/1 +AATGTCTAAAGGTAAAAAACGTTCTGGCGCTCGCCCTG ++HWI-EAS210R_0001:4:10:890:416#0/1 +eggggggggggdcgggggfggfgggggfcgfggggggg +@HWI-EAS210R_0001:4:10:890:654#0/1 +GCAGCAAGGTCCATATCTGACTTTTTGTTAACGTATTT ++HWI-EAS210R_0001:4:10:890:654#0/1 +ggggghggggghggggghgfggggfgdfffgggggggg +@HWI-EAS210R_0001:4:10:890:269#0/1 +CTTGAAGGCTTCCCATTCATTCAGGAACCGCCTTCTGG ++HWI-EAS210R_0001:4:10:890:269#0/1 +eecaadggggggggggggggggdgg\ffffgggggggg +@HWI-EAS210R_0001:4:10:890:657#0/1 +TACCAGCTTTAGCCATAGCACCAGAAACAAAACTAGGG ++HWI-EAS210R_0001:4:10:890:657#0/1 +ggggghggggffgggfggggggfggfggggfffgfggg +@HWI-EAS210R_0001:4:10:890:1449#0/1 +TTCAAGATTGCTGGAGGCCTCCACTATGAAATCGCGTA ++HWI-EAS210R_0001:4:10:890:1449#0/1 +ggggggggggggggggfgggghgggggggggggghhfh +@HWI-EAS210R_0001:4:10:890:305#0/1 +CACGTTGGCTGACGACCGATTAGAGGCGTTTTATGATA ++HWI-EAS210R_0001:4:10:890:305#0/1 +gggegggfgggfggggggdgggdggfgffggggggfhg +@HWI-EAS210R_0001:4:10:890:1190#0/1 +AATAAGCAATGACGGCAGCAATAAACTCAACAGGAGCA ++HWI-EAS210R_0001:4:10:890:1190#0/1 +bbecb_gggggggggggggggggggcgggfVgggeggg +@HWI-EAS210R_0001:4:10:890:1586#0/1 +ATTAGCTGTACCATACTCAGGCACACAAAAATACTGAT ++HWI-EAS210R_0001:4:10:890:1586#0/1 +eggWgaa^O[\`[\_]J_^][`W`\]K^BBBBBBBBBB +@HWI-EAS210R_0001:4:10:890:617#0/1 +AAACGCAAGCCTCAACGCAGCGACGAGCACGAGAGCGG ++HWI-EAS210R_0001:4:10:890:617#0/1 +gggggggggggggggggggggggggggggggggggggg +@HWI-EAS210R_0001:4:10:890:1562#0/1 +GTGGACTGCTGGCGGAAAATGAGAAAATTCGACCTATC ++HWI-EAS210R_0001:4:10:890:1562#0/1 +hggffggggdgggggaageef`Waegggbgggcgddgg +@HWI-EAS210R_0001:4:10:890:1410#0/1 +CATAAAAAAGCCTCCAAGATTTGGAGGCATGAAAACAT ++HWI-EAS210R_0001:4:10:890:1410#0/1 +\geedggfffcd\gd`\edeggY_Zda`gggXgadeeg +@HWI-EAS210R_0001:4:10:890:465#0/1 +AGCGTTGACCCTAATTTTGGTCGTCGGGTACGCAATCG ++HWI-EAS210R_0001:4:10:890:465#0/1 +gggggfgggggggfgggeggggbgggggagf^ffefea +@HWI-EAS210R_0001:4:10:890:1323#0/1 +ATGACAAGTAAAGGACGGTTGTCAGCGTCATAAGAGGT ++HWI-EAS210R_0001:4:10:890:1323#0/1 +gggggggfggdggggggggggggggggggggfgghggg +@HWI-EAS210R_0001:4:10:890:1064#0/1 +TTTCATGGTATTGATAAAGCTGTTGCCGATACTTGGAA ++HWI-EAS210R_0001:4:10:890:1064#0/1 +gggggggcegggfgfggfgfgggdgggggghfggggdf +@HWI-EAS210R_0001:4:10:890:637#0/1 +ATAGTGCCATGCTCAGGAACAAAGAAACGCGGCACAGA ++HWI-EAS210R_0001:4:10:890:637#0/1 +ggggggggggggggggggeggggggfddgggcgggggd +@HWI-EAS210R_0001:4:10:890:1825#0/1 +TCCATCTGAATGCAATGAAGAAAACCACCATTACCAGC ++HWI-EAS210R_0001:4:10:890:1825#0/1 +gggggggghgggffgggggegggffgghgggggggggg +@HWI-EAS210R_0001:4:10:890:1697#0/1 +ACGGTTAATGCTGGTAATGGTGGTTTTCTTCATTGCAT ++HWI-EAS210R_0001:4:10:890:1697#0/1 +eecaadeffedeggcabebeagggdecL_aNTTTTeg_ +@HWI-EAS210R_0001:4:10:890:1479#0/1 +ATTGCTGGCGACCCTGTTTTGTATGGCAACTTGCCGCC ++HWI-EAS210R_0001:4:10:890:1479#0/1 +ggggggggggggggggggggfggggggggghgbgggfg +@HWI-EAS210R_0001:4:10:890:1838#0/1 +ATGTGACCGTTTATCGCAATCTGCCGACCACTCGCGAT ++HWI-EAS210R_0001:4:10:890:1838#0/1 +ggfgfggfggggggfgggggggaggggfegfhggdggg +@HWI-EAS210R_0001:4:10:890:988#0/1 +CATTCAGTCGGCGACTTCACGCCAGAATACGAAAGACC ++HWI-EAS210R_0001:4:10:890:988#0/1 +ggghggfggggggegghgfggfgggggggggggggggg +@HWI-EAS210R_0001:4:10:890:1902#0/1 +CTTCCTCGTACGCCGGGCAATAATGTTTATGTTGGTTT ++HWI-EAS210R_0001:4:10:890:1902#0/1 +gggggggegggggggggggggggggggggfgggggggg +@HWI-EAS210R_0001:4:10:890:1983#0/1 +TTAAGGTACTGAATCTCTTTAGTCGCAGTAGGCGGAAA ++HWI-EAS210R_0001:4:10:890:1983#0/1 +ffdfbaQa_c_Y\]Yeceeeege_gYR^VS]ZW[\Lcc +@HWI-EAS210R_0001:4:10:890:1058#0/1 +GCTCGCCCTGGTCGTCCGCAGCCGTTGCGAGGTACTAA ++HWI-EAS210R_0001:4:10:890:1058#0/1 +ggggdgggggggggggggggggggggggggggfggggg +@HWI-EAS210R_0001:4:10:890:287#0/1 +GTATCCAACCTGCAGAGTTTTATCGCTTCCATGACGCA ++HWI-EAS210R_0001:4:10:890:287#0/1 +ggggggfcggggg^fbe``egggggggdgg`dddfggg +@HWI-EAS210R_0001:4:10:890:335#0/1 +AAGAGGTTTTACCTCCAAATGAAGAAATAACATCATGG ++HWI-EAS210R_0001:4:10:890:335#0/1 +`^_``Q[cccggfgggggggbddg\dXdbegggddfg` +@HWI-EAS210R_0001:4:10:891:149#0/1 +TTTAAGAGCCTCGATACGCTCAAAGTCAAAATAATCAG ++HWI-EAS210R_0001:4:10:891:149#0/1 +ggggghgggggghggggggggggggfgggggggffggg +@HWI-EAS210R_0001:4:10:891:445#0/1 +ATATTTTTCATGGTATTGATAAAGCTGTTGCCGATACT ++HWI-EAS210R_0001:4:10:891:445#0/1 +gggggggddggggggggggggggggggggggggggggg +@HWI-EAS210R_0001:4:10:891:1476#0/1 +GGCGACTCCTTCGAGATGGACGCCGTTGGCGCTCTCCG ++HWI-EAS210R_0001:4:10:891:1476#0/1 +ggegggffghhggfhggfgeggffgggffggagfggfg +@HWI-EAS210R_0001:4:10:891:323#0/1 +TGTAAAACAGGTGCCGAAGAAGCTGGAGTAACAGAAGT ++HWI-EAS210R_0001:4:10:891:323#0/1 +gggcgfefghgggghgfgfgggggggggegaaeeagge +@HWI-EAS210R_0001:4:10:891:69#0/1 +CTTGGCTTCCTTGCTGGTCAGATTGGTCGTCTTATTAC ++HWI-EAS210R_0001:4:10:891:69#0/1 +gggefgggfgafP_Vdbacbgegebd`fbaeggaggQd +@HWI-EAS210R_0001:4:10:891:1379#0/1 +GACCCTGTTTTGTATGGCAACTTGCCGCCGCGTGAAAT ++HWI-EAS210R_0001:4:10:891:1379#0/1 +gggggggggggggggggggggggggggggfgefggggg +@HWI-EAS210R_0001:4:10:891:608#0/1 +CTTATGCTAATTTGCATACTGACCAAGAACGTGATTAC ++HWI-EAS210R_0001:4:10:891:608#0/1 +ggggggggggggggggggggggggfggggggggggggg +@HWI-EAS210R_0001:4:10:891:1831#0/1 +GTGGATTACTATCTGAGTCCGATGCTGTTCAACCACTA ++HWI-EAS210R_0001:4:10:891:1831#0/1 +ggggggggggggggggghggggggggghgggggfgggg +@HWI-EAS210R_0001:4:10:891:412#0/1 +TAAGAAATCATGAGTCAAGTTACTGAACAATCCGTACG ++HWI-EAS210R_0001:4:10:891:412#0/1 +gggggggggggggggggggggggggggggggggggggg +@HWI-EAS210R_0001:4:10:891:346#0/1 +AATTCATCCATTAACTTCTCATCAACATATACAAACTC ++HWI-EAS210R_0001:4:10:891:346#0/1 +^W][_b^be_cRgaggdgdgbL_^_\ZN\R]]`W]cYe +@HWI-EAS210R_0001:4:10:891:174#0/1 +AGTGGAGGCCTCCAGCAATCTTGAACACTCATCCTTAA ++HWI-EAS210R_0001:4:10:891:174#0/1 +ggggggggggggggggggggggcgggggfgggghghcg +@HWI-EAS210R_0001:4:10:891:1415#0/1 +ACCAACCATCAGCATGAGCCTGTCGCATTGCATTCATC ++HWI-EAS210R_0001:4:10:891:1415#0/1 +gggggggggggdghgggggggggggggggggggggggg +@HWI-EAS210R_0001:4:10:891:705#0/1 +TTTCCGTTGCTGCCATCTCAAAAACATTTGGACTGCTC ++HWI-EAS210R_0001:4:10:891:705#0/1 +gfgggggggggfgggggggggggfgddgggggggeggg +@HWI-EAS210R_0001:4:10:891:1398#0/1 +GCTGAACGCCCTCTTAAGGATATTCGCGATGAGTATAA ++HWI-EAS210R_0001:4:10:891:1398#0/1 +ebehggdhdfgffdda`e\ecgfggedgegeggdfdfg +@HWI-EAS210R_0001:4:10:891:971#0/1 +TAGCTTTAAGCGGCTCACCTTTAGCATCAACAGGCCAC ++HWI-EAS210R_0001:4:10:891:971#0/1 +ggfgggghggggggfggfgghggfggggfghggffghg +@HWI-EAS210R_0001:4:10:891:627#0/1 +GTCGCAGTAGGCGGAAAACGAACAAGCGCAAGAGTAAA ++HWI-EAS210R_0001:4:10:891:627#0/1 +dfdffaeaaagggg[bgggghgghgggfgg\efefggd +@HWI-EAS210R_0001:4:10:891:1822#0/1 +GAGTAGTTGAAATGGTAATAAGACGACCAATCTGACCA ++HWI-EAS210R_0001:4:10:891:1822#0/1 +gggeggfggegggggdggggghgfgedgggggcfgdcg +@HWI-EAS210R_0001:4:10:891:1103#0/1 +AAGGGTAATAAGAACGAACCATAAAAAAGCCTCCAAGA ++HWI-EAS210R_0001:4:10:891:1103#0/1 +ggeggfggggggggghggggggggggfffgdggggggb +@HWI-EAS210R_0001:4:10:891:586#0/1 +TATGTCTAATATTCAAACTGGCGCCGAGCGTATGCCGC ++HWI-EAS210R_0001:4:10:891:586#0/1 +ggggggggggggegfgggfcgggggffggggggggggg +@HWI-EAS210R_0001:4:10:891:1620#0/1 +TTTTGTGTGCCTGAGTATGGTACAGCTAATGGCCGTCT ++HWI-EAS210R_0001:4:10:891:1620#0/1 +gggggfggggghgggggghcgggdgggegggefgfT`` +@HWI-EAS210R_0001:4:10:891:42#0/1 +CTAAAGGCAAGCGTAAAGGCGCTCGTCTTTGGTATGTA ++HWI-EAS210R_0001:4:10:891:42#0/1 +ggefggggggfggggfaf_fggggdfggggfaYbfd]b +@HWI-EAS210R_0001:4:10:891:1609#0/1 +GCCATAGCACCAGAAACAAAACTAGGGGCGGCCTCATC ++HWI-EAS210R_0001:4:10:891:1609#0/1 +gggghgggghfgghgggggfffggggcgggfdggfggf +@HWI-EAS210R_0001:4:10:891:1028#0/1 +AATCGCGTAGAGGCTTTGCTATTCAGCGTTTGATGAAT ++HWI-EAS210R_0001:4:10:891:1028#0/1 +gfgggggggdgbggfgegeeggggggggdggegggddg +@HWI-EAS210R_0001:4:10:891:902#0/1 +ATAAACATCATAGGCAGTCGGGAGGGTAGTCGGAACCG ++HWI-EAS210R_0001:4:10:891:902#0/1 +ggggggfgggggggggggggggggfddggggggggggg diff -r f54e602ca23310b8a237af1388d7bc2a37047d34 -r f02a75ce05b71457845c0e5f56f409047a841967 tools/sr_mapping/bowtie2_wrapper.xml --- a/tools/sr_mapping/bowtie2_wrapper.xml +++ b/tools/sr_mapping/bowtie2_wrapper.xml @@ -87,10 +87,10 @@ ## read group information #if str($read_group.selection) == "yes": #if $read_group.rgid and $read_group.rglb and $read_group.rgpl and $read_group.rgsm: - --rg-id $read_group.rgid - --rg LB:$read_group.rglb - --rg PL:$read_group.rgpl - --rg SM:$read_group.rgsm + --rg-id "$read_group.rgid" + --rg "LB:$read_group.rglb" + --rg "PL:$read_group.rgpl" + --rg "SM:$read_group.rgsm" #end if #end if @@ -254,6 +254,17 @@ </outputs><tests> + <test> +  + <param name="type" value="single"/> + <param name="selection" value="no"/> + <param name="full" value="no"/> + <param name="unaligned_file" value="false"/> + <param name="source" value="history" /> + <param name="input_1" value="bowtie2/phix_reads.fastq" ftype="fastqsanger"/> + <param name="own_file" value="bowtie2/phix_genome.fasta" /> + <output name="output" file="bowtie2/phix_mapped.bam" /> + </test></tests><help> Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: dannon: Branch close
by commits-noreply＠bitbucket.org 07 Aug '13

07 Aug '13

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/e68eee8054d2/ Changeset: e68eee8054d2 Branch: sort User: dannon Date: 2013-08-07 19:36:25 Summary: Branch close Affected #: 0 files Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: 2 new changesets
by commits-noreply＠bitbucket.org 07 Aug '13

07 Aug '13

2 new commits in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/1d91e253734e/ Changeset: 1d91e253734e Branch: sort User: ghuls Date: 2013-05-28 15:48:45 Summary: Fix sorting of numbers in scientific notation. The -n option of sort can't handle numbers in scientific notation. The -g option of sort can handle those numbers correctly. Affected #: 3 files diff -r 31714646a7b441f34a43748065278a1b08940c3c -r 1d91e253734edaaafa8f97bb4382ce18fd02371d test-data/sort_in2.bed --- /dev/null +++ b/test-data/sort_in2.bed @@ -0,0 +1,6 @@ +chr10 100 200 feature1 100.01 + +chr20 800 900 feature2 1.1 + +chr2 500 600 feature3 1000.1 + +chr1 300 400 feature4 1.1e-05 + +chr21 300 500 feature5 1.1e2 + +chr15 700 800 feature6 1.1e4 + diff -r 31714646a7b441f34a43748065278a1b08940c3c -r 1d91e253734edaaafa8f97bb4382ce18fd02371d test-data/sort_out3.bed --- /dev/null +++ b/test-data/sort_out3.bed @@ -0,0 +1,6 @@ +chr1 300 400 feature4 1.1e-05 + +chr20 800 900 feature2 1.1 + +chr10 100 200 feature1 100.01 + +chr21 300 500 feature5 1.1e2 + +chr2 500 600 feature3 1000.1 + +chr15 700 800 feature6 1.1e4 + diff -r 31714646a7b441f34a43748065278a1b08940c3c -r 1d91e253734edaaafa8f97bb4382ce18fd02371d tools/filters/sorter.xml --- a/tools/filters/sorter.xml +++ b/tools/filters/sorter.xml @@ -1,19 +1,37 @@ -<tool id="sort1" name="Sort" version="1.0.2"> +<tool id="sort1" name="Sort" version="1.0.3"><description>data in ascending or descending order</description><command interpreter="python"> sorter.py - --input=$input - --output=$out_file1 - #set $style = '' if (str($style) == 'alpha') else 'n' + --input=${input} + --output=${out_file1} + + #if (str($style) == 'num'): + #set $style = 'n' + #elif (str($style) == 'gennum'): + #set $style = 'g' + #else: + #set $style = '' + #end if + #set $order = '' if (str($order) == 'ASC') else 'r' - --key=$column,$column$style$order + + --key=${column},${column}${style}${order} + #for $col in $column_set: - #set $other_column = str($col.other_column) - #set $other_style = '' if (str($col.other_style) == "alpha") else 'n' - #set $other_order = '' if (str($col.other_order) == "ASC") else 'r' - --key=$other_column,$other_column$other_style$other_order + #set $other_column = str($col.other_column) + + #if (str($col.other_style) == 'num'): + #set $other_style = 'n' + #elif (str($col.other_style) == 'gennum'): + #set $other_style = 'g' + #else: + #set $other_style = '' + #end if + + #set $other_order = '' if (str($col.other_order) == "ASC") else 'r' + --key=${other_column},${other_column}${other_style}${other_order} #end for </command><inputs> @@ -21,6 +39,7 @@ <param name="column" label="on column" type="data_column" data_ref="input" accept_default="true"/><param name="style" type="select" label="with flavor"><option value="num">Numerical sort</option> + <option value="gennum">General numeric sort</option><option value="alpha">Alphabetical sort</option></param><param name="order" type="select" label="everything in"> @@ -31,6 +50,7 @@ <param name="other_column" label="on column" type="data_column" data_ref="input" accept_default="true" /><param name="other_style" type="select" label="with flavor"><option value="num">Numerical sort</option> + <option value="gennum">General numeric sort</option><option value="alpha">Alphabetical sort</option></param><param name="other_order" type="select" label="everything in"> @@ -63,6 +83,13 @@ <param name="other_order" value="ASC"/><output name="out_file1" file="sort_out2.bed"/></test> + <test> + <param name="input" value="sort_in2.bed"/> + <param name="column" value="5"/> + <param name="style" value="gennum"/> + <param name="order" value="ASC"/> + <output name="out_file1" file="sort_out3.bed"/> + </test></tests><help> .. class:: infomark @@ -75,8 +102,9 @@ This tool sorts the dataset on any number of columns in either ascending or descending order. -* Numerical sort orders numbers by their magnitude, ignores all characters besides numbers, and evaluates a string of numbers to the value they signify. -* Alphabetical sort is a phonebook type sort based on the conventional order of letters in an alphabet. Each nth letter is compared with the nth letter of other words in the list, starting at the first letter of each word and advancing to the second, third, fourth, and so on, until the order is established. Therefore, in an alphabetical sort, 2 comes after 100 (1 < 2). +* **Numerical sort** orders numbers by their magnitude, ignores all characters besides numbers, and evaluates a string of numbers to the value they signify. +* **General numeric sort** orders numbers by their general numerical value. Unlike the numerical sort option, it can handle numbers in scientific notation too. +* **Alphabetical sort** is a phonebook type sort based on the conventional order of letters in an alphabet. Each nth letter is compared with the nth letter of other words in the list, starting at the first letter of each word and advancing to the second, third, fourth, and so on, until the order is established. Therefore, in an alphabetical sort, 2 comes after 100 (1 < 2). ----- @@ -106,7 +134,7 @@ A g 10 H h 43 A g 4 I h 500 -on columns 1 (alpha), 3 (num), and 6 (num) in ascending order will yield:: +on columns 1 (alphabetical), 3 (numerical), and 6 (numerical) in ascending order will yield:: A kk 4 I h 111 A edf 4 tw b 234 @@ -127,5 +155,34 @@ Pd gf 7 Gthe de 567 Q d 7 II jhu 45 rS hty 90 YY LOp 89 + + +Sorting the following:: + + chr10 100 200 feature1 100.01 + + chr20 800 900 feature2 1.1 + + chr2 500 600 feature3 1000.1 + + chr1 300 400 feature4 1.1e-05 + + chr21 300 500 feature5 1.1e2 + + chr15 700 800 feature6 1.1e4 + + +on column 5 (numerical) in ascending order will yield:: + + chr1 300 400 feature4 1.1e-05 + + chr15 700 800 feature6 1.1e4 + + chr20 800 900 feature2 1.1 + + chr21 300 500 feature5 1.1e2 + + chr10 100 200 feature1 100.01 + + chr2 500 600 feature3 1000.1 + + +on column 5 (general numeric) in ascending order will yield:: + + chr1 300 400 feature4 1.1e-05 + + chr20 800 900 feature2 1.1 + + chr10 100 200 feature1 100.01 + + chr21 300 500 feature5 1.1e2 + + chr2 500 600 feature3 1000.1 + + chr15 700 800 feature6 1.1e4 + + </help></tool> https://bitbucket.org/galaxy/galaxy-central/commits/f54e602ca233/ Changeset: f54e602ca233 User: dannon Date: 2013-08-07 19:33:27 Summary: Merged in ghuls/galaxy-central/sort (pull request #172) Fix sorting of numbers in scientific notation. Affected #: 3 files diff -r 3e6f22a221eb521dddcb13bb7c1a07b4b9551932 -r f54e602ca23310b8a237af1388d7bc2a37047d34 test-data/sort_in2.bed --- /dev/null +++ b/test-data/sort_in2.bed @@ -0,0 +1,6 @@ +chr10 100 200 feature1 100.01 + +chr20 800 900 feature2 1.1 + +chr2 500 600 feature3 1000.1 + +chr1 300 400 feature4 1.1e-05 + +chr21 300 500 feature5 1.1e2 + +chr15 700 800 feature6 1.1e4 + diff -r 3e6f22a221eb521dddcb13bb7c1a07b4b9551932 -r f54e602ca23310b8a237af1388d7bc2a37047d34 test-data/sort_out3.bed --- /dev/null +++ b/test-data/sort_out3.bed @@ -0,0 +1,6 @@ +chr1 300 400 feature4 1.1e-05 + +chr20 800 900 feature2 1.1 + +chr10 100 200 feature1 100.01 + +chr21 300 500 feature5 1.1e2 + +chr2 500 600 feature3 1000.1 + +chr15 700 800 feature6 1.1e4 + diff -r 3e6f22a221eb521dddcb13bb7c1a07b4b9551932 -r f54e602ca23310b8a237af1388d7bc2a37047d34 tools/filters/sorter.xml --- a/tools/filters/sorter.xml +++ b/tools/filters/sorter.xml @@ -1,19 +1,37 @@ -<tool id="sort1" name="Sort" version="1.0.2"> +<tool id="sort1" name="Sort" version="1.0.3"><description>data in ascending or descending order</description><command interpreter="python"> sorter.py - --input=$input - --output=$out_file1 - #set $style = '' if (str($style) == 'alpha') else 'n' + --input=${input} + --output=${out_file1} + + #if (str($style) == 'num'): + #set $style = 'n' + #elif (str($style) == 'gennum'): + #set $style = 'g' + #else: + #set $style = '' + #end if + #set $order = '' if (str($order) == 'ASC') else 'r' - --key=$column,$column$style$order + + --key=${column},${column}${style}${order} + #for $col in $column_set: - #set $other_column = str($col.other_column) - #set $other_style = '' if (str($col.other_style) == "alpha") else 'n' - #set $other_order = '' if (str($col.other_order) == "ASC") else 'r' - --key=$other_column,$other_column$other_style$other_order + #set $other_column = str($col.other_column) + + #if (str($col.other_style) == 'num'): + #set $other_style = 'n' + #elif (str($col.other_style) == 'gennum'): + #set $other_style = 'g' + #else: + #set $other_style = '' + #end if + + #set $other_order = '' if (str($col.other_order) == "ASC") else 'r' + --key=${other_column},${other_column}${other_style}${other_order} #end for </command><inputs> @@ -21,6 +39,7 @@ <param name="column" label="on column" type="data_column" data_ref="input" accept_default="true"/><param name="style" type="select" label="with flavor"><option value="num">Numerical sort</option> + <option value="gennum">General numeric sort</option><option value="alpha">Alphabetical sort</option></param><param name="order" type="select" label="everything in"> @@ -31,6 +50,7 @@ <param name="other_column" label="on column" type="data_column" data_ref="input" accept_default="true" /><param name="other_style" type="select" label="with flavor"><option value="num">Numerical sort</option> + <option value="gennum">General numeric sort</option><option value="alpha">Alphabetical sort</option></param><param name="other_order" type="select" label="everything in"> @@ -63,6 +83,13 @@ <param name="other_order" value="ASC"/><output name="out_file1" file="sort_out2.bed"/></test> + <test> + <param name="input" value="sort_in2.bed"/> + <param name="column" value="5"/> + <param name="style" value="gennum"/> + <param name="order" value="ASC"/> + <output name="out_file1" file="sort_out3.bed"/> + </test></tests><help> .. class:: infomark @@ -75,8 +102,9 @@ This tool sorts the dataset on any number of columns in either ascending or descending order. -* Numerical sort orders numbers by their magnitude, ignores all characters besides numbers, and evaluates a string of numbers to the value they signify. -* Alphabetical sort is a phonebook type sort based on the conventional order of letters in an alphabet. Each nth letter is compared with the nth letter of other words in the list, starting at the first letter of each word and advancing to the second, third, fourth, and so on, until the order is established. Therefore, in an alphabetical sort, 2 comes after 100 (1 < 2). +* **Numerical sort** orders numbers by their magnitude, ignores all characters besides numbers, and evaluates a string of numbers to the value they signify. +* **General numeric sort** orders numbers by their general numerical value. Unlike the numerical sort option, it can handle numbers in scientific notation too. +* **Alphabetical sort** is a phonebook type sort based on the conventional order of letters in an alphabet. Each nth letter is compared with the nth letter of other words in the list, starting at the first letter of each word and advancing to the second, third, fourth, and so on, until the order is established. Therefore, in an alphabetical sort, 2 comes after 100 (1 < 2). ----- @@ -106,7 +134,7 @@ A g 10 H h 43 A g 4 I h 500 -on columns 1 (alpha), 3 (num), and 6 (num) in ascending order will yield:: +on columns 1 (alphabetical), 3 (numerical), and 6 (numerical) in ascending order will yield:: A kk 4 I h 111 A edf 4 tw b 234 @@ -127,5 +155,34 @@ Pd gf 7 Gthe de 567 Q d 7 II jhu 45 rS hty 90 YY LOp 89 + + +Sorting the following:: + + chr10 100 200 feature1 100.01 + + chr20 800 900 feature2 1.1 + + chr2 500 600 feature3 1000.1 + + chr1 300 400 feature4 1.1e-05 + + chr21 300 500 feature5 1.1e2 + + chr15 700 800 feature6 1.1e4 + + +on column 5 (numerical) in ascending order will yield:: + + chr1 300 400 feature4 1.1e-05 + + chr15 700 800 feature6 1.1e4 + + chr20 800 900 feature2 1.1 + + chr21 300 500 feature5 1.1e2 + + chr10 100 200 feature1 100.01 + + chr2 500 600 feature3 1000.1 + + +on column 5 (general numeric) in ascending order will yield:: + + chr1 300 400 feature4 1.1e-05 + + chr20 800 900 feature2 1.1 + + chr10 100 200 feature1 100.01 + + chr21 300 500 feature5 1.1e2 + + chr2 500 600 feature3 1000.1 + + chr15 700 800 feature6 1.1e4 + + </help></tool> Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: 2 new changesets
by commits-noreply＠bitbucket.org 07 Aug '13

07 Aug '13

2 new commits in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/caaf7db30954/ Changeset: caaf7db30954 User: jmchilton Date: 2012-12-14 21:51:20 Summary: Allow display application parameters to be defined with allow_override="True". If parameter X is defined with allow_override="True", then if the dataset link contains the query parameter app_X="foo", the display parameter X will be set to foo regardless of the template text. Affected #: 3 files diff -r eae248415389203907b5b951f139a200024ae069 -r caaf7db309547d9633e9619c89b33fb7e90a92c4 lib/galaxy/datatypes/display_applications/application.py --- a/lib/galaxy/datatypes/display_applications/application.py +++ b/lib/galaxy/datatypes/display_applications/application.py @@ -57,14 +57,17 @@ rval[ key ] = value rval[ DEFAULT_DATASET_NAME ] = data #always have the display dataset name available return rval - def build_parameter_dict( self, data, dataset_hash, user_hash, trans ): + def build_parameter_dict( self, data, dataset_hash, user_hash, trans, app_kwds ): other_values = self.get_inital_values( data, trans ) other_values[ 'DATASET_HASH' ] = dataset_hash other_values[ 'USER_HASH' ] = user_hash for name, param in self.parameters.iteritems(): assert name not in other_values, "The display parameter '%s' has been defined more than once." % name if param.ready( other_values ): - other_values[ name ] = param.get_value( other_values, dataset_hash, user_hash, trans )#subsequent params can rely on this value + if name in app_kwds and param.allow_override: + other_values[ name ] = app_kwds[ name ] + else: + other_values[ name ] = param.get_value( other_values, dataset_hash, user_hash, trans )#subsequent params can rely on this value else: other_values[ name ] = None return False, other_values #need to stop here, next params may need this value @@ -120,13 +123,13 @@ return iter( self.links ) class PopulatedDisplayApplicationLink( object ): - def __init__( self, display_application_link, data, dataset_hash, user_hash, trans ): + def __init__( self, display_application_link, data, dataset_hash, user_hash, trans, app_kwds ): self.link = display_application_link self.data = data self.dataset_hash = dataset_hash self.user_hash = user_hash self.trans = trans - self.ready, self.parameters = self.link.build_parameter_dict( self.data, self.dataset_hash, self.user_hash, trans ) + self.ready, self.parameters = self.link.build_parameter_dict( self.data, self.dataset_hash, self.user_hash, trans, app_kwds ) def display_ready( self ): return self.ready def get_param_value( self, name ): @@ -184,9 +187,9 @@ version = "1.0.0" self.version = version self.links = odict() - def get_link( self, link_name, data, dataset_hash, user_hash, trans ): + def get_link( self, link_name, data, dataset_hash, user_hash, trans, app_kwds ): #returns a link object with data knowledge to generate links - return PopulatedDisplayApplicationLink( self.links[ link_name ], data, dataset_hash, user_hash, trans ) + return PopulatedDisplayApplicationLink( self.links[ link_name ], data, dataset_hash, user_hash, trans, app_kwds ) def filter_by_dataset( self, data, trans ): filtered = DisplayApplication( self.id, self.name, self.datatypes_registry, version = self.version ) for link_name, link_value in self.links.iteritems(): diff -r eae248415389203907b5b951f139a200024ae069 -r caaf7db309547d9633e9619c89b33fb7e90a92c4 lib/galaxy/datatypes/display_applications/parameters.py --- a/lib/galaxy/datatypes/display_applications/parameters.py +++ b/lib/galaxy/datatypes/display_applications/parameters.py @@ -28,6 +28,7 @@ self.viewable = string_as_bool( elem.get( 'viewable', 'False' ) ) #only allow these to be viewed via direct url when explicitly set to viewable self.strip = string_as_bool( elem.get( 'strip', 'False' ) ) self.strip_https = string_as_bool( elem.get( 'strip_https', 'False' ) ) + self.allow_override = string_as_bool( elem.get( 'allow_override', 'False' ) ) # Passing query param app_<name>=<value> to dataset controller allows override if this is true. def get_value( self, other_values, dataset_hash, user_hash, trans ): raise Exception, 'Unimplemented' def prepare( self, other_values, dataset_hash, user_hash, trans ): @@ -126,7 +127,7 @@ def __init__( self, elem, link ): DisplayApplicationParameter.__init__( self, elem, link ) - self.text = elem.text + self.text = elem.text or '' def get_value( self, other_values, dataset_hash, user_hash, trans ): value = fill_template( self.text, context = other_values ) if self.strip: diff -r eae248415389203907b5b951f139a200024ae069 -r caaf7db309547d9633e9619c89b33fb7e90a92c4 lib/galaxy/webapps/galaxy/controllers/dataset.py --- a/lib/galaxy/webapps/galaxy/controllers/dataset.py +++ b/lib/galaxy/webapps/galaxy/controllers/dataset.py @@ -719,6 +719,12 @@ @web.expose def display_application( self, trans, dataset_id=None, user_id=None, app_name = None, link_name = None, app_action = None, action_param = None, **kwds ): """Access to external display applications""" + # Build list of parameters to pass in to display application logic (app_kwds) + app_kwds = {} + for name, value in dict(kwds).iteritems(): # clone kwds because we remove stuff as we go. + if name.startswith( "app_" ): + app_kwds[ name[ len( "app_" ): ] ] = value + del kwds[ name ] if kwds: log.debug( "Unexpected Keywords passed to display_application: %s" % kwds ) #route memory? #decode ids @@ -742,7 +748,7 @@ display_app = trans.app.datatypes_registry.display_applications.get( app_name ) assert display_app, "Unknown display application has been requested: %s" % app_name dataset_hash, user_hash = encode_dataset_user( trans, data, user ) - display_link = display_app.get_link( link_name, data, dataset_hash, user_hash, trans ) + display_link = display_app.get_link( link_name, data, dataset_hash, user_hash, trans, app_kwds ) assert display_link, "Unknown display link has been requested: %s" % link_name if data.state == data.states.ERROR: msg.append( ( 'This dataset is in an error state, you cannot view it at an external display application.', 'error' ) ) https://bitbucket.org/galaxy/galaxy-central/commits/3e6f22a221eb/ Changeset: 3e6f22a221eb User: dannon Date: 2013-08-07 19:27:58 Summary: Merged in jmchilton/galaxy-central-display-application-parameters (pull request #99) Display Application Parameter Enhancements Affected #: 3 files diff -r 6e13df4bc3f0e73b69ba510cc626c62145b0a81c -r 3e6f22a221eb521dddcb13bb7c1a07b4b9551932 lib/galaxy/datatypes/display_applications/application.py --- a/lib/galaxy/datatypes/display_applications/application.py +++ b/lib/galaxy/datatypes/display_applications/application.py @@ -57,14 +57,17 @@ rval[ key ] = value rval[ DEFAULT_DATASET_NAME ] = data #always have the display dataset name available return rval - def build_parameter_dict( self, data, dataset_hash, user_hash, trans ): + def build_parameter_dict( self, data, dataset_hash, user_hash, trans, app_kwds ): other_values = self.get_inital_values( data, trans ) other_values[ 'DATASET_HASH' ] = dataset_hash other_values[ 'USER_HASH' ] = user_hash for name, param in self.parameters.iteritems(): assert name not in other_values, "The display parameter '%s' has been defined more than once." % name if param.ready( other_values ): - other_values[ name ] = param.get_value( other_values, dataset_hash, user_hash, trans )#subsequent params can rely on this value + if name in app_kwds and param.allow_override: + other_values[ name ] = app_kwds[ name ] + else: + other_values[ name ] = param.get_value( other_values, dataset_hash, user_hash, trans )#subsequent params can rely on this value else: other_values[ name ] = None return False, other_values #need to stop here, next params may need this value @@ -120,13 +123,13 @@ return iter( self.links ) class PopulatedDisplayApplicationLink( object ): - def __init__( self, display_application_link, data, dataset_hash, user_hash, trans ): + def __init__( self, display_application_link, data, dataset_hash, user_hash, trans, app_kwds ): self.link = display_application_link self.data = data self.dataset_hash = dataset_hash self.user_hash = user_hash self.trans = trans - self.ready, self.parameters = self.link.build_parameter_dict( self.data, self.dataset_hash, self.user_hash, trans ) + self.ready, self.parameters = self.link.build_parameter_dict( self.data, self.dataset_hash, self.user_hash, trans, app_kwds ) def display_ready( self ): return self.ready def get_param_value( self, name ): @@ -184,9 +187,9 @@ version = "1.0.0" self.version = version self.links = odict() - def get_link( self, link_name, data, dataset_hash, user_hash, trans ): + def get_link( self, link_name, data, dataset_hash, user_hash, trans, app_kwds ): #returns a link object with data knowledge to generate links - return PopulatedDisplayApplicationLink( self.links[ link_name ], data, dataset_hash, user_hash, trans ) + return PopulatedDisplayApplicationLink( self.links[ link_name ], data, dataset_hash, user_hash, trans, app_kwds ) def filter_by_dataset( self, data, trans ): filtered = DisplayApplication( self.id, self.name, self.datatypes_registry, version = self.version ) for link_name, link_value in self.links.iteritems(): diff -r 6e13df4bc3f0e73b69ba510cc626c62145b0a81c -r 3e6f22a221eb521dddcb13bb7c1a07b4b9551932 lib/galaxy/datatypes/display_applications/parameters.py --- a/lib/galaxy/datatypes/display_applications/parameters.py +++ b/lib/galaxy/datatypes/display_applications/parameters.py @@ -28,6 +28,7 @@ self.viewable = string_as_bool( elem.get( 'viewable', 'False' ) ) #only allow these to be viewed via direct url when explicitly set to viewable self.strip = string_as_bool( elem.get( 'strip', 'False' ) ) self.strip_https = string_as_bool( elem.get( 'strip_https', 'False' ) ) + self.allow_override = string_as_bool( elem.get( 'allow_override', 'False' ) ) # Passing query param app_<name>=<value> to dataset controller allows override if this is true. def get_value( self, other_values, dataset_hash, user_hash, trans ): raise Exception, 'Unimplemented' def prepare( self, other_values, dataset_hash, user_hash, trans ): @@ -126,7 +127,7 @@ def __init__( self, elem, link ): DisplayApplicationParameter.__init__( self, elem, link ) - self.text = elem.text + self.text = elem.text or '' def get_value( self, other_values, dataset_hash, user_hash, trans ): value = fill_template( self.text, context = other_values ) if self.strip: diff -r 6e13df4bc3f0e73b69ba510cc626c62145b0a81c -r 3e6f22a221eb521dddcb13bb7c1a07b4b9551932 lib/galaxy/webapps/galaxy/controllers/dataset.py --- a/lib/galaxy/webapps/galaxy/controllers/dataset.py +++ b/lib/galaxy/webapps/galaxy/controllers/dataset.py @@ -720,6 +720,12 @@ @web.expose def display_application( self, trans, dataset_id=None, user_id=None, app_name = None, link_name = None, app_action = None, action_param = None, **kwds ): """Access to external display applications""" + # Build list of parameters to pass in to display application logic (app_kwds) + app_kwds = {} + for name, value in dict(kwds).iteritems(): # clone kwds because we remove stuff as we go. + if name.startswith( "app_" ): + app_kwds[ name[ len( "app_" ): ] ] = value + del kwds[ name ] if kwds: log.debug( "Unexpected Keywords passed to display_application: %s" % kwds ) #route memory? #decode ids @@ -743,7 +749,7 @@ display_app = trans.app.datatypes_registry.display_applications.get( app_name ) assert display_app, "Unknown display application has been requested: %s" % app_name dataset_hash, user_hash = encode_dataset_user( trans, data, user ) - display_link = display_app.get_link( link_name, data, dataset_hash, user_hash, trans ) + display_link = display_app.get_link( link_name, data, dataset_hash, user_hash, trans, app_kwds ) assert display_link, "Unknown display link has been requested: %s" % link_name if data.state == data.states.ERROR: msg.append( ( 'This dataset is in an error state, you cannot view it at an external display application.', 'error' ) ) Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: carlfeberhard: compile_templates.py: add option to save intermediate 'handlebars' files when compiling multiple template files
by commits-noreply＠bitbucket.org 07 Aug '13

07 Aug '13

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/6e13df4bc3f0/ Changeset: 6e13df4bc3f0 User: carlfeberhard Date: 2013-08-07 18:26:42 Summary: compile_templates.py: add option to save intermediate 'handlebars' files when compiling multiple template files Affected #: 1 file diff -r aa9aeb756031a63f4cd7dc4e1b2d84b12522ffef -r 6e13df4bc3f0e73b69ba510cc626c62145b0a81c static/scripts/templates/compile_templates.py --- a/static/scripts/templates/compile_templates.py +++ b/static/scripts/templates/compile_templates.py @@ -208,13 +208,15 @@ print "\nCall this script with the '-h' for more help" # delete multi template intermediate files - print "\nCleaning up intermediate multi-template template files:" - for filename in multi_template_template_filenames: - try: - print 'removing', filename - os.remove( filename ) - except Exception, exc: - print exc + if options.remove_intermediate: + print "\nCleaning up intermediate multi-template template files:" + for filename in multi_template_template_filenames: + try: + print 'removing', filename + os.remove( filename ) + except Exception, exc: + print exc + # ------------------------------------------------------------------------------ if __name__ == '__main__': @@ -226,7 +228,10 @@ help=( 'indicates that files ending with the given string contain multiple ' + 'templates and the script should break those into individual ' + 'handlebars templates (defaults to "%s")' ) % DEFAULT_MULTI_EXT ) - - ( options, args ) = optparser.parse_args() + optparser.add_option( '--no-remove', action='store_false', dest='remove_intermediate', default=True, + help=( 'remove intermediate *.handlebars files when using multiple template' + + 'files (defaults to "True")' ) ) + + ( options, args ) = optparser.parse_args() sys.exit( main( options, args ) ) Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: carlfeberhard: Visualization framework: add config files for built-in visualiztions
by commits-noreply＠bitbucket.org 07 Aug '13

07 Aug '13

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/aa9aeb756031/ Changeset: aa9aeb756031 User: carlfeberhard Date: 2013-08-07 17:48:25 Summary: Visualization framework: add config files for built-in visualiztions Affected #: 4 files diff -r 425272160a7f3764823e18ea395539c54c8e41fd -r aa9aeb756031a63f4cd7dc4e1b2d84b12522ffef config/plugins/visualizations/circster/config/circster.xml --- /dev/null +++ b/config/plugins/visualizations/circster/config/circster.xml @@ -0,0 +1,28 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!DOCTYPE visualization SYSTEM "../../visualization.dtd"> +<visualization name="circster"> + <data_sources> + <data_source> + <model_class>HistoryDatasetAssociation</model_class> + <test type="isinstance" test_attr="datatype" result_type="datatype">data.Data</test> + <to_param param_attr="id">dataset_id</to_param> + <to_param assign="hda">hda_ldda</to_param> + </data_source> + <data_source> + <model_class>LibraryDatasetDatasetAssociation</model_class> + <test type="isinstance" test_attr="datatype" result_type="datatype">data.Data</test> + <to_param param_attr="id">dataset_id</to_param> + <to_param assign="ldda">hda_ldda</to_param> + </data_source> + </data_sources> + <params> + <param type="visualization">id</param> + <param type="hda_or_ldda">dataset_id</param> + <param_modifier type="string" modifies="dataset_id">hda_ldda</param_modifier> + <param type="dbkey">dbkey</param> + </params> +  + <template_root>webapps/galaxy/visualization</template_root> + <template>circster.mako</template> + <render_location>_top</render_location> +</visualization> diff -r 425272160a7f3764823e18ea395539c54c8e41fd -r aa9aeb756031a63f4cd7dc4e1b2d84b12522ffef config/plugins/visualizations/phyloviz/config/phyloviz.xml --- /dev/null +++ b/config/plugins/visualizations/phyloviz/config/phyloviz.xml @@ -0,0 +1,20 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!DOCTYPE visualization SYSTEM "../../visualization.dtd"> +<visualization name="phyloviz"> + <data_sources> + <data_source> + <model_class>HistoryDatasetAssociation</model_class> + <test type="isinstance" test_attr="datatype" result_type="datatype">data.Newick</test> + <test type="isinstance" test_attr="datatype" result_type="datatype">data.Nexus</test> + <to_param param_attr="id">dataset_id</to_param> + </data_source> + </data_sources> + <params> + <param type="dataset" var_name_in_template="hda" required="true">dataset_id</param> + <param type="integer" default="0">tree_index</param> + </params> +  + <template_root>webapps/galaxy/visualization</template_root> + <template>phyloviz.mako</template> + <render_location>_top</render_location> +</visualization> diff -r 425272160a7f3764823e18ea395539c54c8e41fd -r aa9aeb756031a63f4cd7dc4e1b2d84b12522ffef config/plugins/visualizations/sweepster/config/sweepster.xml --- /dev/null +++ b/config/plugins/visualizations/sweepster/config/sweepster.xml @@ -0,0 +1,27 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!DOCTYPE visualization SYSTEM "../../visualization.dtd"> +<visualization name="sweepster"> + <data_sources> + <data_source> + <model_class>HistoryDatasetAssociation</model_class> + <test type="isinstance" test_attr="datatype" result_type="datatype">data.Data</test> + <to_param param_attr="id">dataset_id</to_param> + <to_param assign="hda">hda_ldda</to_param> + </data_source> + <data_source> + <model_class>LibraryDatasetDatasetAssociation</model_class> + <test type="isinstance" test_attr="datatype" result_type="datatype">data.Data</test> + <to_param param_attr="id">dataset_id</to_param> + <to_param assign="ldda">hda_ldda</to_param> + </data_source> + </data_sources> + <params> + <param type="visualization" var_name_in_template="viz">visualization</param> + <param type="hda_or_ldda" var_name_in_template="dataset">dataset_id</param> + <param_modifier type="string" modifies="dataset_id">hda_ldda</param_modifier> + </params> +  + <template_root>webapps/galaxy/visualization</template_root> + <template>sweepster.mako</template> + <render_location>_top</render_location> +</visualization> diff -r 425272160a7f3764823e18ea395539c54c8e41fd -r aa9aeb756031a63f4cd7dc4e1b2d84b12522ffef config/plugins/visualizations/trackster/config/trackster.xml --- /dev/null +++ b/config/plugins/visualizations/trackster/config/trackster.xml @@ -0,0 +1,30 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!DOCTYPE visualization SYSTEM "../../visualization.dtd"> +<visualization name="trackster"> +  + <data_sources> + <data_source> + <model_class>HistoryDatasetAssociation</model_class> + <test type="isinstance" test_attr="datatype" result_type="datatype">data.Data</test> + <to_param param_attr="id">dataset_id</to_param> + <to_param assign="hda">hda_ldda</to_param> + <to_param param_attr="dbkey">dbkey</to_param> + </data_source> + <data_source> + <model_class>LibraryDatasetDatasetAssociation</model_class> + <test type="isinstance" test_attr="datatype" result_type="datatype">data.Data</test> + <to_param param_attr="id">dataset_id</to_param> + <to_param assign="ldda">hda_ldda</to_param> + </data_source> + </data_sources> + <params> + <param type="visualization">id</param> + <param type="dataset">dataset_id</param> + <param type="genome_region">genome_region</param> + <param type="dbkey">dbkey</param> + </params> +  + <template_root>webapps/galaxy/visualization/tracks</template_root> + <template>browser.mako</template> + <render_location>_top</render_location> +</visualization> Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: carlfeberhard: Visualition framework: have 'has_dataprovider' test work on 'test_attr'
by commits-noreply＠bitbucket.org 07 Aug '13

07 Aug '13

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/425272160a7f/ Changeset: 425272160a7f User: carlfeberhard Date: 2013-08-07 17:12:34 Summary: Visualition framework: have 'has_dataprovider' test work on 'test_attr' Affected #: 1 file diff -r f33c054d6d5b75ae545248d71ec559d74b4fa636 -r 425272160a7f3764823e18ea395539c54c8e41fd lib/galaxy/visualization/registry.py --- a/lib/galaxy/visualization/registry.py +++ b/lib/galaxy/visualization/registry.py @@ -530,11 +530,10 @@ #TODO: wish we could take this further but it would mean passing in the datatypes_registry test_fn = lambda o, result: isinstance( getter( o ), result ) - #TODO: needs cleanup - robustiosity-nessness # does the object itself have a datatype attr and does that datatype have the given dataprovider elif test_type == 'has_dataprovider': - test_fn = lambda o, result: ( hasattr( o, 'datatype' ) - and o.datatype.has_dataprovider( result ) ) + test_fn = lambda o, result: ( hasattr( getter( o ), 'has_dataprovider' ) + and getter( o ).has_dataprovider( result ) ) # default to simple (string) equilavance (coercing the test_attr to a string) else: Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: Dave Bouvier: Fix for the download_binary action type incorrectly processing downloaded compressed files.
by commits-noreply＠bitbucket.org 07 Aug '13

07 Aug '13

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/f33c054d6d5b/ Changeset: f33c054d6d5b User: Dave Bouvier Date: 2013-08-07 03:21:10 Summary: Fix for the download_binary action type incorrectly processing downloaded compressed files. Affected #: 3 files diff -r 773941fd26a43c8522c3ff9977cfda5968f65505 -r f33c054d6d5b75ae545248d71ec559d74b4fa636 lib/tool_shed/galaxy_install/tool_dependencies/common_util.py --- a/lib/tool_shed/galaxy_install/tool_dependencies/common_util.py +++ b/lib/tool_shed/galaxy_install/tool_dependencies/common_util.py @@ -71,23 +71,13 @@ __shellquote(env_shell_file_path)) return cmd -def download_binary_from_url( url, work_dir, install_dir ): +def download_binary( url, work_dir ): ''' - Download a pre-compiled binary from the specified URL. If the downloaded file is an archive, - extract it into install_dir and delete the archive. + Download a pre-compiled binary from the specified URL. ''' downloaded_filename = os.path.split( url )[ -1 ] - try: - dir = url_download( work_dir, downloaded_filename, url, extract=True ) - downloaded_filepath = os.path.join( work_dir, downloaded_filename ) - if is_compressed( downloaded_filepath ): - os.remove( downloaded_filepath ) - move_directory_files( current_dir=work_dir, - source_dir=dir, - destination_dir=install_dir ) - return True - except HTTPError: - return False + dir = url_download( work_dir, downloaded_filename, url, extract=False ) + return downloaded_filename def extract_tar( file_name, file_path ): if isgzip( file_name ) or isbz2( file_name ): diff -r 773941fd26a43c8522c3ff9977cfda5968f65505 -r f33c054d6d5b75ae545248d71ec559d74b4fa636 lib/tool_shed/galaxy_install/tool_dependencies/fabric_util.py --- a/lib/tool_shed/galaxy_install/tool_dependencies/fabric_util.py +++ b/lib/tool_shed/galaxy_install/tool_dependencies/fabric_util.py @@ -31,6 +31,15 @@ if int( version.split( "." )[ 0 ] ) < 1: raise NotImplementedError( "Install Fabric version 1.0 or later." ) +def filter_actions_after_binary_installation( actions ): + '''Filter out actions that should not be processed if a binary download succeeded.''' + filtered_actions = [] + for action in actions: + action_type, action_dict = action + if action_type in [ 'set_environment', 'chmod', 'download_binary' ]: + filtered_actions.append( action ) + return filtered_actions + def handle_command( app, tool_dependency, install_dir, cmd, return_output=False ): sa_session = app.model.context.current with settings( warn_only=True ): @@ -165,8 +174,6 @@ actions = actions_dict.get( 'actions', None ) filtered_actions = [] env_shell_file_paths = [] - # Default to false so that the install process will default to compiling. - binary_found = False if actions: with make_tmp_dir() as work_dir: with lcd( work_dir ): @@ -174,20 +181,38 @@ # are currently only two supported processes; download_by_url and clone via a "shell_command" action type. action_type, action_dict = actions[ 0 ] if action_type == 'download_binary': - # Eliminate the download_binary action so remaining actions can be processed correctly. - filtered_actions = actions[ 1: ] url = action_dict[ 'url' ] + # Get the target directory for this download, if the user has specified one. Default to the root of $INSTALL_DIR. + target_directory = action_dict.get( 'target_directory', None ) # Attempt to download a binary from the specified URL. - log.debug( 'Attempting to download from %s', url ) - binary_found = common_util.download_binary_from_url( url, work_dir, install_dir ) - if binary_found: - # If the attempt succeeded, set the action_type to binary_found, in order to skip any download_by_url or shell_command actions. + log.debug( 'Attempting to download from %s to %s', url, str( target_directory ) ) + downloaded_filename = None + try: + downloaded_filename = common_util.download_binary( url, work_dir ) + # Filter out any actions that are not download_binary, chmod, or set_environment. + filtered_actions = filter_actions_after_binary_installation( actions[ 1: ] ) + # Set actions to the same, so that the current download_binary doesn't get re-run in the filtered actions below. actions = filtered_actions - action_type = 'binary_found' - else: + except Exception, e: + log.exception( str( e ) ) # No binary exists, or there was an error downloading the binary from the generated URL. Proceed with the remaining actions. - del actions[ 0 ] - action_type, action_dict = actions[ 0 ] + filtered_actions = actions[ 1: ] + action_type, action_dict = filtered_actions[ 0 ] + # If the downloaded file exists, move it to $INSTALL_DIR. Put this outside the try/catch above so that + # any errors in the move step are correctly sent to the tool dependency error handler. + if downloaded_filename and os.path.exists( os.path.join( work_dir, downloaded_filename ) ): + if target_directory: + target_directory = os.path.realpath( os.path.normpath( os.path.join( install_dir, target_directory ) ) ) + # Make sure the target directory is not outside of $INSTALL_DIR. + if target_directory.startswith( os.path.realpath( install_dir ) ): + full_path_to_dir = os.path.abspath( os.path.join( install_dir, target_directory ) ) + else: + full_path_to_dir = os.path.abspath( install_dir ) + else: + full_path_to_dir = os.path.abspath( install_dir ) + common_util.move_file( current_dir=work_dir, + source=downloaded_filename, + destination_dir=full_path_to_dir ) if action_type == 'download_by_url': # Eliminate the download_by_url action so remaining actions can be processed correctly. filtered_actions = actions[ 1: ] @@ -237,9 +262,6 @@ current_dir = os.path.abspath( os.path.join( work_dir, dir ) ) with lcd( current_dir ): action_type, action_dict = action_tup - # If a binary was found, we only need to process environment variables, file permissions, and any other binary downloads. - if binary_found and action_type not in [ 'set_environment', 'chmod', 'download_binary' ]: - continue if action_type == 'make_directory': common_util.make_directory( full_path=action_dict[ 'full_path' ] ) elif action_type == 'move_directory_files': @@ -374,12 +396,26 @@ os.chmod( target_file, mode ) elif action_type == 'download_binary': url = action_dict[ 'url' ] - binary_found = common_util.download_binary_from_url( url, work_dir, install_dir ) - if binary_found: - log.debug( 'Successfully downloaded binary from %s', url ) - else: - log.error( 'Unable to download binary from %s', url ) - + target_directory = action_dict.get( 'target_directory', None ) + try: + downloaded_filename = common_util.download_binary( url, work_dir ) + except Exception, e: + log.exception( str( e ) ) + # If the downloaded file exists, move it to $INSTALL_DIR. Put this outside the try/catch above so that + # any errors in the move step are correctly sent to the tool dependency error handler. + if downloaded_filename and os.path.exists( os.path.join( work_dir, downloaded_filename ) ): + if target_directory: + target_directory = os.path.realpath( os.path.normpath( os.path.join( install_dir, target_directory ) ) ) + # Make sure the target directory is not outside of $INSTALL_DIR. + if target_directory.startswith( os.path.realpath( install_dir ) ): + full_path_to_dir = os.path.abspath( os.path.join( install_dir, target_directory ) ) + else: + full_path_to_dir = os.path.abspath( install_dir ) + else: + full_path_to_dir = os.path.abspath( install_dir ) + common_util.move_file( current_dir=work_dir, + source=downloaded_filename, + destination_dir=full_path_to_dir ) def log_results( command, fabric_AttributeString, file_path ): """ diff -r 773941fd26a43c8522c3ff9977cfda5968f65505 -r f33c054d6d5b75ae545248d71ec559d74b4fa636 lib/tool_shed/galaxy_install/tool_dependencies/install_util.py --- a/lib/tool_shed/galaxy_install/tool_dependencies/install_util.py +++ b/lib/tool_shed/galaxy_install/tool_dependencies/install_util.py @@ -395,6 +395,7 @@ else: url_template_elem = url_template_elems[ 0 ] action_dict[ 'url' ] = Template( url_template_elem.text ).safe_substitute( platform_info_dict ) + action_dict[ 'target_directory' ] = action_elem.get( 'target_directory', None ) elif action_type == 'shell_command': # <action type="shell_command">make</action> action_elem_text = evaluate_template( action_elem.text ) Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: 2 new changesets
by commits-noreply＠bitbucket.org 06 Aug '13

06 Aug '13

2 new commits in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/3615504d0a43/ Changeset: 3615504d0a43 Branch: next-stable User: jgoecks Date: 2013-08-07 00:06:45 Summary: Update citation for Tophat2. Affected #: 1 file diff -r 9e8c70746ddb12df2c355e09db023dcbea0397b3 -r 3615504d0a43b511c44a4aa6a8fa520260373694 tools/ngs_rna/tophat2_wrapper.xml --- a/tools/ngs_rna/tophat2_wrapper.xml +++ b/tools/ngs_rna/tophat2_wrapper.xml @@ -450,7 +450,8 @@ <help> **Tophat Overview** -TopHat_ is a fast splice junction mapper for RNA-Seq reads. It aligns RNA-Seq reads to mammalian-sized genomes using the ultra high-throughput short read aligner Bowtie, and then analyzes the mapping results to identify splice junctions between exons. Please cite: Trapnell, C., Pachter, L. and Salzberg, S.L. TopHat: discovering splice junctions with RNA-Seq. Bioinformatics 25, 1105-1111 (2009). +TopHat_ is a fast splice junction mapper for RNA-Seq reads. It aligns RNA-Seq reads to mammalian-sized genomes using the ultra high-throughput short read aligner Bowtie(2), and then analyzes the mapping results to identify splice junctions between exons. Please cite: Kim D, Pertea G, Trapnell C, Pimentel H, Kelley R, and Salzberg SL. TopHat2: accurate alignment +of transcriptomes in the presence of insertions, deletions and gene fusions. Genome Biol 14:R36, 2013. .. _Tophat: http://tophat.cbcb.umd.edu/ https://bitbucket.org/galaxy/galaxy-central/commits/773941fd26a4/ Changeset: 773941fd26a4 User: jgoecks Date: 2013-08-07 00:07:15 Summary: Automated merge of next-stable branch Affected #: 1 file diff -r 48220b09b62c17422bf81e1b623b90db41b12c72 -r 773941fd26a43c8522c3ff9977cfda5968f65505 tools/ngs_rna/tophat2_wrapper.xml --- a/tools/ngs_rna/tophat2_wrapper.xml +++ b/tools/ngs_rna/tophat2_wrapper.xml @@ -452,7 +452,8 @@ <help> **Tophat Overview** -TopHat_ is a fast splice junction mapper for RNA-Seq reads. It aligns RNA-Seq reads to mammalian-sized genomes using the ultra high-throughput short read aligner Bowtie, and then analyzes the mapping results to identify splice junctions between exons. Please cite: Trapnell, C., Pachter, L. and Salzberg, S.L. TopHat: discovering splice junctions with RNA-Seq. Bioinformatics 25, 1105-1111 (2009). +TopHat_ is a fast splice junction mapper for RNA-Seq reads. It aligns RNA-Seq reads to mammalian-sized genomes using the ultra high-throughput short read aligner Bowtie(2), and then analyzes the mapping results to identify splice junctions between exons. Please cite: Kim D, Pertea G, Trapnell C, Pimentel H, Kelley R, and Salzberg SL. TopHat2: accurate alignment +of transcriptomes in the presence of insertions, deletions and gene fusions. Genome Biol 14:R36, 2013. .. _Tophat: http://tophat.cbcb.umd.edu/ Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: carlfeberhard: Visualizations framework: fix to_param optionality bug, add has_dataprovider data source test
by commits-noreply＠bitbucket.org 06 Aug '13

06 Aug '13

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/48220b09b62c/ Changeset: 48220b09b62c User: carlfeberhard Date: 2013-08-07 00:00:00 Summary: Visualizations framework: fix to_param optionality bug, add has_dataprovider data source test Affected #: 1 file diff -r 439fecf2f288e776501807c229f54332e3b0fbdf -r 48220b09b62c17422bf81e1b623b90db41b12c72 lib/galaxy/visualization/registry.py --- a/lib/galaxy/visualization/registry.py +++ b/lib/galaxy/visualization/registry.py @@ -449,6 +449,7 @@ returned[ 'tests' ] = tests # to_params (optional, 0 or more) - tells the registry to set certain params based on the model_clas, tests + returned[ 'to_params' ] = {} to_params = self.parse_to_params( xml_tree.findall( 'to_param' ) ) if to_params: returned[ 'to_params' ] = to_params @@ -514,6 +515,7 @@ # test_attr can be a dot separated chain of object attributes (e.g. dataset.datatype) - convert to list #TODO: too dangerous - constrain these to some allowed list + #TODO: does this err if no test_attr - it should... test_attr = test_elem.get( 'test_attr' ) test_attr = test_attr.split( self.ATTRIBUTE_SPLIT_CHAR ) if isinstance( test_attr, str ) else [] # build a lambda function that gets the desired attribute to test @@ -523,14 +525,16 @@ test_result_type = test_elem.get( 'result_type' ) or 'string' # test functions should be sent an object to test, and the parsed result expected from the test - #TODO: currently, isinstance and string equivalance are the only test types supported + # is test_attr attribute an instance of result if test_type == 'isinstance': #TODO: wish we could take this further but it would mean passing in the datatypes_registry test_fn = lambda o, result: isinstance( getter( o ), result ) + #TODO: needs cleanup - robustiosity-nessness + # does the object itself have a datatype attr and does that datatype have the given dataprovider elif test_type == 'has_dataprovider': test_fn = lambda o, result: ( hasattr( o, 'datatype' ) - and o.datatype.has_dataprovier( result ) ) + and o.datatype.has_dataprovider( result ) ) # default to simple (string) equilavance (coercing the test_attr to a string) else: Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: carlfeberhard: Visualization framework: add convenience mako base template
by commits-noreply＠bitbucket.org 06 Aug '13

06 Aug '13

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/439fecf2f288/ Changeset: 439fecf2f288 User: carlfeberhard Date: 2013-08-06 23:30:05 Summary: Visualization framework: add convenience mako base template Affected #: 2 files diff -r e1b3dbeadb5051227f3ea216ea560f9b10bb29ed -r 439fecf2f288e776501807c229f54332e3b0fbdf config/plugins/visualizations/visualization_base.mako --- /dev/null +++ b/config/plugins/visualizations/visualization_base.mako @@ -0,0 +1,67 @@ +# -*- coding: utf-8 -*- +<% _=n_ %> + +%if embedded: + ${self.as_embedded()} +%else: + ${self.as_page()} +%endif + +## render this inside another page or via ajax +<%def name="as_embedded()"> + ${self.stylesheets()} + ${self.javascripts()} + ${self.get_body()} +</%def> + +## render this as it's own page +<%def name="as_page()"> +<!DOCTYPE HTML> +<html> + <head> + <title>${self.title()}</title> + <meta http-equiv="Content-Type" content="text/html; charset=utf-8" /> + ${self.metas()} + ${self.stylesheets()} + ${self.javascripts()} + </head> + <body> + ${self.get_body()} + </body> +</html> +</%def> +##TODO: late_javascripts + +## Default body +<%def name="get_body()"></%def> + +## Default title +<%def name="title()">${visualization_name}</%def> + +## Additional metas can be defined by templates inheriting from this one. +<%def name="metas()"></%def> + +## Default stylesheets +<%def name="stylesheets()"> +${h.css('base')} +</%def> + +## Default javascripts +<%def name="javascripts()"> +${h.js( + "libs/jquery/jquery", + "libs/jquery/jquery.migrate" +)} + +<script type="text/javascript"> + // console protection + window.console = window.console || { + log : function(){}, + debug : function(){}, + info : function(){}, + warn : function(){}, + error : function(){}, + assert : function(){} + }; +</script> +</%def> diff -r e1b3dbeadb5051227f3ea216ea560f9b10bb29ed -r 439fecf2f288e776501807c229f54332e3b0fbdf lib/galaxy/visualization/registry.py --- a/lib/galaxy/visualization/registry.py +++ b/lib/galaxy/visualization/registry.py @@ -524,10 +524,14 @@ # test functions should be sent an object to test, and the parsed result expected from the test #TODO: currently, isinstance and string equivalance are the only test types supported - if test_type == 'isinstance': + if test_type == 'isinstance': #TODO: wish we could take this further but it would mean passing in the datatypes_registry test_fn = lambda o, result: isinstance( getter( o ), result ) + elif test_type == 'has_dataprovider': + test_fn = lambda o, result: ( hasattr( o, 'datatype' ) + and o.datatype.has_dataprovier( result ) ) + # default to simple (string) equilavance (coercing the test_attr to a string) else: test_fn = lambda o, result: str( getter( o ) ) == result Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: carlfeberhard: Visualization framework: update universe_wsgi vis plugin entry with absolute path explanation
by commits-noreply＠bitbucket.org 06 Aug '13

06 Aug '13

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/e1b3dbeadb50/ Changeset: e1b3dbeadb50 User: carlfeberhard Date: 2013-08-06 23:15:17 Summary: Visualization framework: update universe_wsgi vis plugin entry with absolute path explanation Affected #: 1 file diff -r 26467a120cb1d3b8868138d5233602c831db6e79 -r e1b3dbeadb5051227f3ea216ea560f9b10bb29ed universe_wsgi.ini.sample --- a/universe_wsgi.ini.sample +++ b/universe_wsgi.ini.sample @@ -175,6 +175,8 @@ #datatypes_config_file = datatypes_conf.xml # Visualizations config directory: where to look for individual visualization plugins. +# The path is relative to the Galaxy root dir. To use an absolute path begin the path +# with '/'. #visualizations_plugins_directory = config/plugins/visualizations # Each job is given a unique empty directory as its current working directory. Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: 2 new changesets
by commits-noreply＠bitbucket.org 06 Aug '13

06 Aug '13

2 new commits in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/9e8c70746ddb/ Changeset: 9e8c70746ddb Branch: next-stable User: jgoecks Date: 2013-08-06 22:34:20 Summary: Add Bowtie2 wrapper to main's tool conf. Affected #: 1 file diff -r e3f6a95109789272a3c4f0663dbcdab908aa8f26 -r 9e8c70746ddb12df2c355e09db023dcbea0397b3 tool_conf.xml.main --- a/tool_conf.xml.main +++ b/tool_conf.xml.main @@ -260,6 +260,7 @@ </section><section name="NGS: Mapping" id="ngs_mapping"><label text="Illumina" id="illumina"/> + <tool file="sr_mapping/bowtie2_wrapper.xml" /><label text="Roche-454" id="roche_454"/><tool file="metag_tools/megablast_wrapper.xml" /><tool file="metag_tools/megablast_xml_parser.xml" /> https://bitbucket.org/galaxy/galaxy-central/commits/26467a120cb1/ Changeset: 26467a120cb1 User: jgoecks Date: 2013-08-06 22:34:50 Summary: Automated merge of next-stable branch Affected #: 1 file diff -r 12075315e61b01f4b163a1074781eabadc47da00 -r 26467a120cb1d3b8868138d5233602c831db6e79 tool_conf.xml.main --- a/tool_conf.xml.main +++ b/tool_conf.xml.main @@ -260,6 +260,7 @@ </section><section name="NGS: Mapping" id="ngs_mapping"><label text="Illumina" id="illumina"/> + <tool file="sr_mapping/bowtie2_wrapper.xml" /><label text="Roche-454" id="roche_454"/><tool file="metag_tools/megablast_wrapper.xml" /><tool file="metag_tools/megablast_xml_parser.xml" /> Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: dannon: Fix get_hda_list 'get_accessible' variable naming -- previously would blow up as undefined.
by commits-noreply＠bitbucket.org 06 Aug '13

06 Aug '13

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/12075315e61b/ Changeset: 12075315e61b User: dannon Date: 2013-08-06 22:05:09 Summary: Fix get_hda_list 'get_accessible' variable naming -- previously would blow up as undefined. Affected #: 1 file diff -r e08148b86a59421f6faa616529ae262a40138485 -r 12075315e61b01f4b163a1074781eabadc47da00 lib/galaxy/web/base/controller.py --- a/lib/galaxy/web/base/controller.py +++ b/lib/galaxy/web/base/controller.py @@ -537,7 +537,9 @@ hda = None try: hda = self.get_dataset( trans, id, - check_ownership=check_ownership, check_accesible=check_accesible, check_state=check_state ) + check_ownership=check_ownership, + check_accessible=check_accessible, + check_state=check_state ) except Exception, exception: pass hdas.append( hda ) Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: dannon: Cleanup imports in extended_metadata API controller.
by commits-noreply＠bitbucket.org 06 Aug '13

06 Aug '13

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/e08148b86a59/ Changeset: e08148b86a59 User: dannon Date: 2013-08-06 22:02:41 Summary: Cleanup imports in extended_metadata API controller. Affected #: 1 file diff -r 42ef1d78a2f8e2466a0698ed4b5343689da4cd6f -r e08148b86a59421f6faa616529ae262a40138485 lib/galaxy/webapps/galaxy/api/extended_metadata.py --- a/lib/galaxy/webapps/galaxy/api/extended_metadata.py +++ b/lib/galaxy/webapps/galaxy/api/extended_metadata.py @@ -1,17 +1,9 @@ """ API operations on annotations. """ -import logging, os, string, shutil, urllib, re, socket -from cgi import escape, FieldStorage -from galaxy import util, datatypes, jobs, web, util -from galaxy.web.base.controller import BaseAPIController, UsesHistoryMixin, UsesLibraryMixinItems, UsesHistoryDatasetAssociationMixin, UsesStoredWorkflowMixin, UsesExtendedMetadataMixin -from galaxy.util.sanitize_html import sanitize_html -import galaxy.datatypes -from galaxy.util.bunch import Bunch - -import pkg_resources -pkg_resources.require( "Routes" ) -import routes +import logging +from galaxy import web +from galaxy.web.base.controller import BaseAPIController, UsesHistoryMixin, UsesLibraryMixinItems, UsesHistoryDatasetAssociationMixin, UsesStoredWorkflowMixin, UsesExtendedMetadataMixin, HTTPNotImplemented log = logging.getLogger( __name__ ) Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: 2 new changesets
by commits-noreply＠bitbucket.org 06 Aug '13

06 Aug '13

2 new commits in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/42ef1d78a2f8/ Changeset: 42ef1d78a2f8 User: dannon Date: 2013-08-06 21:57:27 Summary: Remove relpath added in pr#149 -- not needed w/ python 2.6+ Affected #: 1 file diff -r 65c255d9c0a56099261a23c7ef1eb5185481ece0 -r 42ef1d78a2f8e2466a0698ed4b5343689da4cd6f lib/galaxy/util/__init__.py --- a/lib/galaxy/util/__init__.py +++ b/lib/galaxy/util/__init__.py @@ -665,41 +665,6 @@ print "ERROR: Unable to read builds for site file %s" %filename return build_sites -def relpath( path, start = None ): - """Return a relative version of a path""" - #modified from python 2.6.1 source code - - #version 2.6+ has it built in, we'll use the 'official' copy - if sys.version_info[:2] >= ( 2, 6 ): - if start is not None: - return os.path.relpath( path, start ) - return os.path.relpath( path ) - - #we need to initialize some local parameters - curdir = os.curdir - pardir = os.pardir - sep = os.sep - commonprefix = os.path.commonprefix - join = os.path.join - if start is None: - start = curdir - - #below is the unedited (but formated) relpath() from posixpath.py of 2.6.1 - #this will likely not function properly on non-posix systems, i.e. windows - if not path: - raise ValueError( "no path specified" ) - - start_list = os.path.abspath( start ).split( sep ) - path_list = os.path.abspath( path ).split( sep ) - - # Work out how much of the filepath is shared by start and path. - i = len( commonprefix( [ start_list, path_list ] ) ) - - rel_list = [ pardir ] * ( len( start_list )- i ) + path_list[ i: ] - if not rel_list: - return curdir - return join( *rel_list ) - def relativize_symlinks( path, start=None, followlinks=False): for root, dirs, files in os.walk( path, followlinks=followlinks ): rel_start = None https://bitbucket.org/galaxy/galaxy-central/commits/a849924d6f9d/ Changeset: a849924d6f9d Branch: extended_metadata User: dannon Date: 2013-08-06 21:58:00 Summary: Close extended_metadata branch Affected #: 0 files Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: 5 new changesets
by commits-noreply＠bitbucket.org 06 Aug '13

06 Aug '13

5 new commits in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/970d44713796/ Changeset: 970d44713796 User: kellrott Date: 2012-12-28 01:38:46 Summary: Adding API for accessing extended metadata Affected #: 3 files diff -r 7073d786cad0f0d251d4ea9b2c42171f698cf953 -r 970d44713796453e3356c6273213bccf3eec7f38 lib/galaxy/model/item_attrs.py --- a/lib/galaxy/model/item_attrs.py +++ b/lib/galaxy/model/item_attrs.py @@ -152,6 +152,83 @@ class_name = '%sAnnotationAssociation' % item.__class__.__name__ return getattr( galaxy.model, class_name, None ) + +class UsesExtendedMetadata: + """ Mixin for getting and setting item extended metadata. """ + + def get_item_extended_metadata_obj( self, db_session, user, item ): + """ + Given an item object (such as a LibraryDatasetDatasetAssociation), find the object + of the associated extended metadata + """ + + extended_metadata = db_session.query( galaxy.model.ExtendedMetadata ) + + if item.__class__ == galaxy.model.LibraryDatasetDatasetAssociation: + extended_metadata = extended_metadata.join( galaxy.model.LibraryDatasetDatasetAssociation ).filter( + galaxy.model.LibraryDatasetDatasetAssociation.id == item.id ) + + return extended_metadata.first() + + + def set_item_extended_metadata_obj( self, db_session, user, item, extmeta_obj): + if item.__class__ == galaxy.model.LibraryDatasetDatasetAssociation: + item.extended_metadata = extmeta_obj + db_session.flush() + + def unlink_item_extended_metadata_obj( self, db_session, user, item): + if item.__class__ == galaxy.model.LibraryDatasetDatasetAssociation: + item.extended_metadata = None + db_session.flush() + + def create_extended_metadata(self, db_session, user, extmeta): + """ + Create/index an extended metadata object. The returned object is + not associated with any items + """ + ex_meta = galaxy.model.ExtendedMetadata(extmeta) + db_session.add( ex_meta ) + db_session.flush() + for path, value in self._scan_json_block(extmeta): + meta_i = galaxy.model.ExtendedMetadataIndex(ex_meta, path, value) + db_session.add(meta_i) + db_session.flush() + return ex_meta + + def delete_extended_metadata( self, db_session, user, item): + if item.__class__ == galaxy.model.ExtendedMetadata: + db_session.delete( item ) + db_session.flush() + + def _scan_json_block(self, meta, prefix=""): + """ + Scan a json style data structure, and emit all fields and their values. + Example paths + + Data + { "data" : [ 1, 2, 3 ] } + + Path: + /data == [1,2,3] + + /data/[0] == 1 + + """ + if isinstance(meta, dict): + for a in meta: + for path, value in self._scan_json_block(meta[a], prefix + "/" + a): + yield path, value + elif isinstance(meta, list): + for i, a in enumerate(meta): + for path, value in self._scan_json_block(a, prefix + "[%d]" % (i)): + yield path, value + else: + #BUG: Everything is cast to string, which can lead to false positives + #for cross type comparisions, ie "True" == True + yield prefix, ("%s" % (meta)).encode("utf8", errors='replace') + + + class APIItem: """ Mixin for api representation. """ #api_collection_visible_keys = ( 'id' ) diff -r 7073d786cad0f0d251d4ea9b2c42171f698cf953 -r 970d44713796453e3356c6273213bccf3eec7f38 lib/galaxy/webapps/galaxy/api/extended_metadata.py --- /dev/null +++ b/lib/galaxy/webapps/galaxy/api/extended_metadata.py @@ -0,0 +1,63 @@ +""" +API operations on annotations. +""" +import logging, os, string, shutil, urllib, re, socket +from cgi import escape, FieldStorage +from galaxy import util, datatypes, jobs, web, util +from galaxy.web.base.controller import * +from galaxy.model.item_attrs import * +from galaxy.util.sanitize_html import sanitize_html +from galaxy.model.orm import * +import galaxy.datatypes +from galaxy.util.bunch import Bunch + +import pkg_resources +pkg_resources.require( "Routes" ) +import routes + +log = logging.getLogger( __name__ ) + +class BaseExtendedMetadataController( BaseAPIController, UsesExtendedMetadata, UsesHistoryMixin, UsesLibraryMixinItems, UsesHistoryDatasetAssociationMixin, UsesStoredWorkflowMixin ): + + @web.expose_api + def index( self, trans, **kwd ): + idnum = kwd[self.exmeta_item_id] + item = self._get_item_from_id(trans, idnum) + if item is not None: + ex_meta = self.get_item_extended_metadata_obj( trans.sa_session, trans.get_user(), item ) + if ex_meta is not None: + return ex_meta.data + + @web.expose_api + def create( self, trans, payload, **kwd ): + idnum = kwd[self.exmeta_item_id] + item = self._get_item_from_id(trans, idnum) + if item is not None: + ex_obj = self.get_item_extended_metadata_obj(trans.sa_session, trans.get_user(), item) + if ex_obj is not None: + self.unlink_item_extended_metadata_obj(trans.sa_session, trans.get_user(), item) + self.delete_extended_metadata(trans.sa_session, trans.get_user(), ex_obj) + ex_obj = self.create_extended_metadata(trans.sa_session, trans.get_user(), payload) + self.set_item_extended_metadata_obj(trans.sa_session, trans.get_user(), item, ex_obj) + + @web.expose_api + def delete( self, trans, **kwd ): + idnum = kwd[self.tagged_item_id] + item = self._get_item_from_id(trans, idnum) + if item is not None: + ex_obj = self.get_item_extended_metadata_obj(trans.sa_session, trans.get_user(), item) + if ex_obj is not None: + self.unlink_item_extended_metadata_obj(trans.sa_session, trans.get_user(), item) + self.delete_extended_metadata(trans.sa_session, trans.get_user(), ex_obj) + + @web.expose_api + def undelete( self, trans, **kwd ): + raise HTTPNotImplemented() + +class LibraryDatasetExtendMetadataController(BaseExtendedMetadataController): + controller_name = "library_dataset_extended_metadata" + exmeta_item_id = "library_content_id" + def _get_item_from_id(self, trans, idstr): + hist = self.get_library_dataset_dataset_association( trans, idstr ) + return hist + diff -r 7073d786cad0f0d251d4ea9b2c42171f698cf953 -r 970d44713796453e3356c6273213bccf3eec7f38 lib/galaxy/webapps/galaxy/buildapp.py --- a/lib/galaxy/webapps/galaxy/buildapp.py +++ b/lib/galaxy/webapps/galaxy/buildapp.py @@ -107,6 +107,11 @@ _add_item_tags_controller( webapp, name_prefix="workflow_", path_prefix='/api/workflows/:workflow_id' ) + + _add_item_extended_metadata_controller( webapp, + name_prefix="library_dataset_", + path_prefix='/api/libraries/:library_id/contents/:library_content_id' ) + webapp.api_mapper.resource( 'dataset', 'datasets', path_prefix='/api' ) webapp.api_mapper.resource_with_deleted( 'library', 'libraries', path_prefix='/api' ) webapp.api_mapper.resource( 'sample', 'samples', path_prefix='/api' ) @@ -187,6 +192,12 @@ conditions=dict(method=["GET"])) +def _add_item_extended_metadata_controller( webapp, name_prefix, path_prefix, **kwd ): + controller = "%sextended_metadata" % name_prefix + name = "%sextended_metadata" % name_prefix + webapp.api_mapper.resource(name, "extended_metadata", path_prefix=path_prefix, controller=controller) + + def wrap_in_middleware( app, global_conf, **local_conf ): """ Based on the configuration wrap `app` in a set of common and useful https://bitbucket.org/galaxy/galaxy-central/commits/7b916e959c09/ Changeset: 7b916e959c09 User: kellrott Date: 2013-01-18 17:46:16 Summary: Removing the '*' imports from the extended_metadata API module Affected #: 1 file diff -r 970d44713796453e3356c6273213bccf3eec7f38 -r 7b916e959c09083e541d1daaf573b1af995ed152 lib/galaxy/webapps/galaxy/api/extended_metadata.py --- a/lib/galaxy/webapps/galaxy/api/extended_metadata.py +++ b/lib/galaxy/webapps/galaxy/api/extended_metadata.py @@ -4,10 +4,9 @@ import logging, os, string, shutil, urllib, re, socket from cgi import escape, FieldStorage from galaxy import util, datatypes, jobs, web, util -from galaxy.web.base.controller import * -from galaxy.model.item_attrs import * +from galaxy.web.base.controller import BaseAPIController, UsesHistoryMixin, UsesLibraryMixinItems, UsesHistoryDatasetAssociationMixin, UsesStoredWorkflowMixin +from galaxy.model.item_attrs import UsesExtendedMetadata from galaxy.util.sanitize_html import sanitize_html -from galaxy.model.orm import * import galaxy.datatypes from galaxy.util.bunch import Bunch https://bitbucket.org/galaxy/galaxy-central/commits/d603df6f2e79/ Changeset: d603df6f2e79 User: kellrott Date: 2013-01-23 20:29:30 Summary: Central Merge Affected #: 248 files diff -r 7b916e959c09083e541d1daaf573b1af995ed152 -r d603df6f2e7978f9c75f64190bfba97bbf59e851 .hgignore --- a/.hgignore +++ b/.hgignore @@ -16,6 +16,7 @@ database/community_files database/compiled_templates database/files +database/job_working_directory database/pbs database/tmp database/*.sqlite @@ -23,6 +24,11 @@ # Python bytecode *.pyc +# Tool Shed Runtime Files +community_webapp.log +community_webapp.pid +hgweb.config* + # Config files universe_wsgi.ini reports_wsgi.ini diff -r 7b916e959c09083e541d1daaf573b1af995ed152 -r d603df6f2e7978f9c75f64190bfba97bbf59e851 LICENSE.txt --- a/LICENSE.txt +++ b/LICENSE.txt @@ -1,25 +1,186 @@ -Copyright (c) 2005 Pennsylvania State University +Copyright (c) 2005-2013 Pennsylvania State University -Permission is hereby granted, free of charge, to any person obtaining -a copy of this software and associated documentation files (the -"Software"), to deal in the Software without restriction, including -without limitation the rights to use, copy, modify, merge, publish, -distribute, sublicense, and/or sell copies of the Software, and to -permit persons to whom the Software is furnished to do so, subject to -the following conditions: +Licensed under the Academic Free License version 3.0 -The above copyright notice and this permission notice shall be -included in all copies or substantial portions of the Software. + 1) Grant of Copyright License. Licensor grants You a worldwide, royalty-free, + non-exclusive, sublicensable license, for the duration of the copyright, to + do the following: -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, -EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF -MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. -IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY -CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, -TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE -SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + a) to reproduce the Original Work in copies, either alone or as part of a + collective work; + + b) to translate, adapt, alter, transform, modify, or arrange the Original + Work, thereby creating derivative works ("Derivative Works") based upon + the Original Work; + + c) to distribute or communicate copies of the Original Work and Derivative + Works to the public, under any license of your choice that does not + contradict the terms and conditions, including Licensor's reserved + rights and remedies, in this Academic Free License; + + d) to perform the Original Work publicly; and + + e) to display the Original Work publicly. + + 2) Grant of Patent License. Licensor grants You a worldwide, royalty-free, + non-exclusive, sublicensable license, under patent claims owned or + controlled by the Licensor that are embodied in the Original Work as + furnished by the Licensor, for the duration of the patents, to make, use, + sell, offer for sale, have made, and import the Original Work and + Derivative Works. + + 3) Grant of Source Code License. The term "Source Code" means the preferred + form of the Original Work for making modifications to it and all available + documentation describing how to modify the Original Work. Licensor agrees + to provide a machine-readable copy of the Source Code of the Original Work + along with each copy of the Original Work that Licensor distributes. + Licensor reserves the right to satisfy this obligation by placing a + machine-readable copy of the Source Code in an information repository + reasonably calculated to permit inexpensive and convenient access by You + for as long as Licensor continues to distribute the Original Work. + + 4) Exclusions From License Grant. Neither the names of Licensor, nor the + names of any contributors to the Original Work, nor any of their + trademarks or service marks, may be used to endorse or promote products + derived from this Original Work without express prior permission of the + Licensor. Except as expressly stated herein, nothing in this License + grants any license to Licensor's trademarks, copyrights, patents, trade + secrets or any other intellectual property. No patent license is granted + to make, use, sell, offer for sale, have made, or import embodiments of + any patent claims other than the licensed claims defined in Section 2. + No license is granted to the trademarks of Licensor even if such marks + are included in the Original Work. Nothing in this License shall be + interpreted to prohibit Licensor from licensing under terms different + from this License any Original Work that Licensor otherwise would have a + right to license. + + 5) External Deployment. The term "External Deployment" means the use, + distribution, or communication of the Original Work or Derivative Works + in any way such that the Original Work or Derivative Works may be used by + anyone other than You, whether those works are distributed or + communicated to those persons or made available as an application + intended for use over a network. As an express condition for the grants + of license hereunder, You must treat any External Deployment by You of + the Original Work or a Derivative Work as a distribution under + section 1(c). + + 6) Attribution Rights. You must retain, in the Source Code of any Derivative + Works that You create, all copyright, patent, or trademark notices from + the Source Code of the Original Work, as well as any notices of licensing + and any descriptive text identified therein as an "Attribution Notice." + You must cause the Source Code for any Derivative Works that You create + to carry a prominent Attribution Notice reasonably calculated to inform + recipients that You have modified the Original Work. + + 7) Warranty of Provenance and Disclaimer of Warranty. Licensor warrants that + the copyright in and to the Original Work and the patent rights granted + herein by Licensor are owned by the Licensor or are sublicensed to You + under the terms of this License with the permission of the contributor(s) + of those copyrights and patent rights. Except as expressly stated in the + immediately preceding sentence, the Original Work is provided under this + License on an "AS IS" BASIS and WITHOUT WARRANTY, either express or + implied, including, without limitation, the warranties of + non-infringement, merchantability or fitness for a particular purpose. + THE ENTIRE RISK AS TO THE QUALITY OF THE ORIGINAL WORK IS WITH YOU. This + DISCLAIMER OF WARRANTY constitutes an essential part of this License. + No license to the Original Work is granted by this License except under + this disclaimer. + + 8) Limitation of Liability. Under no circumstances and under no legal + theory, whether in tort (including negligence), contract, or otherwise, + shall the Licensor be liable to anyone for any indirect, special, + incidental, or consequential damages of any character arising as a result + of this License or the use of the Original Work including, without + limitation, damages for loss of goodwill, work stoppage, computer failure + or malfunction, or any and all other commercial damages or losses. This + limitation of liability shall not apply to the extent applicable law + prohibits such limitation. + + 9) Acceptance and Termination. If, at any time, You expressly assented to + this License, that assent indicates your clear and irrevocable acceptance + of this License and all of its terms and conditions. If You distribute or + communicate copies of the Original Work or a Derivative Work, You must + make a reasonable effort under the circumstances to obtain the express + assent of recipients to the terms of this License. This License + conditions your rights to undertake the activities listed in Section 1, + including your right to create Derivative Works based upon the Original + Work, and doing so without honoring these terms and conditions is + prohibited by copyright law and international treaty. Nothing in this + License is intended to affect copyright exceptions and limitations + (including "fair use" or "fair dealing"). This License shall terminate + immediately and You may no longer exercise any of the rights granted to + You by this License upon your failure to honor the conditions in + Section 1(c). + +10) Termination for Patent Action. This License shall terminate + automatically and You may no longer exercise any of the rights granted + to You by this License as of the date You commence an action, including + a cross-claim or counterclaim, against Licensor or any licensee alleging + that the Original Work infringes a patent. This termination provision + shall not apply for an action alleging patent infringement by + combinations of the Original Work with other software or hardware. + +11) Jurisdiction, Venue and Governing Law. Any action or suit relating to + this License may be brought only in the courts of a jurisdiction wherein + the Licensor resides or in which Licensor conducts its primary business, + and under the laws of that jurisdiction excluding its conflict-of-law + provisions. The application of the United Nations Convention on + Contracts for the International Sale of Goods is expressly excluded. Any + use of the Original Work outside the scope of this License or after its + termination shall be subject to the requirements and penalties of + copyright or patent law in the appropriate jurisdiction. This section + shall survive the termination of this License. + +12) Attorneys' Fees. In any action to enforce the terms of this License or + seeking damages relating thereto, the prevailing party shall be entitled + to recover its costs and expenses, including, without limitation, + reasonable attorneys' fees and costs incurred in connection with such + action, including any appeal of such action. This section shall survive + the termination of this License. + +13) Miscellaneous. If any provision of this License is held to be + unenforceable, such provision shall be reformed only to the extent + necessary to make it enforceable. + +14) Definition of "You" in This License. "You" throughout this License, + whether in upper or lower case, means an individual or a legal entity + exercising rights under, and complying with all of the terms of, this + License. For legal entities, "You" includes any entity that controls, is + controlled by, or is under common control with you. For purposes of this + definition, "control" means (i) the power, direct or indirect, to cause + the direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + +15) Right to Use. You may use the Original Work in all ways not otherwise + restricted or conditioned by this License or by law, and Licensor + promises not to interfere with or be responsible for such uses by You. + +16) Modification of This License. This License is Copyright © 2005 Lawrence + Rosen. Permission is granted to copy, distribute, or communicate this + License without modification. Nothing in this License permits You to + modify this License as applied to the Original Work or to Derivative + Works. However, You may modify the text of this License and copy, + distribute or communicate your modified version (the "Modified + License") and apply it to other original works of authorship subject to + the following conditions: (i) You may not indicate in any way that your + Modified License is the "Academic Free License" or "AFL" and you may not + use those names in the name of your Modified License; (ii) You must + replace the notice specified in the first paragraph above with the + notice "Licensed under <insert your license name here>" or with a notice + of your own that is not confusingly similar to the notice in this + License; and (iii) You may not claim that your original works are open + source software unless your Modified License has been approved by Open + Source Initiative (OSI) and You comply with its license review and + certification process. + Some icons found in Galaxy are from the Silk Icons set, available under the Creative Commons Attribution 2.5 License, from: http://www.famfamfam.com/lab/icons/silk/ + + +Other images and documentation are licensed under the Creative Commons Attribution 3.0 (CC BY 3.0) License. See + +http://creativecommons.org/licenses/by/3.0/ diff -r 7b916e959c09083e541d1daaf573b1af995ed152 -r d603df6f2e7978f9c75f64190bfba97bbf59e851 doc/source/lib/galaxy.jobs.runners.rst --- a/doc/source/lib/galaxy.jobs.runners.rst +++ b/doc/source/lib/galaxy.jobs.runners.rst @@ -72,4 +72,5 @@ galaxy.jobs.runners.cli_job galaxy.jobs.runners.cli_shell + galaxy.jobs.runners.lwr_client diff -r 7b916e959c09083e541d1daaf573b1af995ed152 -r d603df6f2e7978f9c75f64190bfba97bbf59e851 doc/source/lib/galaxy.util.rst --- a/doc/source/lib/galaxy.util.rst +++ b/doc/source/lib/galaxy.util.rst @@ -159,4 +159,5 @@ .. toctree:: galaxy.util.backports + galaxy.util.log diff -r 7b916e959c09083e541d1daaf573b1af995ed152 -r d603df6f2e7978f9c75f64190bfba97bbf59e851 doc/source/lib/galaxy.webapps.community.controllers.rst --- a/doc/source/lib/galaxy.webapps.community.controllers.rst +++ b/doc/source/lib/galaxy.webapps.community.controllers.rst @@ -65,11 +65,3 @@ :undoc-members: :show-inheritance: -:mod:`workflow` Module ----------------------- - -.. automodule:: galaxy.webapps.community.controllers.workflow - :members: - :undoc-members: - :show-inheritance: - diff -r 7b916e959c09083e541d1daaf573b1af995ed152 -r d603df6f2e7978f9c75f64190bfba97bbf59e851 doc/source/lib/galaxy.webapps.community.util.rst --- a/doc/source/lib/galaxy.webapps.community.util.rst +++ b/doc/source/lib/galaxy.webapps.community.util.rst @@ -25,3 +25,11 @@ :undoc-members: :show-inheritance: +:mod:`workflow_util` Module +--------------------------- + +.. automodule:: galaxy.webapps.community.util.workflow_util + :members: + :undoc-members: + :show-inheritance: + diff -r 7b916e959c09083e541d1daaf573b1af995ed152 -r d603df6f2e7978f9c75f64190bfba97bbf59e851 doc/source/lib/galaxy.webapps.galaxy.api.rst --- a/doc/source/lib/galaxy.webapps.galaxy.api.rst +++ b/doc/source/lib/galaxy.webapps.galaxy.api.rst @@ -23,10 +23,6 @@ Quickstart ========== -Set the following option in universe_wsgi.ini and start the server:: - - enable_api = True - Log in as your user, navigate to the API Keys page in the User menu, and generate a new API key. Make a note of the API key, and then pull up a terminal. Now we'll use the display.py script in your galaxy/scripts/api diff -r 7b916e959c09083e541d1daaf573b1af995ed152 -r d603df6f2e7978f9c75f64190bfba97bbf59e851 eggs.ini --- a/eggs.ini +++ b/eggs.ini @@ -29,6 +29,7 @@ simplejson = 2.1.1 threadframe = 0.2 guppy = 0.1.8 +; msgpack_python = 0.2.4 [eggs:noplatform] amqplib = 0.6.1 @@ -65,6 +66,7 @@ Babel = 0.9.4 wchartype = 0.1 Whoosh = 0.3.18 +; fluent_logger = 0.3.3 ; extra version information [tags] diff -r 7b916e959c09083e541d1daaf573b1af995ed152 -r d603df6f2e7978f9c75f64190bfba97bbf59e851 lib/galaxy/app.py --- a/lib/galaxy/app.py +++ b/lib/galaxy/app.py @@ -29,6 +29,7 @@ self.config = config.Configuration( **kwargs ) self.config.check() config.configure_logging( self.config ) + self.configure_fluent_log() # Determine the database url if self.config.database_connection: db_url = self.config.database_connection @@ -54,7 +55,8 @@ db_url, self.config.database_engine_options, database_query_profiling_proxy = self.config.database_query_profiling_proxy, - object_store = self.object_store ) + object_store = self.object_store, + trace_logger=self.trace_logger ) # Manage installed tool shed repositories. self.installed_repository_manager = galaxy.tool_shed.InstalledRepositoryManager( self ) # Create an empty datatypes registry. @@ -149,6 +151,7 @@ self.job_stop_queue = self.job_manager.job_stop_queue # Initialize the external service types self.external_service_types = external_service_types.ExternalServiceTypesCollection( self.config.external_service_type_config_file, self.config.external_service_type_path, self ) + def shutdown( self ): self.job_manager.shutdown() self.object_store.shutdown() @@ -161,3 +164,10 @@ os.unlink( self.datatypes_registry.integrated_datatypes_configs ) except: pass + + def configure_fluent_log( self ): + if self.config.fluent_log: + from galaxy.util.log.fluent_log import FluentTraceLogger + self.trace_logger = FluentTraceLogger( 'galaxy', self.config.fluent_host, self.config.fluent_port ) + else: + self.trace_logger = None diff -r 7b916e959c09083e541d1daaf573b1af995ed152 -r d603df6f2e7978f9c75f64190bfba97bbf59e851 lib/galaxy/config.py --- a/lib/galaxy/config.py +++ b/lib/galaxy/config.py @@ -261,6 +261,10 @@ self.api_folders = string_as_bool( kwargs.get( 'api_folders', False ) ) # This is for testing new library browsing capabilities. self.new_lib_browse = string_as_bool( kwargs.get( 'new_lib_browse', False ) ) + # Logging with fluentd + self.fluent_log = string_as_bool( kwargs.get( 'fluent_log', False ) ) + self.fluent_host = kwargs.get( 'fluent_host', 'localhost' ) + self.fluent_port = int( kwargs.get( 'fluent_port', 24224 ) ) def __read_tool_job_config( self, global_conf_parser, section, key ): try: diff -r 7b916e959c09083e541d1daaf573b1af995ed152 -r d603df6f2e7978f9c75f64190bfba97bbf59e851 lib/galaxy/datatypes/registry.py --- a/lib/galaxy/datatypes/registry.py +++ b/lib/galaxy/datatypes/registry.py @@ -114,7 +114,8 @@ self.datatype_elems.remove( in_memory_elem ) else: # Keep an in-memory list of datatype elems to enable persistence. - self.datatype_elems.append( elem ) + if extension not in self.datatypes_by_extension: + self.datatype_elems.append( elem ) if extension and extension in self.datatypes_by_extension and deactivate: # We are deactivating an installed tool shed repository, so eliminate the datatype from the registry. # TODO: Handle deactivating datatype converters, etc before removing from self.datatypes_by_extension. diff -r 7b916e959c09083e541d1daaf573b1af995ed152 -r d603df6f2e7978f9c75f64190bfba97bbf59e851 lib/galaxy/datatypes/tabular.py --- a/lib/galaxy/datatypes/tabular.py +++ b/lib/galaxy/datatypes/tabular.py @@ -13,7 +13,7 @@ from galaxy.datatypes import metadata from galaxy.datatypes.checkers import is_gzip from galaxy.datatypes.metadata import MetadataElement -from galaxy.datatypes.sniff import get_headers +from galaxy.datatypes.sniff import get_headers, get_test_fname from galaxy.util.json import to_json_string log = logging.getLogger(__name__) @@ -267,20 +267,20 @@ def display_data(self, trans, dataset, preview=False, filename=None, to_ext=None, chunk=None): if chunk: return self.get_chunk(trans, dataset, chunk) + elif to_ext or not preview: + return self._serve_raw(trans, dataset, to_ext) elif dataset.metadata.columns > 50: #Fancy tabular display is only suitable for datasets without an incredibly large number of columns. #We should add a new datatype 'matrix', with it's own draw method, suitable for this kind of data. #For now, default to the old behavior, ugly as it is. Remove this after adding 'matrix'. max_peek_size = 1000000 # 1 MB - if not preview or os.stat( dataset.file_name ).st_size < max_peek_size: + if os.stat( dataset.file_name ).st_size < max_peek_size: return open( dataset.file_name ) else: trans.response.set_content_type( "text/html" ) return trans.stream_template_mako( "/dataset/large_file.mako", truncated_data = open( dataset.file_name ).read(max_peek_size), data = dataset) - elif to_ext or not preview: - return self._serve_raw(trans, dataset, to_ext) else: column_names = 'null' if dataset.metadata.column_names: diff -r 7b916e959c09083e541d1daaf573b1af995ed152 -r d603df6f2e7978f9c75f64190bfba97bbf59e851 lib/galaxy/jobs/runners/__init__.py --- a/lib/galaxy/jobs/runners/__init__.py +++ b/lib/galaxy/jobs/runners/__init__.py @@ -152,6 +152,7 @@ nworkers = self.app.config.cluster_job_queue_workers for i in range( nworkers ): worker = threading.Thread( name="%s.work_thread-%d" % (self.runner_name, i), target=self.run_next ) + worker.setDaemon( True ) worker.start() self.work_threads.append( worker ) diff -r 7b916e959c09083e541d1daaf573b1af995ed152 -r d603df6f2e7978f9c75f64190bfba97bbf59e851 lib/galaxy/jobs/runners/drmaa.py --- a/lib/galaxy/jobs/runners/drmaa.py +++ b/lib/galaxy/jobs/runners/drmaa.py @@ -400,16 +400,18 @@ def stop_job( self, job ): """Attempts to delete a job from the DRM queue""" try: + ext_id = job.get_job_runner_external_id() + assert ext_id not in ( None, 'None' ), 'External job id is None' if self.external_killJob_script is None: - self.ds.control( job.get_job_runner_external_id(), drmaa.JobControlAction.TERMINATE ) + self.ds.control( ext_id, drmaa.JobControlAction.TERMINATE ) else: # FIXME: hardcoded path - subprocess.Popen( [ '/usr/bin/sudo', '-E', self.external_killJob_script, str( job.get_job_runner_external_id() ), str( self.userid ) ], shell=False ) - log.debug( "(%s/%s) Removed from DRM queue at user's request" % ( job.get_id(), job.get_job_runner_external_id() ) ) + subprocess.Popen( [ '/usr/bin/sudo', '-E', self.external_killJob_script, str( ext_id ), str( self.userid ) ], shell=False ) + log.debug( "(%s/%s) Removed from DRM queue at user's request" % ( job.get_id(), ext_id ) ) except drmaa.InvalidJobException: - log.debug( "(%s/%s) User killed running job, but it was already dead" % ( job.get_id(), job.get_job_runner_external_id() ) ) + log.debug( "(%s/%s) User killed running job, but it was already dead" % ( job.get_id(), ext_id ) ) except Exception, e: - log.debug( "(%s/%s) User killed running job, but error encountered removing from DRM queue: %s" % ( job.get_id(), job.get_job_runner_external_id(), e ) ) + log.debug( "(%s/%s) User killed running job, but error encountered removing from DRM queue: %s" % ( job.get_id(), ext_id, e ) ) def recover( self, job, job_wrapper ): """Recovers jobs stuck in the queued/running state when Galaxy started""" diff -r 7b916e959c09083e541d1daaf573b1af995ed152 -r d603df6f2e7978f9c75f64190bfba97bbf59e851 lib/galaxy/jobs/runners/lwr.py --- a/lib/galaxy/jobs/runners/lwr.py +++ b/lib/galaxy/jobs/runners/lwr.py @@ -2,7 +2,7 @@ import subprocess from galaxy import model -from galaxy.jobs.runners import ClusterJobState, ClusterJobRunner +from galaxy.jobs.runners import ClusterJobState, ClusterJobRunner, STOP_SIGNAL import errno from time import sleep @@ -174,8 +174,9 @@ def shutdown( self ): """Attempts to gracefully shut down the worker threads""" log.info( "sending stop signal to worker threads" ) - for i in range( len( self.threads ) ): - self.queue.put( self.STOP_SIGNAL ) + self.monitor_queue.put( STOP_SIGNAL ) + for i in range( len( self.work_threads ) ): + self.work_queue.put( ( STOP_SIGNAL, None ) ) log.info( "local job runner stopped" ) def check_pid( self, pid ): diff -r 7b916e959c09083e541d1daaf573b1af995ed152 -r d603df6f2e7978f9c75f64190bfba97bbf59e851 lib/galaxy/model/__init__.py --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -1335,7 +1335,9 @@ # Loop through sources until viable one is found. for source in source_list: msg = self.convert_dataset( trans, source ) - if msg == self.conversion_messages.PENDING: + # No message or PENDING means that source is viable. No + # message indicates conversion was done and is successful. + if not msg or msg == self.conversion_messages.PENDING: data_source = source break @@ -3150,22 +3152,32 @@ return self.deleted @property def has_repository_dependencies( self ): - return self.metadata and 'repository_dependencies' in self.metadata + if self.metadata: + return 'repository_dependencies' in self.metadata + return False @property def includes_tools( self ): - return self.metadata and 'tools' in self.metadata + if self.metadata: + return 'tools' in self.metadata + return False @property def includes_tool_dependencies( self ): - return self.metadata and 'tool_dependencies' in self.metadata + if self.metadata: + return 'tool_dependencies' in self.metadata + return False @property def includes_workflows( self ): - return self.metadata and 'workflows' in self.metadata + if self.metadata: + return 'workflows' in self.metadata + return False @property def in_error_state( self ): return self.status == self.installation_status.ERROR @property def has_readme_files( self ): - return self.metadata and 'readme_files' in self.metadata + if self.metadata: + return 'readme_files' in self.metadata + return False @property def repository_dependencies( self ): required_repositories = [] diff -r 7b916e959c09083e541d1daaf573b1af995ed152 -r d603df6f2e7978f9c75f64190bfba97bbf59e851 lib/galaxy/model/item_attrs.py --- a/lib/galaxy/model/item_attrs.py +++ b/lib/galaxy/model/item_attrs.py @@ -137,6 +137,12 @@ # Set annotation. annotation_assoc.annotation = annotation return annotation_assoc + + def delete_item_annotation( self, db_session, user, item): + annotation_assoc = self.get_item_annotation_obj( db_session, user, item ) + if annotation_assoc: + db_session.delete(annotation_assoc) + db_session.flush() def copy_item_annotation( self, db_session, source_user, source_item, target_user, target_item ): """ Copy an annotation from a user/item source to a user/item target. """ diff -r 7b916e959c09083e541d1daaf573b1af995ed152 -r d603df6f2e7978f9c75f64190bfba97bbf59e851 lib/galaxy/model/mapping.py --- a/lib/galaxy/model/mapping.py +++ b/lib/galaxy/model/mapping.py @@ -1950,7 +1950,7 @@ # Let this go, it could possibly work with db's we don't support log.error( "database_connection contains an unknown SQLAlchemy database dialect: %s" % dialect ) -def init( file_path, url, engine_options={}, create_tables=False, database_query_profiling_proxy=False, object_store=None ): +def init( file_path, url, engine_options={}, create_tables=False, database_query_profiling_proxy=False, object_store=None, trace_logger=None ): """Connect mappings to the database""" # Connect dataset to the file path Dataset.file_path = file_path @@ -1962,6 +1962,10 @@ if database_query_profiling_proxy: import galaxy.model.orm.logging_connection_proxy as logging_connection_proxy proxy = logging_connection_proxy.LoggingProxy() + # If metlog is enabled, do micrologging + elif trace_logger: + import galaxy.model.orm.logging_connection_proxy as logging_connection_proxy + proxy = logging_connection_proxy.TraceLoggerProxy( trace_logger ) else: proxy = None # Create the database engine diff -r 7b916e959c09083e541d1daaf573b1af995ed152 -r d603df6f2e7978f9c75f64190bfba97bbf59e851 lib/galaxy/model/migrate/versions/0108_add_extended_metadata.py --- a/lib/galaxy/model/migrate/versions/0108_add_extended_metadata.py +++ b/lib/galaxy/model/migrate/versions/0108_add_extended_metadata.py @@ -61,13 +61,20 @@ def downgrade(): metadata.reflect() - ExtendedMetadata_table.drop() - ExtendedMetadataIndex_table.drop() + try: + ExtendedMetadataIndex_table.drop() + except Exception, e: + log.debug( "Dropping 'extended_metadata_index' table failed: %s" % ( str( e ) ) ) + + try: + ExtendedMetadata_table.drop() + except Exception, e: + log.debug( "Dropping 'extended_metadata' table failed: %s" % ( str( e ) ) ) - # Drop the Job table's exit_code column. + # Drop the LDDA table's extended metadata ID column. try: - job_table = Table( "library_dataset_dataset_association", metadata, autoload=True ) - extended_metadata_id = job_table.c.extended_metadata_id + ldda_table = Table( "library_dataset_dataset_association", metadata, autoload=True ) + extended_metadata_id = ldda_table.c.extended_metadata_id extended_metadata_id.drop() except Exception, e: log.debug( "Dropping 'extended_metadata_id' column from library_dataset_dataset_association table failed: %s" % ( str( e ) ) ) diff -r 7b916e959c09083e541d1daaf573b1af995ed152 -r d603df6f2e7978f9c75f64190bfba97bbf59e851 lib/galaxy/model/orm/logging_connection_proxy.py --- a/lib/galaxy/model/orm/logging_connection_proxy.py +++ b/lib/galaxy/model/orm/logging_connection_proxy.py @@ -18,13 +18,31 @@ rval = [] for frame, fname, line, funcname, _, _ in inspect.stack()[2:]: rval.append( "%s:%s@%d" % ( stripwd( fname ), funcname, line ) ) - return " > ".join( rval ) + return rval class LoggingProxy(ConnectionProxy): + """ + Logs SQL statements using standard logging module + """ def cursor_execute(self, execute, cursor, statement, parameters, context, executemany): start = time.clock() rval = execute(cursor, statement, parameters, context) duration = time.clock() - start log.debug( "statement: %r parameters: %r executemany: %r duration: %r stack: %r", - statement, parameters, executemany, duration, pretty_stack() ) + statement, parameters, executemany, duration, " > ".join( pretty_stack() ) ) return rval + +class TraceLoggerProxy(ConnectionProxy): + """ + Logs SQL statements using a metlog client + """ + def __init__( self, trace_logger ): + self.trace_logger = trace_logger + def cursor_execute(self, execute, cursor, statement, parameters, context, executemany): + start = time.clock() + rval = execute(cursor, statement, parameters, context) + duration = time.clock() - start + self.trace_logger.log( "sqlalchemy_query", + message="Query executed", statement=statement, parameters=parameters, + executemany=executemany, duration=duration ) + return rval \ No newline at end of file diff -r 7b916e959c09083e541d1daaf573b1af995ed152 -r d603df6f2e7978f9c75f64190bfba97bbf59e851 lib/galaxy/tool_shed/install_manager.py --- a/lib/galaxy/tool_shed/install_manager.py +++ b/lib/galaxy/tool_shed/install_manager.py @@ -284,7 +284,6 @@ relative_install_dir = os.path.join( relative_clone_dir, name ) install_dir = os.path.join( clone_dir, name ) ctx_rev = suc.get_ctx_rev( tool_shed_url, name, self.repository_owner, installed_changeset_revision ) - print "Adding new row (or updating an existing row) for repository '%s' in the tool_shed_repository table." % name tool_shed_repository = suc.create_or_update_tool_shed_repository( app=self.app, name=name, description=description, diff -r 7b916e959c09083e541d1daaf573b1af995ed152 -r d603df6f2e7978f9c75f64190bfba97bbf59e851 lib/galaxy/tool_shed/update_manager.py --- a/lib/galaxy/tool_shed/update_manager.py +++ b/lib/galaxy/tool_shed/update_manager.py @@ -4,6 +4,7 @@ import threading, urllib2, logging from galaxy.util import string_as_bool import galaxy.util.shed_util as shed_util +from galaxy.model.orm import and_ log = logging.getLogger( __name__ ) diff -r 7b916e959c09083e541d1daaf573b1af995ed152 -r d603df6f2e7978f9c75f64190bfba97bbf59e851 lib/galaxy/tools/__init__.py --- a/lib/galaxy/tools/__init__.py +++ b/lib/galaxy/tools/__init__.py @@ -813,7 +813,7 @@ self.attributes['split_size'] = 20 self.attributes['split_mode'] = 'number_of_parts' -class Tool: +class Tool( object ): """ Represents a computational tool that can be executed through Galaxy. """ @@ -1600,9 +1600,11 @@ try: possible_cases.remove( case.value ) except: - log.warning( "A when tag has been defined for '%s (%s) --> %s', but does not appear to be selectable." % ( group.name, group.test_param.name, case.value ) ) + log.warning( "Tool %s: a when tag has been defined for '%s (%s) --> %s', but does not appear to be selectable." % + ( self.id, group.name, group.test_param.name, case.value ) ) for unspecified_case in possible_cases: - log.warning( "A when tag has not been defined for '%s (%s) --> %s', assuming empty inputs." % ( group.name, group.test_param.name, unspecified_case ) ) + log.warning( "Tool %s: a when tag has not been defined for '%s (%s) --> %s', assuming empty inputs." % + ( self.id, group.name, group.test_param.name, unspecified_case ) ) case = ConditionalWhen() case.value = unspecified_case case.inputs = odict() @@ -2128,16 +2130,16 @@ return params_to_strings( self.inputs, params, app ) def params_from_strings( self, params, app, ignore_errors=False ): return params_from_strings( self.inputs, params, app, ignore_errors ) - def check_and_update_param_values( self, values, trans ): + def check_and_update_param_values( self, values, trans, update_values=True ): """ Check that all parameters have values, and fill in with default values where necessary. This could be called after loading values from a database in case new parameters have been added. """ messages = {} - self.check_and_update_param_values_helper( self.inputs, values, trans, messages ) + self.check_and_update_param_values_helper( self.inputs, values, trans, messages, update_values=update_values ) return messages - def check_and_update_param_values_helper( self, inputs, values, trans, messages, context=None, prefix="" ): + def check_and_update_param_values_helper( self, inputs, values, trans, messages, context=None, prefix="", update_values=True ): """ Recursive helper for `check_and_update_param_values_helper` """ @@ -2181,7 +2183,13 @@ self.check_and_update_param_values_helper( input.cases[current].inputs, group_values, trans, messages, context, prefix ) else: # Regular tool parameter, no recursion needed - pass + try: + #this will fail when a parameter's type has changed to a non-compatible one: e.g. conditional group changed to dataset input + input.value_from_basic( input.value_to_basic( values[ input.name ], trans.app ), trans.app, ignore_errors=False ) + except: + messages[ input.name ] = "Value no longer valid for '%s%s', replaced with default" % ( prefix, input.label ) + if update_values: + values[ input.name ] = input.get_initial_value( trans, context ) def handle_unvalidated_param_values( self, input_values, app ): """ Find any instances of `UnvalidatedValue` within input_values and diff -r 7b916e959c09083e541d1daaf573b1af995ed152 -r d603df6f2e7978f9c75f64190bfba97bbf59e851 lib/galaxy/tools/parameters/basic.py --- a/lib/galaxy/tools/parameters/basic.py +++ b/lib/galaxy/tools/parameters/basic.py @@ -747,9 +747,9 @@ return { "__class__": "RuntimeValue" } return value def value_from_basic( self, value, app, ignore_errors=False ): - if isinstance( value, dict ) and value["__class__"] == "UnvalidatedValue": + if isinstance( value, dict ) and value.get( "__class__", None ) == "UnvalidatedValue": return UnvalidatedValue( value["value"] ) - return super( SelectToolParameter, self ).value_from_basic( value, app ) + return super( SelectToolParameter, self ).value_from_basic( value, app, ignore_errors=ignore_errors ) def need_late_validation( self, trans, context ): """ Determine whether we need to wait to validate this parameters value @@ -943,7 +943,7 @@ if not isinstance( value, list ): value = value.split( '\n' ) for column in value: - for column2 in column.split( ',' ): + for column2 in str( column ).split( ',' ): column2 = column2.strip() if column2: column_list.append( column2 ) @@ -1347,7 +1347,7 @@ rval = [] for val in value: rval.append( get_option_display( val, self.options ) or val ) - return "\n".join( rval ) + suffix + return "\n".join( map( str, rval ) ) + suffix def get_dependencies( self ): """ @@ -1586,8 +1586,11 @@ elif isinstance( value, list) and len(value) > 0 and isinstance( value[0], DummyDataset): return None elif isinstance( value, list ): - return ",".join( [ val if isinstance( val, basestring ) else str(val.id) for val in value] ) - return value.id + return ",".join( [ str( self.to_string( val, app ) ) for val in value ] ) + try: + return value.id + except: + return str( value ) def to_python( self, value, app ): # Both of these values indicate that no dataset is selected. However, 'None' @@ -1608,9 +1611,11 @@ if value and not isinstance( value, list ): value = [ value ] if value: - return ", ".join( [ "%s: %s" % ( item.hid, item.name ) for item in value ] ) - else: - return "No dataset" + try: + return ", ".join( [ "%s: %s" % ( item.hid, item.name ) for item in value ] ) + except: + pass + return "No dataset" def validate( self, value, history=None ): for validator in self.validators: diff -r 7b916e959c09083e541d1daaf573b1af995ed152 -r d603df6f2e7978f9c75f64190bfba97bbf59e851 lib/galaxy/tools/parameters/grouping.py --- a/lib/galaxy/tools/parameters/grouping.py +++ b/lib/galaxy/tools/parameters/grouping.py @@ -68,21 +68,25 @@ return rval def value_from_basic( self, value, app, ignore_errors=False ): rval = [] - for i, d in enumerate( value ): - rval_dict = {} - # If the special __index__ key is not set, create it (for backward - # compatibility) - rval_dict['__index__'] = d.get( '__index__', i ) - # Restore child inputs - for input in self.inputs.itervalues(): - if ignore_errors and input.name not in d: - # If we do not have a value, and are ignoring errors, we simply - # do nothing. There will be no value for the parameter in the - # conditional's values dictionary. - pass - else: - rval_dict[ input.name ] = input.value_from_basic( d[input.name], app, ignore_errors ) - rval.append( rval_dict ) + try: + for i, d in enumerate( value ): + rval_dict = {} + # If the special __index__ key is not set, create it (for backward + # compatibility) + rval_dict['__index__'] = d.get( '__index__', i ) + # Restore child inputs + for input in self.inputs.itervalues(): + if ignore_errors and input.name not in d: + # If we do not have a value, and are ignoring errors, we simply + # do nothing. There will be no value for the parameter in the + # conditional's values dictionary. + pass + else: + rval_dict[ input.name ] = input.value_from_basic( d[input.name], app, ignore_errors ) + rval.append( rval_dict ) + except Exception, e: + if not ignore_errors: + raise e return rval def visit_inputs( self, prefix, value, callback ): for i, d in enumerate( value ): @@ -441,24 +445,28 @@ return rval def value_from_basic( self, value, app, ignore_errors=False ): rval = dict() - current_case = rval['__current_case__'] = value['__current_case__'] - # Test param - if ignore_errors and self.test_param.name not in value: - # If ignoring errors, do nothing. However this is potentially very - # problematic since if we are missing the value of test param, - # the entire conditional is wrong. - pass - else: - rval[ self.test_param.name ] = self.test_param.value_from_basic( value[ self.test_param.name ], app, ignore_errors ) - # Inputs associated with current case - for input in self.cases[current_case].inputs.itervalues(): - if ignore_errors and input.name not in value: - # If we do not have a value, and are ignoring errors, we simply - # do nothing. There will be no value for the parameter in the - # conditional's values dictionary. + try: + current_case = rval['__current_case__'] = value['__current_case__'] + # Test param + if ignore_errors and self.test_param.name not in value: + # If ignoring errors, do nothing. However this is potentially very + # problematic since if we are missing the value of test param, + # the entire conditional is wrong. pass else: - rval[ input.name ] = input.value_from_basic( value[ input.name ], app, ignore_errors ) + rval[ self.test_param.name ] = self.test_param.value_from_basic( value[ self.test_param.name ], app, ignore_errors ) + # Inputs associated with current case + for input in self.cases[current_case].inputs.itervalues(): + if ignore_errors and input.name not in value: + # If we do not have a value, and are ignoring errors, we simply + # do nothing. There will be no value for the parameter in the + # conditional's values dictionary. + pass + else: + rval[ input.name ] = input.value_from_basic( value[ input.name ], app, ignore_errors ) + except Exception, e: + if not ignore_errors: + raise e return rval def visit_inputs( self, prefix, value, callback ): current_case = value['__current_case__'] diff -r 7b916e959c09083e541d1daaf573b1af995ed152 -r d603df6f2e7978f9c75f64190bfba97bbf59e851 lib/galaxy/tools/parameters/validation.py --- a/lib/galaxy/tools/parameters/validation.py +++ b/lib/galaxy/tools/parameters/validation.py @@ -182,10 +182,13 @@ def from_element( cls, param, elem ): return cls( message=elem.get( 'message', None ), check=elem.get( 'check', "" ), skip=elem.get( 'skip', "" ) ) def validate( self, value, history=None ): - if value and value.missing_meta( check = self.check, skip = self.skip ): - if self.message is None: - self.message = "Metadata missing, click the pencil icon in the history item to edit / save the metadata attributes" - raise ValueError( self.message ) + if value: + if not isinstance( value, model.DatasetInstance ): + raise ValueError( 'A non-dataset value was provided.' ) + if value.missing_meta( check = self.check, skip = self.skip ): + if self.message is None: + self.message = "Metadata missing, click the pencil icon in the history item to edit / save the metadata attributes" + raise ValueError( self.message ) class UnspecifiedBuildValidator( Validator ): """ diff -r 7b916e959c09083e541d1daaf573b1af995ed152 -r d603df6f2e7978f9c75f64190bfba97bbf59e851 lib/galaxy/tools/test.py --- a/lib/galaxy/tools/test.py +++ b/lib/galaxy/tools/test.py @@ -35,11 +35,13 @@ value = new_value break if not found_parameter: - raise ValueError( "Unable to determine parameter type of test input '%s'. Ensure that the parameter exists and that any container groups are defined first." % name ) + raise ValueError( "Unable to determine parameter type of test input '%s'. " + "Ensure that the parameter exists and that any container groups are defined first." + % name ) elif isinstance( self.tool.inputs[name], basic.DataToolParameter ): value = self.__add_uploaded_dataset( name, value, extra, self.tool.inputs[name] ) except Exception, e: - log.debug( "Error in add_param for %s: %s" % ( name, e ) ) + log.debug( "Error for tool %s: could not add test parameter %s. %s" % ( self.tool.id, name, e ) ) self.inputs.append( ( name, value, extra ) ) def add_output( self, name, file, extra ): self.outputs.append( ( name, file, extra ) ) diff -r 7b916e959c09083e541d1daaf573b1af995ed152 -r d603df6f2e7978f9c75f64190bfba97bbf59e851 lib/galaxy/util/__init__.py --- a/lib/galaxy/util/__init__.py +++ b/lib/galaxy/util/__init__.py @@ -2,7 +2,7 @@ Utility functions used systemwide. """ -import logging, threading, random, string, re, binascii, pickle, time, datetime, math, re, os, sys, tempfile, stat, grp, smtplib +import logging, threading, random, string, re, binascii, pickle, time, datetime, math, re, os, sys, tempfile, stat, grp, smtplib, errno from email.MIMEText import MIMEText # Older py compatibility diff -r 7b916e959c09083e541d1daaf573b1af995ed152 -r d603df6f2e7978f9c75f64190bfba97bbf59e851 lib/galaxy/util/hash_util.py --- a/lib/galaxy/util/hash_util.py +++ b/lib/galaxy/util/hash_util.py @@ -32,3 +32,10 @@ def hmac_new( key, value ): return hmac.new( key, value, sha ).hexdigest() + +def is_hashable( value ): + try: + hash( value ) + except: + return False + return True diff -r 7b916e959c09083e541d1daaf573b1af995ed152 -r d603df6f2e7978f9c75f64190bfba97bbf59e851 lib/galaxy/util/log/__init__.py --- /dev/null +++ b/lib/galaxy/util/log/__init__.py @@ -0,0 +1,5 @@ +class TraceLogger( object ): + def __init__( self, name ): + self.name = name + def log( **kwargs ): + raise TypeError( "Abstract Method" ) \ No newline at end of file diff -r 7b916e959c09083e541d1daaf573b1af995ed152 -r d603df6f2e7978f9c75f64190bfba97bbf59e851 lib/galaxy/util/log/fluent_log.py --- /dev/null +++ b/lib/galaxy/util/log/fluent_log.py @@ -0,0 +1,39 @@ +""" +Provides a `TraceLogger` implementation that logs to a fluentd collector +""" + +import time +import threading + +import galaxy.eggs +galaxy.eggs.require( "fluent-logger" ) +galaxy.eggs.require( "msgpack_python" ) + +from fluent.sender import FluentSender + + +class FluentTraceLogger( object ): + def __init__( self, name, host='localhost', port=24224 ): + self.lock = threading.Lock() + self.thread_local = threading.local() + self.name = name + self.sender = FluentSender( self.name, host=host, port=port ) + + def context_set( self, key, value ): + self.lock.acquire() + if not hasattr( self.thread_local, 'context' ): + self.thread_local.context = {} + self.thread_local.context[key] = value + self.lock.release() + + def context_remove( self, key ): + self.lock.acquire() + del self.thread_local.context[key] + self.lock.release() + + def log( self, label, **kwargs ): + self.lock.acquire() + if hasattr( self.thread_local, 'context' ): + kwargs.update( self.thread_local.context ) + self.lock.release() + self.sender.emit_with_time( label, int(time.time()), kwargs ) \ No newline at end of file diff -r 7b916e959c09083e541d1daaf573b1af995ed152 -r d603df6f2e7978f9c75f64190bfba97bbf59e851 lib/galaxy/util/shed_util.py --- a/lib/galaxy/util/shed_util.py +++ b/lib/galaxy/util/shed_util.py @@ -1,10 +1,11 @@ -import os, tempfile, shutil, logging, urllib2 +import os, tempfile, shutil, logging, urllib2, threading from galaxy.datatypes import checkers from galaxy.web import url_for from galaxy import util -from galaxy.util.json import from_json_string, to_json_string +from galaxy.util import json from galaxy.webapps.community.util import container_util import shed_util_common as suc +import galaxy.tools from galaxy.tools.search import ToolBoxSearch from galaxy.tool_shed.tool_dependencies.install_util import create_or_update_tool_dependency, install_package, set_environment from galaxy.tool_shed import encoding_util @@ -22,6 +23,38 @@ log = logging.getLogger( __name__ ) +def activate_repository( trans, repository ): + repository_clone_url = suc.generate_clone_url_for_installed_repository( trans.app, repository ) + shed_tool_conf, tool_path, relative_install_dir = suc.get_tool_panel_config_tool_path_install_dir( trans.app, repository ) + repository.deleted = False + repository.status = trans.model.ToolShedRepository.installation_status.INSTALLED + if repository.includes_tools: + metadata = repository.metadata + repository_tools_tups = suc.get_repository_tools_tups( trans.app, metadata ) + # Reload tools into the appropriate tool panel section. + tool_panel_dict = repository.metadata[ 'tool_panel_section' ] + add_to_tool_panel( trans.app, + repository.name, + repository_clone_url, + repository.installed_changeset_revision, + repository_tools_tups, + repository.owner, + shed_tool_conf, + tool_panel_dict, + new_install=False ) + trans.sa_session.add( repository ) + trans.sa_session.flush() + if repository.includes_datatypes: + if tool_path: + repository_install_dir = os.path.abspath ( os.path.join( tool_path, relative_install_dir ) ) + else: + repository_install_dir = os.path.abspath ( relative_install_dir ) + # Activate proprietary datatypes. + installed_repository_dict = load_installed_datatypes( trans.app, repository, repository_install_dir, deactivate=False ) + if installed_repository_dict and 'converter_path' in installed_repository_dict: + load_installed_datatype_converters( trans.app, installed_repository_dict, deactivate=False ) + if installed_repository_dict and 'display_path' in installed_repository_dict: + load_installed_display_applications( trans.app, installed_repository_dict, deactivate=False ) def add_to_shed_tool_config( app, shed_tool_conf_dict, elem_list ): # A tool shed repository is being installed so change the shed_tool_conf file. Parse the config file to generate the entire list # of config_elems instead of using the in-memory list since it will be a subset of the entire list if one or more repositories have @@ -175,66 +208,130 @@ # Attempt to ensure we're copying an appropriate file. if is_data_index_sample_file( filename ): suc.copy_sample_file( app, filename, dest_path=dest_path ) -def create_repository_dependency_objects( trans, tool_path, tool_shed_url, repo_info_dicts, reinstalling=False ): +def create_repository_dependency_objects( trans, tool_path, tool_shed_url, repo_info_dicts, reinstalling=False, install_repository_dependencies=False, + no_changes_checked=False, tool_panel_section=None, new_tool_panel_section=None ): """ Discover all repository dependencies and make sure all tool_shed_repository and associated repository_dependency records exist as well as the dependency relationships between installed repositories. This method is called when new repositories are being installed into a Galaxy instance and when uninstalled repositories are being reinstalled. """ message = '' + # The following list will be maintained within this method to contain all created or updated tool shed repositories, including repository dependencies + # that may not be installed. + all_created_or_updated_tool_shed_repositories = [] + # There will be a one-to-one mapping between items in 3 lists: created_or_updated_tool_shed_repositories, tool_panel_section_keys and filtered_repo_info_dicts. + # The 3 lists will filter out repository dependencies that are not to be installed. created_or_updated_tool_shed_repositories = [] - # Repositories will be filtered (e.g., if already installed, etc), so filter the associated repo_info_dicts accordingly. + tool_panel_section_keys = [] + # Repositories will be filtered (e.g., if already installed, if elected to not be installed, etc), so filter the associated repo_info_dicts accordingly. filtered_repo_info_dicts = [] - # Discover all repository dependencies and retrieve information for installing them. + # Discover all repository dependencies and retrieve information for installing them. Even if the user elected to not install repository dependencies we have + # to make sure all repository dependency objects exist so that the appropriate repository dependency relationships can be built. all_repo_info_dicts = get_required_repo_info_dicts( tool_shed_url, repo_info_dicts ) + if not all_repo_info_dicts: + # No repository dependencies were discovered so process the received repositories. + all_repo_info_dicts = [ rid for rid in repo_info_dicts ] for repo_info_dict in all_repo_info_dicts: for name, repo_info_tuple in repo_info_dict.items(): description, repository_clone_url, changeset_revision, ctx_rev, repository_owner, repository_dependencies, tool_dependencies = \ suc.get_repo_info_tuple_contents( repo_info_tuple ) - clone_dir = os.path.join( tool_path, generate_tool_path( repository_clone_url, changeset_revision ) ) - relative_install_dir = os.path.join( clone_dir, name ) # Make sure the repository was not already installed. - installed_tool_shed_repository, installed_changeset_revision = \ - repository_was_previously_installed( trans, tool_shed_url, name, repo_info_tuple, clone_dir ) + installed_tool_shed_repository, installed_changeset_revision = repository_was_previously_installed( trans, tool_shed_url, name, repo_info_tuple ) if installed_tool_shed_repository: - if reinstalling: - if installed_tool_shed_repository.status in [ trans.model.ToolShedRepository.installation_status.ERROR, - trans.model.ToolShedRepository.installation_status.UNINSTALLED ]: - can_update = True - name = installed_tool_shed_repository.name - description = installed_tool_shed_repository.description - installed_changeset_revision = installed_tool_shed_repository.installed_changeset_revision - metadata_dict = installed_tool_shed_repository.metadata - dist_to_shed = installed_tool_shed_repository.dist_to_shed + tool_section, new_tool_panel_section, tool_panel_section_key = handle_tool_panel_selection( trans=trans, + metadata=installed_tool_shed_repository.metadata, + no_changes_checked=no_changes_checked, + tool_panel_section=tool_panel_section, + new_tool_panel_section=new_tool_panel_section ) + if reinstalling or install_repository_dependencies: + # If the user elected to install repository dependencies, all items in the all_repo_info_dicts list will be processed. However, if + # repository dependencies are not to be installed, only those items contained in the received repo_info_dicts list will be processed. + if is_in_repo_info_dicts( repo_info_dict, repo_info_dicts ) or install_repository_dependencies: + if installed_tool_shed_repository.status in [ trans.model.ToolShedRepository.installation_status.ERROR, + trans.model.ToolShedRepository.installation_status.UNINSTALLED ]: + # The current tool shed repository is not currently installed, so we can update it's record in the database. + can_update = True + name = installed_tool_shed_repository.name + description = installed_tool_shed_repository.description + installed_changeset_revision = installed_tool_shed_repository.installed_changeset_revision + metadata_dict = installed_tool_shed_repository.metadata + dist_to_shed = installed_tool_shed_repository.dist_to_shed + elif installed_tool_shed_repository.status in [ trans.model.ToolShedRepository.installation_status.DEACTIVATED ]: + # The current tool shed repository is deactivated, so updating it's database record is not necessary - just activate it. + activate_repository( trans, installed_tool_shed_repository ) + can_update = False + else: + # The tool shed repository currently being processed is already installed or is in the process of being installed, so it's record + # in the database cannot be updated. + can_update = False else: - # There is a repository already installed which is a dependency of the repository being reinstalled. + # This block will be reached only if reinstalling is True, install_repository_dependencies is False and is_in_repo_info_dicts is False. + # The tool shed repository currently being processed must be a repository dependency that the user elected to not install, so it's + # record in the database cannot be updated. can_update = False else: - # An attempt is being made to install a tool shed repository into a Galaxy instance when the same repository was previously installed. - message += "Revision <b>%s</b> of tool shed repository <b>%s</b> owned by <b>%s</b> " % ( changeset_revision, name, repository_owner ) - if installed_changeset_revision != changeset_revision: - message += "was previously installed using changeset revision <b>%s</b>. " % installed_changeset_revision + # This block will be reached only if reinstalling is False and install_repository_dependencies is False. This implies that the tool shed + # repository currently being processed has already been installed. + if len( all_repo_info_dicts ) == 1: + # If only a single repository is being installed, return an informative message to the user. + message += "Revision <b>%s</b> of tool shed repository <b>%s</b> owned by <b>%s</b> " % ( changeset_revision, name, repository_owner ) + if installed_changeset_revision != changeset_revision: + message += "was previously installed using changeset revision <b>%s</b>. " % installed_changeset_revision + else: + message += "was previously installed. " + if installed_tool_shed_repository.uninstalled: + message += "The repository has been uninstalled, however, so reinstall the original repository instead of installing it again. " + elif installed_tool_shed_repository.deleted: + message += "The repository has been deactivated, however, so activate the original repository instead of installing it again. " + if installed_changeset_revision != changeset_revision: + message += "You can get the latest updates for the repository using the <b>Get updates</b> option from the repository's " + message += "<b>Repository Actions</b> pop-up menu. " + created_or_updated_tool_shed_repositories.append( installed_tool_shed_repository ) + tool_panel_section_keys.append( tool_panel_section_key ) + return created_or_updated_tool_shed_repositories, tool_panel_section_keys, all_repo_info_dicts, filtered_repo_info_dicts, message else: - message += "was previously installed. " - if installed_tool_shed_repository.uninstalled: - message += "The repository has been uninstalled, however, so reinstall the original repository instead of installing it again. " - elif installed_tool_shed_repository.deleted: - message += "The repository has been deactivated, however, so activate the original repository instead of installing it again. " - if installed_changeset_revision != changeset_revision: - message += "You can get the latest updates for the repository using the <b>Get updates</b> option from the repository's " - message += "<b>Repository Actions</b> pop-up menu. " - if len( repo_info_dicts ) == 1: - created_or_updated_tool_shed_repositories.append( installed_tool_shed_repository ) - return created_or_updated_tool_shed_repositories, all_repo_info_dicts, filtered_repo_info_dicts, message + # We're in the process of installing multiple tool shed repositories into Galaxy. Since the repository currently being processed + # has already been installed, skip it and process the next repository in the list. + can_update = False else: - # A tool shed repository is being installed into a Galaxy instance for the first time. We may have the case where a repository - # is being reinstalled where because the repository being newly installed here may be a dependency of the repository being reinstalled. + # A tool shed repository is being installed into a Galaxy instance for the first time, or we're attempting to install it or reinstall it resulted + # in an error. In the latter case, the repository record in the database has no metadata and it's status has been set to 'New'. In either case, + # the repository's database record may be updated. can_update = True installed_changeset_revision = changeset_revision - metadata_dict={} + metadata_dict = {} dist_to_shed = False if can_update: - log.debug( "Adding new row (or updating an existing row) for repository '%s' in the tool_shed_repository table." % name ) + # The database record for the tool shed repository currently being processed can be updated. + if reinstalling or install_repository_dependencies: + # Get the repository metadata to see where it was previously located in the tool panel. + if installed_tool_shed_repository: + # The tool shed repository status is one of 'New', 'Uninstalled', or 'Error'. + tool_section, new_tool_panel_section, tool_panel_section_key = \ + handle_tool_panel_selection( trans=trans, + metadata=installed_tool_shed_repository.metadata, + no_changes_checked=no_changes_checked, + tool_panel_section=tool_panel_section, + new_tool_panel_section=new_tool_panel_section ) + else: + # We're installing a new tool shed repository that does not yet have a database record. This repository is a repository dependency + # of a different repository being installed. + if new_tool_panel_section: + section_id = new_tool_panel_section.lower().replace( ' ', '_' ) + tool_panel_section_key = 'section_%s' % str( section_id ) + elif tool_panel_section: + tool_panel_section_key = 'section_%s' % tool_panel_section + else: + tool_panel_section_key = None + else: + # We're installing a new tool shed repository that does not yet have a database record. + if new_tool_panel_section: + section_id = new_tool_panel_section.lower().replace( ' ', '_' ) + tool_panel_section_key = 'section_%s' % str( section_id ) + elif tool_panel_section: + tool_panel_section_key = 'section_%s' % tool_panel_section + else: + tool_panel_section_key = None tool_shed_repository = suc.create_or_update_tool_shed_repository( app=trans.app, name=name, description=description, @@ -246,9 +343,17 @@ current_changeset_revision=changeset_revision, owner=repository_owner, dist_to_shed=False ) - created_or_updated_tool_shed_repositories.append( tool_shed_repository ) - filtered_repo_info_dicts.append( encoding_util.tool_shed_encode( repo_info_dict ) ) - return created_or_updated_tool_shed_repositories, all_repo_info_dicts, filtered_repo_info_dicts, message + # Add the processed tool shed repository to the list of all processed repositories maintained within this method. + all_created_or_updated_tool_shed_repositories.append( tool_shed_repository ) + # Only append the tool shed repository to the list of created_or_updated_tool_shed_repositories if it is supposed to be installed. + if install_repository_dependencies or is_in_repo_info_dicts( repo_info_dict, repo_info_dicts ): + # Keep the one-to-one mapping between items in 3 lists. + created_or_updated_tool_shed_repositories.append( tool_shed_repository ) + tool_panel_section_keys.append( tool_panel_section_key ) + filtered_repo_info_dicts.append( repo_info_dict ) + # Build repository dependency relationships even if the user chose to not install repository dependencies. + suc.build_repository_dependency_relationships( trans, all_repo_info_dicts, all_created_or_updated_tool_shed_repositories ) + return created_or_updated_tool_shed_repositories, tool_panel_section_keys, all_repo_info_dicts, filtered_repo_info_dicts, message def create_repository_dict_for_proprietary_datatypes( tool_shed, name, owner, installed_changeset_revision, tool_dicts, converter_path=None, display_path=None ): return dict( tool_shed=tool_shed, repository_name=name, @@ -476,6 +581,66 @@ if converter_path and display_path: break return converter_path, display_path +def get_dependencies_for_repository( trans, tool_shed_url, repo_info_dict, includes_tool_dependencies ): + """ + Return dictionaries containing the sets of installed and missing tool dependencies and repository dependencies associated with the repository defined + by the received repo_info_dict. + """ + name = repo_info_dict.keys()[ 0 ] + repo_info_tuple = repo_info_dict[ name ] + description, repository_clone_url, changeset_revision, ctx_rev, repository_owner, repository_dependencies, installed_td = \ + suc.get_repo_info_tuple_contents( repo_info_tuple ) + if repository_dependencies: + missing_td = {} + # Handle the scenario where a repository was installed, then uninstalled and an error occurred during the reinstallation process. + # In this case, a record for the repository will exist in the database with the status of 'New'. + repository = suc.get_repository_for_dependency_relationship( trans.app, tool_shed_url, name, repository_owner, changeset_revision ) + if repository and repository.metadata: + installed_rd, missing_rd = get_installed_and_missing_repository_dependencies( trans, repository ) + else: + installed_rd, missing_rd = get_installed_and_missing_repository_dependencies_for_new_install( trans, repo_info_tuple ) + # Discover all repository dependencies and retrieve information for installing them. + required_repo_info_dicts = get_required_repo_info_dicts( tool_shed_url, util.listify( repo_info_dict ) ) + # Display tool dependencies defined for each of the repository dependencies. + if required_repo_info_dicts: + all_tool_dependencies = {} + for rid in required_repo_info_dicts: + for name, repo_info_tuple in rid.items(): + description, repository_clone_url, changeset_revision, ctx_rev, repository_owner, repository_dependencies, rid_installed_td = \ + suc.get_repo_info_tuple_contents( repo_info_tuple ) + if rid_installed_td: + for td_key, td_dict in rid_installed_td.items(): + if td_key not in all_tool_dependencies: + all_tool_dependencies[ td_key ] = td_dict + if all_tool_dependencies: + if installed_td is None: + installed_td = {} + else: + # Move all tool dependencies to the missing_tool_dependencies container. + for td_key, td_dict in installed_td.items(): + if td_key not in missing_td: + missing_td[ td_key ] = td_dict + installed_td = {} + # Discover and categorize all tool dependencies defined for this repository's repository dependencies. + required_tool_dependencies, required_missing_tool_dependencies = \ + get_installed_and_missing_tool_dependencies_for_new_install( trans, all_tool_dependencies ) + if required_tool_dependencies: + if not includes_tool_dependencies: + includes_tool_dependencies = True + for td_key, td_dict in required_tool_dependencies.items(): + if td_key not in installed_td: + installed_td[ td_key ] = td_dict + if required_missing_tool_dependencies: + if not includes_tool_dependencies: + includes_tool_dependencies = True + for td_key, td_dict in required_missing_tool_dependencies.items(): + if td_key not in missing_td: + missing_td[ td_key ] = td_dict + else: + installed_rd = None + missing_rd = None + missing_td = None + return name, repository_owner, changeset_revision, includes_tool_dependencies, installed_rd, missing_rd, installed_td, missing_td def get_headers( fname, sep, count=60, is_multi_byte=False ): """Returns a list with the first 'count' lines split by 'sep'.""" headers = [] @@ -489,9 +654,14 @@ break return headers def get_installed_and_missing_repository_dependencies( trans, repository ): + """ + Return the installed and missing repository dependencies for a tool shed repository that has a record in the Galaxy database, but + may or may not be installed. In this case, the repository dependencies are associated with the repository in the database. + """ missing_repository_dependencies = {} installed_repository_dependencies = {} - if repository.has_repository_dependencies: + has_repository_dependencies = repository.has_repository_dependencies + if has_repository_dependencies: metadata = repository.metadata installed_rd_tups = [] missing_rd_tups = [] @@ -522,6 +692,50 @@ missing_repository_dependencies[ root_key ] = missing_rd_tups missing_repository_dependencies[ 'description' ] = description return installed_repository_dependencies, missing_repository_dependencies +def get_installed_and_missing_repository_dependencies_for_new_install( trans, repo_info_tuple ): + """ + Parse the received repository_dependencies dictionary that is associated with a repository being installed into Galaxy for the first time + and attempt to determine repository dependencies that are already installed and those that are not. + """ + missing_repository_dependencies = {} + installed_repository_dependencies = {} + missing_rd_tups = [] + installed_rd_tups = [] + description, repository_clone_url, changeset_revision, ctx_rev, repository_owner, repository_dependencies, installed_td = \ + suc.get_repo_info_tuple_contents( repo_info_tuple ) + if repository_dependencies: + description = repository_dependencies[ 'description' ] + root_key = repository_dependencies[ 'root_key' ] + # The repository dependencies container will include only the immediate repository dependencies of this repository, so the container will be + # only a single level in depth. + for key, rd_tups in repository_dependencies.items(): + if key in [ 'description', 'root_key' ]: + continue + for rd_tup in rd_tups: + tool_shed, name, owner, changeset_revision = rd_tup + # Updates to installed repository revisions may have occurred, so make sure to locate the appropriate repository revision if one exists. + repository, current_changeset_revision = repository_was_previously_installed( trans, tool_shed, name, repo_info_tuple ) + if repository: + new_rd_tup = [ tool_shed, name, owner, changeset_revision, repository.id, repository.status ] + if repository.status == trans.model.ToolShedRepository.installation_status.INSTALLED: + if new_rd_tup not in installed_rd_tups: + installed_rd_tups.append( new_rd_tup ) + else: + if new_rd_tup not in missing_rd_tups: + missing_rd_tups.append( new_rd_tup ) + else: + new_rd_tup = [ tool_shed, name, owner, changeset_revision, None, 'Never installed' ] + if new_rd_tup not in missing_rd_tups: + missing_rd_tups.append( new_rd_tup ) + if installed_rd_tups: + installed_repository_dependencies[ 'root_key' ] = root_key + installed_repository_dependencies[ root_key ] = installed_rd_tups + installed_repository_dependencies[ 'description' ] = description + if missing_rd_tups: + missing_repository_dependencies[ 'root_key' ] = root_key + missing_repository_dependencies[ root_key ] = missing_rd_tups + missing_repository_dependencies[ 'description' ] = description + return installed_repository_dependencies, missing_repository_dependencies def get_installed_and_missing_tool_dependencies( trans, repository, all_tool_dependencies ): if all_tool_dependencies: tool_dependencies = {} @@ -536,7 +750,11 @@ if tool_dependency: td_info_dict[ 'repository_id' ] = repository.id td_info_dict[ 'tool_dependency_id' ] = tool_dependency.id - td_info_dict[ 'status' ] = str( tool_dependency.status ) + if tool_dependency.status: + tool_dependency_status = str( tool_dependency.status ) + else: + tool_dependency_status = 'Never installed' + td_info_dict[ 'status' ] = tool_dependency_status val[ index ] = td_info_dict if tool_dependency.status == trans.model.ToolDependency.installation_status.INSTALLED: tool_dependencies[ td_key ] = val @@ -550,7 +768,11 @@ if tool_dependency: val[ 'repository_id' ] = repository.id val[ 'tool_dependency_id' ] = tool_dependency.id - val[ 'status' ] = str( tool_dependency.status ) + if tool_dependency.status: + tool_dependency_status = str( tool_dependency.status ) + else: + tool_dependency_status = 'Never installed' + val[ 'status' ] = tool_dependency_status if tool_dependency.status == trans.model.ToolDependency.installation_status.INSTALLED: tool_dependencies[ td_key ] = val else: @@ -559,6 +781,45 @@ tool_dependencies = None missing_tool_dependencies = None return tool_dependencies, missing_tool_dependencies +def get_installed_and_missing_tool_dependencies_for_new_install( trans, all_tool_dependencies ): + """Return the lists of installed tool dependencies and missing tool dependencies for a set of repositories being installed into Galaxy.""" + # FIXME: this method currently populates and returns only missing tool dependencies since tool dependencies defined for complex repository dependency + # relationships is not currently supported. This method should be enhanced to search for installed tool dependencies defined as complex repository + # dependency relationships when that feature is implemented. + if all_tool_dependencies: + tool_dependencies = {} + missing_tool_dependencies = {} + for td_key, val in all_tool_dependencies.items(): + # Set environment tool dependencies are a list, set each member to never installed. + if td_key == 'set_environment': + new_val = [] + for requirement_dict in val: + requirement_dict[ 'status' ] = trans.model.ToolDependency.installation_status.NEVER_INSTALLED + new_val.append( requirement_dict ) + missing_tool_dependencies[ td_key ] = new_val + else: + # Since we have a new install, missing tool dependencies have never been installed. + val[ 'status' ] = trans.model.ToolDependency.installation_status.NEVER_INSTALLED + missing_tool_dependencies[ td_key ] = val + else: + tool_dependencies = None + missing_tool_dependencies = None + return tool_dependencies, missing_tool_dependencies +def get_readme_files_dict_for_display( trans, tool_shed_url, repo_info_dict ): + """Return a dictionary of README files contained in the single repository being installed so they can be displayed on the tool panel section selection page.""" + name = repo_info_dict.keys()[ 0 ] + repo_info_tuple = repo_info_dict[ name ] + description, repository_clone_url, changeset_revision, ctx_rev, repository_owner, repository_dependencies, installed_td = \ + suc.get_repo_info_tuple_contents( repo_info_tuple ) + # Handle README files. + url = suc.url_join( tool_shed_url, + 'repository/get_readme_files?name=%s&owner=%s&changeset_revision=%s' % \ + ( name, repository_owner, changeset_revision ) ) + response = urllib2.urlopen( url ) + raw_text = response.read() + response.close() + readme_files_dict = json.from_json_string( raw_text ) + return readme_files_dict def get_repository_owner( cleaned_repository_url ): items = cleaned_repository_url.split( 'repos' ) repo_path = items[ 1 ] @@ -573,10 +834,10 @@ """ Inspect the list of repo_info_dicts for repository dependencies and append a repo_info_dict for each of them to the list. All repository_dependencies entries in each of the received repo_info_dicts includes all required repositories, so only one pass through - this methid is required to retrieve all repository dependencies. + this method is required to retrieve all repository dependencies. """ + all_repo_info_dicts = [] if repo_info_dicts: - all_repo_info_dicts = [ rid for rid in repo_info_dicts ] # We'll send tuples of ( tool_shed, repository_name, repository_owner, changeset_revision ) to the tool shed to discover repository ids. required_repository_tups = [] for repo_info_dict in repo_info_dicts: @@ -606,7 +867,7 @@ text = response.read() response.close() if text: - required_repo_info_dict = from_json_string( text ) + required_repo_info_dict = json.from_json_string( text ) required_repo_info_dicts = [] encoded_dict_strings = required_repo_info_dict[ 'repo_info_dicts' ] for encoded_dict_str in encoded_dict_strings: @@ -690,12 +951,16 @@ update_dict = encoding_util.tool_shed_decode( encoded_update_dict ) changeset_revision = update_dict[ 'changeset_revision' ] ctx_rev = update_dict[ 'ctx_rev' ] + includes_tools = update_dict.get( 'includes_tools', False ) + has_repository_dependencies = update_dict.get( 'has_repository_dependencies', False ) response.close() except Exception, e: log.debug( "Error getting change set revision for update from the tool shed for repository '%s': %s" % ( repository.name, str( e ) ) ) + includes_tools = False + has_repository_dependencies = False changeset_revision = None ctx_rev = None - return changeset_revision, ctx_rev + return changeset_revision, ctx_rev, includes_tools, has_repository_dependencies def handle_missing_data_table_entry( app, relative_install_dir, tool_path, repository_tools_tups ): """ Inspect each tool to see if any have input parameters that are dynamically generated select lists that require entries in the @@ -783,6 +1048,60 @@ app.model.ToolDependency.installation_status.ERROR ]: installed_tool_dependencies.append( tool_dependency ) return installed_tool_dependencies +def handle_tool_panel_selection( trans, metadata, no_changes_checked, tool_panel_section, new_tool_panel_section ): + """Handle the selected tool panel location for loading tools included in tool shed repositories when installing or reinstalling them.""" + # Get the location in the tool panel in which each tool was originally loaded. + tool_section = None + tool_panel_section_key = None + if 'tools' in metadata: + if 'tool_panel_section' in metadata: + tool_panel_dict = metadata[ 'tool_panel_section' ] + if not tool_panel_dict: + tool_panel_dict = generate_tool_panel_dict_for_new_install( metadata[ 'tools' ] ) + else: + tool_panel_dict = generate_tool_panel_dict_for_new_install( metadata[ 'tools' ] ) + # This forces everything to be loaded into the same section (or no section) in the tool panel. + tool_section_dicts = tool_panel_dict[ tool_panel_dict.keys()[ 0 ] ] + tool_section_dict = tool_section_dicts[ 0 ] + original_section_id = tool_section_dict[ 'id' ] + original_section_name = tool_section_dict[ 'name' ] + if no_changes_checked: + if original_section_id: + tool_panel_section_key = 'section_%s' % str( original_section_id ) + if tool_panel_section_key in trans.app.toolbox.tool_panel: + tool_section = trans.app.toolbox.tool_panel[ tool_panel_section_key ] + else: + # The section in which the tool was originally loaded used to be in the tool panel, but no longer is. + elem = Element( 'section' ) + elem.attrib[ 'name' ] = original_section_name + elem.attrib[ 'id' ] = original_section_id + elem.attrib[ 'version' ] = '' + tool_section = galaxy.tools.ToolSection( elem ) + trans.app.toolbox.tool_panel[ tool_panel_section_key ] = tool_section + else: + # The user elected to change the tool panel section to contain the tools. + if new_tool_panel_section: + section_id = new_tool_panel_section.lower().replace( ' ', '_' ) + tool_panel_section_key = 'section_%s' % str( section_id ) + if tool_panel_section_key in trans.app.toolbox.tool_panel: + # Appending a tool to an existing section in trans.app.toolbox.tool_panel + log.debug( "Appending to tool panel section: %s" % new_tool_panel_section ) + tool_section = trans.app.toolbox.tool_panel[ tool_panel_section_key ] + else: + # Appending a new section to trans.app.toolbox.tool_panel + log.debug( "Loading new tool panel section: %s" % new_tool_panel_section ) + elem = Element( 'section' ) + elem.attrib[ 'name' ] = new_tool_panel_section + elem.attrib[ 'id' ] = section_id + elem.attrib[ 'version' ] = '' + tool_section = galaxy.tools.ToolSection( elem ) + trans.app.toolbox.tool_panel[ tool_panel_section_key ] = tool_section + elif tool_panel_section: + tool_panel_section_key = 'section_%s' % tool_panel_section + tool_section = trans.app.toolbox.tool_panel[ tool_panel_section_key ] + else: + tool_section = None + return tool_section, new_tool_panel_section, tool_panel_section_key def handle_tool_versions( app, tool_version_dicts, tool_shed_repository ): """ Using the list of tool_version_dicts retrieved from the tool shed (one per changeset revison up to the currently installed changeset revision), @@ -851,6 +1170,18 @@ return False # Default to copying the file if none of the above are true. return True +def is_in_repo_info_dicts( repo_info_dict, repo_info_dicts ): + """Return True if the received repo_info_dict is contained in the list of received repo_info_dicts.""" + for name, repo_info_tuple in repo_info_dict.items(): + for rid in repo_info_dicts: + for rid_name, rid_repo_info_tuple in rid.items(): + if rid_name == name: + if len( rid_repo_info_tuple ) == len( repo_info_tuple ): + for item in rid_repo_info_tuple: + if item not in repo_info_tuple: + return False + return True + return False def load_installed_datatype_converters( app, installed_repository_dict, deactivate=False ): # Load or deactivate proprietary datatype converters app.datatypes_registry.load_datatype_converters( app.toolbox, installed_repository_dict=installed_repository_dict, deactivate=deactivate ) @@ -874,6 +1205,80 @@ def load_installed_display_applications( app, installed_repository_dict, deactivate=False ): # Load or deactivate proprietary datatype display applications app.datatypes_registry.load_display_applications( installed_repository_dict=installed_repository_dict, deactivate=deactivate ) +def merge_containers_dicts_for_new_install( containers_dicts ): + """ + When installing one or more tool shed repositories for the first time, the received list of containers_dicts contains a containers_dict for + each repository being installed. Since the repositories are being installed for the first time, all entries are None except the repository + dependencies and tool dependencies. The entries for missing dependencies are all None since they have previously been merged into the installed + dependencies. This method will merge the dependencies entries into a single container and return it for display. + """ + new_containers_dict = dict( readme_files=None, + datatypes=None, + missing_repository_dependencies=None, + repository_dependencies=None, + missing_tool_dependencies=None, + tool_dependencies=None, + invalid_tools=None, + valid_tools=None, + workflows=None ) + if containers_dicts: + lock = threading.Lock() + lock.acquire( True ) + try: + repository_dependencies_root_folder = None + tool_dependencies_root_folder = None + # Use a unique folder id (hopefully the following is). + folder_id = 867 + for old_container_dict in containers_dicts: + # Merge repository_dependencies. + old_container_repository_dependencies_root = old_container_dict[ 'repository_dependencies' ] + if old_container_repository_dependencies_root: + if repository_dependencies_root_folder is None: + repository_dependencies_root_folder = container_util.Folder( id=folder_id, key='root', label='root', parent=None ) + folder_id += 1 + repository_dependencies_folder = container_util.Folder( id=folder_id, + key='merged', + label='Repository dependencies', + parent=repository_dependencies_root_folder ) + folder_id += 1 + # The old_container_repository_dependencies_root will be a root folder containing a single sub_folder. + old_container_repository_dependencies_folder = old_container_repository_dependencies_root.folders[ 0 ] + # Change the folder id so it won't confict with others being merged. + old_container_repository_dependencies_folder.id = folder_id + folder_id += 1 + # Generate the label by retrieving the repository name. + toolshed, name, owner, changeset_revision = container_util.get_components_from_key( old_container_repository_dependencies_folder.key ) + old_container_repository_dependencies_folder.label = str( name ) + repository_dependencies_folder.folders.append( old_container_repository_dependencies_folder ) + # Merge tool_dependencies. + old_container_tool_dependencies_root = old_container_dict[ 'tool_dependencies' ] + if old_container_tool_dependencies_root: + if tool_dependencies_root_folder is None: + tool_dependencies_root_folder = container_util.Folder( id=folder_id, key='root', label='root', parent=None ) + folder_id += 1 + tool_dependencies_folder = container_util.Folder( id=folder_id, + key='merged', + label='Tool dependencies', + parent=tool_dependencies_root_folder ) + folder_id += 1 + else: + td_list = [ td.listify for td in tool_dependencies_folder.tool_dependencies ] + # The old_container_tool_dependencies_root will be a root folder containing a single sub_folder. + old_container_tool_dependencies_folder = old_container_tool_dependencies_root.folders[ 0 ] + for td in old_container_tool_dependencies_folder.tool_dependencies: + if td.listify not in td_list: + tool_dependencies_folder.tool_dependencies.append( td ) + if repository_dependencies_root_folder: + repository_dependencies_root_folder.folders.append( repository_dependencies_folder ) + new_containers_dict[ 'repository_dependencies' ] = repository_dependencies_root_folder + if tool_dependencies_root_folder: + tool_dependencies_root_folder.folders.append( tool_dependencies_folder ) + new_containers_dict[ 'tool_dependencies' ] = tool_dependencies_root_folder + except Exception, e: + log.debug( "Exception in merge_containers_dicts_for_new_install: %s" % str( e ) ) + finally: + lock.release() + return new_containers_dict def panel_entry_per_tool( tool_section_dict ): # Return True if tool_section_dict looks like this. # {<Tool guid> : [{ tool_config : <tool_config_file>, id: <ToolSection id>, version : <ToolSection version>, name : <TooSection name>}]} @@ -887,9 +1292,9 @@ if k not in [ 'id', 'version', 'name' ]: return True return False -def populate_containers_dict_from_repository_metadata( trans, tool_shed_url, tool_path, repository, reinstalling=False ): +def populate_containers_dict_from_repository_metadata( trans, tool_shed_url, tool_path, repository, reinstalling=False, required_repo_info_dicts=None ): """ - Retrieve necessary information from the received repository's metadata to populate the containers_dict for display. This methos is called only + Retrieve necessary information from the received repository's metadata to populate the containers_dict for display. This method is called only from Galaxy (not the tool shed) when displaying repository dependencies for installed repositories and when displaying them for uninstalled repositories that are being reinstalled. """ @@ -902,38 +1307,41 @@ # Handle README files. if repository.has_readme_files: if reinstalling: - # Since we're reinstalling, we need to sned a request to the tool shed to get the README files. + # Since we're reinstalling, we need to send a request to the tool shed to get the README files. url = suc.url_join( tool_shed_url, 'repository/get_readme_files?name=%s&owner=%s&changeset_revision=%s' % \ ( repository.name, repository.owner, repository.installed_changeset_revision ) ) response = urllib2.urlopen( url ) raw_text = response.read() response.close() - readme_files_dict = from_json_string( raw_text ) + readme_files_dict = json.from_json_string( raw_text ) else: readme_files_dict = suc.build_readme_files_dict( repository.metadata, tool_path ) else: readme_files_dict = None # Handle repository dependencies. installed_repository_dependencies, missing_repository_dependencies = get_installed_and_missing_repository_dependencies( trans, repository ) - # Handle tool dependencies. - all_tool_dependencies = metadata.get( 'tool_dependencies', None ) - installed_tool_dependencies, missing_tool_dependencies = get_installed_and_missing_tool_dependencies( trans, repository, all_tool_dependencies ) + # Handle the current repository's tool dependencies. + repository_tool_dependencies = metadata.get( 'tool_dependencies', None ) + repository_installed_tool_dependencies, repository_missing_tool_dependencies = get_installed_and_missing_tool_dependencies( trans, + repository, + repository_tool_dependencies ) if reinstalling: - # All tool dependencies will be considered missing since we are reinstalling the repository. - if installed_tool_dependencies: - for td in installed_tool_dependencies: - missing_tool_dependencies.append( td ) - installed_tool_dependencies = None + installed_tool_dependencies, missing_tool_dependencies = \ + populate_tool_dependencies_dicts( trans=trans, + tool_shed_url=tool_shed_url, + tool_path=tool_path, + repository_installed_tool_dependencies=repository_installed_tool_dependencies, + repository_missing_tool_dependencies=repository_missing_tool_dependencies, + required_repo_info_dicts=required_repo_info_dicts ) + else: + installed_tool_dependencies = repository_installed_tool_dependencies + missing_tool_dependencies = repository_missing_tool_dependencies # Handle valid tools. valid_tools = metadata.get( 'tools', None ) # Handle workflows. workflows = metadata.get( 'workflows', None ) containers_dict = suc.build_repository_containers_for_galaxy( trans=trans, - toolshed_base_url=tool_shed_url, - repository_name=repository.name, - repository_owner=repository.owner, - changeset_revision=repository.installed_changeset_revision, repository=repository, datatypes=datatypes, invalid_tools=invalid_tools, @@ -943,7 +1351,9 @@ repository_dependencies=installed_repository_dependencies, tool_dependencies=installed_tool_dependencies, valid_tools=valid_tools, - workflows=workflows ) + workflows=workflows, + new_install=False, + reinstalling=reinstalling ) else: containers_dict = dict( datatypes=None, invalid_tools=None, @@ -953,6 +1363,90 @@ valid_tools=None, workflows=None ) return containers_dict +def populate_containers_dict_for_new_install( trans, tool_shed_url, tool_path, readme_files_dict, installed_repository_dependencies, missing_repository_dependencies, + installed_tool_dependencies, missing_tool_dependencies ): + """Return the populated containers for a repository being installed for the first time.""" + installed_tool_dependencies, missing_tool_dependencies = populate_tool_dependencies_dicts( trans=trans, + tool_shed_url=tool_shed_url, + tool_path=tool_path, + repository_installed_tool_dependencies=installed_tool_dependencies, + repository_missing_tool_dependencies=missing_tool_dependencies, + required_repo_info_dicts=None ) + # Since we are installing a new repository, most of the repository contents are set to None since we don't yet know what they are. + containers_dict = suc.build_repository_containers_for_galaxy( trans=trans, + repository=None, + datatypes=None, + invalid_tools=None, + missing_repository_dependencies=missing_repository_dependencies, + missing_tool_dependencies=missing_tool_dependencies, + readme_files_dict=readme_files_dict, + repository_dependencies=installed_repository_dependencies, + tool_dependencies=installed_tool_dependencies, + valid_tools=None, + workflows=None, + new_install=True, + reinstalling=False ) + # Merge the missing_repository_dependencies container contents to the installed_repository_dependencies container. + containers_dict = suc.merge_missing_repository_dependencies_to_installed_container( containers_dict ) + # Merge the missing_tool_dependencies container contents to the installed_tool_dependencies container. + containers_dict = suc.merge_missing_tool_dependencies_to_installed_container( containers_dict ) + return containers_dict +def populate_tool_dependencies_dicts( trans, tool_shed_url, tool_path, repository_installed_tool_dependencies, repository_missing_tool_dependencies, + required_repo_info_dicts ): + """ + Return the populated installed_tool_dependencies and missing_tool_dependencies dictionaries for all repositories defined by entries in the received + required_repo_info_dicts. + """ + installed_tool_dependencies = None + missing_tool_dependencies = None + if repository_installed_tool_dependencies is None: + repository_installed_tool_dependencies = {} + else: + # Add the install_dir attribute to the tool_dependencies. + repository_installed_tool_dependencies = suc.add_installation_directories_to_tool_dependencies( trans=trans, + tool_dependencies=repository_installed_tool_dependencies ) + if repository_missing_tool_dependencies is None: + repository_missing_tool_dependencies = {} + else: + # Add the install_dir attribute to the tool_dependencies. + repository_missing_tool_dependencies = suc.add_installation_directories_to_tool_dependencies( trans=trans, + tool_dependencies=repository_missing_tool_dependencies ) + if required_repo_info_dicts: + # Handle the tool dependencies defined for each of the repository's repository dependencies. + for rid in required_repo_info_dicts: + for name, repo_info_tuple in rid.items(): + description, repository_clone_url, changeset_revision, ctx_rev, repository_owner, repository_dependencies, tool_dependencies = \ + suc.get_repo_info_tuple_contents( repo_info_tuple ) + if tool_dependencies: + # Add the install_dir attribute to the tool_dependencies. + tool_dependencies = suc.add_installation_directories_to_tool_dependencies( trans=trans, + tool_dependencies=tool_dependencies ) + # The required_repository may have been installed with a different changeset revision. + required_repository, installed_changeset_revision = repository_was_previously_installed( trans, tool_shed_url, name, repo_info_tuple ) + if required_repository: + required_repository_installed_tool_dependencies, required_repository_missing_tool_dependencies = \ + get_installed_and_missing_tool_dependencies( trans, required_repository, tool_dependencies ) + if required_repository_installed_tool_dependencies: + # Add the install_dir attribute to the tool_dependencies. + required_repository_installed_tool_dependencies = \ + suc.add_installation_directories_to_tool_dependencies( trans=trans, + tool_dependencies=required_repository_installed_tool_dependencies ) + for td_key, td_dict in required_repository_installed_tool_dependencies.items(): + if td_key not in repository_installed_tool_dependencies: + repository_installed_tool_dependencies[ td_key ] = td_dict + if required_repository_missing_tool_dependencies: + # Add the install_dir attribute to the tool_dependencies. + required_repository_missing_tool_dependencies = \ + suc.add_installation_directories_to_tool_dependencies( trans=trans, + tool_dependencies=required_repository_missing_tool_dependencies ) + for td_key, td_dict in required_repository_missing_tool_dependencies.items(): + if td_key not in repository_missing_tool_dependencies: + repository_missing_tool_dependencies[ td_key ] = td_dict + if repository_installed_tool_dependencies: + installed_tool_dependencies = repository_installed_tool_dependencies + if repository_missing_tool_dependencies: + missing_tool_dependencies = repository_missing_tool_dependencies + return installed_tool_dependencies, missing_tool_dependencies def pull_repository( repo, repository_clone_url, ctx_rev ): """Pull changes from a remote repository to a local one.""" commands.pull( suc.get_configured_ui(), repo, source=repository_clone_url, rev=[ ctx_rev ] ) @@ -1114,7 +1608,7 @@ trans.sa_session.add( tool_dependency ) trans.sa_session.flush() return removed, error_message -def repository_was_previously_installed( trans, tool_shed_url, repository_name, repo_info_tuple, clone_dir ): +def repository_was_previously_installed( trans, tool_shed_url, repository_name, repo_info_tuple ): """ Handle the case where the repository was previously installed using an older changeset_revsion, but later the repository was updated in the tool shed and now we're trying to install the latest changeset revision of the same repository instead of updating the one @@ -1132,7 +1626,6 @@ text = response.read() response.close() if text: - #clone_path, clone_directory = os.path.split( clone_dir ) changeset_revisions = util.listify( text ) for previous_changeset_revision in changeset_revisions: tool_shed_repository = suc.get_tool_shed_repository_by_shed_name_owner_installed_changeset_revision( trans.app, This diff is so big that we needed to truncate the remainder. https://bitbucket.org/galaxy/galaxy-central/commits/4e15bce31ca2/ Changeset: 4e15bce31ca2 User: kellrott Date: 2013-03-08 20:44:29 Summary: Adding security to the extended metadata controller and mixin. Affected #: 3 files diff -r d603df6f2e7978f9c75f64190bfba97bbf59e851 -r 4e15bce31ca2a1a85c4931c794638f77e712cbe9 lib/galaxy/model/item_attrs.py --- a/lib/galaxy/model/item_attrs.py +++ b/lib/galaxy/model/item_attrs.py @@ -159,82 +159,6 @@ return getattr( galaxy.model, class_name, None ) -class UsesExtendedMetadata: - """ Mixin for getting and setting item extended metadata. """ - - def get_item_extended_metadata_obj( self, db_session, user, item ): - """ - Given an item object (such as a LibraryDatasetDatasetAssociation), find the object - of the associated extended metadata - """ - - extended_metadata = db_session.query( galaxy.model.ExtendedMetadata ) - - if item.__class__ == galaxy.model.LibraryDatasetDatasetAssociation: - extended_metadata = extended_metadata.join( galaxy.model.LibraryDatasetDatasetAssociation ).filter( - galaxy.model.LibraryDatasetDatasetAssociation.id == item.id ) - - return extended_metadata.first() - - - def set_item_extended_metadata_obj( self, db_session, user, item, extmeta_obj): - if item.__class__ == galaxy.model.LibraryDatasetDatasetAssociation: - item.extended_metadata = extmeta_obj - db_session.flush() - - def unlink_item_extended_metadata_obj( self, db_session, user, item): - if item.__class__ == galaxy.model.LibraryDatasetDatasetAssociation: - item.extended_metadata = None - db_session.flush() - - def create_extended_metadata(self, db_session, user, extmeta): - """ - Create/index an extended metadata object. The returned object is - not associated with any items - """ - ex_meta = galaxy.model.ExtendedMetadata(extmeta) - db_session.add( ex_meta ) - db_session.flush() - for path, value in self._scan_json_block(extmeta): - meta_i = galaxy.model.ExtendedMetadataIndex(ex_meta, path, value) - db_session.add(meta_i) - db_session.flush() - return ex_meta - - def delete_extended_metadata( self, db_session, user, item): - if item.__class__ == galaxy.model.ExtendedMetadata: - db_session.delete( item ) - db_session.flush() - - def _scan_json_block(self, meta, prefix=""): - """ - Scan a json style data structure, and emit all fields and their values. - Example paths - - Data - { "data" : [ 1, 2, 3 ] } - - Path: - /data == [1,2,3] - - /data/[0] == 1 - - """ - if isinstance(meta, dict): - for a in meta: - for path, value in self._scan_json_block(meta[a], prefix + "/" + a): - yield path, value - elif isinstance(meta, list): - for i, a in enumerate(meta): - for path, value in self._scan_json_block(a, prefix + "[%d]" % (i)): - yield path, value - else: - #BUG: Everything is cast to string, which can lead to false positives - #for cross type comparisions, ie "True" == True - yield prefix, ("%s" % (meta)).encode("utf8", errors='replace') - - - class APIItem: """ Mixin for api representation. """ #api_collection_visible_keys = ( 'id' ) diff -r d603df6f2e7978f9c75f64190bfba97bbf59e851 -r 4e15bce31ca2a1a85c4931c794638f77e712cbe9 lib/galaxy/web/base/controller.py --- a/lib/galaxy/web/base/controller.py +++ b/lib/galaxy/web/base/controller.py @@ -1619,6 +1619,85 @@ log.debug( "In get_item_tag_assoc with tagged_item %s" % tagged_item ) return self.get_tag_handler( trans )._get_item_tag_assoc( user, tagged_item, tag_name ) + + +class UsesExtendedMetadataMixin( SharableItemSecurityMixin ): + """ Mixin for getting and setting item extended metadata. """ + + def get_item_extended_metadata_obj( self, trans, item ): + """ + Given an item object (such as a LibraryDatasetDatasetAssociation), find the object + of the associated extended metadata + """ + + extended_metadata = trans.sa_session.query( galaxy.model.ExtendedMetadata ) + + if item.__class__ == galaxy.model.LibraryDatasetDatasetAssociation: + extended_metadata = extended_metadata.join( galaxy.model.LibraryDatasetDatasetAssociation ).filter( + galaxy.model.LibraryDatasetDatasetAssociation.id == item.id ) + + return extended_metadata.first() + + + def set_item_extended_metadata_obj( self, trans, item, extmeta_obj, check_writable=False): + if item.__class__ == galaxy.model.LibraryDatasetDatasetAssociation: + if not check_writable or trans.app.security_agent.can_modify_library_item( trans.get_current_user_roles(), item, trans.user ): + item.extended_metadata = extmeta_obj + trans.sa_session.flush() + + def unset_item_extended_metadata_obj( self, trans, item, check_writable=False): + if item.__class__ == galaxy.model.LibraryDatasetDatasetAssociation: + if not check_writable or trans.app.security_agent.can_modify_library_item( trans.get_current_user_roles(), item, trans.user ): + item.extended_metadata = None + trans.sa_session.flush() + + def create_extended_metadata(self, trans, extmeta): + """ + Create/index an extended metadata object. The returned object is + not associated with any items + """ + ex_meta = galaxy.model.ExtendedMetadata(extmeta) + trans.sa_session.add( ex_meta ) + trans.sa_session.flush() + for path, value in self._scan_json_block(extmeta): + meta_i = galaxy.model.ExtendedMetadataIndex(ex_meta, path, value) + trans.sa_session.add(meta_i) + trans.sa_session.flush() + return ex_meta + + def delete_extended_metadata( self, trans, item): + if item.__class__ == galaxy.model.ExtendedMetadata: + trans.sa_session.delete( item ) + trans.sa_session.flush() + + def _scan_json_block(self, meta, prefix=""): + """ + Scan a json style data structure, and emit all fields and their values. + Example paths + + Data + { "data" : [ 1, 2, 3 ] } + + Path: + /data == [1,2,3] + + /data/[0] == 1 + + """ + if isinstance(meta, dict): + for a in meta: + for path, value in self._scan_json_block(meta[a], prefix + "/" + a): + yield path, value + elif isinstance(meta, list): + for i, a in enumerate(meta): + for path, value in self._scan_json_block(a, prefix + "[%d]" % (i)): + yield path, value + else: + #BUG: Everything is cast to string, which can lead to false positives + #for cross type comparisions, ie "True" == True + yield prefix, ("%s" % (meta)).encode("utf8", errors='replace') + + """ Deprecated: `BaseController` used to be available under the name `Root` """ diff -r d603df6f2e7978f9c75f64190bfba97bbf59e851 -r 4e15bce31ca2a1a85c4931c794638f77e712cbe9 lib/galaxy/webapps/galaxy/api/extended_metadata.py --- a/lib/galaxy/webapps/galaxy/api/extended_metadata.py +++ b/lib/galaxy/webapps/galaxy/api/extended_metadata.py @@ -4,8 +4,7 @@ import logging, os, string, shutil, urllib, re, socket from cgi import escape, FieldStorage from galaxy import util, datatypes, jobs, web, util -from galaxy.web.base.controller import BaseAPIController, UsesHistoryMixin, UsesLibraryMixinItems, UsesHistoryDatasetAssociationMixin, UsesStoredWorkflowMixin -from galaxy.model.item_attrs import UsesExtendedMetadata +from galaxy.web.base.controller import BaseAPIController, UsesHistoryMixin, UsesLibraryMixinItems, UsesHistoryDatasetAssociationMixin, UsesStoredWorkflowMixin, UsesExtendedMetadataMixin from galaxy.util.sanitize_html import sanitize_html import galaxy.datatypes from galaxy.util.bunch import Bunch @@ -16,38 +15,38 @@ log = logging.getLogger( __name__ ) -class BaseExtendedMetadataController( BaseAPIController, UsesExtendedMetadata, UsesHistoryMixin, UsesLibraryMixinItems, UsesHistoryDatasetAssociationMixin, UsesStoredWorkflowMixin ): +class BaseExtendedMetadataController( BaseAPIController, UsesExtendedMetadataMixin, UsesHistoryMixin, UsesLibraryMixinItems, UsesHistoryDatasetAssociationMixin, UsesStoredWorkflowMixin ): @web.expose_api def index( self, trans, **kwd ): idnum = kwd[self.exmeta_item_id] - item = self._get_item_from_id(trans, idnum) + item = self._get_item_from_id(trans, idnum, check_writable=False) if item is not None: - ex_meta = self.get_item_extended_metadata_obj( trans.sa_session, trans.get_user(), item ) + ex_meta = self.get_item_extended_metadata_obj( trans, item ) if ex_meta is not None: return ex_meta.data @web.expose_api def create( self, trans, payload, **kwd ): idnum = kwd[self.exmeta_item_id] - item = self._get_item_from_id(trans, idnum) + item = self._get_item_from_id(trans, idnum, check_writable=True) if item is not None: - ex_obj = self.get_item_extended_metadata_obj(trans.sa_session, trans.get_user(), item) + ex_obj = self.get_item_extended_metadata_obj(trans, item) if ex_obj is not None: - self.unlink_item_extended_metadata_obj(trans.sa_session, trans.get_user(), item) - self.delete_extended_metadata(trans.sa_session, trans.get_user(), ex_obj) - ex_obj = self.create_extended_metadata(trans.sa_session, trans.get_user(), payload) - self.set_item_extended_metadata_obj(trans.sa_session, trans.get_user(), item, ex_obj) + self.unset_item_extended_metadata_obj(trans, item) + self.delete_extended_metadata(trans, ex_obj) + ex_obj = self.create_extended_metadata(trans, payload) + self.set_item_extended_metadata_obj(trans, item, ex_obj) @web.expose_api def delete( self, trans, **kwd ): idnum = kwd[self.tagged_item_id] - item = self._get_item_from_id(trans, idnum) + item = self._get_item_from_id(trans, idnum, check_writable=True) if item is not None: - ex_obj = self.get_item_extended_metadata_obj(trans.sa_session, trans.get_user(), item) + ex_obj = self.get_item_extended_metadata_obj(trans, item) if ex_obj is not None: - self.unlink_item_extended_metadata_obj(trans.sa_session, trans.get_user(), item) - self.delete_extended_metadata(trans.sa_session, trans.get_user(), ex_obj) + self.unset_item_extended_metadata_obj(trans, item) + self.delete_extended_metadata(trans, ex_obj) @web.expose_api def undelete( self, trans, **kwd ): @@ -56,7 +55,13 @@ class LibraryDatasetExtendMetadataController(BaseExtendedMetadataController): controller_name = "library_dataset_extended_metadata" exmeta_item_id = "library_content_id" - def _get_item_from_id(self, trans, idstr): - hist = self.get_library_dataset_dataset_association( trans, idstr ) - return hist - + def _get_item_from_id(self, trans, idstr, check_writable=True): + if check_writable: + item = self.get_library_dataset_dataset_association( trans, idstr) + if trans.app.security_agent.can_modify_library_item( trans.get_current_user_roles(), item ): + return item + else: + item = self.get_library_dataset_dataset_association( trans, idstr) + if trans.app.security_agent.can_access_library_item( trans.get_current_user_roles(), item, trans.user ): + return item + return None https://bitbucket.org/galaxy/galaxy-central/commits/bd61f8f7f90c/ Changeset: bd61f8f7f90c User: kellrott Date: 2013-03-18 04:30:50 Summary: Adding view of extended metadata on the ldda_info page Affected #: 2 files diff -r 4e15bce31ca2a1a85c4931c794638f77e712cbe9 -r bd61f8f7f90c02419d2a50b21da2a04f970a0fb6 lib/galaxy/util/__init__.py --- a/lib/galaxy/util/__init__.py +++ b/lib/galaxy/util/__init__.py @@ -49,6 +49,9 @@ from inflection import Inflector, English inflector = Inflector(English) +pkg_resources.require( "simplejson" ) +import simplejson + def is_multi_byte( chars ): for char in chars: try: @@ -168,6 +171,11 @@ elem.tail = i + pad return elem +def pretty_print_json(json_data, is_json_string=False): + if is_json_string: + json_data = simplejson.loads(json_data) + return simplejson.dumps(json_data, sort_keys=True, indent=4 * ' ') + # characters that are valid valid_chars = set(string.letters + string.digits + " -=_.()/+*^,:?!") diff -r 4e15bce31ca2a1a85c4931c794638f77e712cbe9 -r bd61f8f7f90c02419d2a50b21da2a04f970a0fb6 templates/library/common/ldda_info.mako --- a/templates/library/common/ldda_info.mako +++ b/templates/library/common/ldda_info.mako @@ -167,6 +167,13 @@ </div></div> %endif + %if ldda.extended_metadata: + <div class="form-row"> + <label>Extended Metadata:</label> + <pre>${util.pretty_print_json(ldda.extended_metadata.data)}</pre> + <div style="clear: both"></div> + </div> + %endif %if trans.user_is_admin() and cntrller == 'library_admin': <div class="form-row"><label>Disk file:</label> Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: carlfeberhard: Visualization framework: allow plugins to be located outside Galaxy root; fix logging import
by commits-noreply＠bitbucket.org 06 Aug '13

06 Aug '13

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/48fd0ab87841/ Changeset: 48fd0ab87841 User: carlfeberhard Date: 2013-08-06 21:40:17 Summary: Visualization framework: allow plugins to be located outside Galaxy root; fix logging import Affected #: 2 files diff -r 73c9018fcafb9beb2f9fb76bae0a01e1c641b913 -r 48fd0ab87841725fe85726606e187cb728ce9a21 lib/galaxy/visualization/registry.py --- a/lib/galaxy/visualization/registry.py +++ b/lib/galaxy/visualization/registry.py @@ -48,6 +48,7 @@ - validating and parsing params into resources (based on a context) used in the visualization template """ + #: name of this plugin #: any built in visualizations that have their own render method in ctrls/visualization # these should be handled somewhat differently - and be passed onto their resp. methods in ctrl.visualization #TODO: change/remove if/when they can be updated to use this system @@ -57,9 +58,6 @@ 'sweepster', 'phyloviz' ] - #: where to search for visualiztion templates (relative to templates/webapps/galaxy) - # this can be overridden individually in the config entries - TEMPLATE_ROOT = 'visualization' #: directories under plugin_directory that aren't plugins non_plugin_directories = [] @@ -67,7 +65,7 @@ return 'VisualizationsRegistry(%s)' %( self.plugin_directory ) def __init__( self, registry_filepath, template_cache_dir ): - super( VisualizationsRegistry, self ).__init__( registry_filepath, template_cache_dir ) + super( VisualizationsRegistry, self ).__init__( registry_filepath, 'visualizations', template_cache_dir ) # what to use to parse query strings into resources/vars for the template self.resource_parser = ResourceParser() diff -r 73c9018fcafb9beb2f9fb76bae0a01e1c641b913 -r 48fd0ab87841725fe85726606e187cb728ce9a21 lib/galaxy/web/base/pluginframework.py --- a/lib/galaxy/web/base/pluginframework.py +++ b/lib/galaxy/web/base/pluginframework.py @@ -14,6 +14,8 @@ pkg_resources.require( 'Mako' ) import mako +import logging +log = logging.getLogger( __name__ ) # ============================================================================= exceptions class PluginFrameworkException( Exception ): @@ -74,7 +76,13 @@ if not config_plugin_directory: return None try: + # create the plugin path and if plugin dir begins with '/' assume absolute path full_plugin_filepath = os.path.join( config.root, config_plugin_directory ) + if config_plugin_directory.startswith( os.path.sep ): + full_plugin_filepath = config_plugin_directory + if not os.path.exists( full_plugin_filepath ): + raise PluginFrameworkException( 'Plugin path not found: %s' %( full_plugin_filepath ) ) + template_cache = config.template_cache if cls.serves_static else None plugin = cls( full_plugin_filepath, template_cache ) @@ -88,13 +96,22 @@ def __str__( self ): return '%s(%s)' %( self.__class__.__name__, self.plugin_directory ) - def __init__( self, plugin_directory, template_cache_dir=None, debug=False ): + def __init__( self, plugin_directory, name=None, template_cache_dir=None, debug=False ): + """ + :type plugin_directory: string + :param plugin_directory: the base directory where plugin code is kept + :type name: (optional) string (default: None) + :param name: the name of this plugin + (that will appear in url pathing, etc.) + :type template_cache_dir: (optional) string (default: None) + :param template_cache_dir: the cache directory to store compiled mako + """ if not os.path.isdir( plugin_directory ): raise PluginFrameworkException( 'Framework plugin directory not found: %s, %s' %( self.__class__.__name__, plugin_directory ) ) - # absolute (?) path self.plugin_directory = plugin_directory - self.name = os.path.basename( self.plugin_directory ) + #TODO: or pass in from config + self.name = name or os.path.basename( self.plugin_directory ) if self.has_config: self.load_configuration() Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: carlfeberhard: Remove debugging code
by commits-noreply＠bitbucket.org 06 Aug '13

06 Aug '13

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/73c9018fcafb/ Changeset: 73c9018fcafb User: carlfeberhard Date: 2013-08-06 21:09:39 Summary: Remove debugging code Affected #: 1 file diff -r db11479ab7584322519463a33b42f4b911fb1a68 -r 73c9018fcafb9beb2f9fb76bae0a01e1c641b913 lib/galaxy/visualization/registry.py --- a/lib/galaxy/visualization/registry.py +++ b/lib/galaxy/visualization/registry.py @@ -61,7 +61,7 @@ # this can be overridden individually in the config entries TEMPLATE_ROOT = 'visualization' #: directories under plugin_directory that aren't plugins - non_plugin_directories = [ 'bler' ] + non_plugin_directories = [] def __str__( self ): return 'VisualizationsRegistry(%s)' %( self.plugin_directory ) Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: 2 new changesets
by commits-noreply＠bitbucket.org 06 Aug '13

06 Aug '13

2 new commits in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/6a8bb831b771/ Changeset: 6a8bb831b771 User: carlfeberhard Date: 2013-08-06 21:02:10 Summary: Visualizations framework: PluginFramework class, serving static and template files from plugins Affected #: 19 files diff -r f760fca38915be8d0ae8f94e703a6d83302065f8 -r 6a8bb831b771b56e79fff3fccfe610d0bf01b885 .hgignore --- a/.hgignore +++ b/.hgignore @@ -60,7 +60,7 @@ job_conf.xml data_manager_conf.xml shed_data_manager_conf.xml -config/visualizations/*.xml +config/* static/welcome.html.* static/welcome.html diff -r f760fca38915be8d0ae8f94e703a6d83302065f8 -r 6a8bb831b771b56e79fff3fccfe610d0bf01b885 config/plugins/visualizations/README.txt --- /dev/null +++ b/config/plugins/visualizations/README.txt @@ -0,0 +1,34 @@ +Custom visualization plugins +---------------------------- + +Visualizations can be added to your Galaxy instance by creating +sub-directories, templates, and static files here. + +Properly configured and written visualizations will be accessible to +the user when they click the 'visualizations' icon for a dataset +in their history panel. + +The framework must be enabled in your 'universe_wsgi.ini' file by +uncommenting (and having a valid path for) the +'visualizations_plugin_directory' entry. + +For more information, see http://wiki.galaxyproject.org/VisualizationsRegistry + + +Sub-directory structure +----------------------- + +In general, sub-directories should follow the pattern: + + my_visualization/ + config/ + my_visualization.xml + static/ + ... any static files the visualization needs (if any) + templates/ + ... any Mako templates the visualization needs + +The XML config file for a visualization plugin can be validated on the command +line using (from your plugin directory): + + xmllint my_visualization/config/my_visualization.xml --valid --noout diff -r f760fca38915be8d0ae8f94e703a6d83302065f8 -r 6a8bb831b771b56e79fff3fccfe610d0bf01b885 config/plugins/visualizations/visualization.dtd --- /dev/null +++ b/config/plugins/visualizations/visualization.dtd @@ -0,0 +1,130 @@ + +<!ELEMENT visualization (data_sources*,params*,template_root*,template,link_text*,render_location*)> + +<!ATTLIST visualization + name CDATA #REQUIRED +> + +<!ELEMENT data_sources (data_source*)> + + <!ELEMENT data_source (model_class,(test|to_param)*)> + <!ELEMENT model_class (#PCDATA)> +  + <!ELEMENT test (#PCDATA)> +  + <!ATTLIST test + type CDATA #IMPLIED + test_attr CDATA #IMPLIED + result_type CDATA #IMPLIED + > + + <!ELEMENT to_param (#PCDATA)> +  + <!ATTLIST to_param + param_attr CDATA #IMPLIED + assign CDATA #IMPLIED + > + +<!ELEMENT params ((param|param_modifier)*)> + + <!ELEMENT param (#PCDATA)> +  + <!ATTLIST param + type CDATA #IMPLIED + default CDATA #IMPLIED + required CDATA #IMPLIED + csv CDATA #IMPLIED + constrain_to CDATA #IMPLIED + var_name_in_template CDATA #IMPLIED + > +  + <!ELEMENT param_modifier (#PCDATA)> + <!ATTLIST param_modifier + modifies CDATA #REQUIRED + type CDATA #IMPLIED + default CDATA #IMPLIED + required CDATA #IMPLIED + csv CDATA #IMPLIED + constrain_to CDATA #IMPLIED + var_name_in_template CDATA #IMPLIED + > + + +<!ELEMENT template_root (#PCDATA)> + +<!ELEMENT template (#PCDATA)> + +<!ELEMENT link_text (#PCDATA)> + + +<!ELEMENT render_location (#PCDATA)> diff -r f760fca38915be8d0ae8f94e703a6d83302065f8 -r 6a8bb831b771b56e79fff3fccfe610d0bf01b885 config/visualizations/circster.xml.sample --- a/config/visualizations/circster.xml.sample +++ /dev/null @@ -1,26 +0,0 @@ -<?xml version="1.0" encoding="UTF-8"?> -<!DOCTYPE visualization SYSTEM "visualization.dtd"> -<visualization name="circster"> - <data_sources> - <data_source> - <model_class>HistoryDatasetAssociation</model_class> - <test type="isinstance" test_attr="datatype" result_type="datatype">data.Data</test> - <to_param param_attr="id">dataset_id</to_param> - <to_param assign="hda">hda_ldda</to_param> - </data_source> - <data_source> - <model_class>LibraryDatasetDatasetAssociation</model_class> - <test type="isinstance" test_attr="datatype" result_type="datatype">data.Data</test> - <to_param param_attr="id">dataset_id</to_param> - <to_param assign="ldda">hda_ldda</to_param> - </data_source> - </data_sources> - <params> - <param type="visualization">id</param> - <param type="hda_or_ldda">dataset_id</param> - <param_modifier type="string" modifies="dataset_id">hda_ldda</param_modifier> - <param type="dbkey">dbkey</param> - </params> - <template>circster.mako</template> - <render_location>_top</render_location> -</visualization> diff -r f760fca38915be8d0ae8f94e703a6d83302065f8 -r 6a8bb831b771b56e79fff3fccfe610d0bf01b885 config/visualizations/phyloviz.xml.sample --- a/config/visualizations/phyloviz.xml.sample +++ /dev/null @@ -1,18 +0,0 @@ -<?xml version="1.0" encoding="UTF-8"?> -<!DOCTYPE visualization SYSTEM "visualization.dtd"> -<visualization name="phyloviz"> - <data_sources> - <data_source> - <model_class>HistoryDatasetAssociation</model_class> - <test type="isinstance" test_attr="datatype" result_type="datatype">data.Newick</test> - <test type="isinstance" test_attr="datatype" result_type="datatype">data.Nexus</test> - <to_param param_attr="id">dataset_id</to_param> - </data_source> - </data_sources> - <params> - <param type="dataset" var_name_in_template="hda" required="true">dataset_id</param> - <param type="integer" default="0">tree_index</param> - </params> - <template>phyloviz.mako</template> - <render_location>_top</render_location> -</visualization> diff -r f760fca38915be8d0ae8f94e703a6d83302065f8 -r 6a8bb831b771b56e79fff3fccfe610d0bf01b885 config/visualizations/scatterplot.xml.sample --- a/config/visualizations/scatterplot.xml.sample +++ /dev/null @@ -1,15 +0,0 @@ -<?xml version="1.0" encoding="UTF-8"?> -<!DOCTYPE visualization SYSTEM "visualization.dtd"> -<visualization name="scatterplot"> - <data_sources> - <data_source> - <model_class>HistoryDatasetAssociation</model_class> - <test type="isinstance" test_attr="datatype" result_type="datatype">tabular.Tabular</test> - <to_param param_attr="id">dataset_id</to_param> - </data_source> - </data_sources> - <params> - <param type="dataset" var_name_in_template="hda" required="true">dataset_id</param> - </params> - <template>scatterplot.mako</template> -</visualization> diff -r f760fca38915be8d0ae8f94e703a6d83302065f8 -r 6a8bb831b771b56e79fff3fccfe610d0bf01b885 config/visualizations/sweepster.xml.sample --- a/config/visualizations/sweepster.xml.sample +++ /dev/null @@ -1,25 +0,0 @@ -<?xml version="1.0" encoding="UTF-8"?> -<!DOCTYPE visualization SYSTEM "visualization.dtd"> -<visualization name="sweepster"> - <data_sources> - <data_source> - <model_class>HistoryDatasetAssociation</model_class> - <test type="isinstance" test_attr="datatype" result_type="datatype">data.Data</test> - <to_param param_attr="id">dataset_id</to_param> - <to_param assign="hda">hda_ldda</to_param> - </data_source> - <data_source> - <model_class>LibraryDatasetDatasetAssociation</model_class> - <test type="isinstance" test_attr="datatype" result_type="datatype">data.Data</test> - <to_param param_attr="id">dataset_id</to_param> - <to_param assign="ldda">hda_ldda</to_param> - </data_source> - </data_sources> - <params> - <param type="visualization" var_name_in_template="viz">visualization</param> - <param type="hda_or_ldda" var_name_in_template="dataset">dataset_id</param> - <param_modifier type="string" modifies="dataset_id">hda_ldda</param_modifier> - </params> - <template>sweepster.mako</template> - <render_location>_top</render_location> -</visualization> diff -r f760fca38915be8d0ae8f94e703a6d83302065f8 -r 6a8bb831b771b56e79fff3fccfe610d0bf01b885 config/visualizations/trackster.xml.sample --- a/config/visualizations/trackster.xml.sample +++ /dev/null @@ -1,29 +0,0 @@ -<?xml version="1.0" encoding="UTF-8"?> -<!DOCTYPE visualization SYSTEM "visualization.dtd"> -<visualization name="trackster"> -  - <data_sources> - <data_source> - <model_class>HistoryDatasetAssociation</model_class> - <test type="isinstance" test_attr="datatype" result_type="datatype">data.Data</test> - <to_param param_attr="id">dataset_id</to_param> - <to_param assign="hda">hda_ldda</to_param> - <to_param param_attr="dbkey">dbkey</to_param> - </data_source> - <data_source> - <model_class>LibraryDatasetDatasetAssociation</model_class> - <test type="isinstance" test_attr="datatype" result_type="datatype">data.Data</test> - <to_param param_attr="id">dataset_id</to_param> - <to_param assign="ldda">hda_ldda</to_param> - </data_source> - </data_sources> - <params> - <param type="visualization">id</param> - <param type="dataset">dataset_id</param> - <param type="genome_region">genome_region</param> - <param type="dbkey">dbkey</param> - </params> - <template_root>tracks</template_root> - <template>browser.mako</template> - <render_location>_top</render_location> -</visualization> diff -r f760fca38915be8d0ae8f94e703a6d83302065f8 -r 6a8bb831b771b56e79fff3fccfe610d0bf01b885 config/visualizations/visualization.dtd --- a/config/visualizations/visualization.dtd +++ /dev/null @@ -1,132 +0,0 @@ - - - -<!ELEMENT visualization (data_sources*,params*,template_root*,template,link_text*,render_location*)> - -<!ATTLIST visualization - name CDATA #REQUIRED -> - -<!ELEMENT data_sources (data_source*)> - - <!ELEMENT data_source (model_class,(test|to_param)*)> - <!ELEMENT model_class (#PCDATA)> -  - <!ELEMENT test (#PCDATA)> -  - <!ATTLIST test - type CDATA #IMPLIED - test_attr CDATA #IMPLIED - result_type CDATA #IMPLIED - > - - <!ELEMENT to_param (#PCDATA)> -  - <!ATTLIST to_param - param_attr CDATA #IMPLIED - assign CDATA #IMPLIED - > - -<!ELEMENT params ((param|param_modifier)*)> - - <!ELEMENT param (#PCDATA)> -  - <!ATTLIST param - type CDATA #IMPLIED - default CDATA #IMPLIED - required CDATA #IMPLIED - csv CDATA #IMPLIED - constrain_to CDATA #IMPLIED - var_name_in_template CDATA #IMPLIED - > -  - <!ELEMENT param_modifier (#PCDATA)> - <!ATTLIST param_modifier - modifies CDATA #REQUIRED - type CDATA #IMPLIED - default CDATA #IMPLIED - required CDATA #IMPLIED - csv CDATA #IMPLIED - constrain_to CDATA #IMPLIED - var_name_in_template CDATA #IMPLIED - > - - -<!ELEMENT template_root (#PCDATA)> - -<!ELEMENT template (#PCDATA)> - -<!ELEMENT link_text (#PCDATA)> - - -<!ELEMENT render_location (#PCDATA)> diff -r f760fca38915be8d0ae8f94e703a6d83302065f8 -r 6a8bb831b771b56e79fff3fccfe610d0bf01b885 lib/galaxy/app.py --- a/lib/galaxy/app.py +++ b/lib/galaxy/app.py @@ -123,10 +123,8 @@ # Load genome indexer tool. load_genome_index_tools( self.toolbox ) # visualizations registry: associates resources with visualizations, controls how to render - self.visualizations_registry = None - if self.config.visualizations_config_directory: - self.visualizations_registry = VisualizationsRegistry( self.config.root, - self.config.visualizations_config_directory ) + self.visualizations_registry = VisualizationsRegistry.from_config( + self.config.visualizations_plugins_directory, self.config ) # Load security policy. self.security_agent = self.model.security_agent self.host_security_agent = galaxy.security.HostAgent( model=self.security_agent.model, permitted_actions=self.security_agent.permitted_actions ) diff -r f760fca38915be8d0ae8f94e703a6d83302065f8 -r 6a8bb831b771b56e79fff3fccfe610d0bf01b885 lib/galaxy/config.py --- a/lib/galaxy/config.py +++ b/lib/galaxy/config.py @@ -291,8 +291,10 @@ self.fluent_log = string_as_bool( kwargs.get( 'fluent_log', False ) ) self.fluent_host = kwargs.get( 'fluent_host', 'localhost' ) self.fluent_port = int( kwargs.get( 'fluent_port', 24224 ) ) - # visualization registries config directory - self.visualizations_config_directory = kwargs.get( 'visualizations_config_directory', None ) + # PLUGINS: + self.plugin_frameworks = [] + # visualization framework + self.visualizations_plugins_directory = kwargs.get( 'visualizations_plugins_directory', None ) @property def sentry_dsn_public( self ): diff -r f760fca38915be8d0ae8f94e703a6d83302065f8 -r 6a8bb831b771b56e79fff3fccfe610d0bf01b885 lib/galaxy/visualization/registry.py --- a/lib/galaxy/visualization/registry.py +++ b/lib/galaxy/visualization/registry.py @@ -12,6 +12,8 @@ import galaxy.model from galaxy.web import url_for +from galaxy.web.base import pluginframework + import logging log = logging.getLogger( __name__ ) @@ -28,17 +30,16 @@ tests: anding, grouping, not has_dataprovider + user is admin data_sources: lists of add description element to visualization. -TESTS to add: - has dataprovider - user is admin +user_pref for ordering/ex/inclusion of particular visualizations """ # ------------------------------------------------------------------- the registry -class VisualizationsRegistry( object ): +class VisualizationsRegistry( pluginframework.PluginFramework ): """ Main responsibilities are: - testing if an object has a visualization that can be applied to it @@ -47,6 +48,7 @@ - validating and parsing params into resources (based on a context) used in the visualization template """ + #: any built in visualizations that have their own render method in ctrls/visualization # these should be handled somewhat differently - and be passed onto their resp. methods in ctrl.visualization #TODO: change/remove if/when they can be updated to use this system BUILT_IN_VISUALIZATIONS = [ @@ -55,57 +57,37 @@ 'sweepster', 'phyloviz' ] - # where to search for visualiztion templates (relative to templates/webapps/galaxy) + #: where to search for visualiztion templates (relative to templates/webapps/galaxy) # this can be overridden individually in the config entries TEMPLATE_ROOT = 'visualization' + #: directories under plugin_directory that aren't plugins + non_plugin_directories = [ 'bler' ] def __str__( self ): - listings_keys_str = ','.join( self.listings.keys() ) if self.listings else '' - return 'VisualizationsRegistry(%s)' %( listings_keys_str ) + return 'VisualizationsRegistry(%s)' %( self.plugin_directory ) - def __init__( self, galaxy_root, configuration_filepath ): - # load the registry from the xml files located in configuration_filepath using the given parser - configuration_filepath = os.path.join( galaxy_root, configuration_filepath ) - self.configuration_filepath = self.check_conf_filepath( configuration_filepath ) - self.move_sample_conf_files() - self.load() + def __init__( self, registry_filepath, template_cache_dir ): + super( VisualizationsRegistry, self ).__init__( registry_filepath, template_cache_dir ) # what to use to parse query strings into resources/vars for the template self.resource_parser = ResourceParser() + log.debug( '%s loaded', str( self ) ) - def check_conf_filepath( self, configuration_filepath ): + def load_configuration( self ): """ - Checks for the existence of the given filepath. - :param configurarion_filepath: full filepath to the visualization config directory - :raises IOError: if the given directory doesn't exist + Builds the registry by parsing the `config/*.xml` files for every plugin + in ``get_plugin_directories`` and stores the results in ``self.listings``. + + ..note:: + This could be used to re-load a new configuration without restarting + the instance. """ - if not os.path.exists( configuration_filepath ): - raise IOError( 'visualization configuration directory (%s) not found' %( configuration_filepath ) ) - return configuration_filepath + try: + self.listings = VisualizationsConfigParser.parse( self.get_plugin_directories() ) - def move_sample_conf_files( self ): - """ - Copies any `*.xml.sample` files in `configuration_filepath` to - `.xml` files of the same names if no file with that name already exists. - - :returns: a list of the files moved - """ - files_moved = [] - for sample_file in glob.glob( os.path.join( self.configuration_filepath, '*.sample' ) ): - new_name = os.path.splitext( sample_file )[0] - if not os.path.exists( new_name ): - shutil.copy2( sample_file, new_name ) - files_moved.append( new_name ) - - def load( self ): - """ - Builds the registry by parsing the xml in `self.configuration_filepath` - and stores the results in `self.listings`. - - Provided as separate method from `__init__` in order to re-load a - new configuration without restarting the instance. - """ - self.listings = VisualizationsConfigParser.parse( self.configuration_filepath ) + except Exception, exc: + log.exception( 'Error parsing visualizations plugins %s', self.plugin_directory ) + raise def get_visualization( self, trans, visualization_name, target_object ): """ @@ -283,11 +265,11 @@ VALID_RENDER_LOCATIONS = [ 'galaxy_main', '_top', '_blank' ] @classmethod - def parse( cls, config_dir, debug=True ): + def parse( cls, plugin_directories, debug=False ): """ Static class interface. """ - return cls( debug ).parse_files( config_dir ) + return cls( debug ).parse_plugins( plugin_directories ) def __init__( self, debug=False ): self.debug = debug @@ -297,33 +279,45 @@ self.param_parser = ParamParser() self.param_modifier_parser = ParamModifierParser() - def parse_files( self, config_dir ): + def parse_plugins( self, plugin_directories ): """ - Parse each XML file in `config_dir` for visualizations config data. + Parses the config files for each plugin sub-dir in `base_path`. + + :param plugin_directories: a list of paths to enabled plugins. + + :returns: registry data in dictionary form + """ + returned = {} + for plugin_path in plugin_directories: + returned.update( self.parse_plugin( plugin_path ) ) + return returned + + def parse_plugin( self, plugin_path ): + """ + Parses any XML files in ``<plugin_path>/config``. If an error occurs while parsing a visualizations entry, it is skipped. :returns: registry data in dictionary form + ..note:: + assumes config files are in a 'config' sub-dir of each plugin """ returned = {} - try: - for xml_filepath in glob.glob( os.path.join( config_dir, '*.xml' ) ): - try: - visualization_name, visualization = self.parse_file( xml_filepath ) - # skip vis' with parsing errors - don't shutdown the startup - except ParsingException, parse_exc: - log.error( 'Skipped visualization config "%s" due to parsing errors: %s', - xml_filepath, str( parse_exc ), exc_info=self.debug ) - if visualization: - returned[ visualization_name ] = visualization - log.debug( 'Visualization config loaded for: %s', visualization_name ) + plugin_config_path = os.path.join( plugin_path, 'config' ) + if not os.path.isdir( plugin_config_path ): + return returned - except Exception, exc: - log.error( 'Error parsing visualizations configuration directory %s: %s', - config_dir, str( exc ), exc_info=( not self.debug ) ) - #TODO: change when this framework is on by default - if self.debug: - raise + for xml_filepath in glob.glob( os.path.join( plugin_config_path, '*.xml' ) ): + try: + visualization_name, visualization = self.parse_file( xml_filepath ) + # skip vis' with parsing errors - don't shutdown the startup + except ParsingException, parse_exc: + log.error( 'Skipped visualization config "%s" due to parsing errors: %s', + xml_filepath, str( parse_exc ), exc_info=self.debug ) + + if visualization: + returned[ visualization_name ] = visualization + log.debug( 'Visualization config loaded for: %s', visualization_name ) return returned diff -r f760fca38915be8d0ae8f94e703a6d83302065f8 -r 6a8bb831b771b56e79fff3fccfe610d0bf01b885 lib/galaxy/web/base/pluginframework.py --- /dev/null +++ b/lib/galaxy/web/base/pluginframework.py @@ -0,0 +1,216 @@ +""" +Base class for plugins - frameworks or systems that may: + * serve static content + * serve templated html + * have some configuration at startup +""" + +import os.path +import glob +import sys + +import pkg_resources +pkg_resources.require( 'MarkupSafe' ) +pkg_resources.require( 'Mako' ) +import mako + + +# ============================================================================= exceptions +class PluginFrameworkException( Exception ): + """Base exception for plugin frameworks. + """ + pass +class PluginFrameworkConfigException( PluginFrameworkException ): + """Exception for plugin framework configuration errors. + """ + pass +class PluginFrameworkStaticException( PluginFrameworkException ): + """Exception for plugin framework static directory set up errors. + """ + pass +class PluginFrameworkTemplateException( PluginFrameworkException ): + """Exception for plugin framework template directory + and template rendering errors. + """ + pass + + +# ============================================================================= base +class PluginFramework( object ): + """ + Plugins are files/directories living outside the Galaxy ``lib`` directory + that serve static files (css, js, images, etc.), use and serve mako templates, + and have some configuration to control the rendering. + + A plugin framework sets up all the above components. + """ + #: does the class need a config file(s) to be parsed? + has_config = True + #: does the class need static files served? + serves_static = True + #: does the class need template files served? + serves_templates = True + #TODO: allow plugin mako inheritance from existing ``/templates`` files + #uses_galaxy_templates = True + #TODO: possibly better as instance var (or a combo) + #: the directories in ``plugin_directory`` with basenames listed here will + #: be ignored for config, static, and templates + non_plugin_directories = [] + + # ------------------------------------------------------------------------- setup + @classmethod + def from_config( cls, config_plugin_directory, config ): + """ + Set up the framework based on data from some config object by: + * constructing it's absolute plugin_directory filepath + * getting a template_cache + * and appending itself to the config object's ``plugin_frameworks`` list + + .. note:: + precondition: config obj should have attributes: + root, template_cache, and (list) plugin_frameworks + """ + # currently called from (base) app.py - defined here to allow override if needed + if not config_plugin_directory: + return None + try: + full_plugin_filepath = os.path.join( config.root, config_plugin_directory ) + template_cache = config.template_cache if cls.serves_static else None + plugin = cls( full_plugin_filepath, template_cache ) + + config.plugin_frameworks.append( plugin ) + return plugin + + except PluginFrameworkException, plugin_exc: + log.exception( "Error loading framework %s. Skipping...", cls.__class__.__name__ ) + return None + + def __str__( self ): + return '%s(%s)' %( self.__class__.__name__, self.plugin_directory ) + + def __init__( self, plugin_directory, template_cache_dir=None, debug=False ): + if not os.path.isdir( plugin_directory ): + raise PluginFrameworkException( 'Framework plugin directory not found: %s, %s' + %( self.__class__.__name__, plugin_directory ) ) + # absolute (?) path + self.plugin_directory = plugin_directory + self.name = os.path.basename( self.plugin_directory ) + + if self.has_config: + self.load_configuration() + # set_up_static_urls will be called during the static middleware creation (if serves_static) + if self.serves_templates: + self.set_up_templates( template_cache_dir ) + + def get_plugin_directories( self ): + """ + Return the plugin directory paths for this plugin. + + Gets any directories within ``plugin_directory`` that are directories + themselves and whose ``basename`` is not in ``plugin_directory``. + """ + # could instead explicitly list on/off in master config file + for plugin_path in glob.glob( os.path.join( self.plugin_directory, '*' ) ): + if not os.path.isdir( plugin_path ): + continue + + if os.path.basename( plugin_path ) in self.non_plugin_directories: + continue + + yield plugin_path + + # ------------------------------------------------------------------------- config + def load_configuration( self ): + """ + Override to load some framework/plugin specifc configuration. + """ + # Abstract method + return True + + # ------------------------------------------------------------------------- serving static files + def get_static_urls_and_paths( self ): + """ + For each plugin, return a 2-tuple where the first element is a url path + to the plugin's static files and the second is a filesystem path to those + same files. + + Meant to be passed to a Static url map. + """ + url_and_paths = [] + # called during the static middleware creation (buildapp.py, wrap_in_static) + + # NOTE: this only searches for static dirs two levels deep (i.e. <plugin_directory>/<plugin-name>/static) + for plugin_path in self.get_plugin_directories(): + # that path is a plugin, search for subdirs named static in THAT dir + plugin_static_path = os.path.join( plugin_path, 'static' ) + if not os.path.isdir( plugin_static_path ): + continue + + # build a url for that static subdir and create a Static urlmap entry for it + plugin_name = os.path.splitext( os.path.basename( plugin_path ) )[0] + plugin_url = self.name + '/' + plugin_name + '/static' + url_and_paths.append( ( plugin_url, plugin_static_path ) ) + + return url_and_paths + + # ------------------------------------------------------------------------- templates + def set_up_templates( self, template_cache_dir ): + """ + Add a ``template_lookup`` attribute to the framework that can be passed + to the mako renderer to find templates. + """ + if not template_cache_dir: + raise PluginFrameworkTemplateException( 'Plugins that serve templates require a template_cache_dir' ) + self.template_lookup = self._create_mako_template_lookup( template_cache_dir, self._get_template_paths() ) + return self.template_lookup + + def _get_template_paths( self ): + """ + Get the paths that will be searched for templates. + """ + return [ self.plugin_directory ] + + def _create_mako_template_lookup( self, cache_dir, paths, collection_size=500, output_encoding='utf-8' ): + """ + Create a ``TemplateLookup`` with defaults. + """ + return mako.lookup.TemplateLookup( + directories = paths, + module_directory = cache_dir, + collection_size = collection_size, + output_encoding = output_encoding ) + + #TODO: do we want to remove trans and app from the plugin template context? + def fill_template( self, trans, template_filename, **kwargs ): + """ + Pass control over to trans and render the ``template_filename``. + """ + # defined here to be overridden + return trans.fill_template( template_filename, template_lookup=self.template_lookup, **kwargs ) + + def fill_template_with_plugin_imports( self, trans, template_filename, **kwargs ): + """ + Returns a rendered plugin template but allows importing modules from inside + the plugin directory within the template. + + ..example:: I.e. given this layout for a plugin: + bler/ + template/ + bler.mako + static/ + conifg/ + my_script.py + this version of `fill_template` allows `bler.mako` to call `import my_script`. + """ + try: + plugin_base_path = os.path.split( os.path.dirname( template_filename ) )[0] + plugin_path = os.path.join( self.plugin_directory, plugin_base_path ) + sys.path.append( plugin_path ) + filled_template = self.fill_template( trans, template_filename, **kwargs ) + + finally: + sys.path.remove( plugin_path ) + + return filled_template + + #TODO: could add plugin template helpers here diff -r f760fca38915be8d0ae8f94e703a6d83302065f8 -r 6a8bb831b771b56e79fff3fccfe610d0bf01b885 lib/galaxy/web/framework/__init__.py --- a/lib/galaxy/web/framework/__init__.py +++ b/lib/galaxy/web/framework/__init__.py @@ -993,10 +993,13 @@ searchList=[kwargs, self.template_context, dict(caller=self, t=self, h=helpers, util=util, request=self.request, response=self.response, app=self.app)] ) return str( template ) - def fill_template_mako( self, filename, **kwargs ): - template = self.webapp.mako_template_lookup.get_template( filename ) + def fill_template_mako( self, filename, template_lookup=None, **kwargs ): + template_lookup = template_lookup or self.webapp.mako_template_lookup + template = template_lookup.get_template( filename ) template.output_encoding = 'utf-8' - data = dict( caller=self, t=self, trans=self, h=helpers, util=util, request=self.request, response=self.response, app=self.app ) + + data = dict( caller=self, t=self, trans=self, h=helpers, util=util, + request=self.request, response=self.response, app=self.app ) data.update( self.template_context ) data.update( kwargs ) return template.render( **data ) diff -r f760fca38915be8d0ae8f94e703a6d83302065f8 -r 6a8bb831b771b56e79fff3fccfe610d0bf01b885 lib/galaxy/web/framework/middleware/static.py --- a/lib/galaxy/web/framework/middleware/static.py +++ b/lib/galaxy/web/framework/middleware/static.py @@ -12,9 +12,11 @@ from paste.urlparser import StaticURLParser class CacheableStaticURLParser( StaticURLParser ): + def __init__( self, directory, cache_seconds=None ): StaticURLParser.__init__( self, directory ) self.cache_seconds = cache_seconds + def __call__( self, environ, start_response ): path_info = environ.get('PATH_INFO', '') if not path_info: diff -r f760fca38915be8d0ae8f94e703a6d83302065f8 -r 6a8bb831b771b56e79fff3fccfe610d0bf01b885 lib/galaxy/webapps/galaxy/buildapp.py --- a/lib/galaxy/webapps/galaxy/buildapp.py +++ b/lib/galaxy/webapps/galaxy/buildapp.py @@ -2,22 +2,26 @@ Provides factory methods to assemble the Galaxy web application """ -import logging, atexit -import os, os.path -import sys, warnings - -from galaxy.util import asbool +import sys +import os +import os.path +import atexit +import warnings +import glob from paste import httpexceptions import pkg_resources -log = logging.getLogger( __name__ ) - -from galaxy import util import galaxy.model import galaxy.model.mapping import galaxy.datatypes.registry import galaxy.web.framework +from galaxy import util +from galaxy.util import asbool + +import logging +log = logging.getLogger( __name__ ) + class GalaxyWebApplication( galaxy.web.framework.WebApplication ): pass @@ -181,7 +185,7 @@ if kwargs.get( 'middleware', True ): webapp = wrap_in_middleware( webapp, global_conf, **kwargs ) if asbool( kwargs.get( 'static_enabled', True ) ): - webapp = wrap_in_static( webapp, global_conf, **kwargs ) + webapp = wrap_in_static( webapp, global_conf, plugin_frameworks=app.config.plugin_frameworks, **kwargs ) if asbool(kwargs.get('pack_scripts', False)): pack_scripts() # Close any pooled database connections before forking @@ -323,7 +327,7 @@ log.debug( "Enabling 'Request ID' middleware" ) return app -def wrap_in_static( app, global_conf, **local_conf ): +def wrap_in_static( app, global_conf, plugin_frameworks=None, **local_conf ): from paste.urlmap import URLMap from galaxy.web.framework.middleware.static import CacheableStaticURLParser as Static urlmap = URLMap() @@ -343,6 +347,16 @@ urlmap["/static/style"] = Static( conf.get( "static_style_dir" ), cache_time ) urlmap["/favicon.ico"] = Static( conf.get( "static_favicon_dir" ), cache_time ) urlmap["/robots.txt"] = Static( conf.get( "static_robots_txt", 'static/robots.txt'), cache_time ) + + # wrap any static dirs for plugins + plugin_frameworks = plugin_frameworks or [] + for static_serving_framework in ( framework for framework in plugin_frameworks if framework.serves_static ): + # invert control to each plugin for finding their own static dirs + for plugin_url, plugin_static_path in static_serving_framework.get_static_urls_and_paths(): + plugin_url = '/plugins/' + plugin_url + urlmap[( plugin_url )] = Static( plugin_static_path, cache_time ) + log.debug( 'added url, path to static middleware: %s, %s', plugin_url, plugin_static_path ) + # URL mapper becomes the root webapp return urlmap diff -r f760fca38915be8d0ae8f94e703a6d83302065f8 -r 6a8bb831b771b56e79fff3fccfe610d0bf01b885 lib/galaxy/webapps/galaxy/controllers/visualization.py --- a/lib/galaxy/webapps/galaxy/controllers/visualization.py +++ b/lib/galaxy/webapps/galaxy/controllers/visualization.py @@ -708,7 +708,7 @@ # validate name vs. registry registry = trans.app.visualizations_registry if not registry: - raise HTTPNotFound( 'No visualization registry (possibly disabled in universe_wsgi.ini)') + raise HTTPNotFound( 'No visualization registry (possibly disabled in universe_wsgi.ini)' ) if visualization_name not in registry.listings: raise HTTPNotFound( 'Unknown or invalid visualization: ' + visualization_name ) # or redirect to list? @@ -722,16 +722,15 @@ resources = registry.query_dict_to_resources( trans, self, visualization_name, kwargs ) # look up template and render - template_root = registry_listing.get( 'template_root', registry.TEMPLATE_ROOT ) - template = registry_listing[ 'template' ] - template_path = os.path.join( template_root, template ) + template_path = registry_listing[ 'template' ] + returned = registry.fill_template( trans, template_path, + visualization_name=visualization_name, query_args=kwargs, + embedded=embedded, shared_vars={}, **resources ) #NOTE: passing *unparsed* kwargs as query_args #NOTE: shared_vars is a dictionary for shared data in the template # this feels hacky to me but it's what mako recommends: # http://docs.makotemplates.org/en/latest/runtime.html #TODO: embedded - returned = trans.fill_template( template_path, visualization_name=visualization_name, - embedded=embedded, query_args=kwargs, shared_vars={}, **resources ) except Exception, exception: log.exception( 'error rendering visualization (%s): %s', visualization_name, str( exception ) ) diff -r f760fca38915be8d0ae8f94e703a6d83302065f8 -r 6a8bb831b771b56e79fff3fccfe610d0bf01b885 static/scripts/mvc/dataset/hda-edit.js --- a/static/scripts/mvc/dataset/hda-edit.js +++ b/static/scripts/mvc/dataset/hda-edit.js @@ -616,7 +616,7 @@ title : "Scatterplot", type : "url", content : url + '/scatterplot?' + $.param(params), - center : true + location : 'center' }); //TODO: this needs to go away @@ -699,4 +699,4 @@ //============================================================================== //return { // HDAView : HDAView, -//};}); \ No newline at end of file +//};}); diff -r f760fca38915be8d0ae8f94e703a6d83302065f8 -r 6a8bb831b771b56e79fff3fccfe610d0bf01b885 universe_wsgi.ini.sample --- a/universe_wsgi.ini.sample +++ b/universe_wsgi.ini.sample @@ -174,10 +174,8 @@ # Galaxy. #datatypes_config_file = datatypes_conf.xml -# Visualizations config directory, where to look for individual visualization -# xml configuration files. Those files define how visualizations apply to -# particular data and how to pass them the necessary parameters -#visualizations_config_directory = config/visualizations +# Visualizations config directory: where to look for individual visualization plugins. +#visualizations_plugins_directory = config/plugins/visualizations # Each job is given a unique empty directory as its current working directory. # This option defines in what parent directory those directories will be https://bitbucket.org/galaxy/galaxy-central/commits/db11479ab758/ Changeset: db11479ab758 User: carlfeberhard Date: 2013-08-06 21:04:05 Summary: Fix centering of scatterplot; pack scripts Affected #: 1 file diff -r 6a8bb831b771b56e79fff3fccfe610d0bf01b885 -r db11479ab7584322519463a33b42f4b911fb1a68 static/scripts/packed/mvc/dataset/hda-edit.js --- a/static/scripts/packed/mvc/dataset/hda-edit.js +++ b/static/scripts/packed/mvc/dataset/hda-edit.js @@ -1,1 +1,1 @@ -var HDAEditView=HDABaseView.extend(LoggableMixin).extend({initialize:function(a){HDABaseView.prototype.initialize.call(this,a);this.defaultPrimaryActionButtonRenderers=[this._render_showParamsButton,this._render_rerunButton]},_setUpBehaviors:function(c){HDABaseView.prototype._setUpBehaviors.call(this,c);var a=this,b=this.urls.purge,d=c.find("#historyItemPurger-"+this.model.get("id"));if(d){d.attr("href",["javascript","void(0)"].join(":"));d.click(function(e){var f=jQuery.ajax(b);f.success(function(i,g,h){a.model.set("purged",true);a.trigger("purged",a)});f.error(function(h,g,i){a.trigger("error",_l("Unable to purge this dataset"),h,g,i)})})}},_render_warnings:function(){return $(jQuery.trim(HDABaseView.templates.messages(_.extend(this.model.toJSON(),{urls:this.urls}))))},_render_titleButtons:function(){var a=$('<div class="historyItemButtons"></div>');a.append(this._render_displayButton());a.append(this._render_editButton());a.append(this._render_deleteButton());return a},_render_editButton:function(){if((this.model.get("state")===HistoryDatasetAssociation.STATES.NEW)||(this.model.get("state")===HistoryDatasetAssociation.STATES.UPLOAD)||(this.model.get("state")===HistoryDatasetAssociation.STATES.NOT_VIEWABLE)||(!this.model.get("accessible"))){this.editButton=null;return null}var c=this.model.get("purged"),a=this.model.get("deleted"),b={title:_l("Edit Attributes"),href:this.urls.edit,target:"galaxy_main",icon_class:"edit"};if(a||c){b.enabled=false;if(c){b.title=_l("Cannot edit attributes of datasets removed from disk")}else{if(a){b.title=_l("Undelete dataset to edit attributes")}}}this.editButton=new IconButtonView({model:new IconButton(b)});return this.editButton.render().$el},_render_deleteButton:function(){if((this.model.get("state")===HistoryDatasetAssociation.STATES.NEW)||(this.model.get("state")===HistoryDatasetAssociation.STATES.NOT_VIEWABLE)||(!this.model.get("accessible"))){this.deleteButton=null;return null}var a=this,b=a.urls["delete"],c={title:_l("Delete"),href:b,id:"historyItemDeleter-"+this.model.get("id"),icon_class:"delete",on_click:function(){$.ajax({url:b,type:"POST",error:function(){a.$el.show()},success:function(){a.model.set({deleted:true})}})}};if(this.model.get("deleted")||this.model.get("purged")){c={title:_l("Dataset is already deleted"),icon_class:"delete",enabled:false}}this.deleteButton=new IconButtonView({model:new IconButton(c)});return this.deleteButton.render().$el},_render_hdaSummary:function(){var a=_.extend(this.model.toJSON(),{urls:this.urls});if(this.model.get("metadata_dbkey")==="?"&&!this.model.isDeletedOrPurged()){_.extend(a,{dbkey_unknown_and_editable:true})}return HDABaseView.templates.hdaSummary(a)},_render_errButton:function(){if(this.model.get("state")!==HistoryDatasetAssociation.STATES.ERROR){this.errButton=null;return null}this.errButton=new IconButtonView({model:new IconButton({title:_l("View or report this error"),href:this.urls.report_error,target:"galaxy_main",icon_class:"bug"})});return this.errButton.render().$el},_render_rerunButton:function(){this.rerunButton=new IconButtonView({model:new IconButton({title:_l("Run this job again"),href:this.urls.rerun,target:"galaxy_main",icon_class:"arrow-circle"})});return this.rerunButton.render().$el},_render_visualizationsButton:function(){var a=this.model.get("visualizations");if((!this.model.hasData())||(_.isEmpty(a))){this.visualizationsButton=null;return null}if(_.isObject(a[0])){return this._render_visualizationsFrameworkButton(a)}if(!this.urls.visualization){this.visualizationsButton=null;return null}var c=this.model.get("dbkey"),f=this.urls.visualization,d={},g={dataset_id:this.model.get("id"),hda_ldda:"hda"};if(c){g.dbkey=c}this.visualizationsButton=new IconButtonView({model:new IconButton({title:_l("Visualize"),href:this.urls.visualization,icon_class:"chart_curve"})});var b=this.visualizationsButton.render().$el;b.addClass("visualize-icon");function e(h){switch(h){case"trackster":return create_trackster_action_fn(f,g,c);case"scatterplot":return create_scatterplot_action_fn(f,g);default:return function(){parent.frame_manager.frame_new({title:"Visualization",type:"url",content:f+"/"+h+"?"+$.param(g)})}}}if(a.length===1){b.attr("title",a[0]);b.click(e(a[0]))}else{_.each(a,function(i){var h=i.charAt(0).toUpperCase()+i.slice(1);d[_l(h)]=e(i)});make_popupmenu(b,d)}return b},_render_visualizationsFrameworkButton:function(a){if(!(this.model.hasData())||!(a&&!_.isEmpty(a))){this.visualizationsButton=null;return null}this.visualizationsButton=new IconButtonView({model:new IconButton({title:_l("Visualize"),icon_class:"chart_curve"})});var c=this.visualizationsButton.render().$el;c.addClass("visualize-icon");if(_.keys(a).length===1){c.attr("title",_.keys(a)[0]);c.attr("href",_.values(a)[0])}else{var d=[];_.each(a,function(e){d.push(e)});var b=new PopupMenu(c,d)}return c},_render_secondaryActionButtons:function(b){var c=$("<div/>"),a=this;c.attr("style","float: right;").attr("id","secondary-actions-"+this.model.get("id"));_.each(b,function(d){c.append(d.call(a))});return c},_render_tagButton:function(){if(!(this.model.hasData())||(!this.urls.tags.get)){this.tagButton=null;return null}this.tagButton=new IconButtonView({model:new IconButton({title:_l("Edit dataset tags"),target:"galaxy_main",href:this.urls.tags.get,icon_class:"tags"})});return this.tagButton.render().$el},_render_annotateButton:function(){if(!(this.model.hasData())||(!this.urls.annotation.get)){this.annotateButton=null;return null}this.annotateButton=new IconButtonView({model:new IconButton({title:_l("Edit dataset annotation"),target:"galaxy_main",icon_class:"annotate"})});return this.annotateButton.render().$el},_render_tagArea:function(){if(!this.urls.tags.set){return null}return $(HDAEditView.templates.tagArea(_.extend(this.model.toJSON(),{urls:this.urls})).trim())},_render_annotationArea:function(){if(!this.urls.annotation.get){return null}return $(HDAEditView.templates.annotationArea(_.extend(this.model.toJSON(),{urls:this.urls})).trim())},_render_body_error:function(a){HDABaseView.prototype._render_body_error.call(this,a);var b=a.find("#primary-actions-"+this.model.get("id"));b.prepend(this._render_errButton())},_render_body_ok:function(a){a.append(this._render_hdaSummary());if(this.model.isDeletedOrPurged()){a.append(this._render_primaryActionButtons([this._render_downloadButton,this._render_showParamsButton,this._render_rerunButton]));return}a.append(this._render_primaryActionButtons([this._render_downloadButton,this._render_showParamsButton,this._render_rerunButton,this._render_visualizationsButton]));a.append(this._render_secondaryActionButtons([this._render_tagButton,this._render_annotateButton]));a.append('<div class="clear"/>');a.append(this._render_tagArea());a.append(this._render_annotationArea());a.append(this._render_displayAppArea());this._render_displayApps(a);a.append(this._render_peek())},events:{"click .historyItemTitle":"toggleBodyVisibility","click a.icon-button.tags":"loadAndDisplayTags","click a.icon-button.annotate":"loadAndDisplayAnnotation"},loadAndDisplayTags:function(c){this.log(this+".loadAndDisplayTags",c);var a=this,d=this.$el.find(".tag-area"),b=d.find(".tag-elt");if(d.is(":hidden")){if(!jQuery.trim(b.html())){$.ajax({url:this.urls.tags.get,error:function(g,e,f){a.log("Tagging failed",g,e,f);a.trigger("error",_l("Tagging failed"),g,e,f)},success:function(e){b.html(e);b.find(".tooltip").tooltip();d.slideDown("fast")}})}else{d.slideDown("fast")}}else{d.slideUp("fast")}return false},loadAndDisplayAnnotation:function(b){this.log(this+".loadAndDisplayAnnotation",b);var d=this.$el.find(".annotation-area"),c=d.find(".annotation-elt"),a=this.urls.annotation.set;if(d.is(":hidden")){if(!jQuery.trim(c.html())){$.ajax({url:this.urls.annotation.get,error:function(){view.log("Annotation failed",xhr,status,error);view.trigger("error",_l("Annotation failed"),xhr,status,error)},success:function(e){if(e===""){e="<em>"+_l("Describe or add notes to dataset")+"</em>"}c.html(e);d.find(".tooltip").tooltip();async_save_text(c.attr("id"),c.attr("id"),a,"new_annotation",18,true,4);d.slideDown("fast")}})}else{d.slideDown("fast")}}else{d.slideUp("fast")}return false},toString:function(){var a=(this.model)?(this.model+""):("(no model)");return"HDAView("+a+")"}});HDAEditView.templates={tagArea:Handlebars.templates["template-hda-tagArea"],annotationArea:Handlebars.templates["template-hda-annotationArea"]};function create_scatterplot_action_fn(a,b){action=function(){parent.frame_manager.frame_new({title:"Scatterplot",type:"url",content:a+"/scatterplot?"+$.param(b),center:true});$("div.popmenu-wrapper").remove();return false};return action}function create_trackster_action_fn(a,c,b){return function(){var d={};if(b){d["f-dbkey"]=b}$.ajax({url:a+"/list_tracks?"+$.param(d),dataType:"html",error:function(){alert(_l("Could not add this dataset to browser")+".")},success:function(e){var f=window.parent;f.show_modal(_l("View Data in a New or Saved Visualization"),"",{Cancel:function(){f.hide_modal()},"View in saved visualization":function(){f.show_modal(_l("Add Data to Saved Visualization"),e,{Cancel:function(){f.hide_modal()},"Add to visualization":function(){$(f.document).find("input[name=id]:checked").each(function(){var g=$(this).val();c.id=g;f.hide_modal();f.frame_manager.frame_new({title:"Trackster",type:"url",content:a+"/trackster?"+$.param(c)})})}})},"View in new visualization":function(){f.hide_modal();f.frame_manager.frame_new({title:"Trackster",type:"url",content:a+"/trackster?"+$.param(c)})}})}});return false}}; \ No newline at end of file +var HDAEditView=HDABaseView.extend(LoggableMixin).extend({initialize:function(a){HDABaseView.prototype.initialize.call(this,a);this.defaultPrimaryActionButtonRenderers=[this._render_showParamsButton,this._render_rerunButton]},_setUpBehaviors:function(c){HDABaseView.prototype._setUpBehaviors.call(this,c);var a=this,b=this.urls.purge,d=c.find("#historyItemPurger-"+this.model.get("id"));if(d){d.attr("href",["javascript","void(0)"].join(":"));d.click(function(e){var f=jQuery.ajax(b);f.success(function(i,g,h){a.model.set("purged",true);a.trigger("purged",a)});f.error(function(h,g,i){a.trigger("error",_l("Unable to purge this dataset"),h,g,i)})})}},_render_warnings:function(){return $(jQuery.trim(HDABaseView.templates.messages(_.extend(this.model.toJSON(),{urls:this.urls}))))},_render_titleButtons:function(){var a=$('<div class="historyItemButtons"></div>');a.append(this._render_displayButton());a.append(this._render_editButton());a.append(this._render_deleteButton());return a},_render_editButton:function(){if((this.model.get("state")===HistoryDatasetAssociation.STATES.NEW)||(this.model.get("state")===HistoryDatasetAssociation.STATES.UPLOAD)||(this.model.get("state")===HistoryDatasetAssociation.STATES.NOT_VIEWABLE)||(!this.model.get("accessible"))){this.editButton=null;return null}var c=this.model.get("purged"),a=this.model.get("deleted"),b={title:_l("Edit Attributes"),href:this.urls.edit,target:"galaxy_main",icon_class:"edit"};if(a||c){b.enabled=false;if(c){b.title=_l("Cannot edit attributes of datasets removed from disk")}else{if(a){b.title=_l("Undelete dataset to edit attributes")}}}this.editButton=new IconButtonView({model:new IconButton(b)});return this.editButton.render().$el},_render_deleteButton:function(){if((this.model.get("state")===HistoryDatasetAssociation.STATES.NEW)||(this.model.get("state")===HistoryDatasetAssociation.STATES.NOT_VIEWABLE)||(!this.model.get("accessible"))){this.deleteButton=null;return null}var a=this,b=a.urls["delete"],c={title:_l("Delete"),href:b,id:"historyItemDeleter-"+this.model.get("id"),icon_class:"delete",on_click:function(){$.ajax({url:b,type:"POST",error:function(){a.$el.show()},success:function(){a.model.set({deleted:true})}})}};if(this.model.get("deleted")||this.model.get("purged")){c={title:_l("Dataset is already deleted"),icon_class:"delete",enabled:false}}this.deleteButton=new IconButtonView({model:new IconButton(c)});return this.deleteButton.render().$el},_render_hdaSummary:function(){var a=_.extend(this.model.toJSON(),{urls:this.urls});if(this.model.get("metadata_dbkey")==="?"&&!this.model.isDeletedOrPurged()){_.extend(a,{dbkey_unknown_and_editable:true})}return HDABaseView.templates.hdaSummary(a)},_render_errButton:function(){if(this.model.get("state")!==HistoryDatasetAssociation.STATES.ERROR){this.errButton=null;return null}this.errButton=new IconButtonView({model:new IconButton({title:_l("View or report this error"),href:this.urls.report_error,target:"galaxy_main",icon_class:"bug"})});return this.errButton.render().$el},_render_rerunButton:function(){this.rerunButton=new IconButtonView({model:new IconButton({title:_l("Run this job again"),href:this.urls.rerun,target:"galaxy_main",icon_class:"arrow-circle"})});return this.rerunButton.render().$el},_render_visualizationsButton:function(){var a=this.model.get("visualizations");if((!this.model.hasData())||(_.isEmpty(a))){this.visualizationsButton=null;return null}if(_.isObject(a[0])){return this._render_visualizationsFrameworkButton(a)}if(!this.urls.visualization){this.visualizationsButton=null;return null}var c=this.model.get("dbkey"),f=this.urls.visualization,d={},g={dataset_id:this.model.get("id"),hda_ldda:"hda"};if(c){g.dbkey=c}this.visualizationsButton=new IconButtonView({model:new IconButton({title:_l("Visualize"),href:this.urls.visualization,icon_class:"chart_curve"})});var b=this.visualizationsButton.render().$el;b.addClass("visualize-icon");function e(h){switch(h){case"trackster":return create_trackster_action_fn(f,g,c);case"scatterplot":return create_scatterplot_action_fn(f,g);default:return function(){parent.frame_manager.frame_new({title:"Visualization",type:"url",content:f+"/"+h+"?"+$.param(g)})}}}if(a.length===1){b.attr("title",a[0]);b.click(e(a[0]))}else{_.each(a,function(i){var h=i.charAt(0).toUpperCase()+i.slice(1);d[_l(h)]=e(i)});make_popupmenu(b,d)}return b},_render_visualizationsFrameworkButton:function(a){if(!(this.model.hasData())||!(a&&!_.isEmpty(a))){this.visualizationsButton=null;return null}this.visualizationsButton=new IconButtonView({model:new IconButton({title:_l("Visualize"),icon_class:"chart_curve"})});var c=this.visualizationsButton.render().$el;c.addClass("visualize-icon");if(_.keys(a).length===1){c.attr("title",_.keys(a)[0]);c.attr("href",_.values(a)[0])}else{var d=[];_.each(a,function(e){d.push(e)});var b=new PopupMenu(c,d)}return c},_render_secondaryActionButtons:function(b){var c=$("<div/>"),a=this;c.attr("style","float: right;").attr("id","secondary-actions-"+this.model.get("id"));_.each(b,function(d){c.append(d.call(a))});return c},_render_tagButton:function(){if(!(this.model.hasData())||(!this.urls.tags.get)){this.tagButton=null;return null}this.tagButton=new IconButtonView({model:new IconButton({title:_l("Edit dataset tags"),target:"galaxy_main",href:this.urls.tags.get,icon_class:"tags"})});return this.tagButton.render().$el},_render_annotateButton:function(){if(!(this.model.hasData())||(!this.urls.annotation.get)){this.annotateButton=null;return null}this.annotateButton=new IconButtonView({model:new IconButton({title:_l("Edit dataset annotation"),target:"galaxy_main",icon_class:"annotate"})});return this.annotateButton.render().$el},_render_tagArea:function(){if(!this.urls.tags.set){return null}return $(HDAEditView.templates.tagArea(_.extend(this.model.toJSON(),{urls:this.urls})).trim())},_render_annotationArea:function(){if(!this.urls.annotation.get){return null}return $(HDAEditView.templates.annotationArea(_.extend(this.model.toJSON(),{urls:this.urls})).trim())},_render_body_error:function(a){HDABaseView.prototype._render_body_error.call(this,a);var b=a.find("#primary-actions-"+this.model.get("id"));b.prepend(this._render_errButton())},_render_body_ok:function(a){a.append(this._render_hdaSummary());if(this.model.isDeletedOrPurged()){a.append(this._render_primaryActionButtons([this._render_downloadButton,this._render_showParamsButton,this._render_rerunButton]));return}a.append(this._render_primaryActionButtons([this._render_downloadButton,this._render_showParamsButton,this._render_rerunButton,this._render_visualizationsButton]));a.append(this._render_secondaryActionButtons([this._render_tagButton,this._render_annotateButton]));a.append('<div class="clear"/>');a.append(this._render_tagArea());a.append(this._render_annotationArea());a.append(this._render_displayAppArea());this._render_displayApps(a);a.append(this._render_peek())},events:{"click .historyItemTitle":"toggleBodyVisibility","click a.icon-button.tags":"loadAndDisplayTags","click a.icon-button.annotate":"loadAndDisplayAnnotation"},loadAndDisplayTags:function(c){this.log(this+".loadAndDisplayTags",c);var a=this,d=this.$el.find(".tag-area"),b=d.find(".tag-elt");if(d.is(":hidden")){if(!jQuery.trim(b.html())){$.ajax({url:this.urls.tags.get,error:function(g,e,f){a.log("Tagging failed",g,e,f);a.trigger("error",_l("Tagging failed"),g,e,f)},success:function(e){b.html(e);b.find(".tooltip").tooltip();d.slideDown("fast")}})}else{d.slideDown("fast")}}else{d.slideUp("fast")}return false},loadAndDisplayAnnotation:function(b){this.log(this+".loadAndDisplayAnnotation",b);var d=this.$el.find(".annotation-area"),c=d.find(".annotation-elt"),a=this.urls.annotation.set;if(d.is(":hidden")){if(!jQuery.trim(c.html())){$.ajax({url:this.urls.annotation.get,error:function(){view.log("Annotation failed",xhr,status,error);view.trigger("error",_l("Annotation failed"),xhr,status,error)},success:function(e){if(e===""){e="<em>"+_l("Describe or add notes to dataset")+"</em>"}c.html(e);d.find(".tooltip").tooltip();async_save_text(c.attr("id"),c.attr("id"),a,"new_annotation",18,true,4);d.slideDown("fast")}})}else{d.slideDown("fast")}}else{d.slideUp("fast")}return false},toString:function(){var a=(this.model)?(this.model+""):("(no model)");return"HDAView("+a+")"}});HDAEditView.templates={tagArea:Handlebars.templates["template-hda-tagArea"],annotationArea:Handlebars.templates["template-hda-annotationArea"]};function create_scatterplot_action_fn(a,b){action=function(){parent.frame_manager.frame_new({title:"Scatterplot",type:"url",content:a+"/scatterplot?"+$.param(b),location:"center"});$("div.popmenu-wrapper").remove();return false};return action}function create_trackster_action_fn(a,c,b){return function(){var d={};if(b){d["f-dbkey"]=b}$.ajax({url:a+"/list_tracks?"+$.param(d),dataType:"html",error:function(){alert(_l("Could not add this dataset to browser")+".")},success:function(e){var f=window.parent;f.show_modal(_l("View Data in a New or Saved Visualization"),"",{Cancel:function(){f.hide_modal()},"View in saved visualization":function(){f.show_modal(_l("Add Data to Saved Visualization"),e,{Cancel:function(){f.hide_modal()},"Add to visualization":function(){$(f.document).find("input[name=id]:checked").each(function(){var g=$(this).val();c.id=g;f.hide_modal();f.frame_manager.frame_new({title:"Trackster",type:"url",content:a+"/trackster?"+$.param(c)})})}})},"View in new visualization":function(){f.hide_modal();f.frame_manager.frame_new({title:"Trackster",type:"url",content:a+"/trackster?"+$.param(c)})}})}});return false}}; \ No newline at end of file Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: greg: Keep track of category associations when exporting repositories from the tool shed.
by commits-noreply＠bitbucket.org 06 Aug '13

06 Aug '13

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/f760fca38915/ Changeset: f760fca38915 User: greg Date: 2013-08-06 17:04:54 Summary: Keep track of category associations when exporting repositories from the tool shed. Affected #: 2 files diff -r b9a24a719716150910de19eefd1189238c243255 -r f760fca38915be8d0ae8f94e703a6d83302065f8 lib/tool_shed/util/export_util.py --- a/lib/tool_shed/util/export_util.py +++ b/lib/tool_shed/util/export_util.py @@ -246,6 +246,12 @@ sub_elements[ 'description' ] = str( repository.description ) sub_elements[ 'long_description' ] = str( repository.long_description ) sub_elements[ 'archive' ] = archive_name + # Keep track of Category associations. + categories = [] + for rca in repository.categories: + category = rca.category + categories.append( ( 'category', str( category.name ) ) ) + sub_elements[ 'categories' ] = categories return attributes, sub_elements def get_repository_ids( trans, repo_info_dicts ): diff -r b9a24a719716150910de19eefd1189238c243255 -r f760fca38915be8d0ae8f94e703a6d83302065f8 lib/tool_shed/util/xml_util.py --- a/lib/tool_shed/util/xml_util.py +++ b/lib/tool_shed/util/xml_util.py @@ -62,7 +62,18 @@ # Don't include fields that are blank. if v: sub_elem = XmlET.SubElement( elem, k ) - sub_elem.text = v + if isinstance( v, list ): + # If the sub_elem is a list, then it must be a list of tuples where the first item is the tag and the second item is the text value. + for v_tuple in v: + if len( v_tuple ) == 2: + v_tag = v_tuple[ 0 ] + v_text = v_tuple[ 1 ] + # Don't include fields that are blank. + if v_text: + v_elem = XmlET.SubElement( sub_elem, v_tag ) + v_elem.text = v_text + else: + sub_elem.text = v return elem return None Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: 3 new changesets
by commits-noreply＠bitbucket.org 06 Aug '13

06 Aug '13

3 new commits in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/46b60df0b241/ Changeset: 46b60df0b241 Branch: next-stable User: jgoecks Date: 2013-08-06 16:59:43 Summary: Add requirements tags to tophat wrapper. Affected #: 1 file diff -r ccabe7fd484c910ad85e7b6550e59f37ea39dfd8 -r 46b60df0b241daf324f16e14cd43e06bd13240c5 tools/ngs_rna/tophat_wrapper.xml --- a/tools/ngs_rna/tophat_wrapper.xml +++ b/tools/ngs_rna/tophat_wrapper.xml @@ -3,6 +3,8 @@ <description>Find splice junctions using RNA-seq data</description><version_command>tophat --version</version_command><requirements> + <requirement type="package">samtools</requirement> + <requirement type="package">bowtie</requirement><requirement type="package">tophat</requirement></requirements><command interpreter="python"> https://bitbucket.org/galaxy/galaxy-central/commits/e3f6a9510978/ Changeset: e3f6a9510978 Branch: next-stable User: jgoecks Date: 2013-08-06 17:01:31 Summary: Add tophat2 to main tool conf. Affected #: 1 file diff -r 46b60df0b241daf324f16e14cd43e06bd13240c5 -r e3f6a95109789272a3c4f0663dbcdab908aa8f26 tool_conf.xml.main --- a/tool_conf.xml.main +++ b/tool_conf.xml.main @@ -326,6 +326,7 @@ <section name="NGS: RNA Analysis" id="ngs-rna-tools"><label text="RNA-seq" id="rna_seq" /><tool file="ngs_rna/tophat_wrapper.xml" /> + <tool file="ngs_rna/tophat2_wrapper.xml" /><tool file="ngs_rna/cufflinks_wrapper.xml" /><tool file="ngs_rna/cuffcompare_wrapper.xml" /><tool file="ngs_rna/cuffmerge_wrapper.xml" /> https://bitbucket.org/galaxy/galaxy-central/commits/b9a24a719716/ Changeset: b9a24a719716 User: jgoecks Date: 2013-08-06 17:02:37 Summary: Automated merge of next-stable branch Affected #: 2 files diff -r 61de03c05cf653aed0418adc98ae3cb6571f2a4e -r b9a24a719716150910de19eefd1189238c243255 tool_conf.xml.main --- a/tool_conf.xml.main +++ b/tool_conf.xml.main @@ -326,6 +326,7 @@ <section name="NGS: RNA Analysis" id="ngs-rna-tools"><label text="RNA-seq" id="rna_seq" /><tool file="ngs_rna/tophat_wrapper.xml" /> + <tool file="ngs_rna/tophat2_wrapper.xml" /><tool file="ngs_rna/cufflinks_wrapper.xml" /><tool file="ngs_rna/cuffcompare_wrapper.xml" /><tool file="ngs_rna/cuffmerge_wrapper.xml" /> diff -r 61de03c05cf653aed0418adc98ae3cb6571f2a4e -r b9a24a719716150910de19eefd1189238c243255 tools/ngs_rna/tophat_wrapper.xml --- a/tools/ngs_rna/tophat_wrapper.xml +++ b/tools/ngs_rna/tophat_wrapper.xml @@ -3,6 +3,8 @@ <description>Find splice junctions using RNA-seq data</description><version_command>tophat --version</version_command><requirements> + <requirement type="package">samtools</requirement> + <requirement type="package">bowtie</requirement><requirement type="package">tophat</requirement></requirements><command interpreter="python"> Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: greg: Fix for displaying the Installation status for tool dependencies installed along with repositories into Galaxy, and fix broken tool shed functional tests.
by commits-noreply＠bitbucket.org 06 Aug '13

06 Aug '13

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/61de03c05cf6/ Changeset: 61de03c05cf6 User: greg Date: 2013-08-06 16:21:37 Summary: Fix for displaying the Installation status for tool dependencies installed along with repositories into Galaxy, and fix broken tool shed functional tests. Affected #: 3 files diff -r cf53a00bcf1d279072ec279e6531f211992b4f3e -r 61de03c05cf653aed0418adc98ae3cb6571f2a4e lib/galaxy/webapps/tool_shed/util/container_util.py --- a/lib/galaxy/webapps/tool_shed/util/container_util.py +++ b/lib/galaxy/webapps/tool_shed/util/container_util.py @@ -257,13 +257,12 @@ class ToolDependency( object ): """Tool dependency object""" - def __init__( self, id=None, name=None, version=None, type=None, install_dir=None, readme=None, installation_status=None, repository_id=None, + def __init__( self, id=None, name=None, version=None, type=None, readme=None, installation_status=None, repository_id=None, tool_dependency_id=None, is_orphan=None ): self.id = id self.name = name self.version = version self.type = type - self.install_dir = install_dir self.readme = readme self.installation_status = installation_status self.repository_id = repository_id @@ -948,12 +947,10 @@ # Insert a header row. tool_dependency_id += 1 if trans.webapp.name == 'galaxy': - # Include the installation directory. tool_dependency = ToolDependency( id=tool_dependency_id, name='Name', version='Version', type='Type', - install_dir='Install directory', readme=None, installation_status='Installation status', repository_id=None, @@ -964,7 +961,6 @@ name='Name', version='Version', type='Type', - install_dir=None, readme=None, installation_status=None, repository_id=None, @@ -995,7 +991,6 @@ name=name, version=None, type=type, - install_dir=None, readme=None, installation_status=installation_status, repository_id=repository_id, @@ -1013,7 +1008,6 @@ name = requirements_dict[ 'name' ] version = requirements_dict[ 'version' ] type = requirements_dict[ 'type' ] - install_dir = requirements_dict.get( 'install_dir', None ) repository_id = requirements_dict.get( 'repository_id', None ) td_id = requirements_dict.get( 'tool_dependency_id', None ) if trans.webapp.name == 'galaxy': @@ -1024,7 +1018,6 @@ name=name, version=version, type=type, - install_dir=install_dir, readme=None, installation_status=installation_status, repository_id=repository_id, diff -r cf53a00bcf1d279072ec279e6531f211992b4f3e -r 61de03c05cf653aed0418adc98ae3cb6571f2a4e templates/webapps/tool_shed/repository/common.mako --- a/templates/webapps/tool_shed/repository/common.mako +++ b/templates/webapps/tool_shed/repository/common.mako @@ -888,11 +888,7 @@ <${cell_type}>${tool_dependency.type | h}</${cell_type}><${cell_type}> %if trans.webapp.name == 'galaxy': - %if is_missing: - ${tool_dependency.installation_status | h} - %elif tool_dependency.install_dir: - ${tool_dependency.install_dir | h} - %endif + ${tool_dependency.installation_status | h} %else: %if row_is_header: ${tool_dependency.is_orphan | h} diff -r cf53a00bcf1d279072ec279e6531f211992b4f3e -r 61de03c05cf653aed0418adc98ae3cb6571f2a4e test/tool_shed/functional/test_1170_complex_prior_installation_required.py --- a/test/tool_shed/functional/test_1170_complex_prior_installation_required.py +++ b/test/tool_shed/functional/test_1170_complex_prior_installation_required.py @@ -63,7 +63,7 @@ uncompress_file=True, remove_repo_files_not_in_tar=False, commit_message='Uploaded matplotlib tool dependency tarball.', - strings_displayed=['orphan'], + strings_displayed=[ 'This repository currently contains a single file named <b>tool_dependencies.xml</b>' ], strings_not_displayed=[] ) def test_0010_create_numpy_repository( self ): @@ -89,7 +89,7 @@ uncompress_file=True, remove_repo_files_not_in_tar=False, commit_message='Uploaded numpy tool dependency tarball.', - strings_displayed=['orphan'], + strings_displayed=[ 'This repository currently contains a single file named <b>tool_dependencies.xml</b>' ], strings_not_displayed=[] ) def test_0015_create_complex_repository_dependency( self ): Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0