- galaxy-commits - lists.galaxyproject.org

commit/galaxy-central: greg: More fixes for installing many repositories from the tool shed.
by commits-noreply＠bitbucket.org 13 May '13

13 May '13

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/b8af8fa7ab2e/ Changeset: b8af8fa7ab2e User: greg Date: 2013-05-13 21:44:58 Summary: More fixes for installing many repositories from the tool shed. Affected #: 4 files diff -r d92704335a2774e1bde2be72f9f78ce63c0447fd -r b8af8fa7ab2e61700875249c62308302a105bb38 lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py --- a/lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py +++ b/lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py @@ -313,11 +313,6 @@ """Return an svg image representation of a workflow dictionary created when the workflow was exported.""" return workflow_util.generate_workflow_image( trans, workflow_name, repository_metadata_id=None, repository_id=repository_id ) - @web.expose - def get_contents_of_file( self, trans, encoded_file_path ): - file_path = encoding_util.tool_shed_decode( encoded_file_path ) - return suc.get_file_contents( file_path ) - @web.json @web.require_admin def get_file_contents( self, trans, file_path ): @@ -388,34 +383,6 @@ tool_version = tool_util.get_tool_version( app, guid ) return tool_version.get_version_ids( app, reverse=True ) - @web.json - def handle_large_repo_info_dict( self, trans, **kwd ): - """ - In some cases the required encoded_str will be long. With apache, the default limit for the length of the request line is 8190 bytes - (http://httpd.apache.org/docs/2.2/mod/core.html#limitrequestline) And if we subtract three bytes for the request method (i.e. GET), - eight bytes for the version information (i.e. HTTP/1.0/HTTP/1.1) and two bytes for the separating space, we end up with 8177 bytes for - the URI path plus query. The referer handles requests longer than this by persisting the repo_info_dict to a temporary file which we can read. - """ - required_repo_info_dict = {} - encoded_tmp_file_name = kwd.get( 'encoded_tmp_file_name', None ) - # The request would have been longer than 8190 bytes if it included the encoded_str, so we'll send a request to the Galaxy instance to get it. - tool_shed_url = kwd.get( 'tool_shed_url', None ) - if tool_shed_url and encoded_tmp_file_name: - url = suc.url_join( tool_shed_url, - '/repository/get_contents_of_file?encoded_tmp_file_name=%s' % encoded_tmp_file_name ) - response = urllib2.urlopen( url ) - text = response.read() - required_repo_info_dict = json.from_json_string( text ) - else: - log.debug( "Invalid tool_shed_url '%s' or encoded_tmp_file_name '%s'." % ( str( tool_shed_url ), str( encoded_tmp_file_name ) ) ) - tmp_file_name = encoding_util.tool_shed_decode( encoded_tmp_file_name ) - if os.path.exists( tmp_file_name ): - try: - os.unkink( tmp_file_name ) - except: - pass - return common_install_util.process_repo_info_dict( trans, required_repo_info_dict ) - @web.expose @web.require_admin def import_workflow( self, trans, workflow_name, repository_id, **kwd ): diff -r d92704335a2774e1bde2be72f9f78ce63c0447fd -r b8af8fa7ab2e61700875249c62308302a105bb38 lib/galaxy/webapps/tool_shed/controllers/repository.py --- a/lib/galaxy/webapps/tool_shed/controllers/repository.py +++ b/lib/galaxy/webapps/tool_shed/controllers/repository.py @@ -4,7 +4,6 @@ import re import string import tempfile -import urllib2 from time import gmtime from time import strftime from datetime import date @@ -1285,11 +1284,6 @@ return encoding_util.tool_shed_encode( update_dict ) @web.expose - def get_contents_of_file( self, trans, encoded_file_path ): - file_path = encoding_util.tool_shed_decode( encoded_file_path ) - return suc.get_file_contents( file_path ) - - @web.expose def get_ctx_rev( self, trans, **kwd ): """Given a repository and changeset_revision, return the correct ctx.rev() value.""" repository_name = kwd[ 'name' ] @@ -1478,27 +1472,27 @@ repo_info_dicts=repo_info_dicts ) @web.json - def get_required_repo_info_dict( self, trans, encoded_str, **kwd ): + def get_required_repo_info_dict( self, trans, encoded_str=None ): """ Retrieve and return a dictionary that includes a list of dictionaries that each contain all of the information needed to install the list of repositories defined by the received encoded_str. """ - encoded_required_repository_str = encoding_util.tool_shed_decode( encoded_str ) - encoded_required_repository_tups = encoded_required_repository_str.split( encoding_util.encoding_sep2 ) - decoded_required_repository_tups = [] - for encoded_required_repository_tup in encoded_required_repository_tups: - decoded_required_repository_tups.append( encoded_required_repository_tup.split( encoding_util.encoding_sep ) ) - encoded_repository_ids = [] - changeset_revisions = [] - for required_repository_tup in decoded_required_repository_tups: - tool_shed, name, owner, changeset_revision, prior_installation_required = suc.parse_repository_dependency_tuple( required_repository_tup ) - repository = suc.get_repository_by_name_and_owner( trans.app, name, owner ) - encoded_repository_ids.append( trans.security.encode_id( repository.id ) ) - changeset_revisions.append( changeset_revision ) - if encoded_repository_ids and changeset_revisions: - repo_info_dict = json.from_json_string( self.get_repository_information( trans, encoded_repository_ids, changeset_revisions ) ) - else: - repo_info_dict = {} + repo_info_dict = {} + if encoded_str: + encoded_required_repository_str = encoding_util.tool_shed_decode( encoded_str ) + encoded_required_repository_tups = encoded_required_repository_str.split( encoding_util.encoding_sep2 ) + decoded_required_repository_tups = [] + for encoded_required_repository_tup in encoded_required_repository_tups: + decoded_required_repository_tups.append( encoded_required_repository_tup.split( encoding_util.encoding_sep ) ) + encoded_repository_ids = [] + changeset_revisions = [] + for required_repository_tup in decoded_required_repository_tups: + tool_shed, name, owner, changeset_revision, prior_installation_required = suc.parse_repository_dependency_tuple( required_repository_tup ) + repository = suc.get_repository_by_name_and_owner( trans.app, name, owner ) + encoded_repository_ids.append( trans.security.encode_id( repository.id ) ) + changeset_revisions.append( changeset_revision ) + if encoded_repository_ids and changeset_revisions: + repo_info_dict = json.from_json_string( self.get_repository_information( trans, encoded_repository_ids, changeset_revisions ) ) return repo_info_dict @web.expose @@ -1660,41 +1654,6 @@ return tool_guid_lineage @web.expose - def handle_large_repo_info_dict( self, trans, **kwd ): - """ - In some cases the required encoded_str will be long. With apache, the default limit for the length of the request line is 8190 bytes - (http://httpd.apache.org/docs/2.2/mod/core.html#limitrequestline) And if we subtract three bytes for the request method (i.e. GET), - eight bytes for the version information (i.e. HTTP/1.0/HTTP/1.1) and two bytes for the separating space, we end up with 8177 bytes for - the URI path plus query. The referer handles requests longer than this by persisting the encoded_str to a temporary file which we can read. - """ - repo_info_dict = {} - encoded_tmp_file_name = kwd.get( 'encoded_tmp_file_name', None ) - # The request would have been longer than 8190 bytes if it included the encoded_str, so we'll send a request to the Galaxy instance to get it. - galaxy_url = suc.handle_galaxy_url( trans, **kwd ) - if galaxy_url and encoded_tmp_file_name: - url = suc.url_join( galaxy_url, - '/admin_toolshed/get_contents_of_file?encoded_tmp_file_name=%s' % encoded_tmp_file_name ) - response = urllib2.urlopen( url ) - encoded_required_repository_str = response.read() - repo_info_dict = self.get_required_repo_info_dict( trans, encoded_str ) - repo_info_dict[ 'encoded_tmp_file_name' ] = encoded_tmp_file_name - else: - log.debug( "Invalid galaxy_url '%s' or encoded_tmp_file_name '%s'." % ( str( galaxy_url ), str( encoded_tmp_file_name ) ) ) - # Persist the large repo_info_dict to a temporary file. - fh = tempfile.NamedTemporaryFile( 'wb' ) - tmp_file_name = fh.name - fh.close() - fh = open( tmp_file_name, 'wb' ) - fh.write( encoded_required_repository_str ) - fh.close() - encoded_tmp_file_name = encoding_util.tool_shed_encode( os.path.abspath( tmp_file_name ) ) - tool_shed_url = web.url_for( '/', qualified=True ) - # Redirect to the tool shed to enable it to read the persisted encoded_required_repository_str. - url = suc.url_join( galaxy_url, - '/admin_toolshed/handle_large_repo_info_dict?encoded_tmp_file_name=%s&tool_shed_url=%s' % ( str( encoded_tmp_file_name ), str( tool_shed_url ) ) ) - return trans.response.send_redirect( url ) - - @web.expose def help( self, trans, **kwd ): params = util.Params( kwd ) message = util.restore_text( params.get( 'message', '' ) ) diff -r d92704335a2774e1bde2be72f9f78ce63c0447fd -r b8af8fa7ab2e61700875249c62308302a105bb38 lib/tool_shed/util/common_install_util.py --- a/lib/tool_shed/util/common_install_util.py +++ b/lib/tool_shed/util/common_install_util.py @@ -1,6 +1,7 @@ import logging import os -import tempfile +import urllib +import urllib2 from galaxy import eggs from galaxy import util from galaxy import web @@ -17,9 +18,6 @@ import pkg_resources -pkg_resources.require( "simplejson" ) -import simplejson - pkg_resources.require( 'elementtree' ) from elementtree import ElementTree from elementtree import ElementInclude @@ -256,6 +254,7 @@ repository_dependencies entries in each of the received repo_info_dicts includes all required repositories, so only one pass through this method is required to retrieve all repository dependencies. """ + all_repo_info_dicts = [] if repo_info_dicts: # We'll send tuples of ( tool_shed, repository_name, repository_owner, changeset_revision ) to the tool shed to discover repository ids. required_repository_tups = [] @@ -288,47 +287,25 @@ encoded_required_repository_tups.append( encoding_util.encoding_sep.join( required_repository_tup ) ) encoded_required_repository_str = encoding_util.encoding_sep2.join( encoded_required_repository_tups ) encoded_required_repository_str = encoding_util.tool_shed_encode( encoded_required_repository_str ) - url = suc.url_join( tool_shed_url, '/repository/get_required_repo_info_dict?encoded_str=%s' % encoded_required_repository_str ) - text = common_util.tool_shed_get( trans.app, tool_shed_url, url ) - if text: + url = suc.url_join( tool_shed_url, '/repository/get_required_repo_info_dict' ) + request = urllib2.Request( url, data=urllib.urlencode( dict( encoded_str=encoded_required_repository_str ) ) ) + response = urllib2.urlopen( request ).read() + if response: try: - required_repo_info_dict = json.from_json_string( text ) - except simplejson.decoder.JSONDecodeError, e: - if len( url ) >= 8177: - message = '\n\nThe selected tool shed repositories cannot be installed until the tool shed at %s and the Galaxy ' % str( tool_shed_url ) - message += 'instance at %s are both updated to at least the June 3, 2013 Galaxy release. These upgrades ' % str( trans.request.base ) - message += 'are necessary because the number of repositories you are attempting to install generates an HTTP request that is longer than ' - message += '8177 bytes which cannot be handled by tool shed or Galaxy instances older than the June 3, 2013 release.\n\n' - log.exception( message ) - else: - log.exception() - return [] + required_repo_info_dict = json.from_json_string( response ) except Exception, e: - log.exception() - return [] - return process_repo_info_dict( trans, required_repo_info_dict ) - return [] - -def handle_large_repo_info_dict( trans, tool_shed_url, encoded_required_repository_str ): - """ - Handle the cases where the received encoded_required_repository_str is long. With apache, the default limit for the length of the request line is 8190 bytes - (http://httpd.apache.org/docs/2.2/mod/core.html#limitrequestline) And if we subtract three bytes for the request method (i.e. GET), eight bytes for the version - information (i.e. HTTP/1.0/HTTP/1.1) and two bytes for the separating space, we end up with 8177 bytes for the URI path plus query. - """ - # Persist the encoded string to a temporary file. - fh = tempfile.NamedTemporaryFile( 'wb' ) - tmp_file_name = fh.name - fh.close() - fh = open( tmp_file_name, 'wb' ) - fh.write( encoded_required_repository_str ) - fh.close() - encoded_tmp_file_name = encoding_util.tool_shed_encode( os.path.abspath( tmp_file_name ) ) - galaxy_url = web.url_for( '/', qualified=True ) - # Redirect to the tool shed to enable it to read the persisted encoded_required_repository_str. - url = suc.url_join( tool_shed_url, - '/repository/handle_large_repo_info_dict?encoded_tmp_file_name=%s&galaxy_url=%s' % \ - ( encoded_tmp_file_name, galaxy_url ) ) - return trans.response.send_redirect( url ) + log.exception( e ) + return all_repo_info_dicts + required_repo_info_dicts = [] + encoded_dict_strings = required_repo_info_dict[ 'repo_info_dicts' ] + for encoded_dict_str in encoded_dict_strings: + decoded_dict = encoding_util.tool_shed_decode( encoded_dict_str ) + required_repo_info_dicts.append( decoded_dict ) + if required_repo_info_dicts: + for required_repo_info_dict in required_repo_info_dicts: + if required_repo_info_dict not in all_repo_info_dicts: + all_repo_info_dicts.append( required_repo_info_dict ) + return all_repo_info_dicts def handle_tool_dependencies( app, tool_shed_repository, tool_dependencies_config, tool_dependencies ): """ @@ -387,24 +364,3 @@ app.model.ToolDependency.installation_status.ERROR ]: installed_tool_dependencies.append( tool_dependency ) return installed_tool_dependencies - -def process_repo_info_dict( trans, required_repo_info_dict ): - all_repo_info_dicts = [] - required_repo_info_dicts = [] - encoded_dict_strings = required_repo_info_dict[ 'repo_info_dicts' ] - for encoded_dict_str in encoded_dict_strings: - decoded_dict = encoding_util.tool_shed_decode( encoded_dict_str ) - required_repo_info_dicts.append( decoded_dict ) - if required_repo_info_dicts: - for required_repo_info_dict in required_repo_info_dicts: - if required_repo_info_dict not in all_repo_info_dicts: - all_repo_info_dicts.append( required_repo_info_dict ) - # Remove the temporary file that stored the long encoded_required_repository_str if possible. - encoded_tmp_file_name = required_repo_info_dict.get( 'encoded_tmp_file_name', None ) - if encoded_tmp_file_name: - tmp_file_name = encoding_util.tool_shed_decode( encoded_tmp_file_name ) - try: - os.unlink( tmp_file_name ) - except: - pass - return all_repo_info_dicts diff -r d92704335a2774e1bde2be72f9f78ce63c0447fd -r b8af8fa7ab2e61700875249c62308302a105bb38 lib/tool_shed/util/shed_util_common.py --- a/lib/tool_shed/util/shed_util_common.py +++ b/lib/tool_shed/util/shed_util_common.py @@ -466,14 +466,6 @@ return manifest_ctx, ctx_file return None, None -def get_file_contents( file_path ): - if os.path.exists( file_path ): - fh = open( file_path ) - contents = fh.read() - fh.close() - return contents - return '' - def get_file_context_from_ctx( ctx, filename ): """Return the mercurial file context for a specified file.""" # We have to be careful in determining if we found the correct file because multiple files with the same name may be in different directories Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: Dave Bouvier: Updated documentation on missing tool dependencies.
by commits-noreply＠bitbucket.org 13 May '13

13 May '13

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/d92704335a27/ Changeset: d92704335a27 User: Dave Bouvier Date: 2013-05-13 19:12:40 Summary: Updated documentation on missing tool dependencies. Affected #: 1 file diff -r ef2b705c7de344c2c052a36ff3592e78ae5115af -r d92704335a2774e1bde2be72f9f78ce63c0447fd test/install_and_test_tool_shed_repositories/functional_tests.py --- a/test/install_and_test_tool_shed_repositories/functional_tests.py +++ b/test/install_and_test_tool_shed_repositories/functional_tests.py @@ -720,8 +720,9 @@ test_toolbox.toolbox = app.toolbox repositories_failed.append( dict( name=name, owner=owner, changeset_revision=changeset_revision ) ) elif missing_tool_dependencies: - # If a tool dependency fails to install correctly, this should be considered a missing test component, - # as the tools depend on the tool dependency being present in order to return coherent results. + # If a tool dependency fails to install correctly, this should be considered an installation error, + # and functional tests should be skipped, since the tool dependency needs to be correctly installed + # for the test to be considered reliable. log.error( 'One or more tool dependencies of this repository are marked as missing.' ) log.error( 'Updating repository and skipping functional tests.' ) # In keeping with the standard display layout, add the error message to the dict for each tool individually. Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: Dave Bouvier: Skip functional tests and log error message if a repository's tool dependencies fail to install correctly. Clean up uninstall method.
by commits-noreply＠bitbucket.org 13 May '13

13 May '13

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/ef2b705c7de3/ Changeset: ef2b705c7de3 User: Dave Bouvier Date: 2013-05-13 19:08:39 Summary: Skip functional tests and log error message if a repository's tool dependencies fail to install correctly. Clean up uninstall method. Affected #: 1 file diff -r 604693b3ebe0283804225eac10adab2f858ec380 -r ef2b705c7de344c2c052a36ff3592e78ae5115af test/install_and_test_tool_shed_repositories/functional_tests.py --- a/test/install_and_test_tool_shed_repositories/functional_tests.py +++ b/test/install_and_test_tool_shed_repositories/functional_tests.py @@ -39,6 +39,7 @@ eggs.require( "pysqlite" ) import install_and_test_tool_shed_repositories.functional.test_install_repositories as test_install_repositories +import install_and_test_tool_shed_repositories.base.test_db_util as test_db_util import functional.test_toolbox as test_toolbox import atexit, logging, os, os.path, sys, tempfile, simplejson @@ -185,7 +186,7 @@ sa_session = app.model.context.current repositories_to_uninstall = sa_session.query( app.model.ToolShedRepository ).all() for repository in repositories_to_uninstall: - if repository.status == app.model.ToolShedRepository.states.UNINSTALLED: + if repository.status == app.model.ToolShedRepository.installation_status.UNINSTALLED: continue if repository.status not in [ app.model.ToolShedRepository.installation_status.UNINSTALLED, app.model.ToolShedRepository.installation_status.ERROR, @@ -196,7 +197,7 @@ name = str( repository.name ) owner = str( repository.owner ) changeset_revision = str( repository.installed_changeset_revision ) - log.debug( 'Changeset revision %s of repository %s queued for uninstallation.' % ( changeset_revision, name ) ) + log.debug( 'Changeset revision %s of repository %s queued for uninstallation.', changeset_revision, name ) repository_dict = dict( name=name, owner=owner, changeset_revision=changeset_revision ) # Generate a test method to uninstall this repository through the embedded Galaxy application's web interface. test_install_repositories.generate_uninstall_method( repository_dict ) @@ -326,7 +327,7 @@ global test_toolbox for key in test_toolbox.__dict__: if key.startswith( 'TestForTool_' ): - log.info( 'Tool test found in test_toolbox, deleting: %s' % key ) + log.info( 'Tool test found in test_toolbox, deleting: %s', key ) # We can't delete this test just yet, we're still iterating over __dict__. tests_to_delete.append( key ) tool_id = key.replace( 'TestForTool_', '' ) @@ -493,7 +494,7 @@ for i in range( 0, 9 ): try: galaxy_test_port = str( random.randint( default_galaxy_test_port_min, default_galaxy_test_port_max ) ) - log.debug( "Attempting to serve app on randomly chosen port: %s" % galaxy_test_port ) + log.debug( "Attempting to serve app on randomly chosen port: %s", galaxy_test_port ) server = httpserver.serve( webapp, host=galaxy_test_host, port=galaxy_test_port, start_loop=False ) break except socket.error, e: @@ -522,10 +523,10 @@ raise Exception( "Test HTTP server did not return '200 OK' after 10 tries" ) log.info( "Embedded galaxy web server started" ) if galaxy_test_proxy_port: - log.info( "The embedded Galaxy application is running on %s:%s" % ( galaxy_test_host, galaxy_test_proxy_port ) ) + log.info( "The embedded Galaxy application is running on %s:%s", galaxy_test_host, galaxy_test_proxy_port ) else: - log.info( "The embedded Galaxy application is running on %s:%s" % ( galaxy_test_host, galaxy_test_port ) ) - log.info( "Repositories will be installed from the tool shed at %s" % galaxy_tool_shed_url ) + log.info( "The embedded Galaxy application is running on %s:%s", galaxy_test_host, galaxy_test_port ) + log.info( "Repositories will be installed from the tool shed at %s", galaxy_tool_shed_url ) success = False # If a tool_data_table_conf.test.xml file was found, add the entries from it into the app's tool data tables. if additional_tool_data_tables: @@ -541,9 +542,9 @@ try: detailed_repository_list = [] # Get a list of repositories to test from the tool shed specified in the GALAXY_INSTALL_TEST_TOOL_SHED_URL environment variable. - log.info( "Retrieving repositories to install from the URL:\n%s\n" % str( galaxy_tool_shed_url ) ) + log.info( "Retrieving repositories to install from the URL:\n%s\n", str( galaxy_tool_shed_url ) ) repositories_to_install = get_repositories_to_install( galaxy_tool_shed_url, source='url' ) - log.info( "Retrieved %d repositories from the API." % len( repositories_to_install ) ) + log.info( "Retrieved %d repositories from the API.", len( repositories_to_install ) ) for repository_to_install_dict in repositories_to_install: # We need to get some details from the tool shed API, such as repository name and owner, to pass on to the # module that will generate the install methods. @@ -552,22 +553,23 @@ # and functionally correct tools that someone has previously installed. Deleted repositories have never been installed, # and therefore do not need to be checked. If they are undeleted, this script will then test them the next time it runs. if repository_info_dict[ 'deleted' ]: - log.info( "Skipping revision %s of repository id %s (%s/%s) since the repository is deleted..." % \ - ( repository_to_install_dict[ 'changeset_revision' ], - repository_to_install_dict[ 'repository_id' ], - repository_info_dict[ 'owner' ], - repository_info_dict[ 'name' ] ) ) + log.info( "Skipping revision %s of repository id %s (%s/%s) since the repository is deleted...", + repository_to_install_dict[ 'changeset_revision' ], + repository_to_install_dict[ 'repository_id' ], + repository_info_dict[ 'owner' ], + repository_info_dict[ 'name' ] ) continue # Now merge the dict returned from /api/repository_revisions with the detailed dict we just retrieved. detailed_repository_list.append( dict( repository_info_dict.items() + repository_to_install_dict.items() ) ) repositories_tested = len( detailed_repository_list ) - log.info( 'After removing deleted repositories from the list, %d remain to be tested.' % repositories_tested ) + log.info( 'After removing deleted repositories from the list, %d remain to be tested.', repositories_tested ) if '-list_repositories' in sys.argv: log.info( "The API returned the following repositories, not counting deleted:" ) for repository_info_dict in detailed_repository_list: - log.info( "%s owned by %s changeset revision %s" % ( repository_info_dict.get( 'name', None ), - repository_info_dict.get( 'owner', None ), - repository_info_dict.get( 'changeset_revision', None ) ) ) + log.info( "%s owned by %s changeset revision %s", + repository_info_dict.get( 'name', None ), + repository_info_dict.get( 'owner', None ), + repository_info_dict.get( 'changeset_revision', None ) ) # This loop will iterate through the list of repositories generated by the above code, having already filtered out any # that were marked as deleted. For each repository, it will generate a test method that will use Twill to install that # repository into the embedded Galaxy application that was started up, selecting to install repository and tool @@ -608,8 +610,8 @@ # Get the name and owner out of the repository info dict. name = str( repository_info_dict[ 'name' ] ) owner = str( repository_info_dict[ 'owner' ] ) - log.info( "Installing and testing revision %s of repository id %s (%s/%s)..." % \ - ( str( changeset_revision ), str( repository_id ), owner, name ) ) + log.info( "Installing and testing revision %s of repository id %s (%s/%s)...", + str( changeset_revision ), str( repository_id ), owner, name ) # Explicitly clear tests from twill's test environment. remove_generated_tests( app ) # Use the repository information dict to generate an install method that will install the repository into the embedded @@ -626,7 +628,7 @@ # If the installation succeeds, configure and run functional tests for this repository. This is equivalent to # sh run_functional_tests.sh -installed if success: - log.debug( 'Installation of %s succeeded, running all defined functional tests.' % name ) + log.debug( 'Installation of %s succeeded, running all defined functional tests.', name ) # Generate the shed_tools_dict that specifies the location of test data contained within this repository. If the repository # does not have a test-data directory, this will return has_test_data = False, and we will set the do_not_test flag to True, # and the tools_functionally_correct flag to False, as well as updating tool_test_results. @@ -663,6 +665,15 @@ # "traceback": "The captured traceback." # }, # ] + # "installation_errors": + # [ + # { + # "name": "The name of the repository.", + # "owner": "The owner of the repository.", + # "changeset_revision": "The changeset revision of the repository.", + # "error_message": "The message stored in tool_dependency.error_message." + # }, + # ] # "missing_test_components": # [ # { @@ -682,14 +693,42 @@ repository_status[ 'test_environment' ] = test_environment repository_status[ 'passed_tests' ] = [] repository_status[ 'failed_tests' ] = [] + repository_status[ 'installation_errors' ] = [] + repository = test_db_util.get_installed_repository_by_name_owner_changeset_revision( name, owner, changeset_revision ) + missing_tool_dependencies = repository.includes_tool_dependencies and repository.missing_tool_dependencies if 'missing_test_components' not in repository_status: repository_status[ 'missing_test_components' ] = [] if not has_test_data: + # If the repository does not have a test-data directory, any functional tests in the tool configuration will + # fail. Mark the repository as failed and skip installation. log.error( 'Test data is missing for this repository. Updating repository and skipping functional tests.' ) # Record the lack of test data. - failed_tests = dict( tool_id=None, tool_version=None, tool_guid=None, - missing_components="Repository %s is missing a test-data directory." % name ) - repository_status[ 'missing_test_components' ].append( failed_tests ) + for tool in repository.metadata[ 'tools' ]: + tool_id = tool[ 'id' ] + tool_version = tool[ 'version' ] + tool_guid = tool[ 'guid' ] + # In keeping with the standard display layout, add the error message to the dict for each tool individually. + failed_tests = dict( tool_id=tool_id, tool_version=tool_version, tool_guid=tool_guid, + missing_components="Repository %s is missing a test-data directory." % name ) + repository_status[ 'missing_test_components' ].append( failed_tests ) + # Record the status of this repository in the tool shed. + register_test_result( galaxy_tool_shed_url, metadata_revision_id, repository_status, passed_tests=False ) + # Run the cleanup method. This removes tool functional test methods from the test_toolbox module and uninstalls the + # repository using Twill. + execute_uninstall_method( app ) + # Set the test_toolbox.toolbox module-level variable to the new app.toolbox. + test_toolbox.toolbox = app.toolbox + repositories_failed.append( dict( name=name, owner=owner, changeset_revision=changeset_revision ) ) + elif missing_tool_dependencies: + # If a tool dependency fails to install correctly, this should be considered a missing test component, + # as the tools depend on the tool dependency being present in order to return coherent results. + log.error( 'One or more tool dependencies of this repository are marked as missing.' ) + log.error( 'Updating repository and skipping functional tests.' ) + # In keeping with the standard display layout, add the error message to the dict for each tool individually. + for dependency in repository.missing_tool_dependencies: + test_result = dict( name=name, owner=owner, changeset_revision=changeset_revision, + error_message=dependency.error_message ) + repository_status[ 'installation_errors' ].append( test_result ) # Record the status of this repository in the tool shed. register_test_result( galaxy_tool_shed_url, metadata_revision_id, repository_status, passed_tests=False ) # Run the cleanup method. This removes tool functional test methods from the test_toolbox module and uninstalls the @@ -702,7 +741,7 @@ # If the repository does have a test-data directory, we write the generated shed_tools_dict to a file, so the functional # test framework can find it. file( galaxy_shed_tools_dict, 'w' ).write( to_json_string( shed_tools_dict ) ) - log.info( 'Saved generated shed_tools_dict to %s\nContents: %s' % ( galaxy_shed_tools_dict, str( shed_tools_dict ) ) ) + log.info( 'Saved generated shed_tools_dict to %s\nContents: %s', galaxy_shed_tools_dict, str( shed_tools_dict ) ) # Set the GALAXY_TOOL_SHED_TEST_FILE environment variable to the path of the shed_tools_dict file, so that test.base.twilltestcase.setUp # will find and parse it properly. os.environ[ 'GALAXY_TOOL_SHED_TEST_FILE' ] = galaxy_shed_tools_dict @@ -727,6 +766,7 @@ break repository_status[ 'passed_tests' ] = [] for test_id in passed_tests: + # Normalize the tool ID and version display. tool_id, tool_version = get_tool_info_from_test_id( test_id ) test_result = dict( test_id=test_id, tool_id=tool_id, tool_version=tool_version ) repository_status[ 'passed_tests' ].append( test_result ) @@ -737,7 +777,7 @@ # updates the time_last_tested field to today's date. repositories_passed.append( dict( name=name, owner=owner, changeset_revision=changeset_revision ) ) register_test_result( galaxy_tool_shed_url, metadata_revision_id, repository_status, passed_tests=True ) - log.debug( 'Revision %s of repository %s installed and passed functional tests.' % ( changeset_revision, name ) ) + log.debug( 'Revision %s of repository %s installed and passed functional tests.', changeset_revision, name ) else: # If the functional tests fail, log the output and update the failed changeset revision's metadata record in the tool shed via the API. for failure in result.failures + result.errors: @@ -782,25 +822,26 @@ # field to today's date. repositories_failed.append( dict( name=name, owner=owner, changeset_revision=changeset_revision ) ) register_test_result( galaxy_tool_shed_url, metadata_revision_id, repository_status, passed_tests=False ) - log.debug( 'Revision %s of repository %s installed successfully, but did not pass functional tests.' % \ - ( changeset_revision, name ) ) + log.debug( 'Revision %s of repository %s installed successfully, but did not pass functional tests.', + changeset_revision, name ) # Run the uninstall method. This removes tool functional test methods from the test_toolbox module and uninstalls the # repository using Twill. - log.debug( 'Uninstalling changeset revision %s of repository %s' % \ - ( repository_info_dict[ 'changeset_revision' ], repository_info_dict[ 'name' ] ) ) + log.debug( 'Uninstalling changeset revision %s of repository %s', + repository_info_dict[ 'changeset_revision' ], + repository_info_dict[ 'name' ] ) success = execute_uninstall_method( app ) if not success: - log.error( 'Repository %s failed to uninstall.' ) + log.error( 'Repository %s failed to uninstall.', repository_info_dict[ 'name' ] ) # Set the test_toolbox.toolbox module-level variable to the new app.toolbox. test_toolbox.toolbox = app.toolbox else: # Even if the repository failed to install, execute the uninstall method, in case a dependency did succeed. - log.debug( 'Uninstalling repository %s' % repository_info_dict[ 'name' ] ) + log.debug( 'Uninstalling repository %s', repository_info_dict[ 'name' ] ) success = execute_uninstall_method( app ) if not success: - log.error( 'Repository %s failed to uninstall.' ) + log.error( 'Repository %s failed to uninstall.', repository_info_dict[ 'name' ] ) repositories_failed_install.append( dict( name=name, owner=owner, changeset_revision=changeset_revision ) ) - log.debug( 'Repository %s failed to install correctly.' % repository_info_dict[ 'name' ] ) + log.debug( 'Repository %s failed to install correctly.', repository_info_dict[ 'name' ] ) except: log.exception( "Failure running tests" ) @@ -822,7 +863,7 @@ try: for dir in [ galaxy_test_tmp_dir ]: if os.path.exists( dir ): - log.info( "Cleaning up temporary files in %s" % dir ) + log.info( "Cleaning up temporary files in %s", dir ) shutil.rmtree( dir ) except: pass Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: greg: Fixes for handling large numbers of tool shed repositories during installation into Galaxy.
by commits-noreply＠bitbucket.org 13 May '13

13 May '13

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/604693b3ebe0/ Changeset: 604693b3ebe0 User: greg Date: 2013-05-13 17:10:50 Summary: Fixes for handling large numbers of tool shed repositories during installation into Galaxy. Affected #: 5 files diff -r 65a81aead95e147f709ee3969d49766f35d6a2e2 -r 604693b3ebe0283804225eac10adab2f858ec380 lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py --- a/lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py +++ b/lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py @@ -313,6 +313,11 @@ """Return an svg image representation of a workflow dictionary created when the workflow was exported.""" return workflow_util.generate_workflow_image( trans, workflow_name, repository_metadata_id=None, repository_id=repository_id ) + @web.expose + def get_contents_of_file( self, trans, encoded_file_path ): + file_path = encoding_util.tool_shed_decode( encoded_file_path ) + return suc.get_file_contents( file_path ) + @web.json @web.require_admin def get_file_contents( self, trans, file_path ): @@ -383,6 +388,34 @@ tool_version = tool_util.get_tool_version( app, guid ) return tool_version.get_version_ids( app, reverse=True ) + @web.json + def handle_large_repo_info_dict( self, trans, **kwd ): + """ + In some cases the required encoded_str will be long. With apache, the default limit for the length of the request line is 8190 bytes + (http://httpd.apache.org/docs/2.2/mod/core.html#limitrequestline) And if we subtract three bytes for the request method (i.e. GET), + eight bytes for the version information (i.e. HTTP/1.0/HTTP/1.1) and two bytes for the separating space, we end up with 8177 bytes for + the URI path plus query. The referer handles requests longer than this by persisting the repo_info_dict to a temporary file which we can read. + """ + required_repo_info_dict = {} + encoded_tmp_file_name = kwd.get( 'encoded_tmp_file_name', None ) + # The request would have been longer than 8190 bytes if it included the encoded_str, so we'll send a request to the Galaxy instance to get it. + tool_shed_url = kwd.get( 'tool_shed_url', None ) + if tool_shed_url and encoded_tmp_file_name: + url = suc.url_join( tool_shed_url, + '/repository/get_contents_of_file?encoded_tmp_file_name=%s' % encoded_tmp_file_name ) + response = urllib2.urlopen( url ) + text = response.read() + required_repo_info_dict = json.from_json_string( text ) + else: + log.debug( "Invalid tool_shed_url '%s' or encoded_tmp_file_name '%s'." % ( str( tool_shed_url ), str( encoded_tmp_file_name ) ) ) + tmp_file_name = encoding_util.tool_shed_decode( encoded_tmp_file_name ) + if os.path.exists( tmp_file_name ): + try: + os.unkink( tmp_file_name ) + except: + pass + return common_install_util.process_repo_info_dict( trans, required_repo_info_dict ) + @web.expose @web.require_admin def import_workflow( self, trans, workflow_name, repository_id, **kwd ): @@ -1406,15 +1439,6 @@ message=message, status=status ) - @web.expose - def stream_file_contents( self, trans, encoded_tmp_file_name ): - tmp_file_name = encoding_util.tool_shed_decode( encoded_tmp_file_name ) - if os.path.exists( tmp_file_name ): - return open( tmp_file_name, 'r' ) - else: - log.debug( "The required temporary file '%s' cannot be located." % str( tmp_file_name ) ) - return '' - @web.json def tool_dependency_status_updates( self, trans, ids=None, status_list=None ): # Avoid caching diff -r 65a81aead95e147f709ee3969d49766f35d6a2e2 -r 604693b3ebe0283804225eac10adab2f858ec380 lib/galaxy/webapps/tool_shed/controllers/repository.py --- a/lib/galaxy/webapps/tool_shed/controllers/repository.py +++ b/lib/galaxy/webapps/tool_shed/controllers/repository.py @@ -1285,6 +1285,11 @@ return encoding_util.tool_shed_encode( update_dict ) @web.expose + def get_contents_of_file( self, trans, encoded_file_path ): + file_path = encoding_util.tool_shed_decode( encoded_file_path ) + return suc.get_file_contents( file_path ) + + @web.expose def get_ctx_rev( self, trans, **kwd ): """Given a repository and changeset_revision, return the correct ctx.rev() value.""" repository_name = kwd[ 'name' ] @@ -1473,32 +1478,12 @@ repo_info_dicts=repo_info_dicts ) @web.json - def get_required_repo_info_dict( self, trans, encoded_str=None, **kwd ): + def get_required_repo_info_dict( self, trans, encoded_str, **kwd ): """ Retrieve and return a dictionary that includes a list of dictionaries that each contain all of the information needed to install the list of repositories defined by the received encoded_str. """ - # In some cases the received encoded_str will be long. With apache, the default limit for the length of the request line is 8190 bytes - # (http://httpd.apache.org/docs/2.2/mod/core.html#limitrequestline) And if we subtract three bytes for the request method (i.e. GET), - # eight bytes for the version information (i.e. HTTP/1.0/HTTP/1.1) and two bytes for the separating space, we end up with 8177 bytes for - # the URI path plus query. The referer handles requests longer than this by persisting the encoded_str to a temporary file which we - # can read. - encoded_tmp_file_name = kwd.get( 'encoded_tmp_file_name', None ) - if encoded_str: - encoded_required_repository_str = encoding_util.tool_shed_decode( encoded_str ) - else: - # The request would have been longer than 8190 bytes if it included the encoded_str, so we'll send a request to the Galaxy instance to - # stream the string to us. - galaxy_url = suc.handle_galaxy_url( trans, **kwd ) - if galaxy_url and encoded_tmp_file_name: - url = suc.url_join( galaxy_url, - 'admin_toolshed/stream_file_contents?encoded_tmp_file_name=%s' % encoded_tmp_file_name ) - response = urllib2.urlopen( url ) - encoded_required_repository_str = response.read() - else: - log.debug( "Invalid galaxy_url '%s' or encoded_tmp_file_name '%s'." % ( str( galaxy_url ), str( encoded_tmp_file_name ) ) ) - repo_info_dict = {} - return repo_info_dict + encoded_required_repository_str = encoding_util.tool_shed_decode( encoded_str ) encoded_required_repository_tups = encoded_required_repository_str.split( encoding_util.encoding_sep2 ) decoded_required_repository_tups = [] for encoded_required_repository_tup in encoded_required_repository_tups: @@ -1675,6 +1660,41 @@ return tool_guid_lineage @web.expose + def handle_large_repo_info_dict( self, trans, **kwd ): + """ + In some cases the required encoded_str will be long. With apache, the default limit for the length of the request line is 8190 bytes + (http://httpd.apache.org/docs/2.2/mod/core.html#limitrequestline) And if we subtract three bytes for the request method (i.e. GET), + eight bytes for the version information (i.e. HTTP/1.0/HTTP/1.1) and two bytes for the separating space, we end up with 8177 bytes for + the URI path plus query. The referer handles requests longer than this by persisting the encoded_str to a temporary file which we can read. + """ + repo_info_dict = {} + encoded_tmp_file_name = kwd.get( 'encoded_tmp_file_name', None ) + # The request would have been longer than 8190 bytes if it included the encoded_str, so we'll send a request to the Galaxy instance to get it. + galaxy_url = suc.handle_galaxy_url( trans, **kwd ) + if galaxy_url and encoded_tmp_file_name: + url = suc.url_join( galaxy_url, + '/admin_toolshed/get_contents_of_file?encoded_tmp_file_name=%s' % encoded_tmp_file_name ) + response = urllib2.urlopen( url ) + encoded_required_repository_str = response.read() + repo_info_dict = self.get_required_repo_info_dict( trans, encoded_str ) + repo_info_dict[ 'encoded_tmp_file_name' ] = encoded_tmp_file_name + else: + log.debug( "Invalid galaxy_url '%s' or encoded_tmp_file_name '%s'." % ( str( galaxy_url ), str( encoded_tmp_file_name ) ) ) + # Persist the large repo_info_dict to a temporary file. + fh = tempfile.NamedTemporaryFile( 'wb' ) + tmp_file_name = fh.name + fh.close() + fh = open( tmp_file_name, 'wb' ) + fh.write( encoded_required_repository_str ) + fh.close() + encoded_tmp_file_name = encoding_util.tool_shed_encode( os.path.abspath( tmp_file_name ) ) + tool_shed_url = web.url_for( '/', qualified=True ) + # Redirect to the tool shed to enable it to read the persisted encoded_required_repository_str. + url = suc.url_join( galaxy_url, + '/admin_toolshed/handle_large_repo_info_dict?encoded_tmp_file_name=%s&tool_shed_url=%s' % ( str( encoded_tmp_file_name ), str( tool_shed_url ) ) ) + return trans.response.send_redirect( url ) + + @web.expose def help( self, trans, **kwd ): params = util.Params( kwd ) message = util.restore_text( params.get( 'message', '' ) ) diff -r 65a81aead95e147f709ee3969d49766f35d6a2e2 -r 604693b3ebe0283804225eac10adab2f858ec380 lib/tool_shed/util/common_install_util.py --- a/lib/tool_shed/util/common_install_util.py +++ b/lib/tool_shed/util/common_install_util.py @@ -17,6 +17,9 @@ import pkg_resources +pkg_resources.require( "simplejson" ) +import simplejson + pkg_resources.require( 'elementtree' ) from elementtree import ElementTree from elementtree import ElementInclude @@ -253,7 +256,6 @@ repository_dependencies entries in each of the received repo_info_dicts includes all required repositories, so only one pass through this method is required to retrieve all repository dependencies. """ - all_repo_info_dicts = [] if repo_info_dicts: # We'll send tuples of ( tool_shed, repository_name, repository_owner, changeset_revision ) to the tool shed to discover repository ids. required_repository_tups = [] @@ -287,59 +289,47 @@ encoded_required_repository_str = encoding_util.encoding_sep2.join( encoded_required_repository_tups ) encoded_required_repository_str = encoding_util.tool_shed_encode( encoded_required_repository_str ) url = suc.url_join( tool_shed_url, '/repository/get_required_repo_info_dict?encoded_str=%s' % encoded_required_repository_str ) - # In some cases the above URL will be long. With apache, the default limit for the length of the request line is 8190 bytes - # (http://httpd.apache.org/docs/2.2/mod/core.html#limitrequestline) And if we subtract three bytes for the request method - # (i.e. GET), eight bytes for the version information (i.e. HTTP/1.0/HTTP/1.1) and two bytes for the separating space, we end - # up with 8177 bytes for the URI path plus query. - if len( url ) >= 8177: - # Persist the encoded string to a temporary file. - fh = tempfile.NamedTemporaryFile( 'wb' ) - tmp_file_name = fh.name - fh.close() - fh = open( tmp_file_name, 'wb' ) - fh.write( encoded_required_repository_str ) - fh.close() - encoded_tmp_file_name = encoding_util.tool_shed_encode( os.path.abspath( tmp_file_name ) ) - galaxy_url = web.url_for( '/', qualified=True ) - # Send a request to the tool shed to enable it to read the temporary file. - url = suc.url_join( tool_shed_url, - '/repository/get_required_repo_info_dict?encoded_tmp_file_name=%s&galaxy_url=%s' % \ - ( encoded_tmp_file_name, galaxy_url ) ) - else: - encoded_tmp_file_name = None - tmp_file_name = None - try: - text = common_util.tool_shed_get( trans.app, tool_shed_url, url ) - except Exception, e: - if encoded_tmp_file_name: - message = 'The selected tool shed repositories cannot be installed until the tool shed at ' - message += '%s and the Galaxy instance at %s ' % ( str( tool_shed_url ), str( trans.request.base ) ) - message += 'are both updated to at least the June 3, 2013 Galaxy release. These upgrades ' - message += 'are necessary because the number of repositories you are attempting to install ' - message += 'generates an HTTP request that is longer than 8177 bytes which cannot be handled ' - message += 'by tool shed or Galaxy instances older than this release.' - log.debug( message ) - else: + text = common_util.tool_shed_get( trans.app, tool_shed_url, url ) + if text: + try: + required_repo_info_dict = json.from_json_string( text ) + except simplejson.decoder.JSONDecodeError, e: + if len( url ) >= 8177: + message = '\n\nThe selected tool shed repositories cannot be installed until the tool shed at %s and the Galaxy ' % str( tool_shed_url ) + message += 'instance at %s are both updated to at least the June 3, 2013 Galaxy release. These upgrades ' % str( trans.request.base ) + message += 'are necessary because the number of repositories you are attempting to install generates an HTTP request that is longer than ' + message += '8177 bytes which cannot be handled by tool shed or Galaxy instances older than the June 3, 2013 release.\n\n' + log.exception( message ) + else: + log.exception() + return [] + except Exception, e: log.exception() - text = None - if tmp_file_name: - try: - os.unlink( tmp_file_name ) - except: - pass - if text: - required_repo_info_dict = json.from_json_string( text ) - required_repo_info_dicts = [] - encoded_dict_strings = required_repo_info_dict[ 'repo_info_dicts' ] - for encoded_dict_str in encoded_dict_strings: - decoded_dict = encoding_util.tool_shed_decode( encoded_dict_str ) - required_repo_info_dicts.append( decoded_dict ) - if required_repo_info_dicts: - for required_repo_info_dict in required_repo_info_dicts: - if required_repo_info_dict not in all_repo_info_dicts: - all_repo_info_dicts.append( required_repo_info_dict ) - return all_repo_info_dicts + return [] + return process_repo_info_dict( trans, required_repo_info_dict ) + return [] +def handle_large_repo_info_dict( trans, tool_shed_url, encoded_required_repository_str ): + """ + Handle the cases where the received encoded_required_repository_str is long. With apache, the default limit for the length of the request line is 8190 bytes + (http://httpd.apache.org/docs/2.2/mod/core.html#limitrequestline) And if we subtract three bytes for the request method (i.e. GET), eight bytes for the version + information (i.e. HTTP/1.0/HTTP/1.1) and two bytes for the separating space, we end up with 8177 bytes for the URI path plus query. + """ + # Persist the encoded string to a temporary file. + fh = tempfile.NamedTemporaryFile( 'wb' ) + tmp_file_name = fh.name + fh.close() + fh = open( tmp_file_name, 'wb' ) + fh.write( encoded_required_repository_str ) + fh.close() + encoded_tmp_file_name = encoding_util.tool_shed_encode( os.path.abspath( tmp_file_name ) ) + galaxy_url = web.url_for( '/', qualified=True ) + # Redirect to the tool shed to enable it to read the persisted encoded_required_repository_str. + url = suc.url_join( tool_shed_url, + '/repository/handle_large_repo_info_dict?encoded_tmp_file_name=%s&galaxy_url=%s' % \ + ( encoded_tmp_file_name, galaxy_url ) ) + return trans.response.send_redirect( url ) + def handle_tool_dependencies( app, tool_shed_repository, tool_dependencies_config, tool_dependencies ): """ Install and build tool dependencies defined in the tool_dependencies_config. This config's tag sets can currently refer to installation @@ -397,3 +387,24 @@ app.model.ToolDependency.installation_status.ERROR ]: installed_tool_dependencies.append( tool_dependency ) return installed_tool_dependencies + +def process_repo_info_dict( trans, required_repo_info_dict ): + all_repo_info_dicts = [] + required_repo_info_dicts = [] + encoded_dict_strings = required_repo_info_dict[ 'repo_info_dicts' ] + for encoded_dict_str in encoded_dict_strings: + decoded_dict = encoding_util.tool_shed_decode( encoded_dict_str ) + required_repo_info_dicts.append( decoded_dict ) + if required_repo_info_dicts: + for required_repo_info_dict in required_repo_info_dicts: + if required_repo_info_dict not in all_repo_info_dicts: + all_repo_info_dicts.append( required_repo_info_dict ) + # Remove the temporary file that stored the long encoded_required_repository_str if possible. + encoded_tmp_file_name = required_repo_info_dict.get( 'encoded_tmp_file_name', None ) + if encoded_tmp_file_name: + tmp_file_name = encoding_util.tool_shed_decode( encoded_tmp_file_name ) + try: + os.unlink( tmp_file_name ) + except: + pass + return all_repo_info_dicts diff -r 65a81aead95e147f709ee3969d49766f35d6a2e2 -r 604693b3ebe0283804225eac10adab2f858ec380 lib/tool_shed/util/common_util.py --- a/lib/tool_shed/util/common_util.py +++ b/lib/tool_shed/util/common_util.py @@ -27,7 +27,7 @@ url = '%s/repository/get_tool_dependencies?name=%s&owner=%s&changeset_revision=%s&from_install_manager=True' % \ ( tool_shed_url, repository_name, REPOSITORY_OWNER, changeset_revision ) try: - text = tool_shed_get(app, tool_shed_url, url) + text = tool_shed_get( app, tool_shed_url, url ) tool_shed_accessible = True except Exception, e: # Tool shed may be unavailable - we have to set tool_shed_accessible since we're looping. diff -r 65a81aead95e147f709ee3969d49766f35d6a2e2 -r 604693b3ebe0283804225eac10adab2f858ec380 lib/tool_shed/util/shed_util_common.py --- a/lib/tool_shed/util/shed_util_common.py +++ b/lib/tool_shed/util/shed_util_common.py @@ -466,6 +466,14 @@ return manifest_ctx, ctx_file return None, None +def get_file_contents( file_path ): + if os.path.exists( file_path ): + fh = open( file_path ) + contents = fh.read() + fh.close() + return contents + return '' + def get_file_context_from_ctx( ctx, filename ): """Return the mercurial file context for a specified file.""" # We have to be careful in determining if we found the correct file because multiple files with the same name may be in different directories Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: greg: Fixes for sending file contents to the tool shed.
by commits-noreply＠bitbucket.org 12 May '13

12 May '13

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/65a81aead95e/ Changeset: 65a81aead95e User: greg Date: 2013-05-12 17:52:55 Summary: Fixes for sending file contents to the tool shed. Affected #: 4 files diff -r bc5cd95d042bc062c688bee6b3e9cbf74d853fb1 -r 65a81aead95e147f709ee3969d49766f35d6a2e2 lib/galaxy/webapps/galaxy/api/tool_shed_repositories.py --- a/lib/galaxy/webapps/galaxy/api/tool_shed_repositories.py +++ b/lib/galaxy/webapps/galaxy/api/tool_shed_repositories.py @@ -362,4 +362,3 @@ elif isinstance( installed_tool_shed_repositories, list ): all_installed_tool_shed_repositories.extend( installed_tool_shed_repositories ) return all_installed_tool_shed_repositories - diff -r bc5cd95d042bc062c688bee6b3e9cbf74d853fb1 -r 65a81aead95e147f709ee3969d49766f35d6a2e2 lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py --- a/lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py +++ b/lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py @@ -1406,6 +1406,15 @@ message=message, status=status ) + @web.expose + def stream_file_contents( self, trans, encoded_tmp_file_name ): + tmp_file_name = encoding_util.tool_shed_decode( encoded_tmp_file_name ) + if os.path.exists( tmp_file_name ): + return open( tmp_file_name, 'r' ) + else: + log.debug( "The required temporary file '%s' cannot be located." % str( tmp_file_name ) ) + return '' + @web.json def tool_dependency_status_updates( self, trans, ids=None, status_list=None ): # Avoid caching diff -r bc5cd95d042bc062c688bee6b3e9cbf74d853fb1 -r 65a81aead95e147f709ee3969d49766f35d6a2e2 lib/galaxy/webapps/tool_shed/controllers/repository.py --- a/lib/galaxy/webapps/tool_shed/controllers/repository.py +++ b/lib/galaxy/webapps/tool_shed/controllers/repository.py @@ -1491,11 +1491,14 @@ # stream the string to us. galaxy_url = suc.handle_galaxy_url( trans, **kwd ) if galaxy_url and encoded_tmp_file_name: - url = suc.url_join( galaxy_url, '/common_install_util/stream_file_contents?encoded_tmp_file_name=%s' % encoded_tmp_file_name ) + url = suc.url_join( galaxy_url, + 'admin_toolshed/stream_file_contents?encoded_tmp_file_name=%s' % encoded_tmp_file_name ) response = urllib2.urlopen( url ) encoded_required_repository_str = response.read() else: - raise Exception( "Required galaxy_url or encoded_tmp_file_name request parameters missing." ) + log.debug( "Invalid galaxy_url '%s' or encoded_tmp_file_name '%s'." % ( str( galaxy_url ), str( encoded_tmp_file_name ) ) ) + repo_info_dict = {} + return repo_info_dict encoded_required_repository_tups = encoded_required_repository_str.split( encoding_util.encoding_sep2 ) decoded_required_repository_tups = [] for encoded_required_repository_tup in encoded_required_repository_tups: diff -r bc5cd95d042bc062c688bee6b3e9cbf74d853fb1 -r 65a81aead95e147f709ee3969d49766f35d6a2e2 lib/tool_shed/util/common_install_util.py --- a/lib/tool_shed/util/common_install_util.py +++ b/lib/tool_shed/util/common_install_util.py @@ -3,6 +3,7 @@ import tempfile from galaxy import eggs from galaxy import util +from galaxy import web from galaxy.util import json from galaxy.webapps.tool_shed.util import container_util import tool_shed.util.shed_util_common as suc @@ -396,7 +397,3 @@ app.model.ToolDependency.installation_status.ERROR ]: installed_tool_dependencies.append( tool_dependency ) return installed_tool_dependencies - -def stream_file_contents( trans, encoded_tmp_file_name ): - tmp_file_name = encoding_util.tool_shed_decode( encoded_tmp_file_name ) - return open( tmp_file_name ) Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: greg: Include the galaxy_url param in the new request to the tool shed.
by commits-noreply＠bitbucket.org 12 May '13

12 May '13

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/bc5cd95d042b/ Changeset: bc5cd95d042b User: greg Date: 2013-05-12 17:05:49 Summary: Include the galaxy_url param in the new request to the tool shed. Affected #: 1 file diff -r dc7d35578676935777d1f315693a4c15498919e8 -r bc5cd95d042bc062c688bee6b3e9cbf74d853fb1 lib/tool_shed/util/common_install_util.py --- a/lib/tool_shed/util/common_install_util.py +++ b/lib/tool_shed/util/common_install_util.py @@ -299,8 +299,11 @@ fh.write( encoded_required_repository_str ) fh.close() encoded_tmp_file_name = encoding_util.tool_shed_encode( os.path.abspath( tmp_file_name ) ) + galaxy_url = web.url_for( '/', qualified=True ) # Send a request to the tool shed to enable it to read the temporary file. - url = suc.url_join( tool_shed_url, '/repository/get_required_repo_info_dict?encoded_tmp_file_name=%s' % encoded_tmp_file_name ) + url = suc.url_join( tool_shed_url, + '/repository/get_required_repo_info_dict?encoded_tmp_file_name=%s&galaxy_url=%s' % \ + ( encoded_tmp_file_name, galaxy_url ) ) else: encoded_tmp_file_name = None tmp_file_name = None @@ -311,7 +314,7 @@ message = 'The selected tool shed repositories cannot be installed until the tool shed at ' message += '%s and the Galaxy instance at %s ' % ( str( tool_shed_url ), str( trans.request.base ) ) message += 'are both updated to at least the June 3, 2013 Galaxy release. These upgrades ' - message += 'are necessary because the number of repositories you are attemping to install ' + message += 'are necessary because the number of repositories you are attempting to install ' message += 'generates an HTTP request that is longer than 8177 bytes which cannot be handled ' message += 'by tool shed or Galaxy instances older than this release.' log.debug( message ) Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: greg: Persist the encoded string resulting from the information needed to install tool shed repositories if the string generates an HTTP request longer than 8177 bytes and enhance the necessary tool shed and Galaxy methods to handle this scenario.
by commits-noreply＠bitbucket.org 12 May '13

12 May '13

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/dc7d35578676/ Changeset: dc7d35578676 User: greg Date: 2013-05-12 16:47:33 Summary: Persist the encoded string resulting from the information needed to install tool shed repositories if the string generates an HTTP request longer than 8177 bytes and enhance the necessary tool shed and Galaxy methods to handle this scenario. Affected #: 2 files diff -r d09a3531b3d35a3921fdaa52beed7481f0391ca8 -r dc7d35578676935777d1f315693a4c15498919e8 lib/galaxy/webapps/tool_shed/controllers/repository.py --- a/lib/galaxy/webapps/tool_shed/controllers/repository.py +++ b/lib/galaxy/webapps/tool_shed/controllers/repository.py @@ -4,9 +4,11 @@ import re import string import tempfile +import urllib2 from time import gmtime from time import strftime -from datetime import date, datetime +from datetime import date +from datetime import datetime from galaxy import util from galaxy import web from galaxy.util.odict import odict @@ -1471,12 +1473,29 @@ repo_info_dicts=repo_info_dicts ) @web.json - def get_required_repo_info_dict( self, trans, encoded_str ): + def get_required_repo_info_dict( self, trans, encoded_str=None, **kwd ): """ Retrieve and return a dictionary that includes a list of dictionaries that each contain all of the information needed to install the list of repositories defined by the received encoded_str. """ - encoded_required_repository_str = encoding_util.tool_shed_decode( encoded_str ) + # In some cases the received encoded_str will be long. With apache, the default limit for the length of the request line is 8190 bytes + # (http://httpd.apache.org/docs/2.2/mod/core.html#limitrequestline) And if we subtract three bytes for the request method (i.e. GET), + # eight bytes for the version information (i.e. HTTP/1.0/HTTP/1.1) and two bytes for the separating space, we end up with 8177 bytes for + # the URI path plus query. The referer handles requests longer than this by persisting the encoded_str to a temporary file which we + # can read. + encoded_tmp_file_name = kwd.get( 'encoded_tmp_file_name', None ) + if encoded_str: + encoded_required_repository_str = encoding_util.tool_shed_decode( encoded_str ) + else: + # The request would have been longer than 8190 bytes if it included the encoded_str, so we'll send a request to the Galaxy instance to + # stream the string to us. + galaxy_url = suc.handle_galaxy_url( trans, **kwd ) + if galaxy_url and encoded_tmp_file_name: + url = suc.url_join( galaxy_url, '/common_install_util/stream_file_contents?encoded_tmp_file_name=%s' % encoded_tmp_file_name ) + response = urllib2.urlopen( url ) + encoded_required_repository_str = response.read() + else: + raise Exception( "Required galaxy_url or encoded_tmp_file_name request parameters missing." ) encoded_required_repository_tups = encoded_required_repository_str.split( encoding_util.encoding_sep2 ) decoded_required_repository_tups = [] for encoded_required_repository_tup in encoded_required_repository_tups: diff -r d09a3531b3d35a3921fdaa52beed7481f0391ca8 -r dc7d35578676935777d1f315693a4c15498919e8 lib/tool_shed/util/common_install_util.py --- a/lib/tool_shed/util/common_install_util.py +++ b/lib/tool_shed/util/common_install_util.py @@ -1,5 +1,6 @@ import logging import os +import tempfile from galaxy import eggs from galaxy import util from galaxy.util import json @@ -285,7 +286,43 @@ encoded_required_repository_str = encoding_util.encoding_sep2.join( encoded_required_repository_tups ) encoded_required_repository_str = encoding_util.tool_shed_encode( encoded_required_repository_str ) url = suc.url_join( tool_shed_url, '/repository/get_required_repo_info_dict?encoded_str=%s' % encoded_required_repository_str ) - text = common_util.tool_shed_get( trans.app, tool_shed_url, url ) + # In some cases the above URL will be long. With apache, the default limit for the length of the request line is 8190 bytes + # (http://httpd.apache.org/docs/2.2/mod/core.html#limitrequestline) And if we subtract three bytes for the request method + # (i.e. GET), eight bytes for the version information (i.e. HTTP/1.0/HTTP/1.1) and two bytes for the separating space, we end + # up with 8177 bytes for the URI path plus query. + if len( url ) >= 8177: + # Persist the encoded string to a temporary file. + fh = tempfile.NamedTemporaryFile( 'wb' ) + tmp_file_name = fh.name + fh.close() + fh = open( tmp_file_name, 'wb' ) + fh.write( encoded_required_repository_str ) + fh.close() + encoded_tmp_file_name = encoding_util.tool_shed_encode( os.path.abspath( tmp_file_name ) ) + # Send a request to the tool shed to enable it to read the temporary file. + url = suc.url_join( tool_shed_url, '/repository/get_required_repo_info_dict?encoded_tmp_file_name=%s' % encoded_tmp_file_name ) + else: + encoded_tmp_file_name = None + tmp_file_name = None + try: + text = common_util.tool_shed_get( trans.app, tool_shed_url, url ) + except Exception, e: + if encoded_tmp_file_name: + message = 'The selected tool shed repositories cannot be installed until the tool shed at ' + message += '%s and the Galaxy instance at %s ' % ( str( tool_shed_url ), str( trans.request.base ) ) + message += 'are both updated to at least the June 3, 2013 Galaxy release. These upgrades ' + message += 'are necessary because the number of repositories you are attemping to install ' + message += 'generates an HTTP request that is longer than 8177 bytes which cannot be handled ' + message += 'by tool shed or Galaxy instances older than this release.' + log.debug( message ) + else: + log.exception() + text = None + if tmp_file_name: + try: + os.unlink( tmp_file_name ) + except: + pass if text: required_repo_info_dict = json.from_json_string( text ) required_repo_info_dicts = [] @@ -356,3 +393,7 @@ app.model.ToolDependency.installation_status.ERROR ]: installed_tool_dependencies.append( tool_dependency ) return installed_tool_dependencies + +def stream_file_contents( trans, encoded_tmp_file_name ): + tmp_file_name = encoding_util.tool_shed_decode( encoded_tmp_file_name ) + return open( tmp_file_name ) Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: greg: Fix for updating metadata for a tool shed repository that defines complex repository dependencies.
by commits-noreply＠bitbucket.org 11 May '13

11 May '13

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/d09a3531b3d3/ Changeset: d09a3531b3d3 User: greg Date: 2013-05-11 19:16:09 Summary: Fix for updating metadata for a tool shed repository that defines complex repository dependencies. Affected #: 3 files diff -r 7dc222a7163a5049cc407ad1a049dd091ac27b03 -r d09a3531b3d35a3921fdaa52beed7481f0391ca8 lib/tool_shed/galaxy_install/repository_util.py --- a/lib/tool_shed/galaxy_install/repository_util.py +++ b/lib/tool_shed/galaxy_install/repository_util.py @@ -487,9 +487,9 @@ tool_shed_repository, trans.model.ToolShedRepository.installation_status.SETTING_TOOL_VERSIONS ) tool_shed_url = suc.get_url_from_tool_shed( trans.app, tool_shed_repository.tool_shed ) - url = suc.url_join( tool_shed_url, - '/repository/get_tool_versions?name=%s&owner=%s&changeset_revision=%s' % - ( tool_shed_repository.name, tool_shed_repository.owner, tool_shed_repository.changeset_revision ) ) + url = suc.url_join( tool_shed_url, + '/repository/get_tool_versions?name=%s&owner=%s&changeset_revision=%s' % + ( tool_shed_repository.name, tool_shed_repository.owner, tool_shed_repository.changeset_revision ) ) text = common_util.tool_shed_get( trans.app, tool_shed_url, url ) if text: tool_version_dicts = json.from_json_string( text ) diff -r 7dc222a7163a5049cc407ad1a049dd091ac27b03 -r d09a3531b3d35a3921fdaa52beed7481f0391ca8 lib/tool_shed/util/metadata_util.py --- a/lib/tool_shed/util/metadata_util.py +++ b/lib/tool_shed/util/metadata_util.py @@ -1857,23 +1857,14 @@ else: tool_shed, name, owner, changeset_revision, prior_installation_required, error_message = repository_dependency_tup prior_installation_required = util.asbool( str( prior_installation_required ) ) - rd_key = container_util.generate_repository_dependencies_key_for_repository( toolshed_base_url=tool_shed, - repository_name=name, - repository_owner=owner, - changeset_revision=changeset_revision, - prior_installation_required=prior_installation_required ) if repository_dependencies_dict: - if rd_key in repository_dependencies_dict: - repository_dependencies = repository_dependencies_dict[ rd_key ] - for repository_dependency_tup in repository_dependency_tups: - if repository_dependency_tup not in repository_dependencies: - repository_dependencies.append( repository_dependency_tup ) - repository_dependencies_dict[ rd_key ] = repository_dependencies - else: - repository_dependencies_dict[ rd_key ] = repository_dependency_tups + repository_dependencies = repository_dependencies_dict.get( 'repository_dependencies', [] ) + for repository_dependency_tup in repository_dependency_tups: + if repository_dependency_tup not in repository_dependencies: + repository_dependencies.append( repository_dependency_tup ) + repository_dependencies_dict[ 'repository_dependencies' ] = repository_dependencies else: - repository_dependencies_dict = dict( root_key=rd_key, - description=description, + repository_dependencies_dict = dict( description=description, repository_dependencies=repository_dependency_tups ) if repository_dependencies_dict: if is_valid: diff -r 7dc222a7163a5049cc407ad1a049dd091ac27b03 -r d09a3531b3d35a3921fdaa52beed7481f0391ca8 lib/tool_shed/util/shed_util_common.py --- a/lib/tool_shed/util/shed_util_common.py +++ b/lib/tool_shed/util/shed_util_common.py @@ -198,7 +198,7 @@ def create_or_update_tool_shed_repository( app, name, description, installed_changeset_revision, ctx_rev, repository_clone_url, metadata_dict, status, current_changeset_revision=None, owner='', dist_to_shed=False ): """ - Update a tool shed repository record i the Galaxy database with the new information received. If a record defined by the received tool shed, repository name + Update a tool shed repository record in the Galaxy database with the new information received. If a record defined by the received tool shed, repository name and owner does not exists, create a new record with the received information. """ # The received value for dist_to_shed will be True if the InstallManager is installing a repository that contains tools or datatypes that used @@ -1052,6 +1052,10 @@ prior_installation_required = util.asbool( str( prior_installation_required ) ) return tool_shed, name, owner, changeset_revision, prior_installation_required +def pretty_print( dict=None ): + if dict: + return json.to_json_string( dict, sort_keys=True, indent=4 * ' ' ) + def remove_dir( dir ): """Attempt to remove a directory from disk.""" if os.path.exists( dir ): Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: jgoecks: Trackster: fix bug in variant track drawing.
by commits-noreply＠bitbucket.org 10 May '13

10 May '13

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/7dc222a7163a/ Changeset: 7dc222a7163a User: jgoecks Date: 2013-05-10 23:16:20 Summary: Trackster: fix bug in variant track drawing. Affected #: 1 file diff -r b4733e42a2c98c42401bca5e0643fc4d4dcf2db6 -r 7dc222a7163a5049cc407ad1a049dd091ac27b03 static/scripts/viz/trackster/tracks.js --- a/static/scripts/viz/trackster/tracks.js +++ b/static/scripts/viz/trackster/tracks.js @@ -4266,7 +4266,7 @@ draw_tile: function(result, ctx, mode, resolution, region, w_scale) { // Data could be coverage data or variant data. if (result.dataset_type === 'bigwig') { - return this._draw_line_track_tile(result, ctx, "Histogram", region, resolution, w_scale); + return this._draw_line_track_tile(result, ctx, "Histogram", resolution, region, w_scale); } else { // result.dataset_type === 'variant' var view = this.view, Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: Dave Bouvier: Add a new tool dependency action download_file to download a file without extracting it.
by commits-noreply＠bitbucket.org 10 May '13

10 May '13

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/b4733e42a2c9/ Changeset: b4733e42a2c9 User: Dave Bouvier Date: 2013-05-10 20:48:15 Summary: Add a new tool dependency action download_file to download a file without extracting it. Affected #: 3 files diff -r f72fd96d3fac5e17ec5c9aaece023b0ae74db696 -r b4733e42a2c98c42401bca5e0643fc4d4dcf2db6 lib/tool_shed/galaxy_install/tool_dependencies/common_util.py --- a/lib/tool_shed/galaxy_install/tool_dependencies/common_util.py +++ b/lib/tool_shed/galaxy_install/tool_dependencies/common_util.py @@ -151,6 +151,9 @@ def isgzip( file_path ): return checkers.is_gzip( file_path ) +def isjar( file_path ): + return iszip( file_path ) and file_path.endswith( '.jar' ) + def istar( file_path ): return tarfile.is_tarfile( file_path ) diff -r f72fd96d3fac5e17ec5c9aaece023b0ae74db696 -r b4733e42a2c98c42401bca5e0643fc4d4dcf2db6 lib/tool_shed/galaxy_install/tool_dependencies/fabric_util.py --- a/lib/tool_shed/galaxy_install/tool_dependencies/fabric_util.py +++ b/lib/tool_shed/galaxy_install/tool_dependencies/fabric_util.py @@ -71,6 +71,8 @@ # <action type="download_by_url">http://sourceforge.net/projects/samtools/files/samtools/0.1.18/samtools-0.1…</action> common_util.extract_tar( downloaded_file_path, work_dir ) dir = common_util.tar_extraction_directory( work_dir, downloaded_filename ) + elif common_util.isjar( downloaded_file_path ): + dir = os.path.curdir elif common_util.iszip( downloaded_file_path ): # <action type="download_by_url">http://downloads.sourceforge.net/project/picard/picard-tools/1.56/picard-to…</action> zip_archive_extracted = common_util.extract_zip( downloaded_file_path, work_dir ) @@ -85,6 +87,14 @@ if return_code: return dir = package_name + elif action_type == 'download_file': + # <action type="download_file">http://effectors.org/download/version/TTSS_GUI-1.0.1.jar</action> + # Download a single file to the working directory. + filtered_actions = actions[ 1: ] + url = action_dict[ 'url' ] + filename = url.split( '/' )[ -1 ] + common_util.url_download( work_dir, filename, url ) + dir = os.path.curdir else: # We're handling a complex repository dependency where we only have a set_environment tag set. # <action type="set_environment"> @@ -137,6 +147,11 @@ return_code = handle_command( app, tool_dependency, install_dir, cmd ) if return_code: return + elif action_type == 'download_file': + # Download a single file to the current directory. + url = action_dict[ 'url' ] + filename = url.split( '/' )[ -1 ] + common_util.url_download( current_dir, filename, url ) def log_results( command, fabric_AttributeString, file_path ): """ diff -r f72fd96d3fac5e17ec5c9aaece023b0ae74db696 -r b4733e42a2c98c42401bca5e0643fc4d4dcf2db6 lib/tool_shed/galaxy_install/tool_dependencies/install_util.py --- a/lib/tool_shed/galaxy_install/tool_dependencies/install_util.py +++ b/lib/tool_shed/galaxy_install/tool_dependencies/install_util.py @@ -341,6 +341,12 @@ action_dict[ 'target_filename' ] = action_elem.attrib[ 'target_filename' ] else: continue + elif action_type == 'download_file': + # <action type="download_file">http://effectors.org/download/version/TTSS_GUI-1.0.1.jar</action> + if action_elem.text: + action_dict[ 'url' ] = action_elem.text + else: + continue elif action_type == 'make_directory': # <action type="make_directory">$INSTALL_DIR/lib/python</action> if action_elem.text: Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: greg: Fixes for displaying README files associated with an uninstalled tool shed repository.
by commits-noreply＠bitbucket.org 10 May '13

10 May '13

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/f72fd96d3fac/ Changeset: f72fd96d3fac User: greg Date: 2013-05-10 20:24:19 Summary: Fixes for displaying README files associated with an uninstalled tool shed repository. Affected #: 2 files diff -r 43e37c4cc6cfbcdaba7e29a8bfac1b96dc5e7990 -r f72fd96d3fac5e17ec5c9aaece023b0ae74db696 lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py --- a/lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py +++ b/lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py @@ -1130,7 +1130,7 @@ """ Select or change the tool panel section to contain the tools included in the tool shed repository being reinstalled. If there are updates available for the repository in the tool shed, the tool_dependencies and repository_dependencies associated with the updated changeset revision - will have been retrieved from the tool shed and passed in the received kwd. In this case, the stored tool shed repository metqdata from the + will have been retrieved from the tool shed and passed in the received kwd. In this case, the stored tool shed repository metadata from the Galaxy database will not be used since it is outdated. """ message = '' @@ -1185,7 +1185,12 @@ includes_tool_dependencies = True if 'workflows' in metadata: includes_workflows = True - readme_files_dict = readme_util.build_readme_files_dict( metadata ) + # Since we're reinstalling, we need to send a request to the tool shed to get the README files. + url = suc.url_join( tool_shed_url, + 'repository/get_readme_files?name=%s&owner=%s&changeset_revision=%s' % \ + ( tool_shed_repository.name, tool_shed_repository.owner, tool_shed_repository.installed_changeset_revision ) ) + raw_text = common_util.tool_shed_get( trans.app, tool_shed_url, url ) + readme_files_dict = json.from_json_string( raw_text ) tool_dependencies = metadata.get( 'tool_dependencies', None ) repository_dependencies = self.get_repository_dependencies( trans=trans, repository_id=repository_id, diff -r 43e37c4cc6cfbcdaba7e29a8bfac1b96dc5e7990 -r f72fd96d3fac5e17ec5c9aaece023b0ae74db696 lib/tool_shed/util/metadata_util.py --- a/lib/tool_shed/util/metadata_util.py +++ b/lib/tool_shed/util/metadata_util.py @@ -1373,7 +1373,8 @@ invalid_tools = metadata.get( 'invalid_tools', None ) # Handle README files. if repository.has_readme_files: - if reinstalling: + if reinstalling or repository.status not in [ trans.model.ToolShedRepository.installation_status.DEACTIVATED, + trans.model.ToolShedRepository.installation_status.INSTALLED ]: # Since we're reinstalling, we need to send a request to the tool shed to get the README files. url = suc.url_join( tool_shed_url, 'repository/get_readme_files?name=%s&owner=%s&changeset_revision=%s' % \ Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: greg: For tool dependencies whose installation ended in an error, still display them when attempting to uninstall them.
by commits-noreply＠bitbucket.org 10 May '13

10 May '13

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/43e37c4cc6cf/ Changeset: 43e37c4cc6cf User: greg Date: 2013-05-10 19:52:28 Summary: For tool dependencies whose installation ended in an error, still display them when attempting to uninstall them. Affected #: 1 file diff -r 60fe1081baac6aa1e2e9d2e4f8e22945df102671 -r 43e37c4cc6cfbcdaba7e29a8bfac1b96dc5e7990 templates/admin/tool_shed_repository/uninstall_tool_dependencies.mako --- a/templates/admin/tool_shed_repository/uninstall_tool_dependencies.mako +++ b/templates/admin/tool_shed_repository/uninstall_tool_dependencies.mako @@ -51,15 +51,15 @@ tool_dependency.tool_shed_repository.owner, tool_dependency.tool_shed_repository.name, tool_dependency.tool_shed_repository.installed_changeset_revision ) + if not os.path.exists( install_dir ): + install_dir = "This dependency's installation directory does not exist, click <b>Uninstall</b> to reset for installation." %> - %if os.path.exists( install_dir ): - <tr> - <td>${tool_dependency.name}</td> - <td>${tool_dependency.version}</td> - <td>${tool_dependency.type}</td> - <td>${install_dir}</td> - </tr> - %endif + <tr> + <td>${tool_dependency.name}</td> + <td>${tool_dependency.version}</td> + <td>${tool_dependency.type}</td> + <td>${install_dir}</td> + </tr> %endfor </table><div style="clear: both"></div> Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: 2 new changesets
by commits-noreply＠bitbucket.org 10 May '13

10 May '13

2 new commits in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/60fe1081baac/ Changeset: 60fe1081baac User: dan Date: 2013-05-10 17:55:40 Summary: use_panels fix for login.mako in toolshed. Affected #: 1 file diff -r c8762ac39f04cc41da1f59444fdcff655559f4d4 -r 60fe1081baac6aa1e2e9d2e4f8e22945df102671 templates/user/login.mako --- a/templates/user/login.mako +++ b/templates/user/login.mako @@ -3,7 +3,7 @@ def inherit(context): if context.get('trans').webapp.name == 'galaxy' and context.get( 'use_panels', True ): return '/webapps/galaxy/base_panels.mako' - elif context.get('trans').webapp.name == 'tool_shed': + elif context.get('trans').webapp.name == 'tool_shed' and context.get( 'use_panels', True ): return '/webapps/tool_shed/base_panels.mako' else: return '/base.mako' https://bitbucket.org/galaxy/galaxy-central/commits/7cf2f256b3d9/ Changeset: 7cf2f256b3d9 Branch: stable User: dan Date: 2013-05-10 17:55:40 Summary: use_panels fix for login.mako in toolshed. Affected #: 1 file diff -r 429636bbc4ab4c5715dea09cee9ff546d94346de -r 7cf2f256b3d9c32f76567f49a0935b0ca9da9f7e templates/user/login.mako --- a/templates/user/login.mako +++ b/templates/user/login.mako @@ -3,7 +3,7 @@ def inherit(context): if context.get('trans').webapp.name == 'galaxy' and context.get( 'use_panels', True ): return '/webapps/galaxy/base_panels.mako' - elif context.get('trans').webapp.name == 'tool_shed': + elif context.get('trans').webapp.name == 'tool_shed' and context.get( 'use_panels', True ): return '/webapps/tool_shed/base_panels.mako' else: return '/base.mako' Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: 2 new changesets
by commits-noreply＠bitbucket.org 10 May '13

10 May '13

2 new commits in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/c8762ac39f04/ Changeset: c8762ac39f04 User: dan Date: 2013-05-10 17:49:16 Summary: use_panels fix for login.mako. Affected #: 1 file diff -r 45c048c8756438ee829498aac0e8c1e29f85d661 -r c8762ac39f04cc41da1f59444fdcff655559f4d4 templates/user/login.mako --- a/templates/user/login.mako +++ b/templates/user/login.mako @@ -1,7 +1,7 @@ <%! #This is a hack, we should restructure templates to avoid this. def inherit(context): - if context.get('trans').webapp.name == 'galaxy': + if context.get('trans').webapp.name == 'galaxy' and context.get( 'use_panels', True ): return '/webapps/galaxy/base_panels.mako' elif context.get('trans').webapp.name == 'tool_shed': return '/webapps/tool_shed/base_panels.mako' https://bitbucket.org/galaxy/galaxy-central/commits/429636bbc4ab/ Changeset: 429636bbc4ab Branch: stable User: dan Date: 2013-05-10 17:49:16 Summary: use_panels fix for login.mako. Affected #: 1 file diff -r 35b87dd34a79aa4b94c9c3dd8ff17b4143cd340c -r 429636bbc4ab4c5715dea09cee9ff546d94346de templates/user/login.mako --- a/templates/user/login.mako +++ b/templates/user/login.mako @@ -1,7 +1,7 @@ <%! #This is a hack, we should restructure templates to avoid this. def inherit(context): - if context.get('trans').webapp.name == 'galaxy': + if context.get('trans').webapp.name == 'galaxy' and context.get( 'use_panels', True ): return '/webapps/galaxy/base_panels.mako' elif context.get('trans').webapp.name == 'tool_shed': return '/webapps/tool_shed/base_panels.mako' Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: dan: Add openid_consumer_cache to .hgignore.
by commits-noreply＠bitbucket.org 10 May '13

10 May '13

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/45c048c87564/ Changeset: 45c048c87564 User: dan Date: 2013-05-10 17:28:57 Summary: Add openid_consumer_cache to .hgignore. Affected #: 1 file diff -r f0da1c8355df9a68c3db5fecd27ed20def89a3ec -r 45c048c8756438ee829498aac0e8c1e29f85d661 .hgignore --- a/.hgignore +++ b/.hgignore @@ -20,6 +20,7 @@ database/pbs database/tmp database/*.sqlite +database/openid_consumer_cache # Python bytecode *.pyc Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: greg: Fix for defining the order for installing repositories from the tool shed where 1 or more repositories defines a repository dependency with prior_installation_required=True.
by commits-noreply＠bitbucket.org 10 May '13

10 May '13

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/f0da1c8355df/ Changeset: f0da1c8355df User: greg Date: 2013-05-10 17:06:35 Summary: Fix for defining the order for installing repositories from the tool shed where 1 or more repositories defines a repository dependency with prior_installation_required=True. Affected #: 2 files diff -r 72f47fad3d219d44988d672dc41f30748ba8d5ff -r f0da1c8355df9a68c3db5fecd27ed20def89a3ec lib/galaxy/webapps/galaxy/api/tool_shed_repositories.py --- a/lib/galaxy/webapps/galaxy/api/tool_shed_repositories.py +++ b/lib/galaxy/webapps/galaxy/api/tool_shed_repositories.py @@ -248,25 +248,31 @@ # Install the repositories, keeping track of each one for later display. for index, tsr_id in enumerate( ordered_tsr_ids ): tool_shed_repository = trans.sa_session.query( trans.model.ToolShedRepository ).get( trans.security.decode_id( tsr_id ) ) - repo_info_dict = ordered_repo_info_dicts[ index ] - tool_panel_section_key = ordered_tool_panel_section_keys[ index ] - repository_util.install_tool_shed_repository( trans, - tool_shed_repository, - repo_info_dict, - tool_panel_section_key, - shed_tool_conf, - tool_path, - install_tool_dependencies, - reinstalling=False ) - tool_shed_repository_dict = tool_shed_repository.as_dict( value_mapper=default_tool_shed_repository_value_mapper( trans, tool_shed_repository ) ) - tool_shed_repository_dict[ 'url' ] = web.url_for( controller='tool_shed_repositories', - action='show', - id=trans.security.encode_id( tool_shed_repository.id ) ) - installed_tool_shed_repositories.append( tool_shed_repository_dict ) - else: + if tool_shed_repository.status in [ trans.model.ToolShedRepository.installation_status.NEW, + trans.model.ToolShedRepository.installation_status.UNINSTALLED ]: + + repo_info_dict = ordered_repo_info_dicts[ index ] + tool_panel_section_key = ordered_tool_panel_section_keys[ index ] + repository_util.install_tool_shed_repository( trans, + tool_shed_repository, + repo_info_dict, + tool_panel_section_key, + shed_tool_conf, + tool_path, + install_tool_dependencies, + reinstalling=False ) + tool_shed_repository_dict = tool_shed_repository.as_dict( value_mapper=default_tool_shed_repository_value_mapper( trans, tool_shed_repository ) ) + tool_shed_repository_dict[ 'url' ] = web.url_for( controller='tool_shed_repositories', + action='show', + id=trans.security.encode_id( tool_shed_repository.id ) ) + installed_tool_shed_repositories.append( tool_shed_repository_dict ) + elif message: log.error( message, exc_info=True ) trans.response.status = 500 return dict( status='error', error=message ) + elif not created_or_updated_tool_shed_repositories and not message: + # We're attempting to install more than 1 repository, and all of them have already been installed. + return dict( status='error', error='All repositories that you are attempting to install have been previously installed.' ) # Display the list of installed repositories. return installed_tool_shed_repositories diff -r 72f47fad3d219d44988d672dc41f30748ba8d5ff -r f0da1c8355df9a68c3db5fecd27ed20def89a3ec lib/tool_shed/galaxy_install/repository_util.py --- a/lib/tool_shed/galaxy_install/repository_util.py +++ b/lib/tool_shed/galaxy_install/repository_util.py @@ -95,6 +95,34 @@ tool_dependencies ) return repo_info_dict +def get_next_prior_install_required_dict_entry( prior_install_required_dict, processed_tsr_ids ): + """ + The order in which the prior_install_required_dict is processed is critical in order to ensure that the ultimate repository installation order is correctly + defined. This method determines the next key / value pair from the received prior_install_required_dict that should be processed. + """ + # Return the first key / value pair that is not yet processed and whose value is an empty list. + for key, value in prior_install_required_dict.items(): + if key in processed_tsr_ids: + continue + if not value: + return key + # Return the first key / value pair that is not yet processed and whose ids in value are all included in processed_tsr_ids. + for key, value in prior_install_required_dict.items(): + if key in processed_tsr_ids: + continue + all_contained = True + for required_repository_id in value: + if required_repository_id not in processed_tsr_ids: + all_contained = False + break + if all_contained: + return key + # Return the first key / value pair that is not yet processed. Hopefully this is all that is necessary at this point. + for key, value in prior_install_required_dict.items(): + if key in processed_tsr_ids: + continue + return key + def get_prior_install_required_dict( trans, tsr_ids, repo_info_dicts ): """ Return a dictionary whose keys are the received tsr_ids and whose values are a list of tsr_ids, each of which is contained in the received list of tsr_ids @@ -591,10 +619,13 @@ # Create a dictionary whose keys are the received tsr_ids and whose values are a list of tsr_ids, each of which is contained in the received list of tsr_ids # and whose associated repository must be installed prior to the repository associated with the tsr_id key. prior_install_required_dict = get_prior_install_required_dict( trans, tsr_ids, repo_info_dicts ) - # Create the ordered_tsr_ids, the ordered_repo_info_dicts and the ordered_tool_panel_section_keys lists. - for tsr_id in tsr_ids: + processed_tsr_ids = [] + while len( processed_tsr_ids ) != len( prior_install_required_dict.keys() ): + tsr_id = get_next_prior_install_required_dict_entry( prior_install_required_dict, processed_tsr_ids ) + processed_tsr_ids.append( tsr_id ) + # Create the ordered_tsr_ids, the ordered_repo_info_dicts and the ordered_tool_panel_section_keys lists. if tsr_id not in ordered_tsr_ids: - prior_install_required_ids = prior_install_required_dict.get( tsr_id, [] ) + prior_install_required_ids = prior_install_required_dict[ tsr_id ] for prior_install_required_id in prior_install_required_ids: if prior_install_required_id not in ordered_tsr_ids: # Install the associated repository dependency first. Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: dan: Fix for generating password via pbkdf2_bin for Python < 2.6.8
by commits-noreply＠bitbucket.org 10 May '13

10 May '13

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/72f47fad3d21/ Changeset: 72f47fad3d21 User: dan Date: 2013-05-10 16:14:49 Summary: Fix for generating password via pbkdf2_bin for Python < 2.6.8 Affected #: 1 file diff -r 41cbbcb908236c14818dfb2b3372ea545ece718b -r 72f47fad3d219d44988d672dc41f30748ba8d5ff lib/galaxy/security/passwords.py --- a/lib/galaxy/security/passwords.py +++ b/lib/galaxy/security/passwords.py @@ -70,7 +70,7 @@ rv = u = _pseudorandom(salt + _pack_int(block)) for i in xrange(iterations - 1): u = _pseudorandom(''.join(map(chr, u))) - rv = starmap(xor, izip(rv, u)) + rv = starmap( xor, zip( rv, u ) ) #Python < 2.6.8: starmap requires function inputs to be tuples, so we need to use zip instead of izip buf.extend(rv) return ''.join(map(chr, buf))[:keylen] Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: jgoecks: Select2 enhancements: (a) bump up z-index so that select2 works in modal dialogs; (b) add select2 to base_panels for pervasive access and remove from individual templates; and (c) use select2 in Trackster and discontinue use of custom jQuery combobox.
by commits-noreply＠bitbucket.org 10 May '13

10 May '13

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/41cbbcb90823/ Changeset: 41cbbcb90823 User: jgoecks Date: 2013-05-10 15:38:51 Summary: Select2 enhancements: (a) bump up z-index so that select2 works in modal dialogs; (b) add select2 to base_panels for pervasive access and remove from individual templates; and (c) use select2 in Trackster and discontinue use of custom jQuery combobox. Affected #: 6 files diff -r 75a49f8d1bd38939a426660e379da8128631cbe2 -r 41cbbcb908236c14818dfb2b3372ea545ece718b static/style/blue/base.css --- a/static/style/blue/base.css +++ b/static/style/blue/base.css @@ -831,11 +831,11 @@ .select2-container .select2-choice span{margin-right:26px;display:block;overflow:hidden;white-space:nowrap;-o-text-overflow:ellipsis;-ms-text-overflow:ellipsis;text-overflow:ellipsis;} .select2-container .select2-choice abbr{display:block;position:absolute;right:26px;top:8px;width:12px;height:12px;font-size:1px;background:url('../images/select2.png') right top no-repeat;cursor:pointer;text-decoration:none;border:0;outline:0;} .select2-container .select2-choice abbr:hover{background-position:right -11px;cursor:pointer;} -.select2-drop{background:#fff;color:#000;border:1px solid #aaa;border-top:0;position:absolute;top:100%;-webkit-box-shadow:0 4px 5px rgba(0, 0, 0, 0.15);-moz-box-shadow:0 4px 5px rgba(0, 0, 0, 0.15);-o-box-shadow:0 4px 5px rgba(0, 0, 0, 0.15);box-shadow:0 4px 5px rgba(0, 0, 0, 0.15);z-index:9999;width:100%;margin-top:-1px;-webkit-border-radius:0 0 4px 4px;-moz-border-radius:0 0 4px 4px;border-radius:0 0 4px 4px;} +.select2-drop{background:#fff;color:#000;border:1px solid #aaa;border-top:0;position:absolute;top:100%;-webkit-box-shadow:0 4px 5px rgba(0, 0, 0, 0.15);-moz-box-shadow:0 4px 5px rgba(0, 0, 0, 0.15);-o-box-shadow:0 4px 5px rgba(0, 0, 0, 0.15);box-shadow:0 4px 5px rgba(0, 0, 0, 0.15);z-index:24999;width:100%;margin-top:-1px;-webkit-border-radius:0 0 4px 4px;-moz-border-radius:0 0 4px 4px;border-radius:0 0 4px 4px;} .select2-drop.select2-drop-above{-webkit-border-radius:4px 4px 0px 0px;-moz-border-radius:4px 4px 0px 0px;border-radius:4px 4px 0px 0px;margin-top:1px;border-top:1px solid #aaa;border-bottom:0;-webkit-box-shadow:0 -4px 5px rgba(0, 0, 0, 0.15);-moz-box-shadow:0 -4px 5px rgba(0, 0, 0, 0.15);-o-box-shadow:0 -4px 5px rgba(0, 0, 0, 0.15);box-shadow:0 -4px 5px rgba(0, 0, 0, 0.15);} .select2-container .select2-choice div{-webkit-border-radius:0 4px 4px 0;-moz-border-radius:0 4px 4px 0;border-radius:0 4px 4px 0;-moz-background-clip:padding;-webkit-background-clip:padding-box;background-clip:padding-box;background:#ccc;background-image:-webkit-gradient(linear, left bottom, left top, color-stop(0, #cccccc), color-stop(0.6, #eeeeee));background-image:-webkit-linear-gradient(center bottom, #cccccc 0%, #eeeeee 60%);background-image:-moz-linear-gradient(center bottom, #cccccc 0%, #eeeeee 60%);background-image:-o-linear-gradient(bottom, #cccccc 0%, #eeeeee 60%);background-image:-ms-linear-gradient(top, #cccccc 0%, #eeeeee 60%);filter:progid:DXImageTransform.Microsoft.gradient(startColorstr='#cccccc', endColorstr='#eeeeee', GradientType=0);background-image:linear-gradient(top, #cccccc 0%, #eeeeee 60%);border-left:1px solid #aaa;position:absolute;right:0;top:0;display:block;height:100%;width:18px;} .select2-container .select2-choice div b{background:url('../images/select2.png') no-repeat 0 1px;display:block;width:100%;height:100%;} -.select2-search{display:inline-block;white-space:nowrap;z-index:10000;min-height:26px;width:100%;margin:0;padding-left:4px;padding-right:4px;} +.select2-search{display:inline-block;white-space:nowrap;z-index:25000;min-height:26px;width:100%;margin:0;padding-left:4px;padding-right:4px;} .select2-search-hidden{display:block;position:absolute;left:-10000px;} .select2-search input{background:#ffffff url('../images/select2.png') no-repeat 100% -22px;background:url('../images/select2.png') no-repeat 100% -22px,-webkit-gradient(linear, left bottom, left top, color-stop(0.85, #ffffff), color-stop(0.99, #eeeeee));background:url('../images/select2.png') no-repeat 100% -22px,-webkit-linear-gradient(center bottom, #ffffff 85%, #eeeeee 99%);background:url('../images/select2.png') no-repeat 100% -22px,-moz-linear-gradient(center bottom, #ffffff 85%, #eeeeee 99%);background:url('../images/select2.png') no-repeat 100% -22px,-o-linear-gradient(bottom, #ffffff 85%, #eeeeee 99%);background:url('../images/select2.png') no-repeat 100% -22px,-ms-linear-gradient(top, #ffffff 85%, #eeeeee 99%);background:url('../images/select2.png') no-repeat 100% -22px,linear-gradient(top, #ffffff 85%, #eeeeee 99%);padding:4px 20px 4px 5px;outline:0;border:1px solid #aaa;font-family:sans-serif;font-size:1em;width:100%;margin:0;height:auto !important;min-height:26px;-webkit-box-shadow:none;-moz-box-shadow:none;box-shadow:none;border-radius:0;-moz-border-radius:0;-webkit-border-radius:0;} .select2-drop.select2-drop-above .select2-search input{margin-top:4px;} diff -r 75a49f8d1bd38939a426660e379da8128631cbe2 -r 41cbbcb908236c14818dfb2b3372ea545ece718b static/style/select2.less --- a/static/style/select2.less +++ b/static/style/select2.less @@ -109,7 +109,7 @@ -moz-box-shadow: 0 4px 5px rgba(0, 0, 0, .15); -o-box-shadow: 0 4px 5px rgba(0, 0, 0, .15); box-shadow: 0 4px 5px rgba(0, 0, 0, .15); - z-index: 9999; + z-index: 24999; width:100%; margin-top:-1px; @@ -166,7 +166,7 @@ .select2-search { display: inline-block; white-space: nowrap; - z-index: 10000; + z-index: 25000; min-height: 26px; width: 100%; margin: 0; diff -r 75a49f8d1bd38939a426660e379da8128631cbe2 -r 41cbbcb908236c14818dfb2b3372ea545ece718b templates/base/base_panels.mako --- a/templates/base/base_panels.mako +++ b/templates/base/base_panels.mako @@ -52,6 +52,7 @@ ${h.js( 'libs/jquery/jquery', 'libs/json2', + 'libs/jquery/select2', 'libs/bootstrap', 'libs/underscore', 'libs/backbone/backbone', diff -r 75a49f8d1bd38939a426660e379da8128631cbe2 -r 41cbbcb908236c14818dfb2b3372ea545ece718b templates/webapps/galaxy/tool_form.mako --- a/templates/webapps/galaxy/tool_form.mako +++ b/templates/webapps/galaxy/tool_form.mako @@ -13,7 +13,7 @@ <%def name="javascripts()"> ${parent.javascripts()} - ${h.js( "galaxy.panels", "libs/jquery/select2", "libs/jquery/jstorage" )} + ${h.js( "galaxy.panels", "libs/jquery/jstorage" )} <script type="text/javascript"> $(function() { $(window).bind("refresh_on_change", function() { diff -r 75a49f8d1bd38939a426660e379da8128631cbe2 -r 41cbbcb908236c14818dfb2b3372ea545ece718b templates/webapps/galaxy/tracks/browser.mako --- a/templates/webapps/galaxy/tracks/browser.mako +++ b/templates/webapps/galaxy/tracks/browser.mako @@ -105,10 +105,7 @@ "Create": function() { $(document).trigger("convert_to_values"); continue_fn(); } }); $("#new-title").focus(); - $("select[name='dbkey']").combobox({ - appendTo: $("#overlay"), - size: 40 - }); + $("select[name='dbkey']").select2({ width: 'resolve'}); // To support the large number of options for dbkey, enable scrolling in overlay. $("#overlay").css("overflow", "auto"); } diff -r 75a49f8d1bd38939a426660e379da8128631cbe2 -r 41cbbcb908236c14818dfb2b3372ea545ece718b templates/webapps/galaxy/workflow/run.mako --- a/templates/webapps/galaxy/workflow/run.mako +++ b/templates/webapps/galaxy/workflow/run.mako @@ -2,7 +2,6 @@ <%def name="javascripts()"> ${parent.javascripts()} - ${h.js( "libs/jquery/select2" )} <script type="text/javascript"> $( function() { function show_tool_body(title){ Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: 5 new changesets
by commits-noreply＠bitbucket.org 09 May '13

09 May '13

5 new commits in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/35b87dd34a79/ Changeset: 35b87dd34a79 Branch: stable User: Dave Bouvier Date: 2013-05-09 20:25:51 Summary: Only look for and process tool help image definitions if the tool XML defines a help block. Affected #: 1 file diff -r 7930e04adb535b8bf3c7973efc5cdce0fb384034 -r 35b87dd34a79aa4b94c9c3dd8ff17b4143cd340c lib/galaxy/tools/__init__.py --- a/lib/galaxy/tools/__init__.py +++ b/lib/galaxy/tools/__init__.py @@ -1357,23 +1357,23 @@ """ # TODO: Allow raw HTML or an external link. self.help = root.find("help") - # Handle tool shelp image display for tools that are contained in repositories that are in the lool shed or installed into Galaxy. - # When tool config files use the speical string $PATH_TO_IMAGES, the folloing code will replace that string with the path on disk. - if self.repository_id and self.help.text.find( '$PATH_TO_IMAGES' ) >= 0: - if self.app.name == 'galaxy': - repository = self.sa_session.query( self.app.model.ToolShedRepository ).get( self.app.security.decode_id( self.repository_id ) ) - if repository: - path_to_images = '/tool_runner/static/images/%s' % self.repository_id - self.help.text = self.help.text.replace( '$PATH_TO_IMAGES', path_to_images ) - elif self.app.name == 'tool_shed': - repository = self.sa_session.query( self.app.model.Repository ).get( self.app.security.decode_id( self.repository_id ) ) - if repository: - path_to_images = '/repository/static/images/%s' % self.repository_id - self.help.text = self.help.text.replace( '$PATH_TO_IMAGES', path_to_images ) self.help_by_page = list() help_header = "" help_footer = "" if self.help is not None: + # Handle tool help image display for tools that are contained in repositories that are in the tool shed or installed into Galaxy. + # When tool config files use the special string $PATH_TO_IMAGES, the following code will replace that string with the path on disk. + if self.repository_id and self.help.text.find( '$PATH_TO_IMAGES' ) >= 0: + if self.app.name == 'galaxy': + repository = self.sa_session.query( self.app.model.ToolShedRepository ).get( self.app.security.decode_id( self.repository_id ) ) + if repository: + path_to_images = '/tool_runner/static/images/%s' % self.repository_id + self.help.text = self.help.text.replace( '$PATH_TO_IMAGES', path_to_images ) + elif self.app.name == 'tool_shed': + repository = self.sa_session.query( self.app.model.Repository ).get( self.app.security.decode_id( self.repository_id ) ) + if repository: + path_to_images = '/repository/static/images/%s' % self.repository_id + self.help.text = self.help.text.replace( '$PATH_TO_IMAGES', path_to_images ) help_pages = self.help.findall( "page" ) help_header = self.help.text try: https://bitbucket.org/galaxy/galaxy-central/commits/d914458eb4f3/ Changeset: d914458eb4f3 User: Dave Bouvier Date: 2013-05-09 20:26:35 Summary: Merge in fixes from stable branch. Affected #: 4 files diff -r 6ba6f297b9694e29ff774a33e9b88b9f666ddee7 -r d914458eb4f31b39f175ccdbc31668b13072ea23 lib/galaxy/tools/__init__.py --- a/lib/galaxy/tools/__init__.py +++ b/lib/galaxy/tools/__init__.py @@ -1266,23 +1266,23 @@ """ # TODO: Allow raw HTML or an external link. self.help = root.find("help") - # Handle tool shelp image display for tools that are contained in repositories that are in the lool shed or installed into Galaxy. - # When tool config files use the speical string $PATH_TO_IMAGES, the folloing code will replace that string with the path on disk. - if self.repository_id and self.help.text.find( '$PATH_TO_IMAGES' ) >= 0: - if self.app.name == 'galaxy': - repository = self.sa_session.query( self.app.model.ToolShedRepository ).get( self.app.security.decode_id( self.repository_id ) ) - if repository: - path_to_images = '/tool_runner/static/images/%s' % self.repository_id - self.help.text = self.help.text.replace( '$PATH_TO_IMAGES', path_to_images ) - elif self.app.name == 'tool_shed': - repository = self.sa_session.query( self.app.model.Repository ).get( self.app.security.decode_id( self.repository_id ) ) - if repository: - path_to_images = '/repository/static/images/%s' % self.repository_id - self.help.text = self.help.text.replace( '$PATH_TO_IMAGES', path_to_images ) self.help_by_page = list() help_header = "" help_footer = "" if self.help is not None: + # Handle tool help image display for tools that are contained in repositories that are in the tool shed or installed into Galaxy. + # When tool config files use the special string $PATH_TO_IMAGES, the following code will replace that string with the path on disk. + if self.repository_id and self.help.text.find( '$PATH_TO_IMAGES' ) >= 0: + if self.app.name == 'galaxy': + repository = self.sa_session.query( self.app.model.ToolShedRepository ).get( self.app.security.decode_id( self.repository_id ) ) + if repository: + path_to_images = '/tool_runner/static/images/%s' % self.repository_id + self.help.text = self.help.text.replace( '$PATH_TO_IMAGES', path_to_images ) + elif self.app.name == 'tool_shed': + repository = self.sa_session.query( self.app.model.Repository ).get( self.app.security.decode_id( self.repository_id ) ) + if repository: + path_to_images = '/repository/static/images/%s' % self.repository_id + self.help.text = self.help.text.replace( '$PATH_TO_IMAGES', path_to_images ) help_pages = self.help.findall( "page" ) help_header = self.help.text try: https://bitbucket.org/galaxy/galaxy-central/commits/7e72e623a2aa/ Changeset: 7e72e623a2aa User: Dave Bouvier Date: 2013-05-09 20:43:45 Summary: Merged. Affected #: 1 file diff -r d914458eb4f31b39f175ccdbc31668b13072ea23 -r 7e72e623a2aa71a7eaae78b9f7f03e3abff4fce6 templates/webapps/tool_shed/repository/common.mako --- a/templates/webapps/tool_shed/repository/common.mako +++ b/templates/webapps/tool_shed/repository/common.mako @@ -544,6 +544,7 @@ <%def name="render_repository_dependency( repository_dependency, pad, parent, row_counter, row_is_header=False )"><% + from galaxy.util import string_as_bool encoded_id = trans.security.encode_id( repository_dependency.id ) if trans.webapp.name == 'galaxy': if repository_dependency.tool_shed_repository_id: @@ -557,6 +558,7 @@ repository_name = str( repository_dependency.repository_name ) repository_owner = str( repository_dependency.repository_owner ) changeset_revision = str( repository_dependency.changeset_revision ) + prior_installation_required = string_as_bool( str( repository_dependency.prior_installation_required ) ) if prior_installation_required: prior_installation_required_str = " <i>(prior install required)</i>" else: @@ -676,6 +678,7 @@ <%def name="render_tool_dependency( tool_dependency, pad, parent, row_counter, row_is_header )"><% + from galaxy.util import string_as_bool encoded_id = trans.security.encode_id( tool_dependency.id ) is_missing = tool_dependency.installation_status not in [ 'Installed' ] if row_is_header: @@ -725,7 +728,17 @@ ${tool_dependency.install_dir | h} %endif %else: - ${tool_dependency.is_orphan | h} + %if row_is_header: + ${tool_dependency.is_orphan | h} + %else: + <% + if string_as_bool( str( tool_dependency.is_orphan ) ): + is_orpan = 'yes' + else: + is_orphan = 'no' + %> + ${is_orphan | h} + %endif %endif </${cell_type}></tr> https://bitbucket.org/galaxy/galaxy-central/commits/6905bc773595/ Changeset: 6905bc773595 User: Dave Bouvier Date: 2013-05-09 20:56:19 Summary: Fix Galaxy side user creation in tool shed functional tests. Affected #: 1 file diff -r 7e72e623a2aa71a7eaae78b9f7f03e3abff4fce6 -r 6905bc773595db5951247d6057e9c42b9c6e735f test/tool_shed/base/twilltestcase.py --- a/test/tool_shed/base/twilltestcase.py +++ b/test/tool_shed/base/twilltestcase.py @@ -312,12 +312,7 @@ def create_user_in_galaxy( self, cntrller='user', email='test(a)bx.psu.edu', password='testuser', username='admin-user', redirect='' ): self.visit_galaxy_url( "/user/create?cntrller=%s&use_panels=False" % cntrller ) - tc.fv( '1', 'email', email ) - tc.fv( '1', 'redirect', redirect ) - tc.fv( '1', 'password', password ) - tc.fv( '1', 'confirm', password ) - tc.fv( '1', 'username', username ) - tc.submit( 'create_user_button' ) + self.submit_form( '1', 'create_user_button', email=email, password=password, confirm=password, username=username, redirect=redirect ) previously_created = False username_taken = False invalid_username = False @@ -509,10 +504,7 @@ self.create_user_in_galaxy( email=email, password=password, username=username, redirect=redirect ) if previously_created: self.visit_galaxy_url( "/user/login?use_panels=False" ) - tc.fv( '1', 'email', email ) - tc.fv( '1', 'redirect', redirect ) - tc.fv( '1', 'password', password ) - tc.submit( 'login_button' ) + self.submit_form( '1', 'login_button', email=email, redirect=redirect, password=password ) def galaxy_logout( self ): self.home() https://bitbucket.org/galaxy/galaxy-central/commits/75a49f8d1bd3/ Changeset: 75a49f8d1bd3 User: Dave Bouvier Date: 2013-05-09 21:50:27 Summary: Merged again. Affected #: 1 file Diff not available. Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: greg: Fix a typo in a tool shed template.
by commits-noreply＠bitbucket.org 09 May '13

09 May '13

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/b56a433e8eab/ Changeset: b56a433e8eab User: greg Date: 2013-05-09 21:43:28 Summary: Fix a typo in a tool shed template. Affected #: 1 file diff -r 1f9a0bf662b376455fbc39dcd679cf443c4d0cfa -r b56a433e8eab3c64ddc31890c6229140cb4e8f40 templates/webapps/tool_shed/repository/common.mako --- a/templates/webapps/tool_shed/repository/common.mako +++ b/templates/webapps/tool_shed/repository/common.mako @@ -733,7 +733,7 @@ %else: <% if string_as_bool( str( tool_dependency.is_orphan ) ): - is_orpan = 'yes' + is_orphan = 'yes' else: is_orphan = 'no' %> Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: greg: Fix for displaying if a repository dependency has the prior_installation_required flag set to True when inside a container, and render the orphan setting for tool dependencies as 'yes' or 'no' instead of True or false.
by commits-noreply＠bitbucket.org 09 May '13

09 May '13

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/1f9a0bf662b3/ Changeset: 1f9a0bf662b3 User: greg Date: 2013-05-09 20:39:11 Summary: Fix for displaying if a repository dependency has the prior_installation_required flag set to True when inside a container, and render the orphan setting for tool dependencies as 'yes' or 'no' instead of True or false. Affected #: 1 file diff -r 6ba6f297b9694e29ff774a33e9b88b9f666ddee7 -r 1f9a0bf662b376455fbc39dcd679cf443c4d0cfa templates/webapps/tool_shed/repository/common.mako --- a/templates/webapps/tool_shed/repository/common.mako +++ b/templates/webapps/tool_shed/repository/common.mako @@ -544,6 +544,7 @@ <%def name="render_repository_dependency( repository_dependency, pad, parent, row_counter, row_is_header=False )"><% + from galaxy.util import string_as_bool encoded_id = trans.security.encode_id( repository_dependency.id ) if trans.webapp.name == 'galaxy': if repository_dependency.tool_shed_repository_id: @@ -557,6 +558,7 @@ repository_name = str( repository_dependency.repository_name ) repository_owner = str( repository_dependency.repository_owner ) changeset_revision = str( repository_dependency.changeset_revision ) + prior_installation_required = string_as_bool( str( repository_dependency.prior_installation_required ) ) if prior_installation_required: prior_installation_required_str = " <i>(prior install required)</i>" else: @@ -676,6 +678,7 @@ <%def name="render_tool_dependency( tool_dependency, pad, parent, row_counter, row_is_header )"><% + from galaxy.util import string_as_bool encoded_id = trans.security.encode_id( tool_dependency.id ) is_missing = tool_dependency.installation_status not in [ 'Installed' ] if row_is_header: @@ -725,7 +728,17 @@ ${tool_dependency.install_dir | h} %endif %else: - ${tool_dependency.is_orphan | h} + %if row_is_header: + ${tool_dependency.is_orphan | h} + %else: + <% + if string_as_bool( str( tool_dependency.is_orphan ) ): + is_orpan = 'yes' + else: + is_orphan = 'no' + %> + ${is_orphan | h} + %endif %endif </${cell_type}></tr> Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: Dave Bouvier: Add missing hda name length check to submit_form.
by commits-noreply＠bitbucket.org 09 May '13

09 May '13

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/6ba6f297b969/ Changeset: 6ba6f297b969 User: Dave Bouvier Date: 2013-05-09 19:49:13 Summary: Add missing hda name length check to submit_form. Affected #: 1 file diff -r 51f4926f6dd8b837c02dd8559587342542cdd7f2 -r 6ba6f297b9694e29ff774a33e9b88b9f666ddee7 test/base/twilltestcase.py --- a/test/base/twilltestcase.py +++ b/test/base/twilltestcase.py @@ -1169,18 +1169,21 @@ # To help with debugging a tool, print out the form controls when the test fails print "form '%s' contains the following controls ( note the values )" % f.name controls = {} + formcontrols = [] hc_prefix = '<HiddenControl(' for i, control in enumerate( f.controls ): - print "control %d: %s" % ( i, str( control ) ) + formcontrols.append( "control %d: %s" % ( i, str( control ) ) ) + for i, control in enumerate( f.controls ): if not hc_prefix in str( control ): try: #check if a repeat element needs to be added - if control.name not in kwd and control.name.endswith( '_add' ): - #control name doesn't exist, could be repeat - repeat_startswith = control.name[0:-4] - if repeat_startswith and not [ c_name for c_name in controls.keys() if c_name.startswith( repeat_startswith ) ] and [ c_name for c_name in kwd.keys() if c_name.startswith( repeat_startswith ) ]: - tc.submit( control.name ) - return self.submit_form( form_no=form_no, button=button, **kwd ) + if control.name is not None: + if control.name not in kwd and control.name.endswith( '_add' ): + #control name doesn't exist, could be repeat + repeat_startswith = control.name[0:-4] + if repeat_startswith and not [ c_name for c_name in controls.keys() if c_name.startswith( repeat_startswith ) ] and [ c_name for c_name in kwd.keys() if c_name.startswith( repeat_startswith ) ]: + tc.submit( control.name ) + return self.submit_form( form_no=form_no, button=button, **kwd ) # Check for refresh_on_change attribute, submit a change if required if hasattr( control, 'attrs' ) and 'refresh_on_change' in control.attrs.keys(): changed = False @@ -1188,9 +1191,10 @@ # This loop gets the filename/label for the selected values. item_labels = [ item.attrs[ 'label' ] for item in control.get_items() if item.selected ] for value in kwd[ control.name ]: - # Galaxy truncates long file names in the dataset_collector in galaxy/tools/parameters/basic.py + # Galaxy truncates long file names in the dataset_collector in galaxy/tools/parameters/basic.py. + # This (and other places where this is done) should be refactored to use the HDA id. if len( value ) > 30 and control.is_of_kind( 'singlelist' ): - field_value = '%s..%s' % ( elem[:17], elem[-11:] ) + field_value = '%s..%s' % ( value[:17], value[-11:] ) else: field_value = value if field_value not in control.value and True not in [ field_value in item_label for item_label in item_labels ]: @@ -1214,7 +1218,9 @@ tc.submit( '___refresh_grouping___' ) return self.submit_form( form_no=form_no, button=button, **kwd ) except Exception, e: - log.debug( "In submit_form, continuing, but caught exception: %s" % str( e ) ) + log.exception( "In submit_form, continuing, but caught exception." ) + for formcontrol in formcontrols: + log.debug( formcontrol ) continue controls[ control.name ] = control # No refresh_on_change attribute found in current form, so process as usual @@ -1254,15 +1260,25 @@ for elem in control_value: try: tc.fv( f.name, control.name, str( elem ) ) - except Exception, e2: - print "Attempting to set control '", control.name, "' to value '", elem, "' threw exception: ", e2 - # Galaxy truncates long file names in the dataset_collector in galaxy/tools/parameters/basic.py - if len( elem ) > 30: - elem_name = '%s..%s' % ( elem[:17], elem[-11:] ) - else: - elem_name = elem - tc.fv( f.name, control.name, str( elem_name ) ) + except Exception: + try: + # Galaxy truncates long file names in the dataset_collector in galaxy/tools/parameters/basic.py + if len( elem ) > 30: + elem_name = '%s..%s' % ( elem[:17], elem[-11:] ) + tc.fv( f.name, control.name, str( elem_name ) ) + pass + else: + raise + except Exception: + raise + except Exception: + for formcontrol in formcontrols: + log.debug( formcontrol ) + log.exception( "Attempting to set control '%s' to value '%s' (also tried '%s') threw exception.", control.name, elem, elem_name ) + pass except Exception, exc: + for formcontrol in formcontrols: + log.debug( formcontrol ) errmsg = "Attempting to set field '%s' to value '%s' in form '%s' threw exception: %s\n" % ( control_name, str( control_value ), f.name, str( exc ) ) errmsg += "control: %s\n" % str( control ) errmsg += "If the above control is a DataToolparameter whose data type class does not include a sniff() method,\n" @@ -1272,6 +1288,7 @@ # Add conditions for other control types here when necessary. pass tc.submit( button ) + def refresh_form( self, control_name, value, form_no=0, **kwd ): """Handles Galaxy's refresh_on_change for forms without ultimately submitting the form""" # control_name is the name of the form field that requires refresh_on_change, and value is Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: jgoecks: Visualization framework: replace summary_tree with bigwig datatype for all feature track coverage data, yielding better performance and a standardized approach for working with coverage data. Update Trackster to work with bigwig coverage data.
by commits-noreply＠bitbucket.org 09 May '13

09 May '13

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/51f4926f6dd8/ Changeset: 51f4926f6dd8 User: jgoecks Date: 2013-05-09 17:44:45 Summary: Visualization framework: replace summary_tree with bigwig datatype for all feature track coverage data, yielding better performance and a standardized approach for working with coverage data. Update Trackster to work with bigwig coverage data. Affected #: 24 files diff -r 38a9047cd5877b9123c06a92c2c53b6b488733cf -r 51f4926f6dd8b837c02dd8559587342542cdd7f2 datatypes_conf.xml.sample --- a/datatypes_conf.xml.sample +++ b/datatypes_conf.xml.sample @@ -9,12 +9,7 @@ <datatype extension="fli" type="galaxy.datatypes.tabular:FeatureLocationIndex" display_in_upload="false"/><datatype extension="bam" type="galaxy.datatypes.binary:Bam" mimetype="application/octet-stream" display_in_upload="true"><converter file="bam_to_bai.xml" target_datatype="bai"/> - <converter file="bam_to_summary_tree_converter.xml" target_datatype="summary_tree"/> -  + <converter file="bam_to_bigwig_converter.xml" target_datatype="bigwig"/><display file="ucsc/bam.xml" /><display file="ensembl/ensembl_bam.xml" /><display file="igv/bam.xml" /> @@ -25,7 +20,7 @@ <converter file="interval_to_coverage.xml" target_datatype="coverage"/><converter file="bed_to_bgzip_converter.xml" target_datatype="bgzip"/><converter file="bed_to_tabix_converter.xml" target_datatype="tabix" depends_on="bgzip"/> - <converter file="bed_to_summary_tree_converter.xml" target_datatype="summary_tree"/> + <converter file="bed_gff_or_vcf_to_bigwig_converter.xml" target_datatype="bigwig"/><converter file="bed_to_fli_converter.xml" target_datatype="fli"/><display file="igb/bed.xml" /> @@ -51,7 +46,7 @@ <datatype extension="chrint" type="galaxy.datatypes.interval:ChromatinInteractions" display_in_upload="True"><converter file="interval_to_bgzip_converter.xml" target_datatype="bgzip"/><converter file="interval_to_tabix_converter.xml" target_datatype="tabix" depends_on="bgzip"/> - <converter file="interval_to_summary_tree_converter.xml" target_datatype="summary_tree"/> + <converter file="bed_gff_or_vcf_to_bigwig_converter.xml" target_datatype="bigwig"/></datatype><datatype extension="csv" type="galaxy.datatypes.tabular:Tabular" subclass="True" display_in_upload="true" /> @@ -93,7 +88,7 @@ <datatype extension="gff" type="galaxy.datatypes.interval:Gff" display_in_upload="true"><converter file="gff_to_bed_converter.xml" target_datatype="bed"/><converter file="gff_to_interval_index_converter.xml" target_datatype="interval_index"/> - <converter file="gff_to_summary_tree_converter.xml" target_datatype="summary_tree"/> + <converter file="bed_gff_or_vcf_to_bigwig_converter.xml" target_datatype="bigwig"/><converter file="gff_to_fli_converter.xml" target_datatype="fli"/><display file="ensembl/ensembl_gff.xml" inherit="True"/> @@ -103,7 +98,7 @@ <datatype extension="gmaj.zip" type="galaxy.datatypes.images:Gmaj" mimetype="application/zip"/><datatype extension="gtf" type="galaxy.datatypes.interval:Gtf" display_in_upload="true"><converter file="gff_to_interval_index_converter.xml" target_datatype="interval_index"/> - <converter file="gff_to_summary_tree_converter.xml" target_datatype="summary_tree"/> + <converter file="bed_gff_or_vcf_to_bigwig_converter.xml" target_datatype="bigwig"/></datatype><datatype extension="toolshed.gz" type="galaxy.datatypes.binary:Binary" mimetype="multipart/x-gzip" subclass="True" /><datatype extension="h5" type="galaxy.datatypes.binary:Binary" mimetype="application/octet-stream" subclass="True" /> @@ -115,7 +110,7 @@ <converter file="interval_to_bed12_converter.xml" target_datatype="bed12"/><converter file="interval_to_bgzip_converter.xml" target_datatype="bgzip"/><converter file="interval_to_tabix_converter.xml" target_datatype="tabix" depends_on="bgzip"/> - <converter file="interval_to_summary_tree_converter.xml" target_datatype="summary_tree"/> + <converter file="interval_to_bigwig_converter.xml" target_datatype="bigwig"/><display file="ensembl/ensembl_interval_as_bed.xml" inherit="True"/><display file="gbrowse/gbrowse_interval_as_bed.xml" inherit="True"/> @@ -156,7 +151,7 @@ <datatype extension="encodepeak" type="galaxy.datatypes.interval:ENCODEPeak" display_in_upload="True"><converter file="encodepeak_to_tabix_converter.xml" target_datatype="tabix" depends_on="bgzip"/><converter file="encodepeak_to_bgzip_converter.xml" target_datatype="bgzip"/> - <converter file="encodepeak_to_summary_tree_converter.xml" target_datatype="summary_tree"/> + <converter file="bed_gff_or_vcf_to_bigwig_converter.xml" target_datatype="bigwig"/></datatype><datatype extension="pdf" type="galaxy.datatypes.images:Pdf" mimetype="application/pdf"/><datatype extension="pileup" type="galaxy.datatypes.tabular:Pileup" display_in_upload="true"> @@ -172,7 +167,7 @@ <datatype extension="Roadmaps" type="galaxy.datatypes.assembly:Roadmaps" display_in_upload="false"/><datatype extension="sam" type="galaxy.datatypes.tabular:Sam" display_in_upload="true"><converter file="sam_to_bam.xml" target_datatype="bam"/> - <converter file="sam_to_summary_tree_converter.xml" target_datatype="summary_tree"/> + <converter file="sam_to_bigwig_converter.xml" target_datatype="bigwig"/></datatype><datatype extension="scf" type="galaxy.datatypes.binary:Scf" mimetype="application/octet-stream" display_in_upload="true"/><datatype extension="Sequences" type="galaxy.datatypes.assembly:Sequences" display_in_upload="false"/> @@ -190,7 +185,7 @@ <converter file="vcf_to_bgzip_converter.xml" target_datatype="bgzip"/><converter file="vcf_to_vcf_bgzip_converter.xml" target_datatype="vcf_bgzip"/><converter file="vcf_to_tabix_converter.xml" target_datatype="tabix" depends_on="bgzip"/> - <converter file="vcf_to_summary_tree_converter.xml" target_datatype="summary_tree"/> + <converter file="bed_gff_or_vcf_to_bigwig_converter.xml" target_datatype="bigwig"/><display file="ucsc/vcf.xml" /><display file="igv/vcf.xml" /><display file="rviewer/vcf.xml" inherit="True"/> @@ -203,7 +198,6 @@ <display file="igb/wig.xml" /></datatype> - <datatype extension="summary_tree" type="galaxy.datatypes.binary:Binary" subclass="True" /><datatype extension="interval_index" type="galaxy.datatypes.binary:Binary" subclass="True" /><datatype extension="tabix" type="galaxy.datatypes.binary:Binary" subclass="True" /><datatype extension="bgzip" type="galaxy.datatypes.binary:Binary" subclass="True" /> diff -r 38a9047cd5877b9123c06a92c2c53b6b488733cf -r 51f4926f6dd8b837c02dd8559587342542cdd7f2 lib/galaxy/datatypes/binary.py --- a/lib/galaxy/datatypes/binary.py +++ b/lib/galaxy/datatypes/binary.py @@ -103,7 +103,7 @@ """Class describing a BAM binary file""" file_ext = "bam" track_type = "ReadTrack" - data_sources = { "data": "bai", "index": [ "bigwig", "summary_tree" ] } + data_sources = { "data": "bai", "index": "bigwig" } MetadataElement( name="bam_index", desc="BAM Index File", param=metadata.FileParameter, file_ext="bai", readonly=True, no_value=None, visible=False, optional=True ) diff -r 38a9047cd5877b9123c06a92c2c53b6b488733cf -r 51f4926f6dd8b837c02dd8559587342542cdd7f2 lib/galaxy/datatypes/converters/bam_to_bigwig_converter.xml --- a/lib/galaxy/datatypes/converters/bam_to_bigwig_converter.xml +++ b/lib/galaxy/datatypes/converters/bam_to_bigwig_converter.xml @@ -1,7 +1,14 @@ <tool id="CONVERTER_bam_to_bigwig_0" name="Convert BAM to BigWig" version="1.0.0" hidden="true"><command> - bedtools genomecov -bg -split -ibam $input -g $chromInfo | wigToBigWig stdin $chromInfo $output + bedtools genomecov -bg -split -ibam $input -g $chromInfo + + ## Streaming the bedgraph file to wigToBigWig is fast but very memory intensive; hence, this + ## should only be used on systems with large RAM. + ## | wigToBigWig stdin $chromInfo $output + + ## This can be used anywhere. + > temp.bg ; bedGraphToBigWig temp.bg $chromInfo $output </command><inputs><param format="bam" name="input" type="data" label="Choose BAM file"/> diff -r 38a9047cd5877b9123c06a92c2c53b6b488733cf -r 51f4926f6dd8b837c02dd8559587342542cdd7f2 lib/galaxy/datatypes/converters/bam_to_summary_tree_converter.xml --- a/lib/galaxy/datatypes/converters/bam_to_summary_tree_converter.xml +++ /dev/null @@ -1,14 +0,0 @@ -<tool id="CONVERTER_bam_to_summary_tree_0" name="Convert BAM to Summary Tree" version="1.0.0" hidden="true"> -  - <command interpreter="python"> - sam_or_bam_to_summary_tree_converter.py --bam $input1 $input1.metadata.bam_index $output1 - </command> - <inputs> - <param format="bam" name="input1" type="data" label="Choose BAM file"/> - </inputs> - <outputs> - <data format="summary_tree" name="output1"/> - </outputs> - <help> - </help> -</tool> diff -r 38a9047cd5877b9123c06a92c2c53b6b488733cf -r 51f4926f6dd8b837c02dd8559587342542cdd7f2 lib/galaxy/datatypes/converters/bed_gff_or_vcf_to_bigwig_converter.xml --- /dev/null +++ b/lib/galaxy/datatypes/converters/bed_gff_or_vcf_to_bigwig_converter.xml @@ -0,0 +1,25 @@ +<tool id="CONVERTER_bed_gff_or_vcf_to_bigwig_0" name="Convert BED, GFF, or VCF to BigWig" version="1.0.0" hidden="true"> +  + <command> + ## Remove comments and sort by chromosome. + grep -v '^#' $input | sort -k1,1 | + + ## Generate coverage bedgraph. + bedtools genomecov -bg -split -i stdin -g $chromInfo + + ## Streaming the bedgraph file to wigToBigWig is fast but very memory intensive; hence, this + ## should only be used on systems with large RAM. + ## | wigToBigWig stdin $chromInfo $output + + ## This can be used anywhere. + > temp.bg ; bedGraphToBigWig temp.bg $chromInfo $output + </command> + <inputs> + <param format="bed,gff,vcf" name="input" type="data" label="Choose input file"/> + </inputs> + <outputs> + <data format="bigwig" name="output"/> + </outputs> + <help> + </help> +</tool> diff -r 38a9047cd5877b9123c06a92c2c53b6b488733cf -r 51f4926f6dd8b837c02dd8559587342542cdd7f2 lib/galaxy/datatypes/converters/bed_to_summary_tree_converter.xml --- a/lib/galaxy/datatypes/converters/bed_to_summary_tree_converter.xml +++ /dev/null @@ -1,14 +0,0 @@ -<tool id="CONVERTER_bed_to_summary_tree_0" name="Convert BED to Summary Tree" version="1.0.0" hidden="true"> - - <command interpreter="python">interval_to_summary_tree_converter.py $input1 $output1</command> - <inputs> - <page> - <param format="bed" name="input1" type="data" label="Choose BED file"/> - </page> - </inputs> - <outputs> - <data format="summary_tree" name="output1"/> - </outputs> - <help> - </help> -</tool> diff -r 38a9047cd5877b9123c06a92c2c53b6b488733cf -r 51f4926f6dd8b837c02dd8559587342542cdd7f2 lib/galaxy/datatypes/converters/encodepeak_to_summary_tree_converter.xml --- a/lib/galaxy/datatypes/converters/encodepeak_to_summary_tree_converter.xml +++ /dev/null @@ -1,20 +0,0 @@ -<tool id="CONVERTER_encodepeak_to_summary_tree_0" name="Convert ENCODEPeak to Summary Tree" version="1.0.0" hidden="true"> - - <command interpreter="python">interval_to_summary_tree_converter.py - -c ${input1.metadata.chromCol} - -s ${input1.metadata.startCol} - -e ${input1.metadata.endCol} - $input1 $output1 - </command> - - <inputs> - <page> - <param format="ENCODEPeak" name="input1" type="data" label="Choose ENCODEPeak file"/> - </page> - </inputs> - <outputs> - <data format="summary_tree" name="output1"/> - </outputs> - <help> - </help> -</tool> diff -r 38a9047cd5877b9123c06a92c2c53b6b488733cf -r 51f4926f6dd8b837c02dd8559587342542cdd7f2 lib/galaxy/datatypes/converters/gff_to_summary_tree_converter.xml --- a/lib/galaxy/datatypes/converters/gff_to_summary_tree_converter.xml +++ /dev/null @@ -1,14 +0,0 @@ -<tool id="CONVERTER_gff_to_summary_tree_0" name="Convert GFF to Summary Tree" version="1.0.0" hidden="true"> - - <command interpreter="python">interval_to_summary_tree_converter.py $input1 $output1 --gff</command> - <inputs> - <page> - <param format="gff" name="input1" type="data" label="Choose GFF file"/> - </page> - </inputs> - <outputs> - <data format="summary_tree" name="output1"/> - </outputs> - <help> - </help> -</tool> diff -r 38a9047cd5877b9123c06a92c2c53b6b488733cf -r 51f4926f6dd8b837c02dd8559587342542cdd7f2 lib/galaxy/datatypes/converters/interval_to_summary_tree_converter.py --- a/lib/galaxy/datatypes/converters/interval_to_summary_tree_converter.py +++ /dev/null @@ -1,63 +0,0 @@ -#!/usr/bin/env python - -""" -Convert from interval file to summary tree file. Default input file format is BED (0-based, half-open intervals). - -usage: %prog <options> in_file out_file - -c, --chr-col: chromosome column, default=1 - -s, --start-col: start column, default=2 - -e, --end-col: end column, default=3 - -t, --strand-col: strand column, default=6 - -G, --gff: input is GFF format, meaning start and end coordinates are 1-based, closed interval -""" -from __future__ import division - -import sys, fileinput, optparse -from galaxy import eggs -import pkg_resources; pkg_resources.require( "bx-python" ) -from galaxy.visualization.tracks.summary import * -from bx.intervals.io import * -from galaxy.datatypes.util.gff_util import * - -def main(): - # Read options, args. - parser = optparse.OptionParser() - parser.add_option( '-c', '--chr-col', type='int', dest='chrom_col', default=1 ) - parser.add_option( '-s', '--start-col', type='int', dest='start_col', default=2 ) - parser.add_option( '-e', '--end-col', type='int', dest='end_col', default=3 ) - parser.add_option( '-t', '--strand-col', type='int', dest='strand_col', default=6 ) - parser.add_option( '-G', '--gff', dest="gff_format", action="store_true" ) - (options, args) = parser.parse_args() - input_fname, output_fname = args - - # Convert column indices to 0-based. - options.chrom_col -= 1 - options.start_col -= 1 - options.end_col -= 1 - options.strand_col -= 1 - - # Do conversion. - if options.gff_format: - reader_wrapper_class = GFFReaderWrapper - chr_col, start_col, end_col, strand_col = ( 0, 3, 4, 6 ) - else: - reader_wrapper_class = NiceReaderWrapper - chr_col, start_col, end_col, strand_col = ( options.chrom_col, options.start_col, options.end_col, options.strand_col ) - reader_wrapper = reader_wrapper_class( fileinput.FileInput( input_fname ), - chrom_col=chr_col, - start_col=start_col, - end_col=end_col, - strand_col=strand_col, - fix_strand=True ) - st = SummaryTree() - for feature in list( reader_wrapper ): - if isinstance( feature, GenomicInterval ): - # Tree expects BED coordinates. - if type( feature ) is GFFFeature: - convert_gff_coords_to_bed( feature ) - st.insert_range( feature.chrom, long( feature.start ), long( feature.end ) ) - - st.write( output_fname ) - -if __name__ == "__main__": - main() diff -r 38a9047cd5877b9123c06a92c2c53b6b488733cf -r 51f4926f6dd8b837c02dd8559587342542cdd7f2 lib/galaxy/datatypes/converters/interval_to_summary_tree_converter.xml --- a/lib/galaxy/datatypes/converters/interval_to_summary_tree_converter.xml +++ /dev/null @@ -1,20 +0,0 @@ -<tool id="CONVERTER_interval_to_summary_tree_0" name="Convert Interval to Summary Tree" version="1.0.0" hidden="true"> - - <command interpreter="python">interval_to_summary_tree_converter.py - -c ${input1.metadata.chromCol} - -s ${input1.metadata.startCol} - -e ${input1.metadata.endCol} - $input1 $output1 - </command> - - <inputs> - <page> - <param format="interval" name="input1" type="data" label="Choose Interval file"/> - </page> - </inputs> - <outputs> - <data format="summary_tree" name="output1"/> - </outputs> - <help> - </help> -</tool> diff -r 38a9047cd5877b9123c06a92c2c53b6b488733cf -r 51f4926f6dd8b837c02dd8559587342542cdd7f2 lib/galaxy/datatypes/converters/sam_or_bam_to_summary_tree_converter.py --- a/lib/galaxy/datatypes/converters/sam_or_bam_to_summary_tree_converter.py +++ /dev/null @@ -1,42 +0,0 @@ -#!/usr/bin/env python - -from __future__ import division - -import sys, os, optparse -sys.stderr = open(os.devnull, 'w') # suppress stderr as cython produces warning on some systems: - # csamtools.so:6: RuntimeWarning: __builtin__.file size changed - -from galaxy import eggs -import pkg_resources - -if sys.version_info[:2] == (2, 4): - pkg_resources.require( "ctypes" ) -pkg_resources.require( "pysam" ) - -from pysam import csamtools -from galaxy.visualization.tracks.summary import * - -def main(): - parser = optparse.OptionParser() - parser.add_option( '-S', '--sam', action="store_true", dest="is_sam" ) - parser.add_option( '-B', '--bam', action="store_true", dest="is_bam" ) - options, args = parser.parse_args() - - if options.is_bam: - input_fname = args[0] - index_fname = args[1] - out_fname = args[2] - samfile = csamtools.Samfile( filename=input_fname, mode='rb', index_filename=index_fname ) - elif options.is_sam: - input_fname = args[0] - out_fname = args[1] - samfile = csamtools.Samfile( filename=input_fname, mode='r' ) - - st = SummaryTree() - for read in samfile.fetch(): - st.insert_range( samfile.getrname( read.rname ), read.pos, read.pos + read.rlen ) - - st.write(out_fname) - -if __name__ == "__main__": - main() diff -r 38a9047cd5877b9123c06a92c2c53b6b488733cf -r 51f4926f6dd8b837c02dd8559587342542cdd7f2 lib/galaxy/datatypes/converters/sam_to_summary_tree_converter.xml --- a/lib/galaxy/datatypes/converters/sam_to_summary_tree_converter.xml +++ /dev/null @@ -1,14 +0,0 @@ -<tool id="CONVERTER_sam_to_summary_tree_0" name="Convert SAM to Summary Tree" version="1.0.0" hidden="true"> - - <command interpreter="python">sam_or_bam_to_summary_tree_converter.py --sam $input1 $output1</command> - <inputs> - <page> - <param format="sam" name="input1" type="data" label="Choose sam file"/> - </page> - </inputs> - <outputs> - <data format="summary_tree" name="output1"/> - </outputs> - <help> - </help> -</tool> diff -r 38a9047cd5877b9123c06a92c2c53b6b488733cf -r 51f4926f6dd8b837c02dd8559587342542cdd7f2 lib/galaxy/datatypes/converters/vcf_to_summary_tree_converter.py --- a/lib/galaxy/datatypes/converters/vcf_to_summary_tree_converter.py +++ /dev/null @@ -1,30 +0,0 @@ -#!/usr/bin/env python - -""" -Convert from VCF file to summary tree file. - -usage: %prog in_file out_file -""" -from __future__ import division - -import optparse -import galaxy_utils.sequence.vcf -from galaxy.visualization.tracks.summary import SummaryTree - -def main(): - # Read options, args. - parser = optparse.OptionParser() - (options, args) = parser.parse_args() - in_file, out_file = args - - # Do conversion. - st = SummaryTree() - for line in list( galaxy_utils.sequence.vcf.Reader( open( in_file ) ) ): - # VCF format provides a chrom and 1-based position for each variant. - # SummaryTree expects 0-based coordinates. - st.insert_range( line.chrom, long( line.pos-1 ), long( line.pos ) ) - - st.write(out_file) - -if __name__ == "__main__": - main() \ No newline at end of file diff -r 38a9047cd5877b9123c06a92c2c53b6b488733cf -r 51f4926f6dd8b837c02dd8559587342542cdd7f2 lib/galaxy/datatypes/converters/vcf_to_summary_tree_converter.xml --- a/lib/galaxy/datatypes/converters/vcf_to_summary_tree_converter.xml +++ /dev/null @@ -1,14 +0,0 @@ -<tool id="CONVERTER_vcf_to_summary_tree_0" name="Convert VCF to Summary Tree" version="1.0.0" hidden="true"> -  - <command interpreter="python">vcf_to_summary_tree_converter.py $input1 $output1</command> - <inputs> - <page> - <param format="vcf" name="input1" type="data" label="Choose VCF file"/> - </page> - </inputs> - <outputs> - <data format="summary_tree" name="output1"/> - </outputs> - <help> - </help> -</tool> diff -r 38a9047cd5877b9123c06a92c2c53b6b488733cf -r 51f4926f6dd8b837c02dd8559587342542cdd7f2 lib/galaxy/datatypes/interval.py --- a/lib/galaxy/datatypes/interval.py +++ b/lib/galaxy/datatypes/interval.py @@ -47,7 +47,7 @@ file_ext = "interval" line_class = "region" track_type = "FeatureTrack" - data_sources = { "data": "tabix", "index": "summary_tree" } + data_sources = { "data": "tabix", "index": "bigwig" } """Add metadata elements""" MetadataElement( name="chromCol", default=1, desc="Chrom column", param=metadata.ColumnParameter ) @@ -354,7 +354,7 @@ class Bed( Interval ): """Tab delimited data in BED format""" file_ext = "bed" - data_sources = { "data": "tabix", "index": "summary_tree", "feature_search": "fli" } + data_sources = { "data": "tabix", "index": "bigwig", "feature_search": "fli" } track_type = Interval.track_type """Add metadata elements""" @@ -569,7 +569,7 @@ """Tab delimited data in Gff format""" file_ext = "gff" column_names = [ 'Seqname', 'Source', 'Feature', 'Start', 'End', 'Score', 'Strand', 'Frame', 'Group' ] - data_sources = { "data": "interval_index", "index": "summary_tree", "feature_search": "fli" } + data_sources = { "data": "interval_index", "index": "bigwig", "feature_search": "fli" } track_type = Interval.track_type """Add metadata elements""" @@ -1288,7 +1288,7 @@ file_ext = "encodepeak" column_names = [ 'Chrom', 'Start', 'End', 'Name', 'Score', 'Strand', 'SignalValue', 'pValue', 'qValue', 'Peak' ] - data_sources = { "data": "tabix", "index": "summary_tree" } + data_sources = { "data": "tabix", "index": "bigwig" } """Add metadata elements""" MetadataElement( name="chromCol", default=1, desc="Chrom column", param=metadata.ColumnParameter ) @@ -1307,7 +1307,7 @@ file_ext = "chrint" track_type = "DiagonalHeatmapTrack" - data_sources = { "data": "tabix", "index": "summary_tree" } + data_sources = { "data": "tabix", "index": "bigwig" } column_names = [ 'Chrom1', 'Start1', 'End1', 'Chrom2', 'Start2', 'End2', 'Value' ] diff -r 38a9047cd5877b9123c06a92c2c53b6b488733cf -r 51f4926f6dd8b837c02dd8559587342542cdd7f2 lib/galaxy/datatypes/tabular.py --- a/lib/galaxy/datatypes/tabular.py +++ b/lib/galaxy/datatypes/tabular.py @@ -359,7 +359,7 @@ class Sam( Tabular ): file_ext = 'sam' track_type = "ReadTrack" - data_sources = { "data": "bam", "index": "summary_tree" } + data_sources = { "data": "bam", "index": "bigwig" } def __init__(self, **kwd): """Initialize taxonomy datatype""" @@ -537,7 +537,7 @@ class Vcf( Tabular ): """ Variant Call Format for describing SNPs and other simple genome variations. """ track_type = "VariantTrack" - data_sources = { "data": "tabix", "index": "summary_tree" } + data_sources = { "data": "tabix", "index": "bigwig" } file_ext = 'vcf' column_names = [ 'Chrom', 'Pos', 'ID', 'Ref', 'Alt', 'Qual', 'Filter', 'Info', 'Format', 'data' ] diff -r 38a9047cd5877b9123c06a92c2c53b6b488733cf -r 51f4926f6dd8b837c02dd8559587342542cdd7f2 lib/galaxy/visualization/data_providers/genome.py --- a/lib/galaxy/visualization/data_providers/genome.py +++ b/lib/galaxy/visualization/data_providers/genome.py @@ -14,7 +14,6 @@ from bx.interval_index_file import Indexes from bx.bbi.bigwig_file import BigWigFile from galaxy.util.lrucache import LRUCache -from galaxy.visualization.tracks.summary import summary_tree_from_file from galaxy.visualization.data_providers.basic import BaseDataProvider from galaxy.visualization.data_providers.cigar import get_ref_based_read_seq_and_cigar from galaxy.datatypes.interval import Bed, Gff, Gtf @@ -800,79 +799,6 @@ return line_filter_iter() -class SummaryTreeDataProvider( GenomeDataProvider ): - """ - Summary tree data provider for the Galaxy track browser. - """ - - dataset_type = 'summary_tree' - - CACHE = LRUCache( 20 ) # Store 20 recently accessed indices for performance - - def valid_chroms( self ): - st = summary_tree_from_file( self.converted_dataset.file_name ) - return st.chrom_blocks.keys() - - def get_data( self, chrom, start, end, level=None, resolution=None, detail_cutoff=None, draw_cutoff=None, **kwargs ): - """ - Returns summary tree data for a given genomic region. - """ - filename = self.converted_dataset.file_name - st = self.CACHE[filename] - if st is None: - st = summary_tree_from_file( self.converted_dataset.file_name ) - self.CACHE[filename] = st - - # Look for chrom in tree using both naming conventions. - if chrom not in st.chrom_blocks: - chrom = _convert_between_ucsc_and_ensemble_naming( chrom ) - if chrom not in st.chrom_blocks: - return None - - # Get or compute level. - if level: - level = int( level ) - elif resolution: - resolution = max( 1, ceil( float( resolution ) ) ) - level = ceil( log( resolution, st.block_size ) ) - 1 - level = int( max( level, 0 ) ) - else: - # Either level or resolution is required. - return None - - if level <= 1: - return "detail" - - # Use level to get results. - stats = st.chrom_stats[ chrom ] - results = st.query( chrom, int(start), int(end), level, detail_cutoff=detail_cutoff, draw_cutoff=draw_cutoff ) - if results == "detail" or results == "draw": - return results - else: - return { - 'dataset_type': self.dataset_type, - 'data': results, - 'max': stats[ level ][ "max" ], - 'avg': stats[ level ][ "avg" ], - 'delta': stats[ level ][ "delta" ], - 'level': level - } - - def has_data( self, chrom ): - """ - Returns true if dataset has data for this chrom - """ - - # Get summary tree. - filename = self.converted_dataset.file_name - st = self.CACHE[filename] - if st is None: - st = summary_tree_from_file( self.converted_dataset.file_name ) - self.CACHE[filename] = st - - # Check for data. - return st.chrom_blocks.get(chrom, None) or st.chrom_blocks.get(_convert_between_ucsc_and_ensemble_naming(chrom), None) - class BamDataProvider( GenomeDataProvider, FilterableMixin ): """ Provides access to intervals from a sorted indexed BAM file. Coordinate diff -r 38a9047cd5877b9123c06a92c2c53b6b488733cf -r 51f4926f6dd8b837c02dd8559587342542cdd7f2 lib/galaxy/visualization/data_providers/registry.py --- a/lib/galaxy/visualization/data_providers/registry.py +++ b/lib/galaxy/visualization/data_providers/registry.py @@ -29,7 +29,6 @@ "interval_index": genome.IntervalIndexDataProvider, "bai": genome.BamDataProvider, "bam": genome.SamDataProvider, - "summary_tree": genome.SummaryTreeDataProvider, "bigwig": genome.BigWigDataProvider, "bigbed": genome.BigBedDataProvider } diff -r 38a9047cd5877b9123c06a92c2c53b6b488733cf -r 51f4926f6dd8b837c02dd8559587342542cdd7f2 lib/galaxy/visualization/tracks/summary.py --- a/lib/galaxy/visualization/tracks/summary.py +++ /dev/null @@ -1,117 +0,0 @@ -''' -This module cannot be moved due to the use of pickling. -''' - -import sys, os -import cPickle - -# TODO: What are the performance implications of setting min level to 1? Data -# structure size and/or query speed? It would be nice to have level 1 data -# so that client does not have to compute it. -MIN_LEVEL = 2 - -class SummaryTree: - ''' - Summary tree data structure for feature aggregation across large genomic regions. - ''' - def __init__( self, block_size=25, levels=6, draw_cutoff=150, detail_cutoff=30 ): - self.chrom_blocks = {} - self.levels = levels - self.draw_cutoff = draw_cutoff - self.detail_cutoff = detail_cutoff - self.block_size = block_size - self.chrom_stats = {} - - def find_block( self, num, level ): - """ Returns block that num is in for level. """ - return ( num / self.block_size ** level ) - - def insert_range( self, chrom, start, end ): - """ Inserts a feature at chrom:start-end into the tree. """ - - # Get or set up chrom blocks. - if chrom in self.chrom_blocks: - blocks = self.chrom_blocks[ chrom ] - else: - blocks = self.chrom_blocks[ chrom ] = {} - self.chrom_stats[ chrom ] = {} - for level in range( MIN_LEVEL, self.levels + 1 ): - blocks[ level ] = {} - - # Insert feature into all matching blocks at all levels. - for level in range( MIN_LEVEL, self.levels + 1 ): - block_level = blocks[ level ] - starting_block = self.find_block( start, level ) - ending_block = self.find_block( end, level ) - for block in range( starting_block, ending_block + 1 ): - if block in block_level: - block_level[ block ] += 1 - else: - block_level[ block ] = 1 - - def finish( self ): - """ Compute stats for levels. """ - - for chrom, blocks in self.chrom_blocks.iteritems(): - for level in range( self.levels, MIN_LEVEL - 1, -1 ): - # Set level's stats. - max_val = max( blocks[ level ].values() ) - self.chrom_stats[ chrom ][ level ] = {} - self.chrom_stats[ chrom ][ level ][ "delta" ] = self.block_size ** level - self.chrom_stats[ chrom ][ level ][ "max" ] = max_val - self.chrom_stats[ chrom ][ level ][ "avg" ] = float( max_val ) / len( blocks[ level ] ) - - self.chrom_blocks[ chrom ] = dict( [ ( key, value ) for key, value in blocks.iteritems() ] ) - - def query( self, chrom, start, end, level, draw_cutoff=None, detail_cutoff=None ): - """ Queries tree for data. """ - - # Set cutoffs to self's attributes if not defined. - if draw_cutoff != 0: - draw_cutoff = self.draw_cutoff - if detail_cutoff != 0: - detail_cutoff = self.detail_cutoff - - # Get data. - if chrom in self.chrom_blocks: - stats = self.chrom_stats[ chrom ] - - # For backwards compatibility: - if "detail_level" in stats and level <= stats[ "detail_level" ]: - return "detail" - elif "draw_level" in stats and level <= stats[ "draw_level" ]: - return "draw" - - # If below draw, detail level, return string to denote this. - max = stats[ level ][ "max" ] - if max < detail_cutoff: - return "detail" - if max < draw_cutoff: - return "draw" - - # Return block data. - blocks = self.chrom_blocks[ chrom ] - results = [] - multiplier = self.block_size ** level - starting_block = self.find_block( start, level ) - ending_block = self.find_block( end, level ) - for block in range( starting_block, ending_block + 1 ): - val = 0 - if block in blocks[ level ]: - val = blocks[ level ][ block ] - results.append( ( block * multiplier, val ) ) - return results - - return None - - def write( self, filename ): - """ Writes tree to file. """ - self.finish() - cPickle.dump( self, open( filename, 'wb' ), 2 ) - -def summary_tree_from_file( filename ): - st_file = open( filename, "rb" ) - st = cPickle.load( st_file ) - st_file.close() - return st - diff -r 38a9047cd5877b9123c06a92c2c53b6b488733cf -r 51f4926f6dd8b837c02dd8559587342542cdd7f2 lib/galaxy/webapps/galaxy/api/datasets.py --- a/lib/galaxy/webapps/galaxy/api/datasets.py +++ b/lib/galaxy/webapps/galaxy/api/datasets.py @@ -145,32 +145,41 @@ extra_info = None mode = kwargs.get( "mode", "Auto" ) - # Handle histogram mode uniquely for now: data_provider_registry = trans.app.data_provider_registry + + # Coverage mode uses index data. if mode == "Coverage": # Get summary using minimal cutoffs. indexer = data_provider_registry.get_data_provider( trans, original_dataset=dataset, source='index' ) - summary = indexer.get_data( chrom, low, high, detail_cutoff=0, draw_cutoff=0, **kwargs ) - if summary == "detail": - # Use maximum level of detail--2--to get summary data no matter the resolution. - summary = indexer.get_data( chrom, low, high, resolution=kwargs[ 'resolution' ], - level=2, detail_cutoff=0, draw_cutoff=0 ) - return summary + return indexer.get_data( chrom, low, high, **kwargs ) - if 'index' in data_sources and data_sources['index']['name'] == "summary_tree" and mode == "Auto": - # Only check for summary_tree if it's Auto mode (which is the default) - # - # Have to choose between indexer and data provider + # TODO: + # (1) add logic back in for no_detail + # (2) handle scenario where mode is Squish/Pack but data requested is large, so reduced data needed to be returned. + + # If mode is Auto, need to determine what type of data to return. + if mode == "Auto": + # Get stats from indexer. indexer = data_provider_registry.get_data_provider( trans, original_dataset=dataset, source='index' ) - summary = indexer.get_data( chrom, low, high, resolution=kwargs[ 'resolution' ] ) - if summary is None: - return { 'dataset_type': indexer.dataset_type, 'data': None } - - if summary == "draw": - kwargs["no_detail"] = True # meh - extra_info = "no_detail" - elif summary != "detail": - return summary + stats = indexer.get_data( chrom, low, high, stats=True ) + + # If stats were requested, return them. + if 'stats' in kwargs: + if stats[ 'data' ][ 'max' ] == 0: + return { 'dataset_type': indexer.dataset_type, 'data': None } + else: + return stats + + # Use heuristic based on max depth and region size to determine whether to + # return coverage data. When zoomed out and region is large, max depth + # is determining factor. However, when sufficiently zoomed in and region is + # small, coverage data is no longer provided. + if int( high ) - int( low ) > 50000 and stats[ 'data' ][ 'max' ] > 1000: + return indexer.get_data( chrom, low, high ) + + # + # Provide individual data points. + # # Get data provider. data_provider = data_provider_registry.get_data_provider( trans, original_dataset=dataset, source='data' ) diff -r 38a9047cd5877b9123c06a92c2c53b6b488733cf -r 51f4926f6dd8b837c02dd8559587342542cdd7f2 static/scripts/utils/config.js --- a/static/scripts/utils/config.js +++ b/static/scripts/utils/config.js @@ -58,7 +58,6 @@ { key: 'mode', type: 'string', default_value: this.mode, hidden: true }, { key: 'reverse_strand_color', label: 'Antisense strand color', type: 'color', default_value: null }, { key: 'show_differences', label: 'Show differences only', type: 'bool', default_value: true }, - { key: 'histogram_max', label: 'Histogram maximum', type: 'float', default_value: null, help: 'Clear value to set automatically' }, { key: 'mode', type: 'string', default_value: this.mode, hidden: true } ] }); diff -r 38a9047cd5877b9123c06a92c2c53b6b488733cf -r 51f4926f6dd8b837c02dd8559587342542cdd7f2 static/scripts/viz/trackster/painters.js --- a/static/scripts/viz/trackster/painters.js +++ b/static/scripts/viz/trackster/painters.js @@ -133,49 +133,6 @@ */ Painter.prototype.draw = function(ctx, width, height, w_scale) {}; -/** - * SummaryTreePainter, a histogram showing number of intervals in a region - */ -var SummaryTreePainter = function(data, view_start, view_end, prefs, mode) { - Painter.call(this, data, view_start, view_end, prefs, mode); -}; - -SummaryTreePainter.prototype.default_prefs = { show_counts: false }; - -SummaryTreePainter.prototype.draw = function(ctx, width, height, w_scale) { - var view_start = this.view_start, - points = this.data.data, - max = (this.prefs.histogram_max ? this.prefs.histogram_max : this.data.max), - // Set base Y so that max label and data do not overlap. Base Y is where rectangle bases - // start. However, height of each rectangle is relative to required_height; hence, the - // max rectangle is required_height. - base_y = height, - delta_x_px = Math.ceil(this.data.delta * w_scale); - ctx.save(); - - for (var i = 0, len = points.length; i < len; i++) { - var x = Math.floor( (points[i][0] - view_start) * w_scale ); - var y = points[i][1]; - - if (!y) { continue; } - var y_px = y / max * height; - if (y !== 0 && y_px < 1) { y_px = 1; } - - ctx.fillStyle = this.prefs.block_color; - ctx.fillRect( x, base_y - y_px, delta_x_px, y_px ); - - // Draw number count if it can fit the number with some padding, otherwise things clump up - var text_padding_req_x = 4; - if (this.prefs.show_counts && (ctx.measureText(y).width + text_padding_req_x) < delta_x_px) { - ctx.fillStyle = this.prefs.label_color; - ctx.textAlign = "center"; - ctx.fillText(y, x + (delta_x_px/2), 10); - } - } - - ctx.restore(); -}; - var LinePainter = function(data, view_start, view_end, prefs, mode) { Painter.call( this, data, view_start, view_end, prefs, mode ); var i, len; @@ -1692,7 +1649,6 @@ return { Scaler: Scaler, - SummaryTreePainter: SummaryTreePainter, LinePainter: LinePainter, LinkedFeaturePainter: LinkedFeaturePainter, ReadPainter: ReadPainter, diff -r 38a9047cd5877b9123c06a92c2c53b6b488733cf -r 51f4926f6dd8b837c02dd8559587342542cdd7f2 static/scripts/viz/trackster/tracks.js --- a/static/scripts/viz/trackster/tracks.js +++ b/static/scripts/viz/trackster/tracks.js @@ -2076,11 +2076,10 @@ */ Tile.prototype.predisplay_actions = function() {}; -var SummaryTreeTile = function(track, region, resolution, canvas, data, max_val) { +var LineTrackTile = function(track, region, resolution, canvas, data) { Tile.call(this, track, region, resolution, canvas, data); - this.max_val = max_val; }; -extend(SummaryTreeTile.prototype, Tile.prototype); +LineTrackTile.prototype.predisplay_actions = function() {}; var FeatureTrackTile = function(track, region, resolution, canvas, data, w_scale, mode, message, all_slotted, feature_mapper) { // Attribute init. @@ -2601,7 +2600,7 @@ track.enabled = false; track.tile_cache.clear(); track.data_manager.clear(); - track.content_div.css("height", "auto"); + track.tiles_div.css("height", "auto"); /* if (!track.content_div.text()) { track.content_div.text(DATA_LOADING); @@ -2669,7 +2668,7 @@ track.tiles_div.css( "height", track.visible_height_px + "px" ); track.enabled = true; // predraw_init may be asynchronous, wait for it and then draw - $.when(track.predraw_init()).done(function() { + $.when.apply($, track.predraw_init()).done(function() { init_deferred.resolve(); track.container_div.removeClass("nodata error pending"); track.request_draw(); @@ -2688,7 +2687,29 @@ /** * Additional initialization required before drawing track for the first time. */ - predraw_init: function() {}, + predraw_init: function() { + var track = this; + return $.getJSON( track.dataset.url(), + { data_type: 'data', stats: true, chrom: track.view.chrom, low: 0, + high: track.view.max_high, hda_ldda: track.dataset.get('hda_ldda') }, function(result) { + track.container_div.addClass( "line-track" ); + var data = result.data; + + // Tracks may not have stat data either because there is no data or data is not yet ready. + if (data && data.min && data.max) { + // Compute default minimum and maximum values + var min_value = data.min, + max_value = data.max; + // If mean and sd are present, use them to compute a ~95% window + // but only if it would shrink the range on one side + min_value = Math.floor( Math.min( 0, Math.max( min_value, data.mean - 2 * data.sd ) ) ); + max_value = Math.ceil( Math.max( 0, Math.min( max_value, data.mean + 2 * data.sd ) ) ); + // Update the prefs + track.prefs.min_value = min_value; + track.prefs.max_value = max_value; + } + }); + }, /** * Returns all drawables in this drawable. @@ -2801,6 +2822,32 @@ }, /** + * Set track bounds for current chromosome. + */ + set_min_max: function() { + var track = this; + + return $.getJSON( track.dataset.url(), + { data_type: 'data', stats: true, chrom: track.view.chrom, low: 0, + high: track.view.max_high, hda_ldda: track.dataset.get('hda_ldda') }, + function(result) { + var data = result.data; + if ( isNaN(parseFloat(track.prefs.min_value)) || isNaN(parseFloat(track.prefs.max_value)) ) { + // Compute default minimum and maximum values + var min_value = data.min, + max_value = data.max; + // If mean and sd are present, use them to compute a ~95% window + // but only if it would shrink the range on one side + min_value = Math.floor( Math.min( 0, Math.max( min_value, data.mean - 2 * data.sd ) ) ); + max_value = Math.ceil( Math.max( 0, Math.min( max_value, data.mean + 2 * data.sd ) ) ); + // Update the prefs + track.prefs.min_value = min_value; + track.prefs.max_value = max_value; + } + }); + }, + + /** * Change track's mode. */ change_mode: function(new_mode) { @@ -2808,6 +2855,10 @@ // TODO: is it necessary to store the mode in two places (.mode and track_config)? track.mode = new_mode; track.config.values['mode'] = new_mode; + // FIXME: find a better way to get Auto data w/o clearing cache; using mode in the + // data manager would work if Auto data were checked for compatibility when a specific + // mode is chosen. + if (new_mode === 'Auto') { this.data_manager.clear(); } track.request_draw(true); this.action_icons.mode_icon.attr("title", "Set display mode (now: " + track.mode + ")"); return track; @@ -2954,10 +3005,11 @@ /** * Add a maximum/minimum label to track. */ - _add_yaxis_label: function(type, val, pref_name, on_change) { + _add_yaxis_label: function(type, on_change) { var track = this, css_class = (type === 'max' ? 'top' : 'bottom'), text = (type === 'max' ? 'max' : 'min'), + pref_name = (type === 'max' ? 'max_value' : 'min_value'), // Default action for on_change is to redraw track. on_change = on_change || function() { track.request_draw(true); @@ -2966,11 +3018,11 @@ if (label.length !== 0) { // Label already exists, so update value. - label.text(val); + label.text(track.prefs[pref_name]); } else { // Add label. - label = $("<div/>").text(val).make_text_editable({ + label = $("<div/>").text(track.prefs[pref_name]).make_text_editable({ num_cols: 12, on_finish: function(new_val) { $(".bs-tooltip").remove(); @@ -2990,6 +3042,29 @@ */ postdraw_actions: function(tiles, width, w_scale, clear_after) { // + // If some tiles have line track tiles (which denote coverage data), redraw all tiles using coverage data. + // + var non_line_track_tiles = _.filter(tiles, function(tile) { + return !(tile instanceof LineTrackTile); + }); + + if (tiles.length !== non_line_track_tiles.length) { + // Clear because this is set when drawing. + this.max_height_px = 0; + var track = this; + _.each(non_line_track_tiles, function(tile) { + tile.html_elt.remove(); + track.draw_helper(true, tile.region, tile.resolution, track.tiles_div, w_scale, { mode: 'Coverage' }); + }); + + track._add_yaxis_label('max'); + } + else { + // Remove Y-axis labels because there are no line track tiles. + this.container_div.find('.yaxislabel').remove(); + } + + // // If some tiles have icons, set padding of tiles without icons so features and rows align. // var icons_present = _.find(tiles, function(tile) { @@ -3004,16 +3079,6 @@ } }); } - - // - // If using SummaryTree tiles, show max. - // - var track = this, - first_tile = tiles[0]; - if (first_tile instanceof SummaryTreeTile) { - var max_val = (this.prefs.histogram_max ? this.prefs.histogram_max : first_tile.max_val); - this._add_yaxis_label('max', max_val, 'histogram_max'); - } }, /** @@ -3021,12 +3086,13 @@ * jQuery.Deferred object that is fulfilled when tile can be drawn again. */ draw_helper: function(force, region, resolution, parent_element, w_scale, kwargs) { + // Init kwargs if necessary to avoid having to check if kwargs defined. + if (!kwargs) { kwargs = {}; } + var track = this, key = this._gen_tile_cache_key(w_scale, region), - is_tile = function(o) { return (o && 'track' in o); }; - - // Init kwargs if necessary to avoid having to check if kwargs defined. - if (!kwargs) { kwargs = {}; } + is_tile = function(o) { return (o && 'track' in o); }, + mode = kwargs.mode || track.mode; // Check tile cache, if found show existing tile in correct position var tile = (force ? undefined : track.tile_cache.get_elt(key)); @@ -3041,7 +3107,7 @@ var can_draw_now = true; // Get the track data, maybe a deferred - var tile_data = track.data_manager.get_data(region, track.mode, resolution, track.data_url_extra_params); + var tile_data = track.data_manager.get_data(region, mode, resolution, track.data_url_extra_params); if ( is_deferred( tile_data ) ) { can_draw_now = false; } @@ -3049,7 +3115,7 @@ // Get reference data if needed, maybe a deferred var seq_data; if ( view.reference_track ) { - seq_data = view.reference_track.data_manager.get_data(region, track.mode, resolution, view.reference_track.data_url_extra_params); + seq_data = view.reference_track.data_manager.get_data(region, mode, resolution, view.reference_track.data_url_extra_params); if ( is_deferred( seq_data ) ) { can_draw_now = false; } @@ -3066,7 +3132,6 @@ // HACK: this is FeatureTrack-specific. // If track mode is Auto, determine mode and update. - var mode = track.mode; if (mode === "Auto" && track.get_mode) { mode = track.get_mode(tile_data); track.update_auto_mode(mode); @@ -3079,7 +3144,6 @@ tile_high = region.get('end'), width = Math.ceil( (tile_high - tile_low) * w_scale ) + track.left_offset, height = track.get_canvas_height(tile_data, mode, w_scale, width); - canvas.width = width; canvas.height = height; var ctx = canvas.getContext('2d'); @@ -3115,12 +3179,14 @@ }, /** - * Draw a track tile for summary tree data. + * Draw line (bigwig) data onto tile. */ - _draw_summary_tree_tile: function(result, ctx, region, resolution, w_scale) { - var painter = new painters.SummaryTreePainter(result, region.get('start'), region.get('end'), this.prefs); - painter.draw(ctx, ctx.canvas.width, ctx.canvas.height, w_scale); - return new SummaryTreeTile(this, region, resolution, ctx.canvas, result.data, result.max); + _draw_line_track_tile: function(result, ctx, mode, resolution, region, w_scale) { + var canvas = ctx.canvas, + painter = new painters.LinePainter(result.data, region.get('start'), region.get('end'), this.prefs, mode); + painter.draw(ctx, canvas.width, canvas.height, w_scale); + + return new LineTrackTile(this, region, resolution, canvas, result.data); }, /** @@ -3277,10 +3343,17 @@ }; extend(LabelTrack.prototype, Track.prototype, { build_header_div: function() {}, + init: function() { // Enable by default because there should always be data when drawing track. this.enabled = true; }, + + /** + * Additional initialization required before drawing track for the first time. + */ + predraw_init: function() {}, + _draw: function() { var view = this.view, range = view.high - view.low, @@ -3623,8 +3696,8 @@ t = function() { track.update_all_min_max(); }; // Add min, max labels. - this._add_yaxis_label('min', this.drawables[0].prefs.min_value, 'min_value', t); - this._add_yaxis_label('max', this.drawables[0].prefs.max_value, 'max_value', t); + this._add_yaxis_label('min', t); + this._add_yaxis_label('max', t); } }); @@ -3655,6 +3728,11 @@ this.enabled = true; }, + /** + * Additional initialization required before drawing track for the first time. + */ + predraw_init: function() {}, + can_draw: Drawable.prototype.can_draw, /** @@ -3716,42 +3794,15 @@ saved_values: obj_dict.prefs, onchange: function() { track.set_name(track.prefs.name); - track.vertical_range = track.prefs.max_value - track.prefs.min_value; track.request_redraw(true); } }); this.prefs = this.config.values; this.visible_height_px = this.config.values.height; - this.vertical_range = this.config.values.max_value - this.config.values.min_value; }; extend(LineTrack.prototype, Drawable.prototype, TiledTrack.prototype, { - predraw_init: function() { - var track = this; - track.vertical_range = undefined; - return $.getJSON( track.dataset.url(), - { data_type: 'data', stats: true, chrom: track.view.chrom, low: 0, - high: track.view.max_high, hda_ldda: track.dataset.get('hda_ldda') }, function(result) { - track.container_div.addClass( "line-track" ); - var data = result.data; - if ( isNaN(parseFloat(track.prefs.min_value)) || isNaN(parseFloat(track.prefs.max_value)) ) { - // Compute default minimum and maximum values - var min_value = data.min, - max_value = data.max; - // If mean and sd are present, use them to compute a ~95% window - // but only if it would shrink the range on one side - min_value = Math.floor( Math.min( 0, Math.max( min_value, data.mean - 2 * data.sd ) ) ); - max_value = Math.ceil( Math.max( 0, Math.min( max_value, data.mean + 2 * data.sd ) ) ); - // Update the prefs - track.prefs.min_value = min_value; - track.prefs.max_value = max_value; - } - track.vertical_range = track.prefs.max_value - track.prefs.min_value; - track.total_frequency = data.total_frequency; - }); - }, - /** * Actions to be taken before drawing. */ @@ -3759,18 +3810,10 @@ before_draw: function() {}, /** - * Draw LineTrack tile. + * Draw track tile. */ draw_tile: function(result, ctx, mode, resolution, region, w_scale) { - // Paint onto canvas. - var - canvas = ctx.canvas, - tile_low = region.get('start'), - tile_high = region.get('end'), - painter = new painters.LinePainter(result.data, tile_low, tile_high, this.prefs, mode); - painter.draw(ctx, canvas.width, canvas.height, w_scale); - - return new Tile(this, region, resolution, canvas, result.data); + return this._draw_line_track_tile(result, ctx, mode, resolution, region, w_scale); }, /** @@ -3811,14 +3854,12 @@ saved_values: obj_dict.prefs, onchange: function() { track.set_name(track.prefs.name); - track.vertical_range = track.prefs.max_value - track.prefs.min_value; this.request_redraw(true); } }); this.prefs = this.config.values; this.visible_height_px = this.config.values.height; - this.vertical_range = this.config.values.max_value - this.config.values.min_value; }; extend(DiagonalHeatmapTrack.prototype, Drawable.prototype, TiledTrack.prototype, { /** @@ -3863,7 +3904,8 @@ { key: 'label_color', label: 'Label color', type: 'color', default_value: 'black' }, { key: 'show_counts', label: 'Show summary counts', type: 'bool', default_value: true, help: 'Show the number of items in each bin when drawing summary histogram' }, - { key: 'histogram_max', label: 'Histogram maximum', type: 'float', default_value: null, help: 'clear value to set automatically' }, + { key: 'min_value', label: 'Histogram minimum', type: 'float', default_value: null, help: 'clear value to set automatically' }, + { key: 'max_value', label: 'Histogram maximum', type: 'float', default_value: null, help: 'clear value to set automatically' }, { key: 'connector_style', label: 'Connector style', type: 'select', default_value: 'fishbones', options: [ { label: 'Line with arrows', value: 'fishbone' }, { label: 'Arcs', value: 'arcs' } ] }, { key: 'mode', type: 'string', default_value: this.mode, hidden: true }, @@ -3905,12 +3947,14 @@ * drawn/fetched and shown. */ postdraw_actions: function(tiles, width, w_scale, clear_after) { - TiledTrack.prototype.postdraw_actions.call(this, tiles, clear_after); + TiledTrack.prototype.postdraw_actions.call(this, tiles, width, w_scale, clear_after); var track = this, i; // If mode is Coverage and tiles do not share max, redraw tiles as necessary using new max. + /* + This code isn't used right now because Coverage mode uses predefined max in preferences. if (track.mode === "Coverage") { // Get global max. var global_max = -1; @@ -3928,7 +3972,8 @@ track.draw_helper(true, tile.index, tile.resolution, tile.html_elt.parent(), w_scale, { more_tile_data: { max: global_max } } ); } } - } + } + */ // // Update filter attributes, UI. @@ -3992,17 +4037,17 @@ this.action_icons.show_more_rows_icon.hide(); } }, + update_auto_mode: function( mode ) { var mode; if ( this.mode === "Auto" ) { if ( mode === "no_detail" ) { mode = "feature spans"; - } else if ( mode === "summary_tree" ) { - mode = "coverage histogram"; } this.action_icons.mode_icon.attr("title", "Set display mode (now: Auto/" + mode + ")"); } }, + /** * Place features in slots for drawing (i.e. pack features). * this.slotters[level] is created in this method. this.slotters[level] @@ -4022,15 +4067,13 @@ return slotter.slot_features( features ); }, + /** * Returns appropriate display mode based on data. */ get_mode: function(data) { - if (data.dataset_type === "summary_tree") { - mode = "summary_tree"; - } // HACK: use no_detail mode track is in overview to prevent overview from being too large. - else if (data.extra_info === "no_detail" || this.is_overview) { + if (data.extra_info === "no_detail" || this.is_overview) { mode = "no_detail"; } else { @@ -4054,12 +4097,13 @@ } return mode; }, + /** * Returns canvas height needed to display data; return value is an integer that denotes the * number of pixels required. */ get_canvas_height: function(result, mode, w_scale, canvas_width) { - if (mode === "summary_tree" || mode === "Coverage") { + if (mode === "Coverage" || result.dataset_type === 'bigwig') { return this.summary_draw_height; } else { @@ -4071,6 +4115,7 @@ return Math.max(MIN_TRACK_HEIGHT, dummy_painter.get_required_height(rows_required, canvas_width) ); } }, + /** * Draw FeatureTrack tile. * @param result result from server @@ -4087,10 +4132,10 @@ tile_low = region.get('start'), tile_high = region.get('end'), left_offset = this.left_offset; - - // Drawing the summary tree. - if (mode === "summary_tree" || mode === "Coverage") { - return this._draw_summary_tree_tile(result, ctx, region, resolution, w_scale); + + // If data is line track data, draw line track tile. + if (result.dataset_type === 'bigwig') { + return this._draw_line_track_tile(result, ctx, 'Histogram', resolution, region, w_scale); } // Handle row-by-row tracks @@ -4147,6 +4192,7 @@ return new FeatureTrackTile(track, region, resolution, canvas, result.data, w_scale, mode, result.message, all_slotted, feature_mapper); }, + /** * Returns true if data is compatible with a given mode. */ @@ -4155,24 +4201,25 @@ if (mode === "Auto") { return true; } - // Histogram mode requires summary_tree data. + // Histogram mode requires bigwig data. else if (mode === "Coverage") { - return data.dataset_type === "summary_tree"; + return data.dataset_type === "bigwig"; } // All other modes--Dense, Squish, Pack--require data + details. - else if (data.extra_info === "no_detail" || data.dataset_type === "summary_tree") { + else if (data.extra_info === "no_detail") { return false; } else { return true; } }, + /** * Returns true if data can be subsetted. */ can_subset: function(data) { - // Do not subset summary tree data, entries with a message, or data with no detail. - if (data.dataset_type === "summary_tree" || data.message || data.extra_info === "no_detail") { + // Do not subset entries with a message or data with no detail. + if (data.dataset_type === 'bigwig' || data.message || data.extra_info === "no_detail") { return false; } @@ -4217,9 +4264,9 @@ * Draw tile. */ draw_tile: function(result, ctx, mode, resolution, region, w_scale) { - // Data could be summary tree data or variant data. - if (result.dataset_type === 'summary_tree') { - return this._draw_summary_tree_tile(result, ctx, region, resolution, w_scale); + // Data could be coverage data or variant data. + if (result.dataset_type === 'bigwig') { + return this._draw_line_track_tile(result, ctx, "Histogram", region, resolution, w_scale); } else { // result.dataset_type === 'variant' var view = this.view, @@ -4235,7 +4282,7 @@ * number of pixels required. */ get_canvas_height: function(result, mode, w_scale, canvas_width) { - if (result.dataset_type === 'summary_tree') { + if (result.dataset_type === 'bigwig') { return this.summary_draw_height; } else { @@ -4264,9 +4311,11 @@ * Additional initialization required before drawing track for the first time. */ predraw_init: function() { + var deferreds = [ Track.prototype.predraw_init.call(this) ]; if (!this.dataset.get_metadata('sample_names')) { - return this.dataset.fetch(); + deferreds.push(this.dataset.fetch()); } + return deferreds; }, /** @@ -4277,7 +4326,7 @@ TiledTrack.prototype.postdraw_actions.call(this, tiles, width, w_scale, clear_after); // Add summary/sample labels if needed and not already included. - if ( !(tiles[0] instanceof SummaryTreeTile) && this.prefs.show_labels) { + if ( !(tiles[0] instanceof LineTrackTile) && this.prefs.show_labels) { var font_size; // Add and/or style labels. @@ -4343,7 +4392,6 @@ { key: 'show_insertions', label: 'Show insertions', type: 'bool', default_value: false }, { key: 'show_differences', label: 'Show differences only', type: 'bool', default_value: true }, { key: 'show_counts', label: 'Show summary counts', type: 'bool', default_value: true }, - { key: 'histogram_max', label: 'Histogram maximum', type: 'float', default_value: null, help: 'Clear value to set automatically' }, { key: 'mode', type: 'string', default_value: this.mode, hidden: true } ], saved_values: obj_dict.prefs, diff -r 38a9047cd5877b9123c06a92c2c53b6b488733cf -r 51f4926f6dd8b837c02dd8559587342542cdd7f2 static/scripts/viz/visualization.js --- a/static/scripts/viz/visualization.js +++ b/static/scripts/viz/visualization.js @@ -505,9 +505,6 @@ // FIXME: constant should go somewhere. extra_params.num_samples = 1000 * detail_multiplier; } - else if (cur_data.dataset_type === 'summary_tree') { - extra_params.level = Math.min(cur_data.level - 1, 2); - } return this.load_data(region, mode, resolution, extra_params); }, @@ -579,7 +576,7 @@ data_point[0] <= subregion.get('end'); }); }, - 'refseq': function(data, subregion) { + refseq: function(data, subregion) { var seq_start = subregion.get('start') - entry.region.get('start'), seq_end = entry.data.length - ( entry.region.get('end') - subregion.get('end') ); return entry.data.slice(seq_start, seq_end); Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: dan: Unicode fixes for previewing large datasets.
by commits-noreply＠bitbucket.org 09 May '13

09 May '13

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/7930e04adb53/ Changeset: 7930e04adb53 Branch: stable User: dan Date: 2013-05-09 16:49:02 Summary: Unicode fixes for previewing large datasets. Affected #: 2 files diff -r 4590977323e76d9448c29c5fdb25f2bb7c500229 -r 7930e04adb535b8bf3c7973efc5cdce0fb384034 lib/galaxy/datatypes/tabular.py --- a/lib/galaxy/datatypes/tabular.py +++ b/lib/galaxy/datatypes/tabular.py @@ -265,7 +265,7 @@ while cursor and ck_data[-1] != '\n': ck_data += cursor cursor = f.read(1) - return to_json_string({'ck_data': ck_data, 'ck_index': ck_index+1}) + return to_json_string( { 'ck_data': util.unicodify( ck_data ), 'ck_index': ck_index + 1 } ) def display_data(self, trans, dataset, preview=False, filename=None, to_ext=None, chunk=None): preview = util.string_as_bool( preview ) diff -r 4590977323e76d9448c29c5fdb25f2bb7c500229 -r 7930e04adb535b8bf3c7973efc5cdce0fb384034 templates/webapps/galaxy/dataset/large_file.mako --- a/templates/webapps/galaxy/dataset/large_file.mako +++ b/templates/webapps/galaxy/dataset/large_file.mako @@ -7,5 +7,5 @@ </div><pre> -${ unicode( truncated_data, 'utf-8' ) | h } +${ util.unicodify( truncated_data ) | h } </pre> Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: dan: Unicode fixes for previewing large datasets.
by commits-noreply＠bitbucket.org 09 May '13

09 May '13

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/38a9047cd587/ Changeset: 38a9047cd587 User: dan Date: 2013-05-09 16:49:02 Summary: Unicode fixes for previewing large datasets. Affected #: 2 files diff -r 80ab774559f8405a46082286c6cf35db420db002 -r 38a9047cd5877b9123c06a92c2c53b6b488733cf lib/galaxy/datatypes/tabular.py --- a/lib/galaxy/datatypes/tabular.py +++ b/lib/galaxy/datatypes/tabular.py @@ -265,7 +265,7 @@ while cursor and ck_data[-1] != '\n': ck_data += cursor cursor = f.read(1) - return to_json_string({'ck_data': ck_data, 'ck_index': ck_index+1}) + return to_json_string( { 'ck_data': util.unicodify( ck_data ), 'ck_index': ck_index + 1 } ) def display_data(self, trans, dataset, preview=False, filename=None, to_ext=None, chunk=None): preview = util.string_as_bool( preview ) diff -r 80ab774559f8405a46082286c6cf35db420db002 -r 38a9047cd5877b9123c06a92c2c53b6b488733cf templates/webapps/galaxy/dataset/large_file.mako --- a/templates/webapps/galaxy/dataset/large_file.mako +++ b/templates/webapps/galaxy/dataset/large_file.mako @@ -10,5 +10,5 @@ </div><pre> -${ unicode( truncated_data, 'utf-8' ) | h } +${ util.unicodify( truncated_data ) | h } </pre> Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0