May 2013 - galaxy-commits - lists.galaxyproject.org

commit/galaxy-central: 3 new changesets
by commits-noreply＠bitbucket.org 15 May '13

15 May '13

3 new commits in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/582a0194b5c5/ Changeset: 582a0194b5c5 User: dannon Date: 2013-05-16 00:47:30 Summary: Library API: import cleanup (fixes incorrect util import) Affected #: 1 file diff -r a87623b5f358a570718afc07d7a714269220db61 -r 582a0194b5c55994e0f961da7b762d0a8779ce2f lib/galaxy/webapps/galaxy/api/libraries.py --- a/lib/galaxy/webapps/galaxy/api/libraries.py +++ b/lib/galaxy/webapps/galaxy/api/libraries.py @@ -1,13 +1,12 @@ """ API operations on a library. """ -import logging, os, string, shutil, urllib, re, socket -from cgi import escape, FieldStorage -from galaxy import util, datatypes, jobs, web, util -from galaxy.web.base.controller import * -from galaxy.util.sanitize_html import sanitize_html -from galaxy.model.orm import * -from paste.httpexceptions import * +import logging +from galaxy import util +from galaxy import web +from galaxy.model.orm import and_, not_, or_ +from galaxy.web.base.controller import BaseAPIController, url_for +from paste.httpexceptions import HTTPBadRequest, HTTPForbidden log = logging.getLogger( __name__ ) https://bitbucket.org/galaxy/galaxy-central/commits/f301a34b2ca6/ Changeset: f301a34b2ca6 User: dannon Date: 2013-05-16 00:47:52 Summary: Remove unused params from library.show Affected #: 1 file diff -r 582a0194b5c55994e0f961da7b762d0a8779ce2f -r f301a34b2ca6ae3c8ab842161e32a5c2d6dd458b lib/galaxy/webapps/galaxy/api/libraries.py --- a/lib/galaxy/webapps/galaxy/api/libraries.py +++ b/lib/galaxy/webapps/galaxy/api/libraries.py @@ -56,7 +56,6 @@ log.debug( "LibraryContentsController.show: enter" ) library_id = id deleted = util.string_as_bool( deleted ) - params = util.Params( kwd ) try: decoded_library_id = trans.security.decode_id( library_id ) except TypeError: https://bitbucket.org/galaxy/galaxy-central/commits/590f27390d0d/ Changeset: 590f27390d0d User: dannon Date: 2013-05-16 00:48:44 Summary: Library Contents API: import cleanup, remove import *'s. Affected #: 1 file diff -r f301a34b2ca6ae3c8ab842161e32a5c2d6dd458b -r 590f27390d0dca446fe804a99ba819663653207e lib/galaxy/webapps/galaxy/api/library_contents.py --- a/lib/galaxy/webapps/galaxy/api/library_contents.py +++ b/lib/galaxy/webapps/galaxy/api/library_contents.py @@ -1,13 +1,10 @@ """ API operations on the contents of a library. """ -import logging, os, string, shutil, urllib, re, socket -from cgi import escape, FieldStorage -from galaxy import util, datatypes, jobs, web, util -from galaxy.web.base.controller import * -from galaxy.util.sanitize_html import sanitize_html -from galaxy.model.orm import * +import logging +from galaxy import web from galaxy.model import ExtendedMetadata, ExtendedMetadataIndex +from galaxy.web.base.controller import BaseAPIController, HTTPBadRequest, url_for, UsesLibraryMixin, UsesLibraryMixinItems log = logging.getLogger( __name__ ) Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: jgoecks: Viz framework: (a) add sam-to-bigwig converter and (b) Trackster bug fixes.
by commits-noreply＠bitbucket.org 15 May '13

15 May '13

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/a87623b5f358/ Changeset: a87623b5f358 User: jgoecks Date: 2013-05-15 22:15:12 Summary: Viz framework: (a) add sam-to-bigwig converter and (b) Trackster bug fixes. Affected #: 2 files diff -r 00ecd6a50591b108d3af3a2c6fd6f9389217d363 -r a87623b5f358a570718afc07d7a714269220db61 lib/galaxy/datatypes/converters/sam_to_bigwig_converter.xml --- /dev/null +++ b/lib/galaxy/datatypes/converters/sam_to_bigwig_converter.xml @@ -0,0 +1,20 @@ +<tool id="CONVERTER_sam_to_bigwig_0" name="Convert SAM to BigWig" version="1.0.0" hidden="true"> + <command> + samtools view -bh $input | bedtools genomecov -bg -split -ibam stdin -g $chromInfo + + ## Streaming the bedgraph file to wigToBigWig is fast but very memory intensive; hence, this + ## should only be used on systems with large RAM. + ## | wigToBigWig stdin $chromInfo $output + + ## This can be used anywhere. + > temp.bg ; bedGraphToBigWig temp.bg $chromInfo $output + </command> + <inputs> + <param format="bam" name="input" type="data" label="Choose BAM file"/> + </inputs> + <outputs> + <data format="bigwig" name="output"/> + </outputs> + <help> + </help> +</tool> diff -r 00ecd6a50591b108d3af3a2c6fd6f9389217d363 -r a87623b5f358a570718afc07d7a714269220db61 static/scripts/viz/trackster/tracks.js --- a/static/scripts/viz/trackster/tracks.js +++ b/static/scripts/viz/trackster/tracks.js @@ -3026,7 +3026,7 @@ num_cols: 12, on_finish: function(new_val) { $(".bs-tooltip").remove(); - var new_val = parseFloat(new_val); + var new_val = round(parseFloat(new_val), 3); track.prefs[ pref_name ] = (!isNaN(new_val) ? new_val : null); on_change(); }, @@ -3134,14 +3134,14 @@ track.tile_cache.set_elt(key, tile_drawn); $.when.apply($, get_tile_data()).then( function() { var tile_data = get_tile_data(), - tracks_data = tile_data.slice(0, tile_data.length - 1), - seq_data = tile_data[tile_data.length - 1]; - + tracks_data = tile_data, + seq_data; + // If sequence data is available, subset to get only data in region. - if (seq_data) { - seq_data = view.reference_track.data_manager.subset_entry(seq_data, region); + if (view.reference_track) { + seq_data = view.reference_track.data_manager.subset_entry(tile_data.pop(), region); } - + // Get drawing modes, heights for all tracks. var drawing_modes = [], drawing_heights = []; @@ -3733,8 +3733,8 @@ */ postdraw_actions: function(tiles, width, w_scale, clear_after) { // Add min, max labels. - this._add_yaxis_label('max', round(this.prefs.max_value, 3), 'max_value'); - this._add_yaxis_label('min', round(this.prefs.min_value, 3), 'min_value'); + this._add_yaxis_label('max'); + this._add_yaxis_label('min'); } }); Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: 5 new changesets
by commits-noreply＠bitbucket.org 15 May '13

15 May '13

5 new commits in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/8d97d7e9e50b/ Changeset: 8d97d7e9e50b User: dan Date: 2013-05-15 22:03:48 Summary: Fix imports in lib/galaxy/webapps/reports/controllers/root.py Affected #: 1 file diff -r 2c3d59e8341e42117537a6d81760801191240a92 -r 8d97d7e9e50b5d2e0d3cbd62b9bf92d4217f562c lib/galaxy/webapps/reports/controllers/root.py --- a/lib/galaxy/webapps/reports/controllers/root.py +++ b/lib/galaxy/webapps/reports/controllers/root.py @@ -1,4 +1,4 @@ -from galaxy.web.base.controller import * +from galaxy.web.base.controller import BaseUIController, web import logging log = logging.getLogger( __name__ ) https://bitbucket.org/galaxy/galaxy-central/commits/87b4c2c7b04c/ Changeset: 87b4c2c7b04c User: dan Date: 2013-05-15 22:04:04 Summary: Fix imports in lib/galaxy/webapps/reports/controllers/sample_tracking.py Affected #: 1 file diff -r 8d97d7e9e50b5d2e0d3cbd62b9bf92d4217f562c -r 87b4c2c7b04c33fa0d89c6b9ca1d799e18df5efc lib/galaxy/webapps/reports/controllers/sample_tracking.py --- a/lib/galaxy/webapps/reports/controllers/sample_tracking.py +++ b/lib/galaxy/webapps/reports/controllers/sample_tracking.py @@ -1,10 +1,10 @@ import calendar, operator, os, socket -from datetime import * +from datetime import datetime, date, timedelta from time import mktime, strftime, localtime -from galaxy.web.base.controller import * +from galaxy.web.base.controller import BaseUIController, web from galaxy import model, util from galaxy.web.framework.helpers import time_ago, iff, grids -from galaxy.model.orm import * +from galaxy.model.orm import and_, not_, or_ import pkg_resources pkg_resources.require( "SQLAlchemy >= 0.4" ) import sqlalchemy as sa https://bitbucket.org/galaxy/galaxy-central/commits/2f18df41a411/ Changeset: 2f18df41a411 User: dan Date: 2013-05-15 22:05:38 Summary: Fix imports in lib/galaxy/webapps/reports/controllers/workflows.py Affected #: 1 file diff -r 87b4c2c7b04c33fa0d89c6b9ca1d799e18df5efc -r 2f18df41a411db2cfa876135b1e2cbeee063b73f lib/galaxy/webapps/reports/controllers/workflows.py --- a/lib/galaxy/webapps/reports/controllers/workflows.py +++ b/lib/galaxy/webapps/reports/controllers/workflows.py @@ -1,10 +1,10 @@ import calendar, operator, os, socket -from datetime import * +from datetime import datetime, date, timedelta from time import mktime, strftime, localtime -from galaxy.web.base.controller import * +from galaxy.web.base.controller import BaseUIController, web from galaxy import model, util from galaxy.web.framework.helpers import time_ago, iff, grids -from galaxy.model.orm import * +from galaxy.model.orm import and_, not_, or_ import pkg_resources pkg_resources.require( "SQLAlchemy >= 0.4" ) import sqlalchemy as sa https://bitbucket.org/galaxy/galaxy-central/commits/c75d043c9d4b/ Changeset: c75d043c9d4b User: dan Date: 2013-05-15 22:08:07 Summary: Fix imports in lib/galaxy/webapps/reports/controllers/users.py Affected #: 1 file diff -r 2f18df41a411db2cfa876135b1e2cbeee063b73f -r c75d043c9d4bcdf006f4e0b69387e0ee35e354ea lib/galaxy/webapps/reports/controllers/users.py --- a/lib/galaxy/webapps/reports/controllers/users.py +++ b/lib/galaxy/webapps/reports/controllers/users.py @@ -1,9 +1,10 @@ -from datetime import * +from datetime import datetime, date, timedelta from time import strftime import calendar, operator -from galaxy.web.base.controller import * +from galaxy.web.base.controller import BaseUIController, web import galaxy.model -from galaxy.model.orm import * +from galaxy import util +from galaxy.model.orm import and_, not_, or_ import pkg_resources pkg_resources.require( "SQLAlchemy >= 0.4" ) import sqlalchemy as sa https://bitbucket.org/galaxy/galaxy-central/commits/00ecd6a50591/ Changeset: 00ecd6a50591 User: dan Date: 2013-05-15 22:10:07 Summary: Fix imports in lib/galaxy/webapps/reports/controllers/system.py Affected #: 1 file diff -r c75d043c9d4bcdf006f4e0b69387e0ee35e354ea -r 00ecd6a50591b108d3af3a2c6fd6f9389217d363 lib/galaxy/webapps/reports/controllers/system.py --- a/lib/galaxy/webapps/reports/controllers/system.py +++ b/lib/galaxy/webapps/reports/controllers/system.py @@ -1,8 +1,8 @@ import operator, os from datetime import datetime, timedelta -from galaxy.web.base.controller import * -from galaxy import model -from galaxy.model.orm import * +from galaxy.web.base.controller import BaseUIController, web +from galaxy import model, util +from galaxy.model.orm import and_, not_, or_, desc import logging log = logging.getLogger( __name__ ) Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: dan: Fix import * in webapps/reports/controllers/jobs.py. Was causing util from galaxy to be overwritten with util from galaxy.model.orm.
by commits-noreply＠bitbucket.org 15 May '13

15 May '13

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/2c3d59e8341e/ Changeset: 2c3d59e8341e User: dan Date: 2013-05-15 21:59:44 Summary: Fix import * in webapps/reports/controllers/jobs.py. Was causing util from galaxy to be overwritten with util from galaxy.model.orm. Affected #: 1 file diff -r b8c951a4e56b7e0cdc8cd176dd96521c2e0eb764 -r 2c3d59e8341e42117537a6d81760801191240a92 lib/galaxy/webapps/reports/controllers/jobs.py --- a/lib/galaxy/webapps/reports/controllers/jobs.py +++ b/lib/galaxy/webapps/reports/controllers/jobs.py @@ -1,10 +1,10 @@ import calendar, operator, os, socket -from datetime import * +from datetime import datetime, date, timedelta from time import mktime, strftime, localtime -from galaxy.web.base.controller import * +from galaxy.web.base.controller import BaseUIController, web from galaxy import model, util from galaxy.web.framework.helpers import time_ago, iff, grids -from galaxy.model.orm import * +from galaxy.model.orm import and_, not_, or_ import pkg_resources pkg_resources.require( "SQLAlchemy >= 0.4" ) import sqlalchemy as sa Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: dan: Add missing message_box_* config elements to Reports app. Broken in cb0432cfcc8a.
by commits-noreply＠bitbucket.org 15 May '13

15 May '13

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/b8c951a4e56b/ Changeset: b8c951a4e56b User: dan Date: 2013-05-15 21:44:58 Summary: Add missing message_box_* config elements to Reports app. Broken in cb0432cfcc8a. Affected #: 1 file diff -r 52b6923d713b0c205cdc8c970e49f4b606710056 -r b8c951a4e56b7e0cdc8cd176dd96521c2e0eb764 lib/galaxy/webapps/reports/config.py --- a/lib/galaxy/webapps/reports/config.py +++ b/lib/galaxy/webapps/reports/config.py @@ -35,6 +35,10 @@ self.allow_user_deletion = string_as_bool( kwargs.get( "allow_user_deletion", "False" ) ) self.log_actions = string_as_bool( kwargs.get( 'log_actions', 'False' ) ) self.brand = kwargs.get( 'brand', None ) + # Configuration for the message box directly below the masthead. + self.message_box_visible = kwargs.get( 'message_box_visible', False ) + self.message_box_content = kwargs.get( 'message_box_content', None ) + self.message_box_class = kwargs.get( 'message_box_class', 'info' ) self.wiki_url = kwargs.get( 'wiki_url', 'http://wiki.g2.bx.psu.edu/FrontPage' ) self.blog_url = kwargs.get( 'blog_url', None ) self.screencasts_url = kwargs.get( 'screencasts_url', None ) Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: Dave Bouvier: Fix incorrect setting of the do_not_test flag when a repository has installation errors.
by commits-noreply＠bitbucket.org 15 May '13

15 May '13

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/52b6923d713b/ Changeset: 52b6923d713b User: Dave Bouvier Date: 2013-05-15 21:25:48 Summary: Fix incorrect setting of the do_not_test flag when a repository has installation errors. Affected #: 1 file diff -r 89b5f6866eeb2fcb955d4e773d5b5328dd214610 -r 52b6923d713b0c205cdc8c970e49f4b606710056 test/install_and_test_tool_shed_repositories/functional_tests.py --- a/test/install_and_test_tool_shed_repositories/functional_tests.py +++ b/test/install_and_test_tool_shed_repositories/functional_tests.py @@ -790,9 +790,8 @@ error_message=dependency.error_message ) repository_status[ 'installation_errors' ][ 'repository_dependencies' ].append( test_result ) # Record the status of this repository in the tool shed. - set_do_not_test = not is_latest_downloadable_revision( galaxy_tool_shed_url, repository_info_dict ) params[ 'tools_functionally_correct' ] = 'false' - params[ 'do_not_test' ] = str( set_do_not_test ) + params[ 'do_not_test' ] = 'false' params[ 'test_install_error' ] = 'true' register_test_result( galaxy_tool_shed_url, metadata_revision_id, @@ -925,10 +924,9 @@ changeset_revision=repository.changeset_revision, error_message=repository.error_message ) repository_status[ 'installation_errors' ][ 'repository_dependencies' ].append( test_result ) - set_do_not_test = not is_latest_downloadable_revision( galaxy_tool_shed_url, repository_info_dict ) params[ 'tools_functionally_correct' ] = 'false' params[ 'test_install_error' ] = 'true' - params[ 'do_not_test' ] = str( set_do_not_test ) + params[ 'do_not_test' ] = 'false' register_test_result( galaxy_tool_shed_url, metadata_revision_id, repository_status, Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: Dave Bouvier: Enhance the install and test functional testing framework to:
by commits-noreply＠bitbucket.org 15 May '13

15 May '13

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/89b5f6866eeb/ Changeset: 89b5f6866eeb User: Dave Bouvier Date: 2013-05-15 21:14:55 Summary: Enhance the install and test functional testing framework to: Skip any repository revision that has been flagged by the owner not to be tested. Set the appropriate flags for failed tests, installations, and missing components. Limit the flagging to revisions that are not the most recent. Affected #: 1 file diff -r 0005fa6a989143068c626cc63103aba910ba22f8 -r 89b5f6866eeb2fcb955d4e773d5b5328dd214610 test/install_and_test_tool_shed_repositories/functional_tests.py --- a/test/install_and_test_tool_shed_repositories/functional_tests.py +++ b/test/install_and_test_tool_shed_repositories/functional_tests.py @@ -253,7 +253,8 @@ params = urllib.urlencode( dict( do_not_test='false', downloadable='true', malicious='false', - includes_tools='true' ) ) + includes_tools='true', + skip_tool_test='false' ) ) api_url = get_api_url( base=location, parts=[ 'repository_revisions' ], params=params ) if format == 'json': return json_from_url( api_url ) @@ -284,23 +285,32 @@ tool_test_results = repository_metadata.get( 'tool_test_errors', {} ) return tool_test_results +def is_latest_downloadable_revision( url, repository_info_dict ): + api_url_parts = [ 'api', 'repositories', 'get_ordered_installable_revisions' ] + params = urllib.urlencode( dict( name=repository_info_dict[ 'name' ], owner=repository_info_dict[ 'owner' ] ) ) + api_url = get_api_url( url, api_url_parts, params ) + # TODO: Remove this try/catch wrapping when the main tool shed is updated. + try: + changeset_revisions = json_from_url( api_url ) + except Exception, e: + return True + latest_revision = changeset_revisions.pop() + return str( repository_info_dict[ 'changeset_revision' ] ) == str( latest_revision ) + def json_from_url( url ): url_handle = urllib.urlopen( url ) url_contents = url_handle.read() - return from_json_string( url_contents ) + try: + parsed_json = from_json_string( url_contents ) + except: + log.exception( 'Error parsing JSON data.' ) + raise + return parsed_json -def register_test_result( url, metadata_id, test_results_dict, passed_tests=False ): +def register_test_result( url, metadata_id, test_results_dict, repository_info_dict, params ): ''' - This script should never set do_not_test = True, because the repositories should always be re-tested - against the most recent code. + Update the repository metadata tool_test_results and appropriate flags using the API. ''' - params = {} - if passed_tests: - params[ 'tools_functionally_correct' ] = 'true' - params[ 'do_not_test' ] = 'false' - else: - params[ 'tools_functionally_correct' ] = 'false' - params[ 'do_not_test' ] = 'false' params[ 'tool_test_results' ] = test_results_dict # BEGIN compatibility code. # TODO: The repository_revisions API controller ignores any received parameter that is not present in the database schema, @@ -602,6 +612,7 @@ } """ repository_status = dict() + params = dict() repository_id = repository_info_dict.get( 'repository_id', None ) changeset_revision = repository_info_dict.get( 'changeset_revision', None ) metadata_revision_id = repository_info_dict.get( 'id', None ) @@ -677,21 +688,50 @@ # }, # ] # "installation_errors": - # [ + # { + # 'tool_dependencies': + # [ + # { + # 'type': 'Type of tool dependency, e.g. package, set_environment, etc.', + # 'name': 'Name of the tool dependency.', + # 'version': 'Version if this is a package, otherwise blank.', + # 'error_message': 'The error message returned when installation was attempted.', + # }, + # ], + # 'repository_dependencies': + # [ + # { + # 'tool_shed': 'The tool shed that this repository was installed from.', + # 'name': 'The name of the repository that failed to install.', + # 'owner': 'Owner of the failed repository.', + # 'changeset_revision': 'Changeset revision of the failed repository.', + # 'error_message': 'The error message that was returned when the repository failed to install.', + # }, + # ], + # 'current_repository': + # [ + # { + # 'tool_shed': 'The tool shed that this repository was installed from.', + # 'name': 'The name of the repository that failed to install.', + # 'owner': 'Owner of the failed repository.', + # 'changeset_revision': 'Changeset revision of the failed repository.', + # 'error_message': 'The error message that was returned when the repository failed to install.', + # }, + # ], # { # "name": "The name of the repository.", # "owner": "The owner of the repository.", # "changeset_revision": "The changeset revision of the repository.", # "error_message": "The message stored in tool_dependency.error_message." # }, - # ] + # } # "missing_test_components": # [ # { - # "tool_id": "The tool ID that does not have functional tests defined.", + # "tool_id": "The tool ID that missing components.", # "tool_version": "The version of the tool." # "tool_guid": "The guid of the tool." - # "missing_components": "A short explanation of what is invalid. + # "missing_components": "Which components are missing, e.g. the test data filename, or the test-data directory." # }, # ] # } @@ -714,7 +754,15 @@ if missing_components not in repository_status[ 'missing_test_components' ]: repository_status[ 'missing_test_components' ].append( missing_components ) # Record the status of this repository in the tool shed. - register_test_result( galaxy_tool_shed_url, metadata_revision_id, repository_status, passed_tests=False ) + set_do_not_test = not is_latest_downloadable_revision( galaxy_tool_shed_url, repository_info_dict ) + params[ 'tools_functionally_correct' ] = 'false' + params[ 'missing_test_components' ] = 'true' + params[ 'do_not_test' ] = str( set_do_not_test ) + register_test_result( galaxy_tool_shed_url, + metadata_revision_id, + repository_status, + repository_info_dict, + params ) # Run the cleanup method. This removes tool functional test methods from the test_toolbox module and uninstalls the # repository using Twill. execute_uninstall_method( app ) @@ -742,13 +790,21 @@ error_message=dependency.error_message ) repository_status[ 'installation_errors' ][ 'repository_dependencies' ].append( test_result ) # Record the status of this repository in the tool shed. - register_test_result( galaxy_tool_shed_url, metadata_revision_id, repository_status, passed_tests=False ) + set_do_not_test = not is_latest_downloadable_revision( galaxy_tool_shed_url, repository_info_dict ) + params[ 'tools_functionally_correct' ] = 'false' + params[ 'do_not_test' ] = str( set_do_not_test ) + params[ 'test_install_error' ] = 'true' + register_test_result( galaxy_tool_shed_url, + metadata_revision_id, + repository_status, + repository_info_dict, + params ) # Run the cleanup method. This removes tool functional test methods from the test_toolbox module and uninstalls the # repository using Twill. execute_uninstall_method( app ) # Set the test_toolbox.toolbox module-level variable to the new app.toolbox. test_toolbox.toolbox = app.toolbox - repositories_failed.append( dict( name=name, owner=owner, changeset_revision=changeset_revision ) ) + repositories_failed_install.append( dict( name=name, owner=owner, changeset_revision=changeset_revision ) ) else: # If the repository does have a test-data directory, we write the generated shed_tools_dict to a file, so the functional # test framework can find it. @@ -788,7 +844,13 @@ # controller with the status of the test. This also sets the do_not_test and tools_functionally correct flags, and # updates the time_last_tested field to today's date. repositories_passed.append( dict( name=name, owner=owner, changeset_revision=changeset_revision ) ) - register_test_result( galaxy_tool_shed_url, metadata_revision_id, repository_status, passed_tests=True ) + params[ 'tools_functionally_correct' ] = 'true' + params[ 'do_not_test' ] = 'false' + register_test_result( galaxy_tool_shed_url, + metadata_revision_id, + repository_status, + repository_info_dict, + params ) log.debug( 'Revision %s of repository %s installed and passed functional tests.', changeset_revision, name ) else: # If the functional tests fail, log the output and update the failed changeset revision's metadata record in the tool shed via the API. @@ -833,7 +895,14 @@ # This also sets the do_not_test and tools_functionally correct flags to the appropriate values, and updates the time_last_tested # field to today's date. repositories_failed.append( dict( name=name, owner=owner, changeset_revision=changeset_revision ) ) - register_test_result( galaxy_tool_shed_url, metadata_revision_id, repository_status, passed_tests=False ) + set_do_not_test = not is_latest_downloadable_revision( galaxy_tool_shed_url, repository_info_dict ) + params[ 'tools_functionally_correct' ] = 'false' + params[ 'do_not_test' ] = str( set_do_not_test ) + register_test_result( galaxy_tool_shed_url, + metadata_revision_id, + repository_status, + repository_info_dict, + params ) log.debug( 'Revision %s of repository %s installed successfully, but did not pass functional tests.', changeset_revision, name ) # Run the uninstall method. This removes tool functional test methods from the test_toolbox module and uninstalls the @@ -856,7 +925,15 @@ changeset_revision=repository.changeset_revision, error_message=repository.error_message ) repository_status[ 'installation_errors' ][ 'repository_dependencies' ].append( test_result ) - register_test_result( galaxy_tool_shed_url, metadata_revision_id, repository_status, passed_tests=False ) + set_do_not_test = not is_latest_downloadable_revision( galaxy_tool_shed_url, repository_info_dict ) + params[ 'tools_functionally_correct' ] = 'false' + params[ 'test_install_error' ] = 'true' + params[ 'do_not_test' ] = str( set_do_not_test ) + register_test_result( galaxy_tool_shed_url, + metadata_revision_id, + repository_status, + repository_info_dict, + params ) success = execute_uninstall_method( app ) if not success: log.error( 'Repository %s failed to uninstall.', repository_info_dict[ 'name' ] ) @@ -899,17 +976,17 @@ if repositories_tested > 0: if repositories_passed: print '# ----------------------------------------------------------------------------------' - print "# %d repositories passed:" % len( repositories_passed ) + print "# %d repositories passed all tests:" % len( repositories_passed ) show_summary_output( repositories_passed ) if repositories_failed: print '# ----------------------------------------------------------------------------------' - print "# %d repositories failed:" % len( repositories_failed ) + print "# %d repositories failed one or more tests:" % len( repositories_failed ) show_summary_output( repositories_failed ) if repositories_failed_install: # Set success to False so that the return code will not be 0. success = False print '# ----------------------------------------------------------------------------------' - print "# %d repositories not installed correctly:" % len( repositories_failed_install ) + print "# %d repositories with installation errors:" % len( repositories_failed_install ) show_summary_output( repositories_failed_install ) else: success = True Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: Dave Bouvier: Add option to filter by repository installation error status in the repository metadata API controller.
by commits-noreply＠bitbucket.org 15 May '13

15 May '13

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/0005fa6a9891/ Changeset: 0005fa6a9891 User: Dave Bouvier Date: 2013-05-15 21:00:15 Summary: Add option to filter by repository installation error status in the repository metadata API controller. Affected #: 1 file diff -r 31616ab57a240c155ac2d5e1272bba28731552be -r 0005fa6a989143068c626cc63103aba910ba22f8 lib/galaxy/webapps/tool_shed/api/repository_revisions.py --- a/lib/galaxy/webapps/tool_shed/api/repository_revisions.py +++ b/lib/galaxy/webapps/tool_shed/api/repository_revisions.py @@ -54,6 +54,10 @@ includes_tools = kwd.get( 'includes_tools', None ) if includes_tools is not None: clause_list.append( trans.model.RepositoryMetadata.table.c.includes_tools == util.string_as_bool( includes_tools ) ) + # Filter by test_install_error if received. + test_install_error = kwd.get( 'test_install_error', None ) + if test_install_error is not None: + clause_list.append( trans.model.RepositoryMetadata.table.c.test_install_error == util.string_as_bool( test_install_error ) ) # Filter by skip_tool_test if received. skip_tool_test = kwd.get( 'skip_tool_test', None ) if skip_tool_test is not None: Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: dan: Unicode fixes for toolshed repositories.
by commits-noreply＠bitbucket.org 15 May '13

15 May '13

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/31616ab57a24/ Changeset: 31616ab57a24 User: dan Date: 2013-05-15 20:50:19 Summary: Unicode fixes for toolshed repositories. Affected #: 3 files diff -r a3e1fb22f5b5afcb776399d8a499e62c795f8279 -r 31616ab57a240c155ac2d5e1272bba28731552be lib/tool_shed/util/shed_util_common.py --- a/lib/tool_shed/util/shed_util_common.py +++ b/lib/tool_shed/util/shed_util_common.py @@ -690,7 +690,7 @@ contents = [] for item in os.listdir( folder_path ): # Skip .hg directories - if str( item ).startswith( '.hg' ): + if item.startswith( '.hg' ): continue if os.path.isdir( os.path.join( folder_path, item ) ): # Append a '/' character so that our jquery dynatree will function properly. diff -r a3e1fb22f5b5afcb776399d8a499e62c795f8279 -r 31616ab57a240c155ac2d5e1272bba28731552be templates/webapps/tool_shed/repository/view_changelog.mako --- a/templates/webapps/tool_shed/repository/view_changelog.mako +++ b/templates/webapps/tool_shed/repository/view_changelog.mako @@ -88,7 +88,7 @@ %endif <div class="form-row"><label>Description:</label> - <a href="${h.url_for( controller='repository', action='view_changeset', id=trans.security.encode_id( repository.id ), ctx_str=ctx_str )}">${changeset[ 'description' ] | h}</a> + <a href="${h.url_for( controller='repository', action='view_changeset', id=trans.security.encode_id( repository.id ), ctx_str=ctx_str )}">${ util.unicodify( changeset[ 'description' ] ) | h}</a></div><div class="form-row"><label>Commit:</label> diff -r a3e1fb22f5b5afcb776399d8a499e62c795f8279 -r 31616ab57a240c155ac2d5e1272bba28731552be templates/webapps/tool_shed/repository/view_changeset.mako --- a/templates/webapps/tool_shed/repository/view_changeset.mako +++ b/templates/webapps/tool_shed/repository/view_changeset.mako @@ -74,7 +74,7 @@ <tr><td><b>Commit message:</b> - <br/>${ctx.description() | h}<br/> + <br/>${ util.unicodify( ctx.description() ) | h}<br/></td></tr> %if modified: @@ -82,7 +82,7 @@ <td><b>modified:</b> %for item in modified: - <br/><a href="#${item}">${item | h}</a> + <br/><a href="#${ util.unicodify( item ) }">${ util.unicodify( item ) | h}</a> %endfor </td></tr> @@ -92,7 +92,7 @@ <td><b>added:</b> %for item in added: - <br/><a href="#${item}">${item | h}</a> + <br/><a href="#${ util.unicodify( item ) }">${ util.unicodify( item ) | h}</a> %endfor </td></tr> @@ -102,7 +102,7 @@ <td><b>removed:</b> %for item in removed: - <br/><a href="#${item}">${item | h}</a> + <br/><a href="#${ util.unicodify( item ) }">${ util.unicodify( item ) | h}</a> %endfor </td></tr> @@ -112,7 +112,7 @@ <td><b>deleted:</b> %for item in deleted: - <br/><a href="#${item}">${item | h}</a> + <br/><a href="#${ util.unicodify( item ) }">${ util.unicodify( item ) | h}</a> %endfor </td></tr> @@ -122,7 +122,7 @@ <td><b>unknown:</b> %for item in unknown: - <br/><a href="#${item}">${item | h}</a> + <br/><a href="#${ util.unicodify( item ) }">${ util.unicodify( item ) | h}</a> %endfor }</td></tr> @@ -132,7 +132,7 @@ <td><b>ignored:</b> %for item in ignored: - <br/><a href="#${item}">${item | h}</a> + <br/><a href="#${ util.unicodify( item ) }">${ util.unicodify( item ) | h}</a> %endfor </td></tr> @@ -142,7 +142,7 @@ <td> clean: %for item in clean: - <br/><a href="#${item}">${item | h}</a> + <br/><a href="#${ util.unicodify( item ) }">${ util.unicodify( item ) | h}</a> %endfor </td></tr> @@ -162,8 +162,8 @@ break ctr += 1 %> - <tr><td bgcolor="#E0E0E0">${anchor_str}</td></tr> - <tr><td>${diff}</td></tr> + <tr><td bgcolor="#E0E0E0">${ util.unicodify( anchor_str ) }</td></tr> + <tr><td>${ util.unicodify( diff ) }</td></tr> %endfor </table></div> Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: greg: Add the ability for a tool shed repository owner to set a flag to skip testing a specified repository revision. Add a new test_install_error column to the repository_metadata table, and add additional filtered list of the latest revision of repositories using these new filters.
by commits-noreply＠bitbucket.org 15 May '13

15 May '13

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/a3e1fb22f5b5/ Changeset: a3e1fb22f5b5 User: greg Date: 2013-05-15 20:03:43 Summary: Add the ability for a tool shed repository owner to set a flag to skip testing a specified repository revision. Add a new test_install_error column to the repository_metadata table, and add additional filtered list of the latest revision of repositories using these new filters. Affected #: 9 files diff -r 036f57f93b23a62205b8541b6fd773561879827e -r a3e1fb22f5b5afcb776399d8a499e62c795f8279 lib/galaxy/webapps/tool_shed/api/repository_revisions.py --- a/lib/galaxy/webapps/tool_shed/api/repository_revisions.py +++ b/lib/galaxy/webapps/tool_shed/api/repository_revisions.py @@ -54,11 +54,25 @@ includes_tools = kwd.get( 'includes_tools', None ) if includes_tools is not None: clause_list.append( trans.model.RepositoryMetadata.table.c.includes_tools == util.string_as_bool( includes_tools ) ) + # Filter by skip_tool_test if received. + skip_tool_test = kwd.get( 'skip_tool_test', None ) + if skip_tool_test is not None: + skip_tool_test = util.string_as_bool( skip_tool_test ) + # Generate and execute the query. try: - query = trans.sa_session.query( trans.app.model.RepositoryMetadata ) \ - .filter( and_( *clause_list ) ) \ - .order_by( trans.app.model.RepositoryMetadata.table.c.repository_id ) \ - .all() + if skip_tool_test: + # The skip_tool_test filter was received as True. + query = trans.sa_session.query( trans.app.model.RepositoryMetadata ) \ + .join( trans.model.SkipToolTest ) \ + .filter( and_( *clause_list ) ) \ + .order_by( trans.app.model.RepositoryMetadata.table.c.repository_id ) \ + .all() + else: + # The skip_tool_test filter was not received or it was received as False. + query = trans.sa_session.query( trans.app.model.RepositoryMetadata ) \ + .filter( and_( *clause_list ) ) \ + .order_by( trans.app.model.RepositoryMetadata.table.c.repository_id ) \ + .all() for repository_metadata in query: repository_metadata_dict = repository_metadata.get_api_value( view='collection', value_mapper=default_value_mapper( trans, repository_metadata ) ) diff -r 036f57f93b23a62205b8541b6fd773561879827e -r a3e1fb22f5b5afcb776399d8a499e62c795f8279 lib/galaxy/webapps/tool_shed/controllers/repository.py --- a/lib/galaxy/webapps/tool_shed/controllers/repository.py +++ b/lib/galaxy/webapps/tool_shed/controllers/repository.py @@ -64,6 +64,8 @@ my_writable_repositories_with_failing_tool_tests_grid = repository_grids.MyWritableRepositoriesWithFailingToolTestsGrid() my_writable_repositories_with_invalid_tools_grid = repository_grids.MyWritableRepositoriesWithInvalidToolsGrid() my_writable_repositories_with_no_failing_tool_tests_grid = repository_grids.MyWritableRepositoriesWithNoFailingToolTestsGrid() + my_writable_repositories_with_skip_tests_checked_grid = repository_grids.MyWritableRepositoriesWithSkipTestsCheckedGrid() + my_writable_repositories_with_test_install_errors_grid = repository_grids.MyWritableRepositoriesWithTestInstallErrorsGrid() repositories_by_user_grid = repository_grids.RepositoriesByUserGrid() repositories_i_own_grid = repository_grids.RepositoriesIOwnGrid() repositories_in_category_grid = repository_grids.RepositoriesInCategoryGrid() @@ -71,6 +73,8 @@ repositories_with_failing_tool_tests_grid = repository_grids.RepositoriesWithFailingToolTestsGrid() repositories_with_invalid_tools_grid = repository_grids.RepositoriesWithInvalidToolsGrid() repositories_with_no_failing_tool_tests_grid = repository_grids.RepositoriesWithNoFailingToolTestsGrid() + repositories_with_skip_tests_checked_grid = repository_grids.RepositoriesWithSkipTestsCheckedGrid() + repositories_with_test_install_errors_grid = repository_grids.RepositoriesWithTestInstallErrorsGrid() repository_dependencies_grid = repository_grids.RepositoryDependenciesGrid() repository_grid = repository_grids.RepositoryGrid() # The repository_metadata_grid is not currently displayed, but is sub-classed by several grids. @@ -222,6 +226,7 @@ message += '<li>you are authorized to update them</li>' message += '<li>the latest installable revision contains at least 1 tool</li>' message += '<li>the latest installable revision is not missing any tool test components</li>' + message += '<li>the latest installable revision has no installation errors</li>' message += '<li>the latest installable revision has at least 1 tool test that fails</li>' message += '</ul>' kwd[ 'message' ] = message @@ -286,6 +291,63 @@ return self.my_writable_repositories_with_no_failing_tool_tests_grid( trans, **kwd ) @web.expose + def browse_my_writable_repositories_with_install_errors( self, trans, **kwd ): + if 'operation' in kwd: + operation = kwd[ 'operation' ].lower() + if operation == "view_or_manage_repository": + return trans.response.send_redirect( web.url_for( controller='repository', + action='view_or_manage_repository', + **kwd ) ) + elif operation == "repositories_by_user": + return trans.response.send_redirect( web.url_for( controller='repository', + action='browse_repositories_by_user', + **kwd ) ) + elif operation in [ 'mark as deprecated', 'mark as not deprecated' ]: + kwd[ 'mark_deprecated' ] = operation == 'mark as deprecated' + return trans.response.send_redirect( web.url_for( controller='repository', + action='deprecate', + **kwd ) ) + if 'message' not in kwd: + message = 'This list contains repositories that match the following criteria:<br>' + message += '<ul>' + message += '<li>you are authorized to update them</li>' + message += '<li>the latest installable revision contains at least 1 tool</li>' + message += '<li>the latest installable revision is not missing any tool test components</li>' + message += '<li>the latest installable revision has installation errors (the repository itself, repository dependencies or tool dependencies)</li>' + message += '</ul>' + kwd[ 'message' ] = message + kwd[ 'status' ] = 'warning' + return self.my_writable_repositories_with_test_install_errors_grid( trans, **kwd ) + + @web.expose + def browse_my_writable_repositories_with_skip_tool_test_checked( self, trans, **kwd ): + if 'operation' in kwd: + operation = kwd[ 'operation' ].lower() + if operation == "view_or_manage_repository": + return trans.response.send_redirect( web.url_for( controller='repository', + action='view_or_manage_repository', + **kwd ) ) + elif operation == "repositories_by_user": + return trans.response.send_redirect( web.url_for( controller='repository', + action='browse_repositories_by_user', + **kwd ) ) + elif operation in [ 'mark as deprecated', 'mark as not deprecated' ]: + kwd[ 'mark_deprecated' ] = operation == 'mark as deprecated' + return trans.response.send_redirect( web.url_for( controller='repository', + action='deprecate', + **kwd ) ) + if 'message' not in kwd: + message = 'This list contains repositories that match the following criteria:<br>' + message += '<ul>' + message += '<li>you are authorized to update them</li>' + message += '<li>the latest installable revision contains at least 1 tool</li>' + message += '<li>the latest installable revision has <b>Skip tool tests</b> checked</li>' + message += '</ul>' + kwd[ 'message' ] = message + kwd[ 'status' ] = 'warning' + return self.my_writable_repositories_with_skip_tests_checked_grid( trans, **kwd ) + + @web.expose def browse_repositories( self, trans, **kwd ): # We add params to the keyword dict in this method in order to rename the param with an "f-" prefix, simulating filtering by clicking a search # link. We have to take this approach because the "-" character is illegal in HTTP requests. @@ -484,6 +546,34 @@ return self.repositories_with_failing_tool_tests_grid( trans, **kwd ) @web.expose + def browse_repositories_with_install_errors( self, trans, **kwd ): + if 'operation' in kwd: + operation = kwd[ 'operation' ].lower() + if operation == "view_or_manage_repository": + return trans.response.send_redirect( web.url_for( controller='repository', + action='view_or_manage_repository', + **kwd ) ) + elif operation == "repositories_by_user": + return trans.response.send_redirect( web.url_for( controller='repository', + action='browse_repositories_by_user', + **kwd ) ) + elif operation in [ 'mark as deprecated', 'mark as not deprecated' ]: + kwd[ 'mark_deprecated' ] = operation == 'mark as deprecated' + return trans.response.send_redirect( web.url_for( controller='repository', + action='deprecate', + **kwd ) ) + if 'message' not in kwd: + message = 'This list contains repositories that match the following criteria:<br>' + message += '<ul>' + message += '<li>the latest installable revision contains at least 1 tool</li>' + message += '<li>the latest installable revision is not missing any tool test components</li>' + message += '<li>the latest installable revision has installation errors (the repository itself, repository dependencies or tool dependencies)</li>' + message += '</ul>' + kwd[ 'message' ] = message + kwd[ 'status' ] = 'warning' + return self.repositories_with_test_install_errors_grid( trans, **kwd ) + + @web.expose def browse_repositories_with_invalid_tools( self, trans, **kwd ): if 'operation' in kwd: operation = kwd[ 'operation' ].lower() @@ -539,6 +629,33 @@ return self.repositories_with_no_failing_tool_tests_grid( trans, **kwd ) @web.expose + def browse_repositories_with_skip_tool_test_checked( self, trans, **kwd ): + if 'operation' in kwd: + operation = kwd[ 'operation' ].lower() + if operation == "view_or_manage_repository": + return trans.response.send_redirect( web.url_for( controller='repository', + action='view_or_manage_repository', + **kwd ) ) + elif operation == "repositories_by_user": + return trans.response.send_redirect( web.url_for( controller='repository', + action='browse_repositories_by_user', + **kwd ) ) + elif operation in [ 'mark as deprecated', 'mark as not deprecated' ]: + kwd[ 'mark_deprecated' ] = operation == 'mark as deprecated' + return trans.response.send_redirect( web.url_for( controller='repository', + action='deprecate', + **kwd ) ) + if 'message' not in kwd: + message = 'This list contains repositories that match the following criteria:<br>' + message += '<ul>' + message += '<li>the latest installable revision contains at least 1 tool</li>' + message += '<li>the latest installable revision has <b>Skip tool tests</b> checked</li>' + message += '</ul>' + kwd[ 'message' ] = message + kwd[ 'status' ] = 'warning' + return self.repositories_with_skip_tests_checked_grid( trans, **kwd ) + + @web.expose def browse_repository( self, trans, id, **kwd ): params = util.Params( kwd ) message = util.restore_text( params.get( 'message', '' ) ) diff -r 036f57f93b23a62205b8541b6fd773561879827e -r a3e1fb22f5b5afcb776399d8a499e62c795f8279 lib/galaxy/webapps/tool_shed/model/__init__.py --- a/lib/galaxy/webapps/tool_shed/model/__init__.py +++ b/lib/galaxy/webapps/tool_shed/model/__init__.py @@ -219,13 +219,14 @@ class RepositoryMetadata( object, APIItem ): api_collection_visible_keys = ( 'id', 'repository_id', 'changeset_revision', 'malicious', 'downloadable', 'has_repository_dependencies', 'includes_datatypes', 'includes_tools', 'includes_tool_dependencies', 'includes_tools_for_display_in_tool_panel', 'includes_workflows' ) - api_element_visible_keys = ( 'id', 'repository_id', 'changeset_revision', 'malicious', 'downloadable', 'tools_functionally_correct', - 'do_not_test', 'time_last_tested', 'tool_test_results', 'has_repository_dependencies', 'includes_datatypes', 'includes_tools', - 'includes_tool_dependencies', 'includes_tools_for_display_in_tool_panel', 'includes_workflows' ) + api_element_visible_keys = ( 'id', 'repository_id', 'changeset_revision', 'malicious', 'downloadable', 'tools_functionally_correct', 'do_not_test', + 'test_install_error', 'time_last_tested', 'tool_test_results', 'has_repository_dependencies', 'includes_datatypes', + 'includes_tools', 'includes_tool_dependencies', 'includes_tools_for_display_in_tool_panel', 'includes_workflows' ) def __init__( self, id=None, repository_id=None, changeset_revision=None, metadata=None, tool_versions=None, malicious=False, downloadable=False, - missing_test_components=None, tools_functionally_correct=False, do_not_test=False, time_last_tested=None, tool_test_results=None, - has_repository_dependencies=False, includes_datatypes=False, includes_tools=False, includes_tool_dependencies=False, includes_workflows=False ): + missing_test_components=None, tools_functionally_correct=False, do_not_test=False, test_install_error=False, time_last_tested=None, + tool_test_results=None, has_repository_dependencies=False, includes_datatypes=False, includes_tools=False, includes_tool_dependencies=False, + includes_workflows=False ): self.id = id self.repository_id = repository_id self.changeset_revision = changeset_revision @@ -236,6 +237,7 @@ self.missing_test_components = missing_test_components self.tools_functionally_correct = tools_functionally_correct self.do_not_test = do_not_test + self.test_install_error = test_install_error self.time_last_tested = time_last_tested self.tool_test_results = tool_test_results self.has_repository_dependencies = has_repository_dependencies @@ -274,6 +276,37 @@ return rval +class SkipToolTest( object, APIItem ): + api_collection_visible_keys = ( 'id', 'repository_metadata_id', 'initial_changeset_revision' ) + api_element_visible_keys = ( 'id', 'repository_metadata_id', 'initial_changeset_revision', 'comment' ) + + def __init__( self, id=None, repository_metadata_id=None, initial_changeset_revision=None, comment=None ): + self.id = id + self.repository_metadata_id = repository_metadata_id + self.initial_changeset_revision = initial_changeset_revision + self.comment = comment + + def as_dict( self, value_mapper=None ): + return self.get_api_value( view='element', value_mapper=value_mapper ) + + def get_api_value( self, view='collection', value_mapper=None ): + if value_mapper is None: + value_mapper = {} + rval = {} + try: + visible_keys = self.__getattribute__( 'api_' + view + '_visible_keys' ) + except AttributeError: + raise Exception( 'Unknown API view: %s' % view ) + for key in visible_keys: + try: + rval[ key ] = self.__getattribute__( key ) + if key in value_mapper: + rval[ key ] = value_mapper.get( key, rval[ key ] ) + except AttributeError: + rval[ key ] = None + return rval + + class RepositoryReview( object, APIItem ): api_collection_visible_keys = ( 'id', 'repository_id', 'changeset_revision', 'user_id', 'rating', 'deleted' ) api_element_visible_keys = ( 'id', 'repository_id', 'changeset_revision', 'user_id', 'rating', 'deleted' ) diff -r 036f57f93b23a62205b8541b6fd773561879827e -r a3e1fb22f5b5afcb776399d8a499e62c795f8279 lib/galaxy/webapps/tool_shed/model/mapping.py --- a/lib/galaxy/webapps/tool_shed/model/mapping.py +++ b/lib/galaxy/webapps/tool_shed/model/mapping.py @@ -132,6 +132,7 @@ Column( "downloadable", Boolean, default=True ), Column( "tools_functionally_correct", Boolean, default=False, index=True ), Column( "do_not_test", Boolean, default=False, index=True ), + Column( "test_install_error", Boolean, default=False, index=True ), Column( "time_last_tested", DateTime, default=None, nullable=True ), Column( "missing_test_components", Boolean, default=False, index=True ), Column( "tool_test_results", JSONType, nullable=True ), @@ -141,6 +142,14 @@ Column( "includes_tool_dependencies", Boolean, default=False, index=True ), Column( "includes_workflows", Boolean, default=False, index=True ) ) +SkipToolTest.table = Table( "skip_tool_test", metadata, + Column( "id", Integer, primary_key=True ), + Column( "create_time", DateTime, default=now ), + Column( "update_time", DateTime, default=now, onupdate=now ), + Column( "repository_metadata_id", Integer, ForeignKey( "repository_metadata.id" ), index=True ), + Column( "initial_changeset_revision", TrimmedString( 255 ), index=True ), + Column( "comment" , TEXT ) ) + RepositoryReview.table = Table( "repository_review", metadata, Column( "id", Integer, primary_key=True ), Column( "create_time", DateTime, default=now ), @@ -267,10 +276,14 @@ foreign_keys=[ RepositoryMetadata.table.c.repository_id, RepositoryMetadata.table.c.changeset_revision ], primaryjoin=( ( RepositoryMetadata.table.c.repository_id == RepositoryReview.table.c.repository_id ) & ( RepositoryMetadata.table.c.changeset_revision == RepositoryReview.table.c.changeset_revision ) ) ) ) ) +mapper( SkipToolTest, SkipToolTest.table, + properties=dict( repository_revision=relation( RepositoryMetadata, + backref='skip_tool_tests' ) ) ) + mapper( RepositoryReview, RepositoryReview.table, properties=dict( repository=relation( Repository, primaryjoin=( RepositoryReview.table.c.repository_id == Repository.table.c.id ) ), - # Take case when using the mapper below! It should be used only when a new review is being created for a repository change set revision. + # Take care when using the mapper below! It should be used only when a new review is being created for a repository change set revision. # Keep in mind that repository_metadata records can be removed from the database for certain change set revisions when metadata is being # reset on a repository! repository_metadata=relation( RepositoryMetadata, diff -r 036f57f93b23a62205b8541b6fd773561879827e -r a3e1fb22f5b5afcb776399d8a499e62c795f8279 lib/galaxy/webapps/tool_shed/model/migrate/versions/0019_add_skip_tool_test_table_and_test_install_error_column.py --- /dev/null +++ b/lib/galaxy/webapps/tool_shed/model/migrate/versions/0019_add_skip_tool_test_table_and_test_install_error_column.py @@ -0,0 +1,82 @@ +""" +Migration script to add the skip_tool_test table and add the test_install_error column to the repository_metadata table. +""" + +from sqlalchemy import * +from sqlalchemy.orm import * +from migrate import * +from migrate.changeset import * +import datetime + +# Need our custom types, but don't import anything else from model +from galaxy.model.custom_types import * + +import sys, logging +log = logging.getLogger( __name__ ) +log.setLevel(logging.DEBUG) +handler = logging.StreamHandler( sys.stdout ) +format = "%(name)s %(levelname)s %(asctime)s %(message)s" +formatter = logging.Formatter( format ) +handler.setFormatter( formatter ) +log.addHandler( handler ) + +now = datetime.datetime.utcnow + +metadata = MetaData() + +SkipToolTest_table = Table( "skip_tool_test", metadata, + Column( "id", Integer, primary_key=True ), + Column( "create_time", DateTime, default=now ), + Column( "update_time", DateTime, default=now, onupdate=now ), + Column( "repository_metadata_id", Integer, ForeignKey( "repository_metadata.id" ), index=True ), + Column( "initial_changeset_revision", TrimmedString( 255 ), index=True ), + Column( "comment" , TEXT ) ) + +def upgrade( migrate_engine ): + print __doc__ + metadata.bind = migrate_engine + metadata.reflect() + # Initialize. + if migrate_engine.name == 'mysql' or migrate_engine.name == 'sqlite': + default_false = "0" + elif migrate_engine.name in [ 'postgresql', 'postgres' ]: + default_false = "false" + + try: + RepositoryMetadata_table = Table( "repository_metadata", metadata, autoload=True ) + except NoSuchTableError: + RepositoryMetadata_table = None + log.debug( "Failed loading table repository_metadata." ) + + if RepositoryMetadata_table is not None: + # Create the test_install_error column. + c = Column( "test_install_error", Boolean, default=False, index=True ) + try: + c.create( RepositoryMetadata_table, index_name="ix_repository_metadata_ttie") + assert c is RepositoryMetadata_table.c.test_install_error + migrate_engine.execute( "UPDATE repository_metadata SET test_install_error=%s" % default_false ) + except Exception, e: + print "Adding test_install_error column to the repository_metadata table failed: %s" % str( e ) + + # Create skip_tool_test table. + try: + SkipToolTest_table.create() + except Exception, e: + print "Creating the skip_tool_test table failed: %s" % str( e ) + +def downgrade( migrate_engine ): + metadata.bind = migrate_engine + metadata.reflect() + + # Drop the skip_tool_test table. + try: + SkipToolTest_table.drop() + except Exception, e: + print "Dropping the skip_tool_test table failed: %s" % str( e ) + + # Drop test_install_error column from the repository_metadata table. + RepositoryMetadata_table = Table( "repository_metadata", metadata, autoload=True ) + try: + RepositoryMetadata_table.c.test_install_error.drop() + except Exception, e: + print "Dropping column test_install_error from the repository_metadata table failed: %s" % str( e ) diff -r 036f57f93b23a62205b8541b6fd773561879827e -r a3e1fb22f5b5afcb776399d8a499e62c795f8279 lib/tool_shed/grids/repository_grids.py --- a/lib/tool_shed/grids/repository_grids.py +++ b/lib/tool_shed/grids/repository_grids.py @@ -604,6 +604,180 @@ .filter( model.Repository.table.c.id < 0 ) +class RepositoriesWithTestInstallErrorsGrid( RepositoryGrid ): + # This grid displays only the latest installable revision of each repository. + title = "Repositories with tool test installation errors" + columns = [ + RepositoryGrid.NameColumn( "Name", + key="name", + link=( lambda item: dict( operation="view_or_manage_repository", id=item.id ) ), + attach_popup=False ), + RepositoryGrid.LatestInstallableRevisionColumn( "Latest Installable Revision" ), + RepositoryGrid.UserColumn( "Owner", + key="User.username", + model_class=model.User, + link=( lambda item: dict( operation="repositories_by_user", id=item.id ) ), + attach_popup=False ) + ] + columns.append( grids.MulticolFilterColumn( "Search repository name", + cols_to_filter=[ columns[0] ], + key="free-text-search", + visible=False, + filterable="standard" ) ) + operations = [] + use_paging = False + + def build_initial_query( self, trans, **kwd ): + # Filter by latest installable revisions that contain tools with missing tool test components. + revision_clause_list = [] + for repository in trans.sa_session.query( model.Repository ) \ + .filter( and_( model.Repository.table.c.deprecated == False, + model.Repository.table.c.deleted == False ) ): + changeset_revision = filter_by_latest_downloadable_changeset_revision_that_has_test_install_errors( trans, repository ) + if changeset_revision: + revision_clause_list.append( model.RepositoryMetadata.table.c.changeset_revision == changeset_revision ) + if revision_clause_list: + return trans.sa_session.query( model.Repository ) \ + .filter( and_( model.Repository.table.c.deprecated == False, + model.Repository.table.c.deleted == False ) ) \ + .join( model.RepositoryMetadata ) \ + .filter( or_( *revision_clause_list ) ) \ + .join( model.User.table ) + # Return an empty query. + return trans.sa_session.query( model.Repository ) \ + .filter( model.Repository.table.c.id < 0 ) + + +class MyWritableRepositoriesWithTestInstallErrorsGrid( RepositoriesWithTestInstallErrorsGrid ): + # This grid displays only the latest installable revision of each repository. + title = "Repositories I can change with tool test installation errors" + columns = [ col for col in RepositoriesWithTestInstallErrorsGrid.columns ] + operations = [] + use_paging = False + + def build_initial_query( self, trans, **kwd ): + # First get all repositories that the current user is authorized to update. + username = trans.user.username + user_clause_list = [] + for repository in trans.sa_session.query( model.Repository ) \ + .filter( and_( model.Repository.table.c.deprecated == False, + model.Repository.table.c.deleted == False ) ): + allow_push = repository.allow_push( trans.app ) + if allow_push: + allow_push_usernames = allow_push.split( ',' ) + if username in allow_push_usernames: + user_clause_list.append( model.Repository.table.c.id == repository.id ) + if user_clause_list: + # We have the list of repositories that the current user is authorized to update, so filter further by latest installable revisions that contain + # tools with missing tool test components. + revision_clause_list = [] + for repository in trans.sa_session.query( model.Repository ) \ + .filter( and_( model.Repository.table.c.deprecated == False, + model.Repository.table.c.deleted == False ) ) \ + .filter( or_( *user_clause_list ) ): + changeset_revision = filter_by_latest_downloadable_changeset_revision_that_has_test_install_errors( trans, repository ) + if changeset_revision: + revision_clause_list.append( model.RepositoryMetadata.table.c.changeset_revision == changeset_revision ) + if revision_clause_list: + return trans.sa_session.query( model.Repository ) \ + .filter( and_( model.Repository.table.c.deprecated == False, + model.Repository.table.c.deleted == False ) ) \ + .join( model.User.table ) \ + .filter( or_( *user_clause_list ) ) \ + .join( model.RepositoryMetadata ) \ + .filter( or_( *revision_clause_list ) ) + # Return an empty query. + return trans.sa_session.query( model.Repository ) \ + .filter( model.Repository.table.c.id < 0 ) + + +class RepositoriesWithSkipTestsCheckedGrid( RepositoryGrid ): + # This grid displays only the latest installable revision of each repository. + title = "Repositories with skip tool tests checked" + columns = [ + RepositoryGrid.NameColumn( "Name", + key="name", + link=( lambda item: dict( operation="view_or_manage_repository", id=item.id ) ), + attach_popup=False ), + RepositoryGrid.LatestInstallableRevisionColumn( "Latest Installable Revision" ), + RepositoryGrid.UserColumn( "Owner", + key="User.username", + model_class=model.User, + link=( lambda item: dict( operation="repositories_by_user", id=item.id ) ), + attach_popup=False ) + ] + columns.append( grids.MulticolFilterColumn( "Search repository name", + cols_to_filter=[ columns[0] ], + key="free-text-search", + visible=False, + filterable="standard" ) ) + operations = [] + use_paging = False + + def build_initial_query( self, trans, **kwd ): + # Filter by latest installable revisions that contain tools with missing tool test components. + revision_clause_list = [] + for repository in trans.sa_session.query( model.Repository ) \ + .filter( and_( model.Repository.table.c.deprecated == False, + model.Repository.table.c.deleted == False ) ): + changeset_revision = filter_by_latest_downloadable_changeset_revision_with_skip_tests_checked( trans, repository ) + if changeset_revision: + revision_clause_list.append( model.RepositoryMetadata.table.c.changeset_revision == changeset_revision ) + if revision_clause_list: + return trans.sa_session.query( model.Repository ) \ + .filter( and_( model.Repository.table.c.deprecated == False, + model.Repository.table.c.deleted == False ) ) \ + .join( model.RepositoryMetadata ) \ + .filter( or_( *revision_clause_list ) ) \ + .join( model.User.table ) + # Return an empty query. + return trans.sa_session.query( model.Repository ) \ + .filter( model.Repository.table.c.id < 0 ) + + +class MyWritableRepositoriesWithSkipTestsCheckedGrid( RepositoriesWithSkipTestsCheckedGrid ): + # This grid displays only the latest installable revision of each repository. + title = "Repositories I can change with skip tool tests checked" + columns = [ col for col in RepositoriesWithSkipTestsCheckedGrid.columns ] + operations = [] + use_paging = False + + def build_initial_query( self, trans, **kwd ): + # First get all repositories that the current user is authorized to update. + username = trans.user.username + user_clause_list = [] + for repository in trans.sa_session.query( model.Repository ) \ + .filter( and_( model.Repository.table.c.deprecated == False, + model.Repository.table.c.deleted == False ) ): + allow_push = repository.allow_push( trans.app ) + if allow_push: + allow_push_usernames = allow_push.split( ',' ) + if username in allow_push_usernames: + user_clause_list.append( model.Repository.table.c.id == repository.id ) + if user_clause_list: + # We have the list of repositories that the current user is authorized to update, so filter further by latest installable revisions that contain + # tools with missing tool test components. + revision_clause_list = [] + for repository in trans.sa_session.query( model.Repository ) \ + .filter( and_( model.Repository.table.c.deprecated == False, + model.Repository.table.c.deleted == False ) ) \ + .filter( or_( *user_clause_list ) ): + changeset_revision = filter_by_latest_downloadable_changeset_revision_with_skip_tests_checked( trans, repository ) + if changeset_revision: + revision_clause_list.append( model.RepositoryMetadata.table.c.changeset_revision == changeset_revision ) + if revision_clause_list: + return trans.sa_session.query( model.Repository ) \ + .filter( and_( model.Repository.table.c.deprecated == False, + model.Repository.table.c.deleted == False ) ) \ + .join( model.User.table ) \ + .filter( or_( *user_clause_list ) ) \ + .join( model.RepositoryMetadata ) \ + .filter( or_( *revision_clause_list ) ) + # Return an empty query. + return trans.sa_session.query( model.Repository ) \ + .filter( model.Repository.table.c.id < 0 ) + + class DeprecatedRepositoriesIOwnGrid( RepositoriesIOwnGrid ): title = "Deprecated repositories I own" columns = [ @@ -1475,7 +1649,10 @@ repo = hg.repository( suc.get_configured_ui(), repository.repo_path( trans.app ) ) tip_ctx = str( repo.changectx( repo.changelog.tip() ) ) repository_metadata = get_latest_installable_repository_metadata_if_it_includes_tools( trans, repository ) - if repository_metadata and not repository_metadata.missing_test_components and not repository_metadata.tools_functionally_correct: + if repository_metadata \ + and not repository_metadata.missing_test_components \ + and not repository_metadata.tools_functionally_correct \ + and not repository_metadata.test_install_error: return repository_metadata.changeset_revision return None @@ -1516,6 +1693,32 @@ return repository_metadata.changeset_revision return None +def filter_by_latest_downloadable_changeset_revision_that_has_test_install_errors( trans, repository ): + """ + Inspect the latest installable changeset revision for the received repository to see if it has tool test installation errors. + """ + encoded_repository_id = trans.security.encode_id( repository.id ) + repo = hg.repository( suc.get_configured_ui(), repository.repo_path( trans.app ) ) + tip_ctx = str( repo.changectx( repo.changelog.tip() ) ) + repository_metadata = get_latest_installable_repository_metadata_if_it_includes_tools( trans, repository ) + if repository_metadata \ + and repository_metadata.test_install_error \ + and not repository_metadata.missing_test_components: + return repository_metadata.changeset_revision + return None + +def filter_by_latest_downloadable_changeset_revision_with_skip_tests_checked( trans, repository ): + """ + Inspect the latest installable changeset revision for the received repository to see if skip tests is checked. + """ + encoded_repository_id = trans.security.encode_id( repository.id ) + repo = hg.repository( suc.get_configured_ui(), repository.repo_path( trans.app ) ) + tip_ctx = str( repo.changectx( repo.changelog.tip() ) ) + repository_metadata = get_latest_installable_repository_metadata_if_it_includes_tools( trans, repository ) + if repository_metadata and repository_metadata.skip_tool_tests: + return repository_metadata.changeset_revision + return None + def get_latest_repository_metadata_if_it_includes_invalid_tools( trans, repository ): """Return the latest repository_metadata record for the received repository that contains invalid tools if one exists.""" encoded_repository_id = trans.security.encode_id( repository.id ) diff -r 036f57f93b23a62205b8541b6fd773561879827e -r a3e1fb22f5b5afcb776399d8a499e62c795f8279 lib/tool_shed/util/metadata_util.py --- a/lib/tool_shed/util/metadata_util.py +++ b/lib/tool_shed/util/metadata_util.py @@ -259,6 +259,8 @@ downloadable = has_repository_dependencies or includes_datatypes or includes_tools or includes_tool_dependencies or includes_workflows repository_metadata = suc.get_repository_metadata_by_changeset_revision( trans, id, changeset_revision ) if repository_metadata: + # A repository metadata record already exists with the received changeset_revision, so we don't need to check the skip_tool_test table. + check_skip_tool_test = False repository_metadata.metadata = metadata_dict repository_metadata.downloadable = downloadable repository_metadata.has_repository_dependencies = has_repository_dependencies @@ -267,6 +269,8 @@ repository_metadata.includes_tool_dependencies = includes_tool_dependencies repository_metadata.includes_workflows = includes_workflows else: + # No repository_metadata record exists for the received changeset_revision, so we may need to update the skip_tool_test table. + check_skip_tool_test = True repository_metadata = trans.model.RepositoryMetadata( repository_id=repository.id, changeset_revision=changeset_revision, metadata=metadata_dict, @@ -279,11 +283,38 @@ # Always set the default values for the following columns. When resetting all metadata on a repository, this will reset the values. repository_metadata.tools_functionally_correct = False repository_metadata.missing_test_components = False + repository_metadata.test_install_error = False repository_metadata.do_not_test = False repository_metadata.time_last_tested = None repository_metadata.tool_test_results = None trans.sa_session.add( repository_metadata ) trans.sa_session.flush() + if check_skip_tool_test: + # Since we created a new repository_metadata record, we may need to update the skip_tool_test table to point to it. Inspect each + # changeset revision in the received repository's changelog (up to the received changeset revision) to see if it is contained in the + # skip_tool_test table. If it is, but is not associated with a repository_metadata record, reset that skip_tool_test record to the + # newly created repository_metadata record. + repo = hg.repository( suc.get_configured_ui(), repository.repo_path( trans.app ) ) + for changeset in repo.changelog: + changeset_hash = str( repo.changectx( changeset ) ) + skip_tool_test = suc.get_skip_tool_test_by_changeset_revision( trans, changeset_hash ) + if skip_tool_test: + # We found a skip_tool_test record associated with the changeset_revision, so see if it has a valid repository_revision. + try: + repository_revision = skip_tool_test.repository_revision + # The skip_tool_test record is associated with a valid repository_metadata record, so proceed. + continue + except: + # We found a skip_tool_test record that is associated with an invalid repository_metadata record, so update it to point to + # the newly created repository_metadata record. In some special cases there may be multiple skip_tool_test records that + # require updating, so we won't break here, we'll continue to inspect the rest of the changelog up to the received + # changeset_revision. + skip_tool_test.repository_revision = repository_metadata + trans.sa_session.add( skip_tool_test ) + trans.sa_session.flush() + if changeset_hash == changeset_revision: + # Proceed no further than the received changeset_revision. + break return repository_metadata def generate_data_manager_metadata( app, repository, repo_dir, data_manager_config_filename, metadata_dict, shed_config_dict=None ): diff -r 036f57f93b23a62205b8541b6fd773561879827e -r a3e1fb22f5b5afcb776399d8a499e62c795f8279 lib/tool_shed/util/shed_util_common.py --- a/lib/tool_shed/util/shed_util_common.py +++ b/lib/tool_shed/util/shed_util_common.py @@ -793,6 +793,13 @@ if shed_tool_conf == file_name: return index, shed_tool_conf_dict +def get_skip_tool_test_by_changeset_revision( trans, changeset_revision ): + """Return a skip_tool_test record whose initial_changeset_revision is the received changeset_revision.""" + # There should only be one, but we'll use first() so callers won't have to handle exceptions. + return trans.sa_session.query( trans.model.SkipToolTest ) \ + .filter( trans.model.SkipToolTest.table.c.initial_changeset_revision == changeset_revision ) \ + .first() + def get_tool_panel_config_tool_path_install_dir( app, repository ): """ Return shed-related tool panel config, the tool_path configured in it, and the relative path to the directory where the repository is installed. diff -r 036f57f93b23a62205b8541b6fd773561879827e -r a3e1fb22f5b5afcb776399d8a499e62c795f8279 templates/webapps/tool_shed/index.mako --- a/templates/webapps/tool_shed/index.mako +++ b/templates/webapps/tool_shed/index.mako @@ -116,9 +116,15 @@ <a target="galaxy_main" href="${h.url_for( controller='repository', action='browse_my_writable_repositories_missing_tool_test_components' )}">Latest revision: missing tool tests</a></div><div class="toolTitle"> + <a target="galaxy_main" href="${h.url_for( controller='repository', action='browse_my_writable_repositories_with_install_errors' )}">Latest revision: installation errors</a> + </div> + <div class="toolTitle"><a target="galaxy_main" href="${h.url_for( controller='repository', action='browse_my_writable_repositories_with_failing_tool_tests' )}">Latest revision: failing tool tests</a></div><div class="toolTitle"> + <a target="galaxy_main" href="${h.url_for( controller='repository', action='browse_my_writable_repositories_with_skip_tool_test_checked' )}">Latest revision: skip tool tests</a> + </div> + <div class="toolTitle"><a target="galaxy_main" href="${h.url_for( controller='repository', action='browse_my_writable_repositories_with_no_failing_tool_tests' )}">Latest revision: all tool tests pass</a></div><div class="toolTitle"> @@ -167,9 +173,15 @@ <a target="galaxy_main" href="${h.url_for( controller='repository', action='browse_repositories_missing_tool_test_components' )}">Latest revision: missing tool tests</a></div><div class="toolTitle"> + <a target="galaxy_main" href="${h.url_for( controller='repository', action='browse_repositories_with_install_errors' )}">Latest revision: installation errors</a> + </div> + <div class="toolTitle"><a target="galaxy_main" href="${h.url_for( controller='repository', action='browse_repositories_with_failing_tool_tests' )}">Latest revision: failing tool tests</a></div><div class="toolTitle"> + <a target="galaxy_main" href="${h.url_for( controller='repository', action='browse_repositories_with_skip_tool_test_checked' )}">Latest revision: skip tool tests</a> + </div> + <div class="toolTitle"><a target="galaxy_main" href="${h.url_for( controller='repository', action='browse_repositories_with_no_failing_tool_tests' )}">Latest revision: all tool tests pass</a></div><div class="toolTitle"> Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0